diff --git a/be/src/agent/be_exec_version_manager.cpp b/be/src/agent/be_exec_version_manager.cpp index 3dad2da010e9d5..811ee8b81f667a 100644 --- a/be/src/agent/be_exec_version_manager.cpp +++ b/be/src/agent/be_exec_version_manager.cpp @@ -132,7 +132,10 @@ void BeExecVersionManager::check_function_compatibility(int current_be_exec_vers // a. support strict ownership hash routing for external table sink writers. // b. support Paimon default fixed-bucket routing in the external sink exchange. -const int BeExecVersionManager::max_be_exec_version = 13; +// 14: start from master +// a. support pluggable hash algorithms for table distribution and bucket-local exchanges. + +const int BeExecVersionManager::max_be_exec_version = SUPPORT_DISTRIBUTION_HASH_TYPE_VERSION; const int BeExecVersionManager::min_be_exec_version = 0; std::map> BeExecVersionManager::_function_change_map {}; std::set BeExecVersionManager::_function_restrict_map; diff --git a/be/src/agent/be_exec_version_manager.h b/be/src/agent/be_exec_version_manager.h index a5f8ac9ced34cb..8f30af4ccae916 100644 --- a/be/src/agent/be_exec_version_manager.h +++ b/be/src/agent/be_exec_version_manager.h @@ -29,6 +29,7 @@ constexpr inline int USE_NEW_FIXED_OBJECT_SERIALIZATION_VERSION = 10; constexpr inline int SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION = 11; constexpr inline int SUPPORT_ICEBERG_VARIANT_VERSION = 12; constexpr inline int SUPPORT_EXTERNAL_TABLE_SINK_HASH_VERSION = 13; +constexpr inline int SUPPORT_DISTRIBUTION_HASH_TYPE_VERSION = 14; class BeExecVersionManager { public: diff --git a/be/src/exec/exchange/local_exchange_sink_operator.cpp b/be/src/exec/exchange/local_exchange_sink_operator.cpp index 40a69d86ff1905..a9e39f9ccfc5dd 100644 --- a/be/src/exec/exchange/local_exchange_sink_operator.cpp +++ b/be/src/exec/exchange/local_exchange_sink_operator.cpp @@ -50,7 +50,17 @@ Status LocalExchangeSinkOperatorX::_create_partitioner(RuntimeState* state, int RETURN_IF_ERROR(_partitioner->init(_texprs)); } else if (_type == TLocalPartitionType::BUCKET_HASH_SHUFFLE) { DCHECK_GT(bucket_count, 0); - _partitioner = std::make_unique>(bucket_count); + switch (_distribution_hash_type) { + case TDistributionHashType::CRC32: + _partitioner = std::make_unique>(bucket_count); + break; + case TDistributionHashType::IDENTITY: + _partitioner = std::make_unique(bucket_count); + break; + default: + return Status::InternalError("unsupported distribution_hash_type {}", + static_cast(_distribution_hash_type)); + } RETURN_IF_ERROR(_partitioner->init(_texprs)); } return Status::OK(); diff --git a/be/src/exec/exchange/local_exchange_sink_operator.h b/be/src/exec/exchange/local_exchange_sink_operator.h index 357da9fd83849c..13dbb9532b6859 100644 --- a/be/src/exec/exchange/local_exchange_sink_operator.h +++ b/be/src/exec/exchange/local_exchange_sink_operator.h @@ -73,8 +73,10 @@ class LocalExchangeSinkOperatorX final : public DataSinkOperatorX; LocalExchangeSinkOperatorX(int sink_id, int dest_id, int num_partitions, const std::vector& texprs, - const std::map& bucket_seq_to_instance_idx) + const std::map& bucket_seq_to_instance_idx, + TDistributionHashType::type distribution_hash_type) : Base(sink_id, dest_id, dest_id), + _distribution_hash_type(distribution_hash_type), _num_partitions(num_partitions), _texprs(texprs), _partitioned_exprs_num(texprs.size()), @@ -85,6 +87,9 @@ class LocalExchangeSinkOperatorX final : public DataSinkOperatorX& shuffle_id_to_instance_idx) : Base(operator_id, tnode, dest_id), _type(tnode.local_exchange_node.partition_type), + _distribution_hash_type(tnode.local_exchange_node.__isset.distribution_hash_type + ? tnode.local_exchange_node.distribution_hash_type + : TDistributionHashType::CRC32), _num_partitions(num_partitions), _texprs(tnode.local_exchange_node.distribute_expr_lists), _partitioned_exprs_num(tnode.local_exchange_node.distribute_expr_lists.size()), @@ -135,6 +140,7 @@ class LocalExchangeSinkOperatorX final : public DataSinkOperatorX& _texprs; const size_t _partitioned_exprs_num; diff --git a/be/src/exec/operator/exchange_sink_operator.cpp b/be/src/exec/operator/exchange_sink_operator.cpp index 449c71e3339281..4abc3a83a068b6 100644 --- a/be/src/exec/operator/exchange_sink_operator.cpp +++ b/be/src/exec/operator/exchange_sink_operator.cpp @@ -138,11 +138,24 @@ Status ExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& inf "Partitioner", fmt::format("Crc32CHashPartitioner({})", _partition_count)); } else if (_part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED) { _partition_count = channels.size(); - _partitioner = std::make_unique>(channels.size()); + switch (p._distribution_hash_type) { + case TDistributionHashType::CRC32: + _partitioner = + std::make_unique>(channels.size()); + custom_profile()->add_info_string( + "Partitioner", fmt::format("Crc32HashPartitioner({})", _partition_count)); + break; + case TDistributionHashType::IDENTITY: + _partitioner = std::make_unique(channels.size()); + custom_profile()->add_info_string( + "Partitioner", fmt::format("IdentityHashPartitioner({})", _partition_count)); + break; + default: + return Status::InternalError("unsupported distribution_hash_type {}", + static_cast(p._distribution_hash_type)); + } RETURN_IF_ERROR(_partitioner->init(p._texprs)); RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc)); - custom_profile()->add_info_string( - "Partitioner", fmt::format("Crc32HashPartitioner({})", _partition_count)); } else if (_part_type == TPartitionType::OLAP_TABLE_SINK_HASH_PARTITIONED) { // in ExchangeOlapWriter we rely on type of _partitioner here _partition_count = channels.size(); @@ -301,6 +314,9 @@ ExchangeSinkOperatorX::ExchangeSinkOperatorX( _texprs(sink.output_partition.partition_exprs), _row_desc(row_desc), _part_type(sink.output_partition.type), + _distribution_hash_type(sink.output_partition.__isset.distribution_hash_type + ? sink.output_partition.distribution_hash_type + : TDistributionHashType::CRC32), _dests(destinations), _dest_node_id(sink.dest_node_id), _transfer_large_data_by_brpc(config::transfer_large_data_by_brpc), diff --git a/be/src/exec/operator/exchange_sink_operator.h b/be/src/exec/operator/exchange_sink_operator.h index 10351154d1d8cd..2c89129fbd19a8 100644 --- a/be/src/exec/operator/exchange_sink_operator.h +++ b/be/src/exec/operator/exchange_sink_operator.h @@ -250,6 +250,7 @@ class ExchangeSinkOperatorX MOCK_REMOVE(final) : public DataSinkOperatorX_partition_expr_ctxs); } +void IdentityHashPartitioner::_do_hash(const ColumnPtr& column, HashValType* __restrict result, + int idx) const { + const PrimitiveType type = _partition_expr_ctxs[idx]->root()->data_type()->get_primitive_type(); + for (size_t row = 0; row < column->size(); ++row) { + auto val = column->get_data_at(row); + result[row] = + RawValue::identity_hash(val.data, val.size, type, result[row], _partition_count); + } +} + +Status IdentityHashPartitioner::clone(RuntimeState* state, + std::unique_ptr& partitioner) { + auto* new_partitioner = new IdentityHashPartitioner(_partition_count); + partitioner.reset(new_partitioner); + return _clone_expr_ctxs(state, new_partitioner->_partition_expr_ctxs); +} + template class Crc32HashPartitioner; template class Crc32HashPartitioner; template class Crc32HashPartitioner; diff --git a/be/src/exec/partitioner/partitioner.h b/be/src/exec/partitioner/partitioner.h index 98607c3623634f..cf67162eed408b 100644 --- a/be/src/exec/partitioner/partitioner.h +++ b/be/src/exec/partitioner/partitioner.h @@ -191,6 +191,26 @@ class Crc32CHashPartitioner : public Crc32HashPartitioner { } }; +// Bucket-shuffle repartitioner for tables bucketed with the identity hash. Each distribution +// column's canonical bytes are interpreted as an unsigned integer with the first byte as the least +// significant, then appended to the preceding columns; the combined value is kept modulo the +// bucket count. Must stay bit-identical with FE HashDistributionPruner and BE tablet routing. +class IdentityHashPartitioner : public Crc32HashPartitioner { +public: + IdentityHashPartitioner(int partition_count) + : Crc32HashPartitioner(partition_count) {} + + Status clone(RuntimeState* state, std::unique_ptr& partitioner) override; + +private: + void _do_hash(const ColumnPtr& column, HashValType* __restrict result, int idx) const override; + + void _initialize_hash_vals(size_t rows) const override { + _hash_vals.resize(rows); + std::ranges::fill(_hash_vals, 0); + } +}; + /// Instantiated once in partitioner.cpp; suppresses per-TU implicit instantiation. extern template class Crc32HashPartitioner; extern template class Crc32HashPartitioner; diff --git a/be/src/exec/pipeline/dependency.h b/be/src/exec/pipeline/dependency.h index 53f9ed9281bb1c..79956a7cdad31f 100644 --- a/be/src/exec/pipeline/dependency.h +++ b/be/src/exec/pipeline/dependency.h @@ -790,11 +790,17 @@ struct DataDistribution { DataDistribution(TLocalPartitionType::type type) : distribution_type(type) {} DataDistribution(TLocalPartitionType::type type, const std::vector& partition_exprs_) : distribution_type(type), partition_exprs(partition_exprs_) {} + DataDistribution(TLocalPartitionType::type type, const std::vector& partition_exprs_, + TDistributionHashType::type distribution_hash_type_) + : distribution_type(type), + partition_exprs(partition_exprs_), + distribution_hash_type(distribution_hash_type_) {} DataDistribution(const DataDistribution& other) = default; bool need_local_exchange() const { return distribution_type != TLocalPartitionType::NOOP; } DataDistribution& operator=(const DataDistribution& other) = default; TLocalPartitionType::type distribution_type; std::vector partition_exprs; + TDistributionHashType::type distribution_hash_type = TDistributionHashType::CRC32; }; class ExchangerBase; diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index f62bd0730816c2..5f441ba14658af 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -1015,9 +1015,14 @@ Status PipelineFragmentContext::_add_local_exchange_impl( const bool use_global_hash_shuffle = bucket_seq_to_instance_idx.empty() && !shuffle_idx_to_instance_idx.contains(-1) && followed_by_shuffled_operator && !_use_serial_source; + if (data_distribution.distribution_type == TLocalPartitionType::BUCKET_HASH_SHUFFLE && + _params.fragment.__isset.distribution_hash_type) { + data_distribution.distribution_hash_type = _params.fragment.distribution_hash_type; + } sink = std::make_shared( sink_id, local_exchange_id, use_global_hash_shuffle ? _total_instances : _num_instances, - data_distribution.partition_exprs, bucket_seq_to_instance_idx); + data_distribution.partition_exprs, bucket_seq_to_instance_idx, + data_distribution.distribution_hash_type); if (bucket_seq_to_instance_idx.empty() && data_distribution.distribution_type == TLocalPartitionType::BUCKET_HASH_SHUFFLE) { data_distribution.distribution_type = diff --git a/be/src/storage/tablet_info.cpp b/be/src/storage/tablet_info.cpp index 9fee41082442c2..6b1a7173144132 100644 --- a/be/src/storage/tablet_info.cpp +++ b/be/src/storage/tablet_info.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include diff --git a/be/src/storage/tablet_info.h b/be/src/storage/tablet_info.h index 1ea346844d89d6..e71c08f3c9500e 100644 --- a/be/src/storage/tablet_info.h +++ b/be/src/storage/tablet_info.h @@ -248,24 +248,40 @@ class VOlapTablePartitionParam { std::map* partition_tablets_buffer = nullptr) const { std::function compute_function; if (!_distributed_slot_locs.empty()) { - //TODO: refactor by saving the hash values. then we can calculate in columnwise. - compute_function = [this](Block* block, uint32_t row, - const VOlapTablePartition& partition) -> uint32_t { - uint32_t hash_val = 0; - for (unsigned short _distributed_slot_loc : _distributed_slot_locs) { - auto* slot_desc = _slots[_distributed_slot_loc]; - auto& column = block->get_by_position(_distributed_slot_loc).column; - auto val = column->get_data_at(row); - if (val.data != nullptr) { - hash_val = RawValue::zlib_crc32(val.data, val.size, - slot_desc->type()->get_primitive_type(), - hash_val); - } else { - hash_val = HashUtil::zlib_crc_hash_null(hash_val); + if (_t_param.distribution_hash_type == TDistributionHashType::IDENTITY) { + compute_function = [this](Block* block, uint32_t row, + const VOlapTablePartition& partition) -> uint32_t { + uint32_t bucket = 0; + for (unsigned short distributed_slot_loc : _distributed_slot_locs) { + auto* slot_desc = _slots[distributed_slot_loc]; + const auto& column = block->get_by_position(distributed_slot_loc).column; + auto val = column->get_data_at(row); + bucket = RawValue::identity_hash( + val.data, val.size, slot_desc->type()->get_primitive_type(), bucket, + cast_set(partition.num_buckets)); } - } - return cast_set(hash_val % partition.num_buckets); - }; + return bucket; + }; + } else { + //TODO: refactor by saving the hash values. then we can calculate in columnwise. + compute_function = [this](Block* block, uint32_t row, + const VOlapTablePartition& partition) -> uint32_t { + uint32_t hash_val = 0; + for (unsigned short _distributed_slot_loc : _distributed_slot_locs) { + auto* slot_desc = _slots[_distributed_slot_loc]; + auto& column = block->get_by_position(_distributed_slot_loc).column; + auto val = column->get_data_at(row); + if (val.data != nullptr) { + hash_val = RawValue::zlib_crc32(val.data, val.size, + slot_desc->type()->get_primitive_type(), + hash_val); + } else { + hash_val = HashUtil::zlib_crc_hash_null(hash_val); + } + } + return cast_set(hash_val % partition.num_buckets); + }; + } } else { // random distribution compute_function = [](Block* block, uint32_t row, const VOlapTablePartition& partition) -> uint32_t { diff --git a/be/src/util/raw_value.h b/be/src/util/raw_value.h index 4babb5c0810037..fd5489916f22f4 100644 --- a/be/src/util/raw_value.h +++ b/be/src/util/raw_value.h @@ -22,6 +22,7 @@ #include +#include "common/check.h" #include "common/consts.h" #include "common/logging.h" #include "core/data_type/define_primitive_type.h" @@ -38,8 +39,98 @@ class RawValue { // Same as the up function, only use in vec exec engine. static uint32_t zlib_crc32(const void* value, size_t len, const PrimitiveType& type, uint32_t seed); + + // Treat the canonical distribution bytes of a value as an unsigned integer with the first byte + // as the least-significant byte, then append it to the preceding distribution columns. The + // returned value is kept modulo mod throughout, so values of any byte width and any number of + // columns do not require a wide integer. + static uint32_t identity_hash(const void* value, size_t len, const PrimitiveType& type, + uint32_t seed, uint32_t mod); }; +inline uint32_t RawValue::identity_hash(const void* v, size_t len, const PrimitiveType& type, + uint32_t seed, uint32_t mod) { + DCHECK_GT(mod, 0); + auto append_little_endian = [&seed, mod](const void* value, size_t size) { + const auto* bytes = reinterpret_cast(value); + uint64_t remainder = seed; + size_t bytes_since_mod = 0; + for (size_t i = size; i > 0; --i) { + remainder = remainder * 256 + bytes[i - 1]; + if (++bytes_since_mod == sizeof(uint32_t)) { + remainder %= mod; + bytes_since_mod = 0; + } + } + seed = static_cast(remainder % mod); + }; + + if (v == nullptr) { + static constexpr uint32_t NULL_VALUE = 0; + append_little_endian(&NULL_VALUE, sizeof(NULL_VALUE)); + return seed; + } + + switch (type) { + case TYPE_VARCHAR: + case TYPE_VARBINARY: + case TYPE_HLL: + case TYPE_STRING: + case TYPE_CHAR: + append_little_endian(v, len); + break; + case TYPE_BOOLEAN: + case TYPE_TINYINT: + append_little_endian(v, 1); + break; + case TYPE_SMALLINT: + append_little_endian(v, 2); + break; + case TYPE_INT: + case TYPE_FLOAT: + case TYPE_DATEV2: + case TYPE_DECIMAL32: + case TYPE_IPV4: + append_little_endian(v, 4); + break; + case TYPE_BIGINT: + case TYPE_DOUBLE: + case TYPE_TIMEV2: + case TYPE_DATETIMEV2: + case TYPE_TIMESTAMPTZ: + case TYPE_DECIMAL64: + append_little_endian(v, 8); + break; + case TYPE_LARGEINT: + case TYPE_DECIMAL128I: + case TYPE_IPV6: + append_little_endian(v, 16); + break; + case TYPE_DECIMAL256: + append_little_endian(v, 32); + break; + case TYPE_DATE: + case TYPE_DATETIME: { + const auto* date_val = reinterpret_cast(v); + char buf[64]; + int date_len = date_val->to_buffer(buf); + append_little_endian(buf, date_len); + break; + } + case TYPE_DECIMALV2: { + const auto* dec_val = reinterpret_cast(v); + int64_t int_val = dec_val->int_value(); + int32_t frac_val = dec_val->frac_value(); + append_little_endian(&frac_val, sizeof(frac_val)); + append_little_endian(&int_val, sizeof(int_val)); + break; + } + default: + DORIS_CHECK(false) << "invalid type: " << type; + } + return seed; +} + // NOTE: this is just for split data, decimal use old doris hash function // Because crc32 hardware is not equal with zlib crc32 inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const PrimitiveType& type, @@ -75,7 +166,7 @@ inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const PrimitiveT return HashUtil::zlib_crc_hash(v, 8, seed); case TYPE_DATE: case TYPE_DATETIME: { - auto* date_val = (const VecDateTimeValue*)v; + const auto* date_val = reinterpret_cast(v); char buf[64]; int date_len = date_val->to_buffer(buf); return HashUtil::zlib_crc_hash(buf, date_len, seed); @@ -94,7 +185,7 @@ inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const PrimitiveT } case TYPE_DECIMALV2: { - const DecimalV2Value* dec_val = (const DecimalV2Value*)v; + const auto* dec_val = reinterpret_cast(v); int64_t int_val = dec_val->int_value(); int32_t frac_val = dec_val->frac_value(); seed = HashUtil::zlib_crc_hash(&int_val, sizeof(int_val), seed); diff --git a/be/test/exec/partitioner/identity_partitioner_test.cpp b/be/test/exec/partitioner/identity_partitioner_test.cpp new file mode 100644 index 00000000000000..9a539aa8d84593 --- /dev/null +++ b/be/test/exec/partitioner/identity_partitioner_test.cpp @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include + +#include "common/object_pool.h" +#include "core/block/block.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/value/decimalv2_value.h" +#include "core/value/ipv4_value.h" +#include "core/value/ipv6_value.h" +#include "core/value/vdatetime_value.h" +#include "exec/partitioner/partitioner.h" +#include "runtime/descriptor_helper.h" +#include "runtime/descriptors.h" +#include "testutil/column_helper.h" +#include "testutil/mock/mock_runtime_state.h" +#include "util/raw_value.h" + +namespace doris { + +// Unit tests for the BE-side identity reshuffle partitioner used by bucket-shuffle join when the +// target table is bucketed with distribution_hash_type = identity. It must interpret every value's +// canonical little-endian bytes as unsigned and compose multiple columns identically to FE pruning +// and BE tablet routing. +class IdentityPartitionerTest : public ::testing::Test { +protected: + void SetUp() override { + TDescriptorTableBuilder dtb; + TTupleDescriptorBuilder tuple_builder; + tuple_builder.add_slot(TSlotDescriptorBuilder() + .type(TYPE_INT) + .nullable(true) + .column_name("c1") + .column_pos(1) + .build()); + tuple_builder.add_slot(TSlotDescriptorBuilder() + .type(TYPE_STRING) + .nullable(false) + .column_name("c2") + .column_pos(2) + .build()); + tuple_builder.build(&dtb); + TDescriptorTable thrift_tbl = dtb.desc_tbl(); + + DescriptorTbl* desc_tbl = nullptr; + auto st = DescriptorTbl::create(&_pool, thrift_tbl, &desc_tbl); + ASSERT_TRUE(st.ok()) << st.to_string(); + _state.set_desc_tbl(desc_tbl); + + _tuple_id = thrift_tbl.tupleDescriptors[0].id; + _row_desc = std::make_unique(*desc_tbl, std::vector {_tuple_id}); + _slot_ids.push_back(thrift_tbl.slotDescriptors[0].id); + _slot_ids.push_back(thrift_tbl.slotDescriptors[1].id); + } + + TExpr make_slot_ref(size_t slot_index, PrimitiveType type, bool nullable) { + TExprNode node; + node.__set_node_type(TExprNodeType::SLOT_REF); + node.__set_num_children(0); + TSlotRef slot_ref; + slot_ref.__set_slot_id(_slot_ids[slot_index]); + slot_ref.__set_tuple_id(_tuple_id); + node.__set_slot_ref(slot_ref); + TTypeDesc type_desc = create_type_desc(type); + type_desc.__set_is_nullable(nullable); + node.__set_type(type_desc); + node.__set_is_nullable(nullable); + TExpr expr; + expr.nodes.emplace_back(std::move(node)); + return expr; + } + + TExpr make_int_slot_ref() { return make_slot_ref(0, TYPE_INT, true); } + + TExpr make_string_slot_ref() { return make_slot_ref(1, TYPE_STRING, false); } + + template + std::vector run(int partition_count, Block block, + std::vector exprs) { + Partitioner partitioner(partition_count); + EXPECT_TRUE(partitioner.init(exprs).ok()); + EXPECT_TRUE(partitioner.prepare(&_state, *_row_desc).ok()); + EXPECT_TRUE(partitioner.open(&_state).ok()); + EXPECT_TRUE(partitioner.do_partitioning(&_state, &block).ok()); + return partitioner.get_channel_ids(); + } + + template + std::vector run(int partition_count, Block block) { + return run(partition_count, std::move(block), {make_int_slot_ref()}); + } + + ObjectPool _pool; + MockRuntimeState _state; + std::unique_ptr _row_desc; + TTupleId _tuple_id = 0; + std::vector _slot_ids; +}; + +// Positive integers retain value-modulo behavior; for a power-of-two bucket count, two's-complement +// unsigned bytes also place negative values in the same buckets as negative-safe signed modulo. +TEST_F(IdentityPartitionerTest, ChannelIsValueModBucketCount) { + constexpr int n = 8; + std::vector values = {3, 8, 100, 999, -1, -8}; + auto channels = + run(n, ColumnHelper::create_block(values)); + ASSERT_EQ(values.size(), channels.size()); + for (size_t i = 0; i < values.size(); i++) { + EXPECT_EQ(static_cast(((values[i] % n) + n) % n), channels[i]) + << "row " << i << " value " << values[i]; + } +} + +// Canonical two's-complement bytes are unsigned, so negative values need no special branch. +TEST_F(IdentityPartitionerTest, NegativeValueUsesUnsignedBytes) { + constexpr int n = 10; + auto channels = + run(n, ColumnHelper::create_block({-1, -8})); + ASSERT_EQ(2u, channels.size()); + EXPECT_EQ(5u, channels[0]); // UINT32_MAX % 10 + EXPECT_EQ(8u, channels[1]); // (UINT32_MAX - 7) % 10 +} + +TEST_F(IdentityPartitionerTest, SupportsMultipleTypedColumns) { + constexpr int n = 257; + auto block = ColumnHelper::create_block({1, 2}); + auto strings = ColumnHelper::create_block({"A", "BC"}); + block.insert(strings.get_by_position(0)); + auto channels = run(n, std::move(block), + {make_int_slot_ref(), make_string_slot_ref()}); + ASSERT_EQ(2u, channels.size()); + EXPECT_EQ(64u, channels[0]); // (1 * 256 + 'A') % 257 + // unsigned_le("BC") = 0x4342; append it after uint32_le(2). + EXPECT_EQ((2u * 256u * 256u + 0x4342u) % n, channels[1]); +} + +// A null distribution value is represented by four zero bytes. +TEST_F(IdentityPartitionerTest, NullGoesToChannelZero) { + constexpr int n = 8; + // row 0 null -> 0; row 1 = 300 -> 300 % 8 = 4 + auto channels = run( + n, ColumnHelper::create_nullable_block({0, 300}, {1, 0})); + ASSERT_EQ(2u, channels.size()); + EXPECT_EQ(0u, channels[0]); + EXPECT_EQ(4u, channels[1]); +} + +// Guard against the two branches being swapped: crc32 reshuffle must differ from identity for at +// least one row (crc32 does not collapse to value % n). +TEST_F(IdentityPartitionerTest, Crc32DiffersFromIdentity) { + constexpr int n = 8; + std::vector values = {3, 8, 100, 999, 5, 6, 7, 12}; + auto identity = + run(n, ColumnHelper::create_block(values)); + auto crc32 = run>( + n, ColumnHelper::create_block(values)); + ASSERT_EQ(values.size(), identity.size()); + ASSERT_EQ(values.size(), crc32.size()); + bool differs = false; + for (size_t i = 0; i < values.size(); i++) { + if (identity[i] != crc32[i]) { + differs = true; + break; + } + } + EXPECT_TRUE(differs); +} + +TEST(IdentityHashTest, FixedWidthAndLegacyTypes) { + constexpr uint32_t n = 257; + auto hash_bytes = [](const void* value, size_t size, uint32_t seed = 0) { + const auto* bytes = reinterpret_cast(value); + uint64_t remainder = seed; + for (size_t i = size; i > 0; --i) { + remainder = (remainder * 256 + bytes[i - 1]) % n; + } + return static_cast(remainder); + }; + + std::array bytes {}; + bytes[0] = 0x34; + bytes[1] = 0x12; + EXPECT_EQ(hash_bytes(bytes.data(), 2), + RawValue::identity_hash(bytes.data(), 2, TYPE_VARCHAR, 0, n)); + EXPECT_EQ(hash_bytes(bytes.data(), 1), + RawValue::identity_hash(bytes.data(), bytes.size(), TYPE_BOOLEAN, 0, n)); + EXPECT_EQ(hash_bytes(bytes.data(), 2), + RawValue::identity_hash(bytes.data(), bytes.size(), TYPE_SMALLINT, 0, n)); + EXPECT_EQ(hash_bytes(bytes.data(), 8), + RawValue::identity_hash(bytes.data(), bytes.size(), TYPE_BIGINT, 0, n)); + EXPECT_EQ(hash_bytes(bytes.data(), 16), + RawValue::identity_hash(bytes.data(), bytes.size(), TYPE_LARGEINT, 0, n)); + EXPECT_EQ(hash_bytes(bytes.data(), bytes.size()), + RawValue::identity_hash(bytes.data(), bytes.size(), TYPE_DECIMAL256, 0, n)); + + auto date = VecDateTimeValue::create_from_olap_date(20260102); + char date_buffer[64]; + const int date_length = date.to_buffer(date_buffer); + EXPECT_EQ(hash_bytes(date_buffer, date_length), + RawValue::identity_hash(&date, sizeof(date), TYPE_DATE, 0, n)); + + const DecimalV2Value decimal(123, 456000000); + const int32_t fraction = decimal.frac_value(); + const int64_t integer = decimal.int_value(); + const uint32_t fraction_hash = hash_bytes(&fraction, sizeof(fraction)); + EXPECT_EQ(hash_bytes(&integer, sizeof(integer), fraction_hash), + RawValue::identity_hash(&decimal, sizeof(decimal), TYPE_DECIMALV2, 0, n)); +} + +TEST(IdentityHashTest, IpCanonicalBytes) { + constexpr uint32_t n = 257; + IPv4 ipv4 = 0; + ASSERT_TRUE(IPv4Value::from_string(ipv4, "1.2.3.4")); + EXPECT_EQ(2u, RawValue::identity_hash(&ipv4, sizeof(ipv4), TYPE_IPV4, 0, n)); + + IPv6 ipv6 = 0; + ASSERT_TRUE(IPv6Value::from_string(ipv6, "::1")); + EXPECT_EQ(1u, RawValue::identity_hash(&ipv6, sizeof(ipv6), TYPE_IPV6, 0, n)); +} + +} // namespace doris diff --git a/be/test/exec/pipeline/local_exchanger_test.cpp b/be/test/exec/pipeline/local_exchanger_test.cpp index 0967c6758bd79f..084e5d78d19915 100644 --- a/be/test/exec/pipeline/local_exchanger_test.cpp +++ b/be/test/exec/pipeline/local_exchanger_test.cpp @@ -18,7 +18,11 @@ #include #include +#include +#include #include +#include +#include #include "common/status.h" #include "core/assert_cast.h" @@ -69,6 +73,32 @@ class LocalExchangerTest : public testing::Test { const int DUMMY_PORT = config::brpc_port; }; +TEST_F(LocalExchangerTest, BucketShufflePartitionerHashType) { + const std::vector exprs; + const std::map bucket_seq_to_instance_idx {{0, 0}}; + + LocalExchangeSinkOperatorX crc32_op(0, 0, 1, exprs, bucket_seq_to_instance_idx, + TDistributionHashType::CRC32); + EXPECT_TRUE(crc32_op.init(_runtime_state.get(), TLocalPartitionType::BUCKET_HASH_SHUFFLE, 1, + bucket_seq_to_instance_idx) + .ok()); + + LocalExchangeSinkOperatorX identity_op(1, 0, 1, exprs, bucket_seq_to_instance_idx, + TDistributionHashType::IDENTITY); + EXPECT_TRUE(identity_op + .init(_runtime_state.get(), TLocalPartitionType::BUCKET_HASH_SHUFFLE, 1, + bucket_seq_to_instance_idx) + .ok()); + + LocalExchangeSinkOperatorX invalid_op( + 2, 0, 1, exprs, bucket_seq_to_instance_idx, + static_cast(std::numeric_limits::max())); + auto status = invalid_op.init(_runtime_state.get(), TLocalPartitionType::BUCKET_HASH_SHUFFLE, 1, + bucket_seq_to_instance_idx); + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("unsupported distribution_hash_type"), std::string::npos); +} + TEST_F(LocalExchangerTest, ShuffleExchanger) { int num_sink = 4; int num_sources = 4; diff --git a/be/test/exec/sink/sink_test_utils.h b/be/test/exec/sink/sink_test_utils.h index 9e653549f906b8..ca8f92f953a37c 100644 --- a/be/test/exec/sink/sink_test_utils.h +++ b/be/test/exec/sink/sink_test_utils.h @@ -222,6 +222,40 @@ inline TOlapTableLocationParam build_location_param() { return location; } +// A single range partition [-1000, 1000) with `num_buckets` tablets (ids 300, 301, ...), +// bucketed by the integer column "c1" using the given distribution hash type. The range spans +// negatives so identity's unsigned two's-complement byte handling can be exercised end-to-end. +inline TOlapTablePartitionParam build_single_col_partition_param( + int64_t schema_index_id, int32_t num_buckets, TDistributionHashType::type hash_type) { + TOlapTablePartitionParam param; + param.db_id = 1; + param.table_id = 2; + param.version = 0; + + param.__set_partition_type(TPartitionType::RANGE_PARTITIONED); + param.__set_partition_columns({"c1"}); + param.__set_distributed_columns({"c1"}); + param.__set_distribution_hash_type(hash_type); + + TOlapTablePartition p1; + p1.id = 1; + p1.num_buckets = num_buckets; + p1.__set_is_mutable(true); + { + TOlapTableIndexTablets index_tablets; + index_tablets.index_id = schema_index_id; + for (int32_t i = 0; i < num_buckets; i++) { + index_tablets.tablets.push_back(300 + i); + } + p1.indexes = {index_tablets}; + } + p1.__set_start_keys({make_int_literal(-1000)}); + p1.__set_end_keys({make_int_literal(1000)}); + + param.partitions = {p1}; + return param; +} + } // namespace sink_test_utils } // namespace doris diff --git a/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp b/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp index ba587b3c3ba89b..c5322a5da5c449 100644 --- a/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp +++ b/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp @@ -275,5 +275,147 @@ TEST(TabletSinkHashPartitionerTest, OlapTabletFinderRoundRobinEveryBatch) { EXPECT_EQ(tablet_index[0], 0); } } + +// identity distribution_hash_type: canonical bytes are interpreted as unsigned, bit-identical +// with FE pruning. +TEST(TabletSinkHashPartitionerTest, IdentityBucketingModsValueByNumBuckets) { + OperatorContext ctx; + constexpr int32_t num_buckets = 8; + + TOlapTableSchemaParam tschema; + TTupleId tablet_sink_tuple_id = 0; + int64_t schema_index_id = 0; + sink_test_utils::build_desc_tbl_and_schema(ctx, tschema, tablet_sink_tuple_id, schema_index_id, + false); + + auto schema = std::make_shared(); + auto st = schema->init(tschema); + ASSERT_TRUE(st.ok()) << st.to_string(); + + auto tpartition = sink_test_utils::build_single_col_partition_param( + schema_index_id, num_buckets, TDistributionHashType::IDENTITY); + auto vpartition = std::make_unique(schema, tpartition); + st = vpartition->init(); + ASSERT_TRUE(st.ok()) << st.to_string(); + + OlapTabletFinder finder(vpartition.get(), + OlapTabletFinder::FindTabletMode::FIND_TABLET_EVERY_ROW); + + // 3 -> 3, 8 -> 0, 100 -> 4, 999 -> 7, -1 -> 7, -8 -> 0. + auto block = ColumnHelper::create_block({3, 8, 100, 999, -1, -8}); + std::vector partitions(block.rows(), nullptr); + std::vector tablet_index(block.rows(), 0); + std::vector skip(block.rows(), false); + + st = finder.find_tablets(&ctx.state, &block, cast_set(block.rows()), partitions, + tablet_index, skip, nullptr); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_EQ(tablet_index[0], 3u); + EXPECT_EQ(tablet_index[1], 0u); + EXPECT_EQ(tablet_index[2], 4u); + EXPECT_EQ(tablet_index[3], 7u); + EXPECT_EQ(tablet_index[4], 7u); // UINT32_MAX % 8 + EXPECT_EQ(tablet_index[5], 0u); // (UINT32_MAX - 7) % 8 +} + +// identity with a null distribution value falls into bucket 0 (FE/BE write the same rule). +TEST(TabletSinkHashPartitionerTest, IdentityNullGoesToBucketZero) { + OperatorContext ctx; + constexpr int32_t num_buckets = 8; + + TOlapTableSchemaParam tschema; + TTupleId tablet_sink_tuple_id = 0; + int64_t schema_index_id = 0; + // nullable distribution column + sink_test_utils::build_desc_tbl_and_schema(ctx, tschema, tablet_sink_tuple_id, schema_index_id, + true); + + auto schema = std::make_shared(); + auto st = schema->init(tschema); + ASSERT_TRUE(st.ok()) << st.to_string(); + + auto tpartition = sink_test_utils::build_single_col_partition_param( + schema_index_id, num_buckets, TDistributionHashType::IDENTITY); + auto vpartition = std::make_unique(schema, tpartition); + st = vpartition->init(); + ASSERT_TRUE(st.ok()) << st.to_string(); + + OlapTabletFinder finder(vpartition.get(), + OlapTabletFinder::FindTabletMode::FIND_TABLET_EVERY_ROW); + + // row 0 null -> bucket 0; row 1 = 300 -> 300 % 8 = 4 + auto block = ColumnHelper::create_nullable_block({0, 300}, {1, 0}); + std::vector partitions(block.rows(), nullptr); + std::vector tablet_index(block.rows(), 0); + std::vector skip(block.rows(), false); + + st = finder.find_tablets(&ctx.state, &block, cast_set(block.rows()), partitions, + tablet_index, skip, nullptr); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_EQ(tablet_index[0], 0u); // null -> 0 + EXPECT_EQ(tablet_index[1], 4u); // 300 % 8 = 4 +} + +// crc32 (default) must NOT collapse to value % n; guards the two branches from being swapped. +TEST(TabletSinkHashPartitionerTest, Crc32DiffersFromIdentity) { + OperatorContext ctx; + constexpr int32_t num_buckets = 8; + + TOlapTableSchemaParam tschema; + TTupleId tablet_sink_tuple_id = 0; + int64_t schema_index_id = 0; + sink_test_utils::build_desc_tbl_and_schema(ctx, tschema, tablet_sink_tuple_id, schema_index_id, + false); + + auto schema = std::make_shared(); + auto st = schema->init(tschema); + ASSERT_TRUE(st.ok()) << st.to_string(); + + std::vector values = {3, 8, 100, 999, 5, 6, 7, 12}; + + auto identity_param = sink_test_utils::build_single_col_partition_param( + schema_index_id, num_buckets, TDistributionHashType::IDENTITY); + auto identity_part = std::make_unique(schema, identity_param); + ASSERT_TRUE(identity_part->init().ok()); + OlapTabletFinder identity_finder(identity_part.get(), + OlapTabletFinder::FindTabletMode::FIND_TABLET_EVERY_ROW); + auto block1 = ColumnHelper::create_block(values); + std::vector parts1(block1.rows(), nullptr); + std::vector identity_index(block1.rows(), 0); + std::vector skip1(block1.rows(), false); + ASSERT_TRUE(identity_finder + .find_tablets(&ctx.state, &block1, cast_set(block1.rows()), parts1, + identity_index, skip1, nullptr) + .ok()); + + auto crc32_param = sink_test_utils::build_single_col_partition_param( + schema_index_id, num_buckets, TDistributionHashType::CRC32); + auto crc32_part = std::make_unique(schema, crc32_param); + ASSERT_TRUE(crc32_part->init().ok()); + OlapTabletFinder crc32_finder(crc32_part.get(), + OlapTabletFinder::FindTabletMode::FIND_TABLET_EVERY_ROW); + auto block2 = ColumnHelper::create_block(values); + std::vector parts2(block2.rows(), nullptr); + std::vector crc32_index(block2.rows(), 0); + std::vector skip2(block2.rows(), false); + ASSERT_TRUE(crc32_finder + .find_tablets(&ctx.state, &block2, cast_set(block2.rows()), parts2, + crc32_index, skip2, nullptr) + .ok()); + + // identity locates value % n; crc32 must differ for at least one row. + for (size_t i = 0; i < values.size(); i++) { + EXPECT_EQ(identity_index[i], + static_cast(((values[i] % num_buckets) + num_buckets) % num_buckets)); + } + bool differs = false; + for (size_t i = 0; i < values.size(); i++) { + if (crc32_index[i] != identity_index[i]) { + differs = true; + break; + } + } + EXPECT_TRUE(differs); +} } // anonymous namespace } // namespace doris diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/IPv4Literal.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/IPv4Literal.java index 1c57e69cf9b0dd..3d34b6523b15f3 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/IPv4Literal.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/IPv4Literal.java @@ -17,11 +17,15 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import com.google.gson.annotations.SerializedName; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + public class IPv4Literal extends LiteralExpr { public static final long IPV4_MIN = 0L; // 0.0.0.0 @@ -159,6 +163,14 @@ public String getStringValue() { return parseLongToIPv4(this.value); } + @Override + public ByteBuffer getHashValue(PrimitiveType type) { + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt((int) value); + buffer.flip(); + return buffer; + } + public long getValue() { return value; } diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/IPv6Literal.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/IPv6Literal.java index fb9a06b7ac8847..265be262338aa7 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/IPv6Literal.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/IPv6Literal.java @@ -17,12 +17,14 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import com.google.gson.annotations.SerializedName; import com.googlecode.ipv6.IPv6Address; +import java.nio.ByteBuffer; import java.util.regex.Pattern; public class IPv6Literal extends LiteralExpr { @@ -141,6 +143,17 @@ public String getStringValue() { return this.value; } + @Override + public ByteBuffer getHashValue(PrimitiveType type) { + byte[] networkOrder = parseAddress(value).toByteArray(); + ByteBuffer buffer = ByteBuffer.allocate(networkOrder.length); + for (int i = networkOrder.length - 1; i >= 0; i--) { + buffer.put(networkOrder[i]); + } + buffer.flip(); + return buffer; + } + public String getValue() { return value; } diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java index 96a4014bd59a00..b01e73f814a170 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java @@ -17,9 +17,13 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + public class TimeV2Literal extends LiteralExpr { public static final TimeV2Literal MIN_VALUE = new TimeV2Literal(838, 59, 59, 999999, 6, true); public static final TimeV2Literal MAX_VALUE = new TimeV2Literal(838, 59, 59, 999999, 6, false); @@ -126,6 +130,14 @@ public String getStringValue() { return sb.toString(); } + @Override + public ByteBuffer getHashValue(PrimitiveType type) { + ByteBuffer buffer = ByteBuffer.allocate(Double.BYTES).order(ByteOrder.LITTLE_ENDIAN); + buffer.putDouble(getValue()); + buffer.flip(); + return buffer; + } + protected static boolean checkRange(int hour, int minute, int second, int microsecond) { return hour > 838 || minute > 59 || second > 59 || microsecond > 999999 || minute < 0 || second < 0 || microsecond < 0; diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/VarBinaryLiteral.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/VarBinaryLiteral.java index b081e7be17f0ba..9cda95a4697aa7 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/VarBinaryLiteral.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/VarBinaryLiteral.java @@ -17,12 +17,14 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import com.google.common.io.BaseEncoding; import com.google.gson.annotations.SerializedName; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; public class VarBinaryLiteral extends LiteralExpr { @@ -115,6 +117,11 @@ public int compareLiteral(LiteralExpr other) { + this + " (" + this.type + ") vs " + other + " (" + ((LiteralExpr) other).type + ")"); } + @Override + public ByteBuffer getHashValue(PrimitiveType type) { + return ByteBuffer.wrap(value); + } + @Override public String getStringValue() { return new String(value, StandardCharsets.ISO_8859_1); diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index aef69d206fe317..24b91131acfbd7 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1996,7 +1996,7 @@ public class Config extends ConfigBase { * Max data version of backends serialize block. */ @ConfField(mutable = false) - public static int max_be_exec_version = 13; + public static int max_be_exec_version = 14; /** * Min data version of backends serialize block. diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java index 8f5fe32bb302b2..5b762390e66dea 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/ErrorCode.java @@ -1136,6 +1136,8 @@ public enum ErrorCode { "Colocate tables distribution columns size must be same: %s should be %s"), ERR_COLOCATE_TABLE_MUST_HAS_SAME_DISTRIBUTION_COLUMN_TYPE(5063, new byte[]{'4', '2', '0', '0', '0'}, "Colocate tables distribution columns must have the same data type: %s should be %s"), + ERR_COLOCATE_TABLE_MUST_HAS_SAME_DISTRIBUTION_HASH_TYPE(5063, new byte[]{'4', '2', '0', '0', '0'}, + "Colocate tables must have same distribution hash type: %s should be %s"), ERR_COLOCATE_NOT_COLOCATE_TABLE(5064, new byte[]{'4', '2', '0', '0', '0'}, "Table %s is not a colocated table"), ERR_INVALID_OPERATION(5065, new byte[]{'4', '2', '0', '0', '0'}, "Operation %s is invalid"), diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java index 746ca81f6f1dc3..07b8e0a77eadfa 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -102,9 +102,11 @@ public final class FeMetaVersion { public static final int VERSION_139 = 139; public static final int VERSION_140 = 140; + // add group-level distribution_hash_type in ColocateGroupSchema + public static final int VERSION_141 = 141; // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_140; + public static final int VERSION_CURRENT = VERSION_141; // all logs meta version should >= the minimum version, so that we could remove many if clause, for example diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/HashDistributionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/HashDistributionDesc.java index 4509a71440c7d7..feb8a002a56dde 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/HashDistributionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/HashDistributionDesc.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DistributionInfo; import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.HashDistributionInfo.HashType; import org.apache.doris.catalog.KeysType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; @@ -33,15 +34,25 @@ public class HashDistributionDesc extends DistributionDesc { private List distributionColumnNames; + private HashType hashType; public HashDistributionDesc(int numBucket, List distributionColumnNames) { super(numBucket); this.distributionColumnNames = distributionColumnNames; + this.hashType = HashType.CRC32; } public HashDistributionDesc(int numBucket, boolean autoBucket, List distributionColumnNames) { super(numBucket, autoBucket); this.distributionColumnNames = distributionColumnNames; + this.hashType = HashType.CRC32; + } + + public HashDistributionDesc(int numBucket, boolean autoBucket, List distributionColumnNames, + HashType hashType) { + super(numBucket, autoBucket); + this.distributionColumnNames = distributionColumnNames; + this.hashType = hashType; } @Override @@ -126,13 +137,16 @@ public DistributionInfo toDistributionInfo(List columns) throws DdlExcep } } - HashDistributionInfo hashDistributionInfo = - new HashDistributionInfo(numBucket, autoBucket, distributionColumns); + HashDistributionInfo hashDistributionInfo + = new HashDistributionInfo(numBucket, autoBucket, distributionColumns, hashType); return hashDistributionInfo; } @Override public DistributionDescriptor toDistributionDescriptor() { - return new DistributionDescriptor(true, this.autoBucket, this.numBucket, this.distributionColumnNames); + DistributionDescriptor descriptor + = new DistributionDescriptor(true, this.autoBucket, this.numBucket, this.distributionColumnNames); + descriptor.updateHashType(hashType); + return descriptor; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java index 0860f89eb169b4..4d9f0f7737a603 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java @@ -21,6 +21,8 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import com.google.common.collect.Lists; @@ -45,17 +47,25 @@ public class ColocateGroupSchema implements Writable { private int bucketsNum; @SerializedName(value = "replicaAlloc") private ReplicaAllocation replicaAlloc; + @SerializedName(value = "hashType") + private HashDistributionInfo.HashType hashType; private ColocateGroupSchema() { } - public ColocateGroupSchema(GroupId groupId, List distributionCols, - int bucketsNum, ReplicaAllocation replicaAlloc) { + public ColocateGroupSchema(GroupId groupId, List distributionCols, int bucketsNum, + ReplicaAllocation replicaAlloc) { + this(groupId, distributionCols, bucketsNum, replicaAlloc, HashDistributionInfo.HashType.CRC32); + } + + public ColocateGroupSchema(GroupId groupId, List distributionCols, int bucketsNum, + ReplicaAllocation replicaAlloc, HashDistributionInfo.HashType hashType) { this.groupId = groupId; this.distributionColTypes = distributionCols.stream().map(c -> c.getType()).collect(Collectors.toList()); this.bucketsNum = bucketsNum; this.replicaAlloc = replicaAlloc; + this.hashType = hashType; } public GroupId getGroupId() { @@ -78,6 +88,12 @@ public List getDistributionColTypes() { return distributionColTypes; } + public HashDistributionInfo.HashType getHashType() { + return hashType == null + ? HashDistributionInfo.HashType.CRC32 + : hashType; + } + public void checkColocateSchema(OlapTable tbl) throws DdlException { checkDistribution(tbl.getDefaultDistributionInfo()); // We add a table with many partitions to the colocate group, @@ -91,6 +107,11 @@ public void checkColocateSchema(OlapTable tbl) throws DdlException { public void checkDistribution(DistributionInfo distributionInfo) throws DdlException { if (distributionInfo instanceof HashDistributionInfo) { HashDistributionInfo info = (HashDistributionInfo) distributionInfo; + // hash type + if (info.getHashType() != getHashType()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COLOCATE_TABLE_MUST_HAS_SAME_DISTRIBUTION_HASH_TYPE, + info.getHashType(), getHashType()); + } // buckets num if (info.getBucketNum() != bucketsNum) { ErrorReport.reportDdlException(ErrorCode.ERR_COLOCATE_TABLE_MUST_HAS_SAME_BUCKET_NUM, @@ -159,6 +180,7 @@ public void write(DataOutput out) throws IOException { } out.writeInt(bucketsNum); this.replicaAlloc.write(out); + Text.writeString(out, getHashType().name()); } public void readFields(DataInput in) throws IOException { @@ -169,5 +191,10 @@ public void readFields(DataInput in) throws IOException { } bucketsNum = in.readInt(); this.replicaAlloc = ReplicaAllocation.read(in); + if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_141) { + this.hashType = HashDistributionInfo.HashType.valueOf(Text.readString(in)); + } else { + this.hashType = HashDistributionInfo.HashType.CRC32; + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java index 29ef3be84d9d67..931efec49f67fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java @@ -223,7 +223,7 @@ public GroupId addTableToGroup(long dbId, OlapTable tbl, String fullGroupName, G HashDistributionInfo distributionInfo = (HashDistributionInfo) tbl.getDefaultDistributionInfo(); ColocateGroupSchema groupSchema = new ColocateGroupSchema(groupId, distributionInfo.getDistributionColumns(), distributionInfo.getBucketNum(), - tbl.getDefaultReplicaAllocation()); + tbl.getDefaultReplicaAllocation(), distributionInfo.getHashType()); groupName2Id.put(fullGroupName, groupId); group2Schema.put(groupId, groupSchema); group2ErrMsgs.put(groupId, ""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 081a2555678e4e..60ed868dd49318 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3986,6 +3986,14 @@ private static void addOlapTablePropertyInfo(OlapTable olapTable, StringBuilder sb.append(colocateTable).append("\""); } + // distribution hash type (only emit when non-default to keep output stable) + DistributionInfo defaultDistInfo = olapTable.getDefaultDistributionInfo(); + if (defaultDistInfo instanceof HashDistributionInfo + && ((HashDistributionInfo) defaultDistInfo).getHashType() != HashDistributionInfo.HashType.CRC32) { + sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_DISTRIBUTION_HASH_TYPE).append("\" = \""); + sb.append(((HashDistributionInfo) defaultDistInfo).getHashType().name().toLowerCase()).append("\""); + } + // dynamic partition if (olapTable.dynamicPartitionExists()) { sb.append(olapTable.getTableProperty().getDynamicPartitionProperty().getProperties(replicaAlloc)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java index a1f4688cb66693..37f19c1e5646d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java @@ -33,28 +33,58 @@ * Hash Distribution Info. */ public class HashDistributionInfo extends DistributionInfo { + + /** + * Hash function type used by HASH distribution to map rows to buckets. + * + * CRC32 (legacy behavior) is the default for backward compatibility. + */ + public enum HashType { + CRC32, IDENTITY; + } + @SerializedName(value = "distributionColumns") private List distributionColumns; + @SerializedName(value = "hashType") + private HashType hashType; + public HashDistributionInfo() { super(); this.distributionColumns = new ArrayList(); + this.hashType = HashType.CRC32; } public HashDistributionInfo(int bucketNum, List distributionColumns) { super(DistributionInfoType.HASH, bucketNum); this.distributionColumns = distributionColumns; + this.hashType = HashType.CRC32; } public HashDistributionInfo(int bucketNum, boolean autoBucket, List distributionColumns) { super(DistributionInfoType.HASH, bucketNum, autoBucket); this.distributionColumns = distributionColumns; + this.hashType = HashType.CRC32; + } + + public HashDistributionInfo(int bucketNum, boolean autoBucket, List distributionColumns, + HashType hashType) { + super(DistributionInfoType.HASH, bucketNum, autoBucket); + this.distributionColumns = distributionColumns; + this.hashType = hashType; } public List getDistributionColumns() { return distributionColumns; } + // null-safe defense against old versions persisted before hashType existed. + public HashType getHashType() { + return hashType == null + ? HashType.CRC32 + : hashType; + } + public static void checkDistributionColumnType(String columnName, Type type) throws DdlException { if (type.isArrayType()) { throw new DdlException("Array Type should not be used in distribution column[" + columnName + "]."); @@ -101,12 +131,12 @@ public boolean equals(Object o) { return false; } HashDistributionInfo that = (HashDistributionInfo) o; - return bucketNum == that.bucketNum && sameDistributionColumns(that); + return bucketNum == that.bucketNum && sameDistributionColumns(that) && getHashType() == that.getHashType(); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), distributionColumns, bucketNum); + return Objects.hash(super.hashCode(), distributionColumns, bucketNum, getHashType()); } @Override @@ -115,7 +145,8 @@ public DistributionDesc toDistributionDesc() { for (Column col : distributionColumns) { distriColNames.add(col.getName()); } - DistributionDesc distributionDesc = new HashDistributionDesc(bucketNum, autoBucket, distriColNames); + DistributionDesc distributionDesc + = new HashDistributionDesc(bucketNum, autoBucket, distriColNames, getHashType()); return distributionDesc; } @@ -169,4 +200,8 @@ public RandomDistributionInfo toRandomDistributionInfo() { public void setDistributionColumns(List column) { this.distributionColumns = column; } + + public void setHashType(HashType hashType) { + this.hashType = hashType; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 88ab99432c9f9e..d2e8724d120670 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -2034,6 +2034,7 @@ public String getSignature(int signatureVersion, List partNames) { HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; sb.append(Util.getSchemaSignatureString(hashDistributionInfo.getDistributionColumns())); sb.append(hashDistributionInfo.getBucketNum()); + sb.append(hashDistributionInfo.getHashType()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java index 20e7b73cdc3aab..036a38182bf32c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java @@ -301,6 +301,10 @@ public String getMetaChecksum() { updateMetaChecksum(digest, (byte) 17, distType == null ? -1L : distType.ordinal()); updateMetaChecksum(digest, (byte) 18, distributionInfo.getBucketNum()); updateMetaChecksum(digest, (byte) 19, distributionInfo.getAutoBucket() ? 1L : 0L); + if (distributionInfo instanceof HashDistributionInfo) { + updateMetaChecksum(digest, (byte) 20, + ((HashDistributionInfo) distributionInfo).getHashType().ordinal()); + } } else { updateMetaChecksum(digest, (byte) 17, -1L); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index 3832876cd84401..d5031fcaeeadd0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -253,6 +253,23 @@ public long getHashValue() { return hashValue.getValue(); } + /** + * Treat each distribution value's canonical bytes as an unsigned integer with the first byte + * as the least-significant byte, then append it to the preceding values. Keeping only the + * remainder avoids constructing an arbitrarily wide integer for multi-column keys. + */ + public int getIdentityHashValue(int hashMod) { + Preconditions.checkArgument(hashMod > 0, "hash modulus must be positive"); + long result = 0; + for (int keyIndex = 0; keyIndex < keys.size(); keyIndex++) { + ByteBuffer buffer = keys.get(keyIndex).getHashValue(types.get(keyIndex)); + for (int byteIndex = buffer.limit() - 1; byteIndex >= 0; byteIndex--) { + result = (result * 256 + Byte.toUnsignedInt(buffer.get(byteIndex))) % hashMod; + } + } + return (int) result; + } + public boolean isMinValue() { for (LiteralExpr literalExpr : keys) { if (!literalExpr.isMinValue()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 45acffcd9fd61e..394dedd90b7f4f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -27,6 +27,7 @@ import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.EnvFactory; +import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PrimitiveType; @@ -117,6 +118,8 @@ public class PropertyAnalyzer { public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE = "light_schema_change"; public static final String PROPERTIES_DISTRIBUTION_TYPE = "distribution_type"; + // hash function type when distribution_type is "HASH" + public static final String PROPERTIES_DISTRIBUTION_HASH_TYPE = "distribution_hash_type"; public static final String PROPERTIES_SEND_CLEAR_ALTER_TASK = "send_clear_alter_tasks"; /* * for upgrade alpha rowset to beta rowset, valid value: v1, v2 @@ -795,6 +798,25 @@ public static String analyzeColocate(Map properties) { return colocateGroup; } + // analyze the hash function type of table; defaults to CRC32 + public static HashDistributionInfo.HashType analyzeDistributionHashType(Map properties) + throws AnalysisException { + HashDistributionInfo.HashType hashType = HashDistributionInfo.HashType.CRC32; + if (properties != null && properties.containsKey(PROPERTIES_DISTRIBUTION_HASH_TYPE)) { + String value = properties.get(PROPERTIES_DISTRIBUTION_HASH_TYPE); + properties.remove(PROPERTIES_DISTRIBUTION_HASH_TYPE); + if (value.equalsIgnoreCase("crc32")) { + hashType = HashDistributionInfo.HashType.CRC32; + } else if (value.equalsIgnoreCase("identity")) { + hashType = HashDistributionInfo.HashType.IDENTITY; + } else { + throw new AnalysisException("Invalid " + PROPERTIES_DISTRIBUTION_HASH_TYPE + ": " + value + + ". Supported values are 'crc32' and 'identity'."); + } + } + return hashType; + } + public static long analyzeTimeout(Map properties, long defaultTimeout) throws AnalysisException { long timeout = defaultTimeout; if (properties != null && properties.containsKey(PROPERTIES_TIMEOUT)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 3a825450292042..0366f86620ceb8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1646,6 +1646,9 @@ public void addPartition(Database db, String tableName, AddPartitionOp addPartit + "new is: " + hashDistributionInfo.getDistributionColumns() + " default is: " + ((HashDistributionInfo) defaultDistributionInfo).getDistributionColumns()); } + // New partition inherits the table's hash type, otherwise BE would bucket rows with one + // hash function while FE prunes with another, making the data unreadable. + hashDistributionInfo.setHashType(((HashDistributionInfo) defaultDistributionInfo).getHashType()); } else if (distributionInfo.getType() == DistributionInfoType.RANDOM) { RandomDistributionInfo randomDistributionInfo = (RandomDistributionInfo) distributionInfo; if (randomDistributionInfo.getBucketNum() <= 0) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 1d6923a409f913..6170e8dfc34725 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -379,6 +379,7 @@ public PlanFragment visitPhysicalDistribute(PhysicalDistribute d // target data partition DataPartition targetDataPartition = toDataPartition(targetDistribution, validOutputIds, context); exchangeNode.setPartitionType(targetDataPartition.getType()); + exchangeNode.setDistributionHashType(targetDataPartition.getHashType()); exchangeNode.setDistributeExprLists(getDistributeExpr(distribute)); exchangeNode.setChildrenDistributeExprLists(upstreamDistributeExprs); // its source partition is targetDataPartition. and outputPartition is UNPARTITIONED now, will be set when @@ -3707,7 +3708,9 @@ private DataPartition toDataPartition(DistributionSpec distributionSpec/* target switch (distributionSpecHash.getShuffleType()) { case STORAGE_BUCKETED: partitionType = TPartitionType.BUCKET_SHFFULE_HASH_PARTITIONED; - break; + // Bucket-shuffle re-partitions the shuffled side to the target table's storage + // layout, so the storage hashType must ride along for BE to pick the right partitioner. + return new DataPartition(partitionType, partitionExprs, distributionSpecHash.getHashType()); case EXECUTION_BUCKETED: partitionType = TPartitionType.HASH_PARTITIONED; break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/ShuffleKeyPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/ShuffleKeyPruner.java index 50c540911e4935..f0b40ac5ad6174 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/ShuffleKeyPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/ShuffleKeyPruner.java @@ -489,7 +489,7 @@ private static PhysicalHashAggregate tryPruneGlobalAgg(PhysicalH private static DistributionSpecHash sliceHashSpec(DistributionSpecHash origin, List newOrderedKeys) { return new DistributionSpecHash(newOrderedKeys, origin.getShuffleType(), - origin.getTableId(), origin.getSelectedIndexId(), origin.getPartitionIds()); + origin.getTableId(), origin.getSelectedIndexId(), origin.getPartitionIds(), origin.getHashType()); } private static PhysicalDistribute rebuildDistribute(PhysicalDistribute origin, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java index 2df7723a7ab052..ef29952fd3d5aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.properties; +import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.nereids.PlanContext; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; @@ -453,6 +454,7 @@ public PhysicalProperties visitPhysicalPartitionTopN(PhysicalPartitionTopN childrenDistribution = childrenOutputProperties.stream() .map(PhysicalProperties::getDistributionSpec) .collect(Collectors.toList()); @@ -531,7 +533,8 @@ public PhysicalProperties visitPhysicalSetOperation(PhysicalSetOperation setOper childDistribution.getShuffleType(), childDistribution.getTableId(), childDistribution.getSelectedIndexId(), - childDistribution.getPartitionIds() + childDistribution.getPartitionIds(), + childDistribution.getHashType() ) ); } @@ -561,10 +564,12 @@ public PhysicalProperties visitPhysicalSetOperation(PhysicalSetOperation setOper } } if (offsetsOfFirstChild == null) { - firstType = ((DistributionSpecHash) childDistribution).getShuffleType(); + firstType = distributionSpecHash.getShuffleType(); + firstHashType = distributionSpecHash.getHashType(); offsetsOfFirstChild = offsetsOfCurrentChild; } else if (!Arrays.equals(offsetsOfFirstChild, offsetsOfCurrentChild) - || firstType != ((DistributionSpecHash) childDistribution).getShuffleType()) { + || firstType != distributionSpecHash.getShuffleType() + || firstHashType != distributionSpecHash.getHashType()) { // NOTICE: if come here, the first child output must be DistributionSpecHash return PhysicalProperties.createAnyFromHash((DistributionSpecHash) childrenDistribution.get(0)); } @@ -574,7 +579,8 @@ public PhysicalProperties visitPhysicalSetOperation(PhysicalSetOperation setOper for (int offset : offsetsOfFirstChild) { request.add(setOperation.getOutput().get(offset).getExprId()); } - return PhysicalProperties.createHash(request, firstType); + return new PhysicalProperties(new DistributionSpecHash(request, firstType, + -1L, -1L, Collections.emptySet(), firstHashType)); } @Override @@ -754,7 +760,8 @@ private DistributionSpecHash mockAnotherSideSpecFromConjuncts( } anotherSideOrderedExprIds.add(rightExprIds.get(index)); } - return new DistributionSpecHash(anotherSideOrderedExprIds, oneSideSpec.getShuffleType()); + return new DistributionSpecHash(anotherSideOrderedExprIds, oneSideSpec.getShuffleType(), + -1L, -1L, Collections.emptySet(), oneSideSpec.getHashType()); } private static boolean isSameHashValue(DataType originType, DataType castType) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java index e94e4f7dfae20f..b8975095bf2b14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java @@ -63,6 +63,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; @@ -496,7 +497,8 @@ public List> visitPhysicalHashJoin( } else if (leftHashSpec.getShuffleType() == ShuffleType.NATURAL && rightHashSpec.getShuffleType() == ShuffleType.STORAGE_BUCKETED) { shouldCheckLeftBucketDownGrade = true; - if (!bothSideShuffleKeysAreSameOrder(leftHashSpec, rightHashSpec, + if (leftHashSpec.getHashType() != rightHashSpec.getHashType() + || !bothSideShuffleKeysAreSameOrder(leftHashSpec, rightHashSpec, (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec())) { updatedForRight = Optional.of(calAnotherSideRequired( @@ -591,7 +593,8 @@ public List> visitPhysicalHashJoin( } else if ((leftHashSpec.getShuffleType() == ShuffleType.STORAGE_BUCKETED && rightHashSpec.getShuffleType() == ShuffleType.STORAGE_BUCKETED)) { - if (!bothSideShuffleKeysAreSameOrder(rightHashSpec, leftHashSpec, + if (leftHashSpec.getHashType() != rightHashSpec.getHashType() + || !bothSideShuffleKeysAreSameOrder(rightHashSpec, leftHashSpec, (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec(), (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec())) { if (children.get(0).getPlan() instanceof PhysicalDistribute) { @@ -785,7 +788,8 @@ && canMapBucketKeysToRequire((DistributionSpecHash) childDistribution, List shuffleSideIds = calAnotherSideRequiredShuffleIds( notNeedShuffleOutput, notShuffleSideRequire, currentRequire); PhysicalProperties target = new PhysicalProperties( - new DistributionSpecHash(shuffleSideIds, ShuffleType.STORAGE_BUCKETED)); + new DistributionSpecHash(shuffleSideIds, ShuffleType.STORAGE_BUCKETED, -1L, -1L, + Collections.emptySet(), notNeedShuffleOutput.getHashType())); updateChildEnforceAndCost(i, target); } } else { @@ -956,7 +960,7 @@ private PhysicalProperties calAnotherSideRequired(ShuffleType shuffleType, notNeedShuffleSideRequired, needShuffleSideRequired); return new PhysicalProperties(new DistributionSpecHash(shuffleSideIds, shuffleType, needShuffleSideOutput.getTableId(), needShuffleSideOutput.getSelectedIndexId(), - needShuffleSideOutput.getPartitionIds())); + needShuffleSideOutput.getPartitionIds(), notNeedShuffleSideOutput.getHashType())); } private void updateChildEnforceAndCost(int index, PhysicalProperties targetProperties) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHash.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHash.java index ab96960684a154..47e9727c05dbe5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHash.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHash.java @@ -17,10 +17,12 @@ package org.apache.doris.nereids.properties; +import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.nereids.annotation.Developing; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.util.Utils; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -54,6 +56,10 @@ public class DistributionSpecHash extends DistributionSpec { private final Set partitionIds; private final long selectedIndexId; + // storage bucketing hash function of the NATURAL side; only equal hashType tables may share + // a distribution (colocate / bucket-shuffle). Non-bucketing specs default to CRC32. + private final HashDistributionInfo.HashType hashType; + /** * Use for no need set table related attributes. */ @@ -69,11 +75,17 @@ public DistributionSpecHash(List orderedShuffledColumns, ShuffleType shu this(orderedShuffledColumns, shuffleType, tableId, -1L, partitionIds); } + public DistributionSpecHash(List orderedShuffledColumns, ShuffleType shuffleType, long tableId, + long selectedIndexId, Set partitionIds) { + this(orderedShuffledColumns, shuffleType, tableId, selectedIndexId, partitionIds, + HashDistributionInfo.HashType.CRC32); + } + /** * Normal constructor. */ public DistributionSpecHash(List orderedShuffledColumns, ShuffleType shuffleType, - long tableId, long selectedIndexId, Set partitionIds) { + long tableId, long selectedIndexId, Set partitionIds, HashDistributionInfo.HashType hashType) { this.orderedShuffledColumns = ImmutableList.copyOf( Objects.requireNonNull(orderedShuffledColumns, "orderedShuffledColumns should not null")); this.shuffleType = Objects.requireNonNull(shuffleType, "shuffleType should not null"); @@ -81,6 +93,7 @@ public DistributionSpecHash(List orderedShuffledColumns, ShuffleType shu Objects.requireNonNull(partitionIds, "partitionIds should not null")); this.tableId = tableId; this.selectedIndexId = selectedIndexId; + this.hashType = Objects.requireNonNull(hashType, "hashType should not null"); ImmutableList.Builder> equivalenceExprIdsBuilder = ImmutableList.builderWithExpectedSize(orderedShuffledColumns.size()); ImmutableMap.Builder exprIdToEquivalenceSetBuilder @@ -101,7 +114,14 @@ public DistributionSpecHash(List orderedShuffledColumns, ShuffleType shu long tableId, Set partitionIds, List> equivalenceExprIds, Map exprIdToEquivalenceSet) { this(orderedShuffledColumns, shuffleType, tableId, -1L, partitionIds, - equivalenceExprIds, exprIdToEquivalenceSet); + equivalenceExprIds, exprIdToEquivalenceSet, HashDistributionInfo.HashType.CRC32); + } + + public DistributionSpecHash(List orderedShuffledColumns, ShuffleType shuffleType, long tableId, + long selectedIndexId, Set partitionIds, List> equivalenceExprIds, + Map exprIdToEquivalenceSet) { + this(orderedShuffledColumns, shuffleType, tableId, selectedIndexId, partitionIds, equivalenceExprIds, + exprIdToEquivalenceSet, HashDistributionInfo.HashType.CRC32); } /** @@ -109,12 +129,13 @@ public DistributionSpecHash(List orderedShuffledColumns, ShuffleType shu */ public DistributionSpecHash(List orderedShuffledColumns, ShuffleType shuffleType, long tableId, long selectedIndexId, Set partitionIds, List> equivalenceExprIds, - Map exprIdToEquivalenceSet) { + Map exprIdToEquivalenceSet, HashDistributionInfo.HashType hashType) { this.orderedShuffledColumns = ImmutableList.copyOf(Objects.requireNonNull(orderedShuffledColumns, "orderedShuffledColumns should not null")); this.shuffleType = Objects.requireNonNull(shuffleType, "shuffleType should not null"); this.tableId = tableId; this.selectedIndexId = selectedIndexId; + this.hashType = Objects.requireNonNull(hashType, "hashType should not null"); this.partitionIds = ImmutableSet.copyOf( Objects.requireNonNull(partitionIds, "partitionIds should not null")); this.equivalenceExprIds = ImmutableList.copyOf( @@ -124,6 +145,9 @@ public DistributionSpecHash(List orderedShuffledColumns, ShuffleType shu } static DistributionSpecHash merge(DistributionSpecHash left, DistributionSpecHash right, ShuffleType shuffleType) { + Preconditions.checkState(left.hashType == right.hashType, + "can not merge distribution specs with different hash types: %s vs %s", + left.hashType, right.hashType); List orderedShuffledColumns = left.getOrderedShuffledColumns(); ImmutableList.Builder> equivalenceExprIds = ImmutableList.builderWithExpectedSize(orderedShuffledColumns.size()); @@ -140,7 +164,7 @@ static DistributionSpecHash merge(DistributionSpecHash left, DistributionSpecHas exprIdToEquivalenceSet.putAll(right.getExprIdToEquivalenceSet()); return new DistributionSpecHash(orderedShuffledColumns, shuffleType, left.getTableId(), left.getSelectedIndexId(), left.getPartitionIds(), equivalenceExprIds.build(), - exprIdToEquivalenceSet.buildKeepingLast()); + exprIdToEquivalenceSet.buildKeepingLast(), left.getHashType()); } static DistributionSpecHash merge(DistributionSpecHash left, DistributionSpecHash right) { @@ -163,6 +187,10 @@ public long getSelectedIndexId() { return selectedIndexId; } + public HashDistributionInfo.HashType getHashType() { + return hashType; + } + public Set getPartitionIds() { return partitionIds; } @@ -202,6 +230,7 @@ public boolean satisfy(DistributionSpec required) { return containsSatisfy(requiredHash.getOrderedShuffledColumns()); } return requiredHash.getShuffleType() == this.getShuffleType() + && this.hashType == requiredHash.hashType && equalsSatisfy(requiredHash.getOrderedShuffledColumns()); } @@ -229,12 +258,12 @@ private boolean equalsSatisfy(List required) { public DistributionSpecHash withShuffleType(ShuffleType shuffleType) { return new DistributionSpecHash(orderedShuffledColumns, shuffleType, tableId, selectedIndexId, partitionIds, - equivalenceExprIds, exprIdToEquivalenceSet); + equivalenceExprIds, exprIdToEquivalenceSet, hashType); } public DistributionSpecHash withShuffleTypeAndForbidColocateJoin(ShuffleType shuffleType) { return new DistributionSpecHash(orderedShuffledColumns, shuffleType, -1, -1, partitionIds, - equivalenceExprIds, exprIdToEquivalenceSet); + equivalenceExprIds, exprIdToEquivalenceSet, hashType); } /** @@ -266,7 +295,7 @@ public DistributionSpecHash withShuffleExprs(List prunedOrderedColumns) } return new DistributionSpecHash(ImmutableList.copyOf(prunedOrderedColumns), shuffleType, tableId, selectedIndexId, partitionIds, equivBuilder.build(), - mapBuilder.buildKeepingLast()); + mapBuilder.buildKeepingLast(), hashType); } /** @@ -304,7 +333,7 @@ public DistributionSpec project(Map projections, } } return new DistributionSpecHash(orderedShuffledColumns, shuffleType, tableId, selectedIndexId, partitionIds, - equivalenceExprIds, exprIdToEquivalenceSet); + equivalenceExprIds, exprIdToEquivalenceSet, hashType); } @Override @@ -313,12 +342,13 @@ public boolean equals(Object o) { return false; } DistributionSpecHash that = (DistributionSpecHash) o; - return shuffleType == that.shuffleType && orderedShuffledColumns.equals(that.orderedShuffledColumns); + return shuffleType == that.shuffleType && hashType == that.hashType + && orderedShuffledColumns.equals(that.orderedShuffledColumns); } @Override public int hashCode() { - return Objects.hash(shuffleType, orderedShuffledColumns); + return Objects.hash(shuffleType, hashType, orderedShuffledColumns); } @Override @@ -326,6 +356,7 @@ public String toString() { return Utils.toSqlString("DistributionSpecHash", "orderedShuffledColumns", orderedShuffledColumns, "shuffleType", shuffleType, + "hashType", hashType, "tableId", tableId, "selectedIndexId", selectedIndexId, "partitionIds", partitionIds, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java index 8448f14831cfa7..cb2a310af8acea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java @@ -86,7 +86,12 @@ public static DistributionSpec convertDistribution(LogicalOlapScan olapScan) { boolean isBelongStableCG = Utils.isBelongStableCG(olapTable); boolean isSelectUnpartition = Utils.isSelectUnpartition(olapTable, olapScan.getSelectedPartitionIds()); // TODO: find a better way to handle both tablet num == 1 and colocate table together in future - if (distributionInfo instanceof HashDistributionInfo && (isBelongStableCG || isSelectUnpartition)) { + // Any HASH-bucketed table advertises a NATURAL distribution carrying its bucketing hashType. + // Colocate / bucket-shuffle compatibility is then gated by comparing both sides' hashType, + // so hash types can participate as long as both sides agree. + boolean isHashBucketed = distributionInfo instanceof HashDistributionInfo; + if (isHashBucketed && (isBelongStableCG || isSelectUnpartition)) { + HashDistributionInfo.HashType hashType = ((HashDistributionInfo) distributionInfo).getHashType(); if (olapScan.getSelectedIndexId() != olapScan.getTable().getBaseIndexId()) { HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; List output = olapScan.getOutput(); @@ -115,7 +120,8 @@ public static DistributionSpec convertDistribution(LogicalOlapScan olapScan) { } } return new DistributionSpecHash(hashColumns, ShuffleType.NATURAL, olapScan.getTable().getId(), - olapScan.getSelectedIndexId(), Sets.newLinkedHashSet(olapScan.getSelectedPartitionIds())); + olapScan.getSelectedIndexId(), Sets.newLinkedHashSet(olapScan.getSelectedPartitionIds()), + hashType); } else { HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; List output = olapScan.getOutput(); @@ -133,7 +139,8 @@ public static DistributionSpec convertDistribution(LogicalOlapScan olapScan) { } } return new DistributionSpecHash(hashColumns, ShuffleType.NATURAL, olapScan.getTable().getId(), - olapScan.getSelectedIndexId(), Sets.newLinkedHashSet(olapScan.getSelectedPartitionIds())); + olapScan.getSelectedIndexId(), Sets.newLinkedHashSet(olapScan.getSelectedPartitionIds()), + hashType); } } else { // RandomDistributionInfo diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java index 20f8c1d6e48d20..3663cc102941f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java @@ -110,6 +110,7 @@ private Collection getSelectedTabletIds(List schema, Map cols; + // Default to CRC32 so hash paths that never call setHashType (e.g. CreateMTMVInfo/CreateTableInfo) still + // translate to a non-null hash type. + private HashType hashType = HashType.CRC32; public DistributionDescriptor(boolean isHash, boolean isAutoBucket, int bucketNum, List cols) { this.isHash = isHash; @@ -69,6 +73,10 @@ public void updateBucketNum(int bucketNum) { this.bucketNum = bucketNum; } + public void updateHashType(HashType hashType) { + this.hashType = hashType; + } + /** * analyze distribution descriptor */ @@ -122,7 +130,7 @@ public void validate(Map columnMap, KeysType keysType) public DistributionDesc translateToCatalogStyle() { if (isHash) { - return new HashDistributionDesc(bucketNum, isAutoBucket, cols); + return new HashDistributionDesc(bucketNum, isAutoBucket, cols, hashType); } return new RandomDistributionDesc(bucketNum, isAutoBucket); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index 20292df56ca7a5..0b6753cf23724b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -262,6 +262,9 @@ public static boolean couldColocateJoin(DistributionSpecHash leftHashSpec, Distr || rightHashSpec.getShuffleType() != ShuffleType.NATURAL) { return false; } + if (leftHashSpec.getHashType() != rightHashSpec.getHashType()) { + return false; + } final long leftTableId = leftHashSpec.getTableId(); final long rightTableId = rightHashSpec.getTableId(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java index 0ef85f8ee67170..afc7fcde240a58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataPartition.java @@ -24,7 +24,9 @@ import org.apache.doris.analysis.ExprToSqlVisitor; import org.apache.doris.analysis.ExprToThriftVisitor; import org.apache.doris.analysis.ToSqlParams; +import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.thrift.TDataPartition; +import org.apache.doris.thrift.TDistributionHashType; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TIcebergPartitionField; import org.apache.doris.thrift.TMergePartitionInfo; @@ -55,6 +57,8 @@ public class DataPartition { // for hash partition: exprs used to compute hash value private ImmutableList partitionExprs; private MergePartitionInfo mergePartitionInfo; + // storage bucketing hash for BUCKET_SHFFULE_HASH_PARTITIONED; defaults to CRC32 (legacy behavior) + private HashDistributionInfo.HashType hashType = HashDistributionInfo.HashType.CRC32; public DataPartition(TPartitionType type, List exprs) { Preconditions.checkNotNull(exprs); @@ -67,6 +71,11 @@ public DataPartition(TPartitionType type, List exprs) { this.partitionExprs = ImmutableList.copyOf(exprs); } + public DataPartition(TPartitionType type, List exprs, HashDistributionInfo.HashType hashType) { + this(type, exprs); + this.hashType = hashType == null ? HashDistributionInfo.HashType.CRC32 : hashType; + } + public DataPartition(TPartitionType type) { Preconditions.checkState(type == TPartitionType.UNPARTITIONED || type == TPartitionType.RANDOM @@ -102,6 +111,17 @@ public List getPartitionExprs() { return partitionExprs; } + public HashDistributionInfo.HashType getHashType() { + return hashType; + } + + public static TDistributionHashType toTHashType(HashDistributionInfo.HashType hashType) { + if (hashType == HashDistributionInfo.HashType.IDENTITY) { + return TDistributionHashType.IDENTITY; + } + return TDistributionHashType.CRC32; + } + public TDataPartition toThrift() { TDataPartition result = new TDataPartition(type); if (partitionExprs != null) { @@ -110,6 +130,9 @@ public TDataPartition toThrift() { if (mergePartitionInfo != null) { result.setMergePartitionInfo(mergePartitionInfo.toThrift()); } + if (type == TPartitionType.BUCKET_SHFFULE_HASH_PARTITIONED) { + result.setDistributionHashType(toTHashType(hashType)); + } return result; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java index 8898987d9a75ef..1fce1429ea3cec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.SortInfo; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.common.Pair; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.planner.LocalExchangeNode.LocalExchangeType; @@ -59,6 +60,8 @@ public class ExchangeNode extends PlanNode { private boolean isRightChildOfBroadcastHashJoin = false; private TPartitionType partitionType; + // storage bucketing hash carried for BUCKET_SHFFULE_HASH_PARTITIONED; defaults to CRC32 (legacy) + private HashDistributionInfo.HashType distributionHashType = HashDistributionInfo.HashType.CRC32; /** * use for Nereids only. @@ -81,6 +84,21 @@ public void setPartitionType(TPartitionType partitionType) { this.partitionType = partitionType; } + public HashDistributionInfo.HashType getDistributionHashType() { + return distributionHashType; + } + + @Override + public HashDistributionInfo.HashType getStorageDistributionHashType() { + return distributionHashType; + } + + public void setDistributionHashType(HashDistributionInfo.HashType distributionHashType) { + this.distributionHashType = distributionHashType == null + ? HashDistributionInfo.HashType.CRC32 + : distributionHashType; + } + public void updateTupleIds(TupleDescriptor outputTupleDesc) { if (outputTupleDesc != null) { clearTupleIds(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashDistributionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashDistributionPruner.java index 747bb17a6ad6bf..0a2705f77f9e39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashDistributionPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashDistributionPruner.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.SlotRef; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.HashDistributionInfo.HashType; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.Tablet; @@ -64,12 +65,20 @@ public class HashDistributionPruner implements DistributionPruner { private final Map distributionColumnFilters; private final int hashMod; + private final HashType hashType; + public HashDistributionPruner(List schema, MaterializedIndex materializedIndex, List columns, Map filters, int hashMod, boolean isBaseIndexSelected) { + this(schema, materializedIndex, columns, filters, hashMod, isBaseIndexSelected, HashType.CRC32); + } + + public HashDistributionPruner(List schema, MaterializedIndex materializedIndex, List columns, + Map filters, int hashMod, boolean isBaseIndexSelected, HashType hashType) { this.tablets = materializedIndex.getTablets(); this.bucketNum = tablets.size(); this.distributionColumns = columns; this.hashMod = hashMod; + this.hashType = hashType; if (isBaseIndexSelected) { this.distributionColumnFilters = filters; } else { @@ -92,8 +101,14 @@ public HashDistributionPruner(List schema, MaterializedIndex materialize public Collection prune(int columnId, PartitionKey hashKey, int complex) { if (columnId == distributionColumns.size()) { // compute Hash Key - long hashValue = hashKey.getHashValue(); - return Lists.newArrayList(getTabletId((int) ((hashValue & 0xffffffff) % hashMod))); + int bucket; + if (hashType == HashType.IDENTITY) { + bucket = hashKey.getIdentityHashValue(hashMod); + } else { + long hashValue = hashKey.getHashValue(); + bucket = (int) ((hashValue & 0xffffffff) % hashMod); + } + return Lists.newArrayList(getTabletId(bucket)); } Column keyColumn = distributionColumns.get(columnId); PartitionColumnFilter filter = distributionColumnFilters.get(keyColumn.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/LocalExchangeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/LocalExchangeNode.java index 66eda40079952f..e6801661724e99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/LocalExchangeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/LocalExchangeNode.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprToThriftVisitor; import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TExpr; import org.apache.doris.thrift.TLocalExchangeNode; @@ -39,6 +40,9 @@ public class LocalExchangeNode extends PlanNode { public static final String EXCHANGE_NODE = "LOCAL-EXCHANGE"; private LocalExchangeType exchangeType; + // storage bucketing hash for BUCKET_HASH_SHUFFLE; inherited from the upstream ExchangeNode's + // bucket-shuffle distribution. Defaults to CRC32 (legacy behavior). + private HashDistributionInfo.HashType distributionHashType = HashDistributionInfo.HashType.CRC32; /** * use for Nereids only. @@ -56,6 +60,12 @@ public LocalExchangeNode(PlanNodeId id, PlanNode inputNode, LocalExchangeType ex this.children.add(inputNode); this.exchangeType = exchangeType; this.fragment = inputNode.getFragment(); + // Preserve the effective storage layout through passthrough/unary nodes as well as direct + // ExchangeNode and OlapScanNode children. + HashDistributionInfo.HashType childHashType = inputNode.getStorageDistributionHashType(); + if (childHashType != null) { + this.distributionHashType = childHashType; + } List hashExprs = distributeExprs; boolean isHashShuffle = (exchangeType == LocalExchangeType.BUCKET_HASH_SHUFFLE @@ -97,6 +107,14 @@ protected void toThrift(TPlanNode msg) { } msg.local_exchange_node.setDistributeExprLists(thriftDistributeExprLists); } + if (exchangeType == LocalExchangeType.BUCKET_HASH_SHUFFLE) { + msg.local_exchange_node.setDistributionHashType(DataPartition.toTHashType(distributionHashType)); + } + } + + @Override + public HashDistributionInfo.HashType getStorageDistributionHashType() { + return distributionHashType; } private List distributeExprLists() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 3418137d57c734..e1ba53dfbf758f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -368,6 +368,14 @@ public OlapTable getOlapTable() { return olapTable; } + @Override + public HashDistributionInfo.HashType getStorageDistributionHashType() { + DistributionInfo distributionInfo = olapTable.getDefaultDistributionInfo(); + return distributionInfo instanceof HashDistributionInfo + ? ((HashDistributionInfo) distributionInfo).getHashType() + : null; + } + public String getTableNameInPlan() { return tableNameInPlan; } @@ -450,7 +458,8 @@ private Collection distributionPrune( info.getDistributionColumns(), columnFilters, info.getBucketNum(), - getSelectedIndexId() == olapTable.getBaseIndexId()); + getSelectedIndexId() == olapTable.getBaseIndexId(), + info.getHashType()); return new ArrayList<>(distributionPruner.prune()); } case RANDOM: { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index d799a5f0557035..619f41a8e8ee7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -71,6 +71,7 @@ import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TDataSink; import org.apache.doris.thrift.TDataSinkType; +import org.apache.doris.thrift.TDistributionHashType; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TNodeInfo; @@ -502,6 +503,14 @@ private void setPartialUpdateInfoForParam(TOlapTableSchemaParam schemaParam, Ola } } + private TDistributionHashType getTDistributionHashType(DistributionInfo distInfo) { + if (distInfo instanceof HashDistributionInfo + && ((HashDistributionInfo) distInfo).getHashType() == HashDistributionInfo.HashType.IDENTITY) { + return TDistributionHashType.IDENTITY; + } + return TDistributionHashType.CRC32; + } + private List getDistColumns(DistributionInfo distInfo) throws UserException { List distColumns = Lists.newArrayList(); switch (distInfo.getType()) { @@ -990,6 +999,7 @@ private TOlapTablePartitionParam createPartition(long dbId, OlapTable table) partitionParam.setTableId(table.getId()); partitionParam.setVersion(0); partitionParam.setPartitionType(partType.toThrift()); + partitionParam.setDistributionHashType(getTDistributionHashType(table.getDefaultDistributionInfo())); // create shadow partition for empty auto partition table. only use in this load. if (enableAutomaticPartition && partitionIds.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java index 98621ccc4f6636..1fc4b6df5376c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java @@ -26,6 +26,7 @@ import org.apache.doris.analysis.JoinOperator; import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.ToSqlParams; +import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.common.TreeNode; import org.apache.doris.nereids.trees.plans.distribute.NereidsSpecifyInstances; import org.apache.doris.nereids.trees.plans.distribute.worker.job.ScanSource; @@ -334,6 +335,11 @@ public TPlanFragment toThrift() { } else { result.setPartition(dataPartitionForThrift.toThrift()); } + HashDistributionInfo.HashType hashType = planRoot == null + ? null : planRoot.getStorageDistributionHashType(); + if (hashType != null) { + result.setDistributionHashType(DataPartition.toTHashType(hashType)); + } // TODO chenhao , calculated by cost result.setMinReservationBytes(0); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index a635ca26730892..3d333b14b12765 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -31,6 +31,7 @@ import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.common.Id; import org.apache.doris.common.Pair; import org.apache.doris.common.TreeNode; @@ -1152,6 +1153,26 @@ protected Pair enforceRequire( return Pair.of(leNode, preferType); } + /** + * Return the effective storage hash type when this subtree has one unambiguous bucket layout. + * Unary nodes preserve their child's layout; multi-input nodes preserve it only when every + * child reports the same layout. + */ + public HashDistributionInfo.HashType getStorageDistributionHashType() { + HashDistributionInfo.HashType hashType = null; + for (PlanNode child : children) { + HashDistributionInfo.HashType childHashType = child.getStorageDistributionHashType(); + if (childHashType == null) { + return null; + } + if (hashType != null && hashType != childHashType) { + return null; + } + hashType = childHashType; + } + return hashType; + } + /** * Create a LocalExchangeNode wrapping child with the given exchange type. * No child-type skip — matches BE's _add_local_exchange which inserts LE for any child diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DistributionHashTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DistributionHashTypeTest.java new file mode 100644 index 00000000000000..1e418993e10d8a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DistributionHashTypeTest.java @@ -0,0 +1,343 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.DistributionDesc; +import org.apache.doris.analysis.HashDistributionDesc; +import org.apache.doris.catalog.ColocateTableIndex.GroupId; +import org.apache.doris.catalog.HashDistributionInfo.HashType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.meta.MetaContext; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.util.List; +import java.util.Map; + +// Tests for the pluggable bucketing hash function carried by the `distribution_hash_type` table +// property. Today HashType has CRC32 (default/legacy) and IDENTITY; more types will be added later, +// so the framework-level cases (gson round-trip, equals, property parse) iterate over +// HashType.values() and stay correct as new constants appear. Identity-specific cases verify that +// canonical bytes from every valid distribution-column type and multiple columns are accepted. +public class DistributionHashTypeTest { + + private Column intCol(String name) { + return new Column(name, PrimitiveType.BIGINT, true); + } + + // ------------------------------------------------------------------ + // Metadata / backward compatibility + // ------------------------------------------------------------------ + + @Test + public void testLegacyConstructorsDefaultToCrc32() { + Assert.assertEquals(HashType.CRC32, new HashDistributionInfo().getHashType()); + Assert.assertEquals(HashType.CRC32, + new HashDistributionInfo(8, Lists.newArrayList(intCol("id"))).getHashType()); + Assert.assertEquals(HashType.CRC32, + new HashDistributionInfo(8, false, Lists.newArrayList(intCol("id"))).getHashType()); + } + + @Test + public void testLegacyMetadataWithoutHashTypeDeserializesToCrc32() { + // Metadata written before hashType existed has no "hashType" key; gson leaves it null and + // getHashType() must fall back to CRC32 so old tables keep their historical bucket layout. + HashDistributionInfo original + = new HashDistributionInfo(8, false, Lists.newArrayList(intCol("id")), HashType.CRC32); + String json = GsonUtils.GSON.toJson(original); + String legacyJson = json.replaceAll(",?\\s*\"hashType\"\\s*:\\s*\"[A-Z0-9_]+\"", ""); + Assert.assertFalse(legacyJson.contains("hashType")); + HashDistributionInfo restored = GsonUtils.GSON.fromJson(legacyJson, HashDistributionInfo.class); + Assert.assertEquals(HashType.CRC32, restored.getHashType()); + } + + @Test + public void testHashTypeSurvivesGsonRoundTrip() { + // Framework-level: every hash type must round-trip. Adding a new HashType automatically + // extends this coverage. + for (HashType type : HashType.values()) { + HashDistributionInfo original = new HashDistributionInfo(8, false, Lists.newArrayList(intCol("id")), type); + HashDistributionInfo restored + = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(original), HashDistributionInfo.class); + Assert.assertEquals("hashType lost in gson round trip: " + type, type, restored.getHashType()); + } + } + + @Test + public void testEqualityAndHashCodeConsiderHashType() { + // Any two distinct hash types must make otherwise-identical infos unequal. + HashType[] types = HashType.values(); + for (int i = 0; i < types.length; i++) { + HashDistributionInfo a = new HashDistributionInfo(8, false, Lists.newArrayList(intCol("id")), types[i]); + HashDistributionInfo aSame = new HashDistributionInfo(8, false, Lists.newArrayList(intCol("id")), types[i]); + Assert.assertEquals(a, aSame); + Assert.assertEquals(a.hashCode(), aSame.hashCode()); + for (int j = i + 1; j < types.length; j++) { + HashDistributionInfo b = new HashDistributionInfo(8, false, Lists.newArrayList(intCol("id")), types[j]); + Assert.assertNotEquals(a, b); + } + } + } + + @Test + public void testToDistributionDescCarriesHashType() throws DdlException { + // toDistributionDesc() is used when a partition deep-copies the table distribution + // (dynamic partition / addMultiPartitions); the hashType must ride along. Verify by + // round-tripping desc back to info (HashDistributionDesc has no getter). + for (HashType type : HashType.values()) { + List columns = Lists.newArrayList(intCol("id")); + HashDistributionInfo info = new HashDistributionInfo(8, false, columns, type); + DistributionDesc desc = info.toDistributionDesc(); + Assert.assertTrue(desc instanceof HashDistributionDesc); + HashDistributionInfo rebuilt = (HashDistributionInfo) desc.toDistributionInfo(columns); + Assert.assertEquals(type, rebuilt.getHashType()); + HashDistributionInfo descriptorRoundTrip = (HashDistributionInfo) desc.toDistributionDescriptor() + .translateToCatalogStyle().toDistributionInfo(columns); + Assert.assertEquals(type, descriptorRoundTrip.getHashType()); + } + } + + @Test + public void testSetHashTypeInheritedByAddPartition() { + // ADD PARTITION with an explicit DISTRIBUTED BY builds a CRC32 info, then + // InternalCatalog.addPartition overwrites hashType with the table's. Verify the setter path. + HashDistributionInfo partition + = new HashDistributionInfo(8, false, Lists.newArrayList(intCol("id")), HashType.CRC32); + Assert.assertEquals(HashType.CRC32, partition.getHashType()); + partition.setHashType(HashType.IDENTITY); + Assert.assertEquals(HashType.IDENTITY, partition.getHashType()); + } + + // ------------------------------------------------------------------ + // Property parsing + // ------------------------------------------------------------------ + + @Test + public void testAnalyzeDistributionHashType() throws AnalysisException { + // missing property -> CRC32 + Assert.assertEquals(HashType.CRC32, PropertyAnalyzer.analyzeDistributionHashType(null)); + Assert.assertEquals(HashType.CRC32, PropertyAnalyzer.analyzeDistributionHashType(Maps.newHashMap())); + + // every hash type parses case-insensitively and the property is consumed (removed) so it is + // not later flagged as an unknown property. + for (HashType type : HashType.values()) { + Map props = Maps.newHashMap(); + props.put(PropertyAnalyzer.PROPERTIES_DISTRIBUTION_HASH_TYPE, mixCase(type.name())); + Assert.assertEquals(type, PropertyAnalyzer.analyzeDistributionHashType(props)); + Assert.assertFalse(props.containsKey(PropertyAnalyzer.PROPERTIES_DISTRIBUTION_HASH_TYPE)); + } + } + + @Test + public void testAnalyzeDistributionHashTypeInvalidValueThrows() { + Map bad = Maps.newHashMap(); + bad.put(PropertyAnalyzer.PROPERTIES_DISTRIBUTION_HASH_TYPE, "murmur3"); + AnalysisException e + = Assert.assertThrows(AnalysisException.class, () -> PropertyAnalyzer.analyzeDistributionHashType(bad)); + Assert.assertTrue(e.getMessage().contains(PropertyAnalyzer.PROPERTIES_DISTRIBUTION_HASH_TYPE)); + } + + // ------------------------------------------------------------------ + // identity accepts canonical bytes from all valid distribution columns + // ------------------------------------------------------------------ + + @Test + public void testToDistributionInfoIdentitySingleIntegerColumn() throws DdlException { + List schema = Lists.newArrayList(intCol("shard_num"), new Column("v", PrimitiveType.INT, false)); + HashDistributionDesc desc + = new HashDistributionDesc(8, false, Lists.newArrayList("shard_num"), HashType.IDENTITY); + HashDistributionInfo info = (HashDistributionInfo) desc.toDistributionInfo(schema); + Assert.assertEquals(HashType.IDENTITY, info.getHashType()); + Assert.assertEquals(1, info.getDistributionColumns().size()); + } + + @Test + public void testToDistributionInfoIdentityAllowsLargeInt() throws DdlException { + List schema = Lists.newArrayList(new Column("big_id", PrimitiveType.LARGEINT, true)); + HashDistributionDesc desc = new HashDistributionDesc(8, false, Lists.newArrayList("big_id"), HashType.IDENTITY); + HashDistributionInfo info = (HashDistributionInfo) desc.toDistributionInfo(schema); + Assert.assertEquals(HashType.IDENTITY, info.getHashType()); + } + + @Test + public void testToDistributionInfoIdentityAllowsNonIntegerColumn() throws DdlException { + List schema = Lists.newArrayList(new Column("s", PrimitiveType.VARCHAR, true)); + HashDistributionDesc desc = new HashDistributionDesc(8, false, Lists.newArrayList("s"), HashType.IDENTITY); + HashDistributionInfo info = (HashDistributionInfo) desc.toDistributionInfo(schema); + Assert.assertEquals(HashType.IDENTITY, info.getHashType()); + Assert.assertEquals(PrimitiveType.VARCHAR, + info.getDistributionColumns().get(0).getType().getPrimitiveType()); + } + + @Test + public void testToDistributionInfoIdentityAllowsMultipleColumns() throws DdlException { + List schema = Lists.newArrayList(intCol("a"), new Column("b", PrimitiveType.VARCHAR, true)); + HashDistributionDesc desc = new HashDistributionDesc(8, false, Lists.newArrayList("a", "b"), + HashType.IDENTITY); + HashDistributionInfo info = (HashDistributionInfo) desc.toDistributionInfo(schema); + Assert.assertEquals(HashType.IDENTITY, info.getHashType()); + Assert.assertEquals(2, info.getDistributionColumns().size()); + } + + @Test + public void testToDistributionInfoCrc32AllowsNonIntegerAndMultiColumn() throws DdlException { + // crc32 (default) keeps its historical freedom: multi-column and non-integer are fine. + List schema = Lists.newArrayList(new Column("a", PrimitiveType.VARCHAR, true), intCol("b")); + HashDistributionDesc desc = new HashDistributionDesc(8, false, Lists.newArrayList("a", "b"), HashType.CRC32); + HashDistributionInfo info = (HashDistributionInfo) desc.toDistributionInfo(schema); + Assert.assertEquals(HashType.CRC32, info.getHashType()); + Assert.assertEquals(2, info.getDistributionColumns().size()); + } + + @Test + public void testTableSignatureConsidersHashType() { + Column key = new Column("id", PrimitiveType.INT, true); + HashDistributionInfo distributionInfo = new HashDistributionInfo(8, Lists.newArrayList(key)); + OlapTable table = new OlapTable(1L, "t", Lists.newArrayList(key), KeysType.DUP_KEYS, + new SinglePartitionInfo(), distributionInfo); + table.addPartition(new Partition(2L, "p", new MaterializedIndex(3L, + MaterializedIndex.IndexState.NORMAL), distributionInfo)); + + String crc32Signature = table.getSignature(1, Lists.newArrayList("p")); + distributionInfo.setHashType(HashType.IDENTITY); + String identitySignature = table.getSignature(1, Lists.newArrayList("p")); + Assert.assertNotEquals(crc32Signature, identitySignature); + } + + // ------------------------------------------------------------------ + // ColocateGroupSchema: hashType participates in colocate compatibility and metadata + // ------------------------------------------------------------------ + + private ColocateGroupSchema schemaWith(HashType type) { + return new ColocateGroupSchema(new GroupId(1L, 2L), Lists.newArrayList(intCol("id")), 8, + new ReplicaAllocation((short) 1), type); + } + + @Test + public void testCheckDistributionAllowsSameHashType() throws DdlException { + // A table whose distribution hashType matches the group's must pass checkDistribution. + for (HashType type : HashType.values()) { + ColocateGroupSchema schema = schemaWith(type); + HashDistributionInfo info = new HashDistributionInfo(8, false, Lists.newArrayList(intCol("id")), type); + schema.checkDistribution(info); // should not throw + } + } + + @Test + public void testCheckDistributionRejectsDifferentHashType() { + // Mixing hash types inside one colocate group would break co-location, so it must be + // rejected before the buckets-num / column checks even when those are identical. + HashType[] types = HashType.values(); + for (int i = 0; i < types.length; i++) { + for (int j = 0; j < types.length; j++) { + if (i == j) { + continue; + } + ColocateGroupSchema schema = schemaWith(types[i]); + HashDistributionInfo info + = new HashDistributionInfo(8, false, Lists.newArrayList(intCol("id")), types[j]); + Assert.assertThrows(DdlException.class, () -> schema.checkDistribution(info)); + } + } + } + + @Test + public void testWritableRoundTripPreservesHashType() throws Exception { + // With a current-version journal, write() appends the hashType name and readFields() must + // restore it verbatim for every hash type. + MetaContext metaContext = new MetaContext(); + metaContext.setMetaVersion(FeMetaVersion.VERSION_141); + metaContext.setThreadLocalInfo(); + try { + for (HashType type : HashType.values()) { + ColocateGroupSchema original = schemaWith(type); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + original.write(new DataOutputStream(bos)); + ColocateGroupSchema restored + = ColocateGroupSchema.read(new DataInputStream(new ByteArrayInputStream(bos.toByteArray()))); + Assert.assertEquals("hashType lost in Writable round trip: " + type, type, restored.getHashType()); + Assert.assertEquals(8, restored.getBucketsNum()); + } + } finally { + MetaContext.remove(); + } + } + + @Test + public void testReadFieldsBeforeVersion141FallsBackToCrc32() throws Exception { + // Build the exact legacy stream, which ended after ReplicaAllocation and had no hash type. + ColocateGroupSchema original = schemaWith(HashType.CRC32); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(bos); + original.getGroupId().write(out); + out.writeInt(original.getDistributionColTypes().size()); + for (Type type : original.getDistributionColTypes()) { + ColumnType.write(out, type); + } + out.writeInt(original.getBucketsNum()); + original.getReplicaAlloc().write(out); + + MetaContext readContext = new MetaContext(); + readContext.setMetaVersion(FeMetaVersion.VERSION_140); + readContext.setThreadLocalInfo(); + try { + ByteArrayInputStream input = new ByteArrayInputStream(bos.toByteArray()); + ColocateGroupSchema restored = ColocateGroupSchema.read(new DataInputStream(input)); + Assert.assertEquals(HashType.CRC32, restored.getHashType()); + Assert.assertEquals(0, input.available()); + } finally { + MetaContext.remove(); + } + } + + @Test + public void testGetHashTypeNullFallsBackToCrc32() { + // Legacy gson metadata has no "hashType" field; getHashType() must not NPE and defaults to + // CRC32, matching HashDistributionInfo's fallback. + ColocateGroupSchema schema = schemaWith(HashType.IDENTITY); + String json = GsonUtils.GSON.toJson(schema); + String legacyJson = json.replaceAll(",?\\s*\"hashType\"\\s*:\\s*\"[A-Z0-9_]+\"", ""); + Assert.assertFalse(legacyJson.contains("hashType")); + ColocateGroupSchema restored = GsonUtils.GSON.fromJson(legacyJson, ColocateGroupSchema.class); + Assert.assertEquals(HashType.CRC32, restored.getHashType()); + } + + // Alternate the case of each character so the parse path is exercised case-insensitively + // regardless of which hash type name it is. + private String mixCase(String s) { + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + sb.append((i & 1) == 0 + ? Character.toUpperCase(c) + : Character.toLowerCase(c)); + } + return sb.toString(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/MaterializedIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/MaterializedIndexTest.java index 6044e35b5ecf64..fbe51efd9f3c4e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/MaterializedIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/MaterializedIndexTest.java @@ -142,6 +142,18 @@ public void testPartitionMetaChecksum() { Assert.assertEquals(firstPartition.getMetaChecksum(), firstPartition.getRemoteMetaChecksum()); } + @Test + public void testPartitionMetaChecksumChangesOnDistributionHashType() { + MaterializedIndex baseIndex = new MaterializedIndex(1L, IndexState.NORMAL); + HashDistributionInfo distributionInfo = new HashDistributionInfo( + 3, List.of(new Column("k1", PrimitiveType.INT))); + Partition partition = new Partition(1L, "p1", baseIndex, distributionInfo); + String crc32Checksum = partition.getMetaChecksum(); + + distributionInfo.setHashType(HashDistributionInfo.HashType.IDENTITY); + Assert.assertNotEquals(crc32Checksum, partition.getMetaChecksum()); + } + @Test public void testPartitionMetaChecksumChangesOnReplicaQueryFields() { // Build a partition with one tablet/replica. diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DistributionSpecHashTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DistributionSpecHashTest.java index da99ec15b6d624..123b2dca809ada 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DistributionSpecHashTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DistributionSpecHashTest.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.properties; +import org.apache.doris.catalog.HashDistributionInfo.HashType; import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -55,7 +56,8 @@ public void testWithShuffleExprsSubset() { -1L, Sets.newHashSet(0L), Lists.newArrayList(Sets.newHashSet(e1, e4), Sets.newHashSet(e2, e5), Sets.newHashSet(e3, e6)), - map + map, + HashType.IDENTITY ); // retain middle slot only (original index 1): map renumbered to 0 in the new spec @@ -67,6 +69,7 @@ public void testWithShuffleExprsSubset() { expectedMiddle.put(e2, 0); expectedMiddle.put(e5, 0); Assertions.assertEquals(expectedMiddle, middleOnly.getExprIdToEquivalenceSet()); + Assertions.assertEquals(HashType.IDENTITY, middleOnly.getHashType()); } @Test @@ -387,4 +390,97 @@ public void testHashEqualSatisfyWithDifferentLength() { Assertions.assertFalse(bucketed1.satisfy(bucketed2)); Assertions.assertFalse(bucketed2.satisfy(bucketed1)); } + + @Test + public void testMergeRejectsDifferentHashTypes() { + DistributionSpecHash crc32 = naturalSpec(HashType.CRC32); + DistributionSpecHash identity = naturalSpec(HashType.IDENTITY); + Assertions.assertThrows(IllegalStateException.class, () -> DistributionSpecHash.merge(crc32, identity)); + } + + // Two NATURAL specs identical except for hashType must be unequal and hash differently, so the + // memo (which keys PhysicalProperties on DistributionSpecHash) never collapses a crc32 and an + // identity distribution into the same group entry and mis-shares their enforcer/cost. + @Test + public void testEqualsAndHashCodeConsiderHashType() { + DistributionSpecHash crc32 = naturalSpec(HashType.CRC32); + DistributionSpecHash crc32Same = naturalSpec(HashType.CRC32); + DistributionSpecHash identity = naturalSpec(HashType.IDENTITY); + + Assertions.assertEquals(crc32, crc32Same); + Assertions.assertEquals(crc32.hashCode(), crc32Same.hashCode()); + Assertions.assertNotEquals(crc32, identity); + Assertions.assertNotEquals(crc32.hashCode(), identity.hashCode()); + } + + // satisfy()'s equal branch (NATURAL/STORAGE_BUCKETED/EXECUTION_BUCKETED target) must reject a + // provider whose hashType differs, otherwise a crc32-bucketed child would be wrongly accepted as + // satisfying an identity NATURAL requirement (and vice versa) and skip the needed reshuffle. + @Test + public void testSatisfyEqualBranchChecksHashType() { + DistributionSpecHash crc32Provider = naturalSpec(HashType.CRC32); + DistributionSpecHash crc32Required = naturalSpec(HashType.CRC32); + DistributionSpecHash identityRequired = naturalSpec(HashType.IDENTITY); + + Assertions.assertTrue(crc32Provider.satisfy(crc32Required)); + Assertions.assertFalse(crc32Provider.satisfy(identityRequired)); + + DistributionSpecHash identityProvider = naturalSpec(HashType.IDENTITY); + Assertions.assertTrue(identityProvider.satisfy(identityRequired)); + Assertions.assertFalse(identityProvider.satisfy(crc32Required)); + } + + // The REQUIRE branch is hashType-agnostic: execution shuffle is always crc32, and a REQUIRE spec + // defaults to CRC32. An identity NATURAL/bucketed provider must still satisfy a plain REQUIRE so + // identity single-table plans are not broken. + @Test + public void testSatisfyRequireBranchIgnoresHashType() { + DistributionSpecHash require = new DistributionSpecHash( + Lists.newArrayList(new ExprId(1), new ExprId(2)), + ShuffleType.REQUIRE, + 1, + Sets.newHashSet(1L), + Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), + requireMap() + ); + + DistributionSpecHash naturalIdentity = new DistributionSpecHash( + Lists.newArrayList(new ExprId(1), new ExprId(2)), + ShuffleType.NATURAL, + 1, + -1L, + Sets.newHashSet(1L), + Lists.newArrayList(Sets.newHashSet(new ExprId(1)), Sets.newHashSet(new ExprId(2))), + requireMap(), + HashType.IDENTITY + ); + + Assertions.assertTrue(naturalIdentity.satisfy(require)); + } + + private Map requireMap() { + Map map = Maps.newHashMap(); + map.put(new ExprId(1), 0); + map.put(new ExprId(2), 1); + return map; + } + + private DistributionSpecHash naturalSpec(HashType hashType) { + Map map = Maps.newHashMap(); + map.put(new ExprId(0), 0); + map.put(new ExprId(1), 0); + map.put(new ExprId(2), 1); + map.put(new ExprId(3), 1); + return new DistributionSpecHash( + Lists.newArrayList(new ExprId(0), new ExprId(2)), + ShuffleType.NATURAL, + 0, + -1L, + Sets.newHashSet(0L), + Lists.newArrayList(Sets.newHashSet(new ExprId(0), new ExprId(1)), + Sets.newHashSet(new ExprId(2), new ExprId(3))), + map, + hashType + ); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/JoinUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/JoinUtilsTest.java index 9d634d58ab9fe7..3b6bff71fa5b24 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/JoinUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/JoinUtilsTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.ColocateTableIndex; import org.apache.doris.catalog.ColocateTableIndex.GroupId; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.HashDistributionInfo.HashType; import org.apache.doris.nereids.properties.DistributionSpecHash; import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; import org.apache.doris.nereids.trees.expressions.Add; @@ -288,4 +289,47 @@ public void testCouldColocateJoinForDiffTableNotInSameGroup() { Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); } } + + // Two NATURAL sides with the same non-crc32 hashType (IDENTITY) can still colocate: same storage + // hash means each side's bucket layout matches, so no reshuffle is needed. + @Test + public void testCouldColocateJoinForSameIdentityHashType() { + ConnectContext ctx = new ConnectContext(); + ctx.setThreadLocalInfo(); + + DistributionSpecHash left = new DistributionSpecHash(Lists.newArrayList(new ExprId(1)), + ShuffleType.NATURAL, 1L, 1L, Collections.emptySet(), HashType.IDENTITY); + DistributionSpecHash right = new DistributionSpecHash(Lists.newArrayList(new ExprId(2)), + ShuffleType.NATURAL, 1L, 1L, Collections.emptySet(), HashType.IDENTITY); + + Expression leftKey1 = new SlotReference(new ExprId(1), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression rightKey1 = new SlotReference(new ExprId(2), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + + List conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey1)); + Assertions.assertTrue(JoinUtils.couldColocateJoin(left, right, conjuncts)); + } + + // Different hashType on the two NATURAL sides (crc32 vs identity) must NOT colocate: the storage + // bucket layouts differ, so a bucket-local join would read mismatched buckets — the correctness + // red line. Guarded by JoinUtils.couldColocateJoin's leftHashSpec/rightHashSpec hashType check. + @Test + public void testCouldNotColocateJoinForDifferentHashType() { + ConnectContext ctx = new ConnectContext(); + ctx.setThreadLocalInfo(); + + DistributionSpecHash left = new DistributionSpecHash(Lists.newArrayList(new ExprId(1)), + ShuffleType.NATURAL, 1L, 1L, Collections.emptySet(), HashType.CRC32); + DistributionSpecHash right = new DistributionSpecHash(Lists.newArrayList(new ExprId(2)), + ShuffleType.NATURAL, 1L, 1L, Collections.emptySet(), HashType.IDENTITY); + + Expression leftKey1 = new SlotReference(new ExprId(1), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression rightKey1 = new SlotReference(new ExprId(2), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + + List conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey1)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/HashDistributionPrunerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/HashDistributionPrunerTest.java index 6cc4194dfe7cc7..4a2979b707eaad 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/HashDistributionPrunerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/HashDistributionPrunerTest.java @@ -18,15 +18,22 @@ package org.apache.doris.planner; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.IPv4Literal; +import org.apache.doris.analysis.IPv6Literal; import org.apache.doris.analysis.InPredicate; +import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.LargeIntLiteral; +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.analysis.NullLiteral; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.analysis.VarBinaryLiteral; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.HashDistributionInfo.HashType; import org.apache.doris.catalog.LocalTablet; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.Tablet; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -34,6 +41,7 @@ import org.junit.Assert; import org.junit.Test; +import java.math.BigInteger; import java.util.Collection; import java.util.List; import java.util.Map; @@ -44,13 +52,9 @@ public class HashDistributionPrunerTest { @Test public void test() { List tabletIds = Lists.newArrayListWithExpectedSize(300); - List indexTablets = Lists.newArrayListWithExpectedSize(300); for (long i = 0; i < 300; i++) { tabletIds.add(i); - indexTablets.add(new LocalTablet(i)); } - MaterializedIndex index = new MaterializedIndex(); - index.appendTablets(indexTablets); // distribution columns Column dealDate = new Column("dealDate", PrimitiveType.DATE, false); @@ -98,6 +102,7 @@ public void test() { filters.put("CHANNEL", channelFilter); filters.put("SHOP_TYPE", shopTypeFilter); + MaterializedIndex index = createMaterializedIndex(tabletIds); HashDistributionPruner pruner = new HashDistributionPruner(null, index, columns, filters, tabletIds.size(), true); @@ -146,6 +151,109 @@ public void test() { Assert.assertEquals(39, tablets.size()); } + // Identity bucketing treats each value's canonical bytes as an unsigned integer with its first + // byte least significant, then appends multiple columns before taking the bucket modulus. This + // must remain bit-identical with BE tablet routing and bucket-shuffle partitioning. + @Test + public void testIdentityPrune() { + List tabletIds = Lists.newArrayListWithExpectedSize(512); + for (long i = 0; i < 512; i++) { + tabletIds.add(i); + } + Column shardNum = new Column("shard_num", PrimitiveType.BIGINT, false); + List columns = Lists.newArrayList(shardNum); + + // in-range: shard_num = 100 -> 100 % 512 = 100 + assertIdentityBucket(tabletIds, columns, "SHARD_NUM", new IntLiteral(100), 100L); + // wraps: 600 % 512 = 88 + assertIdentityBucket(tabletIds, columns, "SHARD_NUM", new IntLiteral(600), 88L); + // Two's-complement bytes are interpreted as unsigned. A power-of-two modulus therefore + // still maps -1 to the final bucket. + assertIdentityBucket(tabletIds, columns, "SHARD_NUM", new IntLiteral(-1), 511L); + + // LARGEINT uses all 128 bits of its canonical little-endian representation. + Column bigId = new Column("big_id", PrimitiveType.LARGEINT, false); + List bigCols = Lists.newArrayList(bigId); + BigInteger huge = BigInteger.ONE.shiftLeft(100).add(BigInteger.valueOf(5)); + long expected = huge.mod(BigInteger.valueOf(512)).longValue(); + assertIdentityBucket(tabletIds, bigCols, "BIG_ID", new LargeIntLiteral(huge), expected); + + // With a non-power-of-two bucket count, -1 is UINT32_MAX rather than signed -1. + List tenTablets = Lists.newArrayListWithExpectedSize(10); + for (long i = 0; i < 10; i++) { + tenTablets.add(i); + } + assertIdentityBucket(tenTablets, columns, "SHARD_NUM", new IntLiteral(-1), 5L); + } + + @Test + public void testIdentityPruneWithMultipleTypedColumns() { + List tabletIds = Lists.newArrayListWithExpectedSize(257); + for (long i = 0; i < 257; i++) { + tabletIds.add(i); + } + List columns = Lists.newArrayList( + new Column("id", PrimitiveType.INT, false), + new Column("name", PrimitiveType.VARCHAR, false)); + + Map filters = new CaseInsensitiveMap(); + PartitionColumnFilter idFilter = new PartitionColumnFilter(); + idFilter.setLowerBound(new IntLiteral(1), true); + idFilter.setUpperBound(new IntLiteral(1), true); + filters.put("ID", idFilter); + PartitionColumnFilter nameFilter = new PartitionColumnFilter(); + nameFilter.setLowerBound(new StringLiteral("A"), true); + nameFilter.setUpperBound(new StringLiteral("A"), true); + filters.put("NAME", nameFilter); + + MaterializedIndex index = createMaterializedIndex(tabletIds); + HashDistributionPruner pruner = new HashDistributionPruner(null, index, columns, filters, + tabletIds.size(), true, HashType.IDENTITY); + // append(uint32_le(1), bytes("A")) = 1 * 256 + 65; 321 % 257 = 64 + Assert.assertEquals(Lists.newArrayList(64L), pruner.prune()); + } + + @Test + public void testIdentityNullCanonicalBytes() { + PartitionKey nullKey = new PartitionKey(); + nullKey.pushColumn(new NullLiteral(), PrimitiveType.INT); + Assert.assertEquals(0, nullKey.getIdentityHashValue(257)); + + nullKey.pushColumn(new StringLiteral("A"), PrimitiveType.VARCHAR); + Assert.assertEquals(65, nullKey.getIdentityHashValue(257)); + } + + @Test + public void testIdentityPruneWithIpAndVarBinaryCanonicalBytes() throws Exception { + PartitionKey ipv4 = new PartitionKey(); + ipv4.pushColumn(new IPv4Literal("1.2.3.4"), PrimitiveType.IPV4); + Assert.assertEquals(2, ipv4.getIdentityHashValue(257)); + + PartitionKey ipv6 = new PartitionKey(); + ipv6.pushColumn(new IPv6Literal("::1"), PrimitiveType.IPV6); + Assert.assertEquals(1, ipv6.getIdentityHashValue(257)); + + PartitionKey varBinary = new PartitionKey(); + varBinary.pushColumn(new VarBinaryLiteral(new byte[] {(byte) 0xff, 0}), PrimitiveType.VARBINARY); + Assert.assertEquals(255, varBinary.getIdentityHashValue(257)); + } + + private void assertIdentityBucket(List tabletIds, List columns, String colName, Expr value, + long expectedBucket) { + PartitionColumnFilter filter = new PartitionColumnFilter(); + filter.setLowerBound((LiteralExpr) value, true); + filter.setUpperBound((LiteralExpr) value, true); + Map filters = new CaseInsensitiveMap(); + filters.put(colName, filter); + + MaterializedIndex index = createMaterializedIndex(tabletIds); + HashDistributionPruner pruner = new HashDistributionPruner(null, index, columns, filters, tabletIds.size(), + true, HashType.IDENTITY); + Collection results = pruner.prune(); + Assert.assertEquals(1, results.size()); + Assert.assertEquals(Long.valueOf(expectedBucket), results.iterator().next()); + } + @Test public void testPruneWithMaterializedIndex() { List tabletIds = Lists.newArrayListWithExpectedSize(8); @@ -185,4 +293,12 @@ public void testPruneWithMaterializedIndex() { Assert.assertEquals(tabletIds, Lists.newArrayList(allIndexTablets)); } + private MaterializedIndex createMaterializedIndex(List tabletIds) { + MaterializedIndex index = new MaterializedIndex(); + for (long tabletId : tabletIds) { + index.addTablet(new LocalTablet(tabletId), null, true); + } + return index; + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java index 59c26166c3f51b..68025613f405cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java @@ -31,6 +31,7 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.FunctionName; +import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; @@ -40,6 +41,7 @@ import org.apache.doris.planner.LocalExchangeNode.LocalExchangeTypeRequire; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.TDistributionHashType; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TPartitionType; import org.apache.doris.thrift.TPlanNode; @@ -58,6 +60,27 @@ public class LocalShuffleNodeCoverageTest { private static final AtomicInteger NEXT_ID = new AtomicInteger(1); + @Test + public void testIdentityHashTypePropagatesThroughLocalExchangeAndFragment() { + TrackingPlanNode identityChild = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP) { + @Override + public HashDistributionInfo.HashType getStorageDistributionHashType() { + return HashDistributionInfo.HashType.IDENTITY; + } + }; + LocalExchangeNode passthrough = new LocalExchangeNode(nextPlanNodeId(), identityChild, + LocalExchangeType.PASSTHROUGH, null); + LocalExchangeNode bucket = new LocalExchangeNode(nextPlanNodeId(), passthrough, + LocalExchangeType.BUCKET_HASH_SHUFFLE, Collections.emptyList()); + Assertions.assertEquals(HashDistributionInfo.HashType.IDENTITY, + bucket.getStorageDistributionHashType()); + Assertions.assertEquals(TDistributionHashType.IDENTITY, + bucket.treeToThrift().getNodes().get(0).getLocalExchangeNode().getDistributionHashType()); + + PlanFragment fragment = new PlanFragment(new PlanFragmentId(1), bucket, DataPartition.UNPARTITIONED); + Assertions.assertEquals(TDistributionHashType.IDENTITY, fragment.toThrift().getDistributionHashType()); + } + @Test public void testRequireSpecificAutoRequireHashPreservesSpecificHash() { // Pass-through operators (union / streaming agg / sort) forward their parent's specific diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 302c57b3180b1a..f6854622cbecd2 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -326,6 +326,8 @@ struct TOlapTablePartitionParam { 13: optional bool partitions_is_fake = false // remote insert fe master address 14: optional Types.TNetworkAddress master_address + // hash function type; CRC32 (legacy behavior) is the default for backward compatibility + 15: optional Types.TDistributionHashType distribution_hash_type = Types.TDistributionHashType.CRC32 } struct TOlapTableIndex { diff --git a/gensrc/thrift/Partitions.thrift b/gensrc/thrift/Partitions.thrift index da172fac735c2b..ccd10b82588f13 100644 --- a/gensrc/thrift/Partitions.thrift +++ b/gensrc/thrift/Partitions.thrift @@ -205,4 +205,6 @@ struct TDataPartition { 2: optional list partition_exprs 3: optional list partition_infos 4: optional TMergePartitionInfo merge_partition_info + // storage bucketing hash for BUCKET_SHFFULE_HASH_PARTITIONED; !__isset means CRC32 (legacy) + 5: optional Types.TDistributionHashType distribution_hash_type = Types.TDistributionHashType.CRC32 } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index af4b9c3eb68619..894663e89082c8 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -1502,6 +1502,8 @@ struct TLocalExchangeNode { // `TPipelineFragmentParams.total_instances`, and mapping global instance index to local instance by // `TPipelineFragmentParams.shuffle_idx_to_instance_idx` 2: optional list distribute_expr_lists + // storage bucketing hash for BUCKET_HASH_SHUFFLE; !__isset means CRC32 (legacy) + 3: optional Types.TDistributionHashType distribution_hash_type = Types.TDistributionHashType.CRC32 } struct TOlapRewriteNode { diff --git a/gensrc/thrift/Planner.thrift b/gensrc/thrift/Planner.thrift index 866d8d45320243..dd5a9cc9cfb60b 100644 --- a/gensrc/thrift/Planner.thrift +++ b/gensrc/thrift/Planner.thrift @@ -64,6 +64,10 @@ struct TPlanFragment { 8: optional i64 initial_reservation_total_claims 9: optional QueryCache.TQueryCacheParam query_cache_param + + // Effective storage bucketing hash used by BE-native bucket local exchanges. If absent, legacy + // fragments use CRC32. + 10: optional Types.TDistributionHashType distribution_hash_type = Types.TDistributionHashType.CRC32 } // location information for a single scan range diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index 006c553f21c485..8ccf932f508cad 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -794,6 +794,12 @@ struct TColumnGroup { 2: required list columns_in_group } +// hash function type used by HASH distribution to map rows to buckets. +enum TDistributionHashType { + CRC32 = 0, + IDENTITY = 1 +} + const i32 TSNAPSHOT_REQ_VERSION1 = 3; // corresponding to alpha rowset const i32 TSNAPSHOT_REQ_VERSION2 = 4; // corresponding to beta rowset // the snapshot request should always set prefer snapshot version to TPREFER_SNAPSHOT_REQ_VERSION diff --git a/regression-test/data/ddl_p0/test_distribution_hash_type_identity.out b/regression-test/data/ddl_p0/test_distribution_hash_type_identity.out new file mode 100644 index 00000000000000..85427945a0c2c7 --- /dev/null +++ b/regression-test/data/ddl_p0/test_distribution_hash_type_identity.out @@ -0,0 +1,122 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !identity_count -- +7 + +-- !identity_eq_0 -- +0 + +-- !identity_eq_1 -- +1 + +-- !identity_eq_7 -- +7 + +-- !identity_eq_8 -- +8 + +-- !identity_eq_513 -- +513 + +-- !identity_eq_negative_1 -- +-1 + +-- !identity_eq_1024 -- +1024 + +-- !identity_in -- +1024 +7 +8 + +-- !identity_string -- +beta 2 + +-- !identity_null -- +9 + +-- !identity_ipv4 -- +4 + +-- !identity_ipv6 -- +1 + +-- !identity_multi -- +-1 A 12 + +-- !identity_typed -- +7 + +-- !crc32_row_count -- +80 + +-- !identity_row_count -- +80 + +-- !crc32_rows_per_id -- +1 10 +2 10 +3 10 +4 10 +5 10 +6 10 +7 10 +8 10 + +-- !identity_rows_per_id -- +1 10 +2 10 +3 10 +4 10 +5 10 +6 10 +7 10 +8 10 + +-- !identity_added_partition -- +513 + +-- !identity_partition_count -- +4 + +-- !identity_colocate_join -- +1 +1024 +7 +8 + +-- !mixed_hash_join -- +1 +1024 +7 +8 + +-- !identity_bucket_shuffle_native -- +-1 5 50 +-8 7 70 +1024 6 60 +513 4 40 +7 2 20 +8 3 30 + +-- !identity_bucket_shuffle_fe -- +-1 5 50 +-8 7 70 +1024 6 60 +513 4 40 +7 2 20 +8 3 30 + +-- !identity_multi_bucket_shuffle -- +-1 A 12 22 +1 A 10 20 +2 BC 13 23 + +-- !identity_nullable_bucket_shuffle -- +10 100 +9 90 + +-- !identity_set_operation_join -- +-1 A 12 22 +1 A 10 20 +2 BC 13 23 + diff --git a/regression-test/suites/check_hash_bucket_table/check_hash_bucket_table.groovy b/regression-test/suites/check_hash_bucket_table/check_hash_bucket_table.groovy index 3fe6713f66b5ee..6ab61bf98ae7a8 100644 --- a/regression-test/suites/check_hash_bucket_table/check_hash_bucket_table.groovy +++ b/regression-test/suites/check_hash_bucket_table/check_hash_bucket_table.groovy @@ -75,6 +75,11 @@ suite("check_hash_bucket_table") { def checkTable = { String db, String tblName -> sql "use `${db}`;" def showStmt = sql_return_maparray("show create table `${tblName}`")[0]["Create Table"] + // TODO: Add hash bucket validation for non-CRC32 tables. + if (showStmt.contains("\"distribution_hash_type\"")) { + logger.info("===== [check] Skip non-CRC32 hash table: ${db}.${tblName}") + return false + } def partitionInfo = sql_return_maparray """ show partitions from `${tblName}`; """ int checkedPartition = 0 partitionInfo.each { diff --git a/regression-test/suites/ddl_p0/test_distribution_hash_type_identity.groovy b/regression-test/suites/ddl_p0/test_distribution_hash_type_identity.groovy new file mode 100644 index 00000000000000..f1eeb7d185f7c9 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_distribution_hash_type_identity.groovy @@ -0,0 +1,555 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_distribution_hash_type_identity") { + + // --------------------------------------------------------------------- + // 1. DDL: create table with distribution_hash_type = identity + // --------------------------------------------------------------------- + sql "DROP TABLE IF EXISTS test_dist_hash_identity" + sql """ + CREATE TABLE `test_dist_hash_identity` ( + `id` BIGINT NOT NULL, + `v` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "identity" + ); + """ + + // SHOW CREATE TABLE round-trip: the property must be echoed back so the table can be rebuilt. + def createStmt = sql "SHOW CREATE TABLE test_dist_hash_identity" + assertTrue(createStmt[0][1].toString().toLowerCase().contains("distribution_hash_type")) + assertTrue(createStmt[0][1].toString().toLowerCase().contains("identity")) + + // default (property absent) is crc32: SHOW CREATE must NOT emit the property. + sql "DROP TABLE IF EXISTS test_dist_hash_default" + sql """ + CREATE TABLE `test_dist_hash_default` ( + `id` BIGINT NOT NULL, + `v` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + def defaultStmt = sql "SHOW CREATE TABLE test_dist_hash_default" + assertFalse(defaultStmt[0][1].toString().toLowerCase().contains("distribution_hash_type")) + + // --------------------------------------------------------------------- + // 2. identity accepts multiple distribution columns and all valid types + // --------------------------------------------------------------------- + sql "DROP TABLE IF EXISTS test_dist_hash_string" + sql """ + CREATE TABLE `test_dist_hash_string` ( + `name` VARCHAR(32) NOT NULL, + `v` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`name`) + DISTRIBUTED BY HASH(`name`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "identity" + ); + """ + + sql "DROP TABLE IF EXISTS test_dist_hash_nullable" + sql """ + CREATE TABLE `test_dist_hash_nullable` ( + `name` VARCHAR(32) NULL, + `v` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`name`) + DISTRIBUTED BY HASH(`name`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "identity" + ); + """ + + sql "DROP TABLE IF EXISTS test_dist_hash_ipv4" + sql """ + CREATE TABLE `test_dist_hash_ipv4` ( + `addr` IPV4 NOT NULL, + `v` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`addr`) + DISTRIBUTED BY HASH(`addr`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "identity" + ); + """ + + sql "DROP TABLE IF EXISTS test_dist_hash_ipv6" + sql """ + CREATE TABLE `test_dist_hash_ipv6` ( + `addr` IPV6 NOT NULL, + `v` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`addr`) + DISTRIBUTED BY HASH(`addr`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "identity" + ); + """ + + sql "DROP TABLE IF EXISTS test_dist_hash_multi_col" + sql """ + CREATE TABLE `test_dist_hash_multi_col` ( + `id` INT NOT NULL, + `name` VARCHAR(32) NOT NULL, + `v` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`, `name`) + DISTRIBUTED BY HASH(`id`, `name`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "identity" + ); + """ + + sql "DROP TABLE IF EXISTS test_dist_hash_typed_multi" + sql """ + CREATE TABLE `test_dist_hash_typed_multi` ( + `d` DATE NOT NULL, + `dt` DATETIMEV2(6) NOT NULL, + `amount` DECIMAL(18, 2) NOT NULL, + `v` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`d`, `dt`) + DISTRIBUTED BY HASH(`d`, `dt`, `amount`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "identity" + ); + """ + + // invalid hash type value rejected + sql "DROP TABLE IF EXISTS test_dist_hash_bad_value" + test { + sql """ + CREATE TABLE `test_dist_hash_bad_value` ( + `id` BIGINT NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "murmur" + ); + """ + exception "Invalid distribution_hash_type" + } + + // --------------------------------------------------------------------- + // 3. colocate: same distribution_hash_type may share a group; different hash types may not. + // A colocate group keeps every table on its storage layout with no reshuffle, so all + // members must bucket rows with the same hash function. + // --------------------------------------------------------------------- + // 3a. two identity tables in the same colocate group -> allowed. + sql "DROP TABLE IF EXISTS test_dist_hash_colo_id1" + sql "DROP TABLE IF EXISTS test_dist_hash_colo_id2" + sql """ + CREATE TABLE `test_dist_hash_colo_id1` ( + `id` BIGINT NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "identity", + "colocate_with" = "test_dist_hash_cg_identity" + ); + """ + sql """ + CREATE TABLE `test_dist_hash_colo_id2` ( + `id` BIGINT NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "identity", + "colocate_with" = "test_dist_hash_cg_identity" + ); + """ + + // 3b. crc32 table joining an existing identity group -> rejected on hash type mismatch. + sql "DROP TABLE IF EXISTS test_dist_hash_colo_crc32" + test { + sql """ + CREATE TABLE `test_dist_hash_colo_crc32` ( + `id` BIGINT NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "colocate_with" = "test_dist_hash_cg_identity" + ); + """ + exception "Colocate tables must have same distribution hash type" + } + + // --------------------------------------------------------------------- + // 4. read/write consistency: identity write then equality query must find the row. + // This is the core guarantee: BE buckets and FE prunes with the same hash function. + // Use explicit assertions (not qt_ recording) so a lost row fails loudly instead of + // silently recording an empty result set. + // --------------------------------------------------------------------- + sql """ INSERT INTO test_dist_hash_identity VALUES + (0, 100), (1, 101), (7, 107), (8, 108), (513, 613), (-1, 200), (1024, 300) """ + + qt_identity_count "SELECT COUNT(*) FROM test_dist_hash_identity" + + // Each equality query drives single-bucket pruning. + qt_identity_eq_0 "SELECT id FROM test_dist_hash_identity WHERE id = 0" + qt_identity_eq_1 "SELECT id FROM test_dist_hash_identity WHERE id = 1" + qt_identity_eq_7 "SELECT id FROM test_dist_hash_identity WHERE id = 7" + qt_identity_eq_8 "SELECT id FROM test_dist_hash_identity WHERE id = 8" + qt_identity_eq_513 "SELECT id FROM test_dist_hash_identity WHERE id = 513" + qt_identity_eq_negative_1 "SELECT id FROM test_dist_hash_identity WHERE id = -1" + qt_identity_eq_1024 "SELECT id FROM test_dist_hash_identity WHERE id = 1024" + + order_qt_identity_in "SELECT id FROM test_dist_hash_identity WHERE id IN (7, 8, 1024)" + + // Non-integer and multi-column identity layouts must use the same canonical bytes in BE + // writes, FE tablet pruning, and bucket shuffle. + sql "INSERT INTO test_dist_hash_string VALUES ('alpha', 1), ('beta', 2)" + qt_identity_string "SELECT name, v FROM test_dist_hash_string WHERE name = 'beta'" + + sql "INSERT INTO test_dist_hash_nullable VALUES (NULL, 9), ('x', 10)" + qt_identity_null "SELECT v FROM test_dist_hash_nullable WHERE name <=> NULL" + + sql "INSERT INTO test_dist_hash_ipv4 VALUES (to_ipv4('1.2.3.4'), 4), (to_ipv4('10.0.0.1'), 10)" + qt_identity_ipv4 "SELECT v FROM test_dist_hash_ipv4 WHERE addr = to_ipv4('1.2.3.4')" + + sql "INSERT INTO test_dist_hash_ipv6 VALUES (to_ipv6('::1'), 1), (to_ipv6('2001:db8::1'), 6)" + qt_identity_ipv6 "SELECT v FROM test_dist_hash_ipv6 WHERE addr = to_ipv6('::1')" + + sql """ INSERT INTO test_dist_hash_multi_col VALUES + (1, 'A', 10), (1, 'B', 11), (-1, 'A', 12), (2, 'BC', 13) """ + order_qt_identity_multi """ + SELECT id, name, v FROM test_dist_hash_multi_col + WHERE id = -1 AND name = 'A' + """ + + sql """ INSERT INTO test_dist_hash_typed_multi VALUES + ('2026-01-02', '2026-01-02 03:04:05.123456', 123.45, 7) """ + qt_identity_typed """ + SELECT v FROM test_dist_hash_typed_multi + WHERE d = '2026-01-02' + AND dt = '2026-01-02 03:04:05.123456' + AND amount = 123.45 + """ + + // --------------------------------------------------------------------- + // 5. bucket data distribution: identity spreads rows evenly, crc32 does not. + // Insert ids 1..8 (10 rows each, 80 rows total) into a crc32 table and an identity + // table, both DISTRIBUTED BY HASH(id) BUCKETS 8. With this key set: + // - crc32(id)%8 folds ids 3 and 8 onto the same bucket and leaves one bucket empty, + // so the row distribution is skewed (one 20-row bucket, one 0-row bucket). + // - identity uses id%8 directly, mapping the 8 distinct ids onto 8 distinct buckets, + // so every bucket holds exactly 10 rows and none is empty. + // crc32(id)%8 for id=1..8 -> {1:7, 2:5, 3:3, 4:0, 5:6, 6:4, 7:2, 8:3}; + // bucket 1 receives no id (empty) while bucket 3 gets both 3 and 8. + // (verify with: select crc32(8)%8; -> same bucket as crc32(3)%8) + // id%8 for id=1..8 -> {1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:0}: 8 buckets, 10 rows each. + // --------------------------------------------------------------------- + // helper: read the per-bucket RowCount via SHOW TABLETS. Each tablet maps to one bucket and + // (single replica here) appears once, so the list of RowCounts is the per-bucket row spread. + // RowCount is reported asynchronously, so poll until the total matches the expected row count + // before trusting the layout. + def bucketRowCounts = { String tbl, int expectedTotal -> + def counts = null + for (int attempt = 0; attempt < 60; attempt++) { + def tablets = sql_return_maparray "SHOW TABLETS FROM ${tbl}" + def perBucket = tablets.collect { (it["RowCount"] as String) as long } + long total = perBucket.sum() as long + if (total == expectedTotal) { + counts = perBucket + break + } + sleep(5000) + } + assertNotNull(counts, "RowCount for ${tbl} never reached ${expectedTotal}".toString()) + return counts + } + + // truncate existing data + // identity table: even distribution, one row per bucket per id. + sql "TRUNCATE TABLE test_dist_hash_identity" + // crc32 (default) table: skewed distribution with an empty bucket. + sql "TRUNCATE TABLE test_dist_hash_default" + + // write ids 1..8, 10 rows each (v = 1..10) -> 80 rows total for both tables. + def bucketValues = [] + (1..8).each { id -> + (1..10).each { v -> bucketValues << "(${id}, ${v})" } + } + def bucketInsert = bucketValues.join(", ") + sql "INSERT INTO test_dist_hash_default VALUES ${bucketInsert}" + sql "INSERT INTO test_dist_hash_identity VALUES ${bucketInsert}" + + // sanity: both tables received all 80 rows with 10 rows per id (no rows dropped on write). + qt_crc32_row_count "SELECT COUNT(*) FROM test_dist_hash_default" + qt_identity_row_count "SELECT COUNT(*) FROM test_dist_hash_identity" + order_qt_crc32_rows_per_id "SELECT id, COUNT(*) FROM test_dist_hash_default GROUP BY id" + order_qt_identity_rows_per_id "SELECT id, COUNT(*) FROM test_dist_hash_identity GROUP BY id" + + // crc32: at least one bucket is empty and at least one bucket is overloaded (>10 rows), + // because crc32(id)%8 collides ids 3 and 8 and skips one bucket for ids 1..8. + def crc32Counts = bucketRowCounts("test_dist_hash_default", 80) + assertTrue(crc32Counts.any { it == 0L }, + "crc32 must leave at least one empty bucket, counts=${crc32Counts}".toString()) + assertTrue(crc32Counts.any { it > 10L }, + "crc32 must overload at least one bucket (>10), counts=${crc32Counts}".toString()) + + // identity: every bucket holds exactly 10 rows -> no empty bucket, perfectly even spread. + def identityCounts = bucketRowCounts("test_dist_hash_identity", 80) + assertEquals(8, identityCounts.size(), + "identity should fill all 8 buckets, counts=${identityCounts}".toString()) + assertFalse(identityCounts.any { it == 0L }, + "identity must NOT leave any empty bucket, counts=${identityCounts}".toString()) + identityCounts.each { c -> + assertEquals(10L, c as long, + "identity bucket must hold exactly 10 rows, counts=${identityCounts}".toString()) + } + + // --------------------------------------------------------------------- + // 6. ADD PARTITION inherits the table hash type (commit: inherit on ADD PARTITION). + // A partitioned identity table; manually added partitions must keep identity so + // writes/reads stay consistent. + // --------------------------------------------------------------------- + sql "DROP TABLE IF EXISTS test_dist_hash_identity_part" + sql """ + CREATE TABLE `test_dist_hash_identity_part` ( + `id` BIGINT NOT NULL, + `dt` INT NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`, `dt`) + PARTITION BY RANGE(`dt`) ( + PARTITION p1 VALUES LESS THAN ("10") + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "identity" + ); + """ + // manual ADD PARTITION: DDL cannot carry distribution_hash_type, so it must be inherited. + sql """ ALTER TABLE test_dist_hash_identity_part ADD PARTITION p2 VALUES LESS THAN ("20") + DISTRIBUTED BY HASH(`id`) BUCKETS 8 """ + + sql """ INSERT INTO test_dist_hash_identity_part VALUES (5, 5), (513, 5), (5, 15), (513, 15) """ + // rows in the newly added partition p2 (dt=15) must be found by equality pruning too; + // if the new partition fell back to crc32, BE/FE hash mismatch would drop these rows. + qt_identity_added_partition "SELECT id FROM test_dist_hash_identity_part WHERE dt = 15 AND id = 513" + qt_identity_partition_count "SELECT COUNT(*) FROM test_dist_hash_identity_part" + + // --------------------------------------------------------------------- + // 7. colocate join: two identity tables in the same colocate group join with no reshuffle. + // Both sides keep their storage layout (same identity hash + same bucket count), so the + // plan must be a COLOCATE join and the result must match the non-optimized join. + // --------------------------------------------------------------------- + sql "set enable_nereids_planner=true" + sql "set disable_colocate_plan=false" + + waitForColocateGroupStable("test_dist_hash_cg_identity") + + sql "INSERT INTO test_dist_hash_colo_id1 VALUES (0), (1), (7), (8), (513), (-1), (1024)" + sql "INSERT INTO test_dist_hash_colo_id2 VALUES (1), (7), (8), (999), (1024)" + + explain { + sql("""SELECT a.id FROM test_dist_hash_colo_id1 a + JOIN test_dist_hash_colo_id2 b ON a.id = b.id""") + contains "HAS_COLO_PLAN_NODE: true" + } + + order_qt_identity_colocate_join """SELECT a.id FROM test_dist_hash_colo_id1 a + JOIN test_dist_hash_colo_id2 b ON a.id = b.id""" + + // a crc32 table joining an identity table must NOT colocate (different hash functions). + sql "DROP TABLE IF EXISTS test_dist_hash_join_crc32" + sql """ + CREATE TABLE `test_dist_hash_join_crc32` ( + `id` BIGINT NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql "INSERT INTO test_dist_hash_join_crc32 VALUES (1), (7), (8), (1024)" + explain { + sql("""SELECT a.id FROM test_dist_hash_colo_id1 a + JOIN test_dist_hash_join_crc32 b ON a.id = b.id""") + contains "HAS_COLO_PLAN_NODE: false" + } + order_qt_mixed_hash_join """SELECT a.id FROM test_dist_hash_colo_id1 a + JOIN test_dist_hash_join_crc32 b ON a.id = b.id""" + + // --------------------------------------------------------------------- + // 8. bucket-shuffle join: an identity table joins a table with a different bucket count. + // The optimizer keeps the identity side on its storage layout and reshuffles the other + // side to that layout. The reshuffle must use the identity hash on BE (not crc32), + // otherwise rows land on the wrong channel and the join result is wrong. + // --------------------------------------------------------------------- + sql "set enable_nereids_planner=true" + sql "set enable_bucket_shuffle_join = true" + // Keep bucket shuffle deterministic across clusters: a positive downgrade ratio may replace it + // with a full PARTITIONED shuffle based on the bucket and parallel-instance counts. + sql "set bucket_shuffle_downgrade_ratio = 0" + + // [shuffle] prevents these tiny test tables from choosing a broadcast join. Together with the + // settings above, it exercises bucket shuffle without depending on table statistics. + def bucketShuffleJoinSql = """ + SELECT l.id, l.v, r.w FROM test_dist_hash_bs_left l + JOIN [shuffle] test_dist_hash_bs_right r ON l.id = r.id + """ + + // With this switch off, BE adds the required local exchange while building pipelines. + sql "set enable_local_shuffle_planner = false" + + sql "DROP TABLE IF EXISTS test_dist_hash_bs_left" + sql "DROP TABLE IF EXISTS test_dist_hash_bs_right" + sql """ + CREATE TABLE `test_dist_hash_bs_left` ( + `id` BIGINT NOT NULL, + `v` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "identity" + ); + """ + sql """ + CREATE TABLE `test_dist_hash_bs_right` ( + `id` BIGINT NOT NULL, + `w` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 5 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "identity" + ); + """ + // include negatives, out-of-range and boundary keys to exercise unsigned binary identity + // reshuffle across channels. + sql """INSERT INTO test_dist_hash_bs_left VALUES + (0, 1), (7, 2), (8, 3), (513, 4), (-1, 5), (1024, 6), (-8, 7)""" + sql """INSERT INTO test_dist_hash_bs_right VALUES + (7, 20), (8, 30), (513, 40), (-1, 50), (1024, 60), (-8, 70), (99, 80)""" + + // Standard EXPLAIN does not expose local-exchange placement, but it must retain the same + // bucket-shuffle join in both planning modes. Query results then validate the BE-native path. + explain { + sql(bucketShuffleJoinSql) + contains "INNER JOIN(BUCKET_SHUFFLE)" + } + order_qt_identity_bucket_shuffle_native "${bucketShuffleJoinSql}" + + // With this switch on, FE inserts explicit local-exchange nodes into the distributed plan. + sql "set enable_local_shuffle_planner = true" + explain { + sql(bucketShuffleJoinSql) + contains "INNER JOIN(BUCKET_SHUFFLE)" + } + order_qt_identity_bucket_shuffle_fe "${bucketShuffleJoinSql}" + sql "set enable_local_shuffle_planner = false" + + // Multi-column mixed-type identity bucket shuffle follows the same composition as storage. + sql "DROP TABLE IF EXISTS test_dist_hash_bs_multi_right" + sql """ + CREATE TABLE `test_dist_hash_bs_multi_right` ( + `id` INT NOT NULL, + `name` VARCHAR(32) NOT NULL, + `w` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`, `name`) + DISTRIBUTED BY HASH(`id`, `name`) BUCKETS 7 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "identity" + ); + """ + sql """ INSERT INTO test_dist_hash_bs_multi_right VALUES + (1, 'A', 20), (-1, 'A', 22), (2, 'BC', 23), (9, 'missing', 24) """ + + explain { + sql("""SELECT l.id, l.name FROM test_dist_hash_multi_col l + JOIN [shuffle] test_dist_hash_bs_multi_right r + ON l.id = r.id AND l.name = r.name""") + contains "INNER JOIN(BUCKET_SHUFFLE)" + } + + order_qt_identity_multi_bucket_shuffle """SELECT l.id, l.name, l.v, r.w + FROM test_dist_hash_multi_col l + JOIN [shuffle] test_dist_hash_bs_multi_right r + ON l.id = r.id AND l.name = r.name""" + + sql "DROP TABLE IF EXISTS test_dist_hash_nullable_right" + sql """ + CREATE TABLE `test_dist_hash_nullable_right` ( + `name` VARCHAR(32) NULL, + `w` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`name`) + DISTRIBUTED BY HASH(`name`) BUCKETS 7 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "distribution_hash_type" = "identity" + ); + """ + sql "INSERT INTO test_dist_hash_nullable_right VALUES (NULL, 90), ('x', 100)" + explain { + sql("""SELECT l.v, r.w FROM test_dist_hash_nullable l + JOIN [shuffle] test_dist_hash_nullable_right r ON l.name <=> r.name""") + contains "INNER JOIN(BUCKET_SHUFFLE)" + } + order_qt_identity_nullable_bucket_shuffle """SELECT l.v, r.w FROM test_dist_hash_nullable l + JOIN [shuffle] test_dist_hash_nullable_right r + ON l.name <=> r.name""" + + // A set operation that preserves an identity storage layout must expose IDENTITY to its parent. + def setOperationJoinSql = """ + SELECT u.id, u.name, u.v, r.w + FROM ( + SELECT id, name, v FROM test_dist_hash_multi_col WHERE id <= 1 + UNION ALL + SELECT id, name, v FROM test_dist_hash_multi_col WHERE id = 2 + ) u + JOIN [shuffle] test_dist_hash_bs_multi_right r + ON u.id = r.id AND u.name = r.name + """ + explain { + sql(setOperationJoinSql) + contains "INNER JOIN(BUCKET_SHUFFLE)" + } + order_qt_identity_set_operation_join "${setOperationJoinSql}" +}