diff --git a/be/src/exprs/function/function_hash.cpp b/be/src/exprs/function/function_hash.cpp index 74eb06e297d416..8d12d0945200cb 100644 --- a/be/src/exprs/function/function_hash.cpp +++ b/be/src/exprs/function/function_hash.cpp @@ -303,7 +303,16 @@ const char* murmur_hash3_get_name_type_bigint_v2_for_test() { template struct XxHashImpl { - static constexpr auto name = ReturnType == TYPE_INT ? "xxhash_32" : "xxhash_64"; + static constexpr auto get_name() { + if constexpr (ReturnType == TYPE_INT) { + return "xxhash_32"; + } else if constexpr (ReturnType == TYPE_BIGINT) { + return "xxhash_64"; + } else { + return "xxhash_128"; + } + } + static constexpr auto name = get_name(); static Status empty_apply(IColumn& icolumn, size_t input_rows_count) { ColumnVector& vec_to = assert_cast&>(icolumn); @@ -341,10 +350,14 @@ struct XxHashImpl { col_to_data[i] = HashUtil::xxHash32WithSeed( reinterpret_cast(&data[current_offset]), offsets[i] - current_offset, col_to_data[i]); - } else { + } else if constexpr (ReturnType == TYPE_BIGINT) { col_to_data[i] = HashUtil::xxHash64WithSeed( reinterpret_cast(&data[current_offset]), offsets[i] - current_offset, col_to_data[i]); + } else { + col_to_data[i] = HashUtil::xxHash128WithSeed( + reinterpret_cast(&data[current_offset]), + offsets[i] - current_offset, static_cast(col_to_data[i])); } current_offset = offsets[i]; } @@ -355,20 +368,23 @@ struct XxHashImpl { if constexpr (ReturnType == TYPE_INT) { col_to_data[i] = HashUtil::xxHash32WithSeed(value.data(), value.size(), col_to_data[i]); - } else { + } else if constexpr (ReturnType == TYPE_BIGINT) { col_to_data[i] = HashUtil::xxHash64WithSeed(value.data(), value.size(), col_to_data[i]); + } else { + col_to_data[i] = + HashUtil::xxHash128WithSeed(value.data(), value.size(), static_cast(col_to_data[i])); } } } else if (const auto* vb_col = check_and_get_column(column)) { for (size_t i = 0; i < input_rows_count; ++i) { auto data_ref = vb_col->get_data_at(i); if constexpr (ReturnType == TYPE_INT) { - col_to_data[i] = HashUtil::xxHash32WithSeed(data_ref.data, data_ref.size, - col_to_data[i]); + col_to_data[i] = HashUtil::xxHash32WithSeed(data_ref.data, data_ref.size, col_to_data[i]); + } else if constexpr (ReturnType == TYPE_BIGINT) { + col_to_data[i] = HashUtil::xxHash64WithSeed(data_ref.data, data_ref.size, col_to_data[i]); } else { - col_to_data[i] = HashUtil::xxHash64WithSeed(data_ref.data, data_ref.size, - col_to_data[i]); + col_to_data[i] = HashUtil::xxHash128WithSeed(data_ref.data, data_ref.size, static_cast(col_to_data[i])); } } } else { @@ -382,6 +398,7 @@ struct XxHashImpl { using FunctionXxHash_32 = FunctionVariadicArgumentsBase>; using FunctionXxHash_64 = FunctionVariadicArgumentsBase>; +using FunctionXxHash_128 = FunctionVariadicArgumentsBase>; void register_function_hash(SimpleFunctionFactory& factory) { factory.register_function(); @@ -393,5 +410,6 @@ void register_function_hash(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_alias("xxhash_64", "xxhash3_64"); + factory.register_function(); } } // namespace doris diff --git a/be/src/util/hash_util.hpp b/be/src/util/hash_util.hpp index ab5b72bffda5e9..f6d03f4a94289d 100644 --- a/be/src/util/hash_util.hpp +++ b/be/src/util/hash_util.hpp @@ -461,6 +461,29 @@ class HashUtil { return XXH64(reinterpret_cast(&INT_VALUE), sizeof(int), seed); } + static inline unsigned __int128 xxh128_to_uint128(XXH128_hash_t h) { + return (static_cast(h.high64) << 64) | h.low64; + } + + static unsigned __int128 xxHash128WithSeed(const char* s, size_t len, xxh_u64 seed) { + return xxh128_to_uint128(XXH3_128bits_withSeed(s, len, seed)); + } + + // same to the up function, just for null value + static unsigned __int128 xxHash128NullWithSeed(xxh_u64 seed) { + static const int INT_VALUE = 0; + return xxh128_to_uint128(XXH3_128bits_withSeed(reinterpret_cast(&INT_VALUE), sizeof(int), seed)); + } + + static unsigned __int128 xxhash128_compat_with_seed(const char* s, size_t len, xxh_u64 seed) { + return xxh128_to_uint128(XXH128(reinterpret_cast(s), len, seed)); + } + + static unsigned __int128 xxhash128_compat_null_with_seed(xxh_u64 seed) { + static const int INT_VALUE = 0; + return xxh128_to_uint128(XXH128(reinterpret_cast(&INT_VALUE), sizeof(int), seed)); + } + #if defined(__clang__) #pragma clang diagnostic pop #endif diff --git a/be/test/exprs/function/function_hash_test.cpp b/be/test/exprs/function/function_hash_test.cpp index 2d3f2ad1cc00c4..883f49f6101e81 100644 --- a/be/test/exprs/function/function_hash_test.cpp +++ b/be/test/exprs/function/function_hash_test.cpp @@ -381,6 +381,71 @@ TEST(HashFunctionTest, xxhash_64_test) { }; } +TEST(HashFunctionTest, xxhash_128_test) { + std::string func_name = "xxhash_128"; + + { + InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR}; + + DataSet data_set = { + {{Null()}, Null()}, + {{std::string("hello")}, pack_murmur_hash3_128_for_test(0xc779cfaa5e523818ULL, 0xb5e9c1ad071b3e7fULL)}}; + + static_cast(check_function(func_name, input_types, data_set)); + }; + + { + InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR}; + + DataSet data_set = { + {{std::string("hello"), std::string("world")}, pack_murmur_hash3_128_for_test(0xfad86fdc7b94209eULL, 0xf8d8c298671f374aULL)}, + {{std::string("hello"), Null()}, Null()}}; + + static_cast(check_function(func_name, input_types, data_set)); + }; + + { + InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR, + PrimitiveType::TYPE_VARCHAR}; + + DataSet data_set = {{{std::string("hello"), std::string("world"), std::string("!")}, + pack_murmur_hash3_128_for_test(0x6ad2986d0444bd84ULL, 0x9409bdfc60ac231cULL)}, + {{std::string("hello"), std::string("world"), Null()}, Null()}}; + + static_cast(check_function(func_name, input_types, data_set)); + }; + + { + InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY}; + + DataSet data_set = {{{Null()}, Null()}, + {{VARBINARY("hello")}, pack_murmur_hash3_128_for_test(0xc779cfaa5e523818ULL, 0xb5e9c1ad071b3e7fULL)}}; + + static_cast(check_function(func_name, input_types, data_set)); + }; + + { + InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARBINARY}; + + DataSet data_set = { + {{VARBINARY("hello"), VARBINARY("world")}, pack_murmur_hash3_128_for_test(0xfad86fdc7b94209eULL, 0xf8d8c298671f374aULL)}, + {{VARBINARY("hello"), Null()}, Null()}}; + + static_cast(check_function(func_name, input_types, data_set)); + }; + + { + InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARBINARY, + PrimitiveType::TYPE_VARBINARY}; + + DataSet data_set = {{{VARBINARY("hello"), VARBINARY("world"), VARBINARY("!")}, + pack_murmur_hash3_128_for_test(0x6ad2986d0444bd84ULL, 0x9409bdfc60ac231cULL)}, + {{VARBINARY("hello"), VARBINARY("world"), Null()}, Null()}}; + + static_cast(check_function(func_name, input_types, data_set)); + }; +} + TEST(HashFunctionTest, murmur_hash3_helper_functions_test) { { std::string input = "hello world"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index e55e1008762e68..ed9e36f0210669 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -582,6 +582,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.WidthBucket; import org.apache.doris.nereids.trees.expressions.functions.scalar.Xor; import org.apache.doris.nereids.trees.expressions.functions.scalar.XpathString; +import org.apache.doris.nereids.trees.expressions.functions.scalar.XxHash128; import org.apache.doris.nereids.trees.expressions.functions.scalar.XxHash32; import org.apache.doris.nereids.trees.expressions.functions.scalar.XxHash64; import org.apache.doris.nereids.trees.expressions.functions.scalar.Year; @@ -1179,6 +1180,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(WidthBucket.class, "width_bucket"), scalar(XxHash32.class, "xxhash_32"), scalar(XxHash64.class, "xxhash_64", "xxhash3_64"), + scalar(XxHash128.class, "xxhash_128"), scalar(Xor.class, "xor"), scalar(XpathString.class, "xpath_string"), scalar(Year.class, "year"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/XxHash128.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/XxHash128.java new file mode 100644 index 00000000000000..f1d481ee50ecb7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/XxHash128.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.LargeIntType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarBinaryType; +import org.apache.doris.nereids.types.VarcharType; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'xxhash_128'. + */ +public class XxHash128 extends ScalarFunction + implements ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(LargeIntType.INSTANCE).varArgs(VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(LargeIntType.INSTANCE).varArgs(StringType.INSTANCE), + FunctionSignature.ret(LargeIntType.INSTANCE).varArgs(VarBinaryType.INSTANCE) + ); + + /** + * constructor with 1 or more arguments. + */ + public XxHash128(Expression arg, Expression... varArgs) { + super("xxhash_128", ExpressionUtils.mergeArguments(arg, varArgs)); + } + + /** constructor for withChildren and reuse signature */ + private XxHash128(ScalarFunctionParams functionParams) { + super(functionParams); + } + + /** + * withChildren. + */ + @Override + public XxHash128 withChildren(List children) { + Preconditions.checkArgument(!children.isEmpty()); + return new XxHash128(getFunctionParams(children)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitXxHash128(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index ec86766df02f2e..e401c9ff91a8c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -598,6 +598,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.WidthBucket; import org.apache.doris.nereids.trees.expressions.functions.scalar.Xor; import org.apache.doris.nereids.trees.expressions.functions.scalar.XpathString; +import org.apache.doris.nereids.trees.expressions.functions.scalar.XxHash128; import org.apache.doris.nereids.trees.expressions.functions.scalar.XxHash32; import org.apache.doris.nereids.trees.expressions.functions.scalar.XxHash64; import org.apache.doris.nereids.trees.expressions.functions.scalar.Year; @@ -2044,6 +2045,10 @@ default R visitXxHash64(XxHash64 xxHash64, C context) { return visitScalarFunction(xxHash64, context); } + default R visitXxHash128(XxHash128 xxHash128, C context) { + return visitScalarFunction(xxHash128, context); + } + default R visitNegative(Negative negative, C context) { return visitScalarFunction(negative, context); } diff --git a/regression-test/suites/query_p0/sql_functions/encryption_digest/test_binary_for_digest.groovy b/regression-test/suites/query_p0/sql_functions/encryption_digest/test_binary_for_digest.groovy index 24a252e895de34..5027908cc4a643 100644 --- a/regression-test/suites/query_p0/sql_functions/encryption_digest/test_binary_for_digest.groovy +++ b/regression-test/suites/query_p0/sql_functions/encryption_digest/test_binary_for_digest.groovy @@ -128,6 +128,12 @@ suite("test_binary_for_digest", "p0,external,mysql,external_docker,external_dock "xxHash64 mismatch for row ${xxhash64_result[i][0]}: VarBinary=${xxhash64_result[i][1]}, VARCHAR=${xxhash64_result[i][2]}") } + def xxhash128_result = sql """select id, xxhash_128(vb), xxhash_128(vc) from ${test_table} order by id""" + for (int i = 0; i < xxhash128_result.size(); i++) { + assertTrue(xxhash128_result[i][1] == xxhash128_result[i][2], + "xxHash32 mismatch for row ${xxhash128_result[i][0]}: VarBinary=${xxhash128_result[i][1]}, VARCHAR=${xxhash128_result[i][2]}") + } + def variadic_xxhash32_result = sql """select id, xxhash_32(vb, vb), xxhash_32(vc, vc) from ${test_table} order by id""" for (int i = 0; i < variadic_xxhash32_result.size(); i++) { assertTrue(variadic_xxhash32_result[i][1] != null && variadic_xxhash32_result[i][2] != null, @@ -140,6 +146,12 @@ suite("test_binary_for_digest", "p0,external,mysql,external_docker,external_dock "Variadic xxHash64 should work with mixed VarBinary and VARCHAR arguments for row ${variadic_xxhash64_result[i][0]}") } + def variadic_xxhash128_result = sql """select id, xxhash_128(vb, vb), xxhash_128(vc, vc) from ${test_table} order by id""" + for (int i = 0; i < variadic_xxhash128_result.size(); i++) { + assertTrue(variadic_xxhash128_result[i][1] != null && variadic_xxhash128_result[i][2] != null, + "Variadic xxHash64 should work with mixed VarBinary and VARCHAR arguments for row ${variadic_xxhash128_result[i][0]}") + } + connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysql_port}?useSSL=false") { try_sql """DROP DATABASE IF EXISTS ${ex_db_name}""" } diff --git a/regression-test/suites/query_p0/sql_functions/hash_functions/test_hash_function.groovy b/regression-test/suites/query_p0/sql_functions/hash_functions/test_hash_function.groovy index 0a15cc73e3b7d8..7db89e6366340f 100644 --- a/regression-test/suites/query_p0/sql_functions/hash_functions/test_hash_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/hash_functions/test_hash_function.groovy @@ -171,6 +171,10 @@ suite("test_hash_function", "arrow_flight_sql") { qt_sql "SELECT xxhash_64(\"hello\");" qt_sql "SELECT xxhash_64(\"hello\", \"world\");" + qt_sql "SELECT xxhash_128(null);" + qt_sql "SELECT xxhash_128(\"hello\");" + qt_sql "SELECT xxhash_128(\"hello\", \"world\");" + def xxhash_res = sql "SELECT xxhash_64(null);" def xxhash3_res = sql "SELECT xxhash3_64(null);" assertEquals(xxhash_res, xxhash3_res);