[improvement](hash) Xxhash 128

be/src/exprs/function/function_hash.cpp

@@ -303,7 +303,16 @@ const char* murmur_hash3_get_name_type_bigint_v2_for_test() {
 
 template <PrimitiveType ReturnType>
 struct XxHashImpl {
-    static constexpr auto name = ReturnType == TYPE_INT ? "xxhash_32" : "xxhash_64";
+    static constexpr auto get_name() {
+        if constexpr (ReturnType == TYPE_INT) {
+            return "xxhash_32";
+        } else if constexpr (ReturnType == TYPE_BIGINT) {
+            return "xxhash_64";
+        } else {
+            return "xxhash_128";
+        }
+    }
+    static constexpr auto name = get_name();
 
     static Status empty_apply(IColumn& icolumn, size_t input_rows_count) {
         ColumnVector<ReturnType>& vec_to = assert_cast<ColumnVector<ReturnType>&>(icolumn);
@@ -341,10 +350,14 @@ struct XxHashImpl {
                     col_to_data[i] = HashUtil::xxHash32WithSeed(
                             reinterpret_cast<const char*>(&data[current_offset]),
                             offsets[i] - current_offset, col_to_data[i]);
-                } else {
+                } else if constexpr (ReturnType == TYPE_BIGINT) {
                     col_to_data[i] = HashUtil::xxHash64WithSeed(
                             reinterpret_cast<const char*>(&data[current_offset]),
                             offsets[i] - current_offset, col_to_data[i]);
+                } else {
+                    col_to_data[i] = HashUtil::xxHash128WithSeed(
+                            reinterpret_cast<const char*>(&data[current_offset]),
+                            offsets[i] - current_offset, static_cast<std::uint64_t>(col_to_data[i]));
                 }
                 current_offset = offsets[i];
             }
@@ -355,20 +368,23 @@ struct XxHashImpl {
                 if constexpr (ReturnType == TYPE_INT) {
                     col_to_data[i] =
                             HashUtil::xxHash32WithSeed(value.data(), value.size(), col_to_data[i]);
-                } else {
+                } else if constexpr (ReturnType == TYPE_BIGINT) {
                     col_to_data[i] =
                             HashUtil::xxHash64WithSeed(value.data(), value.size(), col_to_data[i]);
+                } else {
+                    col_to_data[i] =
+                            HashUtil::xxHash128WithSeed(value.data(), value.size(), static_cast<std::uint64_t>(col_to_data[i]));
                 }
             }
         } else if (const auto* vb_col = check_and_get_column<ColumnVarbinary>(column)) {
             for (size_t i = 0; i < input_rows_count; ++i) {
                 auto data_ref = vb_col->get_data_at(i);
                 if constexpr (ReturnType == TYPE_INT) {
-                    col_to_data[i] = HashUtil::xxHash32WithSeed(data_ref.data, data_ref.size,
-                                                                col_to_data[i]);
+                    col_to_data[i] = HashUtil::xxHash32WithSeed(data_ref.data, data_ref.size, col_to_data[i]);
+                } else if constexpr (ReturnType == TYPE_BIGINT) {
+                    col_to_data[i] = HashUtil::xxHash64WithSeed(data_ref.data, data_ref.size, col_to_data[i]);
                 } else {
-                    col_to_data[i] = HashUtil::xxHash64WithSeed(data_ref.data, data_ref.size,
-                                                                col_to_data[i]);
+                    col_to_data[i] = HashUtil::xxHash128WithSeed(data_ref.data, data_ref.size, static_cast<std::uint64_t>(col_to_data[i]));
                 }
             }
         } else {
@@ -382,6 +398,7 @@ struct XxHashImpl {
 
 using FunctionXxHash_32 = FunctionVariadicArgumentsBase<DataTypeInt32, XxHashImpl<TYPE_INT>>;
 using FunctionXxHash_64 = FunctionVariadicArgumentsBase<DataTypeInt64, XxHashImpl<TYPE_BIGINT>>;
+using FunctionXxHash_128 = FunctionVariadicArgumentsBase<DataTypeInt128, XxHashImpl<TYPE_LARGEINT>>;
 
 void register_function_hash(SimpleFunctionFactory& factory) {
     factory.register_function<FunctionMurmurHash3_32>();
@@ -393,5 +410,6 @@ void register_function_hash(SimpleFunctionFactory& factory) {
     factory.register_function<FunctionXxHash_32>();
     factory.register_function<FunctionXxHash_64>();
     factory.register_alias("xxhash_64", "xxhash3_64");
+    factory.register_function<FunctionXxHash_128>();
 }
 } // namespace doris

be/src/util/hash_util.hpp

@@ -461,6 +461,29 @@ class HashUtil {
         return XXH64(reinterpret_cast<const void*>(&INT_VALUE), sizeof(int), seed);
     }
 
+    static inline unsigned __int128 xxh128_to_uint128(XXH128_hash_t h) {
+        return (static_cast<unsigned __int128>(h.high64) << 64) | h.low64;
+    }
+
+    static unsigned __int128 xxHash128WithSeed(const char* s, size_t len, xxh_u64 seed) {
+        return xxh128_to_uint128(XXH3_128bits_withSeed(s, len, seed));
+    }
+
+    // same to the up function, just for null value
+    static unsigned __int128 xxHash128NullWithSeed(xxh_u64 seed) {
+        static const int INT_VALUE = 0;
+        return xxh128_to_uint128(XXH3_128bits_withSeed(reinterpret_cast<const char*>(&INT_VALUE), sizeof(int), seed));
+    }
+
+    static unsigned __int128 xxhash128_compat_with_seed(const char* s, size_t len, xxh_u64 seed) {
+        return xxh128_to_uint128(XXH128(reinterpret_cast<const void*>(s), len, seed));
+    }
+
+    static unsigned __int128 xxhash128_compat_null_with_seed(xxh_u64 seed) {
+        static const int INT_VALUE = 0;
+        return xxh128_to_uint128(XXH128(reinterpret_cast<const void*>(&INT_VALUE), sizeof(int), seed));
+    }
+
 #if defined(__clang__)
 #pragma clang diagnostic pop
 #endif

be/test/exprs/function/function_hash_test.cpp

@@ -381,6 +381,71 @@ TEST(HashFunctionTest, xxhash_64_test) {
     };
 }
 
+TEST(HashFunctionTest, xxhash_128_test) {
+    std::string func_name = "xxhash_128";
+
+    {
+        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};
+
+        DataSet data_set = {
+                {{Null()}, Null()},
+                {{std::string("hello")}, pack_murmur_hash3_128_for_test(0xc779cfaa5e523818ULL, 0xb5e9c1ad071b3e7fULL)}};
+
+        static_cast<void>(check_function<DataTypeInt128, true>(func_name, input_types, data_set));        
+    };
+
+    {
+        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR};
+
+        DataSet data_set = {
+                {{std::string("hello"), std::string("world")}, pack_murmur_hash3_128_for_test(0xfad86fdc7b94209eULL, 0xf8d8c298671f374aULL)},
+                {{std::string("hello"), Null()}, Null()}};
+
+        static_cast<void>(check_function<DataTypeInt128, true>(func_name, input_types, data_set));
+    };
+
+    {
+        InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR,
+                                    PrimitiveType::TYPE_VARCHAR};
+
+        DataSet data_set = {{{std::string("hello"), std::string("world"), std::string("!")},
+                             pack_murmur_hash3_128_for_test(0x6ad2986d0444bd84ULL, 0x9409bdfc60ac231cULL)},
+                            {{std::string("hello"), std::string("world"), Null()}, Null()}};
+
+        static_cast<void>(check_function<DataTypeInt128, true>(func_name, input_types, data_set));
+    };
+
+    {
+        InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY};
+
+        DataSet data_set = {{{Null()}, Null()},
+                            {{VARBINARY("hello")}, pack_murmur_hash3_128_for_test(0xc779cfaa5e523818ULL, 0xb5e9c1ad071b3e7fULL)}};
+
+        static_cast<void>(check_function<DataTypeInt128, true>(func_name, input_types, data_set));
+    };
+
+    {
+        InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARBINARY};
+
+        DataSet data_set = {
+                {{VARBINARY("hello"), VARBINARY("world")}, pack_murmur_hash3_128_for_test(0xfad86fdc7b94209eULL, 0xf8d8c298671f374aULL)},
+                {{VARBINARY("hello"), Null()}, Null()}};
+
+        static_cast<void>(check_function<DataTypeInt128, true>(func_name, input_types, data_set));
+    };
+
+    {
+        InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARBINARY,
+                                    PrimitiveType::TYPE_VARBINARY};
+
+        DataSet data_set = {{{VARBINARY("hello"), VARBINARY("world"), VARBINARY("!")},
+                             pack_murmur_hash3_128_for_test(0x6ad2986d0444bd84ULL, 0x9409bdfc60ac231cULL)},
+                            {{VARBINARY("hello"), VARBINARY("world"), Null()}, Null()}};
+
+        static_cast<void>(check_function<DataTypeInt128, true>(func_name, input_types, data_set));
+    };
+}
+
 TEST(HashFunctionTest, murmur_hash3_helper_functions_test) {
     {
         std::string input = "hello world";

fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java

@@ -582,6 +582,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.scalar.WidthBucket;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Xor;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.XpathString;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.XxHash128;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.XxHash32;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.XxHash64;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Year;
@@ -1179,6 +1180,7 @@ public class BuiltinScalarFunctions implements FunctionHelper {
             scalar(WidthBucket.class, "width_bucket"),
             scalar(XxHash32.class, "xxhash_32"),
             scalar(XxHash64.class, "xxhash_64", "xxhash3_64"),
+            scalar(XxHash128.class, "xxhash_128"),
             scalar(Xor.class, "xor"),
             scalar(XpathString.class, "xpath_string"),
             scalar(Year.class, "year"),

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/XxHash128.java

@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.scalar;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.LargeIntType;
+import org.apache.doris.nereids.types.StringType;
+import org.apache.doris.nereids.types.VarBinaryType;
+import org.apache.doris.nereids.types.VarcharType;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * ScalarFunction 'xxhash_128'.
+ */
+public class XxHash128 extends ScalarFunction
+        implements ExplicitlyCastableSignature, PropagateNullable {
+
+    public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+            FunctionSignature.ret(LargeIntType.INSTANCE).varArgs(VarcharType.SYSTEM_DEFAULT),
+            FunctionSignature.ret(LargeIntType.INSTANCE).varArgs(StringType.INSTANCE),
+            FunctionSignature.ret(LargeIntType.INSTANCE).varArgs(VarBinaryType.INSTANCE)
+    );
+
+    /**
+     * constructor with 1 or more arguments.
+     */
+    public XxHash128(Expression arg, Expression... varArgs) {
+        super("xxhash_128", ExpressionUtils.mergeArguments(arg, varArgs));
+    }
+
+    /** constructor for withChildren and reuse signature */
+    private XxHash128(ScalarFunctionParams functionParams) {
+        super(functionParams);
+    }
+
+    /**
+     * withChildren.
+     */
+    @Override
+    public XxHash128 withChildren(List<Expression> children) {
+        Preconditions.checkArgument(!children.isEmpty());
+        return new XxHash128(getFunctionParams(children));
+    }
+
+    @Override
+    public List<FunctionSignature> getSignatures() {
+        return SIGNATURES;
+    }
+
+    @Override
+    public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+        return visitor.visitXxHash128(this, context);
+    }
+}

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java

@@ -598,6 +598,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.scalar.WidthBucket;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Xor;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.XpathString;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.XxHash128;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.XxHash32;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.XxHash64;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Year;
@@ -2044,6 +2045,10 @@ default R visitXxHash64(XxHash64 xxHash64, C context) {
         return visitScalarFunction(xxHash64, context);
     }
 
+    default R visitXxHash128(XxHash128 xxHash128, C context) {
+        return visitScalarFunction(xxHash128, context);
+    }
+
     default R visitNegative(Negative negative, C context) {
         return visitScalarFunction(negative, context);
     }

regression-test/suites/query_p0/sql_functions/encryption_digest/test_binary_for_digest.groovy

@@ -128,6 +128,12 @@ suite("test_binary_for_digest", "p0,external,mysql,external_docker,external_dock
                 "xxHash64 mismatch for row ${xxhash64_result[i][0]}: VarBinary=${xxhash64_result[i][1]}, VARCHAR=${xxhash64_result[i][2]}")
         }
 
+        def xxhash128_result = sql """select id, xxhash_128(vb), xxhash_128(vc) from ${test_table} order by id"""
+        for (int i = 0; i < xxhash128_result.size(); i++) {
+            assertTrue(xxhash128_result[i][1] == xxhash128_result[i][2],
+                "xxHash32 mismatch for row ${xxhash128_result[i][0]}: VarBinary=${xxhash128_result[i][1]}, VARCHAR=${xxhash128_result[i][2]}")
+        }
+
         def variadic_xxhash32_result = sql """select id, xxhash_32(vb, vb), xxhash_32(vc, vc) from ${test_table} order by id"""
         for (int i = 0; i < variadic_xxhash32_result.size(); i++) {
             assertTrue(variadic_xxhash32_result[i][1] != null && variadic_xxhash32_result[i][2] != null,
@@ -140,6 +146,12 @@ suite("test_binary_for_digest", "p0,external,mysql,external_docker,external_dock
                 "Variadic xxHash64 should work with mixed VarBinary and VARCHAR arguments for row ${variadic_xxhash64_result[i][0]}")
         }
 
+        def variadic_xxhash128_result = sql """select id, xxhash_128(vb, vb), xxhash_128(vc, vc) from ${test_table} order by id"""
+        for (int i = 0; i < variadic_xxhash128_result.size(); i++) {
+            assertTrue(variadic_xxhash128_result[i][1] != null && variadic_xxhash128_result[i][2] != null,
+                "Variadic xxHash64 should work with mixed VarBinary and VARCHAR arguments for row ${variadic_xxhash128_result[i][0]}")
+        }
+
         connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysql_port}?useSSL=false") {
             try_sql """DROP DATABASE IF EXISTS ${ex_db_name}"""
         }

regression-test/suites/query_p0/sql_functions/hash_functions/test_hash_function.groovy

@@ -171,6 +171,10 @@ suite("test_hash_function", "arrow_flight_sql") {
     qt_sql "SELECT xxhash_64(\"hello\");"
     qt_sql "SELECT xxhash_64(\"hello\", \"world\");"
 
+    qt_sql "SELECT xxhash_128(null);"
+    qt_sql "SELECT xxhash_128(\"hello\");"
+    qt_sql "SELECT xxhash_128(\"hello\", \"world\");"
+
     def xxhash_res = sql "SELECT xxhash_64(null);"
     def xxhash3_res = sql "SELECT xxhash3_64(null);"
     assertEquals(xxhash_res, xxhash3_res);