Merge pull request #108436 from lalitx17/add-xxhash64-spark

Add Spark-compatible xxHash64 function

src/Functions/FunctionsHashing.h

@@ -746,6 +746,33 @@ struct ImplXxHash64
     static constexpr bool return_bigint_instead_of_fixedstring = false;
 };
 
+/// https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.xxhash64.html
+struct ImplXxHash64Spark
+{
+    static constexpr auto name = "xxHash64Spark";
+    using ReturnType = Int64;
+
+    /// Some restrictions apply:
+    /// (1) While the Spark equivalent function supports >1 argument, only a single argument was verified
+    ///     to return the same result in ClickHouse and Spark.
+    /// (2) Only String and NULL arguments were verified to return the same result in ClickHouse and Spark.
+    /// (3) NULL arguments must return a custom hash value
+    static constexpr size_t number_of_arguments = 1; /// (1)
+    static constexpr bool use_default_implementation_for_nulls = false;
+    static constexpr bool check_all_arguments_are_strings = true; /// (2)
+    static constexpr Int64 custom_null_hash = 42; /// (3)
+
+    static auto apply(const char * s, const size_t len) { return XXH_INLINE_XXH64(s, len, 42); }
+
+    static Int64 combineHashes(UInt64 /*h1*/, UInt64 /*h2*/)
+    {
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "{} cannot be called with more than one argument", name);
+    }
+
+    static constexpr bool use_int_hash_for_pods = false;
+    static constexpr bool return_bigint_instead_of_fixedstring = false;
+};
+
 struct ImplXXH3
 {
     static constexpr auto name = "xxh3";
@@ -935,8 +962,9 @@ public:
     static constexpr auto name = Impl::name;
     static constexpr UInt128 NULL_HASH = UInt128({0xc58ad2da03d9a871ul, 0x5715f196cbea7a40ul});
 
-    /// Keep default implementation of useDefaultImplementationForNulls for compatibility.
-    /// E.g. someHash(NULL) is NULL, but someHash(tuple(NULL)) is not NULL.
+    /// Keep default implementation of `useDefaultImplementationForNulls` for compatibility unless an implementation opts
+    /// out via `Impl::use_default_implementation_for_nulls`.
+    /// E.g. `someHash(NULL)` is `NULL`, but `someHash(tuple(NULL))` is not `NULL`.
 
 private:
     using ToType = typename Impl::ReturnType;
@@ -1341,7 +1369,10 @@ private:
             }
             else
             {
-                null_hash = static_cast<ToType>(NULL_HASH);
+                if constexpr (requires { Impl::custom_null_hash; })
+                    null_hash = Impl::custom_null_hash;
+                else
+                    null_hash = static_cast<ToType>(NULL_HASH);
             }
 
             typename ColumnVector<ToType>::Container original_vec_to;
@@ -1490,9 +1521,27 @@ public:
     {
         return name;
     }
-
-    bool isVariadic() const override { return true; }
-    size_t getNumberOfArguments() const override { return 0; }
+    bool isVariadic() const override
+    {
+        if constexpr (requires { Impl::number_of_arguments; })
+            return false;
+        else
+            return true;
+    }
+    size_t getNumberOfArguments() const override
+    {
+        if constexpr (requires { Impl::number_of_arguments; })
+            return Impl::number_of_arguments;
+        else
+            return 0;
+    }
+    bool useDefaultImplementationForNulls() const override
+    {
+        if constexpr (requires { Impl::use_default_implementation_for_nulls; })
+            return Impl::use_default_implementation_for_nulls;
+        else
+            return true;
+    }
     bool useDefaultImplementationForConstants() const override { return true; }
     bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
 
@@ -1500,8 +1549,26 @@ public:
     /// Hash values must remain stable, so we don't want the Variant adaptor to change hash computation.
     bool useDefaultImplementationForVariant() const override { return false; }
 
-    DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override
+    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
     {
+        if constexpr (requires { Impl::check_all_arguments_are_strings; })
+        {
+            if (Impl::check_all_arguments_are_strings)
+            {
+                for (const auto & argument : arguments)
+                {
+                    const DataTypePtr nested_type = removeNullable(argument);
+                    const WhichDataType which(nested_type);
+                    if (!which.isString() && !which.isNothing())
+                        throw Exception(
+                            ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                            "Illegal type {} of argument of function {}. Must be String or NULL",
+                            argument->getName(),
+                            getName());
+                }
+            }
+        }
+
         /// backward-compatible
         if constexpr (std::is_same_v<ToType, UInt128> && !Impl::return_bigint_instead_of_fixedstring)
         {
@@ -1870,6 +1937,7 @@ using FunctionHiveHash = FunctionAnyHash<HiveHashImpl>;
 
 using FunctionXxHash32 = FunctionAnyHash<ImplXxHash32>;
 using FunctionXxHash64 = FunctionAnyHash<ImplXxHash64>;
+using FunctionXxHash64Spark = FunctionAnyHash<ImplXxHash64Spark>;
 using FunctionXXH3 = FunctionAnyHash<ImplXXH3>;
 using FunctionXXH3_128 = FunctionAnyHash<ImplXXH3_128>;
 

src/Functions/FunctionsHashingMisc.cpp

@@ -548,6 +548,38 @@ For the 32-bit version see [`xxHash32`](#xxHash32)
     FunctionDocumentation xxHash64_documentation = {xxHash64_description, xxHash64_syntax, xxHash64_arguments, {}, xxHash64_returned_value, xxHash64_examples, xxHash64_introduced_in, xxHash64_category};
     factory.registerFunction<FunctionXxHash64>(xxHash64_documentation);
 
+    FunctionDocumentation::Description xxHash64Spark_description = R"(
+Calculates a [xxHash](http://cyan4973.github.io/xxHash/) from a string using the same seed as Spark.
+
+This function is the same as [`xxHash64`](#xxHash64), but it uses seed `42` and returns the result as `Int64`.
+Only `String` and `NULL` inputs are supported.
+For `NULL`, the function returns `42`, matching Spark's seed behavior.
+)";
+    FunctionDocumentation::Syntax xxHash64Spark_syntax = "xxHash64Spark(arg)";
+    FunctionDocumentation::Arguments xxHash64Spark_arguments = {{"arg", "Input string to hash.", {"String"}}};
+    FunctionDocumentation::ReturnedValue xxHash64Spark_returned_value
+        = {"Returns the computed 64-bit hash of the input string.", {"Int64"}};
+    FunctionDocumentation::Examples xxHash64Spark_examples
+        = {{"Usage example",
+            "SELECT xxHash64Spark('ABC');",
+            R"(
+┌─xxHash64Spark('ABC')─┐
+│  4105715581806190027 │
+└──────────────────────┘
+        )"}};
+    FunctionDocumentation::IntroducedIn xxHash64Spark_introduced_in = {26, 7};
+    FunctionDocumentation::Category xxHash64Spark_category = FunctionDocumentation::Category::Hash;
+    FunctionDocumentation xxHash64Spark_documentation
+        = {xxHash64Spark_description,
+           xxHash64Spark_syntax,
+           xxHash64Spark_arguments,
+           {},
+           xxHash64Spark_returned_value,
+           xxHash64Spark_examples,
+           xxHash64Spark_introduced_in,
+           xxHash64Spark_category};
+    factory.registerFunction<FunctionXxHash64Spark>(xxHash64Spark_documentation);
+
     FunctionDocumentation::Description xxh3_description = "Computes a [XXH3](https://github.com/Cyan4973/xxHash) 64-bit hash value.";
     FunctionDocumentation::Syntax xxh3_syntax = "xxh3(expr)";
     FunctionDocumentation::Arguments xxh3_argument = {{"expr", "A list of expressions of any data type.", {"Any"}}};

tests/queries/0_stateless/04410_xxhash64_spark.reference

@@ -0,0 +1,5 @@
+Int64
+1
+1
+1
+1

tests/queries/0_stateless/04410_xxhash64_spark.sql

@@ -0,0 +1,18 @@
+SELECT toTypeName(xxHash64Spark('ABC'));
+SELECT xxHash64Spark('ABC') = toInt64(4105715581806190027);
+SELECT xxHash64Spark(NULL) = toInt64(42);
+SELECT xxHash64Spark(if(number = 0, NULL, 'ABC')) = if(number = 0, toInt64(42), toInt64(4105715581806190027)) FROM numbers(2) ORDER BY number;
+
+-- Negative tests
+
+-- Only strings can be hashed at the moment
+SELECT xxHash64Spark(toUInt8(0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT xxHash64Spark(toUInt16(0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT xxHash64Spark(toUInt32(0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT xxHash64Spark(toUInt64(0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT xxHash64Spark(toUInt128(0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT xxHash64Spark(toUInt256(0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT xxHash64Spark(toFloat32(0.0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+
+-- Only single arguments are allowed at the moment
+SELECT xxHash64Spark('a', 'b'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }