Add ZXC compression codec

Integrate zxc (https://github.com/hellobertrand/zxc), an asymmetric LZ codec built for very fast decompression, as a new general-purpose codec `CODEC(ZXC[(level)])` (levels 1..7, default 3, compression method byte 0x9e). `ZXC` trades slow compression for fast decompression at a ratio between `LZ4` and `ZSTD`, which suits the compress-once/read-many pattern of analytical columns and decodes especially fast on modern ARM cores. - `contrib/zxc` submodule plus a curated `contrib/zxc-cmake` build that mirrors zxc's per-ISA variant objects with runtime dispatch, without its own `-march=native`/LTO/PGO defaults (ClickHouse manages its own flags). - `CompressionCodecZXC` uses zxc's one-shot frame API with zxc's internal checksum disabled, since ClickHouse already checksums every compressed block. - Documentation and a stateless roundtrip test (all levels plus error handling). Note: `contrib/zxc` points at the upstream repository as a placeholder; before merging it must be replaced with a fork under github.com/ClickHouse. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
2026-09-26 13:55:07 +00:00 · 2026-07-15 20:23:14 +00:00
parent 83a081939d
commit 0d49daabc7
13 changed files with 315 additions and 0 deletions
@@ -414,3 +414,6 @@
 [submodule "contrib/mapbox-geometry"]
 	path = contrib/mapbox-geometry
 	url = https://github.com/ClickHouse/geometry.hpp
+[submodule "contrib/zxc"]
+	path = contrib/zxc
+	url = https://github.com/hellobertrand/zxc
@@ -97,6 +97,7 @@ add_contrib (brotli-cmake brotli)
 add_contrib (double-conversion-cmake double-conversion)
 add_contrib (croaring-cmake croaring)
 add_contrib (zstd-cmake zstd)
+add_contrib (zxc-cmake zxc)
 add_contrib (bzip2-cmake bzip2)
 add_contrib (minizip-ng-cmake minizip-ng)
 add_contrib (snappy-cmake snappy)
@@ -0,0 +1,83 @@
+# zxc - an asymmetric LZ codec optimized for decompression throughput.
+# Upstream: https://github.com/hellobertrand/zxc
+#
+# This is a curated build. We deliberately do NOT use zxc's own CMakeLists.txt
+# because it enables -march=native, LTO and PGO by default and pulls in a CLI,
+# tests and packaging targets. ClickHouse manages its own compiler flags and
+# needs a portable binary with runtime CPU dispatch, so we mirror only the parts
+# of zxc's build that matter: the per-ISA "variant" objects plus the scalar
+# core, combined into a single static library.
+
+set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/zxc")
+set(SRC_DIR "${LIBRARY_DIR}/src/lib")
+
+set(ZXC_INCLUDE_DIRS
+    "${SRC_DIR}"
+    "${SRC_DIR}/vendors"    # vendored rapidhash.h
+)
+
+# Scalar core, compiled once. zxc_dispatch.c performs runtime CPU detection and
+# routes compress/decompress/huffman to the best available ISA variant below.
+set(ZXC_CORE_SOURCES
+    "${SRC_DIR}/zxc_common.c"
+    "${SRC_DIR}/zxc_pivco_tables.c"
+    "${SRC_DIR}/zxc_dict.c"
+    "${SRC_DIR}/zxc_driver.c"
+    "${SRC_DIR}/zxc_dispatch.c"
+    "${SRC_DIR}/zxc_pstream.c"
+    "${SRC_DIR}/zxc_seekable.c"
+)
+
+# zxc_compress.c / zxc_decompress.c / zxc_huffman.c are recompiled once per ISA
+# variant, each with its own -m flags and a ZXC_FUNCTION_SUFFIX that renames the
+# emitted symbols (e.g. zxc_compress_chunk_wrapper_avx2). zxc_dispatch.c declares
+# these suffixed symbols unconditionally per architecture, so the exact set of
+# variants below must match what the dispatcher expects for each target:
+#   x86-64  -> _default, _sse2, _avx2, _avx512
+#   aarch64 -> _default, _neon
+#   other   -> _default only
+set(ZXC_VARIANT_OBJECTS "")
+
+macro(zxc_add_variant suffix)
+    foreach(_src compress decompress huffman)
+        set(_tgt "_zxc_${_src}${suffix}")
+        add_library(${_tgt} OBJECT "${SRC_DIR}/zxc_${_src}.c")
+        target_include_directories(${_tgt} PRIVATE ${ZXC_INCLUDE_DIRS})
+        # ZXC_STATIC_DEFINE: no dllexport/visibility attributes (static build).
+        # ZXC_FUNCTION_SUFFIX: variant symbol suffix used by the dispatcher.
+        target_compile_definitions(${_tgt} PRIVATE ZXC_STATIC_DEFINE "ZXC_FUNCTION_SUFFIX=${suffix}")
+        if (${ARGC} GREATER 1)
+            target_compile_options(${_tgt} PRIVATE ${ARGN})
+        endif ()
+        list(APPEND ZXC_VARIANT_OBJECTS $<TARGET_OBJECTS:${_tgt}>)
+    endforeach()
+endmacro()
+
+# The default (baseline) variant inherits ClickHouse's target flags, which are
+# already x86-64-v3 (AVX2) / armv8.2-a+simd (NEON). That is a safe minimum: every
+# CPU the ClickHouse binary runs on supports it.
+zxc_add_variant(_default)
+
+if (ARCH_AMD64)
+    # SSE2/AVX2 flags are redundant with the x86-64-v3 baseline but harmless;
+    # AVX512 is NOT in the baseline, so those flags are required for that variant
+    # (only ever entered at runtime on AVX-512 capable CPUs).
+    zxc_add_variant(_sse2   -msse2)
+    zxc_add_variant(_avx2   -mavx2 -mfma -mbmi -mbmi2 -mlzcnt)
+    zxc_add_variant(_avx512 -mavx512f -mavx512bw -mavx512vbmi -mavx512vbmi2 -mbmi -mbmi2 -mlzcnt)
+elseif (ARCH_AARCH64)
+    # NEON is guaranteed by ClickHouse's armv8.2-a+simd baseline; no extra flags
+    # (adding -march=armv8-a would downgrade below the baseline).
+    zxc_add_variant(_neon)
+endif ()
+
+add_library(_zxc ${ZXC_CORE_SOURCES} ${ZXC_VARIANT_OBJECTS})
+
+target_compile_definitions(_zxc PUBLIC ZXC_STATIC_DEFINE)
+target_include_directories(_zxc PRIVATE ${ZXC_INCLUDE_DIRS})
+target_include_directories(_zxc SYSTEM PUBLIC "${LIBRARY_DIR}/include")
+
+# zxc_driver.c uses pthreads for its multi-threaded frame path.
+target_link_libraries(_zxc PRIVATE Threads::Threads)
+
+add_library(ch_contrib::zxc ALIAS _zxc)
@@ -405,6 +405,12 @@ ClickHouse supports general purpose codecs and specialized codecs.

 High compression levels are useful for asymmetric scenarios, like compress once, decompress repeatedly. Higher levels mean better compression and higher CPU usage.

+#### ZXC {#zxc}
+
+`ZXC[(level)]` — asymmetric [`zxc` compression algorithm](https://github.com/hellobertrand/zxc) with configurable `level`. Possible levels: \[1, 7\]. Default level: 3.
+
+`ZXC` trades slow compression for very fast decompression, at a compression ratio between `LZ4` and `ZSTD`. It is a good fit for the compress-once, decompress-many pattern, and decompresses fastest on modern ARM cores. Higher levels mean better compression and slower compression, while decompression stays fast.
+
 #### Obsolete: ZSTD_QAT {#zstd_qat}

 <CloudNotSupportedBadge/>
@@ -546,6 +546,7 @@ target_link_libraries (clickhouse_compression
            clickhouse_parsers
        PRIVATE
            ch_contrib::lz4
+            ch_contrib::zxc
            ch_contrib::roaring
 )
 if (TARGET ch_contrib::fastpfor)
@@ -0,0 +1,104 @@
+#include <Compression/CompressionCodecZXC.h>
+#include <Compression/CompressionInfo.h>
+#include <Compression/CompressionFactory.h>
+#include <Compression/registerCompressionCodecs.h>
+#include <Parsers/IAST.h>
+#include <Parsers/ASTLiteral.h>
+
+#include <zxc.h>
+
+#include <cstring>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int CANNOT_COMPRESS;
+    extern const int CANNOT_DECOMPRESS;
+    extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE;
+    extern const int ILLEGAL_CODEC_PARAMETER;
+}
+
+uint8_t CompressionCodecZXC::getMethodByte() const
+{
+    return static_cast<uint8_t>(CompressionMethodByte::ZXC);
+}
+
+void CompressionCodecZXC::updateHash(SipHash & hash) const
+{
+    getCodecDesc()->updateTreeHash(hash, /*ignore_aliases=*/ true);
+}
+
+UInt32 CompressionCodecZXC::getMaxCompressedDataSize(UInt32 uncompressed_size) const
+{
+    return static_cast<UInt32>(zxc_compress_bound(uncompressed_size));
+}
+
+UInt32 CompressionCodecZXC::doCompressData(const char * source, UInt32 source_size, char * dest) const
+{
+    zxc_compress_opts_t opts;
+    memset(&opts, 0, sizeof(opts));
+    opts.level = level;
+    /// ClickHouse checksums every compressed block itself, so zxc's own checksum
+    /// would be redundant work on the hot decompression path.
+    opts.checksum_enabled = 0;
+
+    int64_t compressed_size = zxc_compress(source, source_size, dest, zxc_compress_bound(source_size), &opts);
+    if (compressed_size < 0)
+        throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress with ZXC codec: error code {}", compressed_size);
+
+    return static_cast<UInt32>(compressed_size);
+}
+
+UInt32 CompressionCodecZXC::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const
+{
+    zxc_decompress_opts_t opts;
+    memset(&opts, 0, sizeof(opts));
+    opts.checksum_enabled = 0;
+
+    int64_t decompressed_size = zxc_decompress(source, source_size, dest, uncompressed_size, &opts);
+    if (decompressed_size < 0)
+        throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress ZXC-encoded data: error code {}", decompressed_size);
+    if (static_cast<UInt32>(decompressed_size) != uncompressed_size)
+        throw Exception(ErrorCodes::CANNOT_DECOMPRESS,
+            "Cannot decompress ZXC-encoded data: decompressed size {} does not match the expected size {}",
+            decompressed_size, uncompressed_size);
+
+    return static_cast<UInt32>(decompressed_size);
+}
+
+CompressionCodecZXC::CompressionCodecZXC(int level_)
+    : level(level_)
+{
+    ASTs arguments;
+    arguments.push_back(make_intrusive<ASTLiteral>(static_cast<UInt64>(level)));
+    setCodecDescription("ZXC", arguments);
+}
+
+void registerCodecZXC(CompressionCodecFactory & factory)
+{
+    UInt8 method_code = static_cast<UInt8>(CompressionMethodByte::ZXC);
+    factory.registerCompressionCodec("ZXC", method_code, [&](const ASTPtr & arguments) -> CompressionCodecPtr
+    {
+        int level = CompressionCodecZXC::ZXC_DEFAULT_LEVEL;
+        if (arguments && !arguments->children.empty())
+        {
+            if (arguments->children.size() != 1)
+                throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "ZXC codec must have 1 parameter, given {}",
+                    arguments->children.size());
+
+            const auto * literal = arguments->children[0]->as<ASTLiteral>();
+            if (!literal)
+                throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "ZXC codec argument must be integer");
+
+            level = static_cast<int>(literal->value.safeGet<UInt64>());
+            if (level < zxc_min_level() || level > zxc_max_level())
+                throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER,
+                    "ZXC codec level must be between {} and {}, given {}",
+                    zxc_min_level(), zxc_max_level(), level);
+        }
+        return std::make_shared<CompressionCodecZXC>(level);
+    });
+}
+
+}
@@ -0,0 +1,41 @@
+#pragma once
+
+#include <Compression/ICompressionCodec.h>
+
+namespace DB
+{
+
+/// zxc (https://github.com/hellobertrand/zxc) is an asymmetric LZ codec: slow to
+/// compress, but very fast to decompress at a ratio between LZ4 and ZSTD. It is a
+/// good fit for write-once/read-many analytical columns.
+class CompressionCodecZXC : public ICompressionCodec
+{
+public:
+    static constexpr int ZXC_DEFAULT_LEVEL = 3;
+
+    explicit CompressionCodecZXC(int level_);
+
+    uint8_t getMethodByte() const override;
+
+    UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override;
+
+    void updateHash(SipHash & hash) const override;
+
+protected:
+    UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
+
+    UInt32 doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
+
+    bool isCompression() const override { return true; }
+    bool isGenericCompression() const override { return true; }
+
+    String getDescription() const override
+    {
+        return "Asymmetric LZ codec (zxc): slow compression, very fast decompression, ratio between LZ4 and ZSTD. Levels 1..7.";
+    }
+
+private:
+    const int level;
+};
+
+}
@@ -271,6 +271,7 @@ CompressionCodecFactory::CompressionCodecFactory()
 #if USE_SZ3
    registerCodecSZ3(*this);
 #endif
+    registerCodecZXC(*this);

    default_codec = get("LZ4", {});
 }
@@ -51,6 +51,7 @@ enum class CompressionMethodByte : uint8_t
    /// ZSTD_QPL        = 0x9b, /// Removed, don't reuse for another codec
    ALP             = 0x9c,
    SZ3             = 0x9d,
+    ZXC             = 0x9e,
 };

 }
@@ -20,6 +20,7 @@ void registerCodecEncrypted(CompressionCodecFactory & factory);
 void registerCodecFPC(CompressionCodecFactory & factory);
 void registerCodecGCD(CompressionCodecFactory & factory);
 void registerCodecALP(CompressionCodecFactory & factory);
+void registerCodecZXC(CompressionCodecFactory & factory);
 void registerCodecSZ3(CompressionCodecFactory & factory);

 CompressionCodecPtr getCompressionCodecLZ4(int level);
@@ -0,0 +1,12 @@
+rows	300000
+l1	1
+l2	1
+l3	1
+l4	1
+l5	1
+l6	1
+l7	1
+ld	1
+v	1
+codec_present	9
+after_attach	1
@@ -0,0 +1,60 @@
+-- ZXC compression codec: roundtrip correctness across all levels, plus error handling.
+
+DROP TABLE IF EXISTS t_zxc_src;
+DROP TABLE IF EXISTS t_zxc;
+
+CREATE TABLE t_zxc_src (id UInt64, s String, v UInt32) ENGINE = MergeTree ORDER BY id;
+INSERT INTO t_zxc_src
+    SELECT number,
+           concat('r', toString(number % 9973), repeat('z', number % 37)),
+           toUInt32((number * 2654435761) % 1000000)
+    FROM numbers(300000);
+
+-- One column per level (1..7), a default-level column, and a numeric column.
+CREATE TABLE t_zxc
+(
+    id UInt64,
+    s1 String CODEC(ZXC(1)),
+    s2 String CODEC(ZXC(2)),
+    s3 String CODEC(ZXC(3)),
+    s4 String CODEC(ZXC(4)),
+    s5 String CODEC(ZXC(5)),
+    s6 String CODEC(ZXC(6)),
+    s7 String CODEC(ZXC(7)),
+    sd String CODEC(ZXC),
+    v  UInt32 CODEC(ZXC(5))
+)
+ENGINE = MergeTree ORDER BY id;
+
+INSERT INTO t_zxc SELECT id, s, s, s, s, s, s, s, s, v FROM t_zxc_src;
+
+SELECT 'rows', count() FROM t_zxc;
+
+-- Each ZXC column must decompress back to exactly the source values (1 = identical).
+SELECT 'l1', (SELECT sum(cityHash64(s1)) FROM t_zxc) = (SELECT sum(cityHash64(s)) FROM t_zxc_src);
+SELECT 'l2', (SELECT sum(cityHash64(s2)) FROM t_zxc) = (SELECT sum(cityHash64(s)) FROM t_zxc_src);
+SELECT 'l3', (SELECT sum(cityHash64(s3)) FROM t_zxc) = (SELECT sum(cityHash64(s)) FROM t_zxc_src);
+SELECT 'l4', (SELECT sum(cityHash64(s4)) FROM t_zxc) = (SELECT sum(cityHash64(s)) FROM t_zxc_src);
+SELECT 'l5', (SELECT sum(cityHash64(s5)) FROM t_zxc) = (SELECT sum(cityHash64(s)) FROM t_zxc_src);
+SELECT 'l6', (SELECT sum(cityHash64(s6)) FROM t_zxc) = (SELECT sum(cityHash64(s)) FROM t_zxc_src);
+SELECT 'l7', (SELECT sum(cityHash64(s7)) FROM t_zxc) = (SELECT sum(cityHash64(s)) FROM t_zxc_src);
+SELECT 'ld', (SELECT sum(cityHash64(sd)) FROM t_zxc) = (SELECT sum(cityHash64(s)) FROM t_zxc_src);
+SELECT 'v',  (SELECT sum(v) FROM t_zxc) = (SELECT sum(v) FROM t_zxc_src);
+
+-- Codec is recorded on all 9 ZXC columns.
+SELECT 'codec_present', count()
+FROM system.columns
+WHERE database = currentDatabase() AND table = 't_zxc' AND compression_codec LIKE '%ZXC%';
+
+-- Data persists across detach/attach (read back from disk).
+DETACH TABLE t_zxc;
+ATTACH TABLE t_zxc;
+SELECT 'after_attach', (SELECT sum(cityHash64(sd)) FROM t_zxc) = (SELECT sum(cityHash64(s)) FROM t_zxc_src);
+
+-- Invalid parameters are rejected.
+CREATE TABLE t_zxc_bad (x UInt64 CODEC(ZXC(0)))  ENGINE = MergeTree ORDER BY x; -- { serverError ILLEGAL_CODEC_PARAMETER }
+CREATE TABLE t_zxc_bad (x UInt64 CODEC(ZXC(8)))  ENGINE = MergeTree ORDER BY x; -- { serverError ILLEGAL_CODEC_PARAMETER }
+CREATE TABLE t_zxc_bad (x UInt64 CODEC(ZXC(1, 2))) ENGINE = MergeTree ORDER BY x; -- { serverError ILLEGAL_SYNTAX_FOR_CODEC_TYPE }
+
+DROP TABLE t_zxc;
+DROP TABLE t_zxc_src;