Preserve dictionary definitions in clickhouse format --obfuscate

Dictionary layout types (`complex_key_hashed`, `range_hashed`,
`ssd_cache`, etc.), source types, and source/layout parameter names
(`PORT`, `PASSWORD`, `DB`, `SIZE_IN_CELLS`, `BLOCK_SIZE`, `SHARDS`,
etc.) were being obfuscated, producing invalid SQL output.

Add `getAllRegisteredNames` to `DictionaryFactory` and
`DictionarySourceFactory`, register their names as known identifiers,
add dictionary parameter keywords to the obfuscator keyword list,
and remove the now-redundant hardcoded dictionary layout/source
keywords that are covered by the factory-based approach.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

programs/format/Format.cpp

@@ -37,6 +37,9 @@
 #include <Formats/registerFormats.h>
 #include <Compression/CompressionFactory.h>
 #include <Storages/MergeTree/MergeTreeIndices.h>
+#include <Dictionaries/DictionaryFactory.h>
+#include <Dictionaries/DictionarySourceFactory.h>
+#include <Dictionaries/registerDictionaries.h>
 #include <Processors/Transforms/getSourceFromASTInsertQuery.h>
 
 #include <boost/algorithm/string/split.hpp>
@@ -186,6 +189,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
             registerDatabases();
             registerStorages();
             registerFormats();
+            registerDictionaries();
 
             std::unordered_set<std::string> additional_names;
 
@@ -196,6 +200,8 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
             auto all_known_index_types = MergeTreeIndexFactory::instance().getAllRegisteredNames();
             auto all_known_codecs = CompressionCodecFactory::instance().getAllRegisteredNames();
             auto all_known_database_engines = DatabaseFactory::instance().getAllRegisteredNames();
+            auto all_known_dict_layouts = DictionaryFactory::instance().getAllRegisteredNames();
+            auto all_known_dict_sources = DictionarySourceFactory::instance().getAllRegisteredNames();
 
             additional_names.insert(all_known_storage_names.begin(), all_known_storage_names.end());
             additional_names.insert(all_known_data_type_names.begin(), all_known_data_type_names.end());
@@ -204,6 +210,8 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
             additional_names.insert(all_known_index_types.begin(), all_known_index_types.end());
             additional_names.insert(all_known_codecs.begin(), all_known_codecs.end());
             additional_names.insert(all_known_database_engines.begin(), all_known_database_engines.end());
+            additional_names.insert(all_known_dict_layouts.begin(), all_known_dict_layouts.end());
+            additional_names.insert(all_known_dict_sources.begin(), all_known_dict_sources.end());
 
             for (auto * it = auto_time_zones; *it; ++it)
             {

src/Dictionaries/DictionaryFactory.h

@@ -54,6 +54,15 @@ public:
 
     void registerLayout(const std::string & layout_type, LayoutCreateFunction create_layout, bool is_layout_complex, bool has_layout_complex = true);
 
+    std::vector<String> getAllRegisteredNames() const
+    {
+        std::vector<String> result;
+        result.reserve(registered_layouts.size());
+        for (const auto & pair : registered_layouts)
+            result.push_back(pair.first);
+        return result;
+    }
+
 private:
     struct RegisteredLayout
     {

src/Dictionaries/DictionarySourceFactory.h

@@ -44,6 +44,15 @@ public:
 
     void registerSource(const std::string & source_type, Creator create_source);
 
+    std::vector<String> getAllRegisteredNames() const
+    {
+        std::vector<String> result;
+        result.reserve(registered_sources.size());
+        for (const auto & pair : registered_sources)
+            result.push_back(pair.first);
+        return result;
+    }
+
     DictionarySourcePtr create(
         const std::string & name,
         const Poco::Util::AbstractConfiguration & config,

src/Parsers/obfuscateQueries.cpp

@@ -89,9 +89,29 @@ const std::unordered_set<std::string> & getObfuscateKeywords()
             "LOADING", "ENABLE", "DISABLE", "NOTIFY", "QUERIES", "SIMILARITY", "VECTOR",
             "POSTINGS", "HEADER", "METADATA", "ALLOCATE", "FREE", "LOAD", "UNLOAD", "UNKNOWN",
             "CLIENT", "MODEL",
-            /// Dictionary layout/source keywords
-            "FLAT", "HASHED", "IP_TRIE", "REGEXP_TREE", "DIRECT", "CLICKHOUSE", "EXECUTABLE",
-            "LIBRARY",
+            /// Dictionary source connection/behavior parameters
+            "PORT", "PASSWORD", "DB", "SOCKET", "ENDPOINT", "COMMAND",
+            "CONNECTION_STRING", "SSLMODE", "SSL_CA", "SSL_CERT", "SSL_KEY",
+            "PATH", "URI", "PROXY_URL", "CYPRESS_PATH",
+            "INVALIDATE_QUERY", "UPDATE_FIELD", "UPDATE_LAG",
+            "CONNECT_TIMEOUT", "RW_TIMEOUT", "MYSQL_CONNECT_TIMEOUT", "MYSQL_RW_TIMEOUT",
+            "CLOSE_CONNECTION", "SHARE_CONNECTION", "FAIL_ON_CONNECTION_LOSS",
+            "OPT_RECONNECT", "ENABLE_LOCAL_INFILE", "BACKGROUND_RECONNECT",
+            "EXECUTE_DIRECT", "IMPLICIT_KEY", "CHECK_EXIT_CODE", "STDERR_REACTION",
+            "COMMAND_READ_TIMEOUT", "COMMAND_WRITE_TIMEOUT", "COMMAND_TERMINATION_TIMEOUT",
+            "COMPRESSION_METHOD", "POOL_SIZE", "DB_INDEX", "STORAGE_TYPE",
+            "COLLECTION", "KEYSPACE", "COLUMN_FAMILY", "CONSISTENCY",
+            "ALLOW_FILTERING", "PARTITION_KEY_PREFIX", "PRIORITY",
+            /// Dictionary layout parameters
+            "SIZE_IN_CELLS", "BLOCK_SIZE", "SHARDS", "SHARD_LOAD_QUEUE_BACKLOG",
+            "PREALLOCATE", "MAX_LOAD_FACTOR",
+            "INITIAL_ARRAY_SIZE", "MAX_ARRAY_SIZE",
+            "FILE_SIZE", "READ_BUFFER_SIZE", "WRITE_BUFFER_SIZE", "MAX_PARTITIONS_COUNT",
+            "ALLOW_READ_EXPIRED_KEYS", "STRICT_MAX_LIFETIME_SECONDS",
+            "MAX_THREADS_FOR_UPDATES", "MAX_UPDATE_QUEUE_SIZE",
+            "QUERY_WAIT_TIMEOUT_MILLISECONDS", "UPDATE_QUEUE_PUSH_TIMEOUT_MILLISECONDS",
+            "CONVERT_NULL_RANGE_BOUND_TO_OPEN", "RANGE_LOOKUP_STRATEGY",
+            "STORE_POLYGON_KEY_COLUMN", "REQUIRE_NONEMPTY",
             /// Special SQL functions/types parsed by special rules
             "EXTRACT", "TRIM", "DECIMAL",
             /// Multi-word data type constituents not registered as standalone keywords

tests/queries/0_stateless/04036_obfuscator_database_engines.reference

@@ -1,3 +1,3 @@
-CREATE DATABASE attention ENGINE = Atomic
-CREATE DATABASE attention ENGINE = Memory
-CREATE DATABASE attention ENGINE = MaterializedPostgreSQL('fzfw:6076', 'tfxl', 'user', 'uqjd')
+CREATE DATABASE db ENGINE = Atomic
+CREATE DATABASE db ENGINE = Memory
+CREATE DATABASE db ENGINE = MaterializedPostgreSQL('fzfw:6076', 'tfxl', 'user', 'uqjd')

tests/queries/0_stateless/04037_obfuscator_dictionary_definitions.reference

@@ -0,0 +1,4 @@
+CREATE DICTIONARY d (id UInt64, name String) PRIMARY KEY id SOURCE(MYSQL(HOST 'xwcuulbce' PORT 2625 USER 'xnst' PASSWORD '' DB 'fnbs' TABLE 'bdtbkw')) LAYOUT(FLAT()) LIFETIME(0)
+CREATE DICTIONARY d (id UInt64, name String) PRIMARY KEY id SOURCE(CLICKHOUSE(TABLE 'f')) LAYOUT(COMPLEX_KEY_HASHED(SHARDS 15 PREALLOCATE 1)) LIFETIME(0)
+CREATE DICTIONARY d (id UInt64, name String) PRIMARY KEY id SOURCE(CLICKHOUSE(TABLE 'f')) LAYOUT(SSD_CACHE(BLOCK_SIZE 7588 FILE_SIZE 16543306 SIZE_IN_CELLS 1000000)) LIFETIME(0)
+CREATE DICTIONARY d (id UInt64, name String) PRIMARY KEY id SOURCE(CLICKHOUSE(TABLE 'f')) LAYOUT(RANGE_HASHED()) LIFETIME(0)

tests/queries/0_stateless/04037_obfuscator_dictionary_definitions.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+obf="$CLICKHOUSE_FORMAT --obfuscate"
+
+echo "CREATE DICTIONARY d (id UInt64, name String) PRIMARY KEY id SOURCE(MYSQL(HOST 'localhost' PORT 3306 USER 'root' PASSWORD '' DB 'test' TABLE 'source')) LAYOUT(FLAT()) LIFETIME(0)" | $obf
+echo "CREATE DICTIONARY d (id UInt64, name String) PRIMARY KEY id SOURCE(CLICKHOUSE(TABLE 't')) LAYOUT(COMPLEX_KEY_HASHED(SHARDS 16 PREALLOCATE 1)) LIFETIME(0)" | $obf
+echo "CREATE DICTIONARY d (id UInt64, name String) PRIMARY KEY id SOURCE(CLICKHOUSE(TABLE 't')) LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 16777216 SIZE_IN_CELLS 1000000)) LIFETIME(0)" | $obf
+echo "CREATE DICTIONARY d (id UInt64, name String) PRIMARY KEY id SOURCE(CLICKHOUSE(TABLE 't')) LAYOUT(RANGE_HASHED()) LIFETIME(0)" | $obf