mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
zeekay
9a90943b8f
13,838 upstream commits, forward from the 26.6.1.1 fork point. 129 conflicts: 37 resolved mechanically (take upstream's file, re-apply the brand sweep) and 92 by hand where our fork and upstream had both changed the same code. The rule throughout: upstream owns the engine, we own the naming and a small set of deliberate deviations. Never resolve by discarding an upstream change — a dropped rename or a missed new parameter becomes a compile error a thousand files away. Deviations verified intact after the merge: ENABLE_XRAY still defaults OFF (CMakeLists.txt:105) X-Datastore-User read by the server (authenticateUserByHTTP.cpp) X-ClickHouse-* in src/ 0 hanzo/research present system.*_log retention bound present ArrowFlight rip held against upstream re-adding it Two things worth naming, because neither shows up as a conflict: src/CMakeLists.txt gained an UNGUARDED `add_object_library(datastore_storages_arrowflight Storages/ArrowFlight)` from upstream, for a directory our gRPC rip deleted. It sat outside every conflict hunk, so a resolver reading only the markers would sail straight past it into a CMake configure error. Removed; the Server/ArrowFlight sibling at line 474 stays because it is inside `if (TARGET ch_contrib::arrow_flight)`. The brand sweep had rewritten github.com/ClickHouse/ClickHouse issue links to github.com/Datastore/Datastore — a dead host. Those point at upstream's own tracker and are restored, alongside the contributors list, sql.clickhouse.com and the ClickHouse Inc copyright: attribution is theirs, not ours to rename. base/glibc-compatibility/memcpy/memcpy.h is deleted, not kept: upstream dropped it for LLVM-libc's, and our only change to it was a comment rebrand.
2019 lines
93 KiB
Python
2019 lines
93 KiB
Python
from abc import abstractmethod
|
|
import xml.etree.ElementTree as ET
|
|
import tempfile
|
|
import multiprocessing
|
|
import random
|
|
import string
|
|
import typing
|
|
|
|
from environment import get_system_timezones
|
|
from integration.helpers.cluster import DatastoreCluster
|
|
from integration.helpers.config_cluster import mongo_pass, mysql_pass, pg_pass
|
|
|
|
|
|
def generate_xml_safe_string(length: int = 10) -> str:
|
|
"""
|
|
Generate a random string that is safe to use as XML value content.
|
|
|
|
Args:
|
|
length (int): Desired length of the string (default: 10)
|
|
|
|
Returns:
|
|
str: Random string containing only XML-safe characters
|
|
"""
|
|
# XML 1.0 valid characters (excluding control chars except tab, LF, CR)
|
|
xml_safe_chars = (
|
|
"\t\n\r" # allowed control chars
|
|
+ string.ascii_letters
|
|
+ string.digits
|
|
+ " !\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" # punctuation
|
|
+ "\u0020\ud7ff\ue000-\ufffd" # other valid Unicode ranges
|
|
)
|
|
# For Python 3, we need to handle the Unicode ranges properly
|
|
# Create a list of valid characters
|
|
valid_chars = []
|
|
# Add basic ASCII characters
|
|
valid_chars.extend(c for c in xml_safe_chars if ord(c) < 0xD800)
|
|
# Add higher Unicode characters (avoid surrogates)
|
|
valid_chars.extend(chr(c) for c in range(0xE000, 0xFFFD + 1))
|
|
# Generate the random string
|
|
return "".join(random.choice(valid_chars) for _ in range(length))
|
|
|
|
|
|
def threshold_generator(
|
|
always_on_prob, always_off_prob, min_val, max_val, bits: int = 64
|
|
):
|
|
def gen():
|
|
tmp = random.random()
|
|
if tmp <= always_on_prob:
|
|
return min_val
|
|
if tmp <= always_on_prob + always_off_prob:
|
|
return max_val
|
|
if (
|
|
tmp <= always_on_prob + always_off_prob + 0.01
|
|
and isinstance(min_val, int)
|
|
and isinstance(max_val, int)
|
|
):
|
|
return 2**bits - 1
|
|
|
|
if isinstance(min_val, int) and isinstance(max_val, int):
|
|
return random.randint(min_val, max_val)
|
|
return random.uniform(min_val, max_val)
|
|
|
|
return gen
|
|
|
|
|
|
def file_size_value(max_val: int, bits: int = 64, min_val: int = 1):
|
|
def gen():
|
|
return str(
|
|
threshold_generator(0.2, 0.2, min_val, max_val, bits)()
|
|
) + random.choice(
|
|
["Ki", "Ki", "Mi", "Gi"] # Increased probability
|
|
)
|
|
|
|
return gen
|
|
|
|
|
|
true_false_lambda = lambda: random.randint(0, 1)
|
|
threads_lambda = lambda: random.randint(0, multiprocessing.cpu_count())
|
|
no_zero_threads_lambda = lambda: random.randint(1, multiprocessing.cpu_count())
|
|
|
|
|
|
rocksdb_properties = {
|
|
"rocksdb": {
|
|
"options": {
|
|
"max_background_jobs": threshold_generator(0.2, 0.2, 0, 100),
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
possible_properties = {
|
|
"access_control_improvements": {
|
|
"on_cluster_queries_require_cluster_grant": true_false_lambda,
|
|
"role_cache_expiration_time_seconds": threshold_generator(0.2, 0.2, 1, 60, 31),
|
|
"select_from_information_schema_requires_grant": true_false_lambda,
|
|
"select_from_system_db_requires_grant": true_false_lambda,
|
|
"settings_constraints_replace_previous": true_false_lambda,
|
|
"table_engines_require_grant": true_false_lambda,
|
|
"throw_on_unmatched_row_policies": true_false_lambda,
|
|
"users_without_row_policies_can_read_rows": true_false_lambda,
|
|
},
|
|
"aggregate_function_group_array_action_when_limit_is_reached": lambda: random.choice(
|
|
["throw", "discard"]
|
|
),
|
|
"aggregate_function_group_array_max_element_size": threshold_generator(
|
|
0.2, 0.2, 0, 10000
|
|
),
|
|
"allow_feature_tier": lambda: random.randint(0, 2),
|
|
"allow_use_jemalloc_memory": true_false_lambda,
|
|
"async_insert_queue_flush_on_shutdown": true_false_lambda,
|
|
"async_insert_threads": threads_lambda,
|
|
"async_load_databases": true_false_lambda,
|
|
"async_load_system_database": true_false_lambda,
|
|
"asynchronous_heavy_metrics_update_period_s": threshold_generator(0.2, 0.2, 1, 60),
|
|
"asynchronous_metrics_enable_heavy_metrics": true_false_lambda,
|
|
"asynchronous_metrics_keeper_metrics_only": true_false_lambda,
|
|
"asynchronous_metrics_update_period_s": threshold_generator(0.2, 0.2, 1, 30),
|
|
"background_buffer_flush_schedule_pool_size": threads_lambda,
|
|
"background_common_pool_size": no_zero_threads_lambda,
|
|
"background_distributed_schedule_pool_size": no_zero_threads_lambda,
|
|
"background_fetches_pool_size": no_zero_threads_lambda,
|
|
# "background_merges_mutations_concurrency_ratio": threshold_generator(
|
|
# 0.2, 0.2, 0.0, 3.0
|
|
# ),
|
|
"background_merges_mutations_scheduling_policy": lambda: random.choice(
|
|
["round_robin", "shortest_task_first"]
|
|
),
|
|
"background_message_broker_schedule_pool_size": no_zero_threads_lambda,
|
|
"background_move_pool_size": no_zero_threads_lambda,
|
|
# "background_pool_size": threads_lambda, has to be in a certain range
|
|
# Clamped to background_schedule_pool_size internally, so any value is safe
|
|
"background_schedule_pool_initial_size": threshold_generator(0.2, 0.2, 0, 128),
|
|
"background_schedule_pool_max_parallel_tasks_per_type_ratio": threshold_generator(
|
|
0.2, 0.2, 0.0, 1.0
|
|
),
|
|
"background_schedule_pool_size": no_zero_threads_lambda,
|
|
"background_streaming_schedule_pool_size": no_zero_threads_lambda,
|
|
"backup_threads": no_zero_threads_lambda,
|
|
"backups_io_thread_pool_queue_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"bcrypt_workfactor": threshold_generator(0.2, 0.2, 0, 20, 31),
|
|
"cache_size_to_ram_max_ratio": threshold_generator(0.2, 0.2, 0.01, 1.0),
|
|
# "cannot_allocate_thread_fault_injection_probability": threshold_generator(0.2, 0.2, 0.0, 1.0), the server may not start
|
|
"compiled_expression_cache_elements_size": threshold_generator(0.2, 0.2, 0, 10000),
|
|
"compiled_expression_cache_size": threshold_generator(0.2, 0.2, 0, 10000),
|
|
"concurrent_threads_lazy_allocation": true_false_lambda,
|
|
"concurrent_threads_scheduler": lambda: random.choice(
|
|
["round_robin", "fair_round_robin", "max_min_fair"]
|
|
),
|
|
"concurrent_threads_soft_limit_num": threads_lambda,
|
|
"concurrent_threads_soft_limit_ratio_to_cores": threads_lambda,
|
|
"cpu_slot_preemption": true_false_lambda,
|
|
"cpu_slot_preemption_timeout_ms": threshold_generator(0.2, 0.2, 0, 5000),
|
|
"cpu_slot_quantum_ns": threshold_generator(0.2, 0.2, 0, 100000000),
|
|
"database_atomic_delay_before_drop_table_sec": threshold_generator(0.2, 0.2, 0, 60),
|
|
"database_catalog_drop_error_cooldown_sec": threshold_generator(0.2, 0.2, 0, 30),
|
|
"database_catalog_drop_table_concurrency": threads_lambda,
|
|
"database_catalog_unused_dir_cleanup_period_sec": threshold_generator(
|
|
0.2, 0.2, 60, 86400
|
|
),
|
|
"database_catalog_unused_dir_hide_timeout_sec": threshold_generator(
|
|
0.2, 0.2, 60, 3600
|
|
),
|
|
"database_catalog_unused_dir_rm_timeout_sec": threshold_generator(
|
|
0.2, 0.2, 3600, 2592000
|
|
),
|
|
"database_replicated_allow_detach_permanently": true_false_lambda,
|
|
"database_replicated_drop_broken_tables": true_false_lambda,
|
|
"dictionaries_lazy_load": true_false_lambda,
|
|
"disable_insertion_and_mutation": true_false_lambda,
|
|
"disable_internal_dns_cache": true_false_lambda,
|
|
"disk_connections_hard_limit": threshold_generator(0.2, 0.2, 0, 400000),
|
|
"disk_connections_rcvbuf": threshold_generator(0.2, 0.2, 0, 16 * 1024 * 1024),
|
|
"disk_connections_sndbuf": threshold_generator(0.2, 0.2, 0, 16 * 1024 * 1024),
|
|
"disk_connections_soft_limit": threshold_generator(0.2, 0.2, 0, 10000),
|
|
"disk_connections_store_limit": threshold_generator(0.2, 0.2, 0, 20000),
|
|
"disk_connections_warn_limit": threshold_generator(0.2, 0.2, 0, 15000),
|
|
"disk_transaction_wait_for_blob_removal": true_false_lambda,
|
|
"display_secrets_in_show_and_select": true_false_lambda,
|
|
"distributed_cache_apply_throttling_settings_from_client": true_false_lambda,
|
|
"distributed_cache_keep_up_free_connections_ratio": threshold_generator(
|
|
0.2, 0.2, 0.0, 1.0
|
|
),
|
|
"distributed_ddl_cleanup_delay_period": threshold_generator(0.2, 0.2, 0, 300),
|
|
"distributed_ddl_max_tasks_in_queue": threshold_generator(0.2, 0.2, 0, 2000),
|
|
"distributed_ddl_pool_size": no_zero_threads_lambda,
|
|
"distributed_ddl_task_max_lifetime": threshold_generator(0.2, 0.2, 0, 604800),
|
|
"distributed_ddl_use_initial_user_and_roles": true_false_lambda,
|
|
"dns_allow_resolve_names_to_ipv4": true_false_lambda,
|
|
"dns_allow_resolve_names_to_ipv6": true_false_lambda,
|
|
"dns_cache_max_entries": threshold_generator(0.2, 0.2, 0, 1024),
|
|
"dns_cache_update_period": threshold_generator(0.2, 0.2, 1, 600),
|
|
"dns_max_consecutive_failures": threshold_generator(0.2, 0.2, 1, 10),
|
|
"drop_distributed_cache_pool_size": threads_lambda,
|
|
"drop_distributed_cache_queue_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"enable_azure_sdk_logging": true_false_lambda,
|
|
"enable_system_unfreeze": true_false_lambda,
|
|
"enable_webterminal": true_false_lambda,
|
|
"encryption_header_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"encryption_header_cache_size": threshold_generator(0.2, 0.2, 0, 104857600),
|
|
"encryption_header_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"format_parsing_thread_pool_queue_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"global_profiler_cpu_time_period_ns": threshold_generator(0.2, 0.2, 0, 1000000000),
|
|
"global_profiler_real_time_period_ns": threshold_generator(0.2, 0.2, 0, 1000000000),
|
|
"handshake_timeout_milliseconds": threshold_generator(0.2, 0.2, 1000, 30000),
|
|
"http_connections_hard_limit": threshold_generator(0.2, 0.2, 0, 400000),
|
|
"http_connections_rcvbuf": threshold_generator(0.2, 0.2, 0, 16 * 1024 * 1024),
|
|
"http_connections_sndbuf": threshold_generator(0.2, 0.2, 0, 16 * 1024 * 1024),
|
|
"http_connections_soft_limit": threshold_generator(0.2, 0.2, 0, 10000),
|
|
"http_connections_store_limit": threshold_generator(0.2, 0.2, 0, 20000),
|
|
"http_connections_warn_limit": threshold_generator(0.2, 0.2, 0, 15000),
|
|
"iceberg_background_schedule_pool_size": threads_lambda,
|
|
"iceberg_catalog_threadpool_pool_size": threads_lambda,
|
|
"iceberg_catalog_threadpool_queue_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"iceberg_compaction_threadpool_pool_size": threads_lambda,
|
|
"iceberg_compaction_threadpool_queue_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"iceberg_metadata_files_cache_max_entries": threshold_generator(0.2, 0.2, 0, 1024),
|
|
"iceberg_metadata_files_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"iceberg_metadata_files_cache_size": threshold_generator(0.2, 0.2, 0, 5368709120),
|
|
"iceberg_metadata_files_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"iceberg_scheduler_compaction_threadpool_pool_size": threads_lambda,
|
|
"iceberg_scheduler_compaction_threadpool_queue_size": threshold_generator(
|
|
0.2, 0.2, 0, 1000
|
|
),
|
|
"ignore_empty_sql_security_in_create_view_query": true_false_lambda,
|
|
"index_mark_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"index_mark_cache_prewarm_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"index_mark_cache_size": threshold_generator(0.2, 0.2, 0, 5368709120),
|
|
"index_mark_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"index_uncompressed_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"index_uncompressed_cache_size": threshold_generator(0.2, 0.2, 0, 5368709120),
|
|
"index_uncompressed_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"io_thread_pool_queue_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"jemalloc_collect_global_profile_samples_in_trace_log": true_false_lambda,
|
|
"jemalloc_enable_background_threads": true_false_lambda,
|
|
"jemalloc_enable_global_profiler": true_false_lambda,
|
|
"jemalloc_flush_profile_interval_bytes": threshold_generator(
|
|
0.2, 0.2, 0, 100 * 1024 * 1024
|
|
),
|
|
"jemalloc_flush_profile_on_memory_exceeded": true_false_lambda,
|
|
"jemalloc_flush_profile_on_memory_exceeded_interval": threshold_generator(
|
|
0.2, 0.2, 0, 10000
|
|
),
|
|
"jemalloc_max_background_threads_num": threads_lambda,
|
|
# log2 of the sampling interval in bytes (default 19 = 512KiB); keep a floor
|
|
# so the profiler doesn't sample nearly every allocation
|
|
"jemalloc_profiler_sampling_rate": threshold_generator(0.2, 0.2, 10, 32),
|
|
"keeper_multiread_batch_size": threshold_generator(0.2, 0.2, 1, 1000),
|
|
"load_marks_threadpool_pool_size": threads_lambda,
|
|
"load_marks_threadpool_queue_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"mark_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"mark_cache_prewarm_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"mark_cache_size": threshold_generator(0.2, 0.2, 0, 5368709120),
|
|
"mark_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"max_active_parts_loading_thread_pool_size": threads_lambda,
|
|
"max_backup_bandwidth_for_server": threshold_generator(0.2, 0.2, 0, 100000),
|
|
"max_backups_io_thread_pool_free_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"max_backups_io_thread_pool_size": threads_lambda,
|
|
"max_build_vector_similarity_index_thread_pool_size": threads_lambda,
|
|
"max_concurrent_insert_queries": threshold_generator(0.2, 0.2, 0, 50),
|
|
"max_concurrent_queries": threshold_generator(0.2, 0.2, 0, 100),
|
|
"max_concurrent_select_queries": threshold_generator(0.2, 0.2, 0, 50),
|
|
"max_connections": threshold_generator(0.2, 0.2, 100, 4096, 31),
|
|
"max_database_num_to_throw": threshold_generator(0.2, 0.2, 0, 10),
|
|
"max_database_replicated_create_table_thread_pool_size": threads_lambda,
|
|
"max_dictionary_num_to_throw": threshold_generator(0.2, 0.2, 0, 10),
|
|
"max_distributed_cache_read_bandwidth_for_server": threshold_generator(
|
|
0.2, 0.2, 0, 100000
|
|
),
|
|
"max_distributed_cache_write_bandwidth_for_server": threshold_generator(
|
|
0.2, 0.2, 0, 100000
|
|
),
|
|
"max_entries_for_hash_table_stats": threshold_generator(0.2, 0.2, 0, 10000),
|
|
"max_fetch_partition_thread_pool_size": threads_lambda,
|
|
"max_format_parsing_thread_pool_free_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"max_format_parsing_thread_pool_size": threads_lambda,
|
|
"max_held_snapshots": threshold_generator(0.2, 0.2, 0, 10),
|
|
"max_http_index_page_size": threshold_generator(0.2, 0.2, 0, 10 * 1024 * 1024),
|
|
"max_io_thread_pool_free_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"max_io_thread_pool_size": threads_lambda,
|
|
"max_local_read_bandwidth_for_server": threshold_generator(0.2, 0.2, 0, 100000),
|
|
"max_local_write_bandwidth_for_server": threshold_generator(0.2, 0.2, 0, 100000),
|
|
"max_materialized_views_count_for_table": threshold_generator(0.2, 0.2, 0, 8),
|
|
"max_merges_bandwidth_for_server": threshold_generator(0.2, 0.2, 0, 100000),
|
|
"max_mutations_bandwidth_for_server": threshold_generator(0.2, 0.2, 0, 100000),
|
|
"max_named_collection_num_to_throw": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"max_open_files": threshold_generator(0.2, 0.2, 0, 100),
|
|
"max_outdated_parts_loading_thread_pool_size": threads_lambda,
|
|
"max_partition_size_to_drop": threshold_generator(0.2, 0.2, 0, 100000),
|
|
"max_parts_cleaning_thread_pool_size": threads_lambda,
|
|
"max_per_cpu_untracked_memory": threshold_generator(0.2, 0.2, 0, 8 * 1024 * 1024),
|
|
"max_prefixes_deserialization_thread_pool_free_size": threshold_generator(
|
|
0.2, 0.2, 0, 1000
|
|
),
|
|
"max_prefixes_deserialization_thread_pool_size": threads_lambda,
|
|
# 0 disables ReaderExecutor connection reuse
|
|
"max_remote_read_connections": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"max_remote_read_network_bandwidth_for_server": threshold_generator(
|
|
0.2, 0.2, 0, 1000
|
|
),
|
|
"max_remote_write_network_bandwidth_for_server": threshold_generator(
|
|
0.2, 0.2, 0, 1000
|
|
),
|
|
"max_replicated_fetches_network_bandwidth_for_server": threshold_generator(
|
|
0.2, 0.2, 0, 1000
|
|
),
|
|
"max_replicated_sends_network_bandwidth_for_server": threshold_generator(
|
|
0.2, 0.2, 0, 1000
|
|
),
|
|
"max_replicated_table_num_to_throw": threshold_generator(0.2, 0.2, 0, 10),
|
|
# "max_server_memory_usage": threshold_generator(0.2, 0.2, 0, 10),
|
|
"max_server_memory_usage_to_ram_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"max_snapshot_commit_thread_pool_free_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"max_snapshot_commit_thread_pool_size": threads_lambda,
|
|
"max_table_num_to_throw": threshold_generator(0.2, 0.2, 0, 10),
|
|
# "max_temporary_data_on_disk_size": threshold_generator(0.2, 0.2, 0, 1000), not worth to mess around
|
|
"max_thread_pool_free_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"max_thread_pool_size": threshold_generator(0.2, 0.2, 700, 10000),
|
|
"max_unexpected_parts_loading_thread_pool_size": threads_lambda,
|
|
"max_view_num_to_throw": threshold_generator(0.2, 0.2, 0, 10),
|
|
"max_waiting_queries": threshold_generator(0.2, 0.2, 0, 100),
|
|
"memory_worker_correct_memory_tracker": true_false_lambda,
|
|
"memory_worker_decay_adjustment_period_ms": threshold_generator(0.2, 0.2, 0, 30000),
|
|
"memory_worker_dynamic_hard_limit": true_false_lambda,
|
|
"memory_worker_period_ms": threshold_generator(0.2, 0.2, 0, 10000),
|
|
"memory_worker_purge_dirty_pages_threshold_ratio": threshold_generator(
|
|
0.2, 0.2, 0.0, 1.0
|
|
),
|
|
"memory_worker_purge_total_memory_threshold_ratio": threshold_generator(
|
|
0.2, 0.2, 0.0, 1.0
|
|
),
|
|
"memory_worker_rss_speculative_reserve_ratio": threshold_generator(
|
|
0.2, 0.2, 0.0, 1.0
|
|
),
|
|
"memory_worker_use_cgroup": true_false_lambda,
|
|
"merges_mutations_memory_usage_soft_limit": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"merges_mutations_memory_usage_to_ram_ratio": threshold_generator(
|
|
0.2, 0.2, 0.0, 1.0
|
|
),
|
|
"message_queue_disable_insertion": true_false_lambda,
|
|
"min_allocation_size_to_throw_on_memory_limit": threshold_generator(
|
|
0.2, 0.2, 0, 1024 * 1024
|
|
),
|
|
"mlock_executable": true_false_lambda,
|
|
"mlock_executable_min_total_memory_amount_bytes": threshold_generator(
|
|
0.2, 0.2, 0, 10 * 1024 * 1024
|
|
),
|
|
"mmap_cache_size": threshold_generator(0.2, 0.2, 0, 2000),
|
|
"oom_canary_enable": true_false_lambda,
|
|
"oom_canary_initial_backoff_seconds": threshold_generator(0.2, 0.2, 0, 10),
|
|
"oom_canary_max_backoff_seconds": threshold_generator(0.2, 0.2, 0, 120),
|
|
"oom_canary_max_rapid_relaunches": threshold_generator(0.2, 0.2, 0, 100),
|
|
"oom_canary_relaunch": true_false_lambda,
|
|
# bits=27 keeps the occasional "huge value" case at 128Mi instead of 2^64
|
|
"oom_canary_size": threshold_generator(0.2, 0.2, 0, 104857600, 27),
|
|
"os_collect_psi_metrics": true_false_lambda,
|
|
"os_threads_nice_value_distributed_cache_tcp_handler": threshold_generator(
|
|
0.2, 0.2, -20, 19
|
|
),
|
|
"os_threads_nice_value_merge_mutate": threshold_generator(0.2, 0.2, -20, 19),
|
|
"os_threads_nice_value_zookeeper_client_send_receive": threshold_generator(
|
|
0.2, 0.2, -20, 19
|
|
),
|
|
"page_cache_free_memory_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"page_cache_history_window_ms": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"page_cache_max_size": threshold_generator(0.2, 0.2, 0, 2097152),
|
|
"page_cache_min_size": threshold_generator(0.2, 0.2, 0, 2097152),
|
|
"page_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"page_cache_shards": threshold_generator(0.2, 0.2, 0, 10),
|
|
"page_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"parquet_metadata_cache_max_entries": threshold_generator(0.2, 0.2, 0, 1024),
|
|
"parquet_metadata_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"parquet_metadata_cache_size": threshold_generator(0.2, 0.2, 0, 5368709120),
|
|
"parquet_metadata_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"parts_kill_delay_period": threshold_generator(0.2, 0.2, 0, 60),
|
|
"parts_kill_delay_period_random_add": threshold_generator(0.2, 0.2, 0, 100),
|
|
"parts_killer_max_condemned_parts_per_batch": threshold_generator(
|
|
0.2, 0.2, 1, 100000
|
|
), # Cloud setting
|
|
"parts_killer_pool_size": threads_lambda, # Cloud setting
|
|
"per_cpu_untracked_memory_thread_buffer": threshold_generator(
|
|
0.2, 0.2, 0, 1024 * 1024
|
|
),
|
|
"point_in_polygon_cache_size": threshold_generator(0.2, 0.2, 0, 104857600),
|
|
"prefetch_threadpool_pool_size": threshold_generator(0.2, 0.2, 1, 1000),
|
|
"prefetch_threadpool_queue_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"prefixes_deserialization_thread_pool_thread_pool_queue_size": threshold_generator(
|
|
0.2, 0.2, 0, 1000
|
|
),
|
|
"prepare_system_log_tables_on_startup": true_false_lambda,
|
|
"primary_index_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"primary_index_cache_prewarm_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"primary_index_cache_size": threshold_generator(0.2, 0.2, 0, 5368709120),
|
|
"primary_index_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"process_query_plan_packet": true_false_lambda,
|
|
"query_cache": {
|
|
"max_entries": threshold_generator(0.2, 0.2, 0, 1024),
|
|
"max_entry_size_in_bytes": threshold_generator(0.2, 0.2, 0, 10 * 1024 * 1024),
|
|
"max_entry_size_in_rows": threshold_generator(0.2, 0.2, 0, 10000),
|
|
"max_size_in_bytes": threshold_generator(0.2, 0.2, 0, 10 * 1024 * 1024),
|
|
},
|
|
"query_cache_max_entries": threshold_generator(0.2, 0.2, 0, 1024),
|
|
"query_cache_max_entry_size_in_bytes": threshold_generator(0.2, 0.2, 0, 1048576),
|
|
"query_cache_max_entry_size_in_rows": threshold_generator(0.2, 0.2, 0, 30000000),
|
|
"query_cache_max_size_in_bytes": threshold_generator(0.2, 0.2, 0, 1073741824),
|
|
"query_condition_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"query_condition_cache_size": threshold_generator(0.2, 0.2, 0, 104857600),
|
|
"query_condition_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"remap_executable": true_false_lambda,
|
|
"restore_threads": no_zero_threads_lambda,
|
|
"s3_allow_server_credentials_for_system_table_disks": true_false_lambda,
|
|
"s3_credentials_provider_max_cache_size": threshold_generator(
|
|
0.2, 0.2, 0, 104857600
|
|
),
|
|
"s3_load_table_anonymously_if_credentials_restricted": true_false_lambda,
|
|
"s3queue_disable_streaming": true_false_lambda,
|
|
"show_license_expiration_warnings": true_false_lambda, # Cloud setting
|
|
"shutdown_wait_backups_and_restores": true_false_lambda,
|
|
"shutdown_wait_unfinished": threshold_generator(0.2, 0.2, 0, 30),
|
|
"shutdown_wait_unfinished_queries": true_false_lambda,
|
|
"skip_check_for_incorrect_settings": true_false_lambda,
|
|
"snapshot_cleaner_period": threshold_generator(0.2, 0.2, 0, 600),
|
|
"snapshot_cleaner_pool_size": threads_lambda,
|
|
"startup_mv_delay_ms": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"storage_connections_hard_limit": threshold_generator(0.2, 0.2, 0, 400000),
|
|
"storage_connections_rcvbuf": threshold_generator(0.2, 0.2, 0, 16 * 1024 * 1024),
|
|
"storage_connections_sndbuf": threshold_generator(0.2, 0.2, 0, 16 * 1024 * 1024),
|
|
"storage_connections_soft_limit": threshold_generator(0.2, 0.2, 0, 10000),
|
|
"storage_connections_store_limit": threshold_generator(0.2, 0.2, 0, 20000),
|
|
"storage_connections_warn_limit": threshold_generator(0.2, 0.2, 0, 15000),
|
|
"storage_metadata_write_full_object_key": true_false_lambda,
|
|
"storage_shared_set_join_use_inner_uuid": true_false_lambda,
|
|
"tables_loader_background_pool_size": threads_lambda,
|
|
"tables_loader_foreground_pool_size": threads_lambda,
|
|
"temporary_data_in_distributed_cache": true_false_lambda,
|
|
"text_index_header_cache_max_entries": threshold_generator(0.2, 0.2, 0, 1024),
|
|
"text_index_header_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"text_index_header_cache_size": threshold_generator(0.2, 0.2, 0, 104857600),
|
|
"text_index_header_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"text_index_postings_cache_max_entries": threshold_generator(0.2, 0.2, 0, 1024),
|
|
"text_index_postings_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"text_index_postings_cache_size": threshold_generator(0.2, 0.2, 0, 104857600),
|
|
"text_index_postings_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"text_index_tokens_cache_max_entries": threshold_generator(0.2, 0.2, 0, 1024),
|
|
"text_index_tokens_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"text_index_tokens_cache_size": threshold_generator(0.2, 0.2, 0, 104857600),
|
|
"text_index_tokens_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"thread_pool_queue_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"threadpool_local_fs_reader_pool_size": threads_lambda,
|
|
"threadpool_local_fs_reader_queue_size": threshold_generator(0.2, 0.2, 0, 20000),
|
|
"threadpool_remote_fs_reader_pool_size": threads_lambda,
|
|
"threadpool_remote_fs_reader_queue_size": threshold_generator(0.2, 0.2, 0, 20000),
|
|
"threadpool_writer_pool_size": threshold_generator(0.2, 0.2, 1, 200),
|
|
"threadpool_writer_queue_size": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"throw_on_unknown_workload": true_false_lambda,
|
|
"total_memory_profiler_sample_max_allocation_size": threshold_generator(
|
|
0.2, 0.2, 0, 1024 * 1024
|
|
),
|
|
"total_memory_profiler_sample_min_allocation_size": threshold_generator(
|
|
0.2, 0.2, 0, 1024 * 1024
|
|
),
|
|
"total_memory_profiler_step": threshold_generator(0.2, 0.2, 0, 4 * 1024 * 1024),
|
|
"total_memory_tracker_sample_probability": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"transaction_log": {
|
|
"fault_probability_after_commit": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"fault_probability_before_commit": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
},
|
|
"uncompressed_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"uncompressed_cache_size": threshold_generator(0.2, 0.2, 0, 2097152),
|
|
"uncompressed_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"unique_key_bitmap_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"unique_key_bitmap_cache_size_bytes": threshold_generator(0.2, 0.2, 0, 5368709120),
|
|
"unique_key_bitmap_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"unique_key_index_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"unique_key_index_cache_size_bytes": threshold_generator(0.2, 0.2, 0, 5368709120),
|
|
"unique_key_index_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"use_separate_cache_arena": true_false_lambda,
|
|
"use_shared_merge_tree_log_pipeline": true_false_lambda, # Cloud setting
|
|
"user_profile_events_per_cpu": true_false_lambda,
|
|
"validate_tcp_client_information": true_false_lambda,
|
|
"vector_similarity_index_cache_max_entries": threshold_generator(0.2, 0.2, 0, 1024),
|
|
"vector_similarity_index_cache_policy": lambda: random.choice(["LRU", "SLRU"]),
|
|
"vector_similarity_index_cache_size": threshold_generator(0.2, 0.2, 0, 5368709120),
|
|
"vector_similarity_index_cache_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"wait_dictionaries_load_at_startup": true_false_lambda,
|
|
"zookeeper": {
|
|
"use_compression": true_false_lambda,
|
|
"zookeeper_load_balancing": lambda: random.choice(
|
|
[
|
|
"random",
|
|
"in_order",
|
|
"nearest_hostname",
|
|
"hostname_levenshtein_distance",
|
|
"first_or_random",
|
|
"round_robin",
|
|
]
|
|
),
|
|
},
|
|
**rocksdb_properties,
|
|
}
|
|
|
|
distributed_properties = {
|
|
"cleanup_delay_period": threshold_generator(0.2, 0.2, 0, 60),
|
|
"max_tasks_in_queue": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"pool_size": no_zero_threads_lambda,
|
|
"task_max_lifetime": threshold_generator(0.2, 0.2, 0, 60),
|
|
}
|
|
|
|
object_storages_properties = {
|
|
"local": {},
|
|
"s3": {
|
|
"list_object_keys_size": threshold_generator(0.2, 0.2, 1, 10 * 1024 * 1024, 31),
|
|
"metadata_keep_free_space_bytes": threshold_generator(
|
|
0.2, 0.2, 0, 10 * 1024 * 1024
|
|
),
|
|
"object_metadata_cache_size": threshold_generator(
|
|
0.2, 0.2, 0, 10 * 1024 * 1024
|
|
),
|
|
"objects_chunk_size_to_delete": threshold_generator(
|
|
0.2, 0.2, 0, 10 * 1024 * 1024, 32
|
|
),
|
|
"remove_shared_recursive_file_limit": threshold_generator(0.2, 0.2, 0, 31),
|
|
"s3_allow_native_copy": true_false_lambda,
|
|
"s3_check_objects_after_upload": true_false_lambda,
|
|
"s3_max_get_burst": threshold_generator(0.2, 0.2, 0, 100),
|
|
"s3_max_get_rps": threshold_generator(0.2, 0.2, 0, 100),
|
|
"s3_max_inflight_parts_for_one_file": threshold_generator(0.2, 0.2, 0, 16),
|
|
"s3_max_put_burst": threshold_generator(0.2, 0.2, 0, 100),
|
|
"s3_max_put_rps": threshold_generator(0.2, 0.2, 0, 100),
|
|
"s3_max_single_part_upload_size": threshold_generator(
|
|
0.2, 0.2, 0, 10 * 1024 * 1024
|
|
),
|
|
"s3_max_single_read_retries": threshold_generator(0.2, 0.2, 1, 16),
|
|
"s3_max_unexpected_write_error_retries": threshold_generator(0.2, 0.2, 1, 16),
|
|
"s3_max_upload_part_size": threshold_generator(
|
|
0.2, 0.2, 16 * 1024 * 1024, 5 * 1024 * 1024 * 1024, 33
|
|
),
|
|
"s3_strict_upload_part_size": threshold_generator(
|
|
0.2, 0.2, 0, 100 * 1024 * 1024
|
|
),
|
|
"s3_upload_part_size_multiply_factor": threshold_generator(0.2, 0.2, 1, 10, 4),
|
|
"s3_upload_part_size_multiply_parts_count_threshold": threshold_generator(
|
|
0.2, 0.2, 1, 1000
|
|
),
|
|
# "server_side_encryption_customer_key_base64": true_false_lambda, not working well
|
|
# "skip_access_check": true_false_lambda, may break the startup
|
|
"support_batch_delete": true_false_lambda,
|
|
"thread_pool_size": threads_lambda,
|
|
"use_insecure_imds_request": true_false_lambda,
|
|
},
|
|
"azure": {
|
|
"check_objects_after_upload": true_false_lambda,
|
|
"list_object_keys_size": threshold_generator(0.2, 0.2, 1, 10 * 1024 * 1024),
|
|
"max_blocks_in_multipart_upload": threshold_generator(0.2, 0.2, 1, 100000),
|
|
"max_inflight_parts_for_one_file": threshold_generator(0.2, 0.2, 1, 100),
|
|
"max_single_download_retries": threshold_generator(0.2, 0.2, 1, 16),
|
|
"max_single_part_copy_size": threshold_generator(
|
|
0.2, 0.2, 0, 1024 * 1024 * 1024
|
|
),
|
|
"max_single_part_upload_size": threshold_generator(
|
|
0.2, 0.2, 0, 10 * 1024 * 1024
|
|
),
|
|
"max_single_read_retries": threshold_generator(0.2, 0.2, 1, 16),
|
|
"max_tries": threshold_generator(0.2, 0.2, 1, 16),
|
|
"max_unexpected_write_error_retries": threshold_generator(0.2, 0.2, 1, 16),
|
|
"max_upload_part_size": threshold_generator(
|
|
0.2, 0.2, 16 * 1024 * 1024, 5 * 1024 * 1024 * 1024, 33
|
|
),
|
|
"metadata_keep_free_space_bytes": threshold_generator(
|
|
0.2, 0.2, 0, 10 * 1024 * 1024
|
|
),
|
|
"min_upload_part_size": threshold_generator(0.2, 0.2, 1, 10 * 1024 * 1024),
|
|
"objects_chunk_size_to_delete": threshold_generator(
|
|
0.2, 0.2, 0, 10 * 1024 * 1024
|
|
),
|
|
"remove_shared_recursive_file_limit": threshold_generator(0.2, 0.2, 0, 31),
|
|
"retry_initial_backoff_ms": threshold_generator(0.2, 0.2, 0, 5000),
|
|
"retry_max_backoff_ms": threshold_generator(0.2, 0.2, 0, 30000),
|
|
# "skip_access_check": true_false_lambda, may break the startup
|
|
"strict_upload_part_size": threshold_generator(0.2, 0.2, 0, 100 * 1024 * 1024),
|
|
"thread_pool_size": threads_lambda,
|
|
"upload_part_size_multiply_factor": threshold_generator(0.2, 0.2, 1, 10, 4),
|
|
"upload_part_size_multiply_parts_count_threshold": threshold_generator(
|
|
0.2, 0.2, 1, 1000
|
|
),
|
|
"use_native_copy": true_false_lambda,
|
|
},
|
|
"web": {},
|
|
}
|
|
|
|
s3_with_keeper_properties = {
|
|
"metadata_cache_cleanup_interval": threshold_generator(0.2, 0.2, 1, 60),
|
|
"metadata_cache_enabled": true_false_lambda,
|
|
"metadata_cache_full_directory_lists": true_false_lambda,
|
|
}
|
|
|
|
metadata_cleanup_properties = {
|
|
"deleted_objects_delay_sec": threshold_generator(0.2, 0.2, 0, 60),
|
|
"enabled": lambda: 1 if random.randint(0, 9) < 9 else 0,
|
|
"interval_sec": threshold_generator(0.2, 0.2, 1, 60),
|
|
"old_transactions_delay_sec": threshold_generator(0.2, 0.2, 0, 60),
|
|
}
|
|
|
|
|
|
cache_storage_properties = {
|
|
"allow_dynamic_cache_resize": true_false_lambda,
|
|
"background_download_max_file_segment_size": threshold_generator(
|
|
0.2, 0.2, 0, 10 * 1024 * 1024
|
|
),
|
|
"background_download_queue_size_limit": threshold_generator(0.2, 0.2, 0, 128),
|
|
"background_download_threads": threads_lambda,
|
|
"boundary_alignment": threshold_generator(0.2, 0.2, 1, 128, bits=7),
|
|
"bypass_cache_threshold": threshold_generator(0.2, 0.2, 0, 1024 * 1024 * 1024),
|
|
"cache_on_write_operations": true_false_lambda,
|
|
"check_cache_probability": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"dynamic_resize_lock_wait_ms": threshold_generator(0.2, 0.2, 0, 5000),
|
|
"enable_bypass_cache_with_threshold": true_false_lambda,
|
|
"enable_filesystem_query_cache_limit": true_false_lambda,
|
|
"expose_prometheus_eviction_metrics": true_false_lambda,
|
|
"expose_prometheus_eviction_metrics_per_user": true_false_lambda,
|
|
"invalidated_entries_cleanup_interval_ms": threshold_generator(0.2, 0.2, 1, 60000),
|
|
"invalidated_entries_cleanup_remove_batch": threshold_generator(0.2, 0.2, 1, 1000),
|
|
"invalidated_entries_cleanup_threshold": threshold_generator(0.2, 0.2, 1, 100000),
|
|
"keep_free_space_elements_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"keep_free_space_eviction_threads": no_zero_threads_lambda,
|
|
"keep_free_space_remove_batch": threshold_generator(0.2, 0.2, 0, 10 * 1024 * 1024),
|
|
"keep_free_space_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"load_metadata_asynchronously": true_false_lambda,
|
|
"load_metadata_threads": no_zero_threads_lambda,
|
|
"max_elements": threshold_generator(0.2, 0.2, 10000, 10000000),
|
|
"max_file_segment_size": threshold_generator(
|
|
0.2, 0.2, 4 * 1024 * 1024, 32 * 1024 * 1024
|
|
), # must be >= default boundary_alignment (4Mi)
|
|
"overcommit_eviction_evict_step": threshold_generator(
|
|
0.2, 0.2, 1, 10 * 1024 * 1024
|
|
),
|
|
# Capped at the file segment size internally, 0 disables reserve-ahead
|
|
"reserve_granularity": threshold_generator(0.2, 0.2, 0, 32 * 1024 * 1024),
|
|
# "max_size_ratio_to_total_space": threshold_generator(0.2, 0.2, 0.0, 1.0), cannot be specified with `max_size` at the same time
|
|
"skip_cache_on_disk_failure": true_false_lambda,
|
|
"slru_size_ratio": threshold_generator(0.2, 0.2, 0.01, 0.99),
|
|
"split_cache_ratio": threshold_generator(0.2, 0.2, 0.01, 0.99),
|
|
"use_split_cache": true_false_lambda,
|
|
"write_cache_per_user_id_directory": true_false_lambda,
|
|
}
|
|
|
|
|
|
policy_properties = {
|
|
"description": lambda: generate_xml_safe_string(random.randint(1, 1024)),
|
|
"least_used_ttl_ms": threshold_generator(0.2, 0.2, 0, 120000),
|
|
"load_balancing": lambda: random.choice(["round_robin", "least_used"]),
|
|
# Cannot set `max_data_part_size_bytes` and `max_data_part_size_ratio` at the same time
|
|
# "max_data_part_size_bytes": threshold_generator(0.2, 0.2, 0, 10 * 1024 * 1024),
|
|
"max_data_part_size_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"move_factor": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"perform_ttl_move_on_insert": true_false_lambda,
|
|
"prefer_not_to_merge": true_false_lambda,
|
|
}
|
|
|
|
|
|
all_disks_properties = {
|
|
"description": lambda: generate_xml_safe_string(random.randint(1, 1024)),
|
|
"keep_free_space_bytes": threshold_generator(0.2, 0.2, 0, 10 * 1024 * 1024),
|
|
"local_disk_check_period_ms": threshold_generator(0.2, 0.2, 0, 60000),
|
|
"min_bytes_for_seek": threshold_generator(0.2, 0.2, 0, 10 * 1024 * 1024),
|
|
"perform_ttl_move_on_insert": true_false_lambda,
|
|
"readonly": lambda: 1 if random.randint(0, 9) < 2 else 0,
|
|
# "skip_access_check": true_false_lambda, may break the startup
|
|
"thread_pool_size": no_zero_threads_lambda,
|
|
}
|
|
|
|
|
|
backup_properties = {
|
|
"allow_concurrent_backups": true_false_lambda,
|
|
"allow_concurrent_restores": true_false_lambda,
|
|
"compare_collected_metadata": true_false_lambda,
|
|
"remove_backup_files_after_failure": true_false_lambda,
|
|
"test_inject_sleep": true_false_lambda,
|
|
"test_randomize_order": true_false_lambda,
|
|
}
|
|
|
|
|
|
class PropertiesGroup:
|
|
def __init__(self):
|
|
pass
|
|
|
|
@abstractmethod
|
|
def apply_properties(
|
|
self,
|
|
top_root: ET.Element,
|
|
property_element: ET.Element,
|
|
args,
|
|
cluster: DatastoreCluster,
|
|
is_private_binary: bool,
|
|
):
|
|
pass
|
|
|
|
|
|
Parameter = typing.Callable[[], int | float] | PropertiesGroup
|
|
|
|
|
|
def sample_from_dict(d: dict[str, Parameter], sample: int) -> dict[str, Parameter]:
|
|
items = random.sample(list(d.items()), sample)
|
|
return dict(items)
|
|
|
|
|
|
def apply_properties_recursively(
|
|
next_root: ET.Element, next_properties: dict[str, Parameter], min_values: int = 1
|
|
):
|
|
is_modified = False
|
|
selected_props = sample_from_dict(
|
|
next_properties, random.randint(min_values, len(next_properties))
|
|
)
|
|
for setting, next_child in selected_props.items():
|
|
if next_root.find(setting) is None:
|
|
is_modified = True
|
|
new_element = ET.SubElement(next_root, setting)
|
|
if isinstance(next_child, dict):
|
|
apply_properties_recursively(new_element, next_child, min_values)
|
|
elif isinstance(next_child, PropertiesGroup):
|
|
raise Exception("Can't use Properties Group here")
|
|
else:
|
|
new_element.text = str(next_child())
|
|
return is_modified
|
|
|
|
|
|
def remove_element(property_element: ET.Element, elem: str):
|
|
remove_xml = ET.SubElement(property_element, elem, attrib={"remove": "remove"})
|
|
remove_xml.text = ""
|
|
|
|
|
|
def normalize_cache_properties(cache_element: ET.Element):
|
|
# `FileCacheSettings::validate` rejects split cache with overcommit policies.
|
|
use_split_cache = cache_element.find("use_split_cache")
|
|
cache_policy = cache_element.find("cache_policy")
|
|
if (
|
|
use_split_cache is not None
|
|
and use_split_cache.text == "1"
|
|
and cache_policy is not None
|
|
and cache_policy.text in ("LRU_OVERCOMMIT", "SLRU_OVERCOMMIT")
|
|
):
|
|
use_split_cache.text = "0"
|
|
|
|
|
|
def add_single_cluster(
|
|
existing_nodes: list[str],
|
|
next_cluster: ET.Element,
|
|
):
|
|
number_elements = (
|
|
1 if random.randint(1, 4) == 1 else random.randint(1, len(existing_nodes))
|
|
)
|
|
next_shard_xml = None
|
|
single_shard = random.randint(1, 100) <= 20
|
|
|
|
input_nodes = list(existing_nodes) # Do deep copy
|
|
random.shuffle(input_nodes)
|
|
# Add secret
|
|
if random.randint(1, 100) <= 30:
|
|
secret_xml = ET.SubElement(next_cluster, "secret")
|
|
secret_xml.text = generate_xml_safe_string(random.randint(1, 128))
|
|
# Add allow_distributed_ddl_queries
|
|
if random.randint(1, 100) <= 16:
|
|
allow_ddl_xml = ET.SubElement(next_cluster, "allow_distributed_ddl_queries")
|
|
allow_ddl_xml.text = "false" if random.randint(1, 4) <= 3 else "true"
|
|
|
|
for j in range(0, number_elements):
|
|
if next_shard_xml is None or (not single_shard and random.randint(1, 3) == 1):
|
|
next_shard_xml = ET.SubElement(next_cluster, "shard")
|
|
# Add internal replication
|
|
if random.randint(1, 100) <= 40:
|
|
internal_replication_xml = ET.SubElement(
|
|
next_shard_xml, "internal_replication"
|
|
)
|
|
internal_replication_xml.text = (
|
|
"false" if random.randint(1, 4) <= 3 else "true"
|
|
)
|
|
next_replica_xml = ET.SubElement(next_shard_xml, "replica")
|
|
next_host_xml = ET.SubElement(next_replica_xml, "host")
|
|
next_host_xml.text = input_nodes[j]
|
|
next_port_xml = ET.SubElement(next_replica_xml, "port")
|
|
if random.randint(1, 100) <= 25:
|
|
secure_xml = ET.SubElement(next_replica_xml, "secure")
|
|
secure_xml.text = "1"
|
|
next_port_xml.text = "9440"
|
|
else:
|
|
next_port_xml.text = "9000"
|
|
|
|
|
|
class ClusterPropertiesGroup(PropertiesGroup):
|
|
|
|
def apply_properties(
|
|
self,
|
|
top_root: ET.Element,
|
|
property_element: ET.Element,
|
|
args,
|
|
cluster: DatastoreCluster,
|
|
is_private_binary: bool,
|
|
):
|
|
# remote_server_config = ET.SubElement(root, "remote_servers")
|
|
existing_nodes = [f"node{i}" for i in range(0, len(args.replica_values))]
|
|
|
|
# Remove default clusters
|
|
if random.randint(1, 2) == 1:
|
|
remove_element(property_element, "default")
|
|
if random.randint(1, 2) == 1:
|
|
remove_element(property_element, "all_groups.default")
|
|
|
|
lower_bound, upper_bound = args.number_servers
|
|
number_clusters = random.randint(lower_bound, upper_bound)
|
|
for i in range(0, number_clusters):
|
|
add_single_cluster(
|
|
existing_nodes,
|
|
ET.SubElement(property_element, f"cluster{i}"),
|
|
)
|
|
|
|
|
|
def add_single_disk(
|
|
i: int,
|
|
args,
|
|
cluster: DatastoreCluster,
|
|
next_disk: ET.Element,
|
|
backups_element: ET.Element,
|
|
disk_type: str,
|
|
created_disks_types: list[tuple[int, str]],
|
|
is_private_binary: bool,
|
|
) -> tuple[int, str, str, str]:
|
|
prev_disk = 0
|
|
if disk_type in ("cache", "encrypted"):
|
|
iter_prev_disk = prev_disk = random.choice(range(0, i))
|
|
|
|
# Cannot create encrypted disk on web storage
|
|
# Cannot create cached disk on local storage or encrypted disk
|
|
while created_disks_types[iter_prev_disk][1] in ("cache", "encrypted"):
|
|
if (
|
|
created_disks_types[iter_prev_disk][1] == "encrypted"
|
|
and disk_type == "cache"
|
|
):
|
|
disk_type = "object_storage"
|
|
break
|
|
iter_prev_disk = created_disks_types[iter_prev_disk][0]
|
|
if created_disks_types[iter_prev_disk][1] == "web" and disk_type == "encrypted":
|
|
disk_type = "cache"
|
|
elif created_disks_types[iter_prev_disk][1] == "local" and disk_type == "cache":
|
|
disk_type = "object_storage"
|
|
|
|
# Add a single disk
|
|
disk_type_xml = ET.SubElement(next_disk, "type")
|
|
disk_type_xml.text = disk_type
|
|
|
|
allowed_disk_xml = ET.SubElement(backups_element, "allowed_disk")
|
|
allowed_disk_xml.text = f"disk{i}"
|
|
final_type = disk_type
|
|
final_super_type = disk_type
|
|
final_metadata_type = "local" # default
|
|
|
|
if disk_type == "object_storage":
|
|
object_storages = ["local"]
|
|
if args.with_minio:
|
|
object_storages.append("s3")
|
|
if is_private_binary:
|
|
# Increased probability
|
|
object_storages.extend(
|
|
["s3_with_keeper", "s3_with_keeper", "s3_with_keeper"]
|
|
)
|
|
if args.with_azurite:
|
|
object_storages.append("azure")
|
|
if args.with_nginx:
|
|
object_storages.append("web")
|
|
final_type = object_storage_type = random.choice(object_storages)
|
|
object_storage_type_xml = ET.SubElement(next_disk, "object_storage_type")
|
|
object_storage_type_xml.text = object_storage_type
|
|
|
|
# Set disk metadata type
|
|
metadata_type = (
|
|
"keeper"
|
|
if object_storage_type == "s3_with_keeper"
|
|
else "web" if object_storage_type == "web" else "local"
|
|
)
|
|
if random.randint(1, 100) <= 70:
|
|
possible_metadata_types = (
|
|
["plain", "web"]
|
|
if object_storage_type == "web"
|
|
else ["local", "plain", "plain_rewritable"]
|
|
)
|
|
if is_private_binary and object_storage_type != "web":
|
|
# Increased probability
|
|
possible_metadata_types.extend(["keeper", "keeper", "keeper"])
|
|
metadata_type = random.choice(possible_metadata_types)
|
|
metadata_xml = ET.SubElement(next_disk, "metadata_type")
|
|
metadata_xml.text = metadata_type
|
|
final_metadata_type = metadata_type
|
|
|
|
# Add endpoint info
|
|
if object_storage_type in ("s3", "s3_with_keeper"):
|
|
endpoint_xml = ET.SubElement(next_disk, "endpoint")
|
|
endpoint_xml.text = f"http://{cluster.minio_host}:{cluster.minio_port}/{cluster.minio_bucket}/data{i}"
|
|
access_key_id_xml = ET.SubElement(next_disk, "access_key_id")
|
|
access_key_id_xml.text = "minio"
|
|
secret_access_key_xml = ET.SubElement(next_disk, "secret_access_key")
|
|
secret_access_key_xml.text = cluster.minio_secret_key
|
|
elif object_storage_type == "azure":
|
|
endpoint_xml = ET.SubElement(next_disk, "endpoint")
|
|
endpoint_xml.text = f"http://{cluster.azurite_host}:{cluster.azurite_port}/{cluster.azurite_account}/data{i}"
|
|
account_name_xml = ET.SubElement(next_disk, "account_name")
|
|
account_name_xml.text = cluster.azurite_account
|
|
account_key_xml = ET.SubElement(next_disk, "account_key")
|
|
account_key_xml.text = cluster.azurite_key
|
|
elif object_storage_type == "web":
|
|
endpoint_xml = ET.SubElement(next_disk, "endpoint")
|
|
endpoint_xml.text = (
|
|
f"http://{cluster.nginx_host}:{cluster.nginx_port}/data{i}/"
|
|
)
|
|
elif object_storage_type == "local":
|
|
path_xml = ET.SubElement(next_disk, "path")
|
|
path_xml.text = f"/var/lib/datastore/disk{i}/"
|
|
allowed_path_xml = ET.SubElement(backups_element, "allowed_path")
|
|
allowed_path_xml.text = f"/var/lib/datastore/disk{i}/"
|
|
|
|
# Add a endpoint_subpath
|
|
if metadata_type == "plain_rewritable" and random.randint(1, 100) <= 70:
|
|
endpoint_subpath_xml = ET.SubElement(next_disk, "endpoint_subpath")
|
|
endpoint_subpath_xml.text = (
|
|
f"node{random.choice(range(0, len(args.replica_values)))}"
|
|
)
|
|
|
|
# Add storage settings
|
|
dict_entry = (
|
|
"s3" if object_storage_type == "s3_with_keeper" else object_storage_type
|
|
)
|
|
if object_storages_properties[dict_entry] and random.randint(1, 100) <= 70:
|
|
apply_properties_recursively(
|
|
next_disk, object_storages_properties[dict_entry]
|
|
)
|
|
if (
|
|
is_private_binary
|
|
and (object_storage_type == "s3_with_keeper" or metadata_type == "keeper")
|
|
and random.randint(1, 100) <= 70
|
|
):
|
|
metadata_xml = ET.SubElement(next_disk, "metadata_background_cleanup")
|
|
apply_properties_recursively(metadata_xml, metadata_cleanup_properties)
|
|
elif disk_type == "s3_with_keeper":
|
|
endpoint_xml = ET.SubElement(next_disk, "endpoint")
|
|
endpoint_xml.text = f"http://{cluster.minio_host}:{cluster.minio_port}/{cluster.minio_bucket}/data{i}"
|
|
access_key_id_xml = ET.SubElement(next_disk, "access_key_id")
|
|
access_key_id_xml.text = "minio"
|
|
secret_access_key_xml = ET.SubElement(next_disk, "secret_access_key")
|
|
secret_access_key_xml.text = cluster.minio_secret_key
|
|
|
|
# Add storage settings
|
|
if random.randint(1, 100) <= 70:
|
|
apply_properties_recursively(next_disk, s3_with_keeper_properties)
|
|
if random.randint(1, 100) <= 70:
|
|
metadata_xml = ET.SubElement(next_disk, "metadata_background_cleanup")
|
|
apply_properties_recursively(metadata_xml, metadata_cleanup_properties)
|
|
final_metadata_type = "keeper"
|
|
elif disk_type in ("cache", "encrypted"):
|
|
disk_xml = ET.SubElement(next_disk, "disk")
|
|
disk_xml.text = f"disk{prev_disk}"
|
|
if disk_type == "cache" or random.randint(1, 2) == 1:
|
|
path_xml = ET.SubElement(next_disk, "path")
|
|
path_xml.text = f"/var/lib/datastore/disk{i}/"
|
|
allowed_path_xml = ET.SubElement(backups_element, "allowed_path")
|
|
allowed_path_xml.text = f"/var/lib/datastore/disk{i}/"
|
|
|
|
if disk_type == "cache":
|
|
max_size_xml = ET.SubElement(next_disk, "max_size")
|
|
max_size_xml.text = file_size_value(100, 4, 5)()
|
|
|
|
# Add random settings
|
|
if random.randint(1, 100) <= 70:
|
|
apply_properties_recursively(next_disk, cache_storage_properties)
|
|
normalize_cache_properties(next_disk)
|
|
else:
|
|
enc_algorithm = random.choice(["aes_128_ctr", "aes_192_ctr", "aes_256_ctr"])
|
|
algorithm_xml = ET.SubElement(next_disk, "algorithm")
|
|
algorithm_xml.text = enc_algorithm
|
|
|
|
if enc_algorithm == "aes_128_ctr":
|
|
key_xml = ET.SubElement(next_disk, "key")
|
|
key_xml.text = "".join([random.choice("0123456789") for _ in range(16)])
|
|
else:
|
|
key_hex_xml = ET.SubElement(next_disk, "key_hex")
|
|
key_hex_xml.text = "".join(
|
|
[
|
|
random.choice("abcdef0123456789")
|
|
for _ in range(48 if enc_algorithm == "aes_192_ctr" else 64)
|
|
]
|
|
)
|
|
|
|
if disk_type != "cache" and random.randint(1, 100) <= 50:
|
|
apply_properties_recursively(next_disk, all_disks_properties)
|
|
return (prev_disk, final_type, final_super_type, final_metadata_type)
|
|
|
|
|
|
class DiskPropertiesGroup(PropertiesGroup):
|
|
|
|
def apply_properties(
|
|
self,
|
|
top_root: ET.Element,
|
|
property_element: ET.Element,
|
|
args,
|
|
cluster: DatastoreCluster,
|
|
is_private_binary: bool,
|
|
):
|
|
disk_element = ET.SubElement(property_element, "disks")
|
|
backups_element = ET.SubElement(top_root, "backups")
|
|
disks_table_engines = ET.SubElement(top_root, "allowed_disks_for_table_engines")
|
|
lower_bound, upper_bound = args.number_disks
|
|
number_disks = random.randint(lower_bound, upper_bound)
|
|
number_policies = 0
|
|
|
|
allowed_disk_xml = ET.SubElement(backups_element, "allowed_disk")
|
|
allowed_disk_xml.text = "default"
|
|
created_disks_types = []
|
|
created_cache_disks = []
|
|
created_keeper_disks = []
|
|
safe_for_database_disk = []
|
|
|
|
for i in range(0, number_disks):
|
|
possible_types = (
|
|
["object_storage"]
|
|
if i == 0
|
|
else ["object_storage", "object_storage", "cache", "encrypted"]
|
|
)
|
|
if args.with_minio and is_private_binary:
|
|
# Increase probability
|
|
possible_types.extend(
|
|
["s3_with_keeper", "s3_with_keeper", "s3_with_keeper"]
|
|
)
|
|
disk_xml_elem = ET.SubElement(disk_element, f"disk{i}")
|
|
next_created_disk_pair = add_single_disk(
|
|
i,
|
|
args,
|
|
cluster,
|
|
disk_xml_elem,
|
|
backups_element,
|
|
random.choice(possible_types),
|
|
created_disks_types,
|
|
is_private_binary,
|
|
)
|
|
created_disks_types.append(next_created_disk_pair)
|
|
if next_created_disk_pair[1] == "cache":
|
|
created_cache_disks.append(i)
|
|
elif (
|
|
is_private_binary
|
|
and args.set_shared_mergetree_disk
|
|
and next_created_disk_pair[2] == "s3_with_keeper"
|
|
and disk_xml_elem.findtext("readonly") != "1"
|
|
):
|
|
created_keeper_disks.append(i)
|
|
if (
|
|
next_created_disk_pair[3] == "local"
|
|
and next_created_disk_pair[2] not in ("cache", "encrypted")
|
|
and disk_xml_elem.findtext("readonly") != "1"
|
|
):
|
|
safe_for_database_disk.append(i)
|
|
|
|
# Allow any disk in any table engine
|
|
disks_table_engines.text = ",".join(
|
|
["default"] + [f"disk{i}" for i in range(0, number_disks)]
|
|
)
|
|
# Add policies sometimes
|
|
if random.randint(1, 100) <= args.add_policy_settings_prob:
|
|
j = 0
|
|
bottom_disks = []
|
|
for val in created_disks_types:
|
|
if val[1] not in ("cache", "encrypted"):
|
|
bottom_disks.append(j)
|
|
j += 1
|
|
number_bottom_disks = len(bottom_disks)
|
|
policies_element = ET.SubElement(property_element, "policies")
|
|
lower_bound, upper_bound = args.number_disks
|
|
number_policies = random.randint(lower_bound, upper_bound)
|
|
|
|
for i in range(0, number_policies):
|
|
next_policy_xml = ET.SubElement(policies_element, f"policy{i}")
|
|
volumes_xml = ET.SubElement(next_policy_xml, "volumes")
|
|
main_xml = None
|
|
volume_counter = 0
|
|
|
|
number_elements = (
|
|
1
|
|
if random.randint(1, 2) == 1
|
|
else random.randint(1, number_bottom_disks)
|
|
)
|
|
input_disks = list(bottom_disks) # Do copy
|
|
random.shuffle(input_disks)
|
|
for i in range(0, number_elements):
|
|
if main_xml is None or random.randint(1, 3) == 1:
|
|
if main_xml is not None and random.randint(1, 100) <= 70:
|
|
apply_properties_recursively(main_xml, policy_properties)
|
|
main_xml = ET.SubElement(volumes_xml, f"volume{volume_counter}")
|
|
volume_counter += 1
|
|
disk_xml = ET.SubElement(main_xml, "disk")
|
|
disk_xml.text = f"disk{input_disks[i]}"
|
|
if main_xml is not None and random.randint(1, 100) <= 70:
|
|
apply_properties_recursively(main_xml, policy_properties)
|
|
# Assign volume_priority as a valid permutation of 1..N (no gaps allowed)
|
|
if volume_counter > 0 and random.randint(1, 100) <= 20:
|
|
priorities = list(range(1, volume_counter + 1))
|
|
random.shuffle(priorities)
|
|
for idx, vol_xml in enumerate(volumes_xml):
|
|
prio_xml = ET.SubElement(vol_xml, "volume_priority")
|
|
prio_xml.text = str(priorities[idx])
|
|
if random.randint(1, 100) <= 70:
|
|
apply_properties_recursively(next_policy_xml, policy_properties)
|
|
|
|
allowed_path_xml1 = ET.SubElement(backups_element, "allowed_path")
|
|
allowed_path_xml1.text = "/var/lib/datastore/"
|
|
allowed_path_xml2 = ET.SubElement(backups_element, "allowed_path")
|
|
allowed_path_xml2.text = "/var/lib/datastore/user_files/"
|
|
if random.randint(1, 100) <= 70:
|
|
apply_properties_recursively(backups_element, backup_properties)
|
|
|
|
if (
|
|
top_root.find("temporary_data_in_cache") is None
|
|
and top_root.find("tmp_policy") is None
|
|
and top_root.find("tmp_path") is None
|
|
):
|
|
next_opt = random.randint(1, 100)
|
|
|
|
if len(created_cache_disks) > 0 and next_opt <= 40:
|
|
chosen_cache_disk = random.choice(created_cache_disks)
|
|
temporary_cache_xml = ET.SubElement(top_root, "temporary_data_in_cache")
|
|
temporary_cache_xml.text = f"disk{chosen_cache_disk}"
|
|
# load_metadata_asynchronously is disallowed when the cache disk is used
|
|
# as temporary storage
|
|
tmp_disk_xml = disk_element.find(f"disk{chosen_cache_disk}")
|
|
if tmp_disk_xml is not None:
|
|
lma_xml = tmp_disk_xml.find("load_metadata_asynchronously")
|
|
if lma_xml is not None:
|
|
tmp_disk_xml.remove(lma_xml)
|
|
# elif number_policies > 0 and next_opt <= 70: the disks must be local
|
|
# tmp_policy_xml = ET.SubElement(root, "tmp_policy")
|
|
# tmp_policy_xml.text = (
|
|
# f"policy{random.choice(range(0, number_policies))}"
|
|
# )
|
|
else:
|
|
tmp_path_xml = ET.SubElement(top_root, "tmp_path")
|
|
tmp_path_xml.text = "/var/lib/datastore/tmp/"
|
|
# Set disk for SMTs
|
|
if top_root.find("shared_merge_tree") is None and len(created_keeper_disks) > 0:
|
|
smt_element = ET.SubElement(top_root, "shared_merge_tree")
|
|
disk_element = ET.SubElement(smt_element, "disk")
|
|
disk_element.text = f"disk{random.choice(created_keeper_disks)}"
|
|
# Optionally set database disk
|
|
if (
|
|
top_root.find("database_disk") is None
|
|
and len(safe_for_database_disk) > 0
|
|
and random.randint(1, 100) <= 30
|
|
):
|
|
dbd_element = ET.SubElement(top_root, "database_disk")
|
|
disk_element = ET.SubElement(dbd_element, "disk")
|
|
disk_element.text = f"disk{random.choice(safe_for_database_disk)}"
|
|
# Add custom_local_disks_base_directory
|
|
if top_root.find("custom_local_disks_base_directory") is None:
|
|
clddb_element = ET.SubElement(top_root, "custom_local_disks_base_directory")
|
|
clddb_element.text = "/var/lib/datastore/disks/"
|
|
|
|
|
|
def add_single_cache(i: int, next_cache: ET.Element):
|
|
max_size_xml = ET.SubElement(next_cache, "max_size")
|
|
max_size_xml.text = file_size_value(100, 4, 5)()
|
|
path_xml = ET.SubElement(next_cache, "path")
|
|
path_xml.text = f"/var/lib/datastore/fcache{i}/"
|
|
|
|
# Add random settings
|
|
if random.randint(1, 100) <= 70:
|
|
apply_properties_recursively(next_cache, cache_storage_properties)
|
|
normalize_cache_properties(next_cache)
|
|
|
|
|
|
class CachePropertiesGroup(PropertiesGroup):
|
|
|
|
def apply_properties(
|
|
self,
|
|
top_root: ET.Element,
|
|
property_element: ET.Element,
|
|
args,
|
|
cluster: DatastoreCluster,
|
|
is_private_binary: bool,
|
|
):
|
|
# filesystem_caches_config = ET.SubElement(root, "filesystem_caches")
|
|
lower_bound, upper_bound = args.number_caches
|
|
number_caches = random.randint(lower_bound, upper_bound)
|
|
for i in range(0, number_caches):
|
|
add_single_cache(i, ET.SubElement(property_element, f"fcache{i}"))
|
|
|
|
|
|
class KeeperMapPropertiesGroup(PropertiesGroup):
|
|
|
|
def apply_properties(
|
|
self,
|
|
top_root: ET.Element,
|
|
property_element: ET.Element,
|
|
args,
|
|
cluster: DatastoreCluster,
|
|
is_private_binary: bool,
|
|
):
|
|
property_element.text = "/keeper_map_tables"
|
|
|
|
|
|
class TransactionsPropertiesGroup(PropertiesGroup):
|
|
|
|
def apply_properties(
|
|
self,
|
|
top_root: ET.Element,
|
|
property_element: ET.Element,
|
|
args,
|
|
cluster: DatastoreCluster,
|
|
is_private_binary: bool,
|
|
):
|
|
property_element.text = "1"
|
|
|
|
|
|
class DistributedDDLPropertiesGroup(PropertiesGroup):
|
|
|
|
def apply_properties(
|
|
self,
|
|
top_root: ET.Element,
|
|
property_element: ET.Element,
|
|
args,
|
|
cluster: DatastoreCluster,
|
|
is_private_binary: bool,
|
|
):
|
|
path_xml = ET.SubElement(property_element, "path")
|
|
path_xml.text = "/datastore/task_queue/ddl"
|
|
replicas_path_xml = ET.SubElement(property_element, "replicas_path")
|
|
replicas_path_xml.text = "/datastore/task_queue/replicas"
|
|
apply_properties_recursively(property_element, distributed_properties, 0)
|
|
|
|
|
|
class DistributedQueryPropertiesGroup(PropertiesGroup):
|
|
|
|
def apply_properties(
|
|
self,
|
|
top_root: ET.Element,
|
|
property_element: ET.Element,
|
|
args,
|
|
cluster: ClickHouseCluster,
|
|
is_private_binary: bool,
|
|
):
|
|
next_opt = random.randint(1, 3)
|
|
|
|
# Streaming exchange server for distributed plan execution. The port only
|
|
# takes effect when at least one listen host is configured, otherwise the
|
|
# server logs an error and skips starting the exchange server.
|
|
if next_opt in (1, 3):
|
|
port_xml = ET.SubElement(property_element, "streaming_exchange_port")
|
|
port_xml.text = "9555"
|
|
listen_host_xml = ET.SubElement(
|
|
property_element, "streaming_exchange_listen_host"
|
|
)
|
|
listen_host_xml.text = "0.0.0.0"
|
|
if random.randint(1, 2) == 1:
|
|
listen_host2_xml = ET.SubElement(
|
|
property_element, "streaming_exchange_listen_host"
|
|
)
|
|
listen_host2_xml.text = "::"
|
|
# Object storage for distributed-plan temporary files. `endpoint_subpath`
|
|
# is read unconditionally when the section is present, so always set it.
|
|
if next_opt in (2, 3):
|
|
storage_xml = ET.SubElement(property_element, "temporary_files_storage")
|
|
storage_types = ["local"]
|
|
if args.with_minio:
|
|
storage_types.extend(["s3", "s3"])
|
|
storage_type = random.choice(storage_types)
|
|
type_xml = ET.SubElement(storage_xml, "type")
|
|
type_xml.text = storage_type
|
|
if storage_type == "s3":
|
|
endpoint_xml = ET.SubElement(storage_xml, "endpoint")
|
|
endpoint_xml.text = f"http://{cluster.minio_host}:{cluster.minio_port}/{cluster.minio_bucket}/dqtmp"
|
|
access_key_id_xml = ET.SubElement(storage_xml, "access_key_id")
|
|
access_key_id_xml.text = "minio"
|
|
secret_access_key_xml = ET.SubElement(storage_xml, "secret_access_key")
|
|
secret_access_key_xml.text = cluster.minio_secret_key
|
|
else:
|
|
path_xml = ET.SubElement(storage_xml, "path")
|
|
path_xml.text = "/var/lib/clickhouse/dqtmp/"
|
|
endpoint_subpath_xml = ET.SubElement(storage_xml, "endpoint_subpath")
|
|
endpoint_subpath_xml.text = "distributed_query_temp_files/"
|
|
|
|
|
|
class SharedCatalogPropertiesGroup(PropertiesGroup):
|
|
|
|
def apply_properties(
|
|
self,
|
|
top_root: ET.Element,
|
|
property_element: ET.Element,
|
|
args,
|
|
cluster: DatastoreCluster,
|
|
is_private_binary: bool,
|
|
):
|
|
number_clusters = 0
|
|
shared_settings = {
|
|
"delay_before_drop_intention_seconds": threshold_generator(
|
|
0.2, 0.2, 0, 60, 32
|
|
),
|
|
"delay_before_drop_table_seconds": threshold_generator(0.2, 0.2, 0, 60, 31),
|
|
"drop_ignore_inactive_replica_after_seconds": threshold_generator(
|
|
0.2, 0.2, 0, 60, 32
|
|
),
|
|
"drop_local_thread_pool_size": threads_lambda,
|
|
"drop_lock_duration_seconds": threshold_generator(0.2, 0.2, 0, 60, 31),
|
|
"drop_zookeeper_thread_pool_size": threads_lambda,
|
|
# "migration_from_database_replicated": true_false_lambda, not suitable for testing
|
|
"state_application_thread_pool_size": threads_lambda,
|
|
}
|
|
remote_servers = top_root.find("remote_servers")
|
|
if remote_servers is not None:
|
|
number_clusters = len(
|
|
[c for c in remote_servers if "remove" not in c.attrib]
|
|
)
|
|
if number_clusters > 0 and random.randint(1, 100) <= 75:
|
|
cluster_name_choices = [f"cluster{i}" for i in range(0, number_clusters)]
|
|
if remote_servers is None or remote_servers.find("default") is None:
|
|
# The default cluster was not removed
|
|
cluster_name_choices.append("default")
|
|
shared_settings["cluster_name"] = lambda: random.choice(
|
|
cluster_name_choices
|
|
)
|
|
apply_properties_recursively(property_element, shared_settings, 0)
|
|
|
|
|
|
class DatabaseReplicatedGroup(PropertiesGroup):
|
|
|
|
def apply_properties(
|
|
self,
|
|
top_root: ET.Element,
|
|
property_element: ET.Element,
|
|
args,
|
|
cluster: DatastoreCluster,
|
|
is_private_binary: bool,
|
|
):
|
|
replicated_settings = {
|
|
"allow_skipping_old_temporary_tables_ddls_of_refreshable_materialized_views": true_false_lambda,
|
|
"check_consistency": true_false_lambda,
|
|
"internal_replication": true_false_lambda,
|
|
"logs_to_keep": threshold_generator(0.2, 0.2, 0, 3000),
|
|
"max_broken_tables_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"max_replication_lag_to_enqueue": threshold_generator(0.2, 0.2, 0, 200),
|
|
}
|
|
apply_properties_recursively(property_element, replicated_settings, 0)
|
|
|
|
|
|
class LogTablePropertiesGroup(PropertiesGroup):
|
|
|
|
def __init__(
|
|
self, _log_table: str, _def_max_size_rows: int, _def_reserved_size_rows: int
|
|
):
|
|
super().__init__()
|
|
self.log_table: str = _log_table
|
|
self.def_max_size_rows: int = _def_max_size_rows
|
|
self.def_reserved_size_rows: int = _def_reserved_size_rows
|
|
|
|
def apply_properties(
|
|
self,
|
|
top_root: ET.Element,
|
|
property_element: ET.Element,
|
|
args,
|
|
cluster: DatastoreCluster,
|
|
is_private_binary: bool,
|
|
):
|
|
database_xml = ET.SubElement(property_element, "database")
|
|
database_xml.text = "system"
|
|
table_xml = ET.SubElement(property_element, "table")
|
|
table_xml.text = self.log_table
|
|
|
|
log_table_properties = {
|
|
"buffer_size_rows_flush_threshold": threshold_generator(0.2, 0.2, 0, 10000),
|
|
"flush_on_crash": true_false_lambda,
|
|
# Setting these may crash the server
|
|
# "max_size_rows": threshold_generator(0.2, 0.2, 1, 10000),
|
|
# "reserved_size_rows": threshold_generator(0.2, 0.2, 1, 10000),
|
|
}
|
|
# Can't use this without the engine parameter?
|
|
# number_policies = 0
|
|
# storage_configuration_xml = top_root.find("storage_configuration")
|
|
# if storage_configuration_xml is not None:
|
|
# policies_xml = storage_configuration_xml.find("policies")
|
|
# if policies_xml is not None:
|
|
# number_policies = len([c for c in policies_xml])
|
|
# if number_policies > 0 and random.randint(1, 100) <= 75:
|
|
# policy_choices = [f"policy{i}" for i in range(0, number_policies)]
|
|
# log_table_properties["storage_policy"] = lambda: random.choice(
|
|
# policy_choices
|
|
# )
|
|
apply_properties_recursively(property_element, log_table_properties, 0)
|
|
# max_size_rows (default 1048576) cannot be smaller than reserved_size_rows (default 8192)
|
|
max_size_rows_xml = property_element.find("max_size_rows")
|
|
reserved_size_rows_xml = property_element.find("reserved_size_rows")
|
|
if max_size_rows_xml is not None or reserved_size_rows_xml is not None:
|
|
max_size_rows_value = (
|
|
self.def_max_size_rows
|
|
if (max_size_rows_xml is None or max_size_rows_xml.text is None)
|
|
else int(max_size_rows_xml.text)
|
|
)
|
|
reserved_size_rows_value = (
|
|
self.def_reserved_size_rows
|
|
if (
|
|
reserved_size_rows_xml is None
|
|
or reserved_size_rows_xml.text is None
|
|
)
|
|
else int(reserved_size_rows_xml.text)
|
|
)
|
|
if max_size_rows_value < reserved_size_rows_value:
|
|
max_size_rows_xml = (
|
|
ET.SubElement(property_element, "max_size_rows")
|
|
if max_size_rows_xml is None
|
|
else max_size_rows_xml
|
|
)
|
|
max_size_rows_xml.text = str(
|
|
max(max_size_rows_value, reserved_size_rows_value)
|
|
)
|
|
|
|
|
|
def add_ssl_settings(next_ssl: ET.Element):
|
|
certificate_xml = ET.SubElement(next_ssl, "certificateFile")
|
|
private_key_xml = ET.SubElement(next_ssl, "privateKeyFile")
|
|
if random.randint(1, 2) == 1:
|
|
certificate_xml.text = "/etc/datastore-server/config.d/server.crt"
|
|
private_key_xml.text = "/etc/datastore-server/config.d/server.key"
|
|
else:
|
|
certificate_xml.text = "/etc/datastore-server/config.d/server-cert.pem"
|
|
private_key_xml.text = "/etc/datastore-server/config.d/server-key.pem"
|
|
ca_config_xml = ET.SubElement(next_ssl, "caConfig")
|
|
ca_config_xml.text = "/etc/datastore-server/config.d/ca-cert.pem"
|
|
if random.randint(1, 2) == 1:
|
|
dh_params_xml = ET.SubElement(next_ssl, "dhParamsFile")
|
|
dh_params_xml.text = "/etc/datastore-server/config.d/dhparam.pem"
|
|
|
|
if random.randint(1, 2) == 1:
|
|
verification_xml = ET.SubElement(next_ssl, "verificationMode")
|
|
verification_xml.text = random.choice(["none", "relaxed", "strict"])
|
|
if random.randint(1, 2) == 1:
|
|
load_ca_xml = ET.SubElement(next_ssl, "loadDefaultCAFile")
|
|
load_ca_xml.text = random.choice(["true", "false"])
|
|
if random.randint(1, 2) == 1:
|
|
cache_sessions_xml = ET.SubElement(next_ssl, "cacheSessions")
|
|
cache_sessions_xml.text = random.choice(["true", "false"])
|
|
if random.randint(1, 2) == 1:
|
|
prefer_server_ciphers_xml = ET.SubElement(next_ssl, "preferServerCiphers")
|
|
prefer_server_ciphers_xml.text = random.choice(["true", "false"])
|
|
if random.randint(1, 2) == 1:
|
|
SSL_PROTOCOLS = ["sslv2", "sslv3", "tlsv1", "tlsv1_1", "tlsv1_2"]
|
|
disabled_list_str = ""
|
|
shuffled_protocols = list(SSL_PROTOCOLS) # Do copy
|
|
random.shuffle(shuffled_protocols)
|
|
for i in range(0, random.randint(1, len(shuffled_protocols))):
|
|
disabled_list_str += "" if i == 0 else ","
|
|
disabled_list_str += shuffled_protocols[i]
|
|
disabled_protocols_xml = ET.SubElement(next_ssl, "disableProtocols")
|
|
disabled_protocols_xml.text = disabled_list_str
|
|
|
|
|
|
def modify_server_settings(
|
|
args,
|
|
cluster: DatastoreCluster,
|
|
is_private_binary: bool,
|
|
input_config_path: str,
|
|
) -> tuple[bool, str, int]:
|
|
modified = False
|
|
number_clusters = 0
|
|
|
|
# Parse the existing XML file
|
|
tree = ET.parse(input_config_path)
|
|
root = tree.getroot()
|
|
if root.tag != "datastore":
|
|
raise Exception("<datastore> element not found")
|
|
|
|
if root.find("tcp_port_secure") is None:
|
|
modified = True
|
|
secure_port_xml = ET.SubElement(root, "tcp_port_secure")
|
|
secure_port_xml.text = "9440"
|
|
if root.find("https_port") is None:
|
|
modified = True
|
|
https_port_xml = ET.SubElement(root, "https_port")
|
|
https_port_xml.text = "8443"
|
|
if args.with_arrowflight and root.find("arrowflight_port") is None:
|
|
modified = True
|
|
arrowflight_port_xml = ET.SubElement(root, "arrowflight_port")
|
|
arrowflight_port_xml.text = "8888"
|
|
if root.find("openSSL") is None:
|
|
modified = True
|
|
openssl_xml = ET.SubElement(root, "openSSL")
|
|
server_xml = ET.SubElement(openssl_xml, "server")
|
|
add_ssl_settings(server_xml)
|
|
|
|
client_xml = ET.SubElement(openssl_xml, "client")
|
|
add_ssl_settings(client_xml)
|
|
if random.randint(1, 2) == 1:
|
|
invalid_handler_xml = ET.SubElement(client_xml, "invalidCertificateHandler")
|
|
name_xml = ET.SubElement(invalid_handler_xml, "name")
|
|
name_xml.text = random.choice(
|
|
["AcceptCertificateHandler", "RejectCertificateHandler"]
|
|
)
|
|
|
|
named_collections_xml = root.find("named_collections")
|
|
if named_collections_xml is None:
|
|
named_collections_xml = ET.SubElement(root, "named_collections")
|
|
modified = True
|
|
if args.with_minio and named_collections_xml.find("s3") is None:
|
|
modified = True
|
|
s3_xml = ET.SubElement(named_collections_xml, "s3")
|
|
ET.SubElement(s3_xml, "url").text = (
|
|
f"http://{cluster.minio_host}:{cluster.minio_port}/{cluster.minio_bucket}/"
|
|
)
|
|
ET.SubElement(s3_xml, "access_key_id").text = "minio"
|
|
ET.SubElement(s3_xml, "secret_access_key").text = cluster.minio_secret_key
|
|
if args.with_azurite and named_collections_xml.find("azure") is None:
|
|
modified = True
|
|
azure_xml = ET.SubElement(named_collections_xml, "azure")
|
|
ET.SubElement(azure_xml, "account_name").text = cluster.azurite_account
|
|
ET.SubElement(azure_xml, "account_key").text = cluster.azurite_key
|
|
ET.SubElement(azure_xml, "container").text = cluster.azure_container_name
|
|
ET.SubElement(azure_xml, "storage_account_url").text = (
|
|
f"http://{cluster.azurite_host}:{cluster.azurite_port}/{cluster.azurite_account}"
|
|
)
|
|
if args.with_mysql and named_collections_xml.find("mysql_remote") is None:
|
|
modified = True
|
|
mysql_xml = ET.SubElement(named_collections_xml, "mysql_remote")
|
|
ET.SubElement(mysql_xml, "host").text = cluster.mysql8_host
|
|
ET.SubElement(mysql_xml, "port").text = str(cluster.mysql8_port)
|
|
ET.SubElement(mysql_xml, "user").text = "root"
|
|
ET.SubElement(mysql_xml, "password").text = mysql_pass
|
|
ET.SubElement(mysql_xml, "database").text = "test"
|
|
if args.with_postgresql and named_collections_xml.find("postgres_remote") is None:
|
|
modified = True
|
|
pg_xml = ET.SubElement(named_collections_xml, "postgres_remote")
|
|
ET.SubElement(pg_xml, "host").text = cluster.postgres_host
|
|
ET.SubElement(pg_xml, "port").text = str(cluster.postgres_port)
|
|
ET.SubElement(pg_xml, "user").text = "postgres"
|
|
ET.SubElement(pg_xml, "password").text = pg_pass
|
|
ET.SubElement(pg_xml, "database").text = "test"
|
|
if args.with_mongodb and named_collections_xml.find("mongo_remote") is None:
|
|
modified = True
|
|
mongo_xml = ET.SubElement(named_collections_xml, "mongo_remote")
|
|
ET.SubElement(mongo_xml, "host").text = cluster.mongo_host
|
|
ET.SubElement(mongo_xml, "port").text = str(cluster.mongo_port)
|
|
ET.SubElement(mongo_xml, "user").text = "root"
|
|
ET.SubElement(mongo_xml, "password").text = mongo_pass
|
|
ET.SubElement(mongo_xml, "database").text = "test"
|
|
if named_collections_xml.find("local") is None:
|
|
modified = True
|
|
ET.SubElement(named_collections_xml, "local")
|
|
|
|
if "timezone" not in possible_properties:
|
|
possible_timezones = get_system_timezones()
|
|
if len(possible_timezones) > 0:
|
|
possible_properties["timezone"] = lambda: random.choice(possible_timezones)
|
|
if "cache_policy" not in cache_storage_properties:
|
|
possible_policies = ["LRU", "SLRU"]
|
|
if is_private_binary:
|
|
possible_policies.extend(["LRU_OVERCOMMIT", "SLRU_OVERCOMMIT"])
|
|
cache_storage_properties["cache_policy"] = lambda: random.choice(
|
|
possible_policies
|
|
)
|
|
|
|
selected_properties = {}
|
|
# Select random properties to the XML
|
|
if random.randint(1, 100) <= args.server_settings_prob:
|
|
selected_properties = sample_from_dict(
|
|
possible_properties, random.randint(0, len(possible_properties))
|
|
)
|
|
|
|
# Add remote server configurations
|
|
if (
|
|
root.find("remote_servers") is None
|
|
and random.randint(1, 100) <= args.add_remote_server_settings_prob
|
|
):
|
|
selected_properties["remote_servers"] = ClusterPropertiesGroup()
|
|
|
|
# Add disk configurations
|
|
if (
|
|
root.find("storage_configuration") is None
|
|
and root.find("backups") is None
|
|
and root.find("allowed_disks_for_table_engines") is None
|
|
and random.randint(1, 100) <= args.add_disk_settings_prob
|
|
):
|
|
selected_properties["storage_configuration"] = DiskPropertiesGroup()
|
|
|
|
# Add filesystem caches
|
|
if (
|
|
root.find("filesystem_caches") is None
|
|
and random.randint(1, 100) <= args.add_filesystem_caches_prob
|
|
):
|
|
selected_properties["filesystem_caches"] = CachePropertiesGroup()
|
|
|
|
# Add keeper_map_path_prefix
|
|
if args.add_keeper_map_prefix and root.find("keeper_map_path_prefix") is None:
|
|
selected_properties["keeper_map_path_prefix"] = KeeperMapPropertiesGroup()
|
|
|
|
# Add experimental transactions
|
|
if args.add_transactions and root.find("allow_experimental_transactions") is None:
|
|
selected_properties["allow_experimental_transactions"] = (
|
|
TransactionsPropertiesGroup()
|
|
)
|
|
|
|
# Add distributed_ddl
|
|
if args.add_distributed_ddl and root.find("distributed_ddl") is None:
|
|
selected_properties["distributed_ddl"] = DistributedDDLPropertiesGroup()
|
|
|
|
# Add distributed_query settings (streaming exchange, temporary files storage)
|
|
if args.add_distributed_query and root.find("distributed_query") is None:
|
|
selected_properties["distributed_query"] = DistributedQueryPropertiesGroup()
|
|
|
|
# Add log tables
|
|
if args.add_log_tables:
|
|
all_log_entries = [
|
|
("aggregated_zookeeper_log", 1048576, 8192),
|
|
("asynchronous_insert_log", 1048576, 8192),
|
|
("asynchronous_metric_log", 1048576, 8192),
|
|
("azure_queue_log", 1048576, 8192),
|
|
("background_schedule_pool_log", 1048576, 8192),
|
|
("backup_log", 1048576, 8192),
|
|
("blob_storage_log", 1048576, 8192),
|
|
("crash_log", 1024, 1024),
|
|
("dead_letter_queue", 1048576, 8192),
|
|
("delta_lake_metadata_log", 1048576, 8192),
|
|
("error_log", 1048576, 8192),
|
|
("filesystem_cache_log", 1048576, 8192),
|
|
("filesystem_read_prefetches_log", 1048576, 8192),
|
|
("iceberg_metadata_log", 1048576, 8192),
|
|
("metric_log", 1048576, 8192),
|
|
("opentelemetry_span_log", 1048576, 8192),
|
|
("part_log", 1048576, 8192),
|
|
("predicate_statistics_log", 1048576, 8192),
|
|
("processors_profile_log", 1048576, 8192),
|
|
("query_log", 1048576, 8192),
|
|
("query_metric_log", 1048576, 8192),
|
|
("query_thread_log", 1048576, 8192),
|
|
("query_views_log", 1048576, 8192),
|
|
("s3queue_log", 1048576, 8192),
|
|
("session_log", 1048576, 8192),
|
|
("text_log", 1048576, 8192),
|
|
("trace_log", 1048576, 8192),
|
|
("transactions_info_log", 1048576, 8192),
|
|
("transposed_metric_log", 1048576, 8192),
|
|
("zookeeper_connection_log", 1048576, 8192),
|
|
("zookeeper_log", 1048576, 8192),
|
|
]
|
|
if random.randint(1, 100) <= 70:
|
|
all_log_entries = random.sample(
|
|
all_log_entries, random.randint(1, len(all_log_entries))
|
|
)
|
|
random.shuffle(all_log_entries)
|
|
for entry in all_log_entries:
|
|
if root.find(entry[0]) is None:
|
|
selected_properties[entry[0]] = LogTablePropertiesGroup(
|
|
entry[0], entry[1], entry[2]
|
|
)
|
|
|
|
# Add shared_database_catalog settings, required for shared catalog to work
|
|
if (
|
|
args.add_shared_catalog
|
|
and is_private_binary
|
|
and root.find("shared_database_catalog") is None
|
|
):
|
|
selected_properties["shared_database_catalog"] = SharedCatalogPropertiesGroup()
|
|
|
|
# Add Replicated databases settings
|
|
if args.add_database_replicated and root.find("database_replicated") is None:
|
|
selected_properties["database_replicated"] = DatabaseReplicatedGroup()
|
|
|
|
# Shuffle selected properties and apply
|
|
selected_properties = dict(
|
|
random.sample(list(selected_properties.items()), len(selected_properties))
|
|
)
|
|
for setting, next_child in selected_properties.items():
|
|
if root.find(setting) is None:
|
|
modified = True
|
|
new_element = ET.SubElement(root, setting)
|
|
if isinstance(next_child, dict):
|
|
apply_properties_recursively(new_element, next_child, 0)
|
|
elif isinstance(next_child, PropertiesGroup):
|
|
next_child.apply_properties(
|
|
root, new_element, args, cluster, is_private_binary
|
|
)
|
|
else:
|
|
new_element.text = str(next_child())
|
|
|
|
if modified:
|
|
# Make sure `path` in distributed_ddl is set
|
|
distributed_ddl_xml = root.find("distributed_ddl")
|
|
if distributed_ddl_xml is not None and distributed_ddl_xml.find("path") is None:
|
|
path_xml = ET.SubElement(distributed_ddl_xml, "path")
|
|
path_xml.text = "/datastore/task_queue/ddl"
|
|
# Make sure `zookeeper_path` in transaction_log is set
|
|
transaction_log_xml = root.find("transaction_log")
|
|
if (
|
|
transaction_log_xml is not None
|
|
and transaction_log_xml.find("zookeeper_path") is None
|
|
):
|
|
zookeeper_path_xml = ET.SubElement(transaction_log_xml, "zookeeper_path")
|
|
zookeeper_path_xml.text = "/datastore/txn"
|
|
|
|
# Get number of clusters if generated, to be used in `users.xml` if needed
|
|
remote_servers = root.find("remote_servers")
|
|
if remote_servers is not None:
|
|
number_clusters = len([c for c in remote_servers if "remove" not in c.attrib])
|
|
|
|
if modified:
|
|
ET.indent(tree, space=" ", level=0) # indent tree
|
|
temp_path = None
|
|
# Create a temporary file
|
|
with tempfile.NamedTemporaryFile(
|
|
dir=args.tmp_files_dir, suffix=".xml", delete=False
|
|
) as temp_file:
|
|
temp_path = temp_file.name
|
|
# Write the modified XML to the temporary file
|
|
tree.write(temp_path, encoding="utf-8", xml_declaration=True)
|
|
return True, temp_path, number_clusters
|
|
return False, input_config_path, number_clusters
|
|
|
|
|
|
def modify_user_settings(
|
|
args, input_config_path: str, number_clusters: int
|
|
) -> tuple[bool, str]:
|
|
modified = False
|
|
|
|
# Parse the existing XML file
|
|
tree = ET.parse(input_config_path)
|
|
root = tree.getroot()
|
|
if root.tag != "datastore":
|
|
raise Exception("<datastore> element not found")
|
|
|
|
if number_clusters > 0:
|
|
modified = True
|
|
profiles_xml = root.find("profiles")
|
|
if profiles_xml is None:
|
|
profiles_xml = ET.SubElement(root, "profiles")
|
|
default_xml = profiles_xml.find("default")
|
|
if default_xml is None:
|
|
default_xml = ET.SubElement(profiles_xml, "default")
|
|
cluster_for_parallel_replicas_xml = default_xml.find(
|
|
"cluster_for_parallel_replicas"
|
|
)
|
|
if cluster_for_parallel_replicas_xml is None:
|
|
cluster_for_parallel_replicas_xml = ET.SubElement(
|
|
default_xml, "cluster_for_parallel_replicas"
|
|
)
|
|
cluster_for_parallel_replicas_xml.text = (
|
|
f"cluster{random.choice(range(0, number_clusters))}"
|
|
)
|
|
|
|
if modified:
|
|
ET.indent(tree, space=" ", level=0) # indent tree
|
|
temp_path = None
|
|
# Create a temporary file
|
|
with tempfile.NamedTemporaryFile(
|
|
dir=args.tmp_files_dir, suffix=".xml", delete=False
|
|
) as temp_file:
|
|
temp_path = temp_file.name
|
|
# Write the modified XML to the temporary file
|
|
tree.write(temp_path, encoding="utf-8", xml_declaration=True)
|
|
return True, temp_path
|
|
return False, input_config_path
|
|
|
|
|
|
KEEPER_PROPERTIES_TEMPLATE = """
|
|
<datastore>
|
|
<listen_try>true</listen_try>
|
|
<listen_host>::</listen_host>
|
|
<listen_host>0.0.0.0</listen_host>
|
|
|
|
<logger>
|
|
<level>trace</level>
|
|
<log>/var/log/datastore-keeper/datastore-keeper.log</log>
|
|
<errorlog>/var/log/datastore-keeper/datastore-keeper.err.log</errorlog>
|
|
</logger>
|
|
|
|
<placement>
|
|
<use_imds>0</use_imds>
|
|
<availability_zone>az-zoo{id}</availability_zone>
|
|
</placement>
|
|
|
|
<keeper_server>
|
|
<tcp_port>2181</tcp_port>
|
|
<server_id>{id}</server_id>
|
|
|
|
<raft_configuration>
|
|
<server>
|
|
<id>1</id>
|
|
<hostname>zoo1</hostname>
|
|
<port>9444</port>
|
|
</server>
|
|
<server>
|
|
<id>2</id>
|
|
<hostname>zoo2</hostname>
|
|
<port>9444</port>
|
|
</server>
|
|
<server>
|
|
<id>3</id>
|
|
<hostname>zoo3</hostname>
|
|
<port>9444</port>
|
|
</server>
|
|
</raft_configuration>
|
|
</keeper_server>
|
|
</datastore>
|
|
"""
|
|
|
|
keeper_settings = {
|
|
"cleanup_old_and_ignore_new_acl": true_false_lambda,
|
|
"coordination_settings": {
|
|
"async_replication": true_false_lambda,
|
|
"auto_forwarding": true_false_lambda,
|
|
"check_node_acl_on_remove": true_false_lambda,
|
|
"commit_logs_cache_entry_count_threshold": threshold_generator(
|
|
0.2, 0.2, 0, 500000
|
|
),
|
|
"commit_logs_cache_size_threshold": threshold_generator(
|
|
0.2, 0.2, 0, 1000 * 1024 * 1024
|
|
),
|
|
"commit_profiler_real_time_period_ns": threshold_generator(
|
|
0.2, 0.2, 0, 1000000000
|
|
),
|
|
"compress_logs": true_false_lambda,
|
|
"compress_snapshots_with_zstd_format": true_false_lambda,
|
|
"configuration_change_tries_count": threshold_generator(0.2, 0.2, 0, 40),
|
|
"dead_session_check_period_ms": threshold_generator(0.2, 0.2, 100, 5000),
|
|
"disk_move_retries_during_init": threshold_generator(0.2, 0.2, 0, 200),
|
|
"disk_move_retries_wait_ms": threshold_generator(0.2, 0.2, 0, 5000),
|
|
"dispatch_busy_wait_sleep_us": threshold_generator(0.2, 0.2, 0, 10000),
|
|
"force_sync": true_false_lambda,
|
|
"fresh_log_gap": threshold_generator(0.2, 0.2, 0, 200),
|
|
"heart_beat_interval_ms": threshold_generator(0.2, 0.2, 100, 1500),
|
|
"latest_logs_cache_entry_count_threshold": threshold_generator(
|
|
0.2, 0.2, 0, 500000
|
|
),
|
|
"latest_logs_cache_size_threshold": threshold_generator(
|
|
0.2, 0.2, 0, 2 * 1024 * 1024 * 1024
|
|
),
|
|
"leadership_expiry_ms": threshold_generator(0.2, 0.2, 0, 1999),
|
|
"log_file_overallocate_size": threshold_generator(
|
|
0.2, 0.2, 0, 100 * 1024 * 1024
|
|
),
|
|
"log_slow_connection_operation_threshold_ms": threshold_generator(
|
|
0.2, 0.2, 0, 10000
|
|
),
|
|
"log_slow_cpu_threshold_ms": threshold_generator(0.2, 0.2, 0, 5000),
|
|
"log_slow_total_threshold_ms": threshold_generator(0.2, 0.2, 0, 30000),
|
|
"max_flush_batch_size": threshold_generator(0.2, 0.2, 0, 2000),
|
|
"max_in_flight_request_batches": threshold_generator(0.2, 0.2, 1, 100),
|
|
"max_log_file_size": threshold_generator(0.2, 0.2, 0, 100 * 1024 * 1024),
|
|
"max_read_batch_bytes_size": threshold_generator(
|
|
0.2, 0.2, 0, 100 * 1024 * 1024
|
|
),
|
|
"max_read_batch_size": threshold_generator(0.2, 0.2, 0, 1000000),
|
|
"max_request_queue_bytes_size": threshold_generator(
|
|
0.2, 0.2, 0, 100 * 1024 * 1024
|
|
),
|
|
"max_request_queue_size": threshold_generator(0.2, 0.2, 0, 100000),
|
|
"max_request_size": threshold_generator(0.2, 0.2, 0, 10 * 1024 * 1024),
|
|
"max_requests_append_bytes_size": threshold_generator(
|
|
0.2, 0.2, 0, 10 * 1024 * 1024
|
|
),
|
|
"max_requests_append_size": threshold_generator(0.2, 0.2, 1, 200),
|
|
"max_requests_batch_bytes_size": threshold_generator(
|
|
0.2, 0.2, 0, 10 * 1024 * 1024
|
|
),
|
|
"max_requests_batch_size": threshold_generator(0.2, 0.2, 1, 100),
|
|
"max_requests_quick_batch_size": threshold_generator(0.2, 0.2, 0, 200),
|
|
"max_response_queue_bytes_size": threshold_generator(
|
|
0.2, 0.2, 0, 100 * 1024 * 1024
|
|
),
|
|
"min_request_size_for_cache": threshold_generator(0.2, 0.2, 0, 100 * 1024),
|
|
"min_session_timeout_ms": threshold_generator(0.2, 0.2, 1000, 10000),
|
|
"nuraft_append_entries_backward_probe_throttle_threshold": threshold_generator(
|
|
0.2, 0.2, 0, 128
|
|
),
|
|
"nuraft_max_bytes_in_flight_in_stream": threshold_generator(
|
|
0.2, 0.2, 0, 256 * 1024 * 1024
|
|
),
|
|
"nuraft_max_log_gap_in_stream": threshold_generator(0.2, 0.2, 0, 1024),
|
|
"nuraft_max_uncommitted_log_entries": threshold_generator(0.2, 0.2, 0, 1000000),
|
|
"nuraft_streaming_mode": true_false_lambda,
|
|
"nuraft_use_bg_thread_for_snapshot_io": true_false_lambda,
|
|
"optimize_read_order": true_false_lambda,
|
|
"parallel_read_chunk_size": threshold_generator(0.2, 0.2, 1, 1024),
|
|
"parallel_read_min_batch": threshold_generator(0.2, 0.2, 0, 4096),
|
|
"parallel_read_threads": threshold_generator(0.2, 0.2, 0, 64),
|
|
"quorum_reads": true_false_lambda,
|
|
"raft_limits_reconnect_limit": threshold_generator(0.2, 0.2, 0, 100),
|
|
"raft_limits_response_limit": threshold_generator(0.2, 0.2, 0, 40),
|
|
"reserved_log_items": threshold_generator(0.2, 0.2, 0, 100000),
|
|
"rotate_log_storage_interval": threshold_generator(0.2, 0.2, 1, 100000),
|
|
"session_shutdown_timeout": threshold_generator(0.2, 0.2, 5000, 30000),
|
|
"shutdown_timeout": threshold_generator(0.2, 0.2, 3000, 30000),
|
|
"sleep_before_leader_change_ms": threshold_generator(0.2, 0.2, 1000, 30000),
|
|
"snapshot_distance": threshold_generator(0.2, 0.2, 0, 100000),
|
|
"snapshot_transfer_chunk_size": threshold_generator(
|
|
0.2, 0.2, 0, 16 * 1024 * 1024
|
|
),
|
|
"snapshots_to_keep": threshold_generator(0.2, 0.2, 0, 5),
|
|
"stale_log_gap": threshold_generator(0.2, 0.2, 0, 10000),
|
|
"startup_timeout": threshold_generator(0.2, 0.2, 1000, 600000),
|
|
"stream_in_flight_drain_timeout_ms": threshold_generator(0.2, 0.2, 0, 10000),
|
|
"stream_suspect_retry_delay_ms": threshold_generator(0.2, 0.2, 0, 5000),
|
|
"ttl_gc_batch_size": threshold_generator(0.2, 0.2, 1, 4096),
|
|
"ttl_gc_period_ms": threshold_generator(0.2, 0.2, 1, 10000),
|
|
"use_new_dispatcher": true_false_lambda,
|
|
"use_xid_64": true_false_lambda,
|
|
"write_snapshot_version": lambda: random.choice([6, 7, 8]),
|
|
},
|
|
"create_snapshot_on_exit": true_false_lambda,
|
|
"digest_enabled": true_false_lambda,
|
|
"digest_enabled_on_commit": true_false_lambda,
|
|
"enable_reconfiguration": true_false_lambda,
|
|
"feature_flags": {
|
|
"check_not_exists": true_false_lambda,
|
|
"check_stat": true_false_lambda,
|
|
"create_if_not_exists": true_false_lambda,
|
|
"create_ttl": true_false_lambda,
|
|
"create_with_stats": true_false_lambda,
|
|
"filtered_list": true_false_lambda,
|
|
"get_children_recursive": true_false_lambda,
|
|
"list_with_stat_and_data": true_false_lambda,
|
|
"multi_read": true_false_lambda,
|
|
"multi_watches": true_false_lambda,
|
|
"persistent_watches": true_false_lambda,
|
|
"remove_recursive": true_false_lambda,
|
|
"try_remove": true_false_lambda,
|
|
},
|
|
"force_recovery": true_false_lambda,
|
|
"hostname_checks_enabled": true_false_lambda,
|
|
"max_memory_usage_soft_limit": threshold_generator(0.2, 0.2, 0, 1000),
|
|
"max_memory_usage_soft_limit_ratio": threshold_generator(0.2, 0.2, 0.0, 1.0),
|
|
"upload_snapshot_on_exit": true_false_lambda,
|
|
**rocksdb_properties,
|
|
}
|
|
|
|
|
|
def modify_keeper_settings(args, is_private_binary: bool) -> list[str]:
|
|
result_configs = []
|
|
selected_settings = {}
|
|
|
|
if random.randint(1, 100) <= args.keeper_settings_prob:
|
|
selected_settings = sample_from_dict(
|
|
keeper_settings, random.randint(0, len(keeper_settings))
|
|
)
|
|
|
|
for i in range(1, 4):
|
|
tree = ET.ElementTree(ET.fromstring(KEEPER_PROPERTIES_TEMPLATE.format(id=i)))
|
|
keeper_server_xml: ET.Element | None = tree.find("keeper_server")
|
|
|
|
if keeper_server_xml is not None and len(selected_settings.items()) > 0:
|
|
for setting, next_child in selected_settings.items():
|
|
new_element = ET.SubElement(keeper_server_xml, setting)
|
|
if isinstance(next_child, dict):
|
|
apply_properties_recursively(new_element, next_child, 0)
|
|
elif isinstance(next_child, PropertiesGroup):
|
|
raise Exception("Can't use Properties Group here")
|
|
else:
|
|
new_element.text = str(next_child())
|
|
|
|
# Set default coordination settings
|
|
coordination_settings_xml = keeper_server_xml.find("coordination_settings")
|
|
if coordination_settings_xml is None:
|
|
coordination_settings_xml = ET.SubElement(
|
|
keeper_server_xml, "coordination_settings"
|
|
)
|
|
operation_timeout_ms_xml = ET.SubElement(
|
|
coordination_settings_xml, "operation_timeout_ms"
|
|
)
|
|
operation_timeout_ms_xml.text = "10000"
|
|
session_timeout_ms_xml = ET.SubElement(
|
|
coordination_settings_xml, "session_timeout_ms"
|
|
)
|
|
session_timeout_ms_xml.text = "15000"
|
|
raft_logs_level_xml = ET.SubElement(
|
|
coordination_settings_xml, "raft_logs_level"
|
|
)
|
|
raft_logs_level_xml.text = "trace"
|
|
election_timeout_lower_bound_ms_xml = ET.SubElement(
|
|
coordination_settings_xml, "election_timeout_lower_bound_ms"
|
|
)
|
|
election_timeout_lower_bound_ms_xml.text = "2000"
|
|
election_timeout_upper_bound_ms_xml = ET.SubElement(
|
|
coordination_settings_xml, "election_timeout_upper_bound_ms"
|
|
)
|
|
election_timeout_upper_bound_ms_xml.text = "4000"
|
|
|
|
# Multi read is required for private binary
|
|
if is_private_binary:
|
|
feature_flags_xml = keeper_server_xml.find("feature_flags")
|
|
if feature_flags_xml is None:
|
|
feature_flags_xml = ET.SubElement(keeper_server_xml, "feature_flags")
|
|
multi_read_xml = feature_flags_xml.find("multi_read")
|
|
if multi_read_xml is None:
|
|
multi_read_xml = ET.SubElement(feature_flags_xml, "multi_read")
|
|
multi_read_xml.text = "1"
|
|
|
|
# The `create_ttl` feature flag requires `write_snapshot_version` >= 8,
|
|
# otherwise the server refuses to start
|
|
feature_flags_xml = keeper_server_xml.find("feature_flags")
|
|
if (
|
|
feature_flags_xml is not None
|
|
and feature_flags_xml.findtext("create_ttl") == "1"
|
|
):
|
|
write_snapshot_version_xml = coordination_settings_xml.find(
|
|
"write_snapshot_version"
|
|
)
|
|
if write_snapshot_version_xml is None:
|
|
write_snapshot_version_xml = ET.SubElement(
|
|
coordination_settings_xml, "write_snapshot_version"
|
|
)
|
|
if (
|
|
write_snapshot_version_xml.text is None
|
|
or int(write_snapshot_version_xml.text) < 8
|
|
):
|
|
write_snapshot_version_xml.text = "8"
|
|
|
|
ET.indent(tree, space=" ", level=0)
|
|
with tempfile.NamedTemporaryFile(
|
|
dir=args.tmp_files_dir, suffix=".xml", delete=False
|
|
) as temp_file:
|
|
result_configs.append(temp_file.name)
|
|
tree.write(temp_file.name, encoding="utf-8", xml_declaration=True)
|
|
return result_configs
|