mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Alexey MilovidovandClaude Opus 4.7
20bff28c93
These integration tests left behind state (tables, replicas, ZooKeeper paths) or had nodes killed by OOM, which caused failures on re-run with `--count > 1`: * `test_cluster_discovery/test_auxiliary_keeper.py`: ensure all nodes are running and drop the leftover table locally on each node before the test starts (`ON CLUSTER` cannot be used while some nodes may still be down from a previous run). * `test_concurrent_ttl_merges/test.py`: enable `stay_alive` on both nodes, restart any node killed by OOM, and reset table/replica state and the `SYSTEM STOP TTL MERGES` flag at the start of each test. * `test_keeper_remove_rejoin_leader/test.py`: replace `keeper_utils.wait_nodes` with per-node `wait_until_connected` (with a longer timeout), so a slow node does not abort the wait early. * `test_storage_s3/test.py`: when checking the version of `dummy_old`, treat a query failure as needs-restart instead of propagating the exception, so a node killed by OOM in a previous run is recovered. These changes were originally part of #100177 and are split out so the parent PR contains code changes only. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
137 lines
4.1 KiB
Python
137 lines
4.1 KiB
Python
import functools
|
|
|
|
import pytest
|
|
|
|
from helpers.cluster import ClickHouseCluster
|
|
|
|
from .common import check_on_cluster
|
|
|
|
cluster = ClickHouseCluster(__file__)
|
|
|
|
shard_configs = {
|
|
"node0": ["config/config.xml", "config/macros0.xml"],
|
|
"node1": ["config/config_shard1.xml", "config/macros1.xml"],
|
|
"node2": ["config/config.xml", "config/macros2.xml"],
|
|
"node3": ["config/config_shard3.xml", "config/macros3.xml"],
|
|
"node4": ["config/config.xml", "config/macros4.xml"],
|
|
"node_observer": ["config/config_observer.xml", "config/macros_o.xml"],
|
|
}
|
|
|
|
nodes = {
|
|
node_name: cluster.add_instance(
|
|
node_name,
|
|
main_configs=shard_config
|
|
+ [
|
|
"config/config_discovery_path_auxiliary_keeper.xml",
|
|
"config/config_keepers.xml",
|
|
],
|
|
stay_alive=True,
|
|
with_zookeeper=True,
|
|
)
|
|
for node_name, shard_config in shard_configs.items()
|
|
}
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def start_cluster():
|
|
try:
|
|
cluster.start()
|
|
yield cluster
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
def test_cluster_discovery_with_auxiliary_keeper_startup_and_stop(start_cluster):
|
|
"""
|
|
Start cluster, check nodes count in system.clusters,
|
|
then stop/start some nodes and check that it (dis)appeared in cluster.
|
|
"""
|
|
|
|
# Ensure all nodes are running (previous run may have left some stopped)
|
|
for name in ["node1", "node3"]:
|
|
nodes[name].start_clickhouse(start_wait_sec=60)
|
|
|
|
# Clean up table from previous run (drop locally on each node since ON CLUSTER
|
|
# requires all nodes to be healthy and discoverable)
|
|
for node in nodes.values():
|
|
node.query("DROP TABLE IF EXISTS tbl SYNC")
|
|
|
|
check_nodes_count = functools.partial(
|
|
check_on_cluster, what="count()", msg="Wrong nodes count in cluster"
|
|
)
|
|
check_shard_num = functools.partial(
|
|
check_on_cluster,
|
|
what="count(DISTINCT shard_num)",
|
|
msg="Wrong shard_num count in cluster",
|
|
)
|
|
|
|
total_shards = 3
|
|
total_nodes = 5
|
|
|
|
check_nodes_count(
|
|
[nodes["node0"], nodes["node2"], nodes["node_observer"]], total_nodes
|
|
)
|
|
check_shard_num(
|
|
[nodes["node0"], nodes["node2"], nodes["node_observer"]], total_shards
|
|
)
|
|
|
|
# test ON CLUSTER query
|
|
nodes["node0"].query(
|
|
"CREATE TABLE tbl ON CLUSTER 'test_auto_cluster' (x UInt64) ENGINE = ReplicatedMergeTree('zookeeper2:/clickhouse/{shard}/tbl', '{replica}') ORDER BY x"
|
|
)
|
|
nodes["node0"].query("INSERT INTO tbl VALUES (1)")
|
|
nodes["node1"].query("INSERT INTO tbl VALUES (2)")
|
|
|
|
assert (
|
|
int(
|
|
nodes["node_observer"]
|
|
.query(
|
|
"SELECT sum(x) FROM clusterAllReplicas(test_auto_cluster, default.tbl) settings serialize_query_plan=0"
|
|
)
|
|
.strip()
|
|
)
|
|
== 3
|
|
)
|
|
|
|
# Query SYSTEM CLEAR DNS CACHE may reload cluster configuration
|
|
# check that it does not affect cluster discovery
|
|
nodes["node1"].query("SYSTEM CLEAR DNS CACHE")
|
|
nodes["node0"].query("SYSTEM CLEAR DNS CACHE")
|
|
|
|
check_shard_num(
|
|
[nodes["node0"], nodes["node2"], nodes["node_observer"]], total_shards
|
|
)
|
|
|
|
nodes["node1"].stop_clickhouse(kill=True)
|
|
check_nodes_count(
|
|
[nodes["node0"], nodes["node2"], nodes["node_observer"]], total_nodes - 1
|
|
)
|
|
|
|
# node1 was the only node in shard '1'
|
|
check_shard_num(
|
|
[nodes["node0"], nodes["node2"], nodes["node_observer"]], total_shards - 1
|
|
)
|
|
|
|
nodes["node3"].stop_clickhouse()
|
|
check_nodes_count(
|
|
[nodes["node0"], nodes["node2"], nodes["node_observer"]], total_nodes - 2
|
|
)
|
|
|
|
nodes["node1"].start_clickhouse()
|
|
check_nodes_count(
|
|
[nodes["node0"], nodes["node2"], nodes["node_observer"]], total_nodes - 1
|
|
)
|
|
|
|
nodes["node3"].start_clickhouse()
|
|
check_nodes_count(
|
|
[nodes["node0"], nodes["node2"], nodes["node_observer"]], total_nodes
|
|
)
|
|
|
|
# regular cluster is not affected
|
|
check_nodes_count(
|
|
[nodes["node1"], nodes["node2"]], 2, cluster_name="two_shards", retries=1
|
|
)
|
|
|
|
# cleanup
|
|
nodes["node0"].query("DROP TABLE tbl ON CLUSTER 'test_auto_cluster' SYNC")
|