mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Alexey MilovidovandClaude Opus 4.8
250798b76f
`test_replication_without_zookeeper/test.py::test_startup_without_zookeeper` intermittently fails with `kazoo.exceptions.NotEmptyError` inside `drop_zk`, which calls `zk.delete(path="/clickhouse", recursive=True)`. The drop happens while `node1` (ClickHouse) is still running, so its `ReplicatedMergeTree` background threads keep re-creating ZooKeeper nodes under `/clickhouse`. kazoo's recursive delete races with that: between `get_children` and `delete` a new child node can appear, and kazoo raises `NotEmptyError` without retrying. The test already used `cluster.run_kazoo_commands_with_retries`, but with the default `repeats=1`. In that helper the retry loop is `for i in range(repeats - 1)`, so `repeats=1` performs zero retries — the callback runs exactly once with no exception handling. Pass `repeats=5` (matching the existing usages in `helpers/cluster.py`) so the drop is retried on the transient `NotEmptyError`. This is a pre-existing flaky test (same failure seen on unrelated PRs, e.g. ClickHouse/ClickHouse#103540 and ClickHouse/ClickHouse#101757), not a regression. CI report: https://s3.amazonaws.com/clickhouse-test-reports/json.html?REF=master&sha=d40ea5d0da4103b54971ac01a9960ef9153242bb&name_0=MasterCI&name_1=Integration%20tests%20%28amd_asan_ubsan%2C%20db%20disk%2C%20old%20analyzer%2C%203%2F6%29 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
68 lines
1.7 KiB
Python
68 lines
1.7 KiB
Python
import time
|
|
|
|
import pytest
|
|
|
|
from helpers.cluster import ClickHouseCluster
|
|
|
|
cluster = ClickHouseCluster(__file__)
|
|
node1 = cluster.add_instance(
|
|
"node1",
|
|
main_configs=["configs/remote_servers.xml"],
|
|
with_zookeeper=True,
|
|
stay_alive=True,
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def start_cluster():
|
|
try:
|
|
cluster.start()
|
|
|
|
node1.query(
|
|
"""
|
|
CREATE DATABASE test;
|
|
CREATE TABLE test_table(date Date, id UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/replicated', 'node1') ORDER BY id PARTITION BY toYYYYMM(date);
|
|
"""
|
|
)
|
|
|
|
yield cluster
|
|
|
|
except Exception as ex:
|
|
print(ex)
|
|
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
def drop_zk(zk):
|
|
zk.delete(path="/clickhouse", recursive=True)
|
|
|
|
|
|
def test_startup_without_zookeeper(start_cluster):
|
|
node1.query(
|
|
"INSERT INTO test_table VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)"
|
|
)
|
|
assert node1.query("SELECT COUNT(*) from test_table") == "3\n"
|
|
assert (
|
|
node1.query("SELECT is_readonly from system.replicas where table='test_table'")
|
|
== "0\n"
|
|
)
|
|
|
|
cluster.run_kazoo_commands_with_retries(drop_zk, repeats=5)
|
|
|
|
time.sleep(5)
|
|
assert node1.query("SELECT COUNT(*) from test_table") == "3\n"
|
|
with pytest.raises(Exception):
|
|
node1.query(
|
|
"INSERT INTO test_table VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)"
|
|
)
|
|
|
|
node1.restart_clickhouse()
|
|
|
|
assert node1.query("SELECT COUNT(*) from test_table") == "3\n"
|
|
assert (
|
|
node1.query("SELECT is_readonly from system.replicas where table='test_table'")
|
|
== "1\n"
|
|
)
|