hanzoai/datastore
2
0
Fork 0
mirror of https://github.com/hanzoai/datastore.git synced 2026-09-26 13:55:07 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
main
datastore/tests/integration/test_zookeeper_connection_log/test.py
T
zooqueen fce0ee72e1 datastore: fix XML config tag mismatch (</clickhouse> closing tags) 
Runtime regression from the final lowercase pass: it renamed <clickhouse> opening tags to
<datastore> but the / exclusion (protecting /clickhouse paths) left </clickhouse> closing tags,
so embedded + shipped configs became <datastore>...</clickhouse> -> SAXParseException 'Tag
mismatch' at runtime (compile was fine; XML is data). Renamed all <clickhouse/</clickhouse XML
tags -> datastore (101 closes across configs, embedded strings, keeper configs). Path strings
(/var/lib/clickhouse) have no angle bracket and are untouched. Build compiled; this fixes runtime
config parsing.
2026-06-22 00:25:01 -07:00

188 lines
8.1 KiB
Python
Raw Permalink Blame History

import pytest
import logging
from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV
cluster = ClickHouseCluster(
__file__, zookeeper_config_path="configs/zookeeper_config.xml"
)
node1 = cluster.add_instance(
"node1",
with_zookeeper=True,
stay_alive=True,
main_configs=[
"configs/zookeeper_connection_log.xml",
"configs/auxiliary_zookeepers.xml",
"configs/config_reloader.xml",
],
keeper_randomize_feature_flags=False,
)
node2 = cluster.add_instance(
"node2",
main_configs=["configs/enable_keeper.xml"],
stay_alive=True,
keeper_randomize_feature_flags=False,
)
@pytest.fixture(scope="module")
def started_cluster():
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def test_zookeeper_connection_log(started_cluster):
node1.query("DROP TABLE IF EXISTS simple SYNC")
node1.query("DROP TABLE IF EXISTS simple2 SYNC")
test_start_time = node1.query("SELECT now64()").strip()
logging.debug(f"Test start time is {test_start_time}")
# Let's restart ClickHouse to make sure there are log entries for initialization.
# By restarting we can also make sure there won't be any config reloads in case of repeated runs.
# The previous run would revert the config, but we need to reset the state of config.
node1.restart_clickhouse()
node1.query(
"CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;"
)
node1.query("INSERT INTO simple VALUES ('2020-08-27', 1)")
node1.query("INSERT INTO simple VALUES ('2020-08-28', 1)")
node1.query("INSERT INTO simple VALUES ('2020-08-29', 1)")
node1.query(
"CREATE TABLE simple2 (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/simple', 'node') ORDER BY tuple() PARTITION BY date;"
)
node1.query(
"ALTER TABLE simple2 FETCH PARTITION '2020-08-27' FROM 'zk_conn_log_test_2:/clickhouse/tables/0/simple';"
)
node1.query(
"ALTER TABLE simple2 FETCH PARTITION '2020-08-28' FROM 'zk_conn_log_test_3:/clickhouse/tables/0/simple';"
)
new_auxiliary_config = """<datastore>
<auxiliary_zookeepers>
<zk_conn_log_test_2>
<node index="1">
<host>zoo3</host>
<port>2181</port>
</node>
</zk_conn_log_test_2>
<zk_conn_log_test_4>
<node index="1">
<host>zoo2</host>
<port>2181</port>
</node>
</zk_conn_log_test_4>
</auxiliary_zookeepers>
</datastore>"""
new_config = """<datastore>
<zookeeper>
<node index="1">
<host>zoo2</host>
<port>2181</port>
</node>
<session_timeout_ms>15000</session_timeout_ms>
</zookeeper>
</datastore>"""
with node1.with_replace_config(
"/etc/clickhouse-server/conf.d/zookeeper_config.xml", new_config
):
with node1.with_replace_config(
"/etc/clickhouse-server/config.d/auxiliary_zookeepers.xml",
new_auxiliary_config,
):
node1.query("SYSTEM RELOAD CONFIG")
def check_8_rows(res):
logging.debug(
f"Checking for 8 rows in zookeeper_connection_log, got: {res}"
)
return int(res) >= 8
node1.query_with_retry(
f"SELECT count() FROM system.zookeeper_connection_log WHERE event_time_microseconds >= '{test_start_time}'",
check_callback=check_8_rows,
)
node1.query(
"ALTER TABLE simple2 FETCH PARTITION '2020-08-29' FROM 'zk_conn_log_test_4:/clickhouse/tables/0/simple';"
)
node1.query("SYSTEM FLUSH LOGS")
logging.debug(
node1.query(
"""SELECT event_time_microseconds, client_id, hostname, type, name, host, port, index, keeper_api_version, enabled_feature_flags, reason
FROM system.zookeeper_connection_log ORDER BY event_time_microseconds"""
)
)
logging.debug(
node1.query(
f"""SELECT hostname, type, name, host, port, reason, count() AS cnt,
groupArray(client_id) AS client_ids
FROM system.zookeeper_connection_log
WHERE event_time_microseconds >= '{test_start_time}'
GROUP BY hostname, type, name, host, port, reason
ORDER BY min(event_time_microseconds)"""
)
)
normalized_rows = node1.query(
f"""SELECT hostname, type, name, host, port, index, keeper_api_version,
arrayFilter(x -> has(enabled_feature_flags, x),
['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS','REMOVE_RECURSIVE','MULTI_WATCHES','LIST_WITH_STAT_AND_DATA']) AS enabled_feature_flags,
reason
FROM system.zookeeper_connection_log
WHERE event_time_microseconds >= '{test_start_time}'
ORDER BY event_time_microseconds
LIMIT 1 BY hostname, type, name, host, port, reason"""
)
logging.debug("Normalized rows used in comparison:\n" + normalized_rows)
expected = TSV(
"""node1 Connected default zoo1 2181 0 0 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS','REMOVE_RECURSIVE','MULTI_WATCHES','LIST_WITH_STAT_AND_DATA'] Initialization
node1 Connected zk_conn_log_test_2 zoo2 2181 0 0 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS','REMOVE_RECURSIVE','MULTI_WATCHES','LIST_WITH_STAT_AND_DATA'] Initialization
node1 Connected zk_conn_log_test_3 zoo3 2181 0 0 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS','REMOVE_RECURSIVE','MULTI_WATCHES','LIST_WITH_STAT_AND_DATA'] Initialization
node1 Disconnected default zoo1 2181 0 0 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS','REMOVE_RECURSIVE','MULTI_WATCHES','LIST_WITH_STAT_AND_DATA'] Config changed
node1 Connected default zoo2 2181 0 0 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS','REMOVE_RECURSIVE','MULTI_WATCHES','LIST_WITH_STAT_AND_DATA'] Config changed
node1 Disconnected zk_conn_log_test_2 zoo2 2181 0 0 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS','REMOVE_RECURSIVE','MULTI_WATCHES','LIST_WITH_STAT_AND_DATA'] Config changed
node1 Connected zk_conn_log_test_2 zoo3 2181 0 0 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS','REMOVE_RECURSIVE','MULTI_WATCHES','LIST_WITH_STAT_AND_DATA'] Config changed
node1 Disconnected zk_conn_log_test_3 zoo3 2181 0 0 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS','REMOVE_RECURSIVE','MULTI_WATCHES','LIST_WITH_STAT_AND_DATA'] Removed from config
node1 Connected zk_conn_log_test_4 zoo2 2181 0 0 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS','REMOVE_RECURSIVE','MULTI_WATCHES','LIST_WITH_STAT_AND_DATA'] Initialization"""
)
assert TSV(normalized_rows) == expected
assert (
int(
node1.query(
"SELECT max(event_per_client_id) FROM (SELECT client_id, count() AS event_per_client_id FROM system.zookeeper_connection_log GROUP BY client_id)"
)
)
== 2
)
def test_connect_on_reload(started_cluster, request):
node2.replace_in_config(
"/etc/clickhouse-server/config.d/enable_keeper.xml",
"use_cluster>1",
"use_cluster>0",
)
node2.restart_clickhouse()
node2.replace_in_config(
"/etc/clickhouse-server/config.d/enable_keeper.xml",
"use_cluster>0",
"use_cluster>1",
)
node2.query("SYSTEM RELOAD CONFIG")
node2.query("SELECT 1")
Reference in New Issue View Git Blame Copy Permalink