mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Alexey MilovidovandClaude Opus 4.6
6ceef72dda
The test partitions the Keeper cluster to verify readiness endpoint behavior during quorum loss, but does not wait for the cluster to recover after the partition heals. When the test is retried, subsequent iterations fail with "No followers in Keeper cluster" because nodes haven't re-established their roles yet. Add `keeper_utils.wait_nodes` after the `PartitionManager` context exits to ensure all nodes are fully connected before the next iteration. https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=96423&sha=b1ec90f0bb0dba81b331dcc0e7c995d666cda6fa&name_0=PR&name_1=Integration%20tests%20%28amd_asan%2C%20targeted%29 Co-Authored-By: Claude Opus 4.6 <[email protected]>
89 lines
2.7 KiB
Python
89 lines
2.7 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import os
|
|
|
|
import pytest
|
|
import requests
|
|
|
|
import helpers.keeper_utils as keeper_utils
|
|
from helpers.cluster import ClickHouseCluster
|
|
from helpers.network import PartitionManager
|
|
|
|
cluster = ClickHouseCluster(__file__)
|
|
CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs")
|
|
|
|
# Disable `with_remote_database_disk` as the test does not use the default Keeper.
|
|
node1 = cluster.add_instance(
|
|
"node1",
|
|
main_configs=["configs/enable_keeper1.xml"],
|
|
stay_alive=True,
|
|
with_remote_database_disk=False,
|
|
)
|
|
node2 = cluster.add_instance(
|
|
"node2",
|
|
main_configs=["configs/enable_keeper2.xml"],
|
|
stay_alive=True,
|
|
with_remote_database_disk=False,
|
|
)
|
|
node3 = cluster.add_instance(
|
|
"node3",
|
|
main_configs=["configs/enable_keeper3.xml"],
|
|
stay_alive=True,
|
|
with_remote_database_disk=False,
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def started_cluster():
|
|
try:
|
|
cluster.start(connection_timeout=450.0)
|
|
yield cluster
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
def test_http_readiness_basic_responses(started_cluster):
|
|
leader = keeper_utils.get_leader(cluster, [node1, node2, node3])
|
|
response = requests.get(
|
|
"http://{host}:{port}/ready".format(host=leader.ip_address, port=9182)
|
|
)
|
|
assert response.status_code == 200
|
|
|
|
readiness_data = response.json()
|
|
assert readiness_data["status"] == "ok"
|
|
assert readiness_data["details"]["role"] == "leader"
|
|
|
|
follower = keeper_utils.get_any_follower(cluster, [node1, node2, node3])
|
|
response = requests.get(
|
|
"http://{host}:{port}/ready".format(host=follower.ip_address, port=9182)
|
|
)
|
|
assert response.status_code == 200
|
|
|
|
readiness_data = response.json()
|
|
assert readiness_data["status"] == "ok"
|
|
assert readiness_data["details"]["role"] == "follower"
|
|
assert readiness_data["details"]["hasLeader"]
|
|
|
|
|
|
def test_http_readiness_partitioned_cluster(started_cluster):
|
|
with PartitionManager() as pm:
|
|
leader = keeper_utils.get_leader(cluster, [node1, node2, node3])
|
|
follower = keeper_utils.get_any_follower(cluster, [node1, node2, node3])
|
|
|
|
pm.partition_instances(leader, follower)
|
|
keeper_utils.wait_until_quorum_lost(cluster, follower)
|
|
|
|
response = requests.get(
|
|
"http://{host}:{port}/ready".format(host=follower.ip_address, port=9182)
|
|
)
|
|
print(response.json())
|
|
assert response.status_code == 503
|
|
|
|
readiness_data = response.json()
|
|
assert readiness_data["status"] == "fail"
|
|
assert readiness_data["details"]["role"] == "follower"
|
|
assert readiness_data["details"]["hasLeader"] == False
|
|
|
|
# Wait for cluster to recover after partition is healed
|
|
keeper_utils.wait_nodes(cluster, [node1, node2, node3])
|