mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Alexey MilovidovandClaude Opus 4.8
295ced78d1
Follow-up to #105171, which added `ruff` with a deliberately narrow, bug-class-only rule set and left the unused-name and star-import rules off pending a dedicated cleanup pass. This enables that second batch in `pyproject.toml` and fixes every violation across the tree (the bulk in integration tests): - `F401` unused-import and `F841` unused-variable: removed, preserving any side-effecting right-hand side (e.g. `result = node.query(...)` becomes `node.query(...)`). Two `try: import ...` feature probes are kept with `# noqa: F401`. - `F541` f-string-missing-placeholders: dropped the redundant `f` prefix. - `F403`/`F405` star imports: every `from module import *` is expanded into explicit imports of the names actually used, taken from each name's real origin, so stdlib/third-party names import from their real source rather than through a test "convenience bundle". The now-empty `tests/integration/helpers/kafka/common_direct.py` bundle is removed and its three remaining consumers import `_VarintBytes`/`KafkaAdminClient` directly. Eliminating the star imports surfaced a real bug the wildcard had masked from `F821`: `generate_presets.py` in `test_prometheus_protocols` called an undefined `execute_instant_query_with_http_api`; fixed to the existing `execute_query_via_http_api`. No `black`/`isort` reformatting was applied (neither is enforced in the style check), so the diff is limited to the lint fixes themselves. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
119 lines
3.6 KiB
Python
119 lines
3.6 KiB
Python
import time
|
|
|
|
import pytest
|
|
|
|
from helpers.cluster import ClickHouseCluster
|
|
from helpers.network import PartitionManager
|
|
from helpers.test_tools import assert_eq_with_retry, get_retry_number
|
|
|
|
cluster = ClickHouseCluster(__file__)
|
|
node1 = cluster.add_instance(
|
|
"node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def start_cluster():
|
|
try:
|
|
cluster.start()
|
|
yield cluster
|
|
except Exception as ex:
|
|
print(ex)
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
# Test that outdated parts are not removed when they cannot be removed from zookeeper
|
|
def test_merge_doesnt_work_without_zookeeper(start_cluster, request):
|
|
retry_number = get_retry_number(request)
|
|
table = f"test_table_{retry_number}"
|
|
|
|
node1.query(
|
|
f"""
|
|
CREATE TABLE {table}(date Date, id UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/replicated', 'node1')
|
|
ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS old_parts_lifetime=4, cleanup_delay_period=1, cleanup_thread_preferred_points_per_iteration=0;
|
|
"""
|
|
)
|
|
|
|
node1.query(
|
|
f"INSERT INTO {table} VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)"
|
|
)
|
|
node1.query(
|
|
f"INSERT INTO {table} VALUES ('2018-10-01', 4), ('2018-10-02', 5), ('2018-10-03', 6)"
|
|
)
|
|
assert (
|
|
node1.query(f"SELECT count(*) from system.parts where table = '{table}'")
|
|
== "2\n"
|
|
)
|
|
|
|
node1.query(f"OPTIMIZE TABLE {table} FINAL")
|
|
assert_eq_with_retry(
|
|
node1,
|
|
f"SELECT count(*) from system.parts where table = '{table}' and active = 1",
|
|
"1",
|
|
)
|
|
|
|
node1.query(f"TRUNCATE TABLE {table}")
|
|
|
|
total_parts = node1.query(
|
|
f"SELECT count(*) from system.parts where table = '{table}'"
|
|
)
|
|
assert total_parts == "0\n" or total_parts == "1\n"
|
|
|
|
assert (
|
|
node1.query(
|
|
f"SELECT count(*) from system.parts where table = '{table}' and active = 1"
|
|
)
|
|
== "0\n"
|
|
)
|
|
|
|
node1.query(f"DETACH TABLE {table} SYNC")
|
|
node1.query(f"ATTACH TABLE {table}")
|
|
|
|
node1.query(
|
|
f"INSERT INTO {table} VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)"
|
|
)
|
|
node1.query(
|
|
f"INSERT INTO {table} VALUES ('2018-10-01', 4), ('2018-10-02', 5), ('2018-10-03', 6)"
|
|
)
|
|
assert (
|
|
node1.query(
|
|
f"SELECT count(*) from system.parts where table = '{table}' and active"
|
|
)
|
|
== "2\n"
|
|
)
|
|
|
|
# Force creation of all *Log tables
|
|
node1.query("SYSTEM FLUSH LOGS")
|
|
|
|
# Parts may be moved to Deleting state and then back in Outdated state.
|
|
# But system.parts returns only Active and Outdated parts if _state column is not queried.
|
|
with PartitionManager() as pm:
|
|
node1.query(f"OPTIMIZE TABLE {table} FINAL")
|
|
pm.drop_instance_zk_connections(node1)
|
|
# unfortunately we can be too fast and delete node before partition with ZK
|
|
if (
|
|
node1.query(
|
|
f"SELECT count(*) from system.parts where table = '{table}' and _state!='dummy'"
|
|
)
|
|
== "1\n"
|
|
):
|
|
print("We were too fast and deleted parts before partition with ZK")
|
|
else:
|
|
time.sleep(10) # > old_parts_lifetime
|
|
assert (
|
|
node1.query(
|
|
f"SELECT count(*) from system.parts where table = '{table}' and _state!='dummy'"
|
|
)
|
|
== "3\n"
|
|
)
|
|
|
|
assert_eq_with_retry(
|
|
node1,
|
|
f"SELECT count(*) from system.parts where table = '{table}' and active = 1",
|
|
"1",
|
|
)
|
|
|
|
node1.query(f"DROP TABLE {table} SYNC")
|