mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Alexey MilovidovandClaude Opus 4.8
295ced78d1
Follow-up to #105171, which added `ruff` with a deliberately narrow, bug-class-only rule set and left the unused-name and star-import rules off pending a dedicated cleanup pass. This enables that second batch in `pyproject.toml` and fixes every violation across the tree (the bulk in integration tests): - `F401` unused-import and `F841` unused-variable: removed, preserving any side-effecting right-hand side (e.g. `result = node.query(...)` becomes `node.query(...)`). Two `try: import ...` feature probes are kept with `# noqa: F401`. - `F541` f-string-missing-placeholders: dropped the redundant `f` prefix. - `F403`/`F405` star imports: every `from module import *` is expanded into explicit imports of the names actually used, taken from each name's real origin, so stdlib/third-party names import from their real source rather than through a test "convenience bundle". The now-empty `tests/integration/helpers/kafka/common_direct.py` bundle is removed and its three remaining consumers import `_VarintBytes`/`KafkaAdminClient` directly. Eliminating the star imports surfaced a real bug the wildcard had masked from `F821`: `generate_presets.py` in `test_prometheus_protocols` called an undefined `execute_instant_query_with_http_api`; fixed to the existing `execute_query_via_http_api`. No `black`/`isort` reformatting was applied (neither is enforced in the style check), so the diff is limited to the lint fixes themselves. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
108 lines
2.9 KiB
Python
108 lines
2.9 KiB
Python
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from helpers.cluster import ClickHouseCluster
|
|
|
|
cluster = ClickHouseCluster(__file__)
|
|
ch1 = cluster.add_instance(
|
|
"ch1",
|
|
main_configs=[
|
|
"config.xml",
|
|
],
|
|
macros={"replica": "node1"},
|
|
with_zookeeper=True,
|
|
with_minio=True,
|
|
)
|
|
|
|
DATABASE_NAME = "stale_moving_parts"
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def started_cluster():
|
|
try:
|
|
cluster.start()
|
|
yield cluster
|
|
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
def q(node, query):
|
|
return node.query(database=DATABASE_NAME, sql=query)
|
|
|
|
|
|
# .../disks/s3/store/
|
|
def get_table_path(node, table):
|
|
return (
|
|
node.query(
|
|
sql=f"SELECT data_paths FROM system.tables WHERE table = '{table}' and database = '{DATABASE_NAME}' LIMIT 1"
|
|
)
|
|
.strip('"\n[]')
|
|
.split(",")[1]
|
|
.strip("'")
|
|
)
|
|
|
|
|
|
def exec(node, cmd, path):
|
|
return node.exec_in_container(
|
|
[
|
|
"bash",
|
|
"-c",
|
|
f"{cmd} {path}",
|
|
],
|
|
nothrow=True,
|
|
)
|
|
|
|
|
|
def wait_part_is_stuck(node, table_moving_path, moving_part):
|
|
num_tries = 10
|
|
while q(node, "SELECT part_name FROM system.moves").strip() != moving_part:
|
|
if num_tries == 0:
|
|
raise Exception("Part has not started to move")
|
|
num_tries -= 1
|
|
time.sleep(1)
|
|
num_tries = 5
|
|
while exec(node, "ls", table_moving_path).strip() != moving_part:
|
|
if num_tries == 0:
|
|
raise Exception("Part is not stuck in the moving directory")
|
|
num_tries -= 1
|
|
time.sleep(1)
|
|
|
|
|
|
def test_remove_stale_moving_parts_without_zookeeper(started_cluster):
|
|
ch1.query(f"DROP DATABASE IF EXISTS {DATABASE_NAME}")
|
|
ch1.query(f"CREATE DATABASE {DATABASE_NAME}")
|
|
|
|
q(
|
|
ch1,
|
|
"CREATE TABLE test_remove ON CLUSTER cluster ( id UInt32 ) ENGINE ReplicatedMergeTree() ORDER BY id;",
|
|
)
|
|
|
|
table_moving_path = Path(get_table_path(ch1, "test_remove")) / "moving"
|
|
|
|
q(ch1, "SYSTEM ENABLE FAILPOINT stop_moving_part_before_swap_with_active")
|
|
q(ch1, "INSERT INTO test_remove SELECT number FROM numbers(100);")
|
|
moving_part = "all_0_0_0"
|
|
move_response = ch1.get_query_request(
|
|
sql=f"ALTER TABLE test_remove MOVE PART '{moving_part}' TO DISK 's3'",
|
|
database=DATABASE_NAME,
|
|
)
|
|
|
|
wait_part_is_stuck(ch1, table_moving_path, moving_part)
|
|
|
|
cluster.stop_zookeeper_nodes(["zoo1", "zoo2", "zoo3"])
|
|
# Stop moves in case table is not read-only yet
|
|
q(ch1, "SYSTEM STOP MOVES")
|
|
q(ch1, "SYSTEM DISABLE FAILPOINT stop_moving_part_before_swap_with_active")
|
|
|
|
assert "Cancelled moving parts" in move_response.get_error()
|
|
assert exec(ch1, "ls", table_moving_path).strip() == ""
|
|
|
|
cluster.start_zookeeper_nodes(["zoo1", "zoo2", "zoo3"])
|
|
cluster.wait_zookeeper_nodes_to_start(["zoo1", "zoo2", "zoo3"])
|
|
q(ch1, "SYSTEM START MOVES")
|
|
|
|
q(ch1, "DROP TABLE test_remove")
|