mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Alexey MilovidovandClaude Opus 4.8
295ced78d1
Follow-up to #105171, which added `ruff` with a deliberately narrow, bug-class-only rule set and left the unused-name and star-import rules off pending a dedicated cleanup pass. This enables that second batch in `pyproject.toml` and fixes every violation across the tree (the bulk in integration tests): - `F401` unused-import and `F841` unused-variable: removed, preserving any side-effecting right-hand side (e.g. `result = node.query(...)` becomes `node.query(...)`). Two `try: import ...` feature probes are kept with `# noqa: F401`. - `F541` f-string-missing-placeholders: dropped the redundant `f` prefix. - `F403`/`F405` star imports: every `from module import *` is expanded into explicit imports of the names actually used, taken from each name's real origin, so stdlib/third-party names import from their real source rather than through a test "convenience bundle". The now-empty `tests/integration/helpers/kafka/common_direct.py` bundle is removed and its three remaining consumers import `_VarintBytes`/`KafkaAdminClient` directly. Eliminating the star imports surfaced a real bug the wildcard had masked from `F821`: `generate_presets.py` in `test_prometheus_protocols` called an undefined `execute_instant_query_with_http_api`; fixed to the existing `execute_query_via_http_api`. No `black`/`isort` reformatting was applied (neither is enforced in the style check), so the diff is limited to the lint fixes themselves. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
144 lines
4.3 KiB
Python
144 lines
4.3 KiB
Python
|
|
import pytest
|
|
|
|
from helpers.cluster import ClickHouseCluster
|
|
from helpers.test_tools import assert_eq_with_retry
|
|
from helpers.wait_for_helpers import (
|
|
wait_for_delete_empty_parts,
|
|
)
|
|
|
|
cluster = ClickHouseCluster(__file__)
|
|
node1 = cluster.add_instance(
|
|
"node1",
|
|
main_configs=["configs/asynchronous_metrics_update_period_s.xml"],
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def started_cluster():
|
|
try:
|
|
cluster.start()
|
|
yield cluster
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
def test_numbers_of_detached_parts(started_cluster):
|
|
cluster.start()
|
|
node1.query("DROP TABLE IF EXISTS t SYNC")
|
|
|
|
query_create = """
|
|
CREATE TABLE t
|
|
(
|
|
id Int64,
|
|
event_time Date
|
|
)
|
|
Engine=MergeTree()
|
|
PARTITION BY toYYYYMMDD(event_time)
|
|
ORDER BY id;
|
|
"""
|
|
node1.query(query_create)
|
|
|
|
# gives us 2 partitions with 3 parts in total
|
|
node1.query("INSERT INTO t VALUES (1, toDate('2022-09-01'));")
|
|
node1.query("INSERT INTO t VALUES (2, toDate('2022-08-29'));")
|
|
node1.query("INSERT INTO t VALUES (3, toDate('2022-09-01'));")
|
|
|
|
query_number_detached_parts_in_async_metric = """
|
|
SELECT value
|
|
FROM system.asynchronous_metrics
|
|
WHERE metric LIKE 'NumberOfDetachedParts';
|
|
"""
|
|
query_number_detached_by_user_parts_in_async_metric = """
|
|
SELECT value
|
|
FROM system.asynchronous_metrics
|
|
WHERE metric LIKE 'NumberOfDetachedByUserParts';
|
|
"""
|
|
query_count_active_parts = """
|
|
SELECT count(*) FROM system.parts WHERE table = 't' AND active
|
|
"""
|
|
query_count_detached_parts = """
|
|
SELECT count(*) FROM system.detached_parts WHERE table = 't'
|
|
"""
|
|
|
|
query_one_partition_name = """
|
|
SELECT name FROM system.parts WHERE table = 't' AND active AND partition = '20220829'
|
|
"""
|
|
partition_name = node1.query(query_one_partition_name).strip()
|
|
|
|
assert 0 == int(node1.query(query_count_detached_parts))
|
|
assert 3 == int(node1.query(query_count_active_parts))
|
|
assert 0 == int(node1.query(query_number_detached_parts_in_async_metric))
|
|
assert 0 == int(node1.query(query_number_detached_by_user_parts_in_async_metric))
|
|
|
|
# detach some parts and wait until asynchronous metrics notice it
|
|
node1.query("ALTER TABLE t DETACH PARTITION '20220901';")
|
|
wait_for_delete_empty_parts(node1, "t")
|
|
|
|
assert 2 == int(node1.query(query_count_detached_parts))
|
|
assert 1 == int(node1.query(query_count_active_parts))
|
|
|
|
assert_eq_with_retry(
|
|
node1,
|
|
query_number_detached_parts_in_async_metric,
|
|
"2\n",
|
|
)
|
|
assert 2 == int(node1.query(query_number_detached_by_user_parts_in_async_metric))
|
|
|
|
# detach the rest parts and wait until asynchronous metrics notice it
|
|
node1.query("ALTER TABLE t DETACH PARTITION ALL")
|
|
wait_for_delete_empty_parts(node1, "t")
|
|
|
|
assert 3 == int(node1.query(query_count_detached_parts))
|
|
assert 0 == int(node1.query(query_count_active_parts))
|
|
|
|
assert_eq_with_retry(
|
|
node1,
|
|
query_number_detached_parts_in_async_metric,
|
|
"3\n",
|
|
)
|
|
assert 3 == int(node1.query(query_number_detached_by_user_parts_in_async_metric))
|
|
|
|
# inject some data directly and wait until asynchronous metrics notice it
|
|
data_path = node1.query(
|
|
"SELECT arrayElement(data_paths, 1) FROM system.tables WHERE database='default' AND name='t'"
|
|
).strip()
|
|
node1.exec_in_container(
|
|
[
|
|
"bash",
|
|
"-c",
|
|
f"mkdir {data_path}/detached/unexpected_all_0_0_0",
|
|
]
|
|
)
|
|
|
|
assert 4 == int(node1.query(query_count_detached_parts))
|
|
assert 0 == int(node1.query(query_count_active_parts))
|
|
|
|
assert_eq_with_retry(
|
|
node1,
|
|
query_number_detached_parts_in_async_metric,
|
|
"4\n",
|
|
)
|
|
assert 3 == int(node1.query(query_number_detached_by_user_parts_in_async_metric))
|
|
|
|
# drop some data directly and wait asynchronous metrics notice it
|
|
node1.exec_in_container(
|
|
[
|
|
"bash",
|
|
"-c",
|
|
f"rm -rf {data_path}/detached/{partition_name}",
|
|
]
|
|
)
|
|
|
|
assert 3 == int(node1.query(query_count_detached_parts))
|
|
assert 0 == int(node1.query(query_count_active_parts))
|
|
|
|
assert_eq_with_retry(
|
|
node1,
|
|
query_number_detached_parts_in_async_metric,
|
|
"3\n",
|
|
)
|
|
assert 2 == int(node1.query(query_number_detached_by_user_parts_in_async_metric))
|
|
|
|
node1.query("DROP TABLE t SYNC")
|