mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Alexey MilovidovandClaude Opus 4.8
295ced78d1
Follow-up to #105171, which added `ruff` with a deliberately narrow, bug-class-only rule set and left the unused-name and star-import rules off pending a dedicated cleanup pass. This enables that second batch in `pyproject.toml` and fixes every violation across the tree (the bulk in integration tests): - `F401` unused-import and `F841` unused-variable: removed, preserving any side-effecting right-hand side (e.g. `result = node.query(...)` becomes `node.query(...)`). Two `try: import ...` feature probes are kept with `# noqa: F401`. - `F541` f-string-missing-placeholders: dropped the redundant `f` prefix. - `F403`/`F405` star imports: every `from module import *` is expanded into explicit imports of the names actually used, taken from each name's real origin, so stdlib/third-party names import from their real source rather than through a test "convenience bundle". The now-empty `tests/integration/helpers/kafka/common_direct.py` bundle is removed and its three remaining consumers import `_VarintBytes`/`KafkaAdminClient` directly. Eliminating the star imports surfaced a real bug the wildcard had masked from `F821`: `generate_presets.py` in `test_prometheus_protocols` called an undefined `execute_instant_query_with_http_api`; fixed to the existing `execute_query_via_http_api`. No `black`/`isort` reformatting was applied (neither is enforced in the style check), so the diff is limited to the lint fixes themselves. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
143 lines
4.9 KiB
Python
Executable File
143 lines
4.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# Tests for the clickhouse_backupview utility.
|
|
# Use pytest ./test_backupview.py to run.
|
|
|
|
import os.path
|
|
import sys
|
|
import tempfile
|
|
|
|
|
|
script_dir = os.path.dirname(os.path.realpath(__file__))
|
|
backupview_dir = os.path.abspath(os.path.join(script_dir, ".."))
|
|
if backupview_dir not in sys.path:
|
|
sys.path.append(backupview_dir)
|
|
from clickhouse_backupview import FileInfo, open_backup
|
|
|
|
|
|
def calculate_num_files(dir):
|
|
count = 0
|
|
for _, _, files in os.walk(dir, topdown=False):
|
|
count += len([1 for name in files if name])
|
|
return count
|
|
|
|
|
|
def calculate_total_size(dir):
|
|
total_size = 0
|
|
for root, _, files in os.walk(dir, topdown=False):
|
|
total_size += sum(
|
|
[os.path.getsize(os.path.join(dir, root, name)) for name in files if name]
|
|
)
|
|
return total_size
|
|
|
|
|
|
###########################################################################################
|
|
# Actual tests
|
|
|
|
|
|
def test_backupview_1():
|
|
with open_backup(os.path.join(script_dir, "test_backup_1.zip")) as b:
|
|
assert b.get_subdirs("/") == ["shards"]
|
|
assert b.dir_exists("/shards")
|
|
assert not b.file_exists("/shards")
|
|
assert b.get_subdirs("/shards/") == ["1"]
|
|
assert b.get_subdirs("/shards/1/replicas/1") == ["data", "metadata"]
|
|
assert b.get_subdirs("/shards/1/replicas/1/metadata/") == ["mydb"]
|
|
assert b.get_files_in_dir("/shards/1/replicas/1/metadata/") == ["mydb.sql"]
|
|
assert b.file_exists("/shards/1/replicas/1/metadata/mydb.sql")
|
|
|
|
assert b.get_subdirs("/shards/1/replicas/1/data/mydb/tbl1") == [
|
|
"all_0_0_0",
|
|
"all_1_1_0",
|
|
"all_2_2_0",
|
|
"all_3_3_0",
|
|
]
|
|
|
|
assert b.get_files_in_dir("/shards/1/replicas/1/data/mydb/tbl1/all_0_0_0") == [
|
|
"checksums.txt",
|
|
"columns.txt",
|
|
"count.txt",
|
|
"data.bin",
|
|
"data.cmrk3",
|
|
"default_compression_codec.txt",
|
|
"metadata_version.txt",
|
|
"serialization.json",
|
|
]
|
|
|
|
assert b.get_databases() == ["mydb"]
|
|
assert b.get_tables(database="mydb") == ["tbl1", "tbl2"]
|
|
assert b.get_tables() == [("mydb", "tbl1"), ("mydb", "tbl2")]
|
|
|
|
assert b.get_create_query(database="mydb").startswith("CREATE DATABASE mydb")
|
|
|
|
assert b.get_create_query(table="mydb.tbl1").startswith(
|
|
"CREATE TABLE mydb.tbl1"
|
|
)
|
|
|
|
assert b.get_create_query(table=("mydb", "tbl1")).startswith(
|
|
"CREATE TABLE mydb.tbl1"
|
|
)
|
|
|
|
assert b.get_create_query(database="mydb", table="tbl2").startswith(
|
|
"CREATE TABLE mydb.tbl2"
|
|
)
|
|
|
|
assert b.get_partitions(table="mydb.tbl2") == ["all"]
|
|
assert b.get_parts(table="mydb.tbl2") == [
|
|
"all_0_0_0",
|
|
"all_1_1_0",
|
|
"all_2_2_0",
|
|
]
|
|
|
|
assert b.get_parts(table="mydb.tbl2", partition="all") == [
|
|
"all_0_0_0",
|
|
"all_1_1_0",
|
|
"all_2_2_0",
|
|
]
|
|
|
|
assert (
|
|
b.get_table_data_path(table="mydb.tbl1")
|
|
== "/shards/1/replicas/1/data/mydb/tbl1/"
|
|
)
|
|
|
|
assert b.get_table_data_files(table="mydb.tbl1", part="all_1_1_0") == [
|
|
"/shards/1/replicas/1/data/mydb/tbl1/checksums.txt",
|
|
"/shards/1/replicas/1/data/mydb/tbl1/columns.txt",
|
|
"/shards/1/replicas/1/data/mydb/tbl1/count.txt",
|
|
"/shards/1/replicas/1/data/mydb/tbl1/data.bin",
|
|
"/shards/1/replicas/1/data/mydb/tbl1/data.cmrk3",
|
|
"/shards/1/replicas/1/data/mydb/tbl1/default_compression_codec.txt",
|
|
"/shards/1/replicas/1/data/mydb/tbl1/metadata_version.txt",
|
|
"/shards/1/replicas/1/data/mydb/tbl1/serialization.json",
|
|
]
|
|
|
|
assert (
|
|
b.read_file(
|
|
"/shards/1/replicas/1/data/mydb/tbl1/all_1_1_0/default_compression_codec.txt"
|
|
)
|
|
== b"CODEC(LZ4)"
|
|
)
|
|
|
|
with b.open_file(
|
|
"/shards/1/replicas/1/data/mydb/tbl1/all_1_1_0/default_compression_codec.txt"
|
|
) as f:
|
|
assert f.read() == b"CODEC(LZ4)"
|
|
|
|
assert b.get_file_info(
|
|
"/shards/1/replicas/1/data/mydb/tbl1/all_1_1_0/default_compression_codec.txt"
|
|
) == FileInfo(
|
|
name="/shards/1/replicas/1/data/mydb/tbl1/all_1_1_0/default_compression_codec.txt",
|
|
size=10,
|
|
checksum="557036eda0fb0a277a7caf9b9c8d4dd6",
|
|
data_file="/shards/1/replicas/1/data/mydb/tbl1/all_0_0_0/default_compression_codec.txt",
|
|
)
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
res = b.extract_table_data(table="mydb.tbl1", out=temp_dir)
|
|
num_files = res.num_files
|
|
num_bytes = res.num_bytes
|
|
assert calculate_num_files(temp_dir) == num_files
|
|
assert calculate_total_size(temp_dir) == num_bytes
|
|
assert num_files == 32
|
|
assert num_bytes == 1728
|