mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Alexey MilovidovandClaude Opus 4.8
295ced78d1
Follow-up to #105171, which added `ruff` with a deliberately narrow, bug-class-only rule set and left the unused-name and star-import rules off pending a dedicated cleanup pass. This enables that second batch in `pyproject.toml` and fixes every violation across the tree (the bulk in integration tests): - `F401` unused-import and `F841` unused-variable: removed, preserving any side-effecting right-hand side (e.g. `result = node.query(...)` becomes `node.query(...)`). Two `try: import ...` feature probes are kept with `# noqa: F401`. - `F541` f-string-missing-placeholders: dropped the redundant `f` prefix. - `F403`/`F405` star imports: every `from module import *` is expanded into explicit imports of the names actually used, taken from each name's real origin, so stdlib/third-party names import from their real source rather than through a test "convenience bundle". The now-empty `tests/integration/helpers/kafka/common_direct.py` bundle is removed and its three remaining consumers import `_VarintBytes`/`KafkaAdminClient` directly. Eliminating the star imports surfaced a real bug the wildcard had masked from `F821`: `generate_presets.py` in `test_prometheus_protocols` called an undefined `execute_instant_query_with_http_api`; fixed to the existing `execute_query_via_http_api`. No `black`/`isort` reformatting was applied (neither is enforced in the style check), so the diff is limited to the lint fixes themselves. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
163 lines
4.5 KiB
Python
163 lines
4.5 KiB
Python
import json
|
|
from datetime import datetime
|
|
from xml.etree import ElementTree as ET
|
|
|
|
import pytest
|
|
|
|
from helpers.cluster import ClickHouseCluster
|
|
|
|
cluster = ClickHouseCluster(__file__)
|
|
node_all_keys = cluster.add_instance(
|
|
"node_all_keys", main_configs=["configs/config_all_keys_json.xml"]
|
|
)
|
|
node_some_keys = cluster.add_instance(
|
|
"node_some_keys", main_configs=["configs/config_some_keys_json.xml"]
|
|
)
|
|
node_no_keys = cluster.add_instance(
|
|
"node_no_keys", main_configs=["configs/config_no_keys_json.xml"]
|
|
)
|
|
|
|
node_json_logging_per_channel = cluster.add_instance(
|
|
"node_json_logging_per_channel", main_configs=["configs/config_json_logging_per_channel.xml"]
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def start_cluster():
|
|
try:
|
|
cluster.start()
|
|
yield cluster
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
def is_json(log_json):
|
|
try:
|
|
json.loads(log_json)
|
|
except ValueError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def validate_log_level(config, logs):
|
|
root = ET.fromstring(config)
|
|
key = root.findtext(".//names/level") or "level"
|
|
|
|
valid_level_values = {
|
|
"Fatal",
|
|
"Critical",
|
|
"Error",
|
|
"Warning",
|
|
"Notice",
|
|
"Information",
|
|
"Debug",
|
|
"Trace",
|
|
"Test",
|
|
}
|
|
|
|
length = min(10, len(logs))
|
|
for i in range(0, length):
|
|
json_log = json.loads(logs[i])
|
|
if json_log[key] not in valid_level_values:
|
|
return False
|
|
return True
|
|
|
|
|
|
def is_valid_utc_datetime(datetime_str):
|
|
try:
|
|
datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%S.%fZ")
|
|
return True
|
|
except ValueError:
|
|
try:
|
|
datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%SZ")
|
|
return True
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def validate_log_config_relation(config, logs, config_type):
|
|
root = ET.fromstring(config)
|
|
keys_in_config = set()
|
|
|
|
if config_type == "config_no_keys":
|
|
keys_in_config.add("date_time")
|
|
keys_in_config.add("date_time_utc")
|
|
keys_in_config.add("thread_name")
|
|
keys_in_config.add("thread_id")
|
|
keys_in_config.add("level")
|
|
keys_in_config.add("query_id")
|
|
keys_in_config.add("logger_name")
|
|
keys_in_config.add("message")
|
|
keys_in_config.add("source_file")
|
|
keys_in_config.add("source_line")
|
|
else:
|
|
for child in root.findall(".//names/*"):
|
|
keys_in_config.add(child.text)
|
|
|
|
try:
|
|
length = min(10, len(logs))
|
|
for i in range(0, length):
|
|
json_log = json.loads(logs[i])
|
|
keys_in_log = set(json_log.keys())
|
|
|
|
if not keys_in_config.issubset(keys_in_log):
|
|
return False
|
|
|
|
# Validate the UTC datetime format in "date_time_utc" if it exists
|
|
if "date_time_utc" in json_log and not is_valid_utc_datetime(
|
|
json_log["date_time_utc"]
|
|
):
|
|
return False
|
|
|
|
except ValueError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def validate_logs(logs):
|
|
length = min(10, len(logs))
|
|
result = True
|
|
for i in range(0, length):
|
|
result = result and is_json(logs[i])
|
|
return result
|
|
|
|
|
|
def validate_everything(config, node, config_type):
|
|
node.query("SELECT 1")
|
|
logs = node.grep_in_log("").split("\n")
|
|
return (
|
|
validate_logs(logs)
|
|
and validate_log_config_relation(config, logs, config_type)
|
|
and validate_log_level(config, logs)
|
|
)
|
|
|
|
|
|
def test_structured_logging_json_format(start_cluster):
|
|
config_all_keys = node_all_keys.exec_in_container(
|
|
["cat", "/etc/clickhouse-server/config.d/config_all_keys_json.xml"]
|
|
)
|
|
config_some_keys = node_some_keys.exec_in_container(
|
|
["cat", "/etc/clickhouse-server/config.d/config_some_keys_json.xml"]
|
|
)
|
|
config_no_keys = node_no_keys.exec_in_container(
|
|
["cat", "/etc/clickhouse-server/config.d/config_no_keys_json.xml"]
|
|
)
|
|
|
|
assert (
|
|
validate_everything(config_all_keys, node_all_keys, "config_all_keys") == True
|
|
)
|
|
assert (
|
|
validate_everything(config_some_keys, node_some_keys, "config_some_keys")
|
|
== True
|
|
)
|
|
assert validate_everything(config_no_keys, node_no_keys, "config_no_keys") == True
|
|
|
|
def test_structured_logging_per_channel(start_cluster):
|
|
logs = node_json_logging_per_channel.grep_in_log("").split("\n")
|
|
assert len(logs) > 0
|
|
assert not validate_logs(logs)
|
|
|
|
error_logs = node_json_logging_per_channel.grep_in_log("", filename="clickhouse-server.err.log").strip().split("\n")
|
|
assert len(error_logs) > 0
|
|
assert validate_logs(error_logs)
|