mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
zeekayandHanzo Dev
b3608b62ab
PrometheusMetricsWriter emits DatastoreProfileEvents_, DatastoreMetrics_, DatastoreAsyncMetrics_, DatastoreErrorMetric_ and Datastore_Info, while four consumers still asked for the ClickHouse spelling, so each was reading a string its own server had stopped writing. The three integration tests failed outright, the prometheus endpoint one twice over, since its allowed-prefix assertion rejects every metric line before the query counter is ever looked up. The keeper stress harness was quieter and worse: an allowlist matching nothing reads as a run that produced no metrics rather than as an error. The keeper-stress-analysis skill keeps the ClickHouse spelling on purpose. It queries keeper_stress_tests.keeper_metrics_ts on play.clickhouse.com, where those names are stored data written by upstream nightlies rather than anything this server emits, and that table holds no Datastore rows at all. Renaming there would turn a query that matches into one that matches nothing. Co-authored-by: Hanzo Dev <[email protected]>
92 lines
2.4 KiB
Python
92 lines
2.4 KiB
Python
import re
|
|
import time
|
|
|
|
import pytest
|
|
import requests
|
|
|
|
from helpers.cluster import ClickHouseCluster
|
|
|
|
cluster = ClickHouseCluster(__file__)
|
|
node = cluster.add_instance("node", main_configs=["configs/prom_conf.xml"])
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def start_cluster():
|
|
try:
|
|
cluster.start()
|
|
yield cluster
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
def parse_response_line(line):
|
|
allowed_prefixes = [
|
|
"Datastore",
|
|
"# HELP",
|
|
"# TYPE",
|
|
]
|
|
assert any(line.startswith(prefix) for prefix in allowed_prefixes)
|
|
|
|
if line.startswith("#"):
|
|
return {}
|
|
match = re.match(r"^([a-zA-Z_:][a-zA-Z0-9_:]+)(\{.*\})? -?(\d+)", line)
|
|
assert match, line
|
|
name, _, val = match.groups()
|
|
return {name: int(val)}
|
|
|
|
|
|
def get_and_check_metrics(retries):
|
|
while True:
|
|
try:
|
|
response = requests.get(
|
|
"http://{host}:{port}/metrics".format(host=node.ip_address, port=8001),
|
|
allow_redirects=False,
|
|
# less than default keep-alive timeout (10 seconds)
|
|
timeout=5,
|
|
)
|
|
|
|
if response.status_code != 200:
|
|
response.raise_for_status()
|
|
|
|
break
|
|
except:
|
|
if retries >= 0:
|
|
retries -= 1
|
|
time.sleep(0.5)
|
|
continue
|
|
else:
|
|
raise
|
|
|
|
assert response.headers["content-type"].startswith("text/plain")
|
|
|
|
results = {}
|
|
for resp_line in response.text.split("\n"):
|
|
resp_line = resp_line.rstrip()
|
|
if not resp_line:
|
|
continue
|
|
res = parse_response_line(resp_line)
|
|
results.update(res)
|
|
return results
|
|
|
|
|
|
def test_prometheus_endpoint(start_cluster):
|
|
metrics_dict = get_and_check_metrics(10)
|
|
assert metrics_dict["DatastoreProfileEvents_Query"] >= 0
|
|
prev_query_count = metrics_dict["DatastoreProfileEvents_Query"]
|
|
|
|
node.query("SELECT 1")
|
|
node.query("SELECT 2")
|
|
node.query("SELECT 3")
|
|
|
|
metrics_dict = get_and_check_metrics(10)
|
|
assert metrics_dict["DatastoreProfileEvents_Query"] >= prev_query_count + 3
|
|
|
|
node.query_and_get_error(
|
|
"SELECT throwIf(1, 'test', toInt16(42)) SETTINGS allow_custom_error_code_in_throwif=1"
|
|
)
|
|
metrics_dict = get_and_check_metrics(10)
|
|
|
|
assert metrics_dict["DatastoreErrorMetric_NUMBER_OF_ARGUMENTS_DOESNT_MATCH"] >= 1
|
|
assert metrics_dict["DatastoreErrorMetric_ALL"] >= 1
|
|
assert metrics_dict["Datastore_Info"] == 1
|