mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
zlareb1andClaude Sonnet 4.6
35e48faf43
Adds a stress testing framework for ClickHouse Keeper under `tests/stress/keeper/`, wired into CI (PR gate + nightly workflows). Key components: - Workloads: `keeper_bench.py` runner with YAML-defined workload profiles (`prod_mix.yaml`, `read_requests.yaml`, `write_requests.yaml`, etc.) - Scenarios: `core_no_faults.yaml` and `core_faults.yaml` with netem-based network fault injection (latency, jitter, packet loss) - Fault framework: `faults/network.py` (netem), `faults/base.py` (step dispatcher with `kind: parallel` support) - Metrics: `MetricsSampler` collects keeper stats, `sink.py` writes JSONL, ingested into CIDB via `ingest_keeper_metrics.py` post-hook - Gates: scenario pass/fail conditions based on RPS, p99 latency, error rate - Grafana dashboards for run details, 1vs1 comparison, historical progression CI integration: - PR job: 3 no-fault scenarios (prod-mix, read-multi, write-multi), ARM_LARGE, 15 min each, runs when `src/Coordination/` or stress test files change - Nightly (no-faults): all no-fault scenarios, default + rocks backends, ARM_LARGE - Nightly (faults): all fault scenarios, default + rocks backends, ARM_LARGE - Single `keeper_stress_job` config; mode detected via `Info().pr_number` and `Info().workflow_name` inside the job script Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
32 lines
985 B
Python
32 lines
985 B
Python
import json
|
|
import os
|
|
import threading
|
|
|
|
from keeper.framework.core.util import env_bool
|
|
|
|
_write_lock = threading.Lock()
|
|
|
|
|
|
def _write_jsonl_lines(path, rows):
|
|
"""Append rows as JSONL to path."""
|
|
with _write_lock, open(path, "a", encoding="utf-8") as f:
|
|
for r in rows:
|
|
if env_bool("KEEPER_METRICS_DEBUG", False):
|
|
print(f"writing row: {json.dumps(r, ensure_ascii=False)} to {path}")
|
|
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
|
|
|
|
def sink_clickhouse(rows):
|
|
"""Write rows to sidecar JSONL file."""
|
|
if not rows:
|
|
return
|
|
sidecar = os.environ.get("KEEPER_METRICS_FILE", "").strip()
|
|
if not sidecar:
|
|
raise AssertionError("KEEPER_METRICS_FILE is not set")
|
|
# Ensure directory exists
|
|
try:
|
|
os.makedirs(os.path.dirname(sidecar) or ".", exist_ok=True)
|
|
except OSError:
|
|
raise AssertionError(f"Failed to create directory for {sidecar}")
|
|
_write_jsonl_lines(sidecar, rows)
|