mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
zlareb1andClaude Sonnet 4.6
35e48faf43
Adds a stress testing framework for ClickHouse Keeper under `tests/stress/keeper/`, wired into CI (PR gate + nightly workflows). Key components: - Workloads: `keeper_bench.py` runner with YAML-defined workload profiles (`prod_mix.yaml`, `read_requests.yaml`, `write_requests.yaml`, etc.) - Scenarios: `core_no_faults.yaml` and `core_faults.yaml` with netem-based network fault injection (latency, jitter, packet loss) - Fault framework: `faults/network.py` (netem), `faults/base.py` (step dispatcher with `kind: parallel` support) - Metrics: `MetricsSampler` collects keeper stats, `sink.py` writes JSONL, ingested into CIDB via `ingest_keeper_metrics.py` post-hook - Gates: scenario pass/fail conditions based on RPS, p99 latency, error rate - Grafana dashboards for run details, 1vs1 comparison, historical progression CI integration: - PR job: 3 no-fault scenarios (prod-mix, read-multi, write-multi), ARM_LARGE, 15 min each, runs when `src/Coordination/` or stress test files change - Nightly (no-faults): all no-fault scenarios, default + rocks backends, ARM_LARGE - Nightly (faults): all fault scenarios, default + rocks backends, ARM_LARGE - Single `keeper_stress_job` config; mode detected via `Info().pr_number` and `Info().workflow_name` inside the job script Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
76 lines
1.8 KiB
Bash
Executable File
76 lines
1.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
JEMALLOC_PROFILER=0
|
|
if [[ ! -v MALLOC_CONF ]]; then
|
|
jemalloc_profiles=/tmp/jemalloc_profiles
|
|
mkdir -p "$jemalloc_profiles"
|
|
|
|
export MALLOC_CONF=prof_active:true,prof_prefix:$jemalloc_profiles/clickhouse.jemalloc
|
|
JEMALLOC_PROFILER=1
|
|
fi
|
|
|
|
PID=0
|
|
|
|
function handle_term()
|
|
{
|
|
echo "Sending TERM to $PID"
|
|
ps aux
|
|
kill -TERM "$PID"
|
|
}
|
|
trap handle_term TERM
|
|
|
|
echo "Running: $*"
|
|
"$@" &
|
|
PID=$!
|
|
# This will be interrupted by SIGTERM that is received by this script
|
|
wait $PID
|
|
server_exit_code=$?
|
|
|
|
function dump_stacktraces_on_shutdown()
|
|
{
|
|
# 60 sec should be enough to finish the server
|
|
for _ in {1..60}; do
|
|
if ! kill -0 "$PID" 2>/dev/null; then
|
|
return
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
if kill -0 "$PID"; then
|
|
echo "Attaching gdb to obtain thread stacktraces"
|
|
gdb -batch -ex 'thread apply all bt' -p "$PID" > /var/log/clickhouse-server/stdout.log
|
|
fi
|
|
}
|
|
dump_stacktraces_on_shutdown &
|
|
|
|
while kill -0 "$PID" 2>/dev/null; do
|
|
wait $PID
|
|
server_exit_code=$?
|
|
done
|
|
echo "Server exited with $server_exit_code"
|
|
|
|
# Wait dump_stacktraces_on_shutdown
|
|
wait
|
|
|
|
if [[ $JEMALLOC_PROFILER -eq 1 ]]; then
|
|
jemalloc_reports=/var/lib/clickhouse/jemalloc
|
|
mkdir -p "$jemalloc_reports"
|
|
|
|
echo "=== jemalloc reports:"
|
|
ls -dlt "$jemalloc_profiles"/* | head
|
|
|
|
bin="$(which clickhouse)"
|
|
last_profile="$(ls -dt "$jemalloc_profiles"/* | head -1)"
|
|
echo "Using $last_profile"
|
|
|
|
if [[ -n $last_profile ]]; then
|
|
jeprof "$bin" "$last_profile" --text > "$jemalloc_reports/jemalloc.txt"
|
|
jeprof "$bin" "$last_profile" --collapsed | flamegraph.pl --color mem --width 2560 > "$jemalloc_reports/jemalloc.svg"
|
|
fi
|
|
fi
|
|
|
|
chmod -R a+rX /var/log/clickhouse-server 2>/dev/null || true
|
|
|
|
# Preserve exit code of the server
|
|
exit $server_exit_code
|