mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
* docs: Add step-by-step process of the basic e2e
Signed-off-by: JiangJiaWei1103 <[email protected]>
* chore: Align test infra setup order with Ray docs
Signed-off-by: JiangJiaWei1103 <[email protected]>
* test: Add locust load test for incr upgrade e2e
Signed-off-by: JiangJiaWei1103 <[email protected]>
* docs: Improve maintainability of locust yaml
Signed-off-by: JiangJiaWei1103 <[email protected]>
* refactor: Remove redundant helper
Signed-off-by: JiangJiaWei1103 <[email protected]>
* fix: Deflake hardcoded sleep for Locust ramp up
Signed-off-by: JiangJiaWei1103 <[email protected]>
* refactor: Remove legacy and extract a helper to get rps index
Signed-off-by: JiangJiaWei1103 <[email protected]>
* test: Ensure remaining traffic routed to the new cluster
Signed-off-by: JiangJiaWei1103 <[email protected]>
* test: Support high-rps serve application with expected rps over 900
Signed-off-by: JiangJiaWei1103 <[email protected]>
* Test CI
Signed-off-by: JiangJiaWei1103 <[email protected]>
* Test CI
Signed-off-by: JiangJiaWei1103 <[email protected]>
* Test CI
Signed-off-by: JiangJiaWei1103 <[email protected]>
* test: Recover basic incr upgrade test
Signed-off-by: JiangJiaWei1103 <[email protected]>
* docs: Improve maintainability
Signed-off-by: JiangJiaWei1103 <[email protected]>
* fix: Deflake CI istio gc installation
Signed-off-by: JiangJiaWei1103 <[email protected]>
* fix: Deflake istio gc installation
Signed-off-by: JiangJiaWei1103 <[email protected]>
* revert: Use orig install order
Signed-off-by: JiangJiaWei1103 <[email protected]>
* test: Support diverse incr upgrade parameter combinations
Signed-off-by: JiangJiaWei1103 <[email protected]>
* Test CI
Signed-off-by: JiangJiaWei1103 <[email protected]>
* fix: Skip transient state check right before promotion
Signed-off-by: JiangJiaWei1103 <[email protected]>
* test: Retest standard gradual incr upgrade
Signed-off-by: JiangJiaWei1103 <[email protected]>
* refactor: Make curl function clearer
Signed-off-by: JiangJiaWei1103 <[email protected]>
* fix: Avoid t.FailNow from non-test goroutines
Signed-off-by: JiangJiaWei1103 <[email protected]>
* fix: Deflake by using commit hash
Signed-off-by: JiangJiaWei1103 <[email protected]>
* refactor: Remove redundant checks
Signed-off-by: JiangJiaWei1103 <[email protected]>
* refactor: Get rps col index without hardcoded int
Signed-off-by: JiangJiaWei1103 <[email protected]>
* refactor: Extract locust warmup constants for tweaking
Signed-off-by: JiangJiaWei1103 <[email protected]>
* Remove redundant line
Signed-off-by: JiangJiaWei1103 <[email protected]>
* test: Split test responsibilities
Signed-off-by: JiangJiaWei1103 <[email protected]>
* refactor: Use eg instead of wg
Signed-off-by: JiangJiaWei1103 <[email protected]>
* refactor: Extract trigger incr upgrade helper and curl const
Signed-off-by: JiangJiaWei1103 <[email protected]>
* fix: Deflake waiting for upgrade complete using a longer timeout
Signed-off-by: JiangJiaWei1103 <[email protected]>
* Improve readability
Signed-off-by: JiangJiaWei1103 <[email protected]>
* fix: Remove data race on err btw goroutines
Signed-off-by: JiangJiaWei1103 <[email protected]>
* fix: Fix last migrate time check
Signed-off-by: JiangJiaWei1103 <[email protected]>
* Fix merge and update rollback test for changes in this PR
Signed-off-by: Ryan O'Leary <[email protected]>
* chore: Use a larger runner for higher RPS
Signed-off-by: JiangJiaWei1103 <[email protected]>
* Revert "chore: Use a larger runner for higher RPS"
This reverts commit 067ba1fb3cbf2f7a0cea5e9a2cd129d60779b3dc.
* Revert "[RayService] Rollback Support for Incremental Upgrades (#4109)"
This reverts commit b8b66d42a5.
* codex issue
Signed-off-by: Future-Outlier <[email protected]>
* chore: Deflake by increasing timeout
Signed-off-by: JiangJiaWei1103 <[email protected]>
* chore: Remove worker rsc limits to align with the orig example
Signed-off-by: JiangJiaWei1103 <[email protected]>
* Revert "chore: Remove worker rsc limits to align with the orig example"
This reverts commit 7e821f36dbc922b32dfa23aa171b0e276e29f98f.
* docs: Better strategy naming
Signed-off-by: JiangJiaWei1103 <[email protected]>
* Revert "Revert: Remove rollback e2e"
This reverts commit baf2d6c735ee33e71702b4b9a5a2e70947afabe5, reversing
changes made to 73e66376807d13aaa316be7ac22a7a176f27d285.
* chore: Use Ray project serve config example link
Signed-off-by: JiangJiaWei1103 <[email protected]>
* chore: Deflake CI by lowering steady state RPS
Signed-off-by: JiangJiaWei1103 <[email protected]>
* fix: Make log msg accurate
Co-authored-by: Jun-Hao Wan <[email protected]>
Signed-off-by: Jia-Wei Jiang <[email protected]>
* fix: Make log msg more accurate
Signed-off-by: JiangJiaWei1103 <[email protected]>
* chore: Decrease replicas to align with buildkite runner vcpu
Signed-off-by: JiangJiaWei1103 <[email protected]>
---------
Signed-off-by: JiangJiaWei1103 <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>
Signed-off-by: Future-Outlier <[email protected]>
Signed-off-by: Jia-Wei Jiang <[email protected]>
Co-authored-by: Ryan O'Leary <[email protected]>
Co-authored-by: Ryan O'Leary <[email protected]>
Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Co-authored-by: Jun-Hao Wan <[email protected]>
134 lines
3.5 KiB
Python
134 lines
3.5 KiB
Python
"""
|
|
This script is based on: https://raw.githubusercontent.com/ray-project/serve_workloads/main/microbenchmarks/locust_runner.py
|
|
|
|
Run Locust on Ray cluster.
|
|
|
|
Run this script on a Ray cluster's head node to launch one Locust worker per
|
|
CPU across the Ray cluster's nodes.
|
|
|
|
Example command:
|
|
|
|
$ python locust_runner.py -f locustfile.py -u 200 -r 50 --host [HOST_URL]
|
|
"""
|
|
|
|
import os
|
|
import ray
|
|
import json
|
|
import time
|
|
import argparse
|
|
import sys
|
|
import subprocess
|
|
from tqdm import tqdm
|
|
|
|
|
|
ray.init()
|
|
|
|
HTML_RESULTS_DIR = os.environ.get("HTML_RESULTS_DIR", "locust_results")
|
|
DEFAULT_RESULT_FILENAME = \
|
|
f"{time.strftime('%Y-%m-%d-%p-%H-%M-%S-results.html')}"
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"--html",
|
|
default=DEFAULT_RESULT_FILENAME,
|
|
type=str,
|
|
help="HTML file to save results to.",
|
|
)
|
|
parser.add_argument(
|
|
"-t",
|
|
"--run-time",
|
|
default=None,
|
|
type=str,
|
|
help="Test duration. Same option as Locust's --run-time.",
|
|
)
|
|
|
|
args, locust_args = parser.parse_known_args()
|
|
|
|
num_locust_workers = int(ray.available_resources()["CPU"])
|
|
master_address = ray.util.get_node_ip_address()
|
|
|
|
if not os.path.exists(HTML_RESULTS_DIR):
|
|
os.mkdir(HTML_RESULTS_DIR)
|
|
|
|
# Required locust args: -f, -u, -r, --host, and any custom locustfile args
|
|
base_locust_cmd = [
|
|
"locust",
|
|
"--headless",
|
|
f"--html={HTML_RESULTS_DIR}/{args.html}",
|
|
f"--csv={HTML_RESULTS_DIR}/test",
|
|
*locust_args,
|
|
]
|
|
|
|
|
|
@ray.remote(num_cpus=1)
|
|
class LocustWorker:
|
|
def __init__(self):
|
|
self.proc = None
|
|
|
|
def start(self):
|
|
worker_locust_cmd = base_locust_cmd + [
|
|
"--worker",
|
|
f"--master-host={master_address}",
|
|
]
|
|
self.proc = subprocess.Popen(worker_locust_cmd)
|
|
|
|
|
|
print(f"Spawning {num_locust_workers} Locust worker Ray tasks.")
|
|
|
|
# Hold reference to each locust worker to prevent them from being torn down
|
|
locust_workers = []
|
|
start_refs = []
|
|
for _ in tqdm(range(num_locust_workers)):
|
|
locust_worker = LocustWorker.remote()
|
|
locust_workers.append(locust_worker)
|
|
start_refs.append(locust_worker.start.remote())
|
|
|
|
print("Waiting for Locust worker processes to start.")
|
|
|
|
|
|
def wait_for_locust_workers(start_refs):
|
|
"""Generator that yields whenever a worker process starts.
|
|
|
|
Use with tqdm to track how many workers have started. If you don't need
|
|
tqdm, use ray.get(start_refs) instead of calling this function.
|
|
"""
|
|
|
|
remaining_start_refs = start_refs
|
|
while remaining_start_refs:
|
|
finished_start_refs, remaining_start_refs = \
|
|
ray.wait(remaining_start_refs)
|
|
for ref in finished_start_refs:
|
|
yield ray.get(ref)
|
|
|
|
|
|
# No-op for-loop to let tqdm track wait_for_locust_workers() progress
|
|
for _ in tqdm(wait_for_locust_workers(start_refs), total=num_locust_workers):
|
|
pass
|
|
|
|
master_locust_cmd = base_locust_cmd + [
|
|
"--master",
|
|
f"--expect-workers={num_locust_workers}",
|
|
"--json",
|
|
]
|
|
print(f"Locust command: {master_locust_cmd}")
|
|
|
|
if args.run_time is not None:
|
|
master_locust_cmd += [f"--run-time={args.run_time}"]
|
|
proc = subprocess.Popen(master_locust_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
stdout, stderr = proc.communicate()
|
|
|
|
print("STDOUT:", stdout.decode())
|
|
print("STDERR:", stderr.decode())
|
|
|
|
data = json.loads(stdout.decode())
|
|
assert len(data) == 1, f"data_len: {len(data)}"
|
|
|
|
num_failures = data[0]["num_failures"]
|
|
num_requests = data[0]["num_requests"]
|
|
|
|
assert num_failures == 0, f"num_failures: {num_failures}"
|
|
assert num_requests != 0, f"num_requests: {num_requests}"
|
|
|
|
print("returncode:", proc.returncode)
|
|
sys.exit(proc.returncode)
|