mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
As part of the 2.9.0 release, this PR updates the latest Ray version used in the KubeRay repo to 2.9.0. Example PR: #1678 We only test the three most recent Ray versions in e2e tests. We unskip a test that was skipped due to [Core][GCS FT] Raylet on the worker node crashes unexpectedly when head crashes ray#41343, which is fixed in Ray 2.9.0. Update the serve e2e test example to remove DAGDriver, which was removed completely in Ray 2.9.0 (Thanks @zcin !) --------- Signed-off-by: Archit Kulkarni <[email protected]> Signed-off-by: cindyz <[email protected]> Co-authored-by: cindyz <[email protected]>
185 lines
7.7 KiB
Python
185 lines
7.7 KiB
Python
import time
|
|
|
|
from python_apiserver_client import *
|
|
|
|
def test_templates():
|
|
apis = KubeRayAPIs(token="12345")
|
|
# create
|
|
toleration = Toleration(key="blah1", operator=TolerationOperation.Exists, effect=TolerationEffect.NoExecute)
|
|
template = Template(name="test-template", namespace="default", cpu=2, memory=8, tolerations=[toleration])
|
|
status, error = apis.create_compute_template(template)
|
|
assert status == 200
|
|
assert error is None
|
|
# duplicate create should fail
|
|
status, error = apis.create_compute_template(template)
|
|
assert status != 200
|
|
assert error is not None
|
|
print(f"\nstatus {status}, error code: {str(error)}")
|
|
# get
|
|
status, error, t = apis.get_compute_template(ns="default", name="test-template")
|
|
assert status == 200
|
|
assert error is None
|
|
assert template.to_string() == t.to_string()
|
|
# list
|
|
status, error, templates = apis.list_compute_templates()
|
|
assert status == 200
|
|
assert error is None
|
|
assert template.to_string() == templates[0].to_string()
|
|
# list ns
|
|
status, error, templates = apis.list_compute_templates_namespace(ns="default")
|
|
assert status == 200
|
|
assert error is None
|
|
assert template.to_string() == templates[0].to_string()
|
|
# delete
|
|
status, error = apis.delete_compute_template(ns="default", name="test-template")
|
|
assert status == 200
|
|
assert error is None
|
|
# duplicate delete should fail
|
|
status, error = apis.delete_compute_template(ns="default", name="test-template")
|
|
assert status != 200
|
|
assert error is not None
|
|
print(f"status: {status}, err = {str(error)}")
|
|
|
|
|
|
def test_cluster():
|
|
apis = KubeRayAPIs(token="12345")
|
|
# Create template first
|
|
template = Template(name="default-template", namespace="default", cpu=2, memory=4)
|
|
status, error = apis.create_compute_template(template)
|
|
assert status == 200
|
|
assert error is None
|
|
# cluster
|
|
volume = ConfigMapVolume(name="code-sample", mount_path="/home/ray/samples", source="ray-job-code-sample",
|
|
items={"sample_code.py": "sample_code.py"})
|
|
environment = EnvironmentVariables(keyvalue={"key": "value"})
|
|
head = HeadNodeSpec(compute_template="default-template",
|
|
ray_start_params={"metrics-export-port": "8080", "num-cpus": "0"},
|
|
image="rayproject/ray:2.9.0-py310", service_type=ServiceType.ClusterIP,
|
|
volumes=[volume], environment=environment)
|
|
worker = WorkerNodeSpec(group_name="small", compute_template="default-template", replicas=1,
|
|
min_replicas=1, max_replicas=1, ray_start_params=DEFAULT_WORKER_START_PARAMS,
|
|
image="rayproject/ray:2.9.0-py310", volumes=[volume], environment=environment)
|
|
cluster = Cluster(name="test", namespace="default", user="boris", version="2.9.0",
|
|
cluster_spec=ClusterSpec(head_node=head, worker_groups=[worker]))
|
|
# create
|
|
status, error = apis.create_cluster(cluster)
|
|
assert status == 200
|
|
assert error is None
|
|
# get
|
|
status, error, c = apis.get_cluster(ns="default", name="test")
|
|
assert status == 200
|
|
assert error is None
|
|
print(f"\ngot cluster: {c.to_string()}")
|
|
# list
|
|
status, error, clusters = apis.list_clusters()
|
|
assert status == 200
|
|
assert error is None
|
|
assert len(clusters) == 1
|
|
print(f"got cluster: {clusters[0].to_string()}")
|
|
# list namespace
|
|
status, error, clusters = apis.list_clusters_namespace(ns="default")
|
|
assert status == 200
|
|
assert error is None
|
|
assert len(clusters) == 1
|
|
print(f"got cluster: {clusters[0].to_string()}")
|
|
# get cluster status
|
|
status, error, cs = apis.get_cluster_status(ns="default", name="test")
|
|
assert status == 200
|
|
assert error is None
|
|
print(f"cluster status is {cs}")
|
|
# Wait for the cluster to get ready
|
|
status, error = apis.wait_cluster_ready(ns="default", name="test")
|
|
assert status == 200
|
|
assert error is None
|
|
# get endpoints
|
|
status, error, endpoint = apis.get_cluster_endpoints(ns="default", name="test")
|
|
assert status == 200
|
|
assert error is None
|
|
print(f"cluster endpoints is {endpoint}")
|
|
# delete cluster
|
|
status, error = apis.delete_cluster(ns="default", name="test")
|
|
assert status == 200
|
|
assert error is None
|
|
# delete template
|
|
status, error = apis.delete_compute_template(ns="default", name="default-template")
|
|
assert status == 200
|
|
assert error is None
|
|
|
|
def test_job_submission():
|
|
apis = KubeRayAPIs()
|
|
# Create template first
|
|
template = Template(name="default-template", namespace="default", cpu=2, memory=4)
|
|
status, error = apis.create_compute_template(template)
|
|
assert status == 200
|
|
assert error is None
|
|
# cluster
|
|
volume = ConfigMapVolume(name="code-sample", mount_path="/home/ray/samples", source="ray-job-code-sample",
|
|
items={"sample_code.py": "sample_code.py"})
|
|
environment = EnvironmentVariables(keyvalue={"key": "value"})
|
|
head = HeadNodeSpec(compute_template="default-template",
|
|
ray_start_params={"metrics-export-port": "8080", "num-cpus": "0"},
|
|
image="rayproject/ray:2.9.0-py310", service_type=ServiceType.ClusterIP,
|
|
volumes=[volume], environment=environment)
|
|
worker = WorkerNodeSpec(group_name="small", compute_template="default-template", replicas=1,
|
|
min_replicas=1, max_replicas=1, ray_start_params=DEFAULT_WORKER_START_PARAMS,
|
|
image="rayproject/ray:2.9.0-py310", volumes=[volume], environment=environment)
|
|
cluster = Cluster(name="test-job", namespace="default", user="boris", version="2.9.0",
|
|
cluster_spec=ClusterSpec(head_node=head, worker_groups=[worker]))
|
|
# create
|
|
status, error = apis.create_cluster(cluster)
|
|
assert status == 200
|
|
assert error is None
|
|
# Wait for the cluster to get ready
|
|
status, error = apis.wait_cluster_ready(ns="default", name="test-job")
|
|
assert status == 200
|
|
assert error is None
|
|
# submit Ray job
|
|
resource_yaml = """
|
|
pip:
|
|
- requests==2.26.0
|
|
- pendulum==2.1.2
|
|
env_vars:
|
|
counter_name: test_counter
|
|
"""
|
|
jobRequest = RayJobRequest(entrypoint="python /home/ray/samples/sample_code.py",
|
|
runtime_env=resource_yaml, num_cpu=.5)
|
|
status, error, sid = apis.submit_job(ns="default", name="test-job", jobrequest=jobRequest)
|
|
assert status == 200
|
|
assert error is None
|
|
# get Ray job info
|
|
status, error, jinfo = apis.get_job_info(ns="default", name="test-job", sid=sid)
|
|
assert status == 200
|
|
assert error is None
|
|
print(f"\njobs info {jinfo.to_string()}")
|
|
# get Ray jobs info
|
|
status, error, jinfos = apis.list_job_info(ns="default", name="test-job")
|
|
assert status == 200
|
|
assert error is None
|
|
print("jobs info")
|
|
for inf in jinfos:
|
|
print(f" {inf.to_string()}")
|
|
# get Ray job log
|
|
time.sleep(5) # wait till log is available
|
|
status, error, jlog = apis.get_job_log(ns="default", name="test-job", sid=sid)
|
|
assert status == 200
|
|
assert error is None
|
|
print(f"job log {jlog}")
|
|
# stop Ray job
|
|
status, error = apis.stop_ray_job(ns="default", name="test-job", sid=sid)
|
|
assert status == 200
|
|
assert error is None
|
|
# delete Ray job
|
|
status, error = apis.delete_ray_job(ns="default", name="test-job", sid=sid)
|
|
assert status == 200
|
|
assert error is None
|
|
# delete cluster
|
|
status, error = apis.delete_cluster(ns="default", name="test-job")
|
|
assert status == 200
|
|
assert error is None
|
|
# delete template
|
|
status, error = apis.delete_compute_template(ns="default", name="default-template")
|
|
assert status == 200
|
|
assert error is None
|
|
|
|
|