Files
kuberay/tests/compatibility-test.py

256 lines
11 KiB
Python
Executable File

#!/usr/bin/env python
import logging
import unittest
import os
import random
import string
import kuberay_utils.utils as utils
from framework.prototype import (
EasyJobRule,
show_cluster_info
)
from framework.utils import (
get_head_pod,
pod_exec_command,
shell_subprocess_run,
CONST,
K8S_CLUSTER_MANAGER,
OperatorManager
)
logger = logging.getLogger(__name__)
logging.basicConfig(
format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
datefmt='%Y-%m-%d:%H:%M:%S',
level=logging.INFO
)
# Default Ray version
ray_version = '2.7.0'
class BasicRayTestCase(unittest.TestCase):
"""Test the basic functionalities of RayCluster by executing simple jobs."""
cluster_template = CONST.REPO_ROOT.joinpath("tests/config/ray-cluster.mini.yaml.template")
ray_cluster_ns = "default"
@classmethod
def setUpClass(cls):
"""Create a Kind cluster, a KubeRay operator, and a RayCluster."""
K8S_CLUSTER_MANAGER.cleanup()
K8S_CLUSTER_MANAGER.initialize_cluster()
operator_manager = OperatorManager.instance()
operator_manager.prepare_operator()
utils.create_ray_cluster(BasicRayTestCase.cluster_template, ray_version, ray_image)
def test_simple_code(self):
"""
Run a simple example in the head Pod to test the basic functionality of the Ray cluster.
The example is from https://docs.ray.io/en/latest/ray-core/walkthrough.html#running-a-task.
"""
headpod = get_head_pod(BasicRayTestCase.ray_cluster_ns)
headpod_name = headpod.metadata.name
pod_exec_command(
headpod_name, BasicRayTestCase.ray_cluster_ns, "python samples/simple_code.py")
def test_cluster_info(self):
"""Execute "print(ray.cluster_resources())" in the head Pod."""
EasyJobRule().assert_rule()
class RayFTTestCase(unittest.TestCase):
"""Test Ray GCS Fault Tolerance"""
cluster_template = CONST.REPO_ROOT.joinpath("tests/config/ray-cluster.ray-ft.yaml.template")
ray_cluster_ns = "default"
@classmethod
def setUpClass(cls):
if not utils.is_feature_supported(ray_version, CONST.RAY_FT):
raise unittest.SkipTest(f"{CONST.RAY_FT} is not supported")
K8S_CLUSTER_MANAGER.cleanup()
K8S_CLUSTER_MANAGER.initialize_cluster()
operator_manager = OperatorManager.instance()
operator_manager.prepare_operator()
utils.create_ray_cluster(RayFTTestCase.cluster_template, ray_version, ray_image)
def test_ray_serve(self):
"""Kill GCS process on the head Pod and then test a deployed Ray Serve model."""
if not utils.is_feature_supported(ray_version, CONST.RAY_SERVE_FT):
raise unittest.SkipTest(
"This test is flaky before Ray 2.6.0, so we only run it for 2.6.0 and later."
)
headpod = get_head_pod(RayFTTestCase.ray_cluster_ns)
headpod_name = headpod.metadata.name
# In `test_detached_actor`, we create 1 head Pod and 1 worker Pod. Afterward, we kill the
# GCS process of the head Pod to trigger its restart. Next, we terminate the head Pod,
# and KubeRay will create a new one in its place. However, Ray may take several seconds to
# realize that the old head Pod is gone. Therefore, using `ray list nodes` might show more
# than 1 "ALIVE" head nodes in the cluster temporarily. This may lead to an issue where the
# Serve controller believes it hasn't been scheduled to the head node, and as a result, it
# raises an exception. To avoid this issue, we will add a retry logic in `test_ray_serve_1`
# to wait until only 1 head node is alive. `ray list nodes` is for debugging purpose only.
pod_exec_command(headpod_name, RayFTTestCase.ray_cluster_ns,
"ray list nodes",
check = False
)
# Deploy a Ray Serve model.
exit_code = pod_exec_command(headpod_name, RayFTTestCase.ray_cluster_ns,
"python samples/test_ray_serve_1.py",
check = False
)
if exit_code != 0:
show_cluster_info(RayFTTestCase.ray_cluster_ns)
self.fail(f"Fail to execute test_ray_serve_1.py. The exit code is {exit_code}.")
old_head_pod = get_head_pod(RayFTTestCase.ray_cluster_ns)
old_head_pod_name = old_head_pod.metadata.name
restart_count = old_head_pod.status.container_statuses[0].restart_count
# Kill the gcs_server process on head node. The head node will crash after 20 seconds
# because the value of `RAY_gcs_rpc_server_reconnect_timeout_s` is "20" in the
# `ray-cluster.ray-ft.yaml.template` file.
pod_exec_command(old_head_pod_name, RayFTTestCase.ray_cluster_ns, "pkill gcs_server")
# Waiting for all pods become ready and running.
utils.wait_for_new_head(CONST.RESTART_OLD_POD, old_head_pod_name, restart_count,
RayFTTestCase.ray_cluster_ns, timeout=300, retry_interval_ms=1000)
# Try to connect to the deployed model again
headpod = get_head_pod(RayFTTestCase.ray_cluster_ns)
headpod_name = headpod.metadata.name
exit_code = pod_exec_command(headpod_name, RayFTTestCase.ray_cluster_ns,
"python samples/test_ray_serve_2.py",
check = False
)
if exit_code != 0:
show_cluster_info(RayFTTestCase.ray_cluster_ns)
self.fail(f"Fail to execute test_ray_serve_2.py. The exit code is {exit_code}.")
def test_detached_actor(self):
"""Kill GCS process on the head Pod and then test a detached actor."""
headpod = get_head_pod(RayFTTestCase.ray_cluster_ns)
headpod_name = headpod.metadata.name
# RAY_NAMESPACE is an abstraction in Ray. It is not a Kubernetes namespace.
ray_namespace = ''.join(random.choices(string.ascii_lowercase, k=10))
logger.info('Ray namespace: %s', ray_namespace)
# Register a detached actor
exit_code = pod_exec_command(headpod_name, RayFTTestCase.ray_cluster_ns,
f" python samples/test_detached_actor_1.py {ray_namespace}",
check = False
)
if exit_code != 0:
show_cluster_info(RayFTTestCase.ray_cluster_ns)
self.fail(f"Fail to execute test_detached_actor_1.py. The exit code is {exit_code}.")
old_head_pod = get_head_pod(RayFTTestCase.ray_cluster_ns)
old_head_pod_name = old_head_pod.metadata.name
restart_count = old_head_pod.status.container_statuses[0].restart_count
# [Test 1: Kill GCS process to "restart" the head Pod]
# Kill the gcs_server process on head node. The head node will crash after 20 seconds
# because the value of `RAY_gcs_rpc_server_reconnect_timeout_s` is "20" in the
# `ray-cluster.ray-ft.yaml.template` file.
pod_exec_command(old_head_pod_name, RayFTTestCase.ray_cluster_ns, "pkill gcs_server")
# Waiting for all pods become ready and running.
utils.wait_for_new_head(CONST.RESTART_OLD_POD, old_head_pod_name, restart_count,
RayFTTestCase.ray_cluster_ns, timeout=300, retry_interval_ms=1000)
# Try to connect to the detached actor again.
# [Note] When all pods become running and ready, the RayCluster still needs tens of seconds
# to relaunch actors. Hence, `test_detached_actor_2.py` will retry until a Ray client
# connection succeeds.
headpod = get_head_pod(RayFTTestCase.ray_cluster_ns)
headpod_name = headpod.metadata.name
expected_output = 3
exit_code = pod_exec_command(headpod_name, RayFTTestCase.ray_cluster_ns,
f" python samples/test_detached_actor_2.py {ray_namespace} {expected_output}",
check = False
)
if exit_code != 0:
show_cluster_info(RayFTTestCase.ray_cluster_ns)
self.fail(f"Fail to execute test_detached_actor_2.py. The exit code is {exit_code}.")
# [Test 2: Delete the head Pod and wait for a new head Pod]
# Delete the head Pod. The `kubectl delete pod` command has a default flag `--wait=true`,
# which waits for resources to be gone before returning.
shell_subprocess_run(
f'kubectl delete pod {headpod_name} -n {RayFTTestCase.ray_cluster_ns}')
restart_count = headpod.status.container_statuses[0].restart_count
# Waiting for all pods become ready and running.
utils.wait_for_new_head(CONST.CREATE_NEW_POD, headpod_name, restart_count,
RayFTTestCase.ray_cluster_ns, timeout=300, retry_interval_ms=1000)
# Try to connect to the detached actor again.
headpod = get_head_pod(RayFTTestCase.ray_cluster_ns)
headpod_name = headpod.metadata.name
expected_output = 4
exit_code = pod_exec_command(headpod_name, RayFTTestCase.ray_cluster_ns,
f" python samples/test_detached_actor_2.py {ray_namespace} {expected_output}",
check = False
)
if exit_code != 0:
show_cluster_info(RayFTTestCase.ray_cluster_ns)
self.fail(f"Fail to execute test_detached_actor_2.py. The exit code is {exit_code}.")
class KubeRayHealthCheckTestCase(unittest.TestCase):
"""Test KubeRay health check"""
cluster_template = CONST.REPO_ROOT.joinpath("tests/config/ray-cluster.sidecar.yaml.template")
ray_cluster_ns = "default"
@classmethod
def setUpClass(cls):
K8S_CLUSTER_MANAGER.cleanup()
K8S_CLUSTER_MANAGER.initialize_cluster()
operator_manager = OperatorManager.instance()
operator_manager.prepare_operator()
utils.create_ray_cluster(
KubeRayHealthCheckTestCase.cluster_template, ray_version, ray_image)
def test_terminated_raycontainer(self):
"""
KubeRay should delete a Pod if its restart policy is "Never" and the Ray container is
terminated no matter whether the Pod is in the "Running" state.
"""
old_head_pod = get_head_pod(KubeRayHealthCheckTestCase.ray_cluster_ns)
old_head_pod_name = old_head_pod.metadata.name
restart_count = old_head_pod.status.container_statuses[0].restart_count
# After the Ray container is terminated by `pkill ray`, the head Pod will still be in the
# "Running" state because there is still a sidecar container running in the Pod. KubeRay
# should delete the Pod and create a new one.
pod_exec_command(old_head_pod_name, KubeRayHealthCheckTestCase.ray_cluster_ns, "pkill ray")
# Set the mode to `CONST.CREATE_NEW_POD` to wait for a new head Pod
# rather than restarting the old head Pod.
utils.wait_for_new_head(CONST.CREATE_NEW_POD, old_head_pod_name, restart_count,
RayFTTestCase.ray_cluster_ns, timeout=300, retry_interval_ms=1000)
def parse_environment():
global ray_version, ray_image, kuberay_operator_image
for k, v in os.environ.items():
if k == 'RAY_IMAGE':
ray_image = v
ray_version = ray_image.split(':')[-1]
elif k == 'OPERATOR_IMAGE':
kuberay_operator_image = v
if __name__ == '__main__':
parse_environment()
logger.info('Setting Ray image to: {}'.format(ray_image))
logger.info('Setting Ray version to: {}'.format(ray_version))
logger.info('Setting KubeRay operator image to: {}'.format(kuberay_operator_image))
unittest.main(verbosity=2)