Files
Kai-Hsun ChenandGitHub dd7ed902bd [Feature] Refactor test framework & test kuberay-operator chart with configuration framework (#759)
Refactors for integration tests --

Test operator chart: This PR uses the kuberay-operator chart to install KubeRay operator. Hence, the operator chart is tested.

Refactor: class CONST and class KubernetesClusterManager should be singleton classes. However, the singleton design pattern is not encouraged, so we need to consider it thoroughly before we convert these two classes into singleton classes.

Refactor: Replace os with subprocess. The following paragraph is from Python's official documentation.

The subprocess module provides more powerful facilities for spawning new processes and retrieving their results; using that module is preferable to using this function. See the Replacing Older Functions with the subprocess Module section in the subprocess documentation for some helpful recipes.

Skip test_kill_head due to

[Bug] Head pod is deleted rather than restarted when gcs_server on head pod is killed. #638
[Bug] Worker pods crash unexpectedly when gcs_server on head pod is killed  #634.
Refactor: Replace all existing k8s api clients with K8S_CLUSTER_MANAGER.

Refactor and relieve flakiness of test_ray_serve_work

working_dir is out-of-date (See this comment for more details), but the tests pass sometimes due to the error of the original test logic. => Solution: Update working_dir in ray-service.yaml.template.
To elaborate, the error of the test logic mentioned above is that it only checks the exit code rather than STDOUT.
When Pods are READY and RUNNING, RayService still needs tens of seconds to be ready for serving requests. The time.sleep(60) function is a workaround, and should be removed when [RayService] Track whether Serve app is ready before switching clusters #730 is merged.
Remove NodePort service in RayServiceTestCase. Use a curl Pod to communicate with Ray via ClusterIP service directly. Originally, using Docker container with network_mode='host' and NodePort service is very weird for me.
Refactor: remove useless RayService template ray-service-cluster-update.yaml.template and ray-service-serve-update.yaml.template. The original buggy test logic only checks the exit code rather than the STDOUT of the curl commands. Hence, the different templates are useless in RayServiceTestCase.

Refactor: Because APIServer is not tested by any test case, remove everything related to APIServer docker image in the compatibility test.
2022-11-30 17:25:46 -08:00

253 lines
11 KiB
Python

#!/usr/bin/env python
from string import Template
import os
import logging
import tarfile
import time
import tempfile
import yaml
from kubernetes.stream import stream
from framework.prototype import (
CONST,
K8S_CLUSTER_MANAGER,
RayClusterAddCREvent,
RayServiceAddCREvent,
shell_subprocess_run
)
raycluster_service_file = 'tests/config/raycluster-service.yaml'
logger = logging.getLogger(__name__)
logging.basicConfig(
format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
datefmt='%Y-%m-%d:%H:%M:%S',
level=logging.INFO
)
def is_feature_supported(ray_version, feature):
"""Return True if `feature` is supported in `ray_version`"""
if ray_version == "nightly":
return True
major, minor, _ = [int(s) for s in ray_version.split('.')]
if feature in [CONST.RAY_FT, CONST.RAY_SERVICE]:
return major * 100 + minor > 113
return False
def create_ray_cluster(template_name, ray_version, ray_image):
"""Create a RayCluster and a NodePort service."""
context = {}
with open(template_name, encoding="utf-8") as ray_cluster_template:
template = Template(ray_cluster_template.read())
yamlfile = template.substitute(
{'ray_image': ray_image, 'ray_version': ray_version}
)
with tempfile.NamedTemporaryFile('w', delete=False) as ray_cluster_yaml:
ray_cluster_yaml.write(yamlfile)
context['filepath'] = ray_cluster_yaml.name
for k8s_object in yaml.safe_load_all(yamlfile):
if k8s_object['kind'] == 'RayCluster':
context['cr'] = k8s_object
break
try:
# Deploy a NodePort service to expose ports for users.
shell_subprocess_run(f'kubectl apply -f {raycluster_service_file}')
# Create a RayCluster
ray_cluster_add_event = RayClusterAddCREvent(
custom_resource_object = context['cr'],
rulesets = [],
timeout = 90,
namespace='default',
filepath = context['filepath']
)
ray_cluster_add_event.trigger()
return ray_cluster_add_event
except Exception as ex:
logger.error(f"RayClusterAddCREvent fails to converge: {str(ex)}")
raise Exception("create_ray_cluster fails")
def create_ray_service(template_name, ray_version, ray_image):
"""Create a RayService without a NodePort service."""
context = {}
with open(template_name, encoding="utf-8") as ray_service_template:
template = Template(ray_service_template.read())
yamlfile = template.substitute(
{'ray_image': ray_image, 'ray_version': ray_version}
)
with tempfile.NamedTemporaryFile('w', delete=False) as ray_service_yaml:
ray_service_yaml.write(yamlfile)
context['filepath'] = ray_service_yaml.name
for k8s_object in yaml.safe_load_all(yamlfile):
if k8s_object['kind'] == 'RayService':
context['cr'] = k8s_object
break
try:
# Create a RayService
ray_service_add_event = RayServiceAddCREvent(
custom_resource_object = context['cr'],
rulesets = [],
timeout = 90,
namespace='default',
filepath = context['filepath']
)
ray_service_add_event.trigger()
return ray_service_add_event
except Exception as ex:
logger.error(f"RayServiceAddCREvent fails to converge: {str(ex)}")
raise Exception("create_ray_service fails")
def copy_to_container(container, src, dest, filename):
oldpwd = os.getcwd()
try:
os.chdir(src)
with tempfile.NamedTemporaryFile(suffix='.tar') as tf:
with tarfile.open(fileobj=tf, mode='w') as tar:
try:
tar.add(filename)
finally:
tar.close()
with open(tf.name, 'rb') as data:
container.put_archive(dest, data.read())
finally:
os.chdir(oldpwd)
def exec_run_container(container, cmd, timeout_sec, silent = False):
"""Executes the command `cmd` in `container`, and logs the output if `silent` is False."""
timeout_cmd = 'timeout {}s {}'.format(timeout_sec, cmd)
# If the exit_code is 124, 125, 126, 127, 137, it is related to the `timeout` command.
# See https://manpages.courier-mta.org/htmlman1/timeout.1.html for more details.
exit_code, output = container.exec_run(cmd = timeout_cmd)
if not silent:
logger.info(f"cmd: {timeout_cmd}")
logger.info(f"exit_code: {exit_code}")
logger.info(f"output: {output.decode()}")
return exit_code, output.decode()
def wait_for_condition(
condition_predictor, timeout=10, retry_interval_ms=100, **kwargs
):
"""Wait until a condition is met or time out with an exception.
Args:
condition_predictor: A function that predicts the condition.
timeout: Maximum timeout in seconds.
retry_interval_ms: Retry interval in milliseconds.
Raises:
RuntimeError: If the condition is not met before the timeout expires.
"""
start = time.time()
last_ex = None
while time.time() - start <= timeout:
try:
if condition_predictor(**kwargs):
return
except Exception as ex:
last_ex = ex
time.sleep(retry_interval_ms / 1000.0)
message = "The condition wasn't met before the timeout expired."
if last_ex is not None:
message += f" Last exception: {last_ex}"
raise RuntimeError(message)
def get_pod(namespace, label_selector):
return K8S_CLUSTER_MANAGER.k8s_client_dict[CONST.K8S_V1_CLIENT_KEY].list_namespaced_pod(
namespace = namespace, label_selector = label_selector
)
def pod_exec_command(pod_name, namespace, exec_command, stderr=True, stdin=False, stdout=True, tty=False, silent=False):
exec_command = ['/bin/sh', '-c'] + exec_command
k8s_v1_api = K8S_CLUSTER_MANAGER.k8s_client_dict[CONST.K8S_V1_CLIENT_KEY]
resp = stream(k8s_v1_api.connect_get_namespaced_pod_exec,
pod_name,
namespace,
command=exec_command,
stderr=stderr, stdin=stdin,
stdout=stdout, tty=tty)
if not silent:
logger.info(f"cmd: {exec_command}")
logger.info(f"response: {resp}")
return resp
def wait_for_new_head(old_head_pod_name, old_restart_count, namespace, timeout, retry_interval_ms):
"""
`wait_for_new_head` is used to wait for new head is ready and running. For example, `test_detached_actor` kills
the gcs_server process on the head pod. It takes nearly 1 min to kill the head pod, and the head pod will still
be in 'Running' and 'Ready' in that minute.
Hence, we need to check `restart_count` or `new_head_pod_name`.
(1) `restart_count`: If the pod is restarted by the restartPolicy of a Pod, `restart_count` will increase by 1.
If the pod is deleted by KubeRay and the reconciler creates a new one, `restart_count` will be 0.
(2) `new_head_pod_name`: If the reconciler creates a new head pod, `new_head_pod_name` will be different from
`old_head_pod_name`.
Next, we check the status of pods to ensure all pods should be "Running" and "Ready".
After the cluster state converges (all pods are "Running" and "Ready"), ray processes still need tens of seconds to
become ready to serve new connections from ray clients. So, users need to retry until a Ray client connection succeeds.
Args:
old_head_pod_name: Name of the old head pod.
old_restart_count: If the Pod is restarted by Kubernetes Pod RestartPolicy, the restart_count will increase by 1.
namespace: Namespace that the head pod is running in.
timeout: Same as `wait_for_condition`.
retry_interval_ms: Same as `wait_for_condition`.
Raises:
RuntimeError: If the condition is not met before the timeout expires, raise the RuntimeError.
"""
k8s_v1_api = K8S_CLUSTER_MANAGER.k8s_client_dict[CONST.K8S_V1_CLIENT_KEY]
def check_status(old_head_pod_name, old_restart_count, namespace) -> bool:
all_pods = k8s_v1_api.list_namespaced_pod(namespace = namespace)
headpods = get_pod(namespace=namespace, label_selector='ray.io/node-type=head')
# KubeRay only allows at most 1 head pod per RayCluster instance at the same time. On the other
# hands, when we kill a worker, the operator will reconcile a new one immediately without waiting
# for the Pod termination to complete. Hence, it is possible to have more than `worker.Replicas`
# worker pods in the cluster.
if len(headpods.items) != 1:
logger.info('Number of headpods is not equal to 1.')
return False
new_head_pod = headpods.items[0]
new_head_pod_name = new_head_pod.metadata.name
new_restart_count = new_head_pod.status.container_statuses[0].restart_count
# The default container restartPolicy of a Pod is `Always`. Hence, when GCS server is killed,
# the head pod will restart the old one rather than create a new one.
if new_head_pod_name != old_head_pod_name:
logger.info(f'If GCS server is killed, the head pod will restart the old one rather than create a new one.' +
f' new_head_pod_name: {new_head_pod_name}, old_head_pod_name: {old_head_pod_name}')
# TODO (kevin85421): We should `return False` here, but currently ray:nightly has a high possibility to create
# a new head pod instead of restarting the old one.
# When GCS server is killed, it takes nearly 1 min to kill the head pod. In the minute, the head
# pod will still be in 'Running' and 'Ready'. Hence, we need to check `restart_count`.
else:
# TODO (kevin85421): We should remove `else` in the future. Currently, ray:nightly has a high possibility to
# create a new head pod instead of restarting the old one. The new pod's `restart_count`
# is 0.
if new_restart_count != old_restart_count + 1:
logger.info(f'new_restart_count != old_restart_count + 1 => new_restart_count: {new_restart_count},' +
f' old_restart_count: {old_restart_count}')
return False
# All pods should be "Running" and "Ready". This check is an overkill. We added this check due to
# the buggy behaviors of Ray HA. To elaborate, when GCS server is killed, the head pod should restart,
# but worker pods should not. However, currently, worker pods will also restart.
# See https://github.com/ray-project/kuberay/issues/634 for more details.
for pod in all_pods.items:
if pod.status.phase != 'Running':
logger.info(f'Pod {pod.metadata.name} is not Running.')
return False
for c in pod.status.container_statuses:
if not c.ready:
logger.info(f'Container {c.name} in {pod.metadata.name} is not ready.')
return False
return True
wait_for_condition(check_status, timeout=timeout, retry_interval_ms=retry_interval_ms,
old_head_pod_name=old_head_pod_name, old_restart_count=old_restart_count, namespace=namespace)
# After the cluster state converges, ray processes still need tens of seconds to become ready.
# TODO (kevin85421): Make ray processes become ready when pods are "Ready" and "Running".