Files
2023-06-16 19:48:14 -07:00

283 lines
12 KiB
Python

"""Utilities for integration tests of KubeRay."""
import subprocess
import logging
from pathlib import Path
import tempfile
from urllib import request
import yaml
import jsonpatch
import time
from kubernetes import client, config
logger = logging.getLogger(__name__)
logging.basicConfig(
format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
datefmt='%Y-%m-%d:%H:%M:%S',
level=logging.INFO)
class CONST:
"""Constants"""
__slots__ = ()
# Docker images
OPERATOR_IMAGE_KEY = "kuberay-operator-image"
RAY_IMAGE_KEY = "ray-image"
KUBERAY_LATEST_RELEASE = "kuberay/operator:v0.5.0"
# Kubernetes API clients
K8S_CR_CLIENT_KEY = "k8s-cr-api-client"
K8S_V1_CLIENT_KEY = "k8s-v1-api-client"
# Paths
REPO_ROOT = Path(__file__).absolute().parent.parent.parent
HELM_CHART_ROOT = REPO_ROOT.joinpath("helm-chart")
DEFAULT_KIND_CONFIG = REPO_ROOT.joinpath("tests/framework/config/kind-config.yaml")
# Ray features
RAY_FT = "RAY_FT"
RAY_SERVICE = "RAY_SERVICE"
# Custom Resource Definitions
RAY_CLUSTER_CRD = "RayCluster"
RAY_SERVICE_CRD = "RayService"
RAY_JOB_CRD = "RayJob"
CONST = CONST()
class KubernetesClusterManager:
"""
KubernetesClusterManager controlls the lifecycle of KinD cluster and Kubernetes API client.
"""
def __init__(self) -> None:
self.k8s_client_dict = {}
def delete_kind_cluster(self) -> None:
"""Delete a KinD cluster"""
shell_subprocess_run("kind delete cluster")
for _, k8s_client in self.k8s_client_dict.items():
k8s_client.api_client.rest_client.pool_manager.clear()
k8s_client.api_client.close()
self.k8s_client_dict = {}
def create_kind_cluster(self, kind_config = None) -> None:
"""Create a KinD cluster"""
# To use NodePort service, `kind_config` needs to set `extraPortMappings` properly.
kind_config = CONST.DEFAULT_KIND_CONFIG if not kind_config else kind_config
shell_subprocess_run(f"kind create cluster --wait 900s --config {kind_config}")
config.load_kube_config()
self.k8s_client_dict.update({
CONST.K8S_V1_CLIENT_KEY: client.CoreV1Api(),
CONST.K8S_CR_CLIENT_KEY: client.CustomObjectsApi()
})
def check_cluster_exist(self) -> bool:
"""Check whether KinD cluster exists or not"""
return shell_subprocess_run("kubectl cluster-info --context kind-kind", check = False) == 0
K8S_CLUSTER_MANAGER = KubernetesClusterManager()
class OperatorManager:
"""
OperatorManager controlls the lifecycle of KubeRay operator. It will download Docker images,
load images into an existing KinD cluster, and install CRD and KubeRay operator.
Parameters:
docker_image_dict : A dict that includes docker images that need to be
downloaded and loaded to the cluster.
namespace : A namespace(string) that KubeRay operator will be installed in.
patch : A jsonpatch that will be applied to the default KubeRay operator config
to create the custom config.
"""
def __init__(self, docker_image_dict,
namespace = 'default', patch = jsonpatch.JsonPatch([])) -> None:
self.docker_image_dict = docker_image_dict
self.namespace = namespace
self.values_yaml = {}
for key in [CONST.OPERATOR_IMAGE_KEY, CONST.RAY_IMAGE_KEY]:
if key not in self.docker_image_dict:
raise Exception(f"Image {key} does not exist!")
repo, tag = self.docker_image_dict[CONST.OPERATOR_IMAGE_KEY].split(':')
if f"{repo}:{tag}" == CONST.KUBERAY_LATEST_RELEASE:
url = ( "https://github.com/ray-project/kuberay-helm"
f"/raw/kuberay-operator-{tag[1:]}/helm-chart/kuberay-operator/values.yaml"
)
else:
url = "file:///" + str(CONST.HELM_CHART_ROOT.joinpath("kuberay-operator/values.yaml"))
with request.urlopen(url) as base_fd:
self.values_yaml = patch.apply(yaml.safe_load(base_fd))
def prepare_operator(self):
"""Prepare KubeRay operator for an existing KinD cluster"""
self.__kind_prepare_images()
self.__install_crd_and_operator()
def __kind_prepare_images(self):
"""Download images and load images into KinD cluster"""
def download_images():
"""Download Docker images from DockerHub"""
logger.info("Download Docker images: %s", self.docker_image_dict)
for key in self.docker_image_dict:
# Only pull the image from DockerHub when the image does not
# exist in the local docker registry.
image = self.docker_image_dict[key]
if shell_subprocess_run(
f'docker image inspect {image} > /dev/null', check = False) != 0:
shell_subprocess_run(f'docker pull {image}')
else:
logger.info("Image %s exists", image)
download_images()
logger.info("Load images into KinD cluster")
for key in self.docker_image_dict:
image = self.docker_image_dict[key]
shell_subprocess_run(f'kind load docker-image {image}')
def __install_crd_and_operator(self):
"""
Install both CRD and KubeRay operator by kuberay-operator chart.
"""
with tempfile.NamedTemporaryFile('w', suffix = '_values.yaml') as values_fd:
# dump the config to a temporary file and use the file as values.yaml in the chart.
yaml.safe_dump(self.values_yaml, values_fd)
repo, tag = self.docker_image_dict[CONST.OPERATOR_IMAGE_KEY].split(':')
if f"{repo}:{tag}" == CONST.KUBERAY_LATEST_RELEASE:
logger.info("Install both CRD and KubeRay operator with the latest release.")
shell_subprocess_run(
"helm repo add kuberay https://ray-project.github.io/kuberay-helm/"
)
shell_subprocess_run(
f"helm install -n {self.namespace} -f {values_fd.name} kuberay-operator "
f"kuberay/kuberay-operator --version {tag[1:]}"
)
else:
logger.info("Install both CRD and KubeRay operator by kuberay-operator chart")
shell_subprocess_run(
f"helm install -n {self.namespace} -f {values_fd.name} kuberay-operator "
f"{CONST.HELM_CHART_ROOT}/kuberay-operator/ "
f"--set image.repository={repo},image.tag={tag}"
)
def shell_subprocess_run(command, check=True, hide_output=False) -> int:
"""Command will be executed through the shell.
Args:
check: If true, an error will be raised if the returncode is nonzero
hide_output: If true, stdout and stderr of the command will be hidden
Returns:
Return code of the subprocess.
"""
logger.info("Execute command: %s", command)
if hide_output:
return subprocess.run(
command,
shell=True,
check=check,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
).returncode
else:
return subprocess.run(command, shell=True, check=check).returncode
def shell_subprocess_check_output(command):
"""
Run command and return STDOUT as encoded bytes.
"""
logger.info("Execute command (check_output): %s", command)
try:
output = subprocess.check_output(command, shell=True)
logger.info("Output: %s", output)
return output
except subprocess.CalledProcessError as e:
logger.info("Exception: %s", e.output)
raise
def get_pod(namespace, label_selector):
"""Gets pods in the `namespace`. Returns the first pod that has `label_filter`.
Returns None if the number of matches is not equal to 1.
"""
pods = K8S_CLUSTER_MANAGER.k8s_client_dict[CONST.K8S_V1_CLIENT_KEY].list_namespaced_pod(
namespace = namespace, label_selector = label_selector
)
if len(pods.items) != 1:
logger.warning(
"There are %d matches for selector %s in namespace %s, but the expected match is 1.",
len(pods.items), label_selector, namespace
)
return None
return pods.items[0]
def get_head_pod(namespace):
"""Gets a head pod in the `namespace`. Returns None if there are no matches."""
return get_pod(namespace, 'ray.io/node-type=head')
def pod_exec_command(pod_name, namespace, exec_command, check = True):
"""kubectl exec the `exec_command` in the given `pod_name` Pod in the given `namespace`.
Both STDOUT and STDERR of `exec_command` will be printed.
"""
return shell_subprocess_run(f"kubectl exec {pod_name} -n {namespace} -- {exec_command}", check)
def start_curl_pod(name: str, namespace: str, timeout_s: int = -1):
shell_subprocess_run(
f"kubectl run {name} --image=radial/busyboxplus:curl -n {namespace} "
"--command -- /bin/sh -c \"while true; do sleep 10;done\""
)
# Wait for curl pod to be created
k8s_v1_api = K8S_CLUSTER_MANAGER.k8s_client_dict[CONST.K8S_V1_CLIENT_KEY]
start_time = time.time()
while time.time() - start_time < timeout_s or timeout_s < 0:
resp = k8s_v1_api.read_namespaced_pod(name=name, namespace=namespace)
if resp.status.phase == 'Running':
return
raise TimeoutError(f"Curl pod wasn't started in {timeout_s}s.")
def create_custom_object(namespace, cr_object):
"""Create a custom resource based on `cr_object` in the given `namespace`."""
k8s_cr_api = K8S_CLUSTER_MANAGER.k8s_client_dict[CONST.K8S_CR_CLIENT_KEY]
crd = cr_object["kind"]
if crd == CONST.RAY_CLUSTER_CRD:
k8s_cr_api.create_namespaced_custom_object(
group = 'ray.io', version = 'v1alpha1', namespace = namespace,
plural = 'rayclusters', body = cr_object)
elif crd == CONST.RAY_SERVICE_CRD:
k8s_cr_api.create_namespaced_custom_object(
group = 'ray.io', version = 'v1alpha1', namespace = namespace,
plural = 'rayservices', body = cr_object)
elif crd == CONST.RAY_JOB_CRD:
k8s_cr_api.create_namespaced_custom_object(
group = 'ray.io', version = 'v1alpha1', namespace = namespace,
plural = 'rayjobs', body = cr_object)
def delete_custom_object(crd, namespace, cr_name):
"""Delete the given `cr_name` custom resource in the given `namespace`."""
k8s_cr_api = K8S_CLUSTER_MANAGER.k8s_client_dict[CONST.K8S_CR_CLIENT_KEY]
if crd == CONST.RAY_CLUSTER_CRD:
k8s_cr_api.delete_namespaced_custom_object(
group = 'ray.io', version = 'v1alpha1', namespace = namespace,
plural = 'rayclusters', name = cr_name)
elif crd == CONST.RAY_SERVICE_CRD:
k8s_cr_api.delete_namespaced_custom_object(
group = 'ray.io', version = 'v1alpha1', namespace = namespace,
plural = 'rayservices', name = cr_name)
elif crd == CONST.RAY_JOB_CRD:
k8s_cr_api.delete_namespaced_custom_object(
group = 'ray.io', version = 'v1alpha1', namespace = namespace,
plural = 'rayjobs', name = cr_name)
def get_custom_object(crd, namespace, cr_name):
"""Get the given `cr_name` custom resource in the given `namespace`."""
k8s_cr_api = K8S_CLUSTER_MANAGER.k8s_client_dict[CONST.K8S_CR_CLIENT_KEY]
if crd == CONST.RAY_CLUSTER_CRD:
return k8s_cr_api.get_namespaced_custom_object(
group = 'ray.io', version = 'v1alpha1', namespace = namespace,
plural = 'rayclusters', name = cr_name)
elif crd == CONST.RAY_SERVICE_CRD:
return k8s_cr_api.get_namespaced_custom_object(
group = 'ray.io', version = 'v1alpha1', namespace = namespace,
plural = 'rayservices', name = cr_name)
elif crd == CONST.RAY_JOB_CRD:
return k8s_cr_api.get_namespaced_custom_object(
group = 'ray.io', version = 'v1alpha1', namespace = namespace,
plural = 'rayjobs', name = cr_name)