mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
* Replace all hardcoded root temporary paths with path constants Signed-off-by: 400Ping <[email protected]> * refractor constant.go Signed-off-by: 400Ping <[email protected]> * update Signed-off-by: 400Ping <[email protected]> * [Fix] Make Ray tmp root configurable via RAY_TMP_ROOT Signed-off-by: 400Ping <[email protected]> * fix Signed-off-by: 400Ping <[email protected]> * update Signed-off-by: 400Ping <[email protected]> * [historyserver] Keep replay socket paths cluster-side * fix Signed-off-by: 400Ping <[email protected]> * [Fix] Resolve pod tmp paths from pod env in test helpers Signed-off-by: 400Ping <[email protected]> --------- Signed-off-by: 400Ping <[email protected]> Signed-off-by: Jie-Kai Chang <[email protected]>
155 lines
5.8 KiB
Go
155 lines
5.8 KiB
Go
package support
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
|
|
. "github.com/onsi/gomega"
|
|
rayutils "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
|
|
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
|
|
. "github.com/ray-project/kuberay/ray-operator/test/support"
|
|
)
|
|
|
|
const RayClusterManifestPath = "../../config/raycluster.yaml"
|
|
|
|
// ApplyRayClusterWithCollectorWithEnvs deploys a Ray cluster with the collector sidecar into the test namespace,
|
|
// adding the specified environment variables to the head pod.
|
|
func ApplyRayClusterWithCollectorWithEnvs(test Test, g *WithT, namespace *corev1.Namespace, envs map[string]string) *rayv1.RayCluster {
|
|
rayClusterFromYaml := DeserializeRayClusterYAML(test, RayClusterManifestPath)
|
|
rayClusterFromYaml.Namespace = namespace.Name
|
|
|
|
headContainer := &rayClusterFromYaml.Spec.HeadGroupSpec.Template.Spec.Containers[rayutils.RayContainerIndex]
|
|
if len(headContainer.Env) == 0 {
|
|
headContainer.Env = []corev1.EnvVar{}
|
|
}
|
|
|
|
for key, value := range envs {
|
|
env := corev1.EnvVar{
|
|
Name: key,
|
|
Value: value,
|
|
}
|
|
headContainer.Env = append(headContainer.Env, env)
|
|
}
|
|
|
|
// Inject namespace name as the ray-cluster-namespace for head group collector
|
|
injectCollectorRayClusterNamespace(rayClusterFromYaml.Spec.HeadGroupSpec.Template.Spec.Containers, namespace.Name)
|
|
|
|
// Inject namespace name as the ray-cluster-namespace for worker group collectors
|
|
for wg := range rayClusterFromYaml.Spec.WorkerGroupSpecs {
|
|
injectCollectorRayClusterNamespace(rayClusterFromYaml.Spec.WorkerGroupSpecs[wg].Template.Spec.Containers, namespace.Name)
|
|
}
|
|
|
|
rayCluster, err := test.Client().Ray().RayV1().
|
|
RayClusters(namespace.Name).
|
|
Create(test.Ctx(), rayClusterFromYaml, metav1.CreateOptions{})
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
LogWithTimestamp(test.T(), "Created RayCluster %s/%s successfully", rayCluster.Namespace, rayCluster.Name)
|
|
|
|
LogWithTimestamp(test.T(), "Waiting for RayCluster %s/%s to become ready", rayCluster.Namespace, rayCluster.Name)
|
|
g.Eventually(RayCluster(test, rayCluster.Namespace, rayCluster.Name), TestTimeoutLong).
|
|
Should(WithTransform(RayClusterState, Equal(rayv1.Ready)))
|
|
|
|
LogWithTimestamp(test.T(), "Waiting for head pod of RayCluster %s/%s to be running and ready", rayCluster.Namespace, rayCluster.Name)
|
|
g.Eventually(HeadPod(test, rayCluster), TestTimeoutMedium).
|
|
Should(WithTransform(IsPodRunningAndReady, BeTrue()))
|
|
|
|
return rayCluster
|
|
}
|
|
|
|
// injectCollectorRayClusterNamespace injects the ray-cluster-namespace argument into all collector containers.
|
|
func injectCollectorRayClusterNamespace(containers []corev1.Container, rayClusterNamespace string) {
|
|
for i := range containers {
|
|
if containers[i].Name == "collector" {
|
|
containers[i].Command = append(
|
|
containers[i].Command,
|
|
fmt.Sprintf("--ray-cluster-namespace=%s", rayClusterNamespace),
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
// GetSessionIDFromHeadPod retrieves the sessionID from the Ray head pod by reading the symlink
|
|
// /tmp/ray/session_latest and getting its basename.
|
|
func GetSessionIDFromHeadPod(test Test, g *WithT, rayCluster *rayv1.RayCluster) string {
|
|
headPod, err := GetHeadPod(test, rayCluster)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
|
|
getSessionIDCmd := `ray_tmp_root="${RAY_TMP_ROOT:-/tmp/ray}"
|
|
if [ -L "${ray_tmp_root}/session_latest" ]; then
|
|
session_path=$(readlink "${ray_tmp_root}/session_latest")
|
|
basename "$session_path"
|
|
else
|
|
echo "session_latest is not a symlink"
|
|
exit 1
|
|
fi`
|
|
output, _ := ExecPodCmd(test, headPod, "ray-head", []string{"sh", "-c", getSessionIDCmd})
|
|
|
|
// Parse output to extract the sessionID.
|
|
sessionID := strings.TrimSpace(output.String())
|
|
LogWithTimestamp(test.T(), "Retrieved sessionID: %s", sessionID)
|
|
g.Expect(sessionID).NotTo(BeEmpty(), "sessionID should not be empty")
|
|
|
|
return sessionID
|
|
}
|
|
|
|
// GetNodeIDFromHeadPod retrieves the nodeID from the Ray head pod by reading /tmp/ray/raylet_node_id.
|
|
func GetNodeIDFromHeadPod(test Test, g *WithT, rayCluster *rayv1.RayCluster) string {
|
|
headPod, err := GetHeadPod(test, rayCluster)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
|
|
getNodeIDCmd := `ray_tmp_root="${RAY_TMP_ROOT:-/tmp/ray}"
|
|
if [ -f "${ray_tmp_root}/raylet_node_id" ]; then
|
|
cat "${ray_tmp_root}/raylet_node_id"
|
|
else
|
|
echo "raylet_node_id not found"
|
|
exit 1
|
|
fi`
|
|
output, _ := ExecPodCmd(test, headPod, "ray-head", []string{"sh", "-c", getNodeIDCmd})
|
|
|
|
nodeID := strings.TrimSpace(output.String())
|
|
LogWithTimestamp(test.T(), "Retrieved nodeID: %s", nodeID)
|
|
g.Expect(nodeID).NotTo(BeEmpty(), "nodeID should not be empty")
|
|
|
|
return nodeID
|
|
}
|
|
|
|
// GetNodeIDFromPod retrieves the nodeID from the Ray head or worker pod by reading /tmp/ray/raylet_node_id.
|
|
func GetNodeIDFromPod(test Test, g *WithT, getPod func() (*corev1.Pod, error), containerName string) string {
|
|
pod, err := getPod()
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
|
|
getNodeIDCmd := `ray_tmp_root="${RAY_TMP_ROOT:-/tmp/ray}"
|
|
if [ -f "${ray_tmp_root}/raylet_node_id" ]; then
|
|
cat "${ray_tmp_root}/raylet_node_id"
|
|
else
|
|
echo "raylet_node_id not found"
|
|
exit 1
|
|
fi`
|
|
output, _ := ExecPodCmd(test, pod, containerName, []string{"sh", "-c", getNodeIDCmd})
|
|
|
|
// Parse output to extract the nodeID.
|
|
nodeID := strings.TrimSpace(output.String())
|
|
LogWithTimestamp(test.T(), "Retrieved nodeID: %s", nodeID)
|
|
g.Expect(nodeID).NotTo(BeEmpty(), "nodeID should not be empty")
|
|
|
|
return nodeID
|
|
}
|
|
|
|
// FirstWorkerPod returns a function that gets the first worker pod from the Ray cluster.
|
|
// It adapts the WorkerPods function to be used with functions expecting a single pod.
|
|
func FirstWorkerPod(test Test, rayCluster *rayv1.RayCluster) func() (*corev1.Pod, error) {
|
|
return func() (*corev1.Pod, error) {
|
|
workerPods, err := GetWorkerPods(test, rayCluster)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(workerPods) == 0 {
|
|
return nil, fmt.Errorf("no worker pods found")
|
|
}
|
|
return &workerPods[0], nil
|
|
}
|
|
}
|