mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
2569 lines
99 KiB
Go
2569 lines
99 KiB
Go
package common
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"maps"
|
||
"os"
|
||
"reflect"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"testing"
|
||
|
||
"github.com/stretchr/testify/assert"
|
||
"github.com/stretchr/testify/require"
|
||
corev1 "k8s.io/api/core/v1"
|
||
"k8s.io/apimachinery/pkg/api/resource"
|
||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||
"k8s.io/apimachinery/pkg/util/intstr"
|
||
"k8s.io/utils/ptr"
|
||
|
||
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
|
||
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
|
||
"github.com/ray-project/kuberay/ray-operator/pkg/features"
|
||
)
|
||
|
||
var testMemoryLimit = resource.MustParse("1Gi")
|
||
|
||
var instance = rayv1.RayCluster{
|
||
ObjectMeta: metav1.ObjectMeta{
|
||
Name: "raycluster-sample",
|
||
Namespace: "default",
|
||
},
|
||
Spec: rayv1.RayClusterSpec{
|
||
HeadGroupSpec: rayv1.HeadGroupSpec{
|
||
RayStartParams: map[string]string{},
|
||
Template: corev1.PodTemplateSpec{
|
||
ObjectMeta: metav1.ObjectMeta{
|
||
Namespace: "default",
|
||
},
|
||
Spec: corev1.PodSpec{
|
||
Containers: []corev1.Container{
|
||
{
|
||
Name: "ray-head",
|
||
Image: "repo/image:custom",
|
||
Env: []corev1.EnvVar{
|
||
{
|
||
Name: "TEST_ENV_NAME",
|
||
Value: "TEST_ENV_VALUE",
|
||
},
|
||
},
|
||
Resources: corev1.ResourceRequirements{
|
||
Requests: corev1.ResourceList{
|
||
corev1.ResourceCPU: resource.MustParse("1"),
|
||
corev1.ResourceMemory: testMemoryLimit,
|
||
},
|
||
Limits: corev1.ResourceList{
|
||
corev1.ResourceCPU: resource.MustParse("1"),
|
||
corev1.ResourceMemory: testMemoryLimit,
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
WorkerGroupSpecs: []rayv1.WorkerGroupSpec{
|
||
{
|
||
Replicas: ptr.To[int32](3),
|
||
MinReplicas: ptr.To[int32](0),
|
||
MaxReplicas: ptr.To[int32](10000),
|
||
GroupName: "small-group",
|
||
RayStartParams: map[string]string{
|
||
"port": "6379",
|
||
},
|
||
Template: corev1.PodTemplateSpec{
|
||
ObjectMeta: metav1.ObjectMeta{
|
||
Namespace: "default",
|
||
},
|
||
Spec: corev1.PodSpec{
|
||
Containers: []corev1.Container{
|
||
{
|
||
Name: "ray-worker",
|
||
Image: "repo/image:custom",
|
||
Resources: corev1.ResourceRequirements{
|
||
Limits: corev1.ResourceList{
|
||
corev1.ResourceCPU: resource.MustParse("1"),
|
||
corev1.ResourceMemory: testMemoryLimit,
|
||
"nvidia.com/gpu": resource.MustParse("3"),
|
||
},
|
||
},
|
||
Env: []corev1.EnvVar{
|
||
{
|
||
Name: "TEST_ENV_NAME",
|
||
Value: "TEST_ENV_VALUE",
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
}
|
||
|
||
var volumesNoAutoscaler = []corev1.Volume{
|
||
{
|
||
Name: "shared-mem",
|
||
VolumeSource: corev1.VolumeSource{
|
||
EmptyDir: &corev1.EmptyDirVolumeSource{
|
||
Medium: corev1.StorageMediumMemory,
|
||
SizeLimit: &testMemoryLimit,
|
||
},
|
||
},
|
||
},
|
||
}
|
||
|
||
var volumesWithAutoscaler = []corev1.Volume{
|
||
{
|
||
Name: "shared-mem",
|
||
VolumeSource: corev1.VolumeSource{
|
||
EmptyDir: &corev1.EmptyDirVolumeSource{
|
||
Medium: corev1.StorageMediumMemory,
|
||
SizeLimit: &testMemoryLimit,
|
||
},
|
||
},
|
||
},
|
||
{
|
||
Name: "ray-logs",
|
||
VolumeSource: corev1.VolumeSource{
|
||
EmptyDir: &corev1.EmptyDirVolumeSource{
|
||
Medium: corev1.StorageMediumDefault,
|
||
},
|
||
},
|
||
},
|
||
}
|
||
|
||
var volumeMountsNoAutoscaler = []corev1.VolumeMount{
|
||
{
|
||
Name: "shared-mem",
|
||
MountPath: "/dev/shm",
|
||
ReadOnly: false,
|
||
},
|
||
}
|
||
|
||
var volumeMountsWithAutoscaler = []corev1.VolumeMount{
|
||
{
|
||
Name: "shared-mem",
|
||
MountPath: "/dev/shm",
|
||
ReadOnly: false,
|
||
},
|
||
{
|
||
Name: "ray-logs",
|
||
MountPath: "/tmp/ray",
|
||
ReadOnly: false,
|
||
},
|
||
}
|
||
|
||
var autoscalerContainer = corev1.Container{
|
||
Name: "autoscaler",
|
||
Image: "repo/image:custom",
|
||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||
Env: []corev1.EnvVar{
|
||
{
|
||
Name: utils.RAY_CLUSTER_NAME,
|
||
ValueFrom: &corev1.EnvVarSource{
|
||
FieldRef: &corev1.ObjectFieldSelector{
|
||
FieldPath: fmt.Sprintf("metadata.labels['%s']", utils.RayClusterLabelKey),
|
||
},
|
||
},
|
||
},
|
||
{
|
||
Name: utils.RAY_CLUSTER_NAMESPACE,
|
||
ValueFrom: &corev1.EnvVarSource{
|
||
FieldRef: &corev1.ObjectFieldSelector{
|
||
FieldPath: "metadata.namespace",
|
||
},
|
||
},
|
||
},
|
||
{
|
||
Name: "RAY_HEAD_POD_NAME",
|
||
ValueFrom: &corev1.EnvVarSource{
|
||
FieldRef: &corev1.ObjectFieldSelector{
|
||
FieldPath: "metadata.name",
|
||
},
|
||
},
|
||
},
|
||
{
|
||
Name: "KUBERAY_CRD_VER",
|
||
Value: "v1",
|
||
},
|
||
{
|
||
Name: utils.KUBERAY_GEN_AUTOSCALER_START_CMD,
|
||
Value: "ray kuberay-autoscaler --cluster-name $(RAY_CLUSTER_NAME) --cluster-namespace $(RAY_CLUSTER_NAMESPACE)",
|
||
},
|
||
},
|
||
Command: []string{
|
||
"/bin/bash",
|
||
"-c",
|
||
"--",
|
||
},
|
||
Args: []string{
|
||
"ray kuberay-autoscaler --cluster-name $(RAY_CLUSTER_NAME) --cluster-namespace $(RAY_CLUSTER_NAMESPACE)",
|
||
},
|
||
Resources: corev1.ResourceRequirements{
|
||
Limits: corev1.ResourceList{
|
||
corev1.ResourceCPU: resource.MustParse("500m"),
|
||
corev1.ResourceMemory: resource.MustParse("512Mi"),
|
||
},
|
||
Requests: corev1.ResourceList{
|
||
corev1.ResourceCPU: resource.MustParse("500m"),
|
||
corev1.ResourceMemory: resource.MustParse("512Mi"),
|
||
},
|
||
},
|
||
VolumeMounts: []corev1.VolumeMount{
|
||
{
|
||
MountPath: "/tmp/ray",
|
||
Name: "ray-logs",
|
||
},
|
||
},
|
||
}
|
||
|
||
var trueFlag = true
|
||
|
||
func TestAddEmptyDirVolumes(t *testing.T) {
|
||
testPod := &corev1.Pod{
|
||
Spec: corev1.PodSpec{
|
||
Containers: []corev1.Container{
|
||
{
|
||
Name: "ray-worker",
|
||
VolumeMounts: []corev1.VolumeMount{
|
||
{
|
||
Name: "shared-mem",
|
||
MountPath: "/dev/shm",
|
||
},
|
||
},
|
||
},
|
||
},
|
||
Volumes: []corev1.Volume{
|
||
{
|
||
Name: "shared-mem",
|
||
VolumeSource: corev1.VolumeSource{
|
||
EmptyDir: &corev1.EmptyDirVolumeSource{
|
||
Medium: corev1.StorageMediumMemory,
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
}
|
||
assert.Len(t, testPod.Spec.Containers[0].VolumeMounts, 1)
|
||
assert.Len(t, testPod.Spec.Volumes, 1)
|
||
addEmptyDir(context.Background(), &testPod.Spec.Containers[0], testPod, "shared-mem2", "/dev/shm2", corev1.StorageMediumDefault)
|
||
assert.Len(t, testPod.Spec.Containers[0].VolumeMounts, 2)
|
||
assert.Len(t, testPod.Spec.Volumes, 2)
|
||
addEmptyDir(context.Background(), &testPod.Spec.Containers[0], testPod, "shared-mem2", "/dev/shm2", corev1.StorageMediumDefault)
|
||
assert.Len(t, testPod.Spec.Containers[0].VolumeMounts, 2)
|
||
assert.Len(t, testPod.Spec.Volumes, 2)
|
||
}
|
||
|
||
func TestGetHeadPort(t *testing.T) {
|
||
headStartParams := make(map[string]string)
|
||
actualResult := GetHeadPort(headStartParams)
|
||
assert.Equal(t, "6379", actualResult)
|
||
|
||
headStartParams["port"] = "9999"
|
||
actualResult = GetHeadPort(headStartParams)
|
||
assert.Equal(t, "9999", actualResult)
|
||
}
|
||
|
||
func getEnvVar(container corev1.Container, envName string) *corev1.EnvVar {
|
||
for _, env := range container.Env {
|
||
if env.Name == envName {
|
||
return &env
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func checkContainerEnv(t *testing.T, container corev1.Container, envName string, expectedValue string) {
|
||
env := getEnvVar(container, envName)
|
||
require.NotNil(t, env)
|
||
|
||
if env.Value != "" {
|
||
assert.Equal(t, expectedValue, env.Value)
|
||
} else {
|
||
// env.ValueFrom is the source for the environment variable's value. Cannot be used if value is not empty.
|
||
// See https://pkg.go.dev/k8s.io/api/core/v1#EnvVar for more details.
|
||
assert.Equal(t, expectedValue, env.ValueFrom.FieldRef.FieldPath)
|
||
}
|
||
}
|
||
|
||
func assertWorkerGCSFaultToleranceConfig(t *testing.T, podTemplate *corev1.PodTemplateSpec, container corev1.Container) {
|
||
assert.Empty(t, podTemplate.Annotations[utils.RayExternalStorageNSAnnotationKey])
|
||
assert.Empty(t, podTemplate.Annotations[utils.RayFTEnabledAnnotationKey])
|
||
assert.True(t, utils.EnvVarExists(utils.RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S, container.Env))
|
||
assert.False(t, utils.EnvVarExists(utils.RAY_EXTERNAL_STORAGE_NS, container.Env))
|
||
assert.False(t, utils.EnvVarExists(utils.RAY_REDIS_ADDRESS, container.Env))
|
||
assert.False(t, utils.EnvVarExists(utils.REDIS_PASSWORD, container.Env))
|
||
}
|
||
|
||
func TestConfigureGCSFaultToleranceWithAnnotations(t *testing.T) {
|
||
tests := []struct {
|
||
name string
|
||
storageNS string
|
||
redisUsernameEnv string
|
||
redisPasswordEnv string
|
||
redisUsernameRayStartParams string
|
||
redisPasswordRayStartParams string
|
||
isHeadPod bool
|
||
gcsFTEnabled bool
|
||
}{
|
||
{
|
||
name: "GCS FT enabled",
|
||
gcsFTEnabled: true,
|
||
isHeadPod: true,
|
||
},
|
||
{
|
||
name: "GCS FT enabled with external storage",
|
||
gcsFTEnabled: true,
|
||
storageNS: "test-ns",
|
||
isHeadPod: true,
|
||
},
|
||
{
|
||
name: "GCS FT enabled with redis password env",
|
||
gcsFTEnabled: true,
|
||
redisPasswordEnv: "test-password",
|
||
isHeadPod: true,
|
||
},
|
||
{
|
||
name: "GCS FT enabled with redis username and password env",
|
||
gcsFTEnabled: true,
|
||
redisUsernameEnv: "test-username",
|
||
redisPasswordEnv: "test-password",
|
||
isHeadPod: true,
|
||
},
|
||
{
|
||
name: "GCS FT enabled with redis password ray start params",
|
||
gcsFTEnabled: true,
|
||
redisPasswordRayStartParams: "test-password",
|
||
isHeadPod: true,
|
||
},
|
||
{
|
||
name: "GCS FT enabled with redis username and password ray start params",
|
||
gcsFTEnabled: true,
|
||
redisUsernameRayStartParams: "test-username",
|
||
redisPasswordRayStartParams: "test-password",
|
||
isHeadPod: true,
|
||
},
|
||
{
|
||
// The most common case.
|
||
name: "GCS FT enabled with redis password env and ray start params referring to env",
|
||
gcsFTEnabled: true,
|
||
redisPasswordEnv: "test-password",
|
||
redisPasswordRayStartParams: "$REDIS_PASSWORD",
|
||
isHeadPod: false,
|
||
},
|
||
{
|
||
name: "GCS FT enabled with redis username and password env and ray start params referring to env",
|
||
gcsFTEnabled: true,
|
||
redisUsernameEnv: "test-username",
|
||
redisUsernameRayStartParams: "$REDIS_USERNAME",
|
||
redisPasswordEnv: "test-password",
|
||
redisPasswordRayStartParams: "$REDIS_PASSWORD",
|
||
isHeadPod: false,
|
||
},
|
||
{
|
||
name: "GCS FT enabled / worker Pod",
|
||
gcsFTEnabled: true,
|
||
isHeadPod: false,
|
||
},
|
||
{
|
||
name: "GCS FT disabled",
|
||
gcsFTEnabled: false,
|
||
isHeadPod: true,
|
||
},
|
||
}
|
||
|
||
for _, test := range tests {
|
||
t.Run(test.name, func(t *testing.T) {
|
||
// Validate the test input
|
||
if test.redisUsernameEnv != "" && test.redisUsernameRayStartParams != "" {
|
||
assert.Equal(t, "$REDIS_USERNAME", test.redisUsernameRayStartParams)
|
||
}
|
||
if test.redisPasswordEnv != "" && test.redisPasswordRayStartParams != "" {
|
||
assert.Equal(t, "$REDIS_PASSWORD", test.redisPasswordRayStartParams)
|
||
}
|
||
|
||
// Prepare the cluster
|
||
cluster := rayv1.RayCluster{
|
||
ObjectMeta: metav1.ObjectMeta{
|
||
Annotations: map[string]string{
|
||
utils.RayFTEnabledAnnotationKey: strconv.FormatBool(test.gcsFTEnabled),
|
||
},
|
||
},
|
||
Spec: rayv1.RayClusterSpec{
|
||
HeadGroupSpec: rayv1.HeadGroupSpec{
|
||
RayStartParams: map[string]string{},
|
||
Template: corev1.PodTemplateSpec{
|
||
Spec: corev1.PodSpec{
|
||
Containers: []corev1.Container{
|
||
{
|
||
Env: []corev1.EnvVar{
|
||
{
|
||
Name: utils.RAY_REDIS_ADDRESS,
|
||
Value: "redis:6379",
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
WorkerGroupSpecs: []rayv1.WorkerGroupSpec{
|
||
{
|
||
Template: corev1.PodTemplateSpec{
|
||
Spec: corev1.PodSpec{
|
||
Containers: []corev1.Container{
|
||
{
|
||
Env: []corev1.EnvVar{},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
}
|
||
if test.storageNS != "" {
|
||
cluster.Annotations[utils.RayExternalStorageNSAnnotationKey] = test.storageNS
|
||
}
|
||
if test.redisUsernameEnv != "" {
|
||
cluster.Spec.HeadGroupSpec.Template.Spec.Containers[utils.RayContainerIndex].Env = append(cluster.Spec.HeadGroupSpec.Template.Spec.Containers[utils.RayContainerIndex].Env, corev1.EnvVar{
|
||
Name: utils.REDIS_USERNAME,
|
||
Value: test.redisUsernameEnv,
|
||
})
|
||
}
|
||
if test.redisPasswordEnv != "" {
|
||
cluster.Spec.HeadGroupSpec.Template.Spec.Containers[utils.RayContainerIndex].Env = append(cluster.Spec.HeadGroupSpec.Template.Spec.Containers[utils.RayContainerIndex].Env, corev1.EnvVar{
|
||
Name: utils.REDIS_PASSWORD,
|
||
Value: test.redisPasswordEnv,
|
||
})
|
||
}
|
||
if test.redisUsernameRayStartParams != "" {
|
||
cluster.Spec.HeadGroupSpec.RayStartParams["redis-username"] = test.redisUsernameRayStartParams
|
||
}
|
||
if test.redisPasswordRayStartParams != "" {
|
||
cluster.Spec.HeadGroupSpec.RayStartParams["redis-password"] = test.redisPasswordRayStartParams
|
||
}
|
||
podTemplate := &cluster.Spec.HeadGroupSpec.Template
|
||
if !test.isHeadPod {
|
||
podTemplate = &cluster.Spec.WorkerGroupSpecs[0].Template
|
||
}
|
||
|
||
// Configure GCS fault tolerance
|
||
if test.isHeadPod {
|
||
configureGCSFaultTolerance(podTemplate, cluster, rayv1.HeadNode)
|
||
} else {
|
||
configureGCSFaultTolerance(podTemplate, cluster, rayv1.WorkerNode)
|
||
}
|
||
|
||
// Check configurations for GCS fault tolerance
|
||
container := podTemplate.Spec.Containers[utils.RayContainerIndex]
|
||
if test.isHeadPod {
|
||
assert.False(t, utils.EnvVarExists(utils.RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S, container.Env))
|
||
assert.Equal(t, podTemplate.Annotations[utils.RayFTEnabledAnnotationKey], strconv.FormatBool(test.gcsFTEnabled))
|
||
if test.storageNS != "" {
|
||
assert.Equal(t, podTemplate.Annotations[utils.RayExternalStorageNSAnnotationKey], test.storageNS)
|
||
assert.True(t, utils.EnvVarExists(utils.RAY_EXTERNAL_STORAGE_NS, container.Env))
|
||
}
|
||
if test.redisUsernameEnv != "" {
|
||
env := getEnvVar(container, utils.REDIS_USERNAME)
|
||
assert.Equal(t, env.Value, test.redisUsernameEnv)
|
||
}
|
||
if test.redisPasswordEnv != "" {
|
||
env := getEnvVar(container, utils.REDIS_PASSWORD)
|
||
assert.Equal(t, env.Value, test.redisPasswordEnv)
|
||
} else if test.redisPasswordRayStartParams != "" {
|
||
env := getEnvVar(container, utils.REDIS_PASSWORD)
|
||
assert.Equal(t, env.Value, test.redisPasswordRayStartParams)
|
||
}
|
||
} else {
|
||
assertWorkerGCSFaultToleranceConfig(t, podTemplate, container)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestConfigureGCSFaultToleranceWithGcsFTOptions(t *testing.T) {
|
||
tests := []struct {
|
||
gcsFTOptions *rayv1.GcsFaultToleranceOptions
|
||
name string
|
||
isHeadPod bool
|
||
}{
|
||
{
|
||
name: "GCS FT enabled",
|
||
gcsFTOptions: &rayv1.GcsFaultToleranceOptions{
|
||
RedisAddress: "redis:6379",
|
||
},
|
||
isHeadPod: true,
|
||
},
|
||
{
|
||
name: "GCS FT enabled with redis password",
|
||
gcsFTOptions: &rayv1.GcsFaultToleranceOptions{
|
||
RedisAddress: "redis:6379",
|
||
RedisPassword: &rayv1.RedisCredential{
|
||
Value: "test-password",
|
||
},
|
||
},
|
||
isHeadPod: true,
|
||
},
|
||
{
|
||
name: "GCS FT enabled with redis username and password",
|
||
gcsFTOptions: &rayv1.GcsFaultToleranceOptions{
|
||
RedisAddress: "redis:6379",
|
||
RedisUsername: &rayv1.RedisCredential{
|
||
Value: "test-username",
|
||
},
|
||
RedisPassword: &rayv1.RedisCredential{
|
||
Value: "test-password",
|
||
},
|
||
},
|
||
isHeadPod: true,
|
||
},
|
||
{
|
||
name: "GCS FT enabled with redis password in secret",
|
||
gcsFTOptions: &rayv1.GcsFaultToleranceOptions{
|
||
RedisAddress: "redis:6379",
|
||
RedisPassword: &rayv1.RedisCredential{
|
||
ValueFrom: &corev1.EnvVarSource{
|
||
FieldRef: &corev1.ObjectFieldSelector{
|
||
FieldPath: "spec.redisPassword",
|
||
},
|
||
},
|
||
},
|
||
},
|
||
isHeadPod: true,
|
||
},
|
||
{
|
||
name: "GCS FT enabled with redis username and password in secret",
|
||
gcsFTOptions: &rayv1.GcsFaultToleranceOptions{
|
||
RedisAddress: "redis:6379",
|
||
RedisUsername: &rayv1.RedisCredential{
|
||
ValueFrom: &corev1.EnvVarSource{
|
||
FieldRef: &corev1.ObjectFieldSelector{
|
||
FieldPath: "spec.redisUsername",
|
||
},
|
||
},
|
||
},
|
||
RedisPassword: &rayv1.RedisCredential{
|
||
ValueFrom: &corev1.EnvVarSource{
|
||
FieldRef: &corev1.ObjectFieldSelector{
|
||
FieldPath: "spec.redisPassword",
|
||
},
|
||
},
|
||
},
|
||
},
|
||
isHeadPod: true,
|
||
},
|
||
{
|
||
name: "GCS FT enabled with redis external storage namespace",
|
||
gcsFTOptions: &rayv1.GcsFaultToleranceOptions{
|
||
RedisAddress: "redis:6379",
|
||
ExternalStorageNamespace: "test-ns",
|
||
},
|
||
isHeadPod: true,
|
||
},
|
||
{
|
||
name: "GCS FT enabled / worker Pod",
|
||
gcsFTOptions: &rayv1.GcsFaultToleranceOptions{
|
||
RedisAddress: "redis:6379",
|
||
},
|
||
isHeadPod: false,
|
||
},
|
||
}
|
||
|
||
emptyPodTemplate := corev1.PodTemplateSpec{
|
||
Spec: corev1.PodSpec{
|
||
Containers: []corev1.Container{
|
||
{
|
||
Env: []corev1.EnvVar{},
|
||
},
|
||
},
|
||
},
|
||
}
|
||
|
||
for _, test := range tests {
|
||
t.Run(test.name, func(t *testing.T) {
|
||
// Prepare the cluster
|
||
cluster := rayv1.RayCluster{
|
||
Spec: rayv1.RayClusterSpec{
|
||
GcsFaultToleranceOptions: test.gcsFTOptions,
|
||
HeadGroupSpec: rayv1.HeadGroupSpec{
|
||
RayStartParams: map[string]string{},
|
||
Template: *emptyPodTemplate.DeepCopy(),
|
||
},
|
||
WorkerGroupSpecs: []rayv1.WorkerGroupSpec{
|
||
{
|
||
Template: *emptyPodTemplate.DeepCopy(),
|
||
},
|
||
},
|
||
},
|
||
}
|
||
|
||
podTemplate := &cluster.Spec.HeadGroupSpec.Template
|
||
nodeType := rayv1.HeadNode
|
||
if !test.isHeadPod {
|
||
podTemplate = &cluster.Spec.WorkerGroupSpecs[0].Template
|
||
nodeType = rayv1.WorkerNode
|
||
}
|
||
configureGCSFaultTolerance(podTemplate, cluster, nodeType)
|
||
container := podTemplate.Spec.Containers[utils.RayContainerIndex]
|
||
|
||
if test.isHeadPod {
|
||
assert.False(t, utils.EnvVarExists(utils.RAY_GCS_RPC_SERVER_RECONNECT_TIMEOUT_S, container.Env))
|
||
assert.Equal(t, podTemplate.Annotations[utils.RayFTEnabledAnnotationKey], strconv.FormatBool(test.gcsFTOptions != nil))
|
||
|
||
env := getEnvVar(container, utils.RAY_REDIS_ADDRESS)
|
||
assert.Equal(t, "redis:6379", env.Value)
|
||
|
||
if test.gcsFTOptions.RedisUsername != nil {
|
||
env := getEnvVar(container, utils.REDIS_USERNAME)
|
||
assert.Equal(t, env.Value, test.gcsFTOptions.RedisUsername.Value)
|
||
assert.Equal(t, env.ValueFrom, test.gcsFTOptions.RedisUsername.ValueFrom)
|
||
}
|
||
|
||
if test.gcsFTOptions.RedisPassword != nil {
|
||
env := getEnvVar(container, utils.REDIS_PASSWORD)
|
||
assert.Equal(t, env.Value, test.gcsFTOptions.RedisPassword.Value)
|
||
assert.Equal(t, env.ValueFrom, test.gcsFTOptions.RedisPassword.ValueFrom)
|
||
}
|
||
if test.gcsFTOptions.ExternalStorageNamespace != "" {
|
||
assert.Equal(t, podTemplate.Annotations[utils.RayExternalStorageNSAnnotationKey], test.gcsFTOptions.ExternalStorageNamespace)
|
||
env := getEnvVar(container, utils.RAY_EXTERNAL_STORAGE_NS)
|
||
assert.Equal(t, env.Value, test.gcsFTOptions.ExternalStorageNamespace)
|
||
}
|
||
} else {
|
||
assertWorkerGCSFaultToleranceConfig(t, podTemplate, container)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestBuildPod(t *testing.T) {
|
||
cluster := instance.DeepCopy()
|
||
ctx := context.Background()
|
||
|
||
// Define default environment variables
|
||
defaultContainerEnvs := []corev1.EnvVar{
|
||
{Name: "TEST_DEFAULT_ENV_NAME", Value: "TEST_ENV_VALUE"},
|
||
}
|
||
|
||
// Test head pod
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
podTemplateSpec := DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
pod := BuildPod(ctx, podTemplateSpec, rayv1.HeadNode, cluster.Spec.HeadGroupSpec.RayStartParams, "6379", false, utils.GetCRDType(""), "", defaultContainerEnvs, "")
|
||
|
||
// Check environment variables
|
||
rayContainer := pod.Spec.Containers[utils.RayContainerIndex]
|
||
checkContainerEnv(t, rayContainer, utils.RAY_ADDRESS, "127.0.0.1:6379")
|
||
checkContainerEnv(t, rayContainer, utils.RAY_USAGE_STATS_KUBERAY_IN_USE, "1")
|
||
checkContainerEnv(t, rayContainer, utils.RAY_CLUSTER_NAME, fmt.Sprintf("metadata.labels['%s']", utils.RayClusterLabelKey))
|
||
checkContainerEnv(t, rayContainer, utils.RAY_CLUSTER_NAMESPACE, "metadata.namespace")
|
||
checkContainerEnv(t, rayContainer, utils.RAY_DASHBOARD_ENABLE_K8S_DISK_USAGE, "1")
|
||
checkContainerEnv(t, rayContainer, utils.RAY_NODE_TYPE_NAME, fmt.Sprintf("metadata.labels['%s']", utils.RayNodeGroupLabelKey))
|
||
checkContainerEnv(t, rayContainer, utils.RAY_USAGE_STATS_EXTRA_TAGS, fmt.Sprintf("kuberay_version=%s;kuberay_crd=%s", utils.KUBERAY_VERSION, utils.RayClusterCRD))
|
||
headRayStartCommandEnv := getEnvVar(rayContainer, utils.KUBERAY_GEN_RAY_START_CMD)
|
||
assert.Contains(t, headRayStartCommandEnv.Value, "ray start")
|
||
|
||
// In head, init container needs FQ_RAY_IP to create a self-signed certificate for its TLS authenticate.
|
||
for _, initContainer := range pod.Spec.InitContainers {
|
||
checkContainerEnv(t, initContainer, utils.FQ_RAY_IP, "raycluster-sample-head-svc.default.svc.cluster.local")
|
||
checkContainerEnv(t, rayContainer, utils.RAY_IP, "raycluster-sample-head-svc")
|
||
}
|
||
|
||
// Check labels.
|
||
assert.Equal(t, cluster.Name, pod.Labels[utils.RayClusterLabelKey])
|
||
assert.Equal(t, string(rayv1.HeadNode), pod.Labels[utils.RayNodeTypeLabelKey])
|
||
assert.Equal(t, utils.RayNodeHeadGroupLabelValue, pod.Labels[utils.RayNodeGroupLabelKey])
|
||
|
||
// Check volumes.
|
||
actualVolumes := pod.Spec.Volumes
|
||
expectedVolumes := volumesNoAutoscaler
|
||
assert.True(t, reflect.DeepEqual(actualVolumes, expectedVolumes))
|
||
|
||
// Check volume mounts.
|
||
actualVolumeMounts := pod.Spec.Containers[0].VolumeMounts
|
||
expectedVolumeMounts := volumeMountsNoAutoscaler
|
||
assert.True(t, reflect.DeepEqual(actualVolumeMounts, expectedVolumeMounts))
|
||
|
||
// testing worker pod
|
||
worker := cluster.Spec.WorkerGroupSpecs[0]
|
||
podName = cluster.Name + utils.DashSymbol + string(rayv1.WorkerNode) + utils.DashSymbol + worker.GroupName + utils.DashSymbol + utils.FormatInt32(0)
|
||
fqdnRayIP := utils.GenerateFQDNServiceName(ctx, *cluster, cluster.Namespace)
|
||
podTemplateSpec = DefaultWorkerPodTemplate(ctx, *cluster, worker, podName, fqdnRayIP, "6379", "", 0, 0)
|
||
pod = BuildPod(ctx, podTemplateSpec, rayv1.WorkerNode, worker.RayStartParams, "6379", false, utils.GetCRDType(""), fqdnRayIP, defaultContainerEnvs, "")
|
||
|
||
// Check resources
|
||
rayContainer = pod.Spec.Containers[utils.RayContainerIndex]
|
||
expectedResources := corev1.ResourceRequirements{
|
||
Limits: corev1.ResourceList{
|
||
corev1.ResourceCPU: resource.MustParse("1"),
|
||
corev1.ResourceMemory: testMemoryLimit,
|
||
"nvidia.com/gpu": resource.MustParse("3"),
|
||
},
|
||
}
|
||
assert.Equal(t, expectedResources.Limits, rayContainer.Resources.Limits, "Resource limits do not match")
|
||
|
||
// Check environment variables
|
||
rayContainer = pod.Spec.Containers[utils.RayContainerIndex]
|
||
checkContainerEnv(t, rayContainer, utils.RAY_ADDRESS, "raycluster-sample-head-svc.default.svc.cluster.local:6379")
|
||
checkContainerEnv(t, rayContainer, utils.FQ_RAY_IP, "raycluster-sample-head-svc.default.svc.cluster.local")
|
||
checkContainerEnv(t, rayContainer, utils.RAY_IP, "raycluster-sample-head-svc")
|
||
checkContainerEnv(t, rayContainer, utils.RAY_CLUSTER_NAME, fmt.Sprintf("metadata.labels['%s']", utils.RayClusterLabelKey))
|
||
checkContainerEnv(t, rayContainer, utils.RAY_CLUSTER_NAMESPACE, "metadata.namespace")
|
||
checkContainerEnv(t, rayContainer, utils.RAY_DASHBOARD_ENABLE_K8S_DISK_USAGE, "1")
|
||
checkContainerEnv(t, rayContainer, utils.RAY_NODE_TYPE_NAME, fmt.Sprintf("metadata.labels['%s']", utils.RayNodeGroupLabelKey))
|
||
workerRayStartCommandEnv := getEnvVar(rayContainer, utils.KUBERAY_GEN_RAY_START_CMD)
|
||
assert.Contains(t, workerRayStartCommandEnv.Value, "ray start")
|
||
|
||
expectedCommandArg := splitAndSort("ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --block --dashboard-agent-listen-port=52365 --memory=1073741824 --num-cpus=1 --num-gpus=3 --address=raycluster-sample-head-svc.default.svc.cluster.local:6379 --port=6379 --metrics-export-port=8080")
|
||
actualCommandArg := splitAndSort(pod.Spec.Containers[0].Args[0])
|
||
assert.Equal(t, expectedCommandArg, actualCommandArg)
|
||
|
||
// Check Envs
|
||
rayContainer = pod.Spec.Containers[utils.RayContainerIndex]
|
||
checkContainerEnv(t, rayContainer, "TEST_ENV_NAME", "TEST_ENV_VALUE")
|
||
|
||
// Test default environment variables injection in ray pods
|
||
checkContainerEnv(t, rayContainer, "TEST_DEFAULT_ENV_NAME", "TEST_ENV_VALUE")
|
||
}
|
||
|
||
func TestBuildPod_WithUlimitOverride(t *testing.T) {
|
||
cluster := instance.DeepCopy()
|
||
ctx := context.Background()
|
||
|
||
// Add RAY_START_ULIMIT_OPEN_FILES to container env
|
||
cluster.Spec.HeadGroupSpec.Template.Spec.Containers[utils.RayContainerIndex].Env = append(
|
||
cluster.Spec.HeadGroupSpec.Template.Spec.Containers[utils.RayContainerIndex].Env,
|
||
corev1.EnvVar{Name: utils.RAY_START_ULIMIT_OPEN_FILES, Value: "1048576"},
|
||
)
|
||
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
podTemplateSpec := DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
pod := BuildPod(ctx, podTemplateSpec, rayv1.HeadNode, cluster.Spec.HeadGroupSpec.RayStartParams, "6379", false, utils.GetCRDType(""), "", nil, "")
|
||
|
||
// The generated command arg still uses ${RAY_START_ULIMIT_OPEN_FILES:-65536} because the shell resolves it at runtime.
|
||
expectedCommandArg := splitAndSort("ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head --block --dashboard-agent-listen-port=52365 --memory=1073741824 --num-cpus=1 --metrics-export-port=8080 --dashboard-host=0.0.0.0")
|
||
actualCommandArg := splitAndSort(pod.Spec.Containers[0].Args[0])
|
||
assert.Equal(t, expectedCommandArg, actualCommandArg)
|
||
|
||
// Verify that the environment variable exists in the container.
|
||
rayContainer := pod.Spec.Containers[utils.RayContainerIndex]
|
||
checkContainerEnv(t, rayContainer, utils.RAY_START_ULIMIT_OPEN_FILES, "1048576")
|
||
}
|
||
|
||
func TestBuildAutoscalerContainer(t *testing.T) {
|
||
const autoscalerImage = "rayproject/ray:2.52.0"
|
||
const expectedCmd = "ray kuberay-autoscaler --cluster-name $(RAY_CLUSTER_NAME) --cluster-namespace $(RAY_CLUSTER_NAMESPACE)"
|
||
|
||
t.Run("KUBERAY_GEN_AUTOSCALER_START_CMD is always injected with the generated command", func(t *testing.T) {
|
||
container := BuildAutoscalerContainer(autoscalerImage)
|
||
|
||
env := getEnvVar(container, utils.KUBERAY_GEN_AUTOSCALER_START_CMD)
|
||
require.NotNil(t, env, "KUBERAY_GEN_AUTOSCALER_START_CMD env var should always be present")
|
||
assert.Equal(t, expectedCmd, env.Value)
|
||
})
|
||
|
||
t.Run("Default Args equal KUBERAY_GEN_AUTOSCALER_START_CMD value", func(t *testing.T) {
|
||
container := BuildAutoscalerContainer(autoscalerImage)
|
||
|
||
require.Len(t, container.Args, 1)
|
||
assert.Equal(t, expectedCmd, container.Args[0],
|
||
"Default Args should match KUBERAY_GEN_AUTOSCALER_START_CMD so they are consistent")
|
||
})
|
||
|
||
t.Run("AutoscalerOptions.Command override replaces command", func(t *testing.T) {
|
||
container := BuildAutoscalerContainer(autoscalerImage)
|
||
customCMD := []string{"/bin/bash", "-lc", "--"}
|
||
mergeAutoscalerOverrides(&container, &rayv1.AutoscalerOptions{
|
||
Command: customCMD,
|
||
})
|
||
|
||
// Args must reflect the override.
|
||
assert.Equal(t, customCMD, container.Command)
|
||
|
||
// KUBERAY_GEN_AUTOSCALER_START_CMD must still hold the original generated command.
|
||
env := getEnvVar(container, utils.KUBERAY_GEN_AUTOSCALER_START_CMD)
|
||
require.NotNil(t, env)
|
||
assert.Equal(t, expectedCmd, env.Value)
|
||
})
|
||
|
||
t.Run("AutoscalerOptions.Args override replaces Args but not KUBERAY_GEN_AUTOSCALER_START_CMD", func(t *testing.T) {
|
||
container := BuildAutoscalerContainer(autoscalerImage)
|
||
customArgs := []string{"ulimit -n 65536; $KUBERAY_GEN_AUTOSCALER_START_CMD"}
|
||
mergeAutoscalerOverrides(&container, &rayv1.AutoscalerOptions{
|
||
Args: customArgs,
|
||
})
|
||
|
||
// Args must reflect the override.
|
||
assert.Equal(t, customArgs, container.Args)
|
||
|
||
// KUBERAY_GEN_AUTOSCALER_START_CMD must still hold the original generated command.
|
||
env := getEnvVar(container, utils.KUBERAY_GEN_AUTOSCALER_START_CMD)
|
||
require.NotNil(t, env)
|
||
assert.Equal(t, expectedCmd, env.Value)
|
||
})
|
||
|
||
t.Run("AutoscalerOptions.Env additions do not overwrite KUBERAY_GEN_AUTOSCALER_START_CMD", func(t *testing.T) {
|
||
container := BuildAutoscalerContainer(autoscalerImage)
|
||
mergeAutoscalerOverrides(&container, &rayv1.AutoscalerOptions{
|
||
Env: []corev1.EnvVar{
|
||
{Name: "AUTOSCALER_UPDATE_INTERVAL_S", Value: "10"},
|
||
},
|
||
})
|
||
|
||
// The user-supplied env var must be present.
|
||
userEnv := getEnvVar(container, "AUTOSCALER_UPDATE_INTERVAL_S")
|
||
require.NotNil(t, userEnv)
|
||
assert.Equal(t, "10", userEnv.Value)
|
||
|
||
// KUBERAY_GEN_AUTOSCALER_START_CMD must still reflect the KubeRay-generated command.
|
||
env := getEnvVar(container, utils.KUBERAY_GEN_AUTOSCALER_START_CMD)
|
||
require.NotNil(t, env)
|
||
assert.Equal(t, expectedCmd, env.Value)
|
||
})
|
||
}
|
||
|
||
func TestBuildPod_WithPlasmaDirectory(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
testCases := []struct {
|
||
rayStartParams map[string]string
|
||
name string
|
||
expectSharedMemory bool
|
||
}{
|
||
{
|
||
name: "unset plasma-directory keeps shared memory mount",
|
||
rayStartParams: map[string]string{},
|
||
expectSharedMemory: true,
|
||
},
|
||
{
|
||
name: "plasma-directory set to /dev/shm skips shared memory mount",
|
||
rayStartParams: map[string]string{
|
||
PlasmaDirectoryParamKey: "/dev/shm",
|
||
},
|
||
expectSharedMemory: false,
|
||
},
|
||
{
|
||
name: "plasma-directory set to /dev/shm/ skips shared memory mount",
|
||
rayStartParams: map[string]string{
|
||
PlasmaDirectoryParamKey: "/dev/shm/",
|
||
},
|
||
expectSharedMemory: false,
|
||
},
|
||
{
|
||
name: "non /dev/shm plasma-directory skips shared memory mount",
|
||
rayStartParams: map[string]string{
|
||
PlasmaDirectoryParamKey: "/tmp/ray/plasma",
|
||
},
|
||
expectSharedMemory: false,
|
||
},
|
||
}
|
||
|
||
for _, tc := range testCases {
|
||
t.Run(tc.name, func(t *testing.T) {
|
||
cluster := instance.DeepCopy()
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
podTemplateSpec := DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
pod := BuildPod(ctx, podTemplateSpec, rayv1.HeadNode, maps.Clone(tc.rayStartParams), "6379", false, utils.GetCRDType(""), "", nil, "")
|
||
|
||
assert.Equal(t, tc.expectSharedMemory, checkIfVolumeMounted(&pod.Spec.Containers[utils.RayContainerIndex], SharedMemoryVolumeMountPath))
|
||
assert.Equal(t, tc.expectSharedMemory, checkIfVolumeExists(&pod, SharedMemoryVolumeName))
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestBuildPod_WithEnableK8sTokenAuth(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
cluster := instance.DeepCopy()
|
||
cluster.Spec.AuthOptions = &rayv1.AuthOptions{
|
||
Mode: rayv1.AuthModeToken,
|
||
EnableK8sTokenAuth: new(true),
|
||
}
|
||
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
podTemplateSpec := DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
pod := BuildPod(ctx, podTemplateSpec, rayv1.HeadNode, cluster.Spec.HeadGroupSpec.RayStartParams, "6379", false, utils.GetCRDType(""), "", nil, "")
|
||
|
||
rayContainer := pod.Spec.Containers[utils.RayContainerIndex]
|
||
checkContainerEnv(t, rayContainer, utils.RAY_ENABLE_K8S_TOKEN_AUTH_ENV_VAR, "true")
|
||
|
||
foundVolumeMount := false
|
||
for _, vm := range rayContainer.VolumeMounts {
|
||
if vm.Name == utils.RayTokenVolumeName && vm.MountPath == utils.RayTokenMountPath && vm.ReadOnly {
|
||
foundVolumeMount = true
|
||
break
|
||
}
|
||
}
|
||
if !foundVolumeMount {
|
||
t.Errorf("Ray container should have the ray-token volume mount")
|
||
}
|
||
|
||
foundVolume := false
|
||
for _, v := range pod.Spec.Volumes {
|
||
if v.Name == utils.RayTokenVolumeName && v.Projected != nil {
|
||
foundVolume = true
|
||
break
|
||
}
|
||
}
|
||
if !foundVolume {
|
||
t.Errorf("Pod should have the ray-token volume")
|
||
}
|
||
|
||
cluster = instance.DeepCopy()
|
||
cluster.Spec.AuthOptions = &rayv1.AuthOptions{
|
||
Mode: rayv1.AuthModeToken,
|
||
EnableK8sTokenAuth: new(false),
|
||
}
|
||
podTemplateSpec = DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
pod = BuildPod(ctx, podTemplateSpec, rayv1.HeadNode, cluster.Spec.HeadGroupSpec.RayStartParams, "6379", false, utils.GetCRDType(""), "", nil, "")
|
||
rayContainer = pod.Spec.Containers[utils.RayContainerIndex]
|
||
for _, env := range rayContainer.Env {
|
||
if env.Name == utils.RAY_ENABLE_K8S_TOKEN_AUTH_ENV_VAR {
|
||
t.Errorf("RAY_ENABLE_K8S_TOKEN_AUTH should not be set")
|
||
}
|
||
}
|
||
|
||
cluster = instance.DeepCopy()
|
||
cluster.Spec.AuthOptions = &rayv1.AuthOptions{
|
||
Mode: rayv1.AuthModeToken,
|
||
EnableK8sTokenAuth: nil,
|
||
}
|
||
podTemplateSpec = DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
pod = BuildPod(ctx, podTemplateSpec, rayv1.HeadNode, cluster.Spec.HeadGroupSpec.RayStartParams, "6379", false, utils.GetCRDType(""), "", nil, "")
|
||
rayContainer = pod.Spec.Containers[utils.RayContainerIndex]
|
||
for _, env := range rayContainer.Env {
|
||
if env.Name == utils.RAY_ENABLE_K8S_TOKEN_AUTH_ENV_VAR {
|
||
t.Errorf("RAY_ENABLE_K8S_TOKEN_AUTH should not be set")
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestBuildPod_WithEnableK8sTokenAuth_InitContainer(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
cluster := instance.DeepCopy()
|
||
cluster.Spec.AuthOptions = &rayv1.AuthOptions{
|
||
Mode: rayv1.AuthModeToken,
|
||
EnableK8sTokenAuth: new(true),
|
||
}
|
||
|
||
worker := cluster.Spec.WorkerGroupSpecs[0]
|
||
podName := cluster.Name + utils.DashSymbol + string(rayv1.WorkerNode) + utils.DashSymbol + worker.GroupName + utils.DashSymbol + utils.FormatInt32(0)
|
||
fqdnRayIP := utils.GenerateFQDNServiceName(ctx, *cluster, cluster.Namespace)
|
||
podTemplateSpec := DefaultWorkerPodTemplate(ctx, *cluster, worker, podName, fqdnRayIP, "6379", "", 0, 0)
|
||
pod := BuildPod(ctx, podTemplateSpec, rayv1.WorkerNode, worker.RayStartParams, "6379", false, utils.GetCRDType(""), fqdnRayIP, nil, "")
|
||
|
||
foundInitContainer := false
|
||
for _, container := range pod.Spec.InitContainers {
|
||
if container.Name == "wait-gcs-ready" {
|
||
foundInitContainer = true
|
||
foundVolumeMount := false
|
||
for _, vm := range container.VolumeMounts {
|
||
if vm.Name == utils.RayTokenVolumeName && vm.MountPath == utils.RayTokenMountPath && vm.ReadOnly {
|
||
foundVolumeMount = true
|
||
break
|
||
}
|
||
}
|
||
if !foundVolumeMount {
|
||
t.Errorf("wait-gcs-ready init container should have the ray-token volume mount")
|
||
}
|
||
}
|
||
}
|
||
if !foundInitContainer {
|
||
t.Errorf("wait-gcs-ready init container should be present")
|
||
}
|
||
}
|
||
|
||
func TestBuildPod_WithNoCPULimits(t *testing.T) {
|
||
cluster := instance.DeepCopy()
|
||
ctx := context.Background()
|
||
|
||
cluster.Spec.HeadGroupSpec.Template.Spec.Containers[utils.RayContainerIndex].Resources = corev1.ResourceRequirements{
|
||
Requests: corev1.ResourceList{
|
||
corev1.ResourceCPU: resource.MustParse("2"),
|
||
corev1.ResourceMemory: testMemoryLimit,
|
||
},
|
||
Limits: corev1.ResourceList{
|
||
corev1.ResourceMemory: testMemoryLimit,
|
||
},
|
||
}
|
||
cluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[utils.RayContainerIndex].Resources = corev1.ResourceRequirements{
|
||
Requests: corev1.ResourceList{
|
||
corev1.ResourceCPU: resource.MustParse("2"),
|
||
corev1.ResourceMemory: testMemoryLimit,
|
||
},
|
||
|
||
Limits: corev1.ResourceList{
|
||
corev1.ResourceMemory: testMemoryLimit,
|
||
"nvidia.com/gpu": resource.MustParse("3"),
|
||
},
|
||
}
|
||
|
||
// Test head pod
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
podTemplateSpec := DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
pod := BuildPod(ctx, podTemplateSpec, rayv1.HeadNode, cluster.Spec.HeadGroupSpec.RayStartParams, "6379", false, utils.GetCRDType(""), "", nil, "")
|
||
expectedCommandArg := splitAndSort("ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head --block --dashboard-agent-listen-port=52365 --memory=1073741824 --num-cpus=2 --metrics-export-port=8080 --dashboard-host=0.0.0.0")
|
||
actualCommandArg := splitAndSort(pod.Spec.Containers[0].Args[0])
|
||
assert.Equal(t, expectedCommandArg, actualCommandArg)
|
||
|
||
// testing worker pod
|
||
worker := cluster.Spec.WorkerGroupSpecs[0]
|
||
podName = cluster.Name + utils.DashSymbol + string(rayv1.WorkerNode) + utils.DashSymbol + worker.GroupName + utils.DashSymbol + utils.FormatInt32(0)
|
||
fqdnRayIP := utils.GenerateFQDNServiceName(ctx, *cluster, cluster.Namespace)
|
||
podTemplateSpec = DefaultWorkerPodTemplate(ctx, *cluster, worker, podName, fqdnRayIP, "6379", "", 0, 0)
|
||
pod = BuildPod(ctx, podTemplateSpec, rayv1.WorkerNode, worker.RayStartParams, "6379", false, utils.GetCRDType(""), fqdnRayIP, nil, "")
|
||
expectedCommandArg = splitAndSort("ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --block --dashboard-agent-listen-port=52365 --memory=1073741824 --num-cpus=2 --num-gpus=3 --address=raycluster-sample-head-svc.default.svc.cluster.local:6379 --port=6379 --metrics-export-port=8080")
|
||
actualCommandArg = splitAndSort(pod.Spec.Containers[0].Args[0])
|
||
assert.Equal(t, expectedCommandArg, actualCommandArg)
|
||
}
|
||
|
||
func TestBuildPod_WithOverwriteCommand(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
cluster := instance.DeepCopy()
|
||
cluster.Annotations = map[string]string{
|
||
// When the value of the annotation is "true", KubeRay will not generate the command and args for the container.
|
||
// Instead, it will use the command and args specified by the use.
|
||
utils.RayOverwriteContainerCmdAnnotationKey: "true",
|
||
}
|
||
cluster.Spec.HeadGroupSpec.Template.Spec.Containers[utils.RayContainerIndex].Command = []string{"I am head"}
|
||
cluster.Spec.HeadGroupSpec.Template.Spec.Containers[utils.RayContainerIndex].Args = []string{"I am head again"}
|
||
cluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[utils.RayContainerIndex].Command = []string{"I am worker"}
|
||
cluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[utils.RayContainerIndex].Args = []string{"I am worker again"}
|
||
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
podTemplateSpec := DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
headPod := BuildPod(ctx, podTemplateSpec, rayv1.HeadNode, cluster.Spec.HeadGroupSpec.RayStartParams, "6379", false, utils.GetCRDType(""), "", nil, "")
|
||
headContainer := headPod.Spec.Containers[utils.RayContainerIndex]
|
||
assert.Equal(t, []string{"I am head"}, headContainer.Command)
|
||
assert.Equal(t, []string{"I am head again"}, headContainer.Args)
|
||
|
||
worker := cluster.Spec.WorkerGroupSpecs[0]
|
||
podName = cluster.Name + utils.DashSymbol + string(rayv1.WorkerNode) + utils.DashSymbol + worker.GroupName + utils.DashSymbol + utils.FormatInt32(0)
|
||
fqdnRayIP := utils.GenerateFQDNServiceName(ctx, *cluster, cluster.Namespace)
|
||
podTemplateSpec = DefaultWorkerPodTemplate(ctx, *cluster, worker, podName, fqdnRayIP, "6379", "", 0, 0)
|
||
workerPod := BuildPod(ctx, podTemplateSpec, rayv1.WorkerNode, worker.RayStartParams, "6379", false, utils.GetCRDType(""), fqdnRayIP, nil, "")
|
||
workerContainer := workerPod.Spec.Containers[utils.RayContainerIndex]
|
||
assert.Equal(t, []string{"I am worker"}, workerContainer.Command)
|
||
assert.Equal(t, []string{"I am worker again"}, workerContainer.Args)
|
||
}
|
||
|
||
func TestBuildPod_WithAutoscalerEnabled(t *testing.T) {
|
||
ctx := context.Background()
|
||
cluster := instance.DeepCopy()
|
||
cluster.Spec.EnableInTreeAutoscaling = &trueFlag
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
podTemplateSpec := DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
pod := BuildPod(ctx, podTemplateSpec, rayv1.HeadNode, cluster.Spec.HeadGroupSpec.RayStartParams, "6379", true, utils.GetCRDType(""), "", nil, "")
|
||
|
||
assert.Equal(t, cluster.Name, pod.Labels[utils.RayClusterLabelKey])
|
||
assert.Equal(t, string(rayv1.HeadNode), pod.Labels[utils.RayNodeTypeLabelKey])
|
||
assert.Equal(t, utils.RayNodeHeadGroupLabelValue, pod.Labels[utils.RayNodeGroupLabelKey])
|
||
|
||
// verify no-monitoring is set. conversion only happens in BuildPod so we can only check it here.
|
||
assert.Contains(t, pod.Spec.Containers[0].Args[0], "--no-monitor")
|
||
|
||
actualVolumes := pod.Spec.Volumes
|
||
expectedVolumes := volumesWithAutoscaler
|
||
assert.True(t, reflect.DeepEqual(actualVolumes, expectedVolumes))
|
||
|
||
actualVolumeMounts := pod.Spec.Containers[0].VolumeMounts
|
||
expectedVolumeMounts := volumeMountsWithAutoscaler
|
||
assert.True(t, reflect.DeepEqual(actualVolumeMounts, expectedVolumeMounts))
|
||
|
||
// Make sure autoscaler container was formatted correctly.
|
||
assert.Len(t, pod.Spec.Containers, 2)
|
||
index := getAutoscalerContainerIndex(pod)
|
||
actualContainer := pod.Spec.Containers[index]
|
||
expectedContainer := autoscalerContainer
|
||
assert.True(t, reflect.DeepEqual(expectedContainer, actualContainer))
|
||
}
|
||
|
||
func TestBuildPod_WithCreatedByRayService(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
cluster := instance.DeepCopy()
|
||
cluster.Spec.EnableInTreeAutoscaling = &trueFlag
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
podTemplateSpec := DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
pod := BuildPod(ctx, podTemplateSpec, rayv1.HeadNode, cluster.Spec.HeadGroupSpec.RayStartParams, "6379", true, utils.RayServiceCRD, "", nil, "")
|
||
|
||
val, ok := pod.Labels[utils.RayClusterServingServiceLabelKey]
|
||
assert.True(t, ok, "Expected serve label is not present")
|
||
assert.Equal(t, utils.EnableRayClusterServingServiceFalse, val, "Wrong serve label value")
|
||
utils.EnvVarExists(utils.RAY_TIMEOUT_MS_TASK_WAIT_FOR_DEATH_INFO, pod.Spec.Containers[utils.RayContainerIndex].Env)
|
||
|
||
worker := cluster.Spec.WorkerGroupSpecs[0]
|
||
podName = cluster.Name + utils.DashSymbol + string(rayv1.WorkerNode) + utils.DashSymbol + worker.GroupName + utils.DashSymbol + utils.FormatInt32(0)
|
||
fqdnRayIP := utils.GenerateFQDNServiceName(ctx, *cluster, cluster.Namespace)
|
||
podTemplateSpec = DefaultWorkerPodTemplate(ctx, *cluster, worker, podName, fqdnRayIP, "6379", "", 0, 0)
|
||
pod = BuildPod(ctx, podTemplateSpec, rayv1.WorkerNode, worker.RayStartParams, "6379", false, utils.RayServiceCRD, fqdnRayIP, nil, "")
|
||
|
||
val, ok = pod.Labels[utils.RayClusterServingServiceLabelKey]
|
||
assert.True(t, ok, "Expected serve label is not present")
|
||
assert.Equal(t, utils.EnableRayClusterServingServiceTrue, val, "Wrong serve label value")
|
||
utils.EnvVarExists(utils.RAY_TIMEOUT_MS_TASK_WAIT_FOR_DEATH_INFO, pod.Spec.Containers[utils.RayContainerIndex].Env)
|
||
}
|
||
|
||
func TestBuildPod_WithLoginBash(t *testing.T) {
|
||
tests := []struct {
|
||
name string
|
||
expectedCmd []string
|
||
enableLoginBash bool
|
||
}{
|
||
{
|
||
name: "With login bash enabled",
|
||
enableLoginBash: true,
|
||
expectedCmd: []string{"/bin/bash", "-cl", "--"},
|
||
},
|
||
{
|
||
name: "Without login bash enabled",
|
||
enableLoginBash: false,
|
||
expectedCmd: []string{"/bin/bash", "-c", "--"},
|
||
},
|
||
}
|
||
|
||
for _, tc := range tests {
|
||
t.Run(tc.name, func(t *testing.T) {
|
||
ctx := context.Background()
|
||
if tc.enableLoginBash {
|
||
os.Setenv(utils.ENABLE_LOGIN_SHELL, "true")
|
||
defer os.Unsetenv(utils.ENABLE_LOGIN_SHELL)
|
||
}
|
||
|
||
cluster := instance.DeepCopy()
|
||
cluster.Spec.EnableInTreeAutoscaling = &trueFlag
|
||
|
||
// Test head pod
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
podTemplateSpec := DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
headPod := BuildPod(ctx, podTemplateSpec, rayv1.HeadNode, cluster.Spec.HeadGroupSpec.RayStartParams, "6379", true, utils.RayServiceCRD, "", nil, "")
|
||
|
||
// Verify head container command
|
||
headContainer := headPod.Spec.Containers[utils.RayContainerIndex]
|
||
assert.Equal(t, tc.expectedCmd, headContainer.Command)
|
||
|
||
// Verify autoscaler container command
|
||
index := getAutoscalerContainerIndex(headPod)
|
||
autoscalerContainer := headPod.Spec.Containers[index]
|
||
assert.Equal(t, tc.expectedCmd, autoscalerContainer.Command)
|
||
|
||
// Test worker pod
|
||
worker := cluster.Spec.WorkerGroupSpecs[0]
|
||
podName = cluster.Name + utils.DashSymbol + string(rayv1.WorkerNode) + utils.DashSymbol + worker.GroupName + utils.DashSymbol + utils.FormatInt32(0)
|
||
fqdnRayIP := utils.GenerateFQDNServiceName(ctx, *cluster, cluster.Namespace)
|
||
podTemplateSpec = DefaultWorkerPodTemplate(ctx, *cluster, worker, podName, fqdnRayIP, "6379", "", 0, 0)
|
||
workerPod := BuildPod(ctx, podTemplateSpec, rayv1.WorkerNode, worker.RayStartParams, "6379", false, utils.RayServiceCRD, fqdnRayIP, nil, "")
|
||
|
||
// Verify worker container command
|
||
workerContainer := workerPod.Spec.Containers[utils.RayContainerIndex]
|
||
assert.Equal(t, tc.expectedCmd, workerContainer.Command)
|
||
|
||
// Verify init container command
|
||
initContainers := workerPod.Spec.InitContainers
|
||
for _, initContainer := range initContainers {
|
||
if initContainer.Name == "wait-gcs-ready" {
|
||
assert.Equal(t, tc.expectedCmd, initContainer.Command)
|
||
}
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// Check that autoscaler container overrides work as expected.
|
||
func TestBuildPodWithAutoscalerOptions(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
cluster := instance.DeepCopy()
|
||
cluster.Spec.EnableInTreeAutoscaling = &trueFlag
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
|
||
customAutoscalerImage := "custom-autoscaler-xxx"
|
||
customPullPolicy := corev1.PullIfNotPresent
|
||
customTimeout := int32(100)
|
||
customUpscaling := rayv1.UpscalingMode("Aggressive")
|
||
customResources := corev1.ResourceRequirements{
|
||
Requests: corev1.ResourceList{
|
||
corev1.ResourceCPU: resource.MustParse("1"),
|
||
corev1.ResourceMemory: testMemoryLimit,
|
||
},
|
||
Limits: corev1.ResourceList{
|
||
corev1.ResourceCPU: resource.MustParse("1"),
|
||
corev1.ResourceMemory: testMemoryLimit,
|
||
},
|
||
}
|
||
customEnv := []corev1.EnvVar{{Name: "fooEnv", Value: "fooValue"}}
|
||
customEnvFrom := []corev1.EnvFromSource{{Prefix: "Pre"}}
|
||
customVolumeMounts := []corev1.VolumeMount{
|
||
{
|
||
Name: "ca-tls",
|
||
MountPath: "/etc/ca/tls",
|
||
ReadOnly: true,
|
||
},
|
||
{
|
||
Name: "ray-tls",
|
||
MountPath: "/etc/ray/tls",
|
||
},
|
||
}
|
||
|
||
// Define a custom security profile.
|
||
allowPrivilegeEscalation := false
|
||
capabilities := corev1.Capabilities{
|
||
Drop: []corev1.Capability{"ALL"},
|
||
}
|
||
seccompProfile := corev1.SeccompProfile{
|
||
Type: corev1.SeccompProfileTypeRuntimeDefault,
|
||
}
|
||
runAsNonRoot := true
|
||
customSecurityContext := corev1.SecurityContext{
|
||
AllowPrivilegeEscalation: &allowPrivilegeEscalation,
|
||
Capabilities: &capabilities,
|
||
RunAsNonRoot: &runAsNonRoot,
|
||
SeccompProfile: &seccompProfile,
|
||
}
|
||
|
||
cluster.Spec.AutoscalerOptions = &rayv1.AutoscalerOptions{
|
||
UpscalingMode: &customUpscaling,
|
||
IdleTimeoutSeconds: &customTimeout,
|
||
Image: &customAutoscalerImage,
|
||
ImagePullPolicy: &customPullPolicy,
|
||
Resources: &customResources,
|
||
Env: customEnv,
|
||
EnvFrom: customEnvFrom,
|
||
VolumeMounts: customVolumeMounts,
|
||
SecurityContext: &customSecurityContext,
|
||
}
|
||
podTemplateSpec := DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
pod := BuildPod(ctx, podTemplateSpec, rayv1.HeadNode, cluster.Spec.HeadGroupSpec.RayStartParams, "6379", true, utils.GetCRDType(""), "", nil, "")
|
||
expectedContainer := *autoscalerContainer.DeepCopy()
|
||
expectedContainer.Image = customAutoscalerImage
|
||
expectedContainer.ImagePullPolicy = customPullPolicy
|
||
expectedContainer.Resources = customResources
|
||
expectedContainer.EnvFrom = customEnvFrom
|
||
expectedContainer.Env = append(expectedContainer.Env, customEnv...)
|
||
expectedContainer.VolumeMounts = append(customVolumeMounts, expectedContainer.VolumeMounts...)
|
||
expectedContainer.SecurityContext = &customSecurityContext
|
||
index := getAutoscalerContainerIndex(pod)
|
||
actualContainer := pod.Spec.Containers[index]
|
||
assert.True(t, reflect.DeepEqual(expectedContainer, actualContainer))
|
||
}
|
||
|
||
func TestHeadPodTemplate_WithAutoscalingEnabled(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
cluster := instance.DeepCopy()
|
||
cluster.Spec.EnableInTreeAutoscaling = &trueFlag
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
podTemplateSpec := DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
|
||
// autoscaler container is injected into head pod
|
||
assert.Len(t, podTemplateSpec.Spec.Containers, 2)
|
||
|
||
assert.Equal(t, cluster.Name, podTemplateSpec.Spec.ServiceAccountName)
|
||
|
||
// Repeat ServiceAccountName check with long cluster name.
|
||
cluster.Name = longString(t) // 200 chars long
|
||
podTemplateSpec = DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
assert.Equal(t, shortString(t), podTemplateSpec.Spec.ServiceAccountName)
|
||
}
|
||
|
||
func TestDefaultHeadPodTemplate_Autoscaling(t *testing.T) {
|
||
clusterNoAutoscaling := instance.DeepCopy()
|
||
clusterAutoscalingVersionNotSet := instance.DeepCopy()
|
||
clusterAutoscalingVersionNotSet.Spec.EnableInTreeAutoscaling = new(true)
|
||
clusterAutoscalingV1 := instance.DeepCopy()
|
||
clusterAutoscalingV1.Spec.EnableInTreeAutoscaling = new(true)
|
||
clusterAutoscalingV1.Spec.AutoscalerOptions = &rayv1.AutoscalerOptions{
|
||
Version: ptr.To(rayv1.AutoscalerVersionV1),
|
||
}
|
||
clusterAutoscalingV2 := instance.DeepCopy()
|
||
clusterAutoscalingV2.Spec.EnableInTreeAutoscaling = new(true)
|
||
clusterAutoscalingV2.Spec.AutoscalerOptions = &rayv1.AutoscalerOptions{
|
||
Version: ptr.To(rayv1.AutoscalerVersionV2),
|
||
}
|
||
|
||
ctx := context.Background()
|
||
podName := strings.ToLower(instance.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
|
||
tests := map[string]struct {
|
||
expectedRestartPolicy corev1.RestartPolicy
|
||
cluster rayv1.RayCluster
|
||
expectedHeadContainers int
|
||
expectedAutoscalerV2EnvVar bool
|
||
expectedAutoscalerV1EnvVar bool
|
||
}{
|
||
"Pod template with autoscaling disabled should not have autoscaler container or other autoscaler related fields": {
|
||
cluster: *clusterNoAutoscaling,
|
||
expectedHeadContainers: 1,
|
||
expectedAutoscalerV2EnvVar: false,
|
||
expectedAutoscalerV1EnvVar: false,
|
||
expectedRestartPolicy: "",
|
||
},
|
||
"Pod template with autoscaling version not set should not have autoscaler env var": {
|
||
cluster: *clusterAutoscalingVersionNotSet,
|
||
expectedHeadContainers: 2,
|
||
expectedAutoscalerV2EnvVar: false,
|
||
expectedAutoscalerV1EnvVar: false,
|
||
expectedRestartPolicy: "",
|
||
},
|
||
"Pod template with autoscaling v1 enabled should the correct autoscaler v1 fields": {
|
||
cluster: *clusterAutoscalingV1,
|
||
expectedHeadContainers: 2,
|
||
expectedAutoscalerV2EnvVar: false,
|
||
expectedAutoscalerV1EnvVar: true,
|
||
expectedRestartPolicy: "",
|
||
},
|
||
"Pod template with autoscaling v2 enabled should the correct autoscaler v2 fields": {
|
||
cluster: *clusterAutoscalingV2,
|
||
expectedHeadContainers: 2,
|
||
expectedAutoscalerV2EnvVar: true,
|
||
expectedAutoscalerV1EnvVar: false,
|
||
expectedRestartPolicy: corev1.RestartPolicyNever,
|
||
},
|
||
}
|
||
|
||
for name, tc := range tests {
|
||
t.Run(name, func(t *testing.T) {
|
||
podTemplateSpec := DefaultHeadPodTemplate(ctx, tc.cluster, tc.cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
|
||
// if autoscaling is enabled, the head pod should have the autoscaler container appended for a total of 2 containers
|
||
if utils.IsAutoscalingEnabled(&tc.cluster.Spec) {
|
||
assert.Len(t, podTemplateSpec.Spec.Containers, tc.expectedHeadContainers)
|
||
}
|
||
|
||
if tc.expectedAutoscalerV2EnvVar {
|
||
assert.Contains(t, podTemplateSpec.Spec.Containers[0].Env, corev1.EnvVar{
|
||
Name: utils.RAY_ENABLE_AUTOSCALER_V2,
|
||
Value: "true",
|
||
})
|
||
} else {
|
||
assert.NotContains(t, podTemplateSpec.Spec.Containers[0].Env, corev1.EnvVar{
|
||
Name: utils.RAY_ENABLE_AUTOSCALER_V2,
|
||
Value: "true",
|
||
})
|
||
}
|
||
|
||
if tc.expectedAutoscalerV1EnvVar {
|
||
assert.Contains(t, podTemplateSpec.Spec.Containers[0].Env, corev1.EnvVar{
|
||
Name: utils.RAY_ENABLE_AUTOSCALER_V2,
|
||
Value: "false",
|
||
})
|
||
} else {
|
||
assert.NotContains(t, podTemplateSpec.Spec.Containers[0].Env, corev1.EnvVar{
|
||
Name: utils.RAY_ENABLE_AUTOSCALER_V2,
|
||
Value: "false",
|
||
})
|
||
}
|
||
|
||
assert.Equal(t, tc.expectedRestartPolicy, podTemplateSpec.Spec.RestartPolicy)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestHeadPodTemplate_AutoscalerImage(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
cluster := instance.DeepCopy()
|
||
cluster.Spec.EnableInTreeAutoscaling = &trueFlag
|
||
cluster.Spec.AutoscalerOptions = nil
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
|
||
// Case 1: If `AutoscalerOptions.Image` is not set, the Autoscaler container should use the Ray head container's image by default.
|
||
expectedAutoscalerImage := cluster.Spec.HeadGroupSpec.Template.Spec.Containers[utils.RayContainerIndex].Image
|
||
podTemplateSpec := DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
pod := corev1.Pod{
|
||
Spec: podTemplateSpec.Spec,
|
||
}
|
||
autoscalerContainerIndex := getAutoscalerContainerIndex(pod)
|
||
assert.Equal(t, expectedAutoscalerImage, podTemplateSpec.Spec.Containers[autoscalerContainerIndex].Image)
|
||
|
||
// Case 2: If `AutoscalerOptions.Image` is set, the Autoscaler container should use the specified image.
|
||
customAutoscalerImage := "custom-autoscaler-xxx"
|
||
cluster = instance.DeepCopy()
|
||
cluster.Spec.EnableInTreeAutoscaling = &trueFlag
|
||
cluster.Spec.AutoscalerOptions = &rayv1.AutoscalerOptions{
|
||
Image: &customAutoscalerImage,
|
||
}
|
||
podTemplateSpec = DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
pod.Spec = podTemplateSpec.Spec
|
||
autoscalerContainerIndex = getAutoscalerContainerIndex(pod)
|
||
assert.Equal(t, customAutoscalerImage, podTemplateSpec.Spec.Containers[autoscalerContainerIndex].Image)
|
||
}
|
||
|
||
// If no service account is specified in the RayCluster,
|
||
// the head pod's service account should be an empty string.
|
||
func TestHeadPodTemplate_WithNoServiceAccount(t *testing.T) {
|
||
cluster := instance.DeepCopy()
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
pod := DefaultHeadPodTemplate(context.Background(), *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
|
||
assert.Empty(t, pod.Spec.ServiceAccountName)
|
||
}
|
||
|
||
// If a service account is specified in the RayCluster and EnableInTreeAutoscaling is set to false,
|
||
// the head pod's service account should be the same.
|
||
func TestHeadPodTemplate_WithServiceAccountNoAutoscaling(t *testing.T) {
|
||
cluster := instance.DeepCopy()
|
||
serviceAccount := "head-service-account"
|
||
cluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = serviceAccount
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
pod := DefaultHeadPodTemplate(context.Background(), *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
|
||
assert.Equal(t, serviceAccount, pod.Spec.ServiceAccountName)
|
||
}
|
||
|
||
// If a service account is specified in the RayCluster and EnableInTreeAutoscaling is set to true,
|
||
// the head pod's service account should be the same.
|
||
func TestHeadPodTemplate_WithServiceAccount(t *testing.T) {
|
||
cluster := instance.DeepCopy()
|
||
serviceAccount := "head-service-account"
|
||
cluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = serviceAccount
|
||
cluster.Spec.EnableInTreeAutoscaling = &trueFlag
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
pod := DefaultHeadPodTemplate(context.Background(), *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
|
||
assert.Equal(t, serviceAccount, pod.Spec.ServiceAccountName)
|
||
}
|
||
|
||
func splitAndSort(s string) []string {
|
||
strs := strings.Split(s, " ")
|
||
result := make([]string, 0, len(strs))
|
||
for _, s := range strs {
|
||
if len(s) > 0 {
|
||
result = append(result, s)
|
||
}
|
||
}
|
||
sort.Strings(result)
|
||
return result
|
||
}
|
||
|
||
func TestDefaultWorkerPodTemplateWithName(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
cluster := instance.DeepCopy()
|
||
fqdnRayIP := utils.GenerateFQDNServiceName(ctx, *cluster, cluster.Namespace)
|
||
worker := cluster.Spec.WorkerGroupSpecs[0]
|
||
worker.Template.ObjectMeta.Name = "ray-worker-test"
|
||
podName := cluster.Name + utils.DashSymbol + string(rayv1.WorkerNode) + utils.DashSymbol + worker.GroupName + utils.DashSymbol + utils.FormatInt32(0)
|
||
expectedWorker := *worker.DeepCopy()
|
||
|
||
// Pass a deep copy of worker (*worker.DeepCopy()) to prevent "worker" from updating.
|
||
podTemplateSpec := DefaultWorkerPodTemplate(ctx, *cluster, *worker.DeepCopy(), podName, fqdnRayIP, "6379", "", 0, 0)
|
||
assert.Empty(t, podTemplateSpec.ObjectMeta.Name)
|
||
assert.Equal(t, expectedWorker, worker)
|
||
}
|
||
|
||
func TestDeafultWorkerPodTemplateWithReplicaGrpAndIndex(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
cluster := instance.DeepCopy()
|
||
|
||
fqdnRayIP := utils.GenerateFQDNServiceName(ctx, *cluster, cluster.Namespace)
|
||
worker := cluster.Spec.WorkerGroupSpecs[0]
|
||
|
||
features.SetFeatureGateDuringTest(t, features.RayMultiHostIndexing, true)
|
||
|
||
worker.Template.ObjectMeta.Name = "ray-worker-test"
|
||
worker.NumOfHosts = 4
|
||
podName := cluster.Name + utils.DashSymbol + string(rayv1.WorkerNode) + utils.DashSymbol + worker.GroupName + utils.DashSymbol + utils.FormatInt32(0)
|
||
groupReplicaName := utils.GenerateRayWorkerReplicaGroupName(worker.GroupName)
|
||
|
||
// Pass a deep copy of worker (*worker.DeepCopy()) to prevent "worker" from updating.
|
||
podTemplateSpec := DefaultWorkerPodTemplate(ctx, *cluster, *worker.DeepCopy(), podName, fqdnRayIP, "6379", groupReplicaName, 0, 2)
|
||
assert.Empty(t, podTemplateSpec.ObjectMeta.Name)
|
||
assert.Equal(t, podTemplateSpec.Labels[utils.RayWorkerReplicaNameKey], groupReplicaName)
|
||
assert.Equal(t, "0", podTemplateSpec.Labels[utils.RayWorkerReplicaIndexKey])
|
||
assert.Equal(t, "2", podTemplateSpec.Labels[utils.RayHostIndexKey])
|
||
}
|
||
|
||
func containerPortExists(ports []corev1.ContainerPort, containerPort int32) error {
|
||
name := utils.MetricsPortName
|
||
for _, port := range ports {
|
||
if port.Name == name {
|
||
if port.ContainerPort != containerPort {
|
||
return fmt.Errorf("expected `%v` but got `%v` for `%v` port", containerPort, port.ContainerPort, name)
|
||
}
|
||
return nil
|
||
}
|
||
}
|
||
return fmt.Errorf("couldn't find `%v` port", name)
|
||
}
|
||
|
||
func TestDefaultHeadPodTemplateWithConfigurablePorts(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
cluster := instance.DeepCopy()
|
||
cluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{}
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
podTemplateSpec := DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
// DefaultHeadPodTemplate will add the default metrics port if user doesn't specify it.
|
||
// Verify the default metrics port exists.
|
||
require.NoError(t, containerPortExists(podTemplateSpec.Spec.Containers[0].Ports, int32(utils.DefaultMetricsPort)))
|
||
customMetricsPort := int32(utils.DefaultMetricsPort) + 1
|
||
metricsPort := corev1.ContainerPort{
|
||
Name: utils.MetricsPortName,
|
||
ContainerPort: customMetricsPort,
|
||
}
|
||
cluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{metricsPort}
|
||
podTemplateSpec = DefaultHeadPodTemplate(ctx, *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
// Verify the custom metrics port exists.
|
||
require.NoError(t, containerPortExists(podTemplateSpec.Spec.Containers[0].Ports, customMetricsPort))
|
||
}
|
||
|
||
func TestDefaultWorkerPodTemplateWithConfigurablePorts(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
cluster := instance.DeepCopy()
|
||
cluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Ports = []corev1.ContainerPort{}
|
||
worker := cluster.Spec.WorkerGroupSpecs[0]
|
||
podName := cluster.Name + utils.DashSymbol + string(rayv1.WorkerNode) + utils.DashSymbol + worker.GroupName + utils.DashSymbol + utils.FormatInt32(0)
|
||
fqdnRayIP := utils.GenerateFQDNServiceName(ctx, *cluster, cluster.Namespace)
|
||
podTemplateSpec := DefaultWorkerPodTemplate(ctx, *cluster, worker, podName, fqdnRayIP, "6379", "", 0, 0)
|
||
// DefaultWorkerPodTemplate will add the default metrics port if user doesn't specify it.
|
||
// Verify the default metrics port exists.
|
||
require.NoError(t, containerPortExists(podTemplateSpec.Spec.Containers[0].Ports, int32(utils.DefaultMetricsPort)))
|
||
customMetricsPort := int32(utils.DefaultMetricsPort) + 1
|
||
metricsPort := corev1.ContainerPort{
|
||
Name: utils.MetricsPortName,
|
||
ContainerPort: customMetricsPort,
|
||
}
|
||
cluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Ports = []corev1.ContainerPort{metricsPort}
|
||
podTemplateSpec = DefaultWorkerPodTemplate(ctx, *cluster, worker, podName, fqdnRayIP, "6379", "", 0, 0)
|
||
// Verify the custom metrics port exists.
|
||
require.NoError(t, containerPortExists(podTemplateSpec.Spec.Containers[0].Ports, customMetricsPort))
|
||
}
|
||
|
||
func TestDefaultWorkerPodTemplate_Autoscaling(t *testing.T) {
|
||
clusterNoAutoscaling := instance.DeepCopy()
|
||
clusterAutoscalingV1 := instance.DeepCopy()
|
||
clusterAutoscalingV1.Spec.EnableInTreeAutoscaling = new(true)
|
||
clusterAutoscalingV2 := instance.DeepCopy()
|
||
clusterAutoscalingV2.Spec.EnableInTreeAutoscaling = new(true)
|
||
clusterAutoscalingV2.Spec.AutoscalerOptions = &rayv1.AutoscalerOptions{
|
||
Version: ptr.To(rayv1.AutoscalerVersionV2),
|
||
}
|
||
|
||
ctx := context.Background()
|
||
podName := strings.ToLower(instance.Name + utils.DashSymbol + string(rayv1.WorkerNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
fqdnRayIP := utils.GenerateFQDNServiceName(ctx, instance, instance.Namespace)
|
||
|
||
tests := map[string]struct {
|
||
expectedRestartPolicy corev1.RestartPolicy
|
||
cluster rayv1.RayCluster
|
||
}{
|
||
"Pod template with autoscaling disabled should not have autoscaler container or other autoscaler related fields": {
|
||
cluster: *clusterNoAutoscaling,
|
||
expectedRestartPolicy: "",
|
||
},
|
||
"Pod template with autoscaling v1 enabled should the correct autoscaler v1 fields": {
|
||
cluster: *clusterAutoscalingV1,
|
||
expectedRestartPolicy: "",
|
||
},
|
||
"Pod template with autoscaling v2 enabled should the correct autoscaler v2 fields": {
|
||
cluster: *clusterAutoscalingV2,
|
||
expectedRestartPolicy: corev1.RestartPolicyNever,
|
||
},
|
||
}
|
||
|
||
for name, tc := range tests {
|
||
t.Run(name, func(t *testing.T) {
|
||
podTemplateSpec := DefaultWorkerPodTemplate(ctx, tc.cluster, tc.cluster.Spec.WorkerGroupSpecs[0], podName, fqdnRayIP, "6379", "", 0, 0)
|
||
assert.Equal(t, tc.expectedRestartPolicy, podTemplateSpec.Spec.RestartPolicy)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestDefaultInitContainer(t *testing.T) {
|
||
ctx := context.Background()
|
||
// A default init container to check the health of GCS is expected to be added.
|
||
cluster := instance.DeepCopy()
|
||
fqdnRayIP := utils.GenerateFQDNServiceName(ctx, *cluster, cluster.Namespace)
|
||
worker := cluster.Spec.WorkerGroupSpecs[0]
|
||
podName := cluster.Name + utils.DashSymbol + string(rayv1.WorkerNode) + utils.DashSymbol + worker.GroupName + utils.DashSymbol + utils.FormatInt32(0)
|
||
expectedResult := len(cluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers) + 1
|
||
|
||
// Pass a deep copy of worker (*worker.DeepCopy()) to prevent "worker" from updating.
|
||
podTemplateSpec := DefaultWorkerPodTemplate(ctx, *cluster, *worker.DeepCopy(), podName, fqdnRayIP, "6379", "", 0, 0)
|
||
numInitContainers := len(podTemplateSpec.Spec.InitContainers)
|
||
assert.Equal(t, expectedResult, numInitContainers, "A default init container is expected to be added.")
|
||
|
||
// This health-check init container requires certain environment variables to establish a secure connection
|
||
// with the Ray head using TLS authentication. Currently, we simply copied all environment variables from
|
||
// Ray container to the init container. This may be changed in the future.
|
||
healthCheckContainer := podTemplateSpec.Spec.InitContainers[numInitContainers-1]
|
||
rayContainer := worker.Template.Spec.Containers[utils.RayContainerIndex]
|
||
|
||
assert.NotEmpty(t, rayContainer.Env, "The test only makes sense if the Ray container has environment variables.")
|
||
assert.Len(t, healthCheckContainer.Env, len(rayContainer.Env))
|
||
for _, env := range rayContainer.Env {
|
||
// env.ValueFrom is the source for the environment variable's value. Cannot be used if value is not empty.
|
||
if env.Value != "" {
|
||
checkContainerEnv(t, healthCheckContainer, env.Name, env.Value)
|
||
} else {
|
||
checkContainerEnv(t, healthCheckContainer, env.Name, env.ValueFrom.FieldRef.FieldPath)
|
||
}
|
||
}
|
||
|
||
assert.NotEmpty(t, rayContainer.Resources, "The test only makes sense if the Ray container has resource limit/request.")
|
||
}
|
||
|
||
func TestDefaultInitContainerImagePullPolicy(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
cluster := instance.DeepCopy()
|
||
fqdnRayIP := utils.GenerateFQDNServiceName(ctx, *cluster, cluster.Namespace)
|
||
worker := cluster.Spec.WorkerGroupSpecs[0]
|
||
podName := cluster.Name + utils.DashSymbol + string(rayv1.WorkerNode) + utils.DashSymbol + worker.GroupName + utils.DashSymbol + utils.FormatInt32(0)
|
||
|
||
cases := []struct {
|
||
name string
|
||
imagePullPolicy corev1.PullPolicy
|
||
expectedPullPolicy corev1.PullPolicy
|
||
}{
|
||
{
|
||
name: "Always",
|
||
imagePullPolicy: corev1.PullAlways,
|
||
expectedPullPolicy: corev1.PullAlways,
|
||
},
|
||
{
|
||
name: "IfNotPresent",
|
||
imagePullPolicy: corev1.PullIfNotPresent,
|
||
expectedPullPolicy: corev1.PullIfNotPresent,
|
||
},
|
||
{
|
||
name: "Never",
|
||
imagePullPolicy: corev1.PullNever,
|
||
expectedPullPolicy: corev1.PullNever,
|
||
},
|
||
}
|
||
|
||
for _, tc := range cases {
|
||
t.Run(tc.name, func(t *testing.T) {
|
||
// set ray container imagePullPolicy
|
||
worker.Template.Spec.Containers[utils.RayContainerIndex].ImagePullPolicy = tc.imagePullPolicy
|
||
|
||
podTemplateSpec := DefaultWorkerPodTemplate(ctx, *cluster, *worker.DeepCopy(), podName, fqdnRayIP, "6379", "", 0, 0)
|
||
|
||
healthCheckContainer := podTemplateSpec.Spec.InitContainers[len(podTemplateSpec.Spec.InitContainers)-1]
|
||
assert.Equal(t, tc.expectedPullPolicy, healthCheckContainer.ImagePullPolicy, "The ImagePullPolicy of the init container should be the same as the Ray container.")
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestSetMissingRayStartParamsAddress(t *testing.T) {
|
||
ctx := context.Background()
|
||
// The address option is automatically injected into RayStartParams with a default value of <Fully Qualified Domain Name (FQDN)>:<headPort> for workers only, which is used to connect to the Ray cluster.
|
||
// The head should not include the address option in RayStartParams, as it can access the GCS server, which also runs on the head, via localhost.
|
||
// Although not recommended, users can manually set the address option for both head and workers.
|
||
|
||
headPort := "6379"
|
||
fqdnRayIP := "raycluster-kuberay-head-svc.default.svc.cluster.local"
|
||
customAddress := "custom-address:1234"
|
||
|
||
testCases := []struct {
|
||
name string
|
||
assertion func(t *testing.T, rayStartParams map[string]string)
|
||
rayStartParams map[string]string
|
||
fqdnRayIP string
|
||
nodeType rayv1.RayNodeType
|
||
}{
|
||
{
|
||
name: "Head node with no address option set.",
|
||
rayStartParams: map[string]string{},
|
||
fqdnRayIP: "",
|
||
nodeType: rayv1.HeadNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.NotContains(t, rayStartParams, "address", "Head node should not have an address option set by default.")
|
||
},
|
||
},
|
||
{
|
||
name: "Head node with custom address option set.",
|
||
rayStartParams: map[string]string{"address": customAddress},
|
||
fqdnRayIP: "",
|
||
nodeType: rayv1.HeadNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.Equalf(t, customAddress, rayStartParams["address"], "Expected `%v` but got `%v`", customAddress, rayStartParams["address"])
|
||
},
|
||
},
|
||
{
|
||
name: "Worker node with no address option set.",
|
||
rayStartParams: map[string]string{},
|
||
fqdnRayIP: fqdnRayIP,
|
||
nodeType: rayv1.WorkerNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
expectedAddress := fmt.Sprintf("%s:%s", fqdnRayIP, headPort)
|
||
assert.Equalf(t, expectedAddress, rayStartParams["address"], "Expected `%v` but got `%v`", expectedAddress, rayStartParams["address"])
|
||
},
|
||
},
|
||
{
|
||
name: "Worker node with custom address option set.",
|
||
rayStartParams: map[string]string{"address": customAddress},
|
||
fqdnRayIP: fqdnRayIP,
|
||
nodeType: rayv1.WorkerNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.Equalf(t, customAddress, rayStartParams["address"], "Expected `%v` but got `%v`", customAddress, rayStartParams["address"])
|
||
},
|
||
},
|
||
}
|
||
for _, testCase := range testCases {
|
||
t.Run(testCase.name, func(t *testing.T) {
|
||
testCase.rayStartParams = setMissingRayStartParams(ctx, testCase.rayStartParams, testCase.nodeType, headPort, testCase.fqdnRayIP)
|
||
testCase.assertion(t, testCase.rayStartParams)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestSetMissingRayStartParamsMetricsExportPort(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
// The metrics-export-port option is automatically injected into RayStartParams with a default value of DefaultMetricsPort for both head and workers.
|
||
// Users can manually set the metrics-export-port option to customize the metrics export port and scrape Ray’s metrics using Prometheus via <ip>:<custom metrics export port>.
|
||
// See https://github.com/ray-project/kuberay/pull/954 for more details.
|
||
|
||
headPort := "6379"
|
||
fqdnRayIP := "raycluster-kuberay-head-svc.default.svc.cluster.local"
|
||
customMetricsPort := utils.DefaultMetricsPort + 1
|
||
testCases := []struct {
|
||
name string
|
||
assertion func(t *testing.T, rayStartParams map[string]string)
|
||
rayStartParams map[string]string
|
||
fqdnRayIP string
|
||
nodeType rayv1.RayNodeType
|
||
}{
|
||
{
|
||
name: "Head node with no metrics-export-port option set.",
|
||
rayStartParams: map[string]string{},
|
||
fqdnRayIP: "",
|
||
nodeType: rayv1.HeadNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.Equalf(t, fmt.Sprint(utils.DefaultMetricsPort), rayStartParams["metrics-export-port"], "Expected `%v` but got `%v`", utils.DefaultMetricsPort, rayStartParams["metrics-export-port"])
|
||
},
|
||
},
|
||
{
|
||
name: "Head node with custom metrics-export-port option set.",
|
||
rayStartParams: map[string]string{"metrics-export-port": fmt.Sprint(customMetricsPort)},
|
||
fqdnRayIP: "",
|
||
nodeType: rayv1.HeadNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.Equalf(t, fmt.Sprint(customMetricsPort), rayStartParams["metrics-export-port"], "Expected `%v` but got `%v`", customMetricsPort, rayStartParams["metrics-export-port"])
|
||
},
|
||
},
|
||
{
|
||
name: "Worker node with no metrics-export-port option set.",
|
||
rayStartParams: map[string]string{},
|
||
fqdnRayIP: fqdnRayIP,
|
||
nodeType: rayv1.WorkerNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.Equalf(t, fmt.Sprint(utils.DefaultMetricsPort), rayStartParams["metrics-export-port"], "Expected `%v` but got `%v`", utils.DefaultMetricsPort, rayStartParams["metrics-export-port"])
|
||
},
|
||
},
|
||
{
|
||
name: "Worker node with custom metrics-export-port option set.",
|
||
rayStartParams: map[string]string{"metrics-export-port": fmt.Sprint(customMetricsPort)},
|
||
fqdnRayIP: fqdnRayIP,
|
||
nodeType: rayv1.WorkerNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.Equalf(t, fmt.Sprint(customMetricsPort), rayStartParams["metrics-export-port"], "Expected `%v` but got `%v`", customMetricsPort, rayStartParams["metrics-export-port"])
|
||
},
|
||
},
|
||
}
|
||
|
||
for _, testCase := range testCases {
|
||
t.Run(testCase.name, func(t *testing.T) {
|
||
testCase.rayStartParams = setMissingRayStartParams(ctx, testCase.rayStartParams, testCase.nodeType, headPort, testCase.fqdnRayIP)
|
||
testCase.assertion(t, testCase.rayStartParams)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestSetMissingRayStartParamsBlock(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
// The --block option is automatically injected into RayStartParams with a default value of true for both head and workers.
|
||
// Although not recommended, users can manually set the --block option.
|
||
// See https://github.com/ray-project/kuberay/pull/675 for more details.
|
||
|
||
headPort := "6379"
|
||
fqdnRayIP := "raycluster-kuberay-head-svc.default.svc.cluster.local"
|
||
testCases := []struct {
|
||
name string
|
||
assertion func(t *testing.T, rayStartParams map[string]string)
|
||
rayStartParams map[string]string
|
||
fqdnRayIP string
|
||
nodeType rayv1.RayNodeType
|
||
}{
|
||
{
|
||
name: "Head node with no --block option set.",
|
||
rayStartParams: map[string]string{},
|
||
fqdnRayIP: "",
|
||
nodeType: rayv1.HeadNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.Equalf(t, "true", rayStartParams["block"], "Expected `%v` but got `%v`", "true", rayStartParams["block"])
|
||
},
|
||
},
|
||
{
|
||
name: "Head node with --block option set to false.",
|
||
rayStartParams: map[string]string{"block": "false"},
|
||
fqdnRayIP: "",
|
||
nodeType: rayv1.HeadNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.Equalf(t, "true", rayStartParams["block"], "Expected `%v` but got `%v`", "false", rayStartParams["block"])
|
||
},
|
||
},
|
||
{
|
||
name: "Worker node with no --block option set.",
|
||
rayStartParams: map[string]string{},
|
||
fqdnRayIP: fqdnRayIP,
|
||
nodeType: rayv1.WorkerNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.Equalf(t, "true", rayStartParams["block"], "Expected `%v` but got `%v`", "true", rayStartParams["block"])
|
||
},
|
||
},
|
||
{
|
||
name: "Worker node with --block option set to false.",
|
||
rayStartParams: map[string]string{"block": "false"},
|
||
fqdnRayIP: fqdnRayIP,
|
||
nodeType: rayv1.WorkerNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.Equalf(t, "true", rayStartParams["block"], "Expected `%v` but got `%v`", "false", rayStartParams["block"])
|
||
},
|
||
},
|
||
}
|
||
for _, testCase := range testCases {
|
||
t.Run(testCase.name, func(t *testing.T) {
|
||
testCase.rayStartParams = setMissingRayStartParams(ctx, testCase.rayStartParams, testCase.nodeType, headPort, testCase.fqdnRayIP)
|
||
testCase.assertion(t, testCase.rayStartParams)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestSetMissingRayStartParamsDashboardHost(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
// The dashboard-host option is automatically injected into RayStartParams with a default value of "0.0.0.0" for head only as workers do not have dashborad server.
|
||
// Users can manually set the dashboard-host option to customize the host the dashboard server binds to, either "localhost" (127.0.0.1) or "0.0.0.0" (available from all interfaces).
|
||
headPort := "6379"
|
||
fqdnRayIP := "raycluster-kuberay-head-svc.default.svc.cluster.local"
|
||
testCases := []struct {
|
||
name string
|
||
assertion func(t *testing.T, rayStartParams map[string]string)
|
||
rayStartParams map[string]string
|
||
fqdnRayIP string
|
||
nodeType rayv1.RayNodeType
|
||
}{
|
||
{
|
||
name: "Head node with no dashboard-host option set.",
|
||
rayStartParams: map[string]string{},
|
||
fqdnRayIP: "",
|
||
nodeType: rayv1.HeadNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.Equalf(t, "0.0.0.0", rayStartParams["dashboard-host"], "Expected `%v` but got `%v`", "0.0.0.0", rayStartParams["dashboard-host"])
|
||
},
|
||
},
|
||
{
|
||
name: "Head node with dashboard-host option set.",
|
||
rayStartParams: map[string]string{"dashboard-host": "localhost"},
|
||
fqdnRayIP: "",
|
||
nodeType: rayv1.HeadNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.Equalf(t, "localhost", rayStartParams["dashboard-host"], "Expected `%v` but got `%v`", "localhost", rayStartParams["dashboard-host"])
|
||
},
|
||
},
|
||
{
|
||
name: "Worker node with no dashboard-host option set.",
|
||
rayStartParams: map[string]string{},
|
||
fqdnRayIP: fqdnRayIP,
|
||
nodeType: rayv1.WorkerNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.NotContains(t, rayStartParams, "dashboard-host", "workers should not have an dashboard-host option set.")
|
||
},
|
||
},
|
||
{
|
||
name: "Worker node with dashboard-host option set.",
|
||
rayStartParams: map[string]string{"dashboard-host": "localhost"},
|
||
fqdnRayIP: fqdnRayIP,
|
||
nodeType: rayv1.WorkerNode,
|
||
assertion: func(t *testing.T, rayStartParams map[string]string) {
|
||
assert.Equalf(t, "localhost", rayStartParams["dashboard-host"], "Expected `%v` but got `%v`", "localhost", rayStartParams["dashboard-host"])
|
||
},
|
||
},
|
||
}
|
||
|
||
for _, testCase := range testCases {
|
||
t.Run(testCase.name, func(t *testing.T) {
|
||
testCase.rayStartParams = setMissingRayStartParams(ctx, testCase.rayStartParams, testCase.nodeType, headPort, testCase.fqdnRayIP)
|
||
testCase.assertion(t, testCase.rayStartParams)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestGetCustomWorkerInitImage(t *testing.T) {
|
||
// cleanup
|
||
defer os.Unsetenv(EnableInitContainerInjectionEnvKey)
|
||
|
||
// not set the env
|
||
b := getEnableInitContainerInjection()
|
||
assert.True(t, b)
|
||
// set the env with "true"
|
||
os.Setenv(EnableInitContainerInjectionEnvKey, "true")
|
||
b = getEnableInitContainerInjection()
|
||
assert.True(t, b)
|
||
// set the env with "True"
|
||
os.Setenv(EnableInitContainerInjectionEnvKey, "True")
|
||
b = getEnableInitContainerInjection()
|
||
assert.True(t, b)
|
||
// set the env with "false"
|
||
os.Setenv(EnableInitContainerInjectionEnvKey, "false")
|
||
b = getEnableInitContainerInjection()
|
||
assert.False(t, b)
|
||
// set the env with "False"
|
||
os.Setenv(EnableInitContainerInjectionEnvKey, "False")
|
||
b = getEnableInitContainerInjection()
|
||
assert.False(t, b)
|
||
}
|
||
|
||
func TestGetEnableProbesInjection(t *testing.T) {
|
||
// cleanup
|
||
defer os.Unsetenv(utils.ENABLE_PROBES_INJECTION)
|
||
|
||
// not set the env
|
||
os.Unsetenv(utils.ENABLE_PROBES_INJECTION)
|
||
b := getEnableProbesInjection()
|
||
assert.True(t, b)
|
||
// set the env with "true"
|
||
os.Setenv(utils.ENABLE_PROBES_INJECTION, "true")
|
||
b = getEnableProbesInjection()
|
||
assert.True(t, b)
|
||
// set the env with "True"
|
||
os.Setenv(utils.ENABLE_PROBES_INJECTION, "True")
|
||
b = getEnableProbesInjection()
|
||
assert.True(t, b)
|
||
// set the env with "false"
|
||
os.Setenv(utils.ENABLE_PROBES_INJECTION, "false")
|
||
b = getEnableProbesInjection()
|
||
assert.False(t, b)
|
||
// set the env with "False"
|
||
os.Setenv(utils.ENABLE_PROBES_INJECTION, "False")
|
||
b = getEnableProbesInjection()
|
||
assert.False(t, b)
|
||
}
|
||
|
||
func TestInitLivenessAndReadinessProbe(t *testing.T) {
|
||
cluster := instance.DeepCopy()
|
||
podName := strings.ToLower(cluster.Name + utils.DashSymbol + string(rayv1.HeadNode) + utils.DashSymbol + utils.FormatInt32(0))
|
||
podTemplateSpec := DefaultHeadPodTemplate(context.Background(), *cluster, cluster.Spec.HeadGroupSpec, podName, "6379")
|
||
rayContainer := &podTemplateSpec.Spec.Containers[utils.RayContainerIndex]
|
||
rayStartParams := make(map[string]string)
|
||
|
||
// Test 1: User defines a custom HTTPGet probe.
|
||
httpGetProbe := corev1.Probe{
|
||
ProbeHandler: corev1.ProbeHandler{
|
||
HTTPGet: &corev1.HTTPGetAction{
|
||
// Check Raylet status
|
||
Path: fmt.Sprintf("/%s", utils.RayAgentRayletHealthPath),
|
||
Port: intstr.FromInt(utils.DefaultDashboardAgentListenPort),
|
||
},
|
||
},
|
||
}
|
||
|
||
rayContainer.LivenessProbe = &httpGetProbe
|
||
rayContainer.ReadinessProbe = &httpGetProbe
|
||
initLivenessAndReadinessProbe(rayContainer, rayv1.HeadNode, "", rayStartParams, "")
|
||
assert.NotNil(t, rayContainer.LivenessProbe.HTTPGet)
|
||
assert.NotNil(t, rayContainer.ReadinessProbe.HTTPGet)
|
||
assert.Nil(t, rayContainer.LivenessProbe.Exec)
|
||
assert.Nil(t, rayContainer.ReadinessProbe.Exec)
|
||
|
||
// Test 2: User does not define a custom probe. KubeRay will inject Exec probe for worker pod.
|
||
// Here we test the case where the Ray Pod originates from RayServiceCRD,
|
||
// implying that an additional serve health check will be added to the readiness probe.
|
||
rayContainer.LivenessProbe = nil
|
||
rayContainer.ReadinessProbe = nil
|
||
initLivenessAndReadinessProbe(rayContainer, rayv1.WorkerNode, utils.RayServiceCRD, rayStartParams, "")
|
||
assert.NotNil(t, rayContainer.LivenessProbe.Exec)
|
||
assert.NotNil(t, rayContainer.ReadinessProbe.Exec)
|
||
assert.NotContains(t, strings.Join(rayContainer.LivenessProbe.Exec.Command, " "), utils.RayServeProxyHealthPath)
|
||
assert.Contains(t, strings.Join(rayContainer.ReadinessProbe.Exec.Command, " "), utils.RayServeProxyHealthPath)
|
||
assert.Equal(t, int32(2), rayContainer.LivenessProbe.TimeoutSeconds)
|
||
assert.Equal(t, int32(2), rayContainer.ReadinessProbe.TimeoutSeconds)
|
||
|
||
// Test 3: User does not define a custom probe. KubeRay will inject Exec probe for head pod.
|
||
// Here we test the case where the Ray Pod originates from RayServiceCRD,
|
||
// implying that an additional serve health check will be added to the readiness probe.
|
||
rayContainer.LivenessProbe = nil
|
||
rayContainer.ReadinessProbe = nil
|
||
initLivenessAndReadinessProbe(rayContainer, rayv1.HeadNode, utils.RayServiceCRD, rayStartParams, "")
|
||
assert.NotNil(t, rayContainer.LivenessProbe.Exec)
|
||
assert.NotNil(t, rayContainer.ReadinessProbe.Exec)
|
||
// head pod should not have Ray Serve proxy health probes
|
||
assert.NotContains(t, strings.Join(rayContainer.LivenessProbe.Exec.Command, " "), utils.RayServeProxyHealthPath)
|
||
assert.NotContains(t, strings.Join(rayContainer.ReadinessProbe.Exec.Command, " "), utils.RayServeProxyHealthPath)
|
||
assert.Equal(t, int32(5), rayContainer.LivenessProbe.TimeoutSeconds)
|
||
assert.Equal(t, int32(5), rayContainer.ReadinessProbe.TimeoutSeconds)
|
||
|
||
// Test 4: Test custom ports in rayStartParams for head node.
|
||
rayContainer.LivenessProbe = nil
|
||
rayContainer.ReadinessProbe = nil
|
||
customRayStartParams := map[string]string{
|
||
"dashboard-agent-listen-port": "8266",
|
||
"dashboard-port": "8365",
|
||
}
|
||
initLivenessAndReadinessProbe(rayContainer, rayv1.HeadNode, utils.RayClusterCRD, customRayStartParams, "")
|
||
assert.NotNil(t, rayContainer.LivenessProbe.Exec)
|
||
assert.NotNil(t, rayContainer.ReadinessProbe.Exec)
|
||
|
||
livenessCommand := strings.Join(rayContainer.LivenessProbe.Exec.Command, " ")
|
||
readinessCommand := strings.Join(rayContainer.ReadinessProbe.Exec.Command, " ")
|
||
|
||
assert.Contains(t, livenessCommand, ":8266", "Head pod liveness probe should use custom dashboard-agent-listen-port")
|
||
assert.Contains(t, livenessCommand, ":8365", "Head pod liveness probe should use custom dashboard-port")
|
||
assert.Contains(t, readinessCommand, ":8266", "Head pod readiness probe should use custom dashboard-agent-listen-port")
|
||
assert.Contains(t, readinessCommand, ":8365", "Head pod readiness probe should use custom dashboard-port")
|
||
|
||
// Test 5: Test custom ports in rayStartParams for worker node
|
||
rayContainer.LivenessProbe = nil
|
||
rayContainer.ReadinessProbe = nil
|
||
workerRayStartParams := map[string]string{
|
||
"dashboard-agent-listen-port": "9000",
|
||
}
|
||
initLivenessAndReadinessProbe(rayContainer, rayv1.WorkerNode, utils.RayClusterCRD, workerRayStartParams, "")
|
||
assert.NotNil(t, rayContainer.LivenessProbe.Exec)
|
||
assert.NotNil(t, rayContainer.ReadinessProbe.Exec)
|
||
|
||
workerLivenessCommand := strings.Join(rayContainer.LivenessProbe.Exec.Command, " ")
|
||
workerReadinessCommand := strings.Join(rayContainer.ReadinessProbe.Exec.Command, " ")
|
||
|
||
assert.Contains(t, workerLivenessCommand, ":9000", "Worker pod should use custom dashboard-agent-listen-port")
|
||
assert.Contains(t, workerReadinessCommand, ":9000", "Worker pod should use custom dashboard-agent-listen-port")
|
||
assert.NotContains(t, workerLivenessCommand, fmt.Sprintf(":%d", utils.DefaultDashboardPort), "Worker pod should not check dashboard-port")
|
||
assert.NotContains(t, workerReadinessCommand, fmt.Sprintf(":%d", utils.DefaultDashboardPort), "Worker pod should not check dashboard-port")
|
||
|
||
// Test 6: Test RayService worker with custom ports and serve proxy health check
|
||
rayContainer.LivenessProbe = nil
|
||
rayContainer.ReadinessProbe = nil
|
||
rayContainer.Ports = []corev1.ContainerPort{
|
||
{
|
||
Name: utils.ServingPortName,
|
||
ContainerPort: int32(utils.DefaultServingPort),
|
||
},
|
||
}
|
||
rayServiceWorkerParams := map[string]string{
|
||
"dashboard-agent-listen-port": "8500",
|
||
}
|
||
initLivenessAndReadinessProbe(rayContainer, rayv1.WorkerNode, utils.RayServiceCRD, rayServiceWorkerParams, "")
|
||
rayServiceReadinessCommand := strings.Join(rayContainer.ReadinessProbe.Exec.Command, " ")
|
||
assert.Contains(t, rayServiceReadinessCommand, ":8500", "RayService worker should use custom dashboard-agent-listen-port")
|
||
assert.Contains(t, rayServiceReadinessCommand, utils.RayServeProxyHealthPath, "RayService worker should include serve proxy health check")
|
||
assert.Equal(t, int32(utils.ServeReadinessProbeFailureThreshold), rayContainer.ReadinessProbe.FailureThreshold, "RayService worker should have correct failure threshold")
|
||
|
||
// Test 8: Test invalid port values (should fall back to defaults)
|
||
rayContainer.LivenessProbe = nil
|
||
rayContainer.ReadinessProbe = nil
|
||
invalidPortParams := map[string]string{
|
||
"dashboard-agent-listen-port": "invalid-port",
|
||
"dashboard-port": "not-a-number",
|
||
}
|
||
initLivenessAndReadinessProbe(rayContainer, rayv1.HeadNode, utils.RayClusterCRD, invalidPortParams, "")
|
||
|
||
invalidPortLivenessCommand := strings.Join(rayContainer.LivenessProbe.Exec.Command, " ")
|
||
|
||
// Should fall back to default ports when invalid values are provided
|
||
assert.Contains(t, invalidPortLivenessCommand, fmt.Sprintf(":%d", utils.DefaultDashboardAgentListenPort), "Should fall back to default dashboard-agent-listen-port for invalid input")
|
||
assert.Contains(t, invalidPortLivenessCommand, fmt.Sprintf(":%d", utils.DefaultDashboardPort), "Should fall back to default dashboard-port for invalid input")
|
||
|
||
// Test 9: Test unified health check at ray version 2.53.0 and later.
|
||
rayContainer.LivenessProbe = nil
|
||
rayContainer.ReadinessProbe = nil
|
||
// Verify head node has http probes and no exec probes.
|
||
initLivenessAndReadinessProbe(rayContainer, rayv1.HeadNode, utils.RayClusterCRD, rayStartParams, "2.53.0")
|
||
assert.NotNil(t, rayContainer.LivenessProbe.HTTPGet)
|
||
assert.NotNil(t, rayContainer.ReadinessProbe.HTTPGet)
|
||
assert.Nil(t, rayContainer.LivenessProbe.Exec)
|
||
assert.Nil(t, rayContainer.ReadinessProbe.Exec)
|
||
assert.Equal(t, utils.RayNodeHealthPath, rayContainer.LivenessProbe.HTTPGet.Path)
|
||
assert.Equal(t, int32(utils.DefaultDashboardAgentListenPort), rayContainer.LivenessProbe.HTTPGet.Port.IntVal)
|
||
|
||
// Worker nodes also should have only http probes.
|
||
rayContainer.LivenessProbe = nil
|
||
rayContainer.ReadinessProbe = nil
|
||
initLivenessAndReadinessProbe(rayContainer, rayv1.WorkerNode, utils.RayClusterCRD, rayStartParams, "2.53.0")
|
||
assert.Nil(t, rayContainer.LivenessProbe.Exec)
|
||
assert.Nil(t, rayContainer.ReadinessProbe.Exec)
|
||
assert.NotNil(t, rayContainer.LivenessProbe.HTTPGet)
|
||
assert.NotNil(t, rayContainer.ReadinessProbe.HTTPGet)
|
||
|
||
// Ray Serve workers still use exec probes for readiness to check the proxy actor.
|
||
rayContainer.LivenessProbe = nil
|
||
rayContainer.ReadinessProbe = nil
|
||
initLivenessAndReadinessProbe(rayContainer, rayv1.WorkerNode, utils.RayServiceCRD, rayStartParams, "2.53.0")
|
||
assert.NotNil(t, rayContainer.LivenessProbe.HTTPGet)
|
||
assert.Nil(t, rayContainer.LivenessProbe.Exec)
|
||
assert.Nil(t, rayContainer.ReadinessProbe.HTTPGet)
|
||
assert.NotNil(t, rayContainer.ReadinessProbe.Exec)
|
||
assert.Contains(t, strings.Join(rayContainer.ReadinessProbe.Exec.Command, " "), utils.RayServeProxyHealthPath)
|
||
|
||
// Versions parsed below 2.53 must use exec probes.
|
||
rayContainer.LivenessProbe = nil
|
||
rayContainer.ReadinessProbe = nil
|
||
initLivenessAndReadinessProbe(rayContainer, rayv1.HeadNode, utils.RayClusterCRD, rayStartParams, "2.52.0")
|
||
assert.NotNil(t, rayContainer.LivenessProbe.Exec)
|
||
assert.NotNil(t, rayContainer.ReadinessProbe.Exec)
|
||
}
|
||
|
||
func TestGenerateRayStartCommand(t *testing.T) {
|
||
tests := []struct {
|
||
rayStartParams map[string]string
|
||
name string
|
||
expected string
|
||
nodeType rayv1.RayNodeType
|
||
resource corev1.ResourceRequirements
|
||
}{
|
||
{
|
||
name: "WorkerNode with GPU",
|
||
nodeType: rayv1.WorkerNode,
|
||
rayStartParams: map[string]string{},
|
||
resource: corev1.ResourceRequirements{
|
||
Limits: corev1.ResourceList{
|
||
"nvidia.com/gpu": resource.MustParse("1"),
|
||
},
|
||
},
|
||
expected: "ray start --num-gpus=1 ",
|
||
},
|
||
{
|
||
name: "WorkerNode with MIG GPU",
|
||
nodeType: rayv1.WorkerNode,
|
||
rayStartParams: map[string]string{},
|
||
resource: corev1.ResourceRequirements{
|
||
Limits: corev1.ResourceList{
|
||
"nvidia.com/mig-2g.32gb": resource.MustParse("1"),
|
||
},
|
||
},
|
||
expected: "ray start --num-gpus=1 ",
|
||
},
|
||
{
|
||
name: "WorkerNode with TPU",
|
||
nodeType: rayv1.WorkerNode,
|
||
rayStartParams: map[string]string{},
|
||
resource: corev1.ResourceRequirements{
|
||
Limits: corev1.ResourceList{
|
||
"google.com/tpu": resource.MustParse("4"),
|
||
},
|
||
},
|
||
expected: `ray start --resources='{"TPU":4}' `,
|
||
},
|
||
{
|
||
name: "HeadNode with Neuron Cores",
|
||
nodeType: rayv1.HeadNode,
|
||
rayStartParams: map[string]string{},
|
||
resource: corev1.ResourceRequirements{
|
||
Limits: corev1.ResourceList{
|
||
"aws.amazon.com/neuroncore": resource.MustParse("4"),
|
||
},
|
||
},
|
||
expected: `ray start --head --resources='{"neuron_cores":4}' `,
|
||
},
|
||
{
|
||
name: "HeadNode with multiple accelerators",
|
||
nodeType: rayv1.HeadNode,
|
||
rayStartParams: map[string]string{},
|
||
resource: corev1.ResourceRequirements{
|
||
Limits: corev1.ResourceList{
|
||
"aws.amazon.com/neuroncore": resource.MustParse("4"),
|
||
"nvidia.com/gpu": resource.MustParse("1"),
|
||
},
|
||
},
|
||
expected: `ray start --head --num-gpus=1 --resources='{"neuron_cores":4}' `,
|
||
},
|
||
{
|
||
name: "HeadNode with multiple custom accelerators",
|
||
nodeType: rayv1.HeadNode,
|
||
rayStartParams: map[string]string{},
|
||
resource: corev1.ResourceRequirements{
|
||
Limits: corev1.ResourceList{
|
||
"google.com/tpu": resource.MustParse("8"),
|
||
"aws.amazon.com/neuroncore": resource.MustParse("4"),
|
||
"nvidia.com/gpu": resource.MustParse("1"),
|
||
},
|
||
},
|
||
expected: `ray start --head --num-gpus=1 --resources='{"neuron_cores":4}' `,
|
||
},
|
||
{
|
||
name: "HeadNode with existing resources",
|
||
nodeType: rayv1.HeadNode,
|
||
rayStartParams: map[string]string{
|
||
"resources": `"{"custom_resource":2}"`,
|
||
},
|
||
resource: corev1.ResourceRequirements{
|
||
Limits: corev1.ResourceList{
|
||
"aws.amazon.com/neuroncore": resource.MustParse("4"),
|
||
},
|
||
},
|
||
expected: `ray start --head --resources='{"custom_resource":2,"neuron_cores":4}' `,
|
||
},
|
||
{
|
||
name: "HeadNode with existing neuron_cores resources",
|
||
nodeType: rayv1.HeadNode,
|
||
rayStartParams: map[string]string{
|
||
"resources": `'{"custom_resource":2,"neuron_cores":3}'`,
|
||
},
|
||
resource: corev1.ResourceRequirements{
|
||
Limits: corev1.ResourceList{
|
||
"aws.amazon.com/neuroncore": resource.MustParse("4"),
|
||
},
|
||
},
|
||
expected: `ray start --head --resources='{"custom_resource":2,"neuron_cores":3}' `,
|
||
},
|
||
{
|
||
name: "HeadNode with existing TPU resources",
|
||
nodeType: rayv1.HeadNode,
|
||
rayStartParams: map[string]string{
|
||
"resources": `'{"custom_resource":2,"TPU":4}'`,
|
||
},
|
||
resource: corev1.ResourceRequirements{
|
||
Limits: corev1.ResourceList{
|
||
"google.com/tpu": resource.MustParse("8"),
|
||
},
|
||
},
|
||
expected: `ray start --head --resources='{"custom_resource":2,"TPU":4}' `,
|
||
},
|
||
{
|
||
name: "HeadNode with invalid resources string",
|
||
nodeType: rayv1.HeadNode,
|
||
rayStartParams: map[string]string{
|
||
"resources": "{",
|
||
},
|
||
resource: corev1.ResourceRequirements{
|
||
Limits: corev1.ResourceList{
|
||
"aws.amazon.com/neuroncore": resource.MustParse("4"),
|
||
},
|
||
},
|
||
expected: "ray start --head --resources={ ",
|
||
},
|
||
{
|
||
name: "Invalid node type",
|
||
nodeType: "InvalidType",
|
||
rayStartParams: map[string]string{},
|
||
resource: corev1.ResourceRequirements{},
|
||
expected: "",
|
||
},
|
||
{
|
||
name: "HeadNode with include-log-monitor=false",
|
||
nodeType: rayv1.HeadNode,
|
||
rayStartParams: map[string]string{
|
||
"include-log-monitor": "false",
|
||
},
|
||
resource: corev1.ResourceRequirements{},
|
||
expected: "ray start --head --include-log-monitor=false ",
|
||
},
|
||
{
|
||
name: "HeadNode with include-log-monitor=true",
|
||
nodeType: rayv1.HeadNode,
|
||
rayStartParams: map[string]string{
|
||
"include-log-monitor": "true",
|
||
},
|
||
resource: corev1.ResourceRequirements{},
|
||
expected: "ray start --head --include-log-monitor=true ",
|
||
},
|
||
}
|
||
|
||
for _, tt := range tests {
|
||
t.Run(tt.name, func(t *testing.T) {
|
||
result := generateRayStartCommand(context.TODO(), tt.nodeType, tt.rayStartParams, tt.resource)
|
||
assert.Equal(t, tt.expected, result)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestSetAutoscalerV2EnvVars(t *testing.T) {
|
||
tests := map[string]struct {
|
||
podTemplate *corev1.PodTemplateSpec
|
||
expectedEnvVars []corev1.EnvVar
|
||
}{
|
||
"Pod without env vars should have autoscaler v2 env var set to true": {
|
||
podTemplate: &corev1.PodTemplateSpec{
|
||
Spec: corev1.PodSpec{
|
||
Containers: []corev1.Container{
|
||
{},
|
||
},
|
||
},
|
||
},
|
||
expectedEnvVars: []corev1.EnvVar{
|
||
{
|
||
Name: utils.RAY_ENABLE_AUTOSCALER_V2,
|
||
Value: "true",
|
||
},
|
||
},
|
||
},
|
||
"Pod without autoscaler v2 env var should have autoscaler v2 env var set to true": {
|
||
podTemplate: &corev1.PodTemplateSpec{
|
||
Spec: corev1.PodSpec{
|
||
Containers: []corev1.Container{
|
||
{
|
||
Env: []corev1.EnvVar{
|
||
{
|
||
Name: "papal",
|
||
Value: "conclave",
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
expectedEnvVars: []corev1.EnvVar{
|
||
{
|
||
Name: "papal",
|
||
Value: "conclave",
|
||
},
|
||
{
|
||
Name: utils.RAY_ENABLE_AUTOSCALER_V2,
|
||
Value: "true",
|
||
},
|
||
},
|
||
},
|
||
}
|
||
|
||
for name, tc := range tests {
|
||
t.Run(name, func(t *testing.T) {
|
||
setAutoscalerV2EnvVars(tc.podTemplate)
|
||
assert.Equal(t, tc.expectedEnvVars, tc.podTemplate.Spec.Containers[0].Env)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestSetAutoscalerV1EnvVars(t *testing.T) {
|
||
tests := map[string]struct {
|
||
podTemplate *corev1.PodTemplateSpec
|
||
expectedEnvVars []corev1.EnvVar
|
||
}{
|
||
"Pod without env vars should have autoscaler v2 env var set to false": {
|
||
podTemplate: &corev1.PodTemplateSpec{
|
||
Spec: corev1.PodSpec{
|
||
Containers: []corev1.Container{
|
||
{},
|
||
},
|
||
},
|
||
},
|
||
expectedEnvVars: []corev1.EnvVar{
|
||
{
|
||
Name: utils.RAY_ENABLE_AUTOSCALER_V2,
|
||
Value: "false",
|
||
},
|
||
},
|
||
},
|
||
"Pod without autoscaler v2 env var should have autoscaler v2 env var set to false": {
|
||
podTemplate: &corev1.PodTemplateSpec{
|
||
Spec: corev1.PodSpec{
|
||
Containers: []corev1.Container{
|
||
{
|
||
Env: []corev1.EnvVar{
|
||
{
|
||
Name: "papal",
|
||
Value: "conclave",
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
expectedEnvVars: []corev1.EnvVar{
|
||
{
|
||
Name: "papal",
|
||
Value: "conclave",
|
||
},
|
||
{
|
||
Name: utils.RAY_ENABLE_AUTOSCALER_V2,
|
||
Value: "false",
|
||
},
|
||
},
|
||
},
|
||
}
|
||
|
||
for name, tc := range tests {
|
||
t.Run(name, func(t *testing.T) {
|
||
setAutoscalerV1EnvVars(tc.podTemplate)
|
||
assert.Equal(t, tc.expectedEnvVars, tc.podTemplate.Spec.Containers[0].Env)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestMergeLabels(t *testing.T) {
|
||
tests := map[string]struct {
|
||
templateSpecLabels map[string]string
|
||
workerGroupLabels map[string]string
|
||
expectedLabels map[string]string
|
||
}{
|
||
"Non-overlapping labels don't override.": {
|
||
templateSpecLabels: map[string]string{"pod-label-key": "pod-label-value"},
|
||
workerGroupLabels: map[string]string{"ray/io:some-label": "ray-node-label-value"},
|
||
expectedLabels: map[string]string{"pod-label-key": "pod-label-value", "ray/io:some-label": "ray-node-label-value"},
|
||
},
|
||
"Overlapping labels override with precedence for group `Labels`.": {
|
||
templateSpecLabels: map[string]string{"accelerator-type": "GPU", "market-type": "spot"},
|
||
workerGroupLabels: map[string]string{"accelerator-type": "TPU-V6E"},
|
||
expectedLabels: map[string]string{"accelerator-type": "TPU-V6E", "market-type": "spot"},
|
||
},
|
||
"Empty Pod Spec labels.": {
|
||
templateSpecLabels: map[string]string{},
|
||
workerGroupLabels: map[string]string{"group-labels": "group-label-value"},
|
||
expectedLabels: map[string]string{"group-labels": "group-label-value"},
|
||
},
|
||
"Empty `Labels` field for group.": {
|
||
templateSpecLabels: map[string]string{"pod-label": "pod-label-value"},
|
||
workerGroupLabels: map[string]string{},
|
||
expectedLabels: map[string]string{"pod-label": "pod-label-value"},
|
||
},
|
||
"Both `Labels` and labels in Pod spec nil.": {
|
||
templateSpecLabels: nil,
|
||
workerGroupLabels: nil,
|
||
expectedLabels: map[string]string{},
|
||
},
|
||
}
|
||
|
||
for name, tc := range tests {
|
||
t.Run(name, func(t *testing.T) {
|
||
merged := mergeLabels(tc.templateSpecLabels, tc.workerGroupLabels)
|
||
assert.Equal(t, tc.expectedLabels, merged)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestUpdateRayStartParamsLabels(t *testing.T) {
|
||
tests := map[string]struct {
|
||
initialRayStartParams map[string]string
|
||
groupLabels map[string]string
|
||
expectedRayStartParams map[string]string
|
||
}{
|
||
"Set labels in group `Labels` to `--labels` empty rayStartParams.": {
|
||
initialRayStartParams: map[string]string{},
|
||
groupLabels: map[string]string{
|
||
"topology.kubernetes.io/zone": "us-central2",
|
||
"ray.io/node-group": "worker-group-1",
|
||
"cloud.google.com/gke-spot": "true",
|
||
},
|
||
// The output string is sorted alphabetically by key.
|
||
expectedRayStartParams: map[string]string{
|
||
"labels": "cloud.google.com/gke-spot=true,ray.io/node-group=worker-group-1,topology.kubernetes.io/zone=us-central2",
|
||
},
|
||
},
|
||
" group `Labels overwrites an existing '--labels' parameter in rayStartParams": {
|
||
initialRayStartParams: map[string]string{
|
||
"labels": "old=label,to-be=replaced",
|
||
"resources": "some-resources", // should be retained
|
||
},
|
||
groupLabels: map[string]string{
|
||
"new": "label",
|
||
},
|
||
expectedRayStartParams: map[string]string{
|
||
"labels": "new=label",
|
||
"resources": "some-resources",
|
||
},
|
||
},
|
||
"No-op when group `Labels` is nil": {
|
||
initialRayStartParams: map[string]string{"labels": "some=labels"},
|
||
groupLabels: nil,
|
||
expectedRayStartParams: map[string]string{"labels": "some=labels"},
|
||
},
|
||
"No-op when group `Labels` is empty": {
|
||
initialRayStartParams: map[string]string{"labels": "some=labels"},
|
||
groupLabels: map[string]string{},
|
||
expectedRayStartParams: map[string]string{"labels": "some=labels"},
|
||
},
|
||
}
|
||
|
||
for name, tc := range tests {
|
||
t.Run(name, func(t *testing.T) {
|
||
// copy of rayStartParams for test
|
||
rayStartParams := make(map[string]string)
|
||
maps.Copy(rayStartParams, tc.initialRayStartParams)
|
||
|
||
updateRayStartParamsLabels(rayStartParams, tc.groupLabels)
|
||
|
||
assert.Equal(t, tc.expectedRayStartParams, rayStartParams)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestUpdateRayStartParamsResources(t *testing.T) {
|
||
ctx := context.Background()
|
||
|
||
tests := map[string]struct {
|
||
initialRayStartParams map[string]string
|
||
groupResources map[string]string
|
||
expectedRayStartParams map[string]string
|
||
expectedK8sResources corev1.ResourceList
|
||
}{
|
||
"No-op when group `Resources` is nil or empty": {
|
||
initialRayStartParams: map[string]string{"existing": "true"},
|
||
groupResources: nil,
|
||
expectedRayStartParams: map[string]string{"existing": "true"},
|
||
},
|
||
"Basic CPU and Memory set in `Resources` override rayStartParams": {
|
||
initialRayStartParams: map[string]string{},
|
||
groupResources: map[string]string{
|
||
string(corev1.ResourceCPU): "2",
|
||
string(corev1.ResourceMemory): "4Gi",
|
||
},
|
||
expectedRayStartParams: map[string]string{
|
||
"num-cpus": "2",
|
||
"memory": "4294967296", // 4Gi in bytes
|
||
},
|
||
},
|
||
"Uppercase CPU and GPU resource names set in `Resources` override rayStartParams": {
|
||
initialRayStartParams: map[string]string{},
|
||
groupResources: map[string]string{
|
||
"CPU": "2",
|
||
"GPU": "4",
|
||
},
|
||
expectedRayStartParams: map[string]string{
|
||
"num-cpus": "2",
|
||
"num-gpus": "4",
|
||
},
|
||
},
|
||
"GPU and custom TPU resource set in `Resources` override rayStartParams": {
|
||
initialRayStartParams: map[string]string{},
|
||
groupResources: map[string]string{
|
||
"nvidia.com/gpu": "1",
|
||
"TPU": "4",
|
||
},
|
||
expectedRayStartParams: map[string]string{
|
||
"num-gpus": "1",
|
||
"resources": "'{\"TPU\":4}'",
|
||
},
|
||
},
|
||
"Top-level `Resources` should override existing resource params": {
|
||
initialRayStartParams: map[string]string{
|
||
"num-cpus": "1",
|
||
"memory": "1000",
|
||
"resources": "'{\"Custom-Resource\": 10}'",
|
||
},
|
||
groupResources: map[string]string{
|
||
string(corev1.ResourceCPU): "4",
|
||
"Custom-Resource": "5",
|
||
},
|
||
expectedRayStartParams: map[string]string{
|
||
"num-cpus": "4",
|
||
"memory": "1000", // preserved
|
||
"resources": "'{\"Custom-Resource\":5}'",
|
||
},
|
||
},
|
||
}
|
||
|
||
for name, tc := range tests {
|
||
t.Run(name, func(t *testing.T) {
|
||
rayStartParams := make(map[string]string)
|
||
maps.Copy(rayStartParams, tc.initialRayStartParams)
|
||
|
||
updateRayStartParamsResources(ctx, rayStartParams, tc.groupResources)
|
||
|
||
// Verify rayStartParams are updated based on the top-level Resources values.
|
||
assert.Equal(t, tc.expectedRayStartParams, rayStartParams)
|
||
})
|
||
}
|
||
}
|