Files
kuberay/ray-operator/controllers/ray/rayjob_controller_unit_test.go
Andrew Sy KimandGitHub 8682b2d9c9 [RayJob] Add spec.backoffLimit for retrying RayJobs with new clusters (#2192)
* [RayJob] Add spec.backoffLimit for retrying RayJobs

Signed-off-by: Andrew Sy Kim <[email protected]>

* add method checkBackoffLimitAndUpdateStatusIfNeeded to consolidate retry checks into a single place

Signed-off-by: Andrew Sy Kim <[email protected]>

* call checkBackoffLimitAndUpdateStatusIfNeeded twice inside Running switch case

Signed-off-by: Andrew Sy Kim <[email protected]>

---------

Signed-off-by: Andrew Sy Kim <[email protected]>
2024-07-01 09:31:29 -07:00

373 lines
12 KiB
Go

package ray
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"k8s.io/utils/ptr"
clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake"
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
utils "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
)
func TestCreateK8sJobIfNeed(t *testing.T) {
newScheme := runtime.NewScheme()
_ = rayv1.AddToScheme(newScheme)
_ = batchv1.AddToScheme(newScheme)
_ = corev1.AddToScheme(newScheme)
rayCluster := &rayv1.RayCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "test-raycluster",
Namespace: "default",
},
Spec: rayv1.RayClusterSpec{
HeadGroupSpec: rayv1.HeadGroupSpec{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "rayproject/ray",
},
},
},
},
},
},
}
rayJob := &rayv1.RayJob{
ObjectMeta: metav1.ObjectMeta{
Name: "test-rayjob",
Namespace: "default",
},
}
k8sJob := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: "test-rayjob",
Namespace: "default",
},
}
// Test 1: Return the existing k8s job if it already exists
fakeClient := clientFake.NewClientBuilder().WithScheme(newScheme).WithRuntimeObjects(k8sJob, rayCluster, rayJob).Build()
ctx := context.TODO()
rayJobReconciler := &RayJobReconciler{
Client: fakeClient,
Scheme: newScheme,
Recorder: &record.FakeRecorder{},
}
err := rayJobReconciler.createK8sJobIfNeed(ctx, rayJob, rayCluster)
assert.NoError(t, err)
// Test 2: Create a new k8s job if it does not already exist
fakeClient = clientFake.NewClientBuilder().WithScheme(newScheme).WithRuntimeObjects(rayCluster, rayJob).Build()
rayJobReconciler.Client = fakeClient
err = rayJobReconciler.createK8sJobIfNeed(ctx, rayJob, rayCluster)
assert.NoError(t, err)
err = fakeClient.Get(ctx, types.NamespacedName{
Namespace: k8sJob.Namespace,
Name: k8sJob.Name,
}, k8sJob, nil)
assert.NoError(t, err)
assert.Equal(t, k8sJob.Labels[utils.RayOriginatedFromCRNameLabelKey], rayJob.Name)
assert.Equal(t, k8sJob.Labels[utils.RayOriginatedFromCRDLabelKey], utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD))
}
func TestGetSubmitterTemplate(t *testing.T) {
// RayJob instance with user-provided submitter pod template.
rayJobInstanceWithTemplate := &rayv1.RayJob{
Spec: rayv1.RayJobSpec{
Entrypoint: "echo hello world",
SubmitterPodTemplate: &corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Command: []string{"user-command"},
},
},
},
},
},
Status: rayv1.RayJobStatus{
DashboardURL: "test-url",
JobId: "test-job-id",
},
}
// RayJob instance without user-provided submitter pod template.
// In this case we should use the image of the Ray Head, so specify the image so we can test it.
rayJobInstanceWithoutTemplate := &rayv1.RayJob{
Spec: rayv1.RayJobSpec{
Entrypoint: "echo hello world",
RayClusterSpec: &rayv1.RayClusterSpec{
HeadGroupSpec: rayv1.HeadGroupSpec{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "rayproject/ray:custom-version",
},
},
},
},
},
},
},
Status: rayv1.RayJobStatus{
DashboardURL: "test-url",
JobId: "test-job-id",
},
}
rayClusterInstance := &rayv1.RayCluster{
Spec: rayv1.RayClusterSpec{
HeadGroupSpec: rayv1.HeadGroupSpec{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "rayproject/ray:custom-version",
},
},
},
},
},
},
}
r := &RayJobReconciler{}
ctx := context.Background()
// Test 1: User provided template with command
submitterTemplate, err := r.getSubmitterTemplate(ctx, rayJobInstanceWithTemplate, nil)
assert.NoError(t, err)
assert.Equal(t, "user-command", submitterTemplate.Spec.Containers[utils.RayContainerIndex].Command[0])
// Test 2: User provided template without command
rayJobInstanceWithTemplate.Spec.SubmitterPodTemplate.Spec.Containers[utils.RayContainerIndex].Command = []string{}
submitterTemplate, err = r.getSubmitterTemplate(ctx, rayJobInstanceWithTemplate, nil)
assert.NoError(t, err)
assert.Equal(t, []string{"ray", "job", "submit", "--address", "http://test-url", "--submission-id", "test-job-id", "--", "echo", "hello", "world"}, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Command)
// Test 3: User did not provide template, should use the image of the Ray Head
submitterTemplate, err = r.getSubmitterTemplate(ctx, rayJobInstanceWithoutTemplate, rayClusterInstance)
assert.NoError(t, err)
assert.Equal(t, []string{"ray", "job", "submit", "--address", "http://test-url", "--submission-id", "test-job-id", "--", "echo", "hello", "world"}, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Command)
assert.Equal(t, "rayproject/ray:custom-version", submitterTemplate.Spec.Containers[utils.RayContainerIndex].Image)
// Test 4: Check default PYTHONUNBUFFERED setting
submitterTemplate, err = r.getSubmitterTemplate(ctx, rayJobInstanceWithoutTemplate, rayClusterInstance)
assert.NoError(t, err)
envVar, found := utils.EnvVarByName(PythonUnbufferedEnvVarName, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Env)
assert.True(t, found)
assert.Equal(t, "1", envVar.Value)
// Test 5: Check default RAY_DASHBOARD_ADDRESS env var
submitterTemplate, err = r.getSubmitterTemplate(ctx, rayJobInstanceWithTemplate, nil)
assert.NoError(t, err)
envVar, found = utils.EnvVarByName(utils.RAY_DASHBOARD_ADDRESS, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Env)
assert.True(t, found)
assert.Equal(t, "test-url", envVar.Value)
// Test 6: Check default RAY_JOB_SUBMISSION_ID env var
envVar, found = utils.EnvVarByName(utils.RAY_JOB_SUBMISSION_ID, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Env)
assert.True(t, found)
assert.Equal(t, "test-job-id", envVar.Value)
}
func TestUpdateStatusToSuspendingIfNeeded(t *testing.T) {
newScheme := runtime.NewScheme()
_ = rayv1.AddToScheme(newScheme)
tests := map[string]struct {
status rayv1.JobDeploymentStatus
suspend bool
expectedShouldUpdate bool
}{
// When Autoscaler is enabled, the random Pod deletion is controleld by the feature flag `ENABLE_RANDOM_POD_DELETE`.
"Suspend is false": {
suspend: false,
status: rayv1.JobDeploymentStatusInitializing,
expectedShouldUpdate: false,
},
"Suspend is true, but the status is not allowed to transition to suspending": {
suspend: true,
status: rayv1.JobDeploymentStatusComplete,
expectedShouldUpdate: false,
},
"Suspend is true, and the status is allowed to transition to suspending": {
suspend: true,
status: rayv1.JobDeploymentStatusInitializing,
expectedShouldUpdate: true,
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
name := "test-rayjob"
namespace := "default"
rayJob := &rayv1.RayJob{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: rayv1.RayJobSpec{
Suspend: tc.suspend,
},
Status: rayv1.RayJobStatus{
JobDeploymentStatus: tc.status,
},
}
// Initialize a fake client with newScheme and runtimeObjects.
fakeClient := clientFake.NewClientBuilder().
WithScheme(newScheme).
WithRuntimeObjects(rayJob).
WithStatusSubresource(rayJob).Build()
ctx := context.Background()
// Initialize a new RayClusterReconciler.
testRayJobReconciler := &RayJobReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: newScheme,
}
shouldUpdate := testRayJobReconciler.updateStatusToSuspendingIfNeeded(ctx, rayJob)
assert.Equal(t, tc.expectedShouldUpdate, shouldUpdate)
if tc.expectedShouldUpdate {
assert.Equal(t, rayv1.JobDeploymentStatusSuspending, rayJob.Status.JobDeploymentStatus)
} else {
assert.Equal(t, tc.status, rayJob.Status.JobDeploymentStatus)
}
})
}
}
func TestUpdateRayJobStatus(t *testing.T) {
newScheme := runtime.NewScheme()
_ = rayv1.AddToScheme(newScheme)
rayJobTemplate := &rayv1.RayJob{
ObjectMeta: metav1.ObjectMeta{
Name: "test-rayjob",
Namespace: "default",
},
Status: rayv1.RayJobStatus{
JobDeploymentStatus: rayv1.JobDeploymentStatusRunning,
JobStatus: rayv1.JobStatusRunning,
Message: "old message",
},
}
newMessage := "new message"
tests := map[string]struct {
isJobDeploymentStatusChanged bool
}{
"JobDeploymentStatus is not changed": {
isJobDeploymentStatusChanged: false,
},
"JobDeploymentStatus is changed": {
isJobDeploymentStatusChanged: true,
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
oldRayJob := rayJobTemplate.DeepCopy()
// Initialize a fake client with newScheme and runtimeObjects.
fakeClient := clientFake.NewClientBuilder().
WithScheme(newScheme).
WithRuntimeObjects(oldRayJob).
WithStatusSubresource(oldRayJob).Build()
ctx := context.Background()
newRayJob := &rayv1.RayJob{}
err := fakeClient.Get(ctx, types.NamespacedName{Namespace: oldRayJob.Namespace, Name: oldRayJob.Name}, newRayJob)
assert.NoError(t, err)
// Update the status
newRayJob.Status.Message = newMessage
if tc.isJobDeploymentStatusChanged {
newRayJob.Status.JobDeploymentStatus = rayv1.JobDeploymentStatusSuspending
}
// Initialize a new RayClusterReconciler.
testRayJobReconciler := &RayJobReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: newScheme,
}
err = testRayJobReconciler.updateRayJobStatus(ctx, oldRayJob, newRayJob)
assert.NoError(t, err)
err = fakeClient.Get(ctx, types.NamespacedName{Namespace: newRayJob.Namespace, Name: newRayJob.Name}, newRayJob)
assert.NoError(t, err)
assert.Equal(t, newRayJob.Status.Message == newMessage, tc.isJobDeploymentStatusChanged)
})
}
}
func TestValidateRayJobSpec(t *testing.T) {
err := validateRayJobSpec(&rayv1.RayJob{})
assert.Error(t, err, "The RayJob is invalid because both `RayClusterSpec` and `ClusterSelector` are empty")
err = validateRayJobSpec(&rayv1.RayJob{
Spec: rayv1.RayJobSpec{
Suspend: true,
ShutdownAfterJobFinishes: false,
},
})
assert.Error(t, err, "The RayJob is invalid because a RayJob with shutdownAfterJobFinishes set to false is not allowed to be suspended.")
err = validateRayJobSpec(&rayv1.RayJob{
Spec: rayv1.RayJobSpec{
Suspend: true,
ShutdownAfterJobFinishes: true,
RayClusterSpec: &rayv1.RayClusterSpec{},
},
})
assert.NoError(t, err, "The RayJob is valid.")
err = validateRayJobSpec(&rayv1.RayJob{
Spec: rayv1.RayJobSpec{
Suspend: true,
ClusterSelector: map[string]string{
"key": "value",
},
},
})
assert.Error(t, err, "The RayJob is invalid because the ClusterSelector mode doesn't support the suspend operation.")
err = validateRayJobSpec(&rayv1.RayJob{
Spec: rayv1.RayJobSpec{
RuntimeEnvYAML: "invalid_yaml_str",
},
})
assert.Error(t, err, "The RayJob is invalid because the runtimeEnvYAML is invalid.")
err = validateRayJobSpec(&rayv1.RayJob{
Spec: rayv1.RayJobSpec{
BackoffLimit: ptr.To[int32](-1),
},
})
assert.Error(t, err, "The RayJob is invalid because the backoffLimit must be a positive integer.")
}