Files
kuberay/ray-operator/controllers/ray/rayjob_controller_unit_test.go
4d53e84325 [release-1.3] v1.3.2 backports (#3264)
* [RayJob][Fix] Use --no-wait for job submission to avoid carrying the error return code to the log tailing (#3216)

* [RayJob][Fix] Use --no-wait for job submission to avoid carrying the error return code to the log tailing

Signed-off-by: Rueian <[email protected]>

* [RayJob][Fix] Use --no-wait for job submission to avoid carrying the error return code to the log tailing

Signed-off-by: Rueian <[email protected]>

* chore: update comments

Signed-off-by: Rueian <[email protected]>

* chore: add a comment about bash -e

Signed-off-by: Rueian <[email protected]>

---------

Signed-off-by: Rueian <[email protected]>

* kubectl ray job submit: provide entrypoint (#3186)

* [kubectl-plugin] Add head/worker node selector option (#3228)

* add node selector option for kubectl plugin create cluster

Signed-off-by: Troy Chiu <[email protected]>

* nit

Signed-off-by: Troy Chiu <[email protected]>

---------

Signed-off-by: Troy Chiu <[email protected]>

* add node selector option for kubectl plugin create worker group (#3235)

* add node selector option for kubectl plugin create work group

Signed-off-by: Troy Chiu <[email protected]>

* nit

Signed-off-by: Troy Chiu <[email protected]>

* code review: fix usage

Signed-off-by: Troy Chiu <[email protected]>

---------

Signed-off-by: Troy Chiu <[email protected]>

* [kubectl-plugin] remove CPU limits by default (#3243)

Signed-off-by: Andrew Sy Kim <[email protected]>

* [Chore][CI] Limit the release-image-build github workflow to only take tag as input (#3117)

* remove all inputs from workflow_dispatch

Signed-off-by: Tina Wu <[email protected]>

* use tag only

Signed-off-by: Tina Wu <[email protected]>

* align case

Signed-off-by: Tina Wu <[email protected]>

* change sha

Signed-off-by: Tina Wu <[email protected]>

* extract tag

* lint fix

Signed-off-by: Tina Wu <[email protected]>

* update github_env

Signed-off-by: Tina Wu <[email protected]>

* directly take tag

Signed-off-by: Tina Wu <[email protected]>

* add env,

Signed-off-by: Tina Wu <[email protected]>

* directly use tag

Signed-off-by: Tina Wu <[email protected]>

* use env. when in script

Signed-off-by: Tina Wu <[email protected]>

* env.tag when with

Signed-off-by: Tina Wu <[email protected]>

* use env.tag for all

Signed-off-by: Tina Wu <[email protected]>

---------

Signed-off-by: Tina Wu <[email protected]>
Co-authored-by: tinaxfwu <[email protected]>

* [CI] Remove create tag step from release (#3249)

Signed-off-by: Chi-Sheng Liu <[email protected]>

---------

Signed-off-by: Rueian <[email protected]>
Signed-off-by: Troy Chiu <[email protected]>
Signed-off-by: Andrew Sy Kim <[email protected]>
Signed-off-by: Tina Wu <[email protected]>
Signed-off-by: Chi-Sheng Liu <[email protected]>
Co-authored-by: Rueian <[email protected]>
Co-authored-by: Spencer Peterson <[email protected]>
Co-authored-by: Troy Chiu <[email protected]>
Co-authored-by: Tina Wu <[email protected]>
Co-authored-by: tinaxfwu <[email protected]>
Co-authored-by: Chi-Sheng Liu <[email protected]>
2025-04-02 15:43:57 -04:00

528 lines
16 KiB
Go

package ray
import (
"context"
"errors"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
utils "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
"github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme"
)
func TestCreateRayJobSubmitterIfNeed(t *testing.T) {
newScheme := runtime.NewScheme()
_ = rayv1.AddToScheme(newScheme)
_ = batchv1.AddToScheme(newScheme)
_ = corev1.AddToScheme(newScheme)
rayCluster := &rayv1.RayCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "test-raycluster",
Namespace: "default",
},
Spec: rayv1.RayClusterSpec{
HeadGroupSpec: rayv1.HeadGroupSpec{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "rayproject/ray",
},
},
},
},
},
},
}
rayJob := &rayv1.RayJob{
ObjectMeta: metav1.ObjectMeta{
Name: "test-rayjob",
Namespace: "default",
},
}
k8sJob := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: "test-rayjob",
Namespace: "default",
},
}
// Test 1: Return the existing k8s job if it already exists
fakeClient := clientFake.NewClientBuilder().WithScheme(newScheme).WithRuntimeObjects(k8sJob, rayCluster, rayJob).Build()
ctx := context.TODO()
rayJobReconciler := &RayJobReconciler{
Client: fakeClient,
Scheme: newScheme,
Recorder: &record.FakeRecorder{},
}
err := rayJobReconciler.createK8sJobIfNeed(ctx, rayJob, rayCluster)
require.NoError(t, err)
// Test 2: Create a new k8s job if it does not already exist
fakeClient = clientFake.NewClientBuilder().WithScheme(newScheme).WithRuntimeObjects(rayCluster, rayJob).Build()
rayJobReconciler.Client = fakeClient
err = rayJobReconciler.createK8sJobIfNeed(ctx, rayJob, rayCluster)
require.NoError(t, err)
err = fakeClient.Get(ctx, types.NamespacedName{
Namespace: k8sJob.Namespace,
Name: k8sJob.Name,
}, k8sJob, nil)
require.NoError(t, err)
assert.Equal(t, k8sJob.Labels[utils.RayOriginatedFromCRNameLabelKey], rayJob.Name)
assert.Equal(t, k8sJob.Labels[utils.RayOriginatedFromCRDLabelKey], utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD))
}
func TestGetSubmitterTemplate(t *testing.T) {
// RayJob instance with user-provided submitter pod template.
rayJobInstanceWithTemplate := &rayv1.RayJob{
Spec: rayv1.RayJobSpec{
Entrypoint: "echo no quote 'single quote' \"double quote\"",
SubmitterPodTemplate: &corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Command: []string{"user-command"},
},
},
},
},
},
Status: rayv1.RayJobStatus{
DashboardURL: "test-url",
JobId: "test-job-id",
},
}
// RayJob instance without user-provided submitter pod template.
// In this case we should use the image of the Ray Head, so specify the image so we can test it.
rayJobInstanceWithoutTemplate := &rayv1.RayJob{
Spec: rayv1.RayJobSpec{
Entrypoint: "echo no quote 'single quote' \"double quote\"",
RayClusterSpec: &rayv1.RayClusterSpec{
HeadGroupSpec: rayv1.HeadGroupSpec{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "rayproject/ray:custom-version",
},
},
},
},
},
},
},
Status: rayv1.RayJobStatus{
DashboardURL: "test-url",
JobId: "test-job-id",
},
}
rayClusterInstance := &rayv1.RayCluster{
Spec: rayv1.RayClusterSpec{
HeadGroupSpec: rayv1.HeadGroupSpec{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "rayproject/ray:custom-version",
},
},
},
},
},
},
}
ctx := context.Background()
// Test 1: User provided template with command
submitterTemplate, err := getSubmitterTemplate(ctx, rayJobInstanceWithTemplate, nil)
require.NoError(t, err)
assert.Equal(t, "user-command", submitterTemplate.Spec.Containers[utils.RayContainerIndex].Command[0])
// Test 2: User provided template without command
rayJobInstanceWithTemplate.Spec.SubmitterPodTemplate.Spec.Containers[utils.RayContainerIndex].Command = []string{}
submitterTemplate, err = getSubmitterTemplate(ctx, rayJobInstanceWithTemplate, nil)
require.NoError(t, err)
assert.Equal(t, []string{"/bin/bash"}, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Command)
assert.Equal(t, []string{"-ce", "if ! ray job status --address http://test-url test-job-id >/dev/null 2>&1 ; then ray job submit --address http://test-url --no-wait --submission-id test-job-id -- echo no quote 'single quote' \"double quote\" ; fi ; ray job logs --address http://test-url --follow test-job-id"}, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Args)
// Test 3: User did not provide template, should use the image of the Ray Head
submitterTemplate, err = getSubmitterTemplate(ctx, rayJobInstanceWithoutTemplate, rayClusterInstance)
require.NoError(t, err)
assert.Equal(t, []string{"/bin/bash"}, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Command)
assert.Equal(t, []string{"-ce", "if ! ray job status --address http://test-url test-job-id >/dev/null 2>&1 ; then ray job submit --address http://test-url --no-wait --submission-id test-job-id -- echo no quote 'single quote' \"double quote\" ; fi ; ray job logs --address http://test-url --follow test-job-id"}, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Args)
assert.Equal(t, "rayproject/ray:custom-version", submitterTemplate.Spec.Containers[utils.RayContainerIndex].Image)
// Test 4: Check default PYTHONUNBUFFERED setting
submitterTemplate, err = getSubmitterTemplate(ctx, rayJobInstanceWithoutTemplate, rayClusterInstance)
require.NoError(t, err)
envVar, found := utils.EnvVarByName(PythonUnbufferedEnvVarName, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Env)
assert.True(t, found)
assert.Equal(t, "1", envVar.Value)
// Test 5: Check default RAY_DASHBOARD_ADDRESS env var
submitterTemplate, err = getSubmitterTemplate(ctx, rayJobInstanceWithTemplate, nil)
require.NoError(t, err)
envVar, found = utils.EnvVarByName(utils.RAY_DASHBOARD_ADDRESS, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Env)
assert.True(t, found)
assert.Equal(t, "test-url", envVar.Value)
// Test 6: Check default RAY_JOB_SUBMISSION_ID env var
envVar, found = utils.EnvVarByName(utils.RAY_JOB_SUBMISSION_ID, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Env)
assert.True(t, found)
assert.Equal(t, "test-job-id", envVar.Value)
}
func TestUpdateStatusToSuspendingIfNeeded(t *testing.T) {
newScheme := runtime.NewScheme()
_ = rayv1.AddToScheme(newScheme)
tests := []struct {
name string
status rayv1.JobDeploymentStatus
suspend bool
expectedShouldUpdate bool
}{
// When Autoscaler is enabled, the random Pod deletion is controleld by the feature flag `ENABLE_RANDOM_POD_DELETE`.
{
name: "Suspend is false",
suspend: false,
status: rayv1.JobDeploymentStatusInitializing,
expectedShouldUpdate: false,
},
{
name: "Suspend is true, but the status is not allowed to transition to suspending",
suspend: true,
status: rayv1.JobDeploymentStatusComplete,
expectedShouldUpdate: false,
},
{
name: "Suspend is true, and the status is allowed to transition to suspending",
suspend: true,
status: rayv1.JobDeploymentStatusInitializing,
expectedShouldUpdate: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
name := "test-rayjob"
namespace := "default"
rayJob := &rayv1.RayJob{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: rayv1.RayJobSpec{
Suspend: tc.suspend,
},
Status: rayv1.RayJobStatus{
JobDeploymentStatus: tc.status,
},
}
ctx := context.Background()
shouldUpdate := updateStatusToSuspendingIfNeeded(ctx, rayJob)
assert.Equal(t, tc.expectedShouldUpdate, shouldUpdate)
if tc.expectedShouldUpdate {
assert.Equal(t, rayv1.JobDeploymentStatusSuspending, rayJob.Status.JobDeploymentStatus)
} else {
assert.Equal(t, tc.status, rayJob.Status.JobDeploymentStatus)
}
})
}
}
func TestUpdateRayJobStatus(t *testing.T) {
newScheme := runtime.NewScheme()
_ = rayv1.AddToScheme(newScheme)
rayJobTemplate := &rayv1.RayJob{
ObjectMeta: metav1.ObjectMeta{
Name: "test-rayjob",
Namespace: "default",
},
Status: rayv1.RayJobStatus{
JobDeploymentStatus: rayv1.JobDeploymentStatusRunning,
JobStatus: rayv1.JobStatusRunning,
Message: "old message",
},
}
newMessage := "new message"
tests := []struct {
name string
isJobDeploymentStatusChanged bool
}{
{
name: "JobDeploymentStatus is not changed",
isJobDeploymentStatusChanged: false,
},
{
name: "JobDeploymentStatus is changed",
isJobDeploymentStatusChanged: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
oldRayJob := rayJobTemplate.DeepCopy()
// Initialize a fake client with newScheme and runtimeObjects.
fakeClient := clientFake.NewClientBuilder().
WithScheme(newScheme).
WithRuntimeObjects(oldRayJob).
WithStatusSubresource(oldRayJob).Build()
ctx := context.Background()
newRayJob := &rayv1.RayJob{}
err := fakeClient.Get(ctx, types.NamespacedName{Namespace: oldRayJob.Namespace, Name: oldRayJob.Name}, newRayJob)
require.NoError(t, err)
// Update the status
newRayJob.Status.Message = newMessage
if tc.isJobDeploymentStatusChanged {
newRayJob.Status.JobDeploymentStatus = rayv1.JobDeploymentStatusSuspending
}
// Initialize a new RayClusterReconciler.
testRayJobReconciler := &RayJobReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: newScheme,
}
err = testRayJobReconciler.updateRayJobStatus(ctx, oldRayJob, newRayJob)
require.NoError(t, err)
err = fakeClient.Get(ctx, types.NamespacedName{Namespace: newRayJob.Namespace, Name: newRayJob.Name}, newRayJob)
require.NoError(t, err)
assert.Equal(t, newRayJob.Status.Message == newMessage, tc.isJobDeploymentStatusChanged)
})
}
}
func TestFailedToCreateRayJobSubmitterEvent(t *testing.T) {
rayJob := &rayv1.RayJob{
ObjectMeta: metav1.ObjectMeta{
Name: "test-rayjob",
Namespace: "default",
},
}
submitterTemplate := corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: "test-submit-pod",
Namespace: "default",
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-submit",
Image: "rayproject/ray:latest",
},
},
},
}
fakeClient := clientFake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{
Create: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.CreateOption) error {
return errors.New("random")
},
}).WithScheme(scheme.Scheme).Build()
recorder := record.NewFakeRecorder(100)
reconciler := &RayJobReconciler{
Client: fakeClient,
Recorder: recorder,
Scheme: scheme.Scheme,
}
err := reconciler.createNewK8sJob(context.Background(), rayJob, submitterTemplate)
require.Error(t, err, "Expected error due to simulated job creation failure")
var foundFailureEvent bool
events := []string{}
for len(recorder.Events) > 0 {
event := <-recorder.Events
if strings.Contains(event, "Failed to create new Kubernetes Job") {
foundFailureEvent = true
break
}
events = append(events, event)
}
assert.Truef(t, foundFailureEvent, "Expected event to be generated for job creation failure, got events: %s", strings.Join(events, "\n"))
}
func TestFailedCreateRayClusterEvent(t *testing.T) {
rayJob := &rayv1.RayJob{
ObjectMeta: metav1.ObjectMeta{
Name: "test-rayjob",
Namespace: "default",
},
Spec: rayv1.RayJobSpec{
RayClusterSpec: &rayv1.RayClusterSpec{},
},
}
fakeClient := clientFake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{
Create: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.CreateOption) error {
return errors.New("random")
},
}).WithScheme(scheme.Scheme).Build()
recorder := record.NewFakeRecorder(100)
reconciler := &RayJobReconciler{
Client: fakeClient,
Recorder: recorder,
Scheme: scheme.Scheme,
}
_, err := reconciler.getOrCreateRayClusterInstance(context.Background(), rayJob)
require.Error(t, err, "Expected error due to cluster creation failure")
var foundFailureEvent bool
events := []string{}
for len(recorder.Events) > 0 {
event := <-recorder.Events
if strings.Contains(event, "Failed to create RayCluster") {
foundFailureEvent = true
break
}
events = append(events, event)
}
assert.Truef(t, foundFailureEvent, "Expected event to be generated for cluster creation failure, got events: %s", strings.Join(events, "\n"))
}
func TestFailedDeleteRayJobSubmitterEvent(t *testing.T) {
newScheme := runtime.NewScheme()
_ = batchv1.AddToScheme(newScheme)
rayJob := &rayv1.RayJob{
ObjectMeta: metav1.ObjectMeta{
Name: "test-rayjob",
Namespace: "default",
},
}
submitter := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: "test-rayjob",
Namespace: "default",
},
}
fakeClient := clientFake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{
Delete: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.DeleteOption) error {
return errors.New("random")
},
}).WithScheme(newScheme).WithRuntimeObjects(submitter).Build()
recorder := record.NewFakeRecorder(100)
reconciler := &RayJobReconciler{
Client: fakeClient,
Recorder: recorder,
Scheme: scheme.Scheme,
}
_, err := reconciler.deleteSubmitterJob(context.Background(), rayJob)
require.Error(t, err, "Expected error due to job deletion failure")
var foundFailureEvent bool
events := []string{}
for len(recorder.Events) > 0 {
event := <-recorder.Events
if strings.Contains(event, "Failed to delete submitter K8s Job") {
foundFailureEvent = true
break
}
events = append(events, event)
}
assert.Truef(t, foundFailureEvent, "Expected event to be generated for cluster deletion failure, got events: %s", strings.Join(events, "\n"))
}
func TestFailedDeleteRayClusterEvent(t *testing.T) {
newScheme := runtime.NewScheme()
_ = rayv1.AddToScheme(newScheme)
rayCluster := &rayv1.RayCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "test-raycluster",
Namespace: "default",
},
}
rayJob := &rayv1.RayJob{
ObjectMeta: metav1.ObjectMeta{
Name: "test-rayjob",
Namespace: "default",
},
Status: rayv1.RayJobStatus{
RayClusterName: "test-raycluster",
},
}
fakeClient := clientFake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{
Delete: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.DeleteOption) error {
return errors.New("random")
},
}).WithScheme(newScheme).WithRuntimeObjects(rayCluster).Build()
recorder := record.NewFakeRecorder(100)
reconciler := &RayJobReconciler{
Client: fakeClient,
Recorder: recorder,
Scheme: scheme.Scheme,
}
_, err := reconciler.deleteClusterResources(context.Background(), rayJob)
require.Error(t, err, "Expected error due to cluster deletion failure")
var foundFailureEvent bool
events := []string{}
for len(recorder.Events) > 0 {
event := <-recorder.Events
if strings.Contains(event, "Failed to delete cluster") {
foundFailureEvent = true
break
}
events = append(events, event)
}
assert.Truef(t, foundFailureEvent, "Expected event to be generated for cluster deletion failure, got events: %s", strings.Join(events, "\n"))
}