mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
578 lines
17 KiB
Go
578 lines
17 KiB
Go
package ray
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
batchv1 "k8s.io/api/batch/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/client-go/tools/record"
|
|
"k8s.io/utils/ptr"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
|
|
|
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
|
|
utils "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
|
|
"github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme"
|
|
)
|
|
|
|
func TestCreateRayJobSubmitterIfNeed(t *testing.T) {
|
|
newScheme := runtime.NewScheme()
|
|
_ = rayv1.AddToScheme(newScheme)
|
|
_ = batchv1.AddToScheme(newScheme)
|
|
_ = corev1.AddToScheme(newScheme)
|
|
|
|
rayCluster := &rayv1.RayCluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-raycluster",
|
|
Namespace: "default",
|
|
},
|
|
Spec: rayv1.RayClusterSpec{
|
|
HeadGroupSpec: rayv1.HeadGroupSpec{
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Image: "rayproject/ray",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
rayJob := &rayv1.RayJob{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-rayjob",
|
|
Namespace: "default",
|
|
},
|
|
}
|
|
|
|
k8sJob := &batchv1.Job{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-rayjob",
|
|
Namespace: "default",
|
|
},
|
|
}
|
|
|
|
// Test 1: Return the existing k8s job if it already exists
|
|
fakeClient := clientFake.NewClientBuilder().WithScheme(newScheme).WithRuntimeObjects(k8sJob, rayCluster, rayJob).Build()
|
|
ctx := context.TODO()
|
|
|
|
rayJobReconciler := &RayJobReconciler{
|
|
Client: fakeClient,
|
|
Scheme: newScheme,
|
|
Recorder: &record.FakeRecorder{},
|
|
}
|
|
|
|
err := rayJobReconciler.createK8sJobIfNeed(ctx, rayJob, rayCluster)
|
|
assert.NoError(t, err)
|
|
|
|
// Test 2: Create a new k8s job if it does not already exist
|
|
fakeClient = clientFake.NewClientBuilder().WithScheme(newScheme).WithRuntimeObjects(rayCluster, rayJob).Build()
|
|
rayJobReconciler.Client = fakeClient
|
|
|
|
err = rayJobReconciler.createK8sJobIfNeed(ctx, rayJob, rayCluster)
|
|
assert.NoError(t, err)
|
|
|
|
err = fakeClient.Get(ctx, types.NamespacedName{
|
|
Namespace: k8sJob.Namespace,
|
|
Name: k8sJob.Name,
|
|
}, k8sJob, nil)
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, k8sJob.Labels[utils.RayOriginatedFromCRNameLabelKey], rayJob.Name)
|
|
assert.Equal(t, k8sJob.Labels[utils.RayOriginatedFromCRDLabelKey], utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD))
|
|
}
|
|
|
|
func TestGetSubmitterTemplate(t *testing.T) {
|
|
// RayJob instance with user-provided submitter pod template.
|
|
rayJobInstanceWithTemplate := &rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{
|
|
Entrypoint: "echo hello world",
|
|
SubmitterPodTemplate: &corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Command: []string{"user-command"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Status: rayv1.RayJobStatus{
|
|
DashboardURL: "test-url",
|
|
JobId: "test-job-id",
|
|
},
|
|
}
|
|
|
|
// RayJob instance without user-provided submitter pod template.
|
|
// In this case we should use the image of the Ray Head, so specify the image so we can test it.
|
|
rayJobInstanceWithoutTemplate := &rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{
|
|
Entrypoint: "echo hello world",
|
|
RayClusterSpec: &rayv1.RayClusterSpec{
|
|
HeadGroupSpec: rayv1.HeadGroupSpec{
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Image: "rayproject/ray:custom-version",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Status: rayv1.RayJobStatus{
|
|
DashboardURL: "test-url",
|
|
JobId: "test-job-id",
|
|
},
|
|
}
|
|
rayClusterInstance := &rayv1.RayCluster{
|
|
Spec: rayv1.RayClusterSpec{
|
|
HeadGroupSpec: rayv1.HeadGroupSpec{
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Image: "rayproject/ray:custom-version",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
r := &RayJobReconciler{}
|
|
ctx := context.Background()
|
|
|
|
// Test 1: User provided template with command
|
|
submitterTemplate, err := r.getSubmitterTemplate(ctx, rayJobInstanceWithTemplate, nil)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, "user-command", submitterTemplate.Spec.Containers[utils.RayContainerIndex].Command[0])
|
|
|
|
// Test 2: User provided template without command
|
|
rayJobInstanceWithTemplate.Spec.SubmitterPodTemplate.Spec.Containers[utils.RayContainerIndex].Command = []string{}
|
|
submitterTemplate, err = r.getSubmitterTemplate(ctx, rayJobInstanceWithTemplate, nil)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, []string{"ray", "job", "submit", "--address", "http://test-url", "--submission-id", "test-job-id", "--", "echo", "hello", "world"}, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Command)
|
|
|
|
// Test 3: User did not provide template, should use the image of the Ray Head
|
|
submitterTemplate, err = r.getSubmitterTemplate(ctx, rayJobInstanceWithoutTemplate, rayClusterInstance)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, []string{"ray", "job", "submit", "--address", "http://test-url", "--submission-id", "test-job-id", "--", "echo", "hello", "world"}, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Command)
|
|
assert.Equal(t, "rayproject/ray:custom-version", submitterTemplate.Spec.Containers[utils.RayContainerIndex].Image)
|
|
|
|
// Test 4: Check default PYTHONUNBUFFERED setting
|
|
submitterTemplate, err = r.getSubmitterTemplate(ctx, rayJobInstanceWithoutTemplate, rayClusterInstance)
|
|
assert.NoError(t, err)
|
|
|
|
envVar, found := utils.EnvVarByName(PythonUnbufferedEnvVarName, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Env)
|
|
assert.True(t, found)
|
|
assert.Equal(t, "1", envVar.Value)
|
|
|
|
// Test 5: Check default RAY_DASHBOARD_ADDRESS env var
|
|
submitterTemplate, err = r.getSubmitterTemplate(ctx, rayJobInstanceWithTemplate, nil)
|
|
assert.NoError(t, err)
|
|
|
|
envVar, found = utils.EnvVarByName(utils.RAY_DASHBOARD_ADDRESS, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Env)
|
|
assert.True(t, found)
|
|
assert.Equal(t, "test-url", envVar.Value)
|
|
|
|
// Test 6: Check default RAY_JOB_SUBMISSION_ID env var
|
|
envVar, found = utils.EnvVarByName(utils.RAY_JOB_SUBMISSION_ID, submitterTemplate.Spec.Containers[utils.RayContainerIndex].Env)
|
|
assert.True(t, found)
|
|
assert.Equal(t, "test-job-id", envVar.Value)
|
|
}
|
|
|
|
func TestUpdateStatusToSuspendingIfNeeded(t *testing.T) {
|
|
newScheme := runtime.NewScheme()
|
|
_ = rayv1.AddToScheme(newScheme)
|
|
tests := map[string]struct {
|
|
status rayv1.JobDeploymentStatus
|
|
suspend bool
|
|
expectedShouldUpdate bool
|
|
}{
|
|
// When Autoscaler is enabled, the random Pod deletion is controleld by the feature flag `ENABLE_RANDOM_POD_DELETE`.
|
|
"Suspend is false": {
|
|
suspend: false,
|
|
status: rayv1.JobDeploymentStatusInitializing,
|
|
expectedShouldUpdate: false,
|
|
},
|
|
"Suspend is true, but the status is not allowed to transition to suspending": {
|
|
suspend: true,
|
|
status: rayv1.JobDeploymentStatusComplete,
|
|
expectedShouldUpdate: false,
|
|
},
|
|
"Suspend is true, and the status is allowed to transition to suspending": {
|
|
suspend: true,
|
|
status: rayv1.JobDeploymentStatusInitializing,
|
|
expectedShouldUpdate: true,
|
|
},
|
|
}
|
|
|
|
for name, tc := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
name := "test-rayjob"
|
|
namespace := "default"
|
|
rayJob := &rayv1.RayJob{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: namespace,
|
|
},
|
|
Spec: rayv1.RayJobSpec{
|
|
Suspend: tc.suspend,
|
|
},
|
|
Status: rayv1.RayJobStatus{
|
|
JobDeploymentStatus: tc.status,
|
|
},
|
|
}
|
|
|
|
// Initialize a fake client with newScheme and runtimeObjects.
|
|
fakeClient := clientFake.NewClientBuilder().
|
|
WithScheme(newScheme).
|
|
WithRuntimeObjects(rayJob).
|
|
WithStatusSubresource(rayJob).Build()
|
|
ctx := context.Background()
|
|
|
|
// Initialize a new RayClusterReconciler.
|
|
testRayJobReconciler := &RayJobReconciler{
|
|
Client: fakeClient,
|
|
Recorder: &record.FakeRecorder{},
|
|
Scheme: newScheme,
|
|
}
|
|
shouldUpdate := testRayJobReconciler.updateStatusToSuspendingIfNeeded(ctx, rayJob)
|
|
assert.Equal(t, tc.expectedShouldUpdate, shouldUpdate)
|
|
|
|
if tc.expectedShouldUpdate {
|
|
assert.Equal(t, rayv1.JobDeploymentStatusSuspending, rayJob.Status.JobDeploymentStatus)
|
|
} else {
|
|
assert.Equal(t, tc.status, rayJob.Status.JobDeploymentStatus)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestUpdateRayJobStatus(t *testing.T) {
|
|
newScheme := runtime.NewScheme()
|
|
_ = rayv1.AddToScheme(newScheme)
|
|
|
|
rayJobTemplate := &rayv1.RayJob{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-rayjob",
|
|
Namespace: "default",
|
|
},
|
|
Status: rayv1.RayJobStatus{
|
|
JobDeploymentStatus: rayv1.JobDeploymentStatusRunning,
|
|
JobStatus: rayv1.JobStatusRunning,
|
|
Message: "old message",
|
|
},
|
|
}
|
|
newMessage := "new message"
|
|
|
|
tests := map[string]struct {
|
|
isJobDeploymentStatusChanged bool
|
|
}{
|
|
"JobDeploymentStatus is not changed": {
|
|
isJobDeploymentStatusChanged: false,
|
|
},
|
|
"JobDeploymentStatus is changed": {
|
|
isJobDeploymentStatusChanged: true,
|
|
},
|
|
}
|
|
|
|
for name, tc := range tests {
|
|
t.Run(name, func(t *testing.T) {
|
|
oldRayJob := rayJobTemplate.DeepCopy()
|
|
|
|
// Initialize a fake client with newScheme and runtimeObjects.
|
|
fakeClient := clientFake.NewClientBuilder().
|
|
WithScheme(newScheme).
|
|
WithRuntimeObjects(oldRayJob).
|
|
WithStatusSubresource(oldRayJob).Build()
|
|
ctx := context.Background()
|
|
|
|
newRayJob := &rayv1.RayJob{}
|
|
err := fakeClient.Get(ctx, types.NamespacedName{Namespace: oldRayJob.Namespace, Name: oldRayJob.Name}, newRayJob)
|
|
assert.NoError(t, err)
|
|
|
|
// Update the status
|
|
newRayJob.Status.Message = newMessage
|
|
if tc.isJobDeploymentStatusChanged {
|
|
newRayJob.Status.JobDeploymentStatus = rayv1.JobDeploymentStatusSuspending
|
|
}
|
|
|
|
// Initialize a new RayClusterReconciler.
|
|
testRayJobReconciler := &RayJobReconciler{
|
|
Client: fakeClient,
|
|
Recorder: &record.FakeRecorder{},
|
|
Scheme: newScheme,
|
|
}
|
|
|
|
err = testRayJobReconciler.updateRayJobStatus(ctx, oldRayJob, newRayJob)
|
|
assert.NoError(t, err)
|
|
|
|
err = fakeClient.Get(ctx, types.NamespacedName{Namespace: newRayJob.Namespace, Name: newRayJob.Name}, newRayJob)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, newRayJob.Status.Message == newMessage, tc.isJobDeploymentStatusChanged)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestValidateRayJobSpec(t *testing.T) {
|
|
err := validateRayJobSpec(&rayv1.RayJob{})
|
|
assert.Error(t, err, "The RayJob is invalid because both `RayClusterSpec` and `ClusterSelector` are empty")
|
|
|
|
err = validateRayJobSpec(&rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{
|
|
Suspend: true,
|
|
ShutdownAfterJobFinishes: false,
|
|
},
|
|
})
|
|
assert.Error(t, err, "The RayJob is invalid because a RayJob with shutdownAfterJobFinishes set to false is not allowed to be suspended.")
|
|
|
|
err = validateRayJobSpec(&rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{
|
|
Suspend: true,
|
|
ShutdownAfterJobFinishes: true,
|
|
RayClusterSpec: &rayv1.RayClusterSpec{},
|
|
},
|
|
})
|
|
assert.NoError(t, err, "The RayJob is valid.")
|
|
|
|
err = validateRayJobSpec(&rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{
|
|
Suspend: true,
|
|
ClusterSelector: map[string]string{
|
|
"key": "value",
|
|
},
|
|
},
|
|
})
|
|
assert.Error(t, err, "The RayJob is invalid because the ClusterSelector mode doesn't support the suspend operation.")
|
|
|
|
err = validateRayJobSpec(&rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{
|
|
RuntimeEnvYAML: "invalid_yaml_str",
|
|
},
|
|
})
|
|
assert.Error(t, err, "The RayJob is invalid because the runtimeEnvYAML is invalid.")
|
|
|
|
err = validateRayJobSpec(&rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{
|
|
BackoffLimit: ptr.To[int32](-1),
|
|
},
|
|
})
|
|
assert.Error(t, err, "The RayJob is invalid because the backoffLimit must be a positive integer.")
|
|
}
|
|
|
|
func TestFailedToCreateRayJobSubmitterEvent(t *testing.T) {
|
|
rayJob := &rayv1.RayJob{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-rayjob",
|
|
Namespace: "default",
|
|
},
|
|
}
|
|
|
|
submitterTemplate := corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-submit-pod",
|
|
Namespace: "default",
|
|
},
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "ray-submit",
|
|
Image: "rayproject/ray:latest",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
fakeClient := clientFake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{
|
|
Create: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.CreateOption) error {
|
|
return errors.New("random")
|
|
},
|
|
}).WithScheme(scheme.Scheme).Build()
|
|
|
|
recorder := record.NewFakeRecorder(100)
|
|
|
|
reconciler := &RayJobReconciler{
|
|
Client: fakeClient,
|
|
Recorder: recorder,
|
|
Scheme: scheme.Scheme,
|
|
}
|
|
|
|
err := reconciler.createNewK8sJob(context.Background(), rayJob, submitterTemplate)
|
|
|
|
assert.NotNil(t, err, "Expected error due to simulated job creation failure")
|
|
|
|
var foundFailureEvent bool
|
|
events := []string{}
|
|
for len(recorder.Events) > 0 {
|
|
event := <-recorder.Events
|
|
if strings.Contains(event, "Failed to create new Kubernetes Job") {
|
|
foundFailureEvent = true
|
|
break
|
|
}
|
|
events = append(events, event)
|
|
}
|
|
|
|
assert.Truef(t, foundFailureEvent, "Expected event to be generated for job creation failure, got events: %s", strings.Join(events, "\n"))
|
|
}
|
|
|
|
func TestFailedCreateRayClusterEvent(t *testing.T) {
|
|
rayJob := &rayv1.RayJob{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-rayjob",
|
|
Namespace: "default",
|
|
},
|
|
Spec: rayv1.RayJobSpec{
|
|
RayClusterSpec: &rayv1.RayClusterSpec{},
|
|
},
|
|
}
|
|
|
|
fakeClient := clientFake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{
|
|
Create: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.CreateOption) error {
|
|
return errors.New("random")
|
|
},
|
|
}).WithScheme(scheme.Scheme).Build()
|
|
|
|
recorder := record.NewFakeRecorder(100)
|
|
|
|
reconciler := &RayJobReconciler{
|
|
Client: fakeClient,
|
|
Recorder: recorder,
|
|
Scheme: scheme.Scheme,
|
|
}
|
|
|
|
_, err := reconciler.getOrCreateRayClusterInstance(context.Background(), rayJob)
|
|
|
|
assert.NotNil(t, err, "Expected error due to cluster creation failure")
|
|
|
|
var foundFailureEvent bool
|
|
events := []string{}
|
|
for len(recorder.Events) > 0 {
|
|
event := <-recorder.Events
|
|
if strings.Contains(event, "Failed to create RayCluster") {
|
|
foundFailureEvent = true
|
|
break
|
|
}
|
|
events = append(events, event)
|
|
}
|
|
|
|
assert.Truef(t, foundFailureEvent, "Expected event to be generated for cluster creation failure, got events: %s", strings.Join(events, "\n"))
|
|
}
|
|
|
|
func TestFailedDeleteRayJobSubmitterEvent(t *testing.T) {
|
|
newScheme := runtime.NewScheme()
|
|
_ = batchv1.AddToScheme(newScheme)
|
|
|
|
rayJob := &rayv1.RayJob{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-rayjob",
|
|
Namespace: "default",
|
|
},
|
|
}
|
|
submitter := &batchv1.Job{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-rayjob",
|
|
Namespace: "default",
|
|
},
|
|
}
|
|
|
|
fakeClient := clientFake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{
|
|
Delete: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.DeleteOption) error {
|
|
return errors.New("random")
|
|
},
|
|
}).WithScheme(newScheme).WithRuntimeObjects(submitter).Build()
|
|
|
|
recorder := record.NewFakeRecorder(100)
|
|
|
|
reconciler := &RayJobReconciler{
|
|
Client: fakeClient,
|
|
Recorder: recorder,
|
|
Scheme: scheme.Scheme,
|
|
}
|
|
|
|
_, err := reconciler.deleteSubmitterJob(context.Background(), rayJob)
|
|
|
|
assert.NotNil(t, err, "Expected error due to job deletion failure")
|
|
|
|
var foundFailureEvent bool
|
|
events := []string{}
|
|
for len(recorder.Events) > 0 {
|
|
event := <-recorder.Events
|
|
if strings.Contains(event, "Failed to delete submitter K8s Job") {
|
|
foundFailureEvent = true
|
|
break
|
|
}
|
|
events = append(events, event)
|
|
}
|
|
|
|
assert.Truef(t, foundFailureEvent, "Expected event to be generated for cluster deletion failure, got events: %s", strings.Join(events, "\n"))
|
|
}
|
|
|
|
func TestFailedDeleteRayClusterEvent(t *testing.T) {
|
|
newScheme := runtime.NewScheme()
|
|
_ = rayv1.AddToScheme(newScheme)
|
|
|
|
rayCluster := &rayv1.RayCluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-raycluster",
|
|
Namespace: "default",
|
|
},
|
|
}
|
|
|
|
rayJob := &rayv1.RayJob{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-rayjob",
|
|
Namespace: "default",
|
|
},
|
|
Status: rayv1.RayJobStatus{
|
|
RayClusterName: "test-raycluster",
|
|
},
|
|
}
|
|
|
|
fakeClient := clientFake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{
|
|
Delete: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.DeleteOption) error {
|
|
return errors.New("random")
|
|
},
|
|
}).WithScheme(newScheme).WithRuntimeObjects(rayCluster).Build()
|
|
|
|
recorder := record.NewFakeRecorder(100)
|
|
|
|
reconciler := &RayJobReconciler{
|
|
Client: fakeClient,
|
|
Recorder: recorder,
|
|
Scheme: scheme.Scheme,
|
|
}
|
|
|
|
_, err := reconciler.deleteClusterResources(context.Background(), rayJob)
|
|
|
|
assert.NotNil(t, err, "Expected error due to cluster deletion failure")
|
|
|
|
var foundFailureEvent bool
|
|
events := []string{}
|
|
for len(recorder.Events) > 0 {
|
|
event := <-recorder.Events
|
|
if strings.Contains(event, "Failed to delete cluster") {
|
|
foundFailureEvent = true
|
|
break
|
|
}
|
|
events = append(events, event)
|
|
}
|
|
|
|
assert.Truef(t, foundFailureEvent, "Expected event to be generated for cluster deletion failure, got events: %s", strings.Join(events, "\n"))
|
|
}
|