Files
kuberay/ray-operator/controllers/ray/raycluster_controller_fake_test.go
2023-06-06 13:25:18 -07:00

1282 lines
42 KiB
Go

/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ray
import (
"context"
"fmt"
"testing"
"github.com/ray-project/kuberay/ray-operator/controllers/ray/common"
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
. "github.com/onsi/ginkgo"
rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1"
"github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme"
"github.com/stretchr/testify/assert"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/selection"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/utils/pointer"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
// +kubebuilder:scaffold:imports
)
var (
namespaceStr string
instanceName string
enableInTreeAutoscaling bool
headGroupNameStr string
headGroupServiceAccount string
groupNameStr string
expectReplicaNum int32
testPods []runtime.Object
testPodsNoHeadIP []runtime.Object
testRayCluster *rayv1alpha1.RayCluster
headSelector labels.Selector
headNodeIP string
testServices []runtime.Object
workerSelector labels.Selector
workersToDelete []string
)
func setupTest(t *testing.T) {
logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
// To verify the failure logic, you can change this PrioritizeWorkersToDelete to false.
PrioritizeWorkersToDelete = true
namespaceStr = "default"
instanceName = "raycluster-sample"
enableInTreeAutoscaling = true
headGroupNameStr = "head-group"
headGroupServiceAccount = "head-service-account"
groupNameStr = "small-group"
expectReplicaNum = 3
workersToDelete = []string{"pod1", "pod2"}
headNodeIP = "1.2.3.4"
testPods = []runtime.Object{
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "headNode",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeTypeLabelKey: string(rayv1alpha1.HeadNode),
common.RayNodeGroupLabelKey: headGroupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-head",
Image: "rayproject/autoscaler",
Command: []string{"python"},
Args: []string{"/opt/code.py"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
PodIP: headNodeIP,
},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod1",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/autoscaler",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod2",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/autoscaler",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod3",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/autoscaler",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod4",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/ray:2.4.0",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod5",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/ray:2.4.0",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
}
testPodsNoHeadIP = []runtime.Object{
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "headNode",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeTypeLabelKey: string(rayv1alpha1.HeadNode),
common.RayNodeGroupLabelKey: headGroupNameStr,
},
},
Status: corev1.PodStatus{
Phase: corev1.PodPending,
PodIP: "",
},
},
}
testRayCluster = &rayv1alpha1.RayCluster{
ObjectMeta: metav1.ObjectMeta{
Name: instanceName,
Namespace: namespaceStr,
},
Spec: rayv1alpha1.RayClusterSpec{
RayVersion: "1.0",
EnableInTreeAutoscaling: &enableInTreeAutoscaling,
HeadGroupSpec: rayv1alpha1.HeadGroupSpec{
Replicas: pointer.Int32Ptr(1),
RayStartParams: map[string]string{
"port": "6379",
"object-manager-port": "12345",
"node-manager-port": "12346",
"object-store-memory": "100000000",
"num-cpus": "1",
},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-head",
Image: "rayproject/ray:2.4.0",
Command: []string{"python"},
Args: []string{"/opt/code.py"},
Env: []corev1.EnvVar{
{
Name: "MY_POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
},
},
},
},
},
},
},
WorkerGroupSpecs: []rayv1alpha1.WorkerGroupSpec{
{
Replicas: pointer.Int32Ptr(expectReplicaNum),
MinReplicas: pointer.Int32Ptr(0),
MaxReplicas: pointer.Int32Ptr(10000),
GroupName: groupNameStr,
RayStartParams: map[string]string{
"port": "6379",
"num-cpus": "1",
},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/ray:2.4.0",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
Env: []corev1.EnvVar{
{
Name: "MY_POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
},
},
},
},
},
},
ScaleStrategy: rayv1alpha1.ScaleStrategy{
WorkersToDelete: workersToDelete,
},
},
},
},
}
headService, err := common.BuildServiceForHeadPod(*testRayCluster, nil, nil)
if err != nil {
t.Errorf("failed to build head service: %v", err)
}
// K8s automatically sets TargetPort to the same value as Port. So we mimic that behavior here.
for i, port := range headService.Spec.Ports {
headService.Spec.Ports[i].TargetPort = intstr.IntOrString{IntVal: port.Port}
}
dashboardService, err := common.BuildDashboardService(*testRayCluster)
if err != nil {
t.Errorf("failed to build dashboard service: %v", err)
}
for i, port := range dashboardService.Spec.Ports {
headService.Spec.Ports[i].TargetPort = intstr.IntOrString{IntVal: port.Port}
}
testServices = []runtime.Object{
headService,
dashboardService,
}
instanceReqValue := []string{instanceName}
instanceReq, err := labels.NewRequirement(
common.RayClusterLabelKey,
selection.Equals,
instanceReqValue)
assert.Nil(t, err, "Fail to create requirement")
groupNameReqValue := []string{groupNameStr}
groupNameReq, err := labels.NewRequirement(
common.RayNodeGroupLabelKey,
selection.Equals,
groupNameReqValue)
assert.Nil(t, err, "Fail to create requirement")
headNameReqValue := []string{headGroupNameStr}
headNameReq, err := labels.NewRequirement(
common.RayNodeGroupLabelKey,
selection.Equals,
headNameReqValue)
assert.Nil(t, err, "Fail to create requirement")
headSelector = labels.NewSelector().Add(*headNameReq)
workerSelector = labels.NewSelector().Add(*instanceReq).Add(*groupNameReq)
}
func tearDown(t *testing.T) {
PrioritizeWorkersToDelete = false
}
// TestReconcile_UnhealthyEvent tests the case where we have unhealthy events
// and we want to update the corresponding pods.
func TestReconcile_UnhealthyEvent(t *testing.T) {
setupTest(t)
defer tearDown(t)
testPodName := "eventPod"
// testPodEventName is the name of the event that will be created for testPodName
// The name of the event is generated by concatenating the pod name and a
// meaningless random string
testPodEventName := fmt.Sprintf("%s.15f0c0c5c5c5c5c5", testPodName)
// add a pod in a different namespace
newPods := []runtime.Object{
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: testPodName,
UID: types.UID(testPodName),
Namespace: "ns2",
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/ray:2.2.0",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "newPod2",
UID: types.UID("newPod2"),
Namespace: namespaceStr,
Labels: map[string]string{
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/ray:2.2.0",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: testPodName,
UID: types.UID(testPodName),
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
Annotations: map[string]string{
common.RayFTEnabledAnnotationKey: "true",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/ray:2.2.0",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
}
testPods = append(testPods, newPods...)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
ctx := context.Background()
podList := corev1.PodList{}
err := fakeClient.List(ctx, &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, len(testPods)-1, len(podList.Items), "Init pod list len is wrong")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
// add event for reconcile
err = fakeClient.Create(ctx, &corev1.Event{
ObjectMeta: metav1.ObjectMeta{
Name: testPodEventName,
Namespace: namespaceStr,
},
InvolvedObject: corev1.ObjectReference{
Kind: "Pod",
Name: testPodName,
Namespace: namespaceStr,
},
Reason: "Unhealthy",
Type: "Warning",
Message: "Readiness probe failed",
})
assert.Nil(t, err, "Fail to create event")
if _, err := testRayClusterReconciler.Reconcile(ctx, ctrl.Request{
NamespacedName: types.NamespacedName{
Namespace: namespaceStr,
Name: testPodEventName,
},
}); err != nil {
assert.Nil(t, err, "Fail to reconcile")
}
err = fakeClient.List(ctx, &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
for _, pod := range podList.Items {
if pod.Name == testPodName && pod.Namespace == namespaceStr {
assert.Equal(t, pod.Annotations[common.RayNodeHealthStateAnnotationKey],
common.PodUnhealthy, "Pod annotation is wrong")
}
}
for _, pod := range podList.Items {
assert.Equal(t, corev1.PodRunning, pod.Status.Phase, "Pod phase is wrong")
}
}
func TestReconcile_RemoveWorkersToDelete_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
ctx := context.Background()
podList := corev1.PodList{}
err := fakeClient.List(ctx, &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, len(testPods), len(podList.Items), "Init pod list len is wrong")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcilePods(ctx, testRayCluster)
assert.Nil(t, err, "Fail to reconcile Pods")
err = fakeClient.List(ctx, &podList, &client.ListOptions{
LabelSelector: workerSelector,
Namespace: namespaceStr,
})
assert.Nil(t, err, "Fail to get pod list after reconcile")
assert.Equal(t, int(expectReplicaNum), len(podList.Items),
"Replica number is wrong after reconcile expect %d actual %d", expectReplicaNum, len(podList.Items))
}
func TestReconcile_RandomDelete_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
var localExpectReplicaNum int32 = 2
testRayCluster.Spec.WorkerGroupSpecs[0].Replicas = &localExpectReplicaNum
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
ctx := context.Background()
podList := corev1.PodList{}
err := fakeClient.List(ctx, &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, len(testPods), len(podList.Items), "Init pod list len is wrong")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcilePods(ctx, testRayCluster)
assert.Nil(t, err, "Fail to reconcile Pods")
err = fakeClient.List(ctx, &podList, &client.ListOptions{
LabelSelector: workerSelector,
Namespace: namespaceStr,
})
assert.Nil(t, err, "Fail to get pod list after reconcile")
assert.Equal(t, int(localExpectReplicaNum), len(podList.Items),
"Replica number is wrong after reconcile expect %d actual %d", expectReplicaNum, len(podList.Items))
for i := 0; i < len(podList.Items); i++ {
if contains(workersToDelete, podList.Items[i].Name) {
t.Fatalf("WorkersToDelete is not actually deleted, %s", podList.Items[i].Name)
}
}
}
func TestReconcile_PodDeleted_Diff0_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
// TODO (kevin85421): The tests in this file are not independent. As a workaround,
// I added the assertion to prevent the test logic from being affected by other changes.
// However, we should refactor the tests in the future.
// This test makes some assumptions about the testRayCluster object.
// (1) 1 workerGroup (2) The goal state of the workerGroup is 3 replicas. (3) Set the workersToDelete to empty.
assert.Equal(t, 1, len(testRayCluster.Spec.WorkerGroupSpecs), "This test assumes only one worker group.")
testRayCluster.Spec.WorkerGroupSpecs[0].ScaleStrategy.WorkersToDelete = []string{}
expectedNumWorkerPods := int(*testRayCluster.Spec.WorkerGroupSpecs[0].Replicas)
assert.Equal(t, 3, expectedNumWorkerPods, "This test assumes the expected number of worker pods is 3.")
// This test makes some assumptions about the testPods object.
// `testPods` contains 6 pods, including 1 head pod and 5 worker pods.
assert.Equal(t, 6, len(testPods), "This test assumes the testPods object contains 6 pods.")
numHeadPods := 1
oldNumWorkerPods := len(testPods) - numHeadPods
// Initialize a fake client with newScheme and runtimeObjects.
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
ctx := context.Background()
// Get the pod list from the fake client.
podList := corev1.PodList{}
err := fakeClient.List(ctx, &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, oldNumWorkerPods+numHeadPods, len(podList.Items), "Init pod list len is wrong")
// Simulate the deletion of 2 worker Pods. After the deletion, the number of worker Pods should be 3.
err = fakeClient.Delete(ctx, &podList.Items[3])
assert.Nil(t, err, "Fail to delete pod")
err = fakeClient.Delete(ctx, &podList.Items[4])
assert.Nil(t, err, "Fail to delete pod")
// Initialize a new RayClusterReconciler.
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
// Since the desired state of the workerGroup is 3 replicas,
// the controller will not create or delete any worker Pods.
err = testRayClusterReconciler.reconcilePods(ctx, testRayCluster)
assert.Nil(t, err, "Fail to reconcile Pods")
err = fakeClient.List(ctx, &podList, &client.ListOptions{
LabelSelector: workerSelector,
Namespace: namespaceStr,
})
assert.Nil(t, err, "Fail to get pod list after reconcile")
assert.Equal(t, expectedNumWorkerPods, len(podList.Items),
"Replica number is wrong after reconcile expect %d actual %d", expectedNumWorkerPods, len(podList.Items))
}
func TestReconcile_PodDeleted_DiffLess0_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
// TODO (kevin85421): The tests in this file are not independent. As a workaround,
// I added the assertion to prevent the test logic from being affected by other changes.
// However, we should refactor the tests in the future.
// This test makes some assumptions about the testRayCluster object.
// (1) 1 workerGroup (2) The goal state of the workerGroup is 3 replicas. (3) Set the workersToDelete to empty.
assert.Equal(t, 1, len(testRayCluster.Spec.WorkerGroupSpecs), "This test assumes only one worker group.")
testRayCluster.Spec.WorkerGroupSpecs[0].ScaleStrategy.WorkersToDelete = []string{}
expectedNumWorkerPods := int(*testRayCluster.Spec.WorkerGroupSpecs[0].Replicas)
assert.Equal(t, 3, expectedNumWorkerPods, "This test assumes the expected number of worker pods is 3.")
// This test makes some assumptions about the testPods object.
// `testPods` contains 6 pods, including 1 head pod and 5 worker pods.
assert.Equal(t, 6, len(testPods), "This test assumes the testPods object contains 6 pods.")
numHeadPods := 1
oldNumWorkerPods := len(testPods) - numHeadPods
// Initialize a fake client with newScheme and runtimeObjects.
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
// Get the pod list from the fake client.
podList := corev1.PodList{}
ctx := context.Background()
err := fakeClient.List(ctx, &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, oldNumWorkerPods+numHeadPods, len(podList.Items), "Init pod list len is wrong")
// Simulate the deletion of 2 worker Pods. After the deletion, the number of worker Pods should be 4.
err = fakeClient.Delete(ctx, &podList.Items[3])
assert.Nil(t, err, "Fail to delete pod")
// Initialize a new RayClusterReconciler.
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
// Since the desired state of the workerGroup is 3 replicas,
// the controller will delete a worker Pod to reach the goal state.
err = testRayClusterReconciler.reconcilePods(ctx, testRayCluster)
assert.Nil(t, err, "Fail to reconcile Pods")
err = fakeClient.List(ctx, &podList, &client.ListOptions{
LabelSelector: workerSelector,
Namespace: namespaceStr,
})
assert.Nil(t, err, "Fail to get pod list after reconcile")
// The number of worker Pods should be 3.
assert.Equal(t, expectedNumWorkerPods, len(podList.Items),
"Replica number is wrong after reconcile expect %d actual %d", expectReplicaNum, len(podList.Items))
}
func TestReconcile_PodDCrash_Diff0_WorkersToDelete_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
// TODO (kevin85421): The tests in this file are not independent. As a workaround,
// I added the assertion to prevent the test logic from being affected by other changes.
// However, we should refactor the tests in the future.
// This test makes some assumptions about the testRayCluster object.
// (1) 1 workerGroup
// (2) The goal state of the workerGroup is 3 replicas.
// (3) The workersToDelete has 2 worker Pods (pod3 and pod4). => Simulate the autoscaler scale-down.
assert.Equal(t, 1, len(testRayCluster.Spec.WorkerGroupSpecs), "This test assumes only one worker group.")
expectedNumWorkerPods := int(*testRayCluster.Spec.WorkerGroupSpecs[0].Replicas)
assert.Equal(t, 3, expectedNumWorkerPods, "This test assumes the expected number of worker pods is 3.")
testRayCluster.Spec.WorkerGroupSpecs[0].ScaleStrategy.WorkersToDelete = []string{"pod3", "pod4"}
// This test makes some assumptions about the testPods object.
// `testPods` contains 6 pods, including 1 head pod and 5 worker pods.
assert.Equal(t, 6, len(testPods), "This test assumes the testPods object contains 6 pods.")
numHeadPods := 1
oldNumWorkerPods := len(testPods) - numHeadPods
// Initialize a fake client with newScheme and runtimeObjects.
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
ctx := context.Background()
// Get the pod list from the fake client.
podList := corev1.PodList{}
err := fakeClient.List(ctx, &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, oldNumWorkerPods+numHeadPods, len(podList.Items), "Init pod list len is wrong")
// Simulate 2 Pod fails. Because the workersToDelete also contains pod3 and pod4, the controller will
// delete these two Pods. After the deletion, the number of worker Pods should be 3.
podList.Items[3].Status.Phase = corev1.PodFailed
podList.Items[4].Status.Phase = corev1.PodFailed
err = fakeClient.Update(ctx, &podList.Items[3])
assert.Nil(t, err, "Fail to get update pod status")
err = fakeClient.Update(ctx, &podList.Items[4])
assert.Nil(t, err, "Fail to get update pod status")
// Initialize a new RayClusterReconciler.
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
// Since the desired state of the workerGroup is 3 replicas, the controller
// will not create or delete any worker Pods.
err = testRayClusterReconciler.reconcilePods(ctx, testRayCluster)
assert.Nil(t, err, "Fail to reconcile Pods")
err = fakeClient.List(ctx, &podList, &client.ListOptions{
LabelSelector: workerSelector,
Namespace: namespaceStr,
})
assert.Nil(t, err, "Fail to get pod list after reconcile")
// Failed Pods (pod3, pod4) should be deleted because of the workersToDelete.
// Hence, no failed Pods should exist in `podList`.
assert.Equal(t, expectedNumWorkerPods, len(podList.Items))
assert.Equal(t, expectedNumWorkerPods, getNotFailedPodItemNum(podList),
"Replica number is wrong after reconcile expect %d actual %d", expectReplicaNum, getNotFailedPodItemNum(podList))
}
func TestReconcile_PodDCrash_DiffLess0_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
// TODO (kevin85421): The tests in this file are not independent. As a workaround,
// I added the assertion to prevent the test logic from being affected by other changes.
// However, we should refactor the tests in the future.
// This test makes some assumptions about the testRayCluster object.
// (1) 1 workerGroup
// (2) The goal state of the workerGroup is 3 replicas.
// (3) The workersToDelete has 1 worker Pod (pod3). => Simulate the autoscaler scale-down.
assert.Equal(t, 1, len(testRayCluster.Spec.WorkerGroupSpecs), "This test assumes only one worker group.")
expectedNumWorkerPods := int(*testRayCluster.Spec.WorkerGroupSpecs[0].Replicas)
assert.Equal(t, 3, expectedNumWorkerPods, "This test assumes the expected number of worker pods is 3.")
testRayCluster.Spec.WorkerGroupSpecs[0].ScaleStrategy.WorkersToDelete = []string{"pod3"}
// This test makes some assumptions about the testPods object.
// `testPods` contains 6 pods, including 1 head pod and 5 worker pods.
assert.Equal(t, 6, len(testPods), "This test assumes the testPods object contains 6 pods.")
numHeadPods := 1
oldNumWorkerPods := len(testPods) - numHeadPods
// Initialize a fake client with newScheme and runtimeObjects.
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
ctx := context.Background()
// Get the pod list from the fake client.
podList := corev1.PodList{}
err := fakeClient.List(ctx, &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, oldNumWorkerPods+numHeadPods, len(podList.Items), "Init pod list len is wrong")
// Simulate 1 Pod fails. Because the workersToDelete also contains pod3, the controller will
// delete pod3. After the deletion, the number of worker Pods should be 4.
podList.Items[3].Status.Phase = corev1.PodFailed
err = fakeClient.Update(ctx, &podList.Items[3])
assert.Nil(t, err, "Fail to get update pod status")
// Initialize a new RayClusterReconciler.
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
// Since the desired state of the workerGroup is 3 replicas, the controller
// will delete a worker Pod.
err = testRayClusterReconciler.reconcilePods(ctx, testRayCluster)
assert.Nil(t, err, "Fail to reconcile Pods")
err = fakeClient.List(ctx, &podList, &client.ListOptions{
LabelSelector: workerSelector,
Namespace: namespaceStr,
})
assert.Nil(t, err, "Fail to get pod list after reconcile")
// Failed Pods (pod3) should be deleted because of the workersToDelete.
// Hence, no failed Pods should exist in `podList`.
assert.Equal(t, expectedNumWorkerPods, len(podList.Items))
assert.Equal(t, expectedNumWorkerPods, getNotFailedPodItemNum(podList),
"Replica number is wrong after reconcile expect %d actual %d", expectReplicaNum, getNotFailedPodItemNum(podList))
}
func TestReconcile_PodEvicted_DiffLess0_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
ctx := context.Background()
podList := corev1.PodList{}
err := fakeClient.List(ctx, &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, len(testPods), len(podList.Items), "Init pod list len is wrong")
// Simulate head pod get evicted.
podList.Items[0].Status.Phase = corev1.PodFailed
podList.Items[0].Status.Reason = "Evicted"
err = fakeClient.Update(ctx, &podList.Items[0])
assert.Nil(t, err, "Fail to get update pod status")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcilePods(ctx, testRayCluster)
assert.Nil(t, err, "Fail to reconcile Pods")
// Filter head pod
err = fakeClient.List(ctx, &podList, &client.ListOptions{
LabelSelector: headSelector,
Namespace: namespaceStr,
})
assert.Nil(t, err, "Fail to get pod list after reconcile")
assert.Equal(t, 0, len(podList.Items),
"Evicted head should be deleted after reconcile expect %d actual %d", 0, len(podList.Items))
}
func contains(slice []string, item string) bool {
set := make(map[string]struct{}, len(slice))
for _, s := range slice {
set[s] = struct{}{}
}
_, ok := set[item]
return ok
}
func getNotFailedPodItemNum(podList corev1.PodList) int {
count := 0
for _, aPod := range podList.Items {
if aPod.Status.Phase != corev1.PodFailed {
count++
}
}
return count
}
func TestReconcile_AutoscalerServiceAccount(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
ctx := context.Background()
saNamespacedName := types.NamespacedName{
Name: utils.GetHeadGroupServiceAccountName(testRayCluster),
Namespace: namespaceStr,
}
sa := corev1.ServiceAccount{}
err := fakeClient.Get(ctx, saNamespacedName, &sa)
assert.True(t, k8serrors.IsNotFound(err), "Head group service account should not exist yet")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcileAutoscalerServiceAccount(ctx, testRayCluster)
assert.Nil(t, err, "Fail to reconcile autoscaler ServiceAccount")
err = fakeClient.Get(ctx, saNamespacedName, &sa)
assert.Nil(t, err, "Fail to get head group ServiceAccount after reconciliation")
}
func TestReconcile_Autoscaler_ServiceAccountName(t *testing.T) {
setupTest(t)
defer tearDown(t)
// Specify a ServiceAccountName for the head Pod
myServiceAccount := corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "my-sa",
Namespace: namespaceStr,
},
}
cluster := testRayCluster.DeepCopy()
cluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = myServiceAccount.Name
// Case 1: There is no ServiceAccount "my-sa" in the Kubernetes cluster
runtimeObjects := []runtime.Object{}
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(runtimeObjects...).Build()
ctx := context.Background()
// Initialize the reconciler
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
// If users specify ServiceAccountName for the head Pod, they need to create a ServiceAccount themselves.
// However, if KubeRay creates a ServiceAccount for users, the autoscaler may encounter permission issues during
// zero-downtime rolling updates when RayService is performed. See https://github.com/ray-project/kuberay/pull/1128
// for more details.
err := testRayClusterReconciler.reconcileAutoscalerServiceAccount(ctx, cluster)
assert.NotNil(t, err,
"When users specify ServiceAccountName for the head Pod, they need to create a ServiceAccount themselves. "+
"If the ServiceAccount does not exist, the reconciler should return an error. However, err is nil.")
// Case 2: There is a ServiceAccount "my-sa" in the Kubernetes cluster
runtimeObjects = []runtime.Object{&myServiceAccount}
fakeClient = clientFake.NewClientBuilder().WithRuntimeObjects(runtimeObjects...).Build()
// Initialize the reconciler
testRayClusterReconciler = &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcileAutoscalerServiceAccount(ctx, cluster)
assert.Nil(t, err)
}
func TestReconcile_AutoscalerRoleBinding(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
ctx := context.Background()
rbNamespacedName := types.NamespacedName{
Name: instanceName,
Namespace: namespaceStr,
}
rb := rbacv1.RoleBinding{}
err := fakeClient.Get(ctx, rbNamespacedName, &rb)
assert.True(t, k8serrors.IsNotFound(err), "autoscaler RoleBinding should not exist yet")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcileAutoscalerRoleBinding(ctx, testRayCluster)
assert.Nil(t, err, "Fail to reconcile autoscaler RoleBinding")
err = fakeClient.Get(ctx, rbNamespacedName, &rb)
assert.Nil(t, err, "Fail to get autoscaler RoleBinding after reconciliation")
}
func TestReconcile_UpdateClusterReason(t *testing.T) {
setupTest(t)
defer tearDown(t)
newScheme := runtime.NewScheme()
_ = rayv1alpha1.AddToScheme(newScheme)
fakeClient := clientFake.NewClientBuilder().WithScheme(newScheme).WithRuntimeObjects(testRayCluster).Build()
ctx := context.Background()
namespacedName := types.NamespacedName{
Name: instanceName,
Namespace: namespaceStr,
}
cluster := rayv1alpha1.RayCluster{}
err := fakeClient.Get(ctx, namespacedName, &cluster)
assert.Nil(t, err, "Fail to get RayCluster")
assert.Empty(t, cluster.Status.Reason, "Cluster reason should be empty")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
reason := "test reason"
err = testRayClusterReconciler.updateClusterReason(ctx, testRayCluster, reason)
assert.Nil(t, err, "Fail to update cluster reason")
err = fakeClient.Get(ctx, namespacedName, &cluster)
assert.Nil(t, err, "Fail to get RayCluster after updating reason")
assert.Equal(t, cluster.Status.Reason, reason, "Cluster reason should be updated")
}
func TestUpdateEndpoints(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testServices...).Build()
ctx := context.Background()
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
if err := testRayClusterReconciler.updateEndpoints(ctx, testRayCluster); err != nil {
t.Errorf("updateEndpoints failed: %v", err)
}
expected := map[string]string{
"client": "10001",
"dashboard": "8265",
"metrics": "8080",
"redis": "6379",
"serve": "8000",
}
assert.Equal(t, expected, testRayCluster.Status.Endpoints, "RayCluster status endpoints not updated")
}
func TestGetHeadPodIP(t *testing.T) {
setupTest(t)
defer tearDown(t)
extraHeadPod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "unexpectedExtraHeadNode",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayClusterLabelKey: instanceName,
common.RayNodeTypeLabelKey: string(rayv1alpha1.HeadNode),
common.RayNodeGroupLabelKey: headGroupNameStr,
},
},
}
tests := map[string]struct {
pods []runtime.Object
expectedIP string
returnsError bool
}{
"get expected Pod IP if there's one head node": {
pods: testPods,
expectedIP: headNodeIP,
returnsError: false,
},
"no error if there's no head node": {
pods: []runtime.Object{},
expectedIP: "",
returnsError: false,
},
"no error if there's more than one head node": {
pods: append(testPods, extraHeadPod),
expectedIP: "",
returnsError: false,
},
"no error if head pod ip is not yet set": {
pods: testPodsNoHeadIP,
expectedIP: "",
returnsError: false,
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(tc.pods...).Build()
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
ip, err := testRayClusterReconciler.getHeadPodIP(context.TODO(), testRayCluster)
if tc.returnsError {
assert.NotNil(t, err, "getHeadPodIP should return error")
} else {
assert.Nil(t, err, "getHeadPodIP should not return error")
}
assert.Equal(t, tc.expectedIP, ip, "getHeadPodIP returned unexpected IP")
})
}
}
func TestGetHeadServiceIP(t *testing.T) {
setupTest(t)
defer tearDown(t)
headServiceIP := "1.2.3.4"
headService, err := common.BuildServiceForHeadPod(*testRayCluster, nil, nil)
if err != nil {
t.Errorf("failed to build head service: %v", err)
}
headService.Spec.ClusterIP = headServiceIP
testServices = []runtime.Object{
headService,
}
extraHeadService := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "unexpectedExtraHeadService",
Namespace: namespaceStr,
Labels: common.HeadServiceLabels(*testRayCluster),
},
}
tests := map[string]struct {
services []runtime.Object
expectedIP string
returnsError bool
}{
"get expected Service IP if there's one head Service": {
services: testServices,
expectedIP: headServiceIP,
returnsError: false,
},
"get error if there's no head Service": {
services: []runtime.Object{},
expectedIP: "",
returnsError: true,
},
"get error if there's more than one head Service": {
services: append(testServices, extraHeadService),
expectedIP: "",
returnsError: true,
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(tc.services...).Build()
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
ip, err := testRayClusterReconciler.getHeadServiceIP(context.TODO(), testRayCluster)
if tc.returnsError {
assert.NotNil(t, err, "getHeadServiceIP should return error")
} else {
assert.Nil(t, err, "getHeadServiceIP should not return error")
}
assert.Equal(t, tc.expectedIP, ip, "getHeadServiceIP returned unexpected IP")
})
}
}
func TestUpdateStatusObservedGeneration(t *testing.T) {
setupTest(t)
defer tearDown(t)
// Create a new scheme with CRDs, Pod, Service schemes.
newScheme := runtime.NewScheme()
_ = rayv1alpha1.AddToScheme(newScheme)
_ = corev1.AddToScheme(newScheme)
// To update the status of RayCluster with `r.Status().Update()`,
// initialize the runtimeObjects with appropriate context. In KubeRay, the `ClusterIP`
// and `TargetPort` fields are typically set by the cluster's control plane.
headService, err := common.BuildServiceForHeadPod(*testRayCluster, nil, nil)
assert.Nil(t, err, "Failed to build head service.")
headService.Spec.ClusterIP = headNodeIP
for i, port := range headService.Spec.Ports {
headService.Spec.Ports[i].TargetPort = intstr.IntOrString{IntVal: port.Port}
}
runtimeObjects := append(testPods, headService, testRayCluster)
// To facilitate testing, we set an impossible value for ObservedGeneration.
// Note that ObjectMeta's `Generation` and `ResourceVersion` don't behave properly in the fake client.
// [Ref] https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/client/fake
testRayCluster.Status.ObservedGeneration = -1
// Initialize a fake client with newScheme and runtimeObjects.
fakeClient := clientFake.NewClientBuilder().WithScheme(newScheme).WithRuntimeObjects(runtimeObjects...).Build()
ctx := context.Background()
// Verify the initial values of `Generation` and `ObservedGeneration`.
namespacedName := types.NamespacedName{
Name: instanceName,
Namespace: namespaceStr,
}
cluster := rayv1alpha1.RayCluster{}
err = fakeClient.Get(ctx, namespacedName, &cluster)
assert.Nil(t, err, "Fail to get RayCluster")
assert.Equal(t, int64(-1), cluster.Status.ObservedGeneration)
assert.Equal(t, int64(0), cluster.ObjectMeta.Generation)
// Initialize RayCluster reconciler.
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
// Compare the values of `Generation` and `ObservedGeneration` to check if they match.
err = testRayClusterReconciler.updateStatus(ctx, testRayCluster)
assert.Nil(t, err)
err = fakeClient.Get(ctx, namespacedName, &cluster)
assert.Nil(t, err)
assert.Equal(t, cluster.ObjectMeta.Generation, cluster.Status.ObservedGeneration)
}