Files
kuberay/ray-operator/controllers/ray/raycluster_controller_fake_test.go
Wilson WangandGitHub 3c53af6071 Fix issue with operator OOM restart (#946)
Fix issue with operator OOM restart
2023-03-28 10:29:21 -07:00

1184 lines
36 KiB
Go

/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ray
import (
"context"
"fmt"
"testing"
"github.com/ray-project/kuberay/ray-operator/controllers/ray/common"
. "github.com/onsi/ginkgo"
rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1"
"github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme"
"github.com/stretchr/testify/assert"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/selection"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/utils/pointer"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
// +kubebuilder:scaffold:imports
)
var (
namespaceStr string
instanceName string
enableInTreeAutoscaling bool
headGroupNameStr string
headGroupServiceAccount string
groupNameStr string
expectReplicaNum int32
testPods []runtime.Object
testPodsNoHeadIP []runtime.Object
testRayCluster *rayiov1alpha1.RayCluster
headSelector labels.Selector
headNodeIP string
testServices []runtime.Object
workerSelector labels.Selector
workersToDelete []string
)
func setupTest(t *testing.T) {
logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
// To verify the failure logic, you can change this PrioritizeWorkersToDelete to false.
PrioritizeWorkersToDelete = true
namespaceStr = "default"
instanceName = "raycluster-sample"
enableInTreeAutoscaling = true
headGroupNameStr = "head-group"
headGroupServiceAccount = "head-service-account"
groupNameStr = "small-group"
expectReplicaNum = 3
workersToDelete = []string{"pod1", "pod2"}
headNodeIP = "1.2.3.4"
testPods = []runtime.Object{
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "headNode",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeTypeLabelKey: string(rayiov1alpha1.HeadNode),
common.RayNodeGroupLabelKey: headGroupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-head",
Image: "rayproject/autoscaler",
Command: []string{"python"},
Args: []string{"/opt/code.py"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
PodIP: headNodeIP,
},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod1",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/autoscaler",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod2",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/autoscaler",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod3",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/autoscaler",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod4",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/ray:2.3.0",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod5",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/ray:2.3.0",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
}
testPodsNoHeadIP = []runtime.Object{
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "headNode",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeTypeLabelKey: string(rayiov1alpha1.HeadNode),
common.RayNodeGroupLabelKey: headGroupNameStr,
},
},
Status: corev1.PodStatus{
Phase: corev1.PodPending,
PodIP: "",
},
},
}
testRayCluster = &rayiov1alpha1.RayCluster{
ObjectMeta: metav1.ObjectMeta{
Name: instanceName,
Namespace: namespaceStr,
},
Spec: rayiov1alpha1.RayClusterSpec{
RayVersion: "1.0",
EnableInTreeAutoscaling: &enableInTreeAutoscaling,
HeadGroupSpec: rayiov1alpha1.HeadGroupSpec{
ServiceType: "ClusterIP",
Replicas: pointer.Int32Ptr(1),
RayStartParams: map[string]string{
"port": "6379",
"object-manager-port": "12345",
"node-manager-port": "12346",
"object-store-memory": "100000000",
"num-cpus": "1",
},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
ServiceAccountName: headGroupServiceAccount,
Containers: []corev1.Container{
{
Name: "ray-head",
Image: "rayproject/ray:2.3.0",
Command: []string{"python"},
Args: []string{"/opt/code.py"},
Env: []corev1.EnvVar{
{
Name: "MY_POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
},
},
},
},
},
},
},
WorkerGroupSpecs: []rayiov1alpha1.WorkerGroupSpec{
{
Replicas: pointer.Int32Ptr(expectReplicaNum),
MinReplicas: pointer.Int32Ptr(0),
MaxReplicas: pointer.Int32Ptr(10000),
GroupName: groupNameStr,
RayStartParams: map[string]string{
"port": "6379",
"num-cpus": "1",
},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/ray:2.3.0",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
Env: []corev1.EnvVar{
{
Name: "MY_POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
},
},
},
},
},
},
ScaleStrategy: rayiov1alpha1.ScaleStrategy{
WorkersToDelete: workersToDelete,
},
},
},
},
}
headService, err := common.BuildServiceForHeadPod(*testRayCluster, nil, nil)
if err != nil {
t.Errorf("failed to build head service: %v", err)
}
// K8s automatically sets TargetPort to the same value as Port. So we mimic that behavior here.
for i, port := range headService.Spec.Ports {
headService.Spec.Ports[i].TargetPort = intstr.IntOrString{IntVal: port.Port}
}
dashboardService, err := common.BuildDashboardService(*testRayCluster)
if err != nil {
t.Errorf("failed to build dashboard service: %v", err)
}
for i, port := range dashboardService.Spec.Ports {
headService.Spec.Ports[i].TargetPort = intstr.IntOrString{IntVal: port.Port}
}
testServices = []runtime.Object{
headService,
dashboardService,
}
instanceReqValue := []string{instanceName}
instanceReq, err := labels.NewRequirement(
common.RayClusterLabelKey,
selection.Equals,
instanceReqValue)
assert.Nil(t, err, "Fail to create requirement")
groupNameReqValue := []string{groupNameStr}
groupNameReq, err := labels.NewRequirement(
common.RayNodeGroupLabelKey,
selection.Equals,
groupNameReqValue)
assert.Nil(t, err, "Fail to create requirement")
headNameReqValue := []string{headGroupNameStr}
headNameReq, err := labels.NewRequirement(
common.RayNodeGroupLabelKey,
selection.Equals,
headNameReqValue)
assert.Nil(t, err, "Fail to create requirement")
headSelector = labels.NewSelector().Add(*headNameReq)
workerSelector = labels.NewSelector().Add(*instanceReq).Add(*groupNameReq)
}
func tearDown(t *testing.T) {
PrioritizeWorkersToDelete = false
}
// TestReconcile_UnhealthyEvent tests the case where we have unhealthy events
// and we want to update the corresponding pods.
func TestReconcile_UnhealthyEvent(t *testing.T) {
setupTest(t)
defer tearDown(t)
testPodName := "eventPod"
// testPodEventName is the name of the event that will be created for testPodName
// The name of the event is generated by concatenating the pod name and a
// meaningless random string
testPodEventName := fmt.Sprintf("%s.15f0c0c5c5c5c5c5", testPodName)
// add a pod in a different namespace
newPods := []runtime.Object{
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: testPodName,
UID: types.UID(testPodName),
Namespace: "ns2",
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/ray:2.2.0",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "newPod2",
UID: types.UID("newPod2"),
Namespace: namespaceStr,
Labels: map[string]string{
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/ray:2.2.0",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: testPodName,
UID: types.UID(testPodName),
Namespace: namespaceStr,
Labels: map[string]string{
common.RayNodeLabelKey: "yes",
common.RayClusterLabelKey: instanceName,
common.RayNodeGroupLabelKey: groupNameStr,
},
Annotations: map[string]string{
common.RayFTEnabledAnnotationKey: "true",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/ray:2.2.0",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
}
testPods = append(testPods, newPods...)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
podList := corev1.PodList{}
err := fakeClient.List(context.Background(), &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, len(testPods)-1, len(podList.Items), "Init pod list len is wrong")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
// add event for reconcile
err = fakeClient.Create(context.Background(), &corev1.Event{
ObjectMeta: metav1.ObjectMeta{
Name: testPodEventName,
Namespace: namespaceStr,
},
InvolvedObject: corev1.ObjectReference{
Kind: "Pod",
Name: testPodName,
Namespace: namespaceStr,
},
Reason: "Unhealthy",
Type: "Warning",
Message: "Readiness probe failed",
})
assert.Nil(t, err, "Fail to create event")
if _, err := testRayClusterReconciler.Reconcile(context.Background(), ctrl.Request{
NamespacedName: types.NamespacedName{
Namespace: namespaceStr,
Name: testPodEventName,
},
}); err != nil {
assert.Nil(t, err, "Fail to reconcile")
}
err = fakeClient.List(context.Background(), &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
for _, pod := range podList.Items {
if pod.Name == testPodName && pod.Namespace == namespaceStr {
assert.Equal(t, pod.Annotations[common.RayNodeHealthStateAnnotationKey],
common.PodUnhealthy, "Pod annotation is wrong")
}
}
for _, pod := range podList.Items {
assert.Equal(t, corev1.PodRunning, pod.Status.Phase, "Pod phase is wrong")
}
}
func TestReconcile_RemoveWorkersToDelete_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
podList := corev1.PodList{}
err := fakeClient.List(context.Background(), &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, len(testPods), len(podList.Items), "Init pod list len is wrong")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcilePods(testRayCluster)
assert.Nil(t, err, "Fail to reconcile Pods")
err = fakeClient.List(context.Background(), &podList, &client.ListOptions{
LabelSelector: workerSelector,
Namespace: namespaceStr,
})
assert.Nil(t, err, "Fail to get pod list after reconcile")
assert.Equal(t, int(expectReplicaNum), len(podList.Items),
"Replica number is wrong after reconcile expect %d actual %d", expectReplicaNum, len(podList.Items))
}
func TestReconcile_RandomDelete_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
var localExpectReplicaNum int32 = 2
testRayCluster.Spec.WorkerGroupSpecs[0].Replicas = &localExpectReplicaNum
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
podList := corev1.PodList{}
err := fakeClient.List(context.Background(), &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, len(testPods), len(podList.Items), "Init pod list len is wrong")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcilePods(testRayCluster)
assert.Nil(t, err, "Fail to reconcile Pods")
err = fakeClient.List(context.Background(), &podList, &client.ListOptions{
LabelSelector: workerSelector,
Namespace: namespaceStr,
})
assert.Nil(t, err, "Fail to get pod list after reconcile")
assert.Equal(t, int(localExpectReplicaNum), len(podList.Items),
"Replica number is wrong after reconcile expect %d actual %d", expectReplicaNum, len(podList.Items))
for i := 0; i < len(podList.Items); i++ {
if contains(workersToDelete, podList.Items[i].Name) {
t.Fatalf("WorkersToDelete is not actually deleted, %s", podList.Items[i].Name)
}
}
}
func TestReconcile_PodDeleted_Diff0_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
podList := corev1.PodList{}
err := fakeClient.List(context.Background(), &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, len(testPods), len(podList.Items), "Init pod list len is wrong")
// Simulate 2 pod container got deleted.
err = fakeClient.Delete(context.Background(), &podList.Items[3])
assert.Nil(t, err, "Fail to delete pod")
err = fakeClient.Delete(context.Background(), &podList.Items[4])
assert.Nil(t, err, "Fail to delete pod")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcilePods(testRayCluster)
assert.Nil(t, err, "Fail to reconcile Pods")
err = fakeClient.List(context.Background(), &podList, &client.ListOptions{
LabelSelector: workerSelector,
Namespace: namespaceStr,
})
assert.Nil(t, err, "Fail to get pod list after reconcile")
assert.Equal(t, int(expectReplicaNum), len(podList.Items),
"Replica number is wrong after reconcile expect %d actual %d", expectReplicaNum, len(podList.Items))
for i := 0; i < len(podList.Items); i++ {
if contains(workersToDelete, podList.Items[i].Name) {
t.Errorf("WorkersToDelete is not actually deleted, %s", podList.Items[i].Name)
}
}
}
func TestReconcile_PodDeleted_DiffLess0_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
podList := corev1.PodList{}
err := fakeClient.List(context.Background(), &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, len(testPods), len(podList.Items), "Init pod list len is wrong")
// Simulate 1 pod container got deleted.
err = fakeClient.Delete(context.Background(), &podList.Items[3])
assert.Nil(t, err, "Fail to delete pod")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcilePods(testRayCluster)
assert.Nil(t, err, "Fail to reconcile Pods")
err = fakeClient.List(context.Background(), &podList, &client.ListOptions{
LabelSelector: workerSelector,
Namespace: namespaceStr,
})
assert.Nil(t, err, "Fail to get pod list after reconcile")
assert.Equal(t, int(expectReplicaNum), len(podList.Items),
"Replica number is wrong after reconcile expect %d actual %d", expectReplicaNum, len(podList.Items))
for i := 0; i < len(podList.Items); i++ {
if contains(workersToDelete, podList.Items[i].Name) {
t.Errorf("WorkersToDelete is not actually deleted, %s", podList.Items[i].Name)
}
}
}
func TestReconcile_PodDCrash_Diff0_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
podList := corev1.PodList{}
err := fakeClient.List(context.Background(), &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, len(testPods), len(podList.Items), "Init pod list len is wrong")
// Simulate 2 pod container crash.
podList.Items[3].Status.Phase = corev1.PodFailed
podList.Items[4].Status.Phase = corev1.PodFailed
err = fakeClient.Update(context.Background(), &podList.Items[3])
assert.Nil(t, err, "Fail to get update pod status")
err = fakeClient.Update(context.Background(), &podList.Items[4])
assert.Nil(t, err, "Fail to get update pod status")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcilePods(testRayCluster)
assert.Nil(t, err, "Fail to reconcile Pods")
err = fakeClient.List(context.Background(), &podList, &client.ListOptions{
LabelSelector: workerSelector,
Namespace: namespaceStr,
})
assert.Nil(t, err, "Fail to get pod list after reconcile")
assert.Equal(t, int(expectReplicaNum), getNotFailedPodItemNum(podList),
"Replica number is wrong after reconcile expect %d actual %d", expectReplicaNum, getNotFailedPodItemNum(podList))
for i := 0; i < len(podList.Items); i++ {
if contains(workersToDelete, podList.Items[i].Name) {
t.Errorf("WorkersToDelete is not actually deleted, %s", podList.Items[i].Name)
}
}
}
func TestReconcile_PodDCrash_DiffLess0_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
podList := corev1.PodList{}
err := fakeClient.List(context.Background(), &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, len(testPods), len(podList.Items), "Init pod list len is wrong")
// Simulate 1 pod container crash.
podList.Items[3].Status.Phase = corev1.PodFailed
err = fakeClient.Update(context.Background(), &podList.Items[3])
assert.Nil(t, err, "Fail to get update pod status")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcilePods(testRayCluster)
assert.Nil(t, err, "Fail to reconcile Pods")
err = fakeClient.List(context.Background(), &podList, &client.ListOptions{
LabelSelector: workerSelector,
Namespace: namespaceStr,
})
assert.Nil(t, err, "Fail to get pod list after reconcile")
assert.Equal(t, int(expectReplicaNum), getNotFailedPodItemNum(podList),
"Replica number is wrong after reconcile expect %d actual %d", expectReplicaNum, getNotFailedPodItemNum(podList))
for i := 0; i < len(podList.Items); i++ {
if contains(workersToDelete, podList.Items[i].Name) {
t.Errorf("WorkersToDelete is not actually deleted, %s", podList.Items[i].Name)
}
}
}
func TestReconcile_PodEvicted_DiffLess0_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
podList := corev1.PodList{}
err := fakeClient.List(context.Background(), &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, len(testPods), len(podList.Items), "Init pod list len is wrong")
// Simulate head pod get evicted.
podList.Items[0].Status.Phase = corev1.PodFailed
podList.Items[0].Status.Reason = "Evicted"
err = fakeClient.Update(context.Background(), &podList.Items[0])
assert.Nil(t, err, "Fail to get update pod status")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcilePods(testRayCluster)
assert.Nil(t, err, "Fail to reconcile Pods")
// Filter head pod
err = fakeClient.List(context.Background(), &podList, &client.ListOptions{
LabelSelector: headSelector,
Namespace: namespaceStr,
})
assert.Nil(t, err, "Fail to get pod list after reconcile")
assert.Equal(t, 0, len(podList.Items),
"Evicted head should be deleted after reconcile expect %d actual %d", 0, len(podList.Items))
}
func TestReconcile_UpdateLocalWorkersToDelete_OK(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
podList := corev1.PodList{}
err := fakeClient.List(context.Background(), &podList, client.InNamespace(namespaceStr))
assert.Nil(t, err, "Fail to get pod list")
assert.Equal(t, len(testPods), len(podList.Items), "Init pod list len is wrong")
// Simulate 1 pod container crash.
podList.Items[1].Status.Phase = corev1.PodFailed
runningPodsItems := append(podList.Items[:1], podList.Items[2:]...)
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
testWorker := testRayCluster.Spec.WorkerGroupSpecs[0]
// this should remove `pod1` from testWorker.ScaleStrategy.WorkersToDelete
testRayClusterReconciler.updateLocalWorkersToDelete(&testWorker, runningPodsItems)
assert.Len(t, testWorker.ScaleStrategy.WorkersToDelete, 1, "updateLocalWorkersToDelete does not update WorkersToDelete properly")
assert.Equal(t, testWorker.ScaleStrategy.WorkersToDelete[0], "pod2")
}
func contains(slice []string, item string) bool {
set := make(map[string]struct{}, len(slice))
for _, s := range slice {
set[s] = struct{}{}
}
_, ok := set[item]
return ok
}
func getNotFailedPodItemNum(podList corev1.PodList) int {
count := 0
for _, aPod := range podList.Items {
if aPod.Status.Phase != corev1.PodFailed {
count++
}
}
return count
}
func TestReconcile_AutoscalerServiceAccount(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
saNamespacedName := types.NamespacedName{
Name: headGroupServiceAccount,
Namespace: namespaceStr,
}
sa := corev1.ServiceAccount{}
err := fakeClient.Get(context.Background(), saNamespacedName, &sa)
assert.True(t, k8serrors.IsNotFound(err), "Head group service account should not exist yet")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcileAutoscalerServiceAccount(testRayCluster)
assert.Nil(t, err, "Fail to reconcile autoscaler ServiceAccount")
err = fakeClient.Get(context.Background(), saNamespacedName, &sa)
assert.Nil(t, err, "Fail to get head group ServiceAccount after reconciliation")
}
func TestReconcile_AutoscalerRoleBinding(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testPods...).Build()
rbNamespacedName := types.NamespacedName{
Name: instanceName,
Namespace: namespaceStr,
}
rb := rbacv1.RoleBinding{}
err := fakeClient.Get(context.Background(), rbNamespacedName, &rb)
assert.True(t, k8serrors.IsNotFound(err), "autoscaler RoleBinding should not exist yet")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
err = testRayClusterReconciler.reconcileAutoscalerRoleBinding(testRayCluster)
assert.Nil(t, err, "Fail to reconcile autoscaler RoleBinding")
err = fakeClient.Get(context.Background(), rbNamespacedName, &rb)
assert.Nil(t, err, "Fail to get autoscaler RoleBinding after reconciliation")
}
func TestReconcile_UpdateClusterReason(t *testing.T) {
setupTest(t)
defer tearDown(t)
newScheme := runtime.NewScheme()
_ = rayiov1alpha1.AddToScheme(newScheme)
fakeClient := clientFake.NewClientBuilder().WithScheme(newScheme).WithRuntimeObjects(testRayCluster).Build()
namespacedName := types.NamespacedName{
Name: instanceName,
Namespace: namespaceStr,
}
cluster := rayiov1alpha1.RayCluster{}
err := fakeClient.Get(context.Background(), namespacedName, &cluster)
assert.Nil(t, err, "Fail to get RayCluster")
assert.Empty(t, cluster.Status.Reason, "Cluster reason should be empty")
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
reason := "test reason"
err = testRayClusterReconciler.updateClusterReason(testRayCluster, reason)
assert.Nil(t, err, "Fail to update cluster reason")
err = fakeClient.Get(context.Background(), namespacedName, &cluster)
assert.Nil(t, err, "Fail to get RayCluster after updating reason")
assert.Equal(t, cluster.Status.Reason, reason, "Cluster reason should be updated")
}
func TestUpdateEndpoints(t *testing.T) {
setupTest(t)
defer tearDown(t)
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(testServices...).Build()
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
if err := testRayClusterReconciler.updateEndpoints(testRayCluster); err != nil {
t.Errorf("updateEndpoints failed: %v", err)
}
expected := map[string]string{
"client": "10001",
"dashboard": "8265",
"metrics": "8080",
"redis": "6379",
"serve": "8000",
}
assert.Equal(t, expected, testRayCluster.Status.Endpoints, "RayCluster status endpoints not updated")
}
func TestGetHeadPodIP(t *testing.T) {
setupTest(t)
defer tearDown(t)
extraHeadPod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "unexpectedExtraHeadNode",
Namespace: namespaceStr,
Labels: map[string]string{
common.RayClusterLabelKey: instanceName,
common.RayNodeTypeLabelKey: string(rayiov1alpha1.HeadNode),
common.RayNodeGroupLabelKey: headGroupNameStr,
},
},
}
tests := map[string]struct {
pods []runtime.Object
expectedIP string
returnsError bool
}{
"get expected Pod IP if there's one head node": {
pods: testPods,
expectedIP: headNodeIP,
returnsError: false,
},
"no error if there's no head node": {
pods: []runtime.Object{},
expectedIP: "",
returnsError: false,
},
"no error if there's more than one head node": {
pods: append(testPods, extraHeadPod),
expectedIP: "",
returnsError: false,
},
"no error if head pod ip is not yet set": {
pods: testPodsNoHeadIP,
expectedIP: "",
returnsError: false,
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(tc.pods...).Build()
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
ip, err := testRayClusterReconciler.getHeadPodIP(testRayCluster)
if tc.returnsError {
assert.NotNil(t, err, "getHeadPodIP should return error")
} else {
assert.Nil(t, err, "getHeadPodIP should not return error")
}
assert.Equal(t, tc.expectedIP, ip, "getHeadPodIP returned unexpected IP")
})
}
}
func TestGetHeadServiceIP(t *testing.T) {
setupTest(t)
defer tearDown(t)
headServiceIP := "1.2.3.4"
headService, err := common.BuildServiceForHeadPod(*testRayCluster, nil, nil)
if err != nil {
t.Errorf("failed to build head service: %v", err)
}
headService.Spec.ClusterIP = headServiceIP
testServices = []runtime.Object{
headService,
}
extraHeadService := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "unexpectedExtraHeadService",
Namespace: namespaceStr,
Labels: common.HeadServiceLabels(*testRayCluster),
},
}
tests := map[string]struct {
services []runtime.Object
expectedIP string
returnsError bool
}{
"get expected Service IP if there's one head Service": {
services: testServices,
expectedIP: headServiceIP,
returnsError: false,
},
"get error if there's no head Service": {
services: []runtime.Object{},
expectedIP: "",
returnsError: true,
},
"get error if there's more than one head Service": {
services: append(testServices, extraHeadService),
expectedIP: "",
returnsError: true,
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
fakeClient := clientFake.NewClientBuilder().WithRuntimeObjects(tc.services...).Build()
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
ip, err := testRayClusterReconciler.getHeadServiceIP(testRayCluster)
if tc.returnsError {
assert.NotNil(t, err, "getHeadServiceIP should return error")
} else {
assert.Nil(t, err, "getHeadServiceIP should not return error")
}
assert.Equal(t, tc.expectedIP, ip, "getHeadServiceIP returned unexpected IP")
})
}
}
func TestUpdateStatusObservedGeneration(t *testing.T) {
setupTest(t)
defer tearDown(t)
// Create a new scheme with CRDs, Pod, Service schemes.
newScheme := runtime.NewScheme()
_ = rayiov1alpha1.AddToScheme(newScheme)
_ = corev1.AddToScheme(newScheme)
// To update the status of RayCluster with `r.Status().Update()`,
// initialize the runtimeObjects with appropriate context. In KubeRay, the `ClusterIP`
// and `TargetPort` fields are typically set by the cluster's control plane.
headService, err := common.BuildServiceForHeadPod(*testRayCluster, nil, nil)
assert.Nil(t, err, "Failed to build head service.")
headService.Spec.ClusterIP = headNodeIP
for i, port := range headService.Spec.Ports {
headService.Spec.Ports[i].TargetPort = intstr.IntOrString{IntVal: port.Port}
}
runtimeObjects := append(testPods, headService, testRayCluster)
// To facilitate testing, we set an impossible value for ObservedGeneration.
// Note that ObjectMeta's `Generation` and `ResourceVersion` don't behave properly in the fake client.
// [Ref] https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/client/fake
testRayCluster.Status.ObservedGeneration = -1
// Initialize a fake client with newScheme and runtimeObjects.
fakeClient := clientFake.NewClientBuilder().WithScheme(newScheme).WithRuntimeObjects(runtimeObjects...).Build()
// Verify the initial values of `Generation` and `ObservedGeneration`.
namespacedName := types.NamespacedName{
Name: instanceName,
Namespace: namespaceStr,
}
cluster := rayiov1alpha1.RayCluster{}
err = fakeClient.Get(context.Background(), namespacedName, &cluster)
assert.Nil(t, err, "Fail to get RayCluster")
assert.Equal(t, int64(-1), cluster.Status.ObservedGeneration)
assert.Equal(t, int64(0), cluster.ObjectMeta.Generation)
// Initialize RayCluster reconciler.
testRayClusterReconciler := &RayClusterReconciler{
Client: fakeClient,
Recorder: &record.FakeRecorder{},
Scheme: scheme.Scheme,
Log: ctrl.Log.WithName("controllers").WithName("RayCluster"),
}
// Compare the values of `Generation` and `ObservedGeneration` to check if they match.
err = testRayClusterReconciler.updateStatus(testRayCluster)
assert.Nil(t, err)
err = fakeClient.Get(context.Background(), namespacedName, &cluster)
assert.Nil(t, err)
assert.Equal(t, cluster.ObjectMeta.Generation, cluster.Status.ObservedGeneration)
}