Files
kuberay/ray-operator/controllers/raycluster_controller_test.go
Jiaxin ShanandGitHub 61874223e0 Fix flaky tests by retrying 409 conflict error (#73)
Failure is because of error "Operation cannot be fulfilled on rayclusters.ray.io: the object has been modified; please apply your changes to the latest version and try again". This is pretty common and because test code may not use latest revision to update the object since operator updates the object in the middle. Retry on this kind of issues will work.
2021-10-16 12:49:21 -07:00

284 lines
9.5 KiB
Go

/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controllers
import (
"context"
"fmt"
"reflect"
"time"
rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/api/v1alpha1"
"github.com/ray-project/kuberay/ray-operator/controllers/common"
"github.com/ray-project/kuberay/ray-operator/controllers/utils"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/utils/pointer"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
// +kubebuilder:scaffold:imports
)
const (
DefaultAttempts = 4
DefaultSleepDurationInSeconds = 2
)
var _ = Context("Inside the default namespace", func() {
ctx := context.TODO()
var workerPods corev1.PodList
var myRayCluster = &rayiov1alpha1.RayCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "raycluster-sample",
Namespace: "default",
},
Spec: rayiov1alpha1.RayClusterSpec{
RayVersion: "1.0",
HeadGroupSpec: rayiov1alpha1.HeadGroupSpec{
ServiceType: "ClusterIP",
Replicas: pointer.Int32Ptr(1),
RayStartParams: map[string]string{
"port": "6379",
"object-manager-port": "12345",
"node-manager-port": "12346",
"object-store-memory": "100000000",
"redis-password": "LetMeInRay",
"num-cpus": "1",
},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
corev1.Container{
Name: "ray-head",
Image: "rayproject/autoscaler",
Command: []string{"python"},
Args: []string{"/opt/code.py"},
Env: []corev1.EnvVar{
corev1.EnvVar{
Name: "MY_POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
},
},
},
},
},
},
},
WorkerGroupSpecs: []rayiov1alpha1.WorkerGroupSpec{
rayiov1alpha1.WorkerGroupSpec{
Replicas: pointer.Int32Ptr(3),
MinReplicas: pointer.Int32Ptr(0),
MaxReplicas: pointer.Int32Ptr(10000),
GroupName: "small-group",
RayStartParams: map[string]string{
"port": "6379",
"redis-password": "LetMeInRay",
"num-cpus": "1",
},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
corev1.Container{
Name: "ray-worker",
Image: "rayproject/autoscaler",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
Env: []corev1.EnvVar{
corev1.EnvVar{
Name: "MY_POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
},
},
},
},
},
},
},
},
},
}
var filterLabels = client.MatchingLabels{common.RayClusterLabelKey: myRayCluster.Name, common.RayNodeGroupLabelKey: "small-group"}
Describe("When creating a raycluster", func() {
It("should create a raycluster object", func() {
err := k8sClient.Create(ctx, myRayCluster)
Expect(err).NotTo(HaveOccurred(), "failed to create test RayCluster resource")
})
It("should see a raycluster object", func() {
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: myRayCluster.Name, Namespace: "default"}, myRayCluster),
time.Second*3, time.Millisecond*500).Should(BeNil(), "My myRayCluster = %v", myRayCluster.Name)
})
It("should create a new head service resource", func() {
svc := &corev1.Service{}
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: "raycluster-sample-head-svc", Namespace: "default"}, svc),
time.Second*15, time.Millisecond*500).Should(BeNil(), "My head service = %v", svc)
Expect(svc.Spec.Selector[common.RayIDLabelKey]).Should(Equal(utils.GenerateIdentifier(myRayCluster.Name, rayiov1alpha1.HeadNode)))
})
It("should create more than 1 worker", func() {
Eventually(
listResourceFunc(ctx, &workerPods, filterLabels, &client.ListOptions{Namespace: "default"}),
time.Second*15, time.Millisecond*500).Should(Equal(3), fmt.Sprintf("workerGroup %v", workerPods.Items))
if len(workerPods.Items) > 0 {
Expect(workerPods.Items[0].Status.Phase).Should(Or(Equal(v1.PodRunning), Equal(v1.PodPending)))
}
})
It("should create a head pod resource", func() {
var headPods corev1.PodList
filterLabels := client.MatchingLabels{common.RayClusterLabelKey: myRayCluster.Name, common.RayNodeGroupLabelKey: "headgroup"}
k8sClient.List(ctx, &headPods, filterLabels, &client.ListOptions{Namespace: "default"}, client.InNamespace(myRayCluster.Namespace))
Expect(len(headPods.Items)).Should(BeNumerically("==", 1), "My head pod list= %v", headPods.Items)
pod := &corev1.Pod{}
if len(headPods.Items) > 0 {
pod = &headPods.Items[0]
}
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: pod.Name, Namespace: "default"}, pod),
time.Second*3, time.Millisecond*500).Should(BeNil(), "My head pod = %v", pod)
Expect(pod.Status.Phase).Should(Or(Equal(v1.PodPending), Equal(v1.PodRunning)))
})
It("should re-create a deleted worker", func() {
Eventually(
listResourceFunc(ctx, &workerPods, filterLabels, &client.ListOptions{Namespace: "default"}),
time.Second*15, time.Millisecond*500).Should(Equal(3), fmt.Sprintf("workerGroup %v", workerPods.Items))
pod := workerPods.Items[0]
err := k8sClient.Delete(ctx, &pod,
&client.DeleteOptions{GracePeriodSeconds: pointer.Int64Ptr(0)})
Expect(err).NotTo(HaveOccurred(), "failed delete a pod")
//at least 3 pods should be in none-failed phase
Eventually(
listResourceFunc(ctx, &workerPods, filterLabels, &client.ListOptions{Namespace: "default"}),
time.Second*15, time.Millisecond*500).Should(Equal(3), fmt.Sprintf("workerGroup %v", workerPods.Items))
})
It("should update a raycluster object deleting a random pod", func() {
// adding a scale down
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: myRayCluster.Name, Namespace: "default"}, myRayCluster),
time.Second*3, time.Millisecond*500).Should(BeNil(), "My raycluster = %v", myRayCluster)
rep := new(int32)
*rep = 2
myRayCluster.Spec.WorkerGroupSpecs[0].Replicas = rep
// Operator may update revision after we get cluster earlier. Update may result in 409 conflict error.
// We need to handle conflict error and retry the update.
err := retryOnOldRevision(DefaultAttempts, DefaultSleepDurationInSeconds, func() error {
return k8sClient.Update(ctx, myRayCluster)
})
Expect(err).NotTo(HaveOccurred(), "failed to update test RayCluster resource")
})
It("should have only 2 running worker", func() {
// retry listing pods, given that last update may not immediately happen.
Eventually(
listResourceFunc(ctx, &workerPods, filterLabels, &client.ListOptions{Namespace: "default"}),
time.Second*15, time.Millisecond*500).Should(Equal(2), fmt.Sprintf("workerGroup %v", workerPods.Items))
})
It("should update a raycluster object", func() {
// adding a scale strategy
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: myRayCluster.Name, Namespace: "default"}, myRayCluster),
time.Second*3, time.Millisecond*500).Should(BeNil(), "My raycluster = %v", myRayCluster)
podToDelete1 := workerPods.Items[0]
rep := new(int32)
*rep = 1
myRayCluster.Spec.WorkerGroupSpecs[0].Replicas = rep
myRayCluster.Spec.WorkerGroupSpecs[0].ScaleStrategy.WorkersToDelete = []string{podToDelete1.Name}
Expect(k8sClient.Update(ctx, myRayCluster)).Should(Succeed(), "failed to update test RayCluster resource")
})
It("should have only 1 running worker", func() {
// retry listing pods, given that last update may not immediately happen.
Eventually(
listResourceFunc(ctx, &workerPods, filterLabels, &client.ListOptions{Namespace: "default"}),
time.Second*15, time.Millisecond*500).Should(Equal(1), fmt.Sprintf("workerGroup %v", workerPods.Items))
})
})
})
func getResourceFunc(ctx context.Context, key client.ObjectKey, obj client.Object) func() error {
return func() error {
return k8sClient.Get(ctx, key, obj)
}
}
func listResourceFunc(ctx context.Context, workerPods *corev1.PodList, opt ...client.ListOption) func() (int, error) {
return func() (int, error) {
if err := k8sClient.List(ctx, workerPods, opt...); err != nil {
return -1, err
}
count := 0
for _, aPod := range workerPods.Items {
if (reflect.DeepEqual(aPod.Status.Phase, v1.PodRunning) || reflect.DeepEqual(aPod.Status.Phase, v1.PodPending)) && aPod.DeletionTimestamp == nil {
count++
}
}
return count, nil
}
}
func retryOnOldRevision(attempts int, sleep time.Duration, f func() error) error {
var err error
for i := 0; i < attempts; i++ {
if i > 0 {
fmt.Printf("retrying after error: %v", err)
time.Sleep(sleep)
sleep *= 2
}
err = f()
if err == nil {
return nil
}
if !errors.IsConflict(err) {
return nil
}
}
return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
}