mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
338 lines
12 KiB
Go
338 lines
12 KiB
Go
/*
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package ray
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"reflect"
|
|
"time"
|
|
|
|
"github.com/ray-project/kuberay/ray-operator/controllers/ray/common"
|
|
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
|
|
|
|
. "github.com/onsi/ginkgo"
|
|
. "github.com/onsi/gomega"
|
|
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
rbacv1 "k8s.io/api/rbac/v1"
|
|
"k8s.io/utils/pointer"
|
|
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/client-go/util/retry"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
// +kubebuilder:scaffold:imports
|
|
)
|
|
|
|
var _ = Context("Inside the default namespace", func() {
|
|
ctx := context.TODO()
|
|
var workerPods corev1.PodList
|
|
var headPods corev1.PodList
|
|
enableInTreeAutoscaling := true
|
|
|
|
myRayCluster := &rayv1.RayCluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "raycluster-sample",
|
|
Namespace: "default",
|
|
},
|
|
Spec: rayv1.RayClusterSpec{
|
|
EnableInTreeAutoscaling: &enableInTreeAutoscaling,
|
|
HeadGroupSpec: rayv1.HeadGroupSpec{
|
|
RayStartParams: map[string]string{
|
|
"port": "6379",
|
|
"object-manager-port": "12345",
|
|
"node-manager-port": "12346",
|
|
"num-cpus": "1",
|
|
},
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "ray-head",
|
|
Image: "rayproject/ray:2.7.0",
|
|
Command: []string{"python"},
|
|
Args: []string{"/opt/code.py"},
|
|
Env: []corev1.EnvVar{
|
|
{
|
|
Name: "MY_POD_IP",
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
FieldRef: &corev1.ObjectFieldSelector{
|
|
FieldPath: "status.podIP",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
WorkerGroupSpecs: []rayv1.WorkerGroupSpec{
|
|
{
|
|
Replicas: pointer.Int32(3),
|
|
MinReplicas: pointer.Int32(0),
|
|
MaxReplicas: pointer.Int32(4),
|
|
GroupName: "small-group",
|
|
RayStartParams: map[string]string{
|
|
"port": "6379",
|
|
"num-cpus": "1",
|
|
},
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "ray-worker",
|
|
Image: "rayproject/ray:2.7.0",
|
|
Command: []string{"echo"},
|
|
Args: []string{"Hello Ray"},
|
|
Env: []corev1.EnvVar{
|
|
{
|
|
Name: "MY_POD_IP",
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
FieldRef: &corev1.ObjectFieldSelector{
|
|
FieldPath: "status.podIP",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
headFilterLabels := client.MatchingLabels{common.RayClusterLabelKey: myRayCluster.Name, common.RayNodeGroupLabelKey: "headgroup"}
|
|
workerFilterLabels := client.MatchingLabels{common.RayClusterLabelKey: myRayCluster.Name, common.RayNodeGroupLabelKey: "small-group"}
|
|
|
|
Describe("When creating a raycluster", func() {
|
|
It("should create a raycluster object", func() {
|
|
err := k8sClient.Create(ctx, myRayCluster)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to create test RayCluster resource")
|
|
})
|
|
|
|
It("should see a raycluster object", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: myRayCluster.Name, Namespace: "default"}, myRayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(BeNil(), "My myRayCluster = %v", myRayCluster.Name)
|
|
})
|
|
|
|
It("should create a new head service resource", func() {
|
|
svc := &corev1.Service{}
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: "raycluster-sample-head-svc", Namespace: "default"}, svc),
|
|
time.Second*15, time.Millisecond*500).Should(BeNil(), "My head service = %v", svc)
|
|
Expect(svc.Spec.Selector[common.RayIDLabelKey]).Should(Equal(utils.GenerateIdentifier(myRayCluster.Name, rayv1.HeadNode)))
|
|
})
|
|
|
|
It("should create 3 workers", func() {
|
|
Eventually(
|
|
listResourceFunc(ctx, &workerPods, workerFilterLabels, &client.ListOptions{Namespace: "default"}),
|
|
time.Second*15, time.Millisecond*500).Should(Equal(3), fmt.Sprintf("workerGroup %v", workerPods.Items))
|
|
if len(workerPods.Items) > 0 {
|
|
Expect(workerPods.Items[0].Status.Phase).Should(Or(Equal(corev1.PodRunning), Equal(corev1.PodPending)))
|
|
}
|
|
})
|
|
|
|
It("should create a head pod resource", func() {
|
|
err := k8sClient.List(ctx, &headPods, headFilterLabels, &client.ListOptions{Namespace: "default"}, client.InNamespace(myRayCluster.Namespace))
|
|
Expect(err).NotTo(HaveOccurred(), "failed list head pods")
|
|
Expect(len(headPods.Items)).Should(BeNumerically("==", 1), "My head pod list= %v", headPods.Items)
|
|
|
|
pod := &corev1.Pod{}
|
|
if len(headPods.Items) > 0 {
|
|
pod = &headPods.Items[0]
|
|
}
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: pod.Name, Namespace: "default"}, pod),
|
|
time.Second*3, time.Millisecond*500).Should(BeNil(), "My head pod = %v", pod)
|
|
Expect(pod.Status.Phase).Should(Or(Equal(corev1.PodPending), Equal(corev1.PodRunning)))
|
|
})
|
|
|
|
It("should create the autoscaler K8s RoleBinding if it doesn't exist", func() {
|
|
rbName := myRayCluster.Name
|
|
rb := &rbacv1.RoleBinding{}
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rbName, Namespace: myRayCluster.Namespace}, rb),
|
|
time.Second*15, time.Millisecond*500).Should(BeNil(), "autoscaler RoleBinding = %v", rbName)
|
|
})
|
|
|
|
It("should be able to update all Pods to Running", func() {
|
|
// We need to manually update Pod statuses otherwise they'll always be Pending.
|
|
// envtest doesn't create a full K8s cluster. It's only the control plane.
|
|
// There's no container runtime or any other K8s controllers.
|
|
// So Pods are created, but no controller updates them from Pending to Running.
|
|
// See https://book.kubebuilder.io/reference/envtest.html
|
|
|
|
for _, headPod := range headPods.Items {
|
|
headPod.Status.Phase = corev1.PodRunning
|
|
Expect(k8sClient.Status().Update(ctx, &headPod)).Should(BeNil())
|
|
}
|
|
|
|
Eventually(
|
|
isAllPodsRunning(ctx, headPods, headFilterLabels, "default"),
|
|
time.Second*15, time.Millisecond*500).Should(Equal(true), "Head Pod should be running.")
|
|
|
|
for _, workerPod := range workerPods.Items {
|
|
workerPod.Status.Phase = corev1.PodRunning
|
|
Expect(k8sClient.Status().Update(ctx, &workerPod)).Should(BeNil())
|
|
}
|
|
|
|
Eventually(
|
|
isAllPodsRunning(ctx, workerPods, workerFilterLabels, "default"),
|
|
time.Second*15, time.Millisecond*500).Should(Equal(true), "All worker Pods should be running.")
|
|
})
|
|
|
|
It("cluster's .status.state should be updated to 'ready' shortly after all Pods are Running", func() {
|
|
Eventually(
|
|
getClusterState(ctx, "default", myRayCluster.Name),
|
|
time.Second*(common.RAYCLUSTER_DEFAULT_REQUEUE_SECONDS+5), time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
It("should re-create a deleted worker", func() {
|
|
Eventually(
|
|
listResourceFunc(ctx, &workerPods, workerFilterLabels, &client.ListOptions{Namespace: "default"}),
|
|
time.Second*15, time.Millisecond*500).Should(Equal(3), fmt.Sprintf("workerGroup %v", workerPods.Items))
|
|
|
|
pod := workerPods.Items[0]
|
|
err := k8sClient.Delete(ctx, &pod,
|
|
&client.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)})
|
|
|
|
Expect(err).NotTo(HaveOccurred(), "failed delete a pod")
|
|
|
|
// at least 3 pods should be in none-failed phase
|
|
Eventually(
|
|
listResourceFunc(ctx, &workerPods, workerFilterLabels, &client.ListOptions{Namespace: "default"}),
|
|
time.Second*15, time.Millisecond*500).Should(Equal(3), fmt.Sprintf("workerGroup %v", workerPods.Items))
|
|
})
|
|
|
|
It("should update a raycluster object deleting a random pod", func() {
|
|
// adding a scale down
|
|
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: myRayCluster.Name, Namespace: "default"}, myRayCluster),
|
|
time.Second*9, time.Millisecond*500).Should(BeNil(), "My raycluster = %v", myRayCluster)
|
|
myRayCluster.Spec.WorkerGroupSpecs[0].Replicas = pointer.Int32(2)
|
|
|
|
// Operator may update revision after we get cluster earlier. Update may result in 409 conflict error.
|
|
// We need to handle conflict error and retry the update.
|
|
return k8sClient.Update(ctx, myRayCluster)
|
|
})
|
|
|
|
Expect(err).NotTo(HaveOccurred(), "failed to update test RayCluster resource")
|
|
})
|
|
|
|
It("should update a raycluster object", func() {
|
|
// adding a scale strategy
|
|
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: myRayCluster.Name, Namespace: "default"}, myRayCluster),
|
|
time.Second*9, time.Millisecond*500).Should(BeNil(), "My raycluster = %v", myRayCluster)
|
|
podToDelete := workerPods.Items[0]
|
|
myRayCluster.Spec.WorkerGroupSpecs[0].Replicas = pointer.Int32(2)
|
|
myRayCluster.Spec.WorkerGroupSpecs[0].ScaleStrategy.WorkersToDelete = []string{podToDelete.Name}
|
|
return k8sClient.Update(ctx, myRayCluster)
|
|
})
|
|
Expect(err).NotTo(HaveOccurred(), "failed to update test RayCluster resource")
|
|
})
|
|
|
|
It("should have only 2 running workers", func() {
|
|
// retry listing pods, given that last update may not immediately happen.
|
|
Eventually(
|
|
listResourceFunc(ctx, &workerPods, workerFilterLabels, &client.ListOptions{Namespace: "default"}),
|
|
time.Second*15, time.Millisecond*500).Should(Equal(2), fmt.Sprintf("workerGroup %v", workerPods.Items))
|
|
})
|
|
|
|
It("should increase replicas past maxReplicas", func() {
|
|
// increasing replicas to 5, which is greater than maxReplicas (4)
|
|
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: myRayCluster.Name, Namespace: "default"}, myRayCluster),
|
|
time.Second*9, time.Millisecond*500).Should(BeNil(), "My raycluster = %v", myRayCluster)
|
|
myRayCluster.Spec.WorkerGroupSpecs[0].Replicas = pointer.Int32(5)
|
|
|
|
// Operator may update revision after we get cluster earlier. Update may result in 409 conflict error.
|
|
// We need to handle conflict error and retry the update.
|
|
return k8sClient.Update(ctx, myRayCluster)
|
|
})
|
|
|
|
Expect(err).NotTo(HaveOccurred(), "failed to update test RayCluster resource")
|
|
})
|
|
|
|
It("should scale to maxReplicas (4) workers", func() {
|
|
// retry listing pods, given that last update may not immediately happen.
|
|
Eventually(
|
|
listResourceFunc(ctx, &workerPods, workerFilterLabels, &client.ListOptions{Namespace: "default"}),
|
|
time.Second*15, time.Millisecond*500).Should(Equal(4), fmt.Sprintf("workerGroup %v", workerPods.Items))
|
|
})
|
|
|
|
It("should countinue to have only maxReplicas (4) workers", func() {
|
|
// check that pod count stays at 4 for two seconds.
|
|
Consistently(
|
|
listResourceFunc(ctx, &workerPods, workerFilterLabels, &client.ListOptions{Namespace: "default"}),
|
|
time.Second*2, time.Millisecond*200).Should(Equal(4), fmt.Sprintf("workerGroup %v", workerPods.Items))
|
|
})
|
|
})
|
|
})
|
|
|
|
func getResourceFunc(ctx context.Context, key client.ObjectKey, obj client.Object) func() error {
|
|
return func() error {
|
|
return k8sClient.Get(ctx, key, obj)
|
|
}
|
|
}
|
|
|
|
func listResourceFunc(ctx context.Context, workerPods *corev1.PodList, opt ...client.ListOption) func() (int, error) {
|
|
return func() (int, error) {
|
|
if err := k8sClient.List(ctx, workerPods, opt...); err != nil {
|
|
return -1, err
|
|
}
|
|
|
|
count := 0
|
|
for _, aPod := range workerPods.Items {
|
|
if (reflect.DeepEqual(aPod.Status.Phase, corev1.PodRunning) || reflect.DeepEqual(aPod.Status.Phase, corev1.PodPending)) && aPod.DeletionTimestamp == nil {
|
|
count++
|
|
}
|
|
}
|
|
|
|
return count, nil
|
|
}
|
|
}
|
|
|
|
func getClusterState(ctx context.Context, namespace string, clusterName string) func() rayv1.ClusterState {
|
|
return func() rayv1.ClusterState {
|
|
var cluster rayv1.RayCluster
|
|
if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: clusterName}, &cluster); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
return cluster.Status.State
|
|
}
|
|
}
|
|
|
|
func isAllPodsRunning(ctx context.Context, podlist corev1.PodList, filterLabels client.MatchingLabels, namespace string) bool {
|
|
err := k8sClient.List(ctx, &podlist, filterLabels, &client.ListOptions{Namespace: namespace})
|
|
Expect(err).ShouldNot(HaveOccurred(), "failed to list Pods")
|
|
for _, pod := range podlist.Items {
|
|
if pod.Status.Phase != corev1.PodRunning {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|