Files
kuberay/ray-operator/controllers/ray/rayservice_controller_test.go
brucez-anyscaleandGitHub 781eed1e6b Fix a potential reconcile issue for RayService and allow config unhealth time threshold in CR (#384)
* Improve

* update

* Improve unhealthy to read spec

* fix go mod

* fix lint

* fix lint
2022-07-15 19:03:05 -07:00

419 lines
14 KiB
Go

/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ray
import (
"context"
"fmt"
"time"
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
"k8s.io/apimachinery/pkg/api/resource"
"github.com/ray-project/kuberay/ray-operator/controllers/ray/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1"
corev1 "k8s.io/api/core/v1"
"k8s.io/utils/pointer"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
// +kubebuilder:scaffold:imports
)
var runtimeEnvStr = "working_dir:\n - \"https://github.com/ray-project/test_dag/archive/c620251044717ace0a4c19d766d43c5099af8a77.zip\""
var _ = Context("Inside the default namespace", func() {
ctx := context.TODO()
var workerPods corev1.PodList
var numReplicas int32
var numCpus float64
numReplicas = 1
numCpus = 0.1
myRayService := &rayiov1alpha1.RayService{
ObjectMeta: metav1.ObjectMeta{
Name: "rayservice-sample",
Namespace: "default",
},
Spec: rayiov1alpha1.RayServiceSpec{
ServeDeploymentGraphSpec: rayiov1alpha1.ServeDeploymentGraphSpec{
ImportPath: "fruit.deployment_graph",
RuntimeEnv: runtimeEnvStr,
ServeConfigSpecs: []rayiov1alpha1.ServeConfigSpec{
{
Name: "MangoStand",
NumReplicas: &numReplicas,
UserConfig: "price: 3",
RayActorOptions: rayiov1alpha1.RayActorOptionSpec{
NumCpus: &numCpus,
},
},
{
Name: "OrangeStand",
NumReplicas: &numReplicas,
UserConfig: "price: 2",
RayActorOptions: rayiov1alpha1.RayActorOptionSpec{
NumCpus: &numCpus,
},
},
{
Name: "PearStand",
NumReplicas: &numReplicas,
UserConfig: "price: 1",
RayActorOptions: rayiov1alpha1.RayActorOptionSpec{
NumCpus: &numCpus,
},
},
},
},
RayClusterSpec: rayiov1alpha1.RayClusterSpec{
RayVersion: "1.12.1",
HeadGroupSpec: rayiov1alpha1.HeadGroupSpec{
ServiceType: corev1.ServiceTypeClusterIP,
Replicas: pointer.Int32Ptr(1),
RayStartParams: map[string]string{
"port": "6379",
"object-store-memory": "100000000",
"dashboard-host": "0.0.0.0",
"num-cpus": "1",
"node-ip-address": "127.0.0.1",
"block": "true",
"dashboard-agent-listen-port": "52365",
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"rayCluster": "raycluster-sample",
"rayNodeType": "head",
"groupName": "headgroup",
},
Annotations: map[string]string{
"key": "value",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-head",
Image: "rayproject/ray:1.12.1",
Env: []corev1.EnvVar{
{
Name: "MY_POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
},
},
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("2Gi"),
},
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("2Gi"),
},
},
Ports: []corev1.ContainerPort{
{
Name: "gcs-server",
ContainerPort: 6379,
},
{
Name: "dashboard",
ContainerPort: 8265,
},
{
Name: "head",
ContainerPort: 10001,
},
{
Name: "dashboard-agent",
ContainerPort: 52365,
},
{
Name: "serve",
ContainerPort: 8000,
},
},
},
},
},
},
},
WorkerGroupSpecs: []rayiov1alpha1.WorkerGroupSpec{
{
Replicas: pointer.Int32Ptr(3),
MinReplicas: pointer.Int32Ptr(0),
MaxReplicas: pointer.Int32Ptr(10000),
GroupName: "small-group",
RayStartParams: map[string]string{
"port": "6379",
"num-cpus": "1",
"dashboard-agent-listen-port": "52365",
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Labels: map[string]string{
"rayCluster": "raycluster-sample",
"groupName": "small-group",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/ray:1.12.1",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
Env: []corev1.EnvVar{
{
Name: "MY_POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
},
},
Ports: []corev1.ContainerPort{
{
Name: "client",
ContainerPort: 80,
},
{
Name: "dashboard-agent",
ContainerPort: 52365,
},
},
},
},
},
},
},
},
},
},
}
fakeRayDashboardClient := prepareFakeRayDashboardClient()
utils.GetRayDashboardClientFunc = func() utils.RayDashboardClientInterface {
return &fakeRayDashboardClient
}
utils.GetRayHttpProxyClientFunc = utils.GetFakeRayHttpProxyClient
myRayCluster := &rayiov1alpha1.RayCluster{}
Describe("When creating a rayservice", func() {
It("should create a rayservice object", func() {
err := k8sClient.Create(ctx, myRayService)
Expect(err).NotTo(HaveOccurred(), "failed to create test RayService resource")
})
It("should see a rayservice object", func() {
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: myRayService.Name, Namespace: "default"}, myRayService),
time.Second*3, time.Millisecond*500).Should(BeNil(), "My myRayService = %v", myRayService.Name)
})
It("should create a raycluster object", func() {
Eventually(
getRayClusterNameFunc(ctx, myRayService),
time.Second*15, time.Millisecond*500).Should(Not(BeEmpty()), "My RayCluster name = %v", myRayService.Status.ActiveServiceStatus.RayClusterName)
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: myRayService.Status.ActiveServiceStatus.RayClusterName, Namespace: "default"}, myRayCluster),
time.Second*3, time.Millisecond*500).Should(BeNil(), "My myRayCluster = %v", myRayCluster.Name)
})
It("should create more than 1 worker", func() {
filterLabels := client.MatchingLabels{common.RayClusterLabelKey: myRayService.Status.ActiveServiceStatus.RayClusterName, common.RayNodeGroupLabelKey: "small-group"}
Eventually(
listResourceFunc(ctx, &workerPods, filterLabels, &client.ListOptions{Namespace: "default"}),
time.Second*15, time.Millisecond*500).Should(Equal(3), fmt.Sprintf("workerGroup %v", workerPods.Items))
if len(workerPods.Items) > 0 {
Expect(workerPods.Items[0].Status.Phase).Should(Or(Equal(corev1.PodRunning), Equal(corev1.PodPending)))
}
})
It("Dashboard should be healthy", func() {
Eventually(
checkServiceHealth(ctx, myRayService),
time.Second*3, time.Millisecond*500).Should(BeTrue(), "My myRayService status = %v", myRayService.Status)
})
It("should create a new head service resource", func() {
svc := &corev1.Service{}
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: utils.GenerateServiceName(myRayService.Name), Namespace: "default"}, svc),
time.Second*15, time.Millisecond*500).Should(BeNil(), "My head service = %v", svc)
Expect(svc.Spec.Selector[common.RayIDLabelKey]).Should(Equal(utils.GenerateIdentifier(myRayCluster.Name, rayiov1alpha1.HeadNode)))
})
It("should create a new agent service resource", func() {
svc := &corev1.Service{}
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: utils.GenerateDashboardServiceName(myRayCluster.Name), Namespace: "default"}, svc),
time.Second*15, time.Millisecond*500).Should(BeNil(), "My agent service = %v", svc)
Expect(svc.Spec.Selector[common.RayClusterDashboardServiceLabelKey]).Should(Equal(utils.GenerateDashboardAgentLabel(myRayCluster.Name)))
})
It("should create a new serve service resource", func() {
svc := &corev1.Service{}
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: utils.GenerateServeServiceName(myRayService.Name), Namespace: "default"}, svc),
time.Second*15, time.Millisecond*500).Should(BeNil(), "My serve service = %v", svc)
Expect(svc.Spec.Selector[common.RayClusterLabelKey]).Should(Equal(myRayCluster.Name))
})
It("should update a rayservice object and switch to new Ray Cluster", func() {
// adding a scale strategy
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: myRayService.Name, Namespace: "default"}, myRayService),
time.Second*3, time.Millisecond*500).Should(BeNil(), "My myRayService = %v", myRayService.Name)
podToDelete1 := workerPods.Items[0]
rep := new(int32)
*rep = 1
myRayService.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas = rep
myRayService.Spec.RayClusterSpec.WorkerGroupSpecs[0].ScaleStrategy.WorkersToDelete = []string{podToDelete1.Name}
Expect(k8sClient.Update(ctx, myRayService)).Should(Succeed(), "failed to update test RayService resource")
// Confirm switch to a new Ray Cluster.
Eventually(
getRayClusterNameFunc(ctx, myRayService),
time.Second*15, time.Millisecond*500).Should(Not(Equal(myRayCluster.Name)), "My new RayCluster name = %v", myRayService.Status.ActiveServiceStatus.RayClusterName)
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: myRayService.Status.ActiveServiceStatus.RayClusterName, Namespace: "default"}, myRayCluster),
time.Second*3, time.Millisecond*500).Should(BeNil(), "My myRayCluster = %v", myRayCluster.Name)
})
It("Should detect unhealthy status and try to switch to new RayCluster.", func() {
// Set a wrong serve status with unhealthy.
orignialServeDeploymentUnhealthySecondThreshold := ServiceUnhealthySecondThreshold
ServiceUnhealthySecondThreshold = 5
fakeRayDashboardClient.SetServeStatus(generateServeStatus(metav1.NewTime(time.Now().Add(time.Duration(-5)*time.Minute)), "UNHEALTHY"))
Eventually(
getPreparingRayClusterNameFunc(ctx, myRayService),
time.Second*60, time.Millisecond*500).Should(Not(BeEmpty()), "My new RayCluster name = %v", myRayService.Status.PendingServiceStatus.RayClusterName)
ServiceUnhealthySecondThreshold = orignialServeDeploymentUnhealthySecondThreshold
pendingRayClusterName := myRayService.Status.PendingServiceStatus.RayClusterName
fakeRayDashboardClient.SetServeStatus(generateServeStatus(metav1.Now(), "HEALTHY"))
Eventually(
getPreparingRayClusterNameFunc(ctx, myRayService),
time.Second*15, time.Millisecond*500).Should(BeEmpty(), "My new RayCluster name = %v", myRayService.Status.PendingServiceStatus.RayClusterName)
Eventually(
getRayClusterNameFunc(ctx, myRayService),
time.Second*15, time.Millisecond*500).Should(Equal(pendingRayClusterName), "My new RayCluster name = %v", myRayService.Status.ActiveServiceStatus.RayClusterName)
})
})
})
func prepareFakeRayDashboardClient() utils.FakeRayDashboardClient {
client := utils.FakeRayDashboardClient{}
client.SetServeStatus(generateServeStatus(metav1.Now(), "HEALTHY"))
return client
}
func generateServeStatus(time metav1.Time, status string) utils.ServeDeploymentStatuses {
serveStatuses := utils.ServeDeploymentStatuses{
ApplicationStatus: rayiov1alpha1.AppStatus{
Status: "RUNNING",
LastUpdateTime: &time,
HealthLastUpdateTime: &time,
},
DeploymentStatuses: []rayiov1alpha1.ServeDeploymentStatus{
{
Name: "shallow",
Status: status,
Message: "",
LastUpdateTime: &time,
HealthLastUpdateTime: &time,
},
{
Name: "deep",
Status: status,
Message: "",
LastUpdateTime: &time,
HealthLastUpdateTime: &time,
},
{
Name: "one",
Status: status,
Message: "",
LastUpdateTime: &time,
HealthLastUpdateTime: &time,
},
},
}
return serveStatuses
}
func getRayClusterNameFunc(ctx context.Context, rayService *rayiov1alpha1.RayService) func() (string, error) {
return func() (string, error) {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: rayService.Name, Namespace: "default"}, rayService); err != nil {
return "", err
}
return rayService.Status.ActiveServiceStatus.RayClusterName, nil
}
}
func getPreparingRayClusterNameFunc(ctx context.Context, rayService *rayiov1alpha1.RayService) func() (string, error) {
return func() (string, error) {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: rayService.Name, Namespace: "default"}, rayService); err != nil {
return "", err
}
return rayService.Status.PendingServiceStatus.RayClusterName, nil
}
}
func checkServiceHealth(ctx context.Context, rayService *rayiov1alpha1.RayService) func() (bool, error) {
return func() (bool, error) {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: rayService.Name, Namespace: rayService.Namespace}, rayService); err != nil {
return false, err
}
healthy := true
healthy = healthy && rayService.Status.ActiveServiceStatus.DashboardStatus.IsHealthy
healthy = healthy && (len(rayService.Status.ActiveServiceStatus.ServeStatuses) == 3)
healthy = healthy && rayService.Status.ActiveServiceStatus.ServeStatuses[0].Status == "HEALTHY"
healthy = healthy && rayService.Status.ActiveServiceStatus.ServeStatuses[1].Status == "HEALTHY"
healthy = healthy && rayService.Status.ActiveServiceStatus.ServeStatuses[2].Status == "HEALTHY"
return healthy, nil
}
}