Files
kuberay/ray-operator/controllers/ray/rayservice_controller_test.go
Dmitri GekhtmanandGitHub 5d38edac57 Revise sample configs, increase memory requests, update Ray versions (#761)
This PR

- Cleans up and updates sample configs and test files
- Increments Ray versions to the latest Ray release (2.1.0)
- Memory resource requests are increased in some sample configs.
- The Ray head pod has a high risk of running out of memory if it is allocated less than 2Gb memory, so that's fixed.

Signed-off-by: Dmitri Gekhtman <[email protected]>
2022-12-01 14:10:51 -08:00

454 lines
16 KiB
Go

/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ray
import (
"context"
"fmt"
"time"
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
"k8s.io/apimachinery/pkg/api/resource"
"github.com/ray-project/kuberay/ray-operator/controllers/ray/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1"
corev1 "k8s.io/api/core/v1"
"k8s.io/utils/pointer"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
// +kubebuilder:scaffold:imports
)
var runtimeEnvStr = "working_dir:\n - \"https://github.com/ray-project/test_dag/archive/c620251044717ace0a4c19d766d43c5099af8a77.zip\""
var _ = Context("Inside the default namespace", func() {
ctx := context.TODO()
var workerPods corev1.PodList
var numReplicas int32
var numCpus float64
numReplicas = 1
numCpus = 0.1
myRayService := &rayiov1alpha1.RayService{
ObjectMeta: metav1.ObjectMeta{
Name: "rayservice-sample",
Namespace: "default",
},
Spec: rayiov1alpha1.RayServiceSpec{
ServeDeploymentGraphSpec: rayiov1alpha1.ServeDeploymentGraphSpec{
ImportPath: "fruit.deployment_graph",
RuntimeEnv: runtimeEnvStr,
ServeConfigSpecs: []rayiov1alpha1.ServeConfigSpec{
{
Name: "MangoStand",
NumReplicas: &numReplicas,
UserConfig: "price: 3",
RayActorOptions: rayiov1alpha1.RayActorOptionSpec{
NumCpus: &numCpus,
},
},
{
Name: "OrangeStand",
NumReplicas: &numReplicas,
UserConfig: "price: 2",
RayActorOptions: rayiov1alpha1.RayActorOptionSpec{
NumCpus: &numCpus,
},
},
{
Name: "PearStand",
NumReplicas: &numReplicas,
UserConfig: "price: 1",
RayActorOptions: rayiov1alpha1.RayActorOptionSpec{
NumCpus: &numCpus,
},
},
},
},
RayClusterSpec: rayiov1alpha1.RayClusterSpec{
RayVersion: "1.12.1",
HeadGroupSpec: rayiov1alpha1.HeadGroupSpec{
ServiceType: corev1.ServiceTypeClusterIP,
Replicas: pointer.Int32Ptr(1),
RayStartParams: map[string]string{
"port": "6379",
"object-store-memory": "100000000",
"dashboard-host": "0.0.0.0",
"num-cpus": "1",
"node-ip-address": "127.0.0.1",
"block": "true",
"dashboard-agent-listen-port": "52365",
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"rayCluster": "raycluster-sample",
"groupName": "headgroup",
},
Annotations: map[string]string{
"key": "value",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-head",
Image: "rayproject/ray:2.1.0",
Env: []corev1.EnvVar{
{
Name: "MY_POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
},
{
Name: "SAMPLE_ENV_VAR",
Value: "SAMPLE_VALUE",
},
},
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("2Gi"),
},
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("2Gi"),
},
},
Ports: []corev1.ContainerPort{
{
Name: "gcs-server",
ContainerPort: 6379,
},
{
Name: "dashboard",
ContainerPort: 8265,
},
{
Name: "head",
ContainerPort: 10001,
},
{
Name: "dashboard-agent",
ContainerPort: 52365,
},
{
Name: "serve",
ContainerPort: 8000,
},
},
},
},
},
},
},
WorkerGroupSpecs: []rayiov1alpha1.WorkerGroupSpec{
{
Replicas: pointer.Int32Ptr(3),
MinReplicas: pointer.Int32Ptr(0),
MaxReplicas: pointer.Int32Ptr(10000),
GroupName: "small-group",
RayStartParams: map[string]string{
"port": "6379",
"num-cpus": "1",
"dashboard-agent-listen-port": "52365",
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Labels: map[string]string{
"rayCluster": "raycluster-sample",
"groupName": "small-group",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "ray-worker",
Image: "rayproject/ray:2.1.0",
Command: []string{"echo"},
Args: []string{"Hello Ray"},
Env: []corev1.EnvVar{
{
Name: "MY_POD_IP",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
},
{
Name: "SAMPLE_ENV_VAR",
Value: "SAMPLE_VALUE",
},
},
Ports: []corev1.ContainerPort{
{
Name: "client",
ContainerPort: 80,
},
{
Name: "dashboard-agent",
ContainerPort: 52365,
},
},
},
},
},
},
},
},
},
},
}
fakeRayDashboardClient := prepareFakeRayDashboardClient()
utils.GetRayDashboardClientFunc = func() utils.RayDashboardClientInterface {
return &fakeRayDashboardClient
}
utils.GetRayHttpProxyClientFunc = utils.GetFakeRayHttpProxyClient
myRayCluster := &rayiov1alpha1.RayCluster{}
Describe("When creating a rayservice", func() {
It("should create a rayservice object", func() {
err := k8sClient.Create(ctx, myRayService)
Expect(err).NotTo(HaveOccurred(), "failed to create test RayService resource")
})
It("should see a rayservice object", func() {
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: myRayService.Name, Namespace: "default"}, myRayService),
time.Second*3, time.Millisecond*500).Should(BeNil(), "My myRayService = %v", myRayService.Name)
})
It("should create a raycluster object", func() {
Eventually(
getRayClusterNameFunc(ctx, myRayService),
time.Second*15, time.Millisecond*500).Should(Not(BeEmpty()), "My RayCluster name = %v", myRayService.Status.ActiveServiceStatus.RayClusterName)
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: myRayService.Status.ActiveServiceStatus.RayClusterName, Namespace: "default"}, myRayCluster),
time.Second*3, time.Millisecond*500).Should(BeNil(), "My myRayCluster = %v", myRayCluster.Name)
})
It("should create more than 1 worker", func() {
filterLabels := client.MatchingLabels{common.RayClusterLabelKey: myRayService.Status.ActiveServiceStatus.RayClusterName, common.RayNodeGroupLabelKey: "small-group"}
Eventually(
listResourceFunc(ctx, &workerPods, filterLabels, &client.ListOptions{Namespace: "default"}),
time.Second*15, time.Millisecond*500).Should(Equal(3), fmt.Sprintf("workerGroup %v", workerPods.Items))
if len(workerPods.Items) > 0 {
Expect(workerPods.Items[0].Status.Phase).Should(Or(Equal(corev1.PodRunning), Equal(corev1.PodPending)))
}
})
It("Dashboard should be healthy", func() {
Eventually(
checkServiceHealth(ctx, myRayService),
time.Second*3, time.Millisecond*500).Should(BeTrue(), "My myRayService status = %v", myRayService.Status)
})
It("should create a new head service resource", func() {
svc := &corev1.Service{}
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: utils.GenerateServiceName(myRayService.Name), Namespace: "default"}, svc),
time.Second*15, time.Millisecond*500).Should(BeNil(), "My head service = %v", svc)
Expect(svc.Spec.Selector[common.RayIDLabelKey]).Should(Equal(utils.GenerateIdentifier(myRayCluster.Name, rayiov1alpha1.HeadNode)))
})
It("should create a new agent service resource", func() {
svc := &corev1.Service{}
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: utils.GenerateDashboardServiceName(myRayCluster.Name), Namespace: "default"}, svc),
time.Second*15, time.Millisecond*500).Should(BeNil(), "My agent service = %v", svc)
Expect(svc.Spec.Selector[common.RayClusterDashboardServiceLabelKey]).Should(Equal(utils.GenerateDashboardAgentLabel(myRayCluster.Name)))
})
It("should create a new serve service resource", func() {
svc := &corev1.Service{}
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: utils.GenerateServeServiceName(myRayService.Name), Namespace: "default"}, svc),
time.Second*15, time.Millisecond*500).Should(BeNil(), "My serve service = %v", svc)
Expect(svc.Spec.Selector[common.RayClusterLabelKey]).Should(Equal(myRayCluster.Name))
})
It("should update a rayservice object and switch to new Ray Cluster", func() {
// adding a scale strategy
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: myRayService.Name, Namespace: "default"}, myRayService),
time.Second*3, time.Millisecond*500).Should(BeNil(), "My myRayService = %v", myRayService.Name)
podToDelete1 := workerPods.Items[0]
rep := new(int32)
*rep = 1
myRayService.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas = rep
myRayService.Spec.RayClusterSpec.WorkerGroupSpecs[0].ScaleStrategy.WorkersToDelete = []string{podToDelete1.Name}
Expect(k8sClient.Update(ctx, myRayService)).Should(Succeed(), "failed to update test RayService resource")
// Confirm switch to a new Ray Cluster.
Eventually(
getRayClusterNameFunc(ctx, myRayService),
time.Second*15, time.Millisecond*500).Should(Not(Equal(myRayCluster.Name)), "My new RayCluster name = %v", myRayService.Status.ActiveServiceStatus.RayClusterName)
Eventually(
getResourceFunc(ctx, client.ObjectKey{Name: myRayService.Status.ActiveServiceStatus.RayClusterName, Namespace: "default"}, myRayCluster),
time.Second*3, time.Millisecond*500).Should(BeNil(), "My myRayCluster = %v", myRayCluster.Name)
})
It("should detect unhealthy status and try to switch to new RayCluster.", func() {
// Set deployment statuses to UNHEALTHY
orignalServeDeploymentUnhealthySecondThreshold := ServiceUnhealthySecondThreshold
ServiceUnhealthySecondThreshold = 5
fakeRayDashboardClient.SetServeStatus(generateServeStatus(metav1.NewTime(time.Now().Add(time.Duration(-5)*time.Minute)), "UNHEALTHY"))
Eventually(
getPreparingRayClusterNameFunc(ctx, myRayService),
time.Second*60, time.Millisecond*500).Should(Not(BeEmpty()), "My new RayCluster name = %v", myRayService.Status.PendingServiceStatus.RayClusterName)
ServiceUnhealthySecondThreshold = orignalServeDeploymentUnhealthySecondThreshold
pendingRayClusterName := myRayService.Status.PendingServiceStatus.RayClusterName
fakeRayDashboardClient.SetServeStatus(generateServeStatus(metav1.Now(), "HEALTHY"))
Eventually(
getPreparingRayClusterNameFunc(ctx, myRayService),
time.Second*15, time.Millisecond*500).Should(BeEmpty(), "My new RayCluster name = %v", myRayService.Status.PendingServiceStatus.RayClusterName)
Eventually(
getRayClusterNameFunc(ctx, myRayService),
time.Second*15, time.Millisecond*500).Should(Equal(pendingRayClusterName), "My new RayCluster name = %v", myRayService.Status.ActiveServiceStatus.RayClusterName)
})
It("should perform a zero-downtime update after a code change.", func() {
initialClusterName, _ := getRayClusterNameFunc(ctx, myRayService)()
// The cluster shouldn't switch until deployments are finished updating
fakeRayDashboardClient.SetServeStatus(generateServeStatus(metav1.NewTime(time.Now().Add(time.Duration(-5)*time.Minute)), "UPDATING"))
myRayService.Spec.RayClusterSpec.HeadGroupSpec.Template.Spec.Containers[0].Env[1].Value = "UPDATED_VALUE"
myRayService.Spec.RayClusterSpec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env[1].Value = "UPDATED_VALUE"
Expect(k8sClient.Update(ctx, myRayService)).Should(Succeed(), "failed to update test RayService resource")
Eventually(
getPreparingRayClusterNameFunc(ctx, myRayService),
time.Second*60, time.Millisecond*500).Should(Not(BeEmpty()), "My new RayCluster name = %v", myRayService.Status.PendingServiceStatus.RayClusterName)
pendingRayClusterName := myRayService.Status.PendingServiceStatus.RayClusterName
Consistently(
getRayClusterNameFunc(ctx, myRayService),
time.Second*5, time.Millisecond*500).Should(Equal(initialClusterName), "My current RayCluster name = %v", myRayService.Status.ActiveServiceStatus.RayClusterName)
// The cluster should switch once the deployments are finished updating
fakeRayDashboardClient.SetServeStatus(generateServeStatus(metav1.NewTime(time.Now().Add(time.Duration(-5)*time.Minute)), "HEALTHY"))
Eventually(
getRayClusterNameFunc(ctx, myRayService),
time.Second*60, time.Millisecond*500).Should(Equal(pendingRayClusterName), "My current RayCluster name = %v", myRayService.Status.ActiveServiceStatus.RayClusterName)
})
})
})
func prepareFakeRayDashboardClient() utils.FakeRayDashboardClient {
client := utils.FakeRayDashboardClient{}
client.SetServeStatus(generateServeStatus(metav1.Now(), "HEALTHY"))
return client
}
func generateServeStatus(time metav1.Time, status string) utils.ServeDeploymentStatuses {
serveStatuses := utils.ServeDeploymentStatuses{
ApplicationStatus: rayiov1alpha1.AppStatus{
Status: "RUNNING",
LastUpdateTime: &time,
HealthLastUpdateTime: &time,
},
DeploymentStatuses: []rayiov1alpha1.ServeDeploymentStatus{
{
Name: "shallow",
Status: status,
Message: "",
LastUpdateTime: &time,
HealthLastUpdateTime: &time,
},
{
Name: "deep",
Status: status,
Message: "",
LastUpdateTime: &time,
HealthLastUpdateTime: &time,
},
{
Name: "one",
Status: status,
Message: "",
LastUpdateTime: &time,
HealthLastUpdateTime: &time,
},
},
}
return serveStatuses
}
func getRayClusterNameFunc(ctx context.Context, rayService *rayiov1alpha1.RayService) func() (string, error) {
return func() (string, error) {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: rayService.Name, Namespace: "default"}, rayService); err != nil {
return "", err
}
return rayService.Status.ActiveServiceStatus.RayClusterName, nil
}
}
func getPreparingRayClusterNameFunc(ctx context.Context, rayService *rayiov1alpha1.RayService) func() (string, error) {
return func() (string, error) {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: rayService.Name, Namespace: "default"}, rayService); err != nil {
return "", err
}
return rayService.Status.PendingServiceStatus.RayClusterName, nil
}
}
func checkServiceHealth(ctx context.Context, rayService *rayiov1alpha1.RayService) func() (bool, error) {
return func() (bool, error) {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: rayService.Name, Namespace: rayService.Namespace}, rayService); err != nil {
return false, err
}
healthy := true
healthy = healthy && rayService.Status.ActiveServiceStatus.DashboardStatus.IsHealthy
healthy = healthy && (len(rayService.Status.ActiveServiceStatus.ServeStatuses) == 3)
healthy = healthy && rayService.Status.ActiveServiceStatus.ServeStatuses[0].Status == "HEALTHY"
healthy = healthy && rayService.Status.ActiveServiceStatus.ServeStatuses[1].Status == "HEALTHY"
healthy = healthy && rayService.Status.ActiveServiceStatus.ServeStatuses[2].Status == "HEALTHY"
return healthy, nil
}
}