mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
* Updating client-go to v1.36.0-beta.0 (and updating golang to 1.26) Signed-off-by: Mark Rossetti <[email protected]> * Bump golangci-lint to v2.11.4 and fix lint issues for Go 1.26 Signed-off-by: Mark Rossetti <[email protected]> * bump go to 1.26 in ray-operator docker file Signed-off-by: Mark Rossetti <[email protected]> * Migrate webhooks from deprecated CustomValidator to admission.Validation[T] Signed-off-by: Mark Rossetti <[email protected]> * Target K8s 1.36 GA deps Signed-off-by: Mark Rossetti <[email protected]> * Merge branch 'master' into go-client-go-updates Signed-off-by: Rueian <[email protected]> --------- Signed-off-by: Mark Rossetti <[email protected]> Signed-off-by: Rueian <[email protected]> Co-authored-by: Rueian <[email protected]>
266 lines
13 KiB
Go
266 lines
13 KiB
Go
package e2e
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
. "github.com/onsi/gomega"
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
|
|
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
|
|
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
|
|
rayv1ac "github.com/ray-project/kuberay/ray-operator/pkg/client/applyconfiguration/ray/v1"
|
|
. "github.com/ray-project/kuberay/ray-operator/test/support"
|
|
)
|
|
|
|
func TestRayClusterManagedBy(t *testing.T) {
|
|
test := With(t)
|
|
g := NewWithT(t)
|
|
|
|
// Create a namespace
|
|
namespace := test.NewTestNamespace()
|
|
|
|
test.T().Run("Successful creation of cluster, managed by Kuberay Operator", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
rayClusterAC := rayv1ac.RayCluster("raycluster-ok", namespace.Name).
|
|
WithSpec(NewRayClusterSpec().
|
|
WithManagedBy(utils.KubeRayController))
|
|
|
|
rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
LogWithTimestamp(test.T(), "Created RayCluster %s/%s successfully", rayCluster.Namespace, rayCluster.Name)
|
|
|
|
LogWithTimestamp(test.T(), "Waiting for RayCluster %s/%s to become ready", rayCluster.Namespace, rayCluster.Name)
|
|
g.Eventually(RayCluster(test, rayCluster.Namespace, rayCluster.Name), TestTimeoutMedium).
|
|
Should(WithTransform(RayClusterState, Equal(rayv1.Ready)))
|
|
})
|
|
|
|
test.T().Run("Creation of cluster skipped, managed by Kueue", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
rayClusterAC := rayv1ac.RayCluster("raycluster-skip", namespace.Name).
|
|
WithSpec(NewRayClusterSpec().
|
|
WithManagedBy("kueue.x-k8s.io/multikueue"))
|
|
|
|
rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
LogWithTimestamp(test.T(), "Created RayCluster %s/%s successfully", rayCluster.Namespace, rayCluster.Name)
|
|
|
|
LogWithTimestamp(test.T(), "RayCluster %s/%s will not become ready - not reconciled", rayCluster.Namespace, rayCluster.Name)
|
|
g.Consistently(func(gg Gomega) {
|
|
rc, err := RayCluster(test, rayCluster.Namespace, rayCluster.Name)()
|
|
gg.Expect(err).NotTo(HaveOccurred())
|
|
gg.Expect(rc.Status.Conditions).To(BeEmpty())
|
|
}, time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
|
|
// Should not to be able to change managedBy field as it's immutable
|
|
rayClusterAC.Spec.WithManagedBy(utils.KubeRayController)
|
|
rayCluster, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
|
|
g.Expect(err).To(HaveOccurred())
|
|
g.Eventually(RayCluster(test, *rayClusterAC.Namespace, *rayClusterAC.Name)).
|
|
Should(WithTransform(RayClusterManagedBy, Equal(new("kueue.x-k8s.io/multikueue"))))
|
|
})
|
|
|
|
test.T().Run("Failed creation of cluster, managed by external non supported controller", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
rayClusterAC := rayv1ac.RayCluster("raycluster-fail", namespace.Name).
|
|
WithSpec(NewRayClusterSpec().
|
|
WithManagedBy("controller.com/not-supported"))
|
|
|
|
_, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
|
|
g.Expect(err).To(HaveOccurred())
|
|
g.Expect(errors.IsInvalid(err)).To(BeTrue(), "error: %v", err)
|
|
})
|
|
}
|
|
|
|
func TestRayClusterSuspend(t *testing.T) {
|
|
test := With(t)
|
|
g := NewWithT(t)
|
|
// Create a namespace
|
|
namespace := test.NewTestNamespace()
|
|
|
|
rayClusterAC := rayv1ac.RayCluster("raycluster-suspend", namespace.Name).WithSpec(NewRayClusterSpec())
|
|
|
|
rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
LogWithTimestamp(test.T(), "Created RayCluster %s/%s successfully", rayCluster.Namespace, rayCluster.Name)
|
|
|
|
LogWithTimestamp(test.T(), "Waiting for RayCluster %s/%s to become ready", rayCluster.Namespace, rayCluster.Name)
|
|
g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium).
|
|
Should(WithTransform(StatusCondition(rayv1.HeadPodReady), MatchCondition(metav1.ConditionTrue, rayv1.HeadPodRunningAndReady)))
|
|
g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium).
|
|
Should(WithTransform(StatusCondition(rayv1.RayClusterProvisioned), MatchCondition(metav1.ConditionTrue, rayv1.AllPodRunningAndReadyFirstTime)))
|
|
|
|
rayClusterAC = rayClusterAC.WithSpec(rayClusterAC.Spec.WithSuspend(true))
|
|
rayCluster, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
LogWithTimestamp(test.T(), "Suspend RayCluster %s/%s successfully", rayCluster.Namespace, rayCluster.Name)
|
|
|
|
LogWithTimestamp(test.T(), "Waiting for RayCluster %s/%s to be suspended", rayCluster.Namespace, rayCluster.Name)
|
|
g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium).
|
|
Should(WithTransform(StatusCondition(rayv1.RayClusterSuspended), MatchCondition(metav1.ConditionTrue, string(rayv1.RayClusterSuspended))))
|
|
g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium).
|
|
Should(WithTransform(StatusCondition(rayv1.HeadPodReady), MatchCondition(metav1.ConditionFalse, rayv1.HeadPodNotFound)))
|
|
g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium).
|
|
Should(WithTransform(StatusCondition(rayv1.RayClusterProvisioned), MatchCondition(metav1.ConditionFalse, rayv1.RayClusterPodsProvisioning)))
|
|
|
|
rayClusterAC = rayClusterAC.WithSpec(rayClusterAC.Spec.WithSuspend(false))
|
|
rayCluster, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
LogWithTimestamp(test.T(), "Resume RayCluster %s/%s successfully", rayCluster.Namespace, rayCluster.Name)
|
|
|
|
LogWithTimestamp(test.T(), "Waiting for RayCluster %s/%s to be resumed", rayCluster.Namespace, rayCluster.Name)
|
|
g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium).
|
|
Should(WithTransform(StatusCondition(rayv1.RayClusterSuspended), MatchCondition(metav1.ConditionFalse, string(rayv1.RayClusterSuspended))))
|
|
g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium).
|
|
Should(WithTransform(StatusCondition(rayv1.HeadPodReady), MatchCondition(metav1.ConditionTrue, rayv1.HeadPodRunningAndReady)))
|
|
g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium).
|
|
Should(WithTransform(StatusCondition(rayv1.RayClusterProvisioned), MatchCondition(metav1.ConditionTrue, rayv1.AllPodRunningAndReadyFirstTime)))
|
|
}
|
|
|
|
func TestRayClusterWithResourceQuota(t *testing.T) {
|
|
test := With(t)
|
|
g := NewWithT(t)
|
|
|
|
// Create a namespace
|
|
namespace := test.NewTestNamespace()
|
|
|
|
// Create a resource quota
|
|
CreateResourceQuota(test, namespace.Name, "test-quota", "0.1", "0.1Gi")
|
|
|
|
rayClusterAC := rayv1ac.RayCluster("raycluster-resource-quota", namespace.Name).WithSpec(NewRayClusterSpec())
|
|
|
|
rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
LogWithTimestamp(test.T(), "Created RayCluster %s/%s successfully", rayCluster.Namespace, rayCluster.Name)
|
|
|
|
LogWithTimestamp(test.T(), "Waiting for RayCluster %s/%s to have ReplicaFailure condition", rayCluster.Namespace, rayCluster.Name)
|
|
g.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutShort).
|
|
Should(WithTransform(StatusCondition(rayv1.RayClusterReplicaFailure), MatchConditionContainsMessage(metav1.ConditionTrue, utils.ErrFailedCreateHeadPod.Error(), "forbidden: exceeded quota")))
|
|
}
|
|
|
|
func TestRayClusterScalingDown(t *testing.T) {
|
|
test := With(t)
|
|
g := NewWithT(t)
|
|
|
|
// Create a namespace
|
|
namespace := test.NewTestNamespace()
|
|
|
|
rayClusterAC := rayv1ac.RayCluster("raycluster-scaling-down", namespace.Name).
|
|
WithSpec(rayv1ac.RayClusterSpec().
|
|
WithRayVersion(GetRayVersion()).
|
|
WithHeadGroupSpec(rayv1ac.HeadGroupSpec().
|
|
WithRayStartParams(map[string]string{"dashboard-host": "0.0.0.0"}).
|
|
WithTemplate(HeadPodTemplateApplyConfiguration().WithFinalizers("test.kuberay.io/finalizers"))).
|
|
WithWorkerGroupSpecs(rayv1ac.WorkerGroupSpec().
|
|
WithReplicas(2).
|
|
WithMinReplicas(1).
|
|
WithMaxReplicas(5).
|
|
WithGroupName("small-group").
|
|
WithRayStartParams(map[string]string{"num-cpus": "1"}).
|
|
WithTemplate(WorkerPodTemplateApplyConfiguration().WithFinalizers("test.kuberay.io/finalizers"))))
|
|
|
|
rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
LogWithTimestamp(test.T(), "Created RayCluster %s/%s successfully", namespace.Name, rayCluster.Name)
|
|
|
|
LogWithTimestamp(test.T(), "Waiting for RayCluster %s/%s to become ready", namespace.Name, rayCluster.Name)
|
|
g.Eventually(RayCluster(test, rayCluster.Namespace, rayCluster.Name), TestTimeoutMedium).
|
|
Should(WithTransform(RayClusterState, Equal(rayv1.Ready)))
|
|
|
|
headPod, err := GetHeadPod(test, rayCluster)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
workerPods, err := GetWorkerPods(test, rayCluster)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
allPods := append([]corev1.Pod{*headPod}, workerPods...)
|
|
|
|
LogWithTimestamp(test.T(), "Scaling down replicas of RayCluster %s/%s by 1", namespace.Name, rayCluster.Name)
|
|
rayClusterAC.Spec.WorkerGroupSpecs[0].WithReplicas(1)
|
|
rayCluster, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
|
|
g.Expect(err).NotTo(HaveOccurred(), "Failed to scale down RayCluster")
|
|
|
|
time.Sleep(5 * time.Second)
|
|
|
|
headPod, err = GetHeadPod(test, rayCluster)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
g.Expect(headPod.DeletionTimestamp).To(BeNil(), "Head pod should not have deletionTimestamp")
|
|
|
|
workerPods, err = GetWorkerPods(test, rayCluster)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
deletingCount := 0
|
|
for _, pod := range workerPods {
|
|
if pod.DeletionTimestamp != nil {
|
|
deletingCount++
|
|
}
|
|
}
|
|
g.Expect(deletingCount).To(Equal(1), "Should have only one worker pod having deletionTimestamp")
|
|
|
|
LogWithTimestamp(test.T(), "Removing finalizers from pods")
|
|
for _, pod := range allPods {
|
|
patchBytes := []byte(`{"metadata":{"finalizers":[]}}`)
|
|
_, err := test.Client().Core().CoreV1().Pods(namespace.Name).Patch(test.Ctx(), pod.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{})
|
|
g.Expect(err).NotTo(HaveOccurred(), "Failed to remove finalizer from pod %s/%s", namespace.Name, pod.Name)
|
|
}
|
|
}
|
|
|
|
func TestRayClusterUpgradeStrategy(t *testing.T) {
|
|
test := With(t)
|
|
g := NewWithT(t)
|
|
|
|
namespace := test.NewTestNamespace()
|
|
|
|
rayClusterAC := rayv1ac.RayCluster("raycluster-upgrade-recreate", namespace.Name).WithSpec(NewRayClusterSpec())
|
|
rayClusterAC.Spec.UpgradeStrategy = rayv1ac.RayClusterUpgradeStrategy().WithType(rayv1.RayClusterRecreate)
|
|
|
|
rayCluster, err := test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
LogWithTimestamp(test.T(), "Created RayCluster %s/%s successfully", namespace.Name, rayCluster.Name)
|
|
|
|
LogWithTimestamp(test.T(), "Waiting for RayCluster %s/%s to become ready", namespace.Name, rayCluster.Name)
|
|
g.Eventually(RayCluster(test, rayCluster.Namespace, rayCluster.Name), TestTimeoutMedium).
|
|
Should(WithTransform(RayClusterState, Equal(rayv1.Ready)))
|
|
|
|
headPod, err := GetHeadPod(test, rayCluster)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
initialHeadPodName := headPod.Name
|
|
initialHeadPodHash := headPod.Annotations[utils.UpgradeStrategyRecreateHashKey]
|
|
|
|
workerPods, err := GetWorkerPods(test, rayCluster)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
g.Expect(workerPods).To(HaveLen(1))
|
|
|
|
LogWithTimestamp(test.T(), "Updating RayCluster %s/%s rayVersion and container image to trigger upgrade", rayCluster.Namespace, rayCluster.Name)
|
|
// Update rayVersion and container image to trigger Recreate upgrade
|
|
rayClusterAC.Spec.WithRayVersion("2.51.0")
|
|
rayClusterAC.Spec.HeadGroupSpec.Template.Spec.Containers[0].WithImage("rayproject/ray:2.51.0")
|
|
rayClusterAC.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].WithImage("rayproject/ray:2.51.0")
|
|
rayCluster, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Apply(test.Ctx(), rayClusterAC, TestApplyOptions)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
LogWithTimestamp(test.T(), "Updated RayCluster pod template")
|
|
|
|
LogWithTimestamp(test.T(), "Waiting for new head pod to be running after recreate")
|
|
g.Eventually(func() bool {
|
|
newHeadPod, err := GetHeadPod(test, rayCluster)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return newHeadPod.Name != initialHeadPodName && newHeadPod.Status.Phase == corev1.PodRunning
|
|
}, TestTimeoutMedium).Should(BeTrue())
|
|
|
|
newHeadPod, err := GetHeadPod(test, rayCluster)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
g.Expect(newHeadPod.Name).NotTo(Equal(initialHeadPodName))
|
|
|
|
newHeadPodHash := newHeadPod.Annotations[utils.UpgradeStrategyRecreateHashKey]
|
|
g.Expect(newHeadPodHash).NotTo(Equal(initialHeadPodHash))
|
|
|
|
newWorkerPods, err := GetWorkerPods(test, rayCluster)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
g.Expect(newWorkerPods).To(HaveLen(1))
|
|
}
|