mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
+10









Andrew Sy Kim
GitHub
Ping
Han-Ju Chen
Ryan O'Leary
Kai-Hsun Chen
Kavish
Neo Chien
cchung100m
JustinYeh
Jun-Hao Wan
Divyam Raj
Nary Yeh
Alima Azamat
Spencer Peterson
dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Rueian
Cheng-Yeh Chung
fscnick
Copilot
f68857e9ff
* [Bug] Sidecar mode shouldn't restart head pod when head pod is deleted (#4141) * [Bug] Sidecar mode shouldn't restart head pod when head pod is deleted Signed-off-by: 400Ping <[email protected]> * [Fix] Fix e2e error Signed-off-by: 400Ping <[email protected]> * [Fix] fix according to rueian's comment Signed-off-by: 400Ping <[email protected]> * [Chore] fix ci error Signed-off-by: 400Ping <[email protected]> * Update ray-operator/controllers/ray/raycluster_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ping <[email protected]> * Update ray-operator/controllers/ray/rayjob_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ping <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * Trigger CI Signed-off-by: Future-Outlier <[email protected]> --------- Signed-off-by: 400Ping <[email protected]> Signed-off-by: Ping <[email protected]> Signed-off-by: Future-Outlier <[email protected]> Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> * fix: dashboard build for kuberay 1.5.0 (#4161) Signed-off-by: Future-Outlier <[email protected]> * [Feature Enhancement] Set ordered replica index label to support multi-slice (#4163) * [Feature Enhancement] Set ordered replica index label to support multi-slice Signed-off-by: Ryan O'Leary <[email protected]> * rename replica-id -> replica-name Signed-off-by: Ryan O'Leary <[email protected]> * Separate replica index feature gate logic Signed-off-by: Ryan O'Leary <[email protected]> * remove index arg in createWorkerPod Signed-off-by: Ryan O'Leary <[email protected]> --------- Signed-off-by: Ryan O'Leary <[email protected]> * update stale feature gate comments (#4174) Signed-off-by: Andrew Sy Kim <[email protected]> * [RayCluster] Add more context why we don't recreate head Pod for RayJob (#4175) Signed-off-by: Kai-Hsun Chen <[email protected]> * feature: Remove empty resource list initialization. (#4168) Fixes #4142. * [Dockerfile] [KubeRay Dashboard]: Fix Dockerfile warnings (ENV format, CMD JSON args) (#4167) * [#4166] improvement: Fix Dockerfile warnings (ENV format, CMD JSON args) * extract the hostname from CMD Signed-off-by: Neo Chien <[email protected]> --------- Signed-off-by: Neo Chien <[email protected]> Co-authored-by: cchung100m <[email protected]> * [Fix] Resolve int32 overflow by having the calculation in int64 and c… (#4158) * [Fix] Resolve int32 overflow by having the calculation in int64 and cap it if the count is over math.MaxInt32 Signed-off-by: justinyeh1995 <[email protected]> * [Test] Add unit tests for CalculateReadyReplicas Signed-off-by: justinyeh1995 <[email protected]> * [Fix] Add a nosec comment to pass the Lint (pre-commit) test Signed-off-by: justinyeh1995 <[email protected]> * [Refactor] Add CapInt64ToInt32 to replace #nosec directives Signed-off-by: justinyeh1995 <[email protected]> * [Refactor] Rename function to SafeInt64ToInt32 and add a underflowing prevention (it also help pass the lint test) Signed-off-by: justinyeh1995 <[email protected]> * [Refactor] Remove the early return as SafeInt64ToInt32 handles the int32 overflow and underflow checking. Signed-off-by: justinyeh1995 <[email protected]> --------- Signed-off-by: justinyeh1995 <[email protected]> * Add RayService incremental upgrade sample for guide (#4164) Signed-off-by: Ryan O'Leary <[email protected]> * Edit RayCluster example config for label selectors (#4151) Signed-off-by: Ryan O'Leary <[email protected]> * [RayJob] update light weight submitter image from quay.io (#4181) Signed-off-by: Future-Outlier <[email protected]> * [flaky] RayJob fails when head Pod is deleted when job is running (#4182) Signed-off-by: Future-Outlier <[email protected]> * [CI] Pin Docker api version to avoid API version mismatch (#4188) Signed-off-by: win5923 <[email protected]> * Make replicas configurable for kuberay-operator #4180 (#4195) * Make replicas configurable for kuberay-operator #4180 * Make replicas configurable for kuberay-operator #4180 * [Fix] rayjob update raycluster status (#4192) * feat: check if raycluster status update in rayjob * test: e2e test to check the rayjob raycluster status update * fix: dashboard http client tests discovered and passing (#4173) Signed-off-by: alimaazamat <[email protected]> * [RayJob] Lift cluster status while initializing (#4191) Signed-off-by: Spencer Peterson <[email protected]> * [RayJob] Remove updateJobStatus call (#4198) Fast follow to #4191 Signed-off-by: Spencer Peterson <[email protected]> * Add support for Ray token auth (#4179) * Add support for Ray token auth Signed-off-by: Andrew Sy Kim <[email protected]> * add e2e test for Ray cluster auth Signed-off-by: Andrew Sy Kim <[email protected]> * address nits from Ruiean Signed-off-by: Andrew Sy Kim <[email protected]> * update RAY_auth_mode -> RAY_AUTH_MODE Signed-off-by: Andrew Sy Kim <[email protected]> * configure auth for Ray autoscaler Signed-off-by: Andrew Sy Kim <[email protected]> --------- Signed-off-by: Andrew Sy Kim <[email protected]> * Bump js-yaml from 4.1.0 to 4.1.1 in /dashboard (#4194) Bumps [js-yaml](https://github.com/nodeca/js-yaml) from 4.1.0 to 4.1.1. - [Changelog](https://github.com/nodeca/js-yaml/blob/master/CHANGELOG.md) - [Commits](https://github.com/nodeca/js-yaml/compare/4.1.0...4.1.1) --- updated-dependencies: - dependency-name: js-yaml dependency-version: 4.1.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * update minimum Ray version required for token authentication to 2.52.0 (#4201) * update minimum Ray version required for token authentication to 2.52.0 Signed-off-by: Andrew Sy Kim <[email protected]> * update RayCluster auth e2e test to use Ray v2.52 Signed-off-by: Andrew Sy Kim <[email protected]> --------- Signed-off-by: Andrew Sy Kim <[email protected]> * add samples for RayCluster token auth (#4200) Signed-off-by: Andrew Sy Kim <[email protected]> * update (#4208) Signed-off-by: Future-Outlier <[email protected]> * [RayJob] Add token authentication support for All mode (#4210) * dashboard client authentication support Signed-off-by: Future-Outlier <[email protected]> * support rayjob Signed-off-by: Future-Outlier <[email protected]> * update to fix api serverr err Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * updarte Signed-off-by: Future-Outlier <[email protected]> * Rayjob sidecar mode auth token mode support Signed-off-by: Future-Outlier <[email protected]> * RayJob support k8s job mode Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * Address Andrew's advice Signed-off-by: Future-Outlier <[email protected]> * add todo x-ray-authorization comments Signed-off-by: Future-Outlier <[email protected]> --------- Signed-off-by: Future-Outlier <[email protected]> * [RayCluster] Enable Secret informer watch/list and remove unused RBAC verbs (#4202) * Add authentication secret reconciliation support Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * fix flaky test Signed-off-by: Future-Outlier <[email protected]> * remove test fix Signed-off-by: Rueian <[email protected]> --------- Signed-off-by: Future-Outlier <[email protected]> Signed-off-by: Rueian <[email protected]> Co-authored-by: Rueian <[email protected]> * [APIServer][Docs] Add user guide for retry behavior & configuration (#4144) * [Docs] Add the draft description about feature intro, configurations, and usecases Signed-off-by: justinyeh1995 <[email protected]> * [Fix] Update the retry walk-through Signed-off-by: justinyeh1995 <[email protected]> * [Doc] rewrite the first 2 sections Signed-off-by: justinyeh1995 <[email protected]> * [Doc] Revise documentation wording and add Observing Retry Behavior section Signed-off-by: justinyeh1995 <[email protected]> * [Fix] fix linting issue by running pre-commit run berfore commiting Signed-off-by: justinyeh1995 <[email protected]> * [Fix] fix linting errors in the Markdown linting Signed-off-by: justinyeh1995 <[email protected]> * [Fix] Clean up the math equation Signed-off-by: justinyeh1995 <[email protected]> * Update the math formula of Backoff calculation. Co-authored-by: Nary Yeh <[email protected]> Signed-off-by: JustinYeh <[email protected]> * [Fix] Explicitly mentioned exponential backoff and removed the customization parts Signed-off-by: justinyeh1995 <[email protected]> * [Docs] Clarify naming by replacing “APIServer” with “KubeRay APIServer” Co-authored-by: Cheng-Yeh Chung <[email protected]> Signed-off-by: JustinYeh <[email protected]> * [Docs] Rename retry-configuration.md to retry-behavior.md for accuracy Signed-off-by: justinyeh1995 <[email protected]> * Update Title to KubeRay APIServer Retry Behavior Co-authored-by: Cheng-Yeh Chung <[email protected]> Signed-off-by: JustinYeh <[email protected]> * [Docs] Add a note about the limitation of retry configuration Signed-off-by: justinyeh1995 <[email protected]> --------- Signed-off-by: justinyeh1995 <[email protected]> Signed-off-by: JustinYeh <[email protected]> Co-authored-by: Nary Yeh <[email protected]> Co-authored-by: Cheng-Yeh Chung <[email protected]> * Support X-Ray-Authorization fallback header for accepting auth token via proxy (#4213) * Support X-Ray-Authorization fallback header for accepting auth token in dashboard Signed-off-by: Future-Outlier <[email protected]> * remove todo comment Signed-off-by: Future-Outlier <[email protected]> --------- Signed-off-by: Future-Outlier <[email protected]> * [RayCluster] make auth token secret name consistency (#4216) Signed-off-by: fscnick <[email protected]> * [RayCluster] Status includes head containter status message (#4196) * [RayCluster] Status includes head containter status message Signed-off-by: Spencer Peterson <[email protected]> * lint Signed-off-by: Spencer Peterson <[email protected]> * [RayCluster] Containers not ready status reflects structured reason Signed-off-by: Spencer Peterson <[email protected]> * nit Signed-off-by: Spencer Peterson <[email protected]> --------- Signed-off-by: Spencer Peterson <[email protected]> * Remove erroneous call in applyServeTargetCapacity (#4212) Signed-off-by: Ryan O'Leary <[email protected]> * [RayJob] Add token authentication support for light weight job submitter (#4215) * [RayJob] light weight job submitter auth token support Signed-off-by: Future-Outlier <[email protected]> * X-Ray-Authorization Signed-off-by: Rueian <[email protected]> --------- Signed-off-by: Future-Outlier <[email protected]> Signed-off-by: Rueian <[email protected]> Co-authored-by: Rueian <[email protected]> * feat: kubectl ray get token command (#4218) * feat: kubectl ray get token command Signed-off-by: Rueian <[email protected]> * Update kubectl-plugin/pkg/cmd/get/get_token_test.go Co-authored-by: Copilot <[email protected]> Signed-off-by: Rueian <[email protected]> * Update kubectl-plugin/pkg/cmd/get/get_token.go Co-authored-by: Copilot <[email protected]> Signed-off-by: Rueian <[email protected]> * make sure the raycluster exists before getting the secret Signed-off-by: Rueian <[email protected]> * better ux Signed-off-by: Rueian <[email protected]> * Update kubectl-plugin/pkg/cmd/get/get_token.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Rueian <[email protected]> --------- Signed-off-by: Rueian <[email protected]> Co-authored-by: Copilot <[email protected]> Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> --------- Signed-off-by: 400Ping <[email protected]> Signed-off-by: Ping <[email protected]> Signed-off-by: Future-Outlier <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> Signed-off-by: Andrew Sy Kim <[email protected]> Signed-off-by: Kai-Hsun Chen <[email protected]> Signed-off-by: Neo Chien <[email protected]> Signed-off-by: justinyeh1995 <[email protected]> Signed-off-by: win5923 <[email protected]> Signed-off-by: alimaazamat <[email protected]> Signed-off-by: Spencer Peterson <[email protected]> Signed-off-by: dependabot[bot] <[email protected]> Signed-off-by: Rueian <[email protected]> Signed-off-by: JustinYeh <[email protected]> Signed-off-by: fscnick <[email protected]> Co-authored-by: Ping <[email protected]> Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Co-authored-by: Ryan O'Leary <[email protected]> Co-authored-by: Kai-Hsun Chen <[email protected]> Co-authored-by: Kavish <[email protected]> Co-authored-by: Neo Chien <[email protected]> Co-authored-by: cchung100m <[email protected]> Co-authored-by: JustinYeh <[email protected]> Co-authored-by: Jun-Hao Wan <[email protected]> Co-authored-by: Divyam Raj <[email protected]> Co-authored-by: Nary Yeh <[email protected]> Co-authored-by: Alima Azamat <[email protected]> Co-authored-by: Spencer Peterson <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Rueian <[email protected]> Co-authored-by: Cheng-Yeh Chung <[email protected]> Co-authored-by: fscnick <[email protected]> Co-authored-by: Copilot <[email protected]>
3946 lines
177 KiB
Go
3946 lines
177 KiB
Go
/*
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package ray
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
batchv1 "k8s.io/api/batch/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/utils/ptr"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
|
|
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
|
|
"github.com/ray-project/kuberay/ray-operator/controllers/ray/common"
|
|
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
|
|
utiltypes "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils/types"
|
|
"github.com/ray-project/kuberay/ray-operator/pkg/features"
|
|
"github.com/ray-project/kuberay/ray-operator/test/support"
|
|
)
|
|
|
|
func rayJobTemplate(name string, namespace string) *rayv1.RayJob {
|
|
return &rayv1.RayJob{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: namespace,
|
|
Annotations: map[string]string{
|
|
"foo": "bar",
|
|
},
|
|
},
|
|
Spec: rayv1.RayJobSpec{
|
|
Entrypoint: "sleep 999",
|
|
SubmissionMode: rayv1.K8sJobMode,
|
|
ShutdownAfterJobFinishes: true,
|
|
RayClusterSpec: &rayv1.RayClusterSpec{
|
|
RayVersion: support.GetRayVersion(),
|
|
HeadGroupSpec: rayv1.HeadGroupSpec{
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "ray-head",
|
|
Image: support.GetRayImage(),
|
|
Resources: corev1.ResourceRequirements{
|
|
Limits: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse("1"),
|
|
corev1.ResourceMemory: resource.MustParse("2Gi"),
|
|
},
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse("1"),
|
|
corev1.ResourceMemory: resource.MustParse("2Gi"),
|
|
},
|
|
},
|
|
Ports: []corev1.ContainerPort{
|
|
{
|
|
Name: utils.GcsServerPortName,
|
|
ContainerPort: utils.DefaultGcsServerPort,
|
|
},
|
|
{
|
|
Name: utils.DashboardPortName,
|
|
ContainerPort: utils.DefaultDashboardPort,
|
|
},
|
|
{
|
|
Name: utils.ClientPortName,
|
|
ContainerPort: utils.DefaultClientPort,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
WorkerGroupSpecs: []rayv1.WorkerGroupSpec{
|
|
{
|
|
Replicas: ptr.To[int32](3),
|
|
MinReplicas: ptr.To[int32](0),
|
|
MaxReplicas: ptr.To[int32](10000),
|
|
GroupName: "small-group",
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "ray-worker",
|
|
Image: support.GetRayImage(),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
var _ = Context("RayJob with different submission modes", func() {
|
|
Context("RayJob in K8sJobMode", func() {
|
|
Describe("RayJob SubmitterConfig BackoffLimit", Ordered, func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJobWithDefaultSubmitterConfigBackoffLimit := rayJobTemplate("rayjob-default", namespace)
|
|
rayJobWithNonDefaultSubmitterConfigBackoffLimit := rayJobTemplate("rayjob-non-default", namespace)
|
|
rayJobWithNonDefaultSubmitterConfigBackoffLimit.Spec.SubmitterConfig = &rayv1.SubmitterConfig{
|
|
BackoffLimit: ptr.To[int32](88),
|
|
}
|
|
rayJobs := make(map[*rayv1.RayJob]int32)
|
|
rayJobs[rayJobWithDefaultSubmitterConfigBackoffLimit] = int32(2)
|
|
rayJobs[rayJobWithNonDefaultSubmitterConfigBackoffLimit] = int32(88)
|
|
|
|
It("Verify RayJob spec", func() {
|
|
for rayJob := range rayJobs {
|
|
// Make sure the submission mode is K8sJobMode.
|
|
Expect(rayJob.Spec.SubmissionMode).To(Equal(rayv1.K8sJobMode))
|
|
}
|
|
})
|
|
|
|
It("Create RayJob custom resources", func() {
|
|
for rayJob := range rayJobs {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob: %v", rayJob.Name)
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
}
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
for rayJob := range rayJobs {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
}
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
for rayJob := range rayJobs {
|
|
rayCluster := &rayv1.RayCluster{}
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Make RayCluster.Status.State to be rayv1.Ready.
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
|
|
// RayJobs's JobDeploymentStatus transitions to Running.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
}
|
|
})
|
|
|
|
It("Verify K8s Job BackoffLimit", func() {
|
|
for rayJob, backoffLimit := range rayJobs {
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
Expect(*(job.Spec.BackoffLimit)).To(Equal(backoffLimit))
|
|
}
|
|
})
|
|
})
|
|
|
|
Describe("Successful RayJob in K8sJobMode", Ordered, func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test", namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
It("Verify RayJob spec", func() {
|
|
// This test case simulates the most common scenario in the RayJob code path.
|
|
// (1) The submission mode is K8sJobMode.
|
|
// (2) `shutdownAfterJobFinishes` is true.
|
|
// In this test, RayJob passes through the following states: New -> Initializing -> Running -> Complete
|
|
Expect(rayJob.Spec.SubmissionMode).To(Equal(rayv1.K8sJobMode))
|
|
Expect(rayJob.Spec.ShutdownAfterJobFinishes).To(BeTrue())
|
|
|
|
// This test assumes that there is only one worker group.
|
|
Expect(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs).To(HaveLen(1))
|
|
})
|
|
|
|
It("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
It("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
It("In Initializing state, the JobStatus should show the RayCluster status", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningNotReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// Now the cluster should have nonzero conditions.
|
|
Eventually(
|
|
func() int {
|
|
status := getClusterStatus(ctx, namespace, rayCluster.Name)()
|
|
return len(status.Conditions)
|
|
},
|
|
time.Second*3, time.Millisecond*500).ShouldNot(Equal(0))
|
|
|
|
// We expect the RayJob's RayClusterStatus to eventually mirror the cluster's status.
|
|
Eventually(
|
|
func() (int, error) {
|
|
currentRayJob := &rayv1.RayJob{}
|
|
err := k8sClient.Get(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, currentRayJob)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return len(currentRayJob.Status.RayClusterStatus.Conditions), nil
|
|
},
|
|
time.Second*3, time.Millisecond*500,
|
|
).ShouldNot(Equal(0))
|
|
})
|
|
|
|
It("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Succeeded" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusSucceeded, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusComplete), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
It("If shutdownAfterJobFinishes is true, RayCluster should be deleted but not the submitter Job.", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster)())
|
|
},
|
|
time.Second*3, time.Millisecond*500).Should(BeTrue())
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
})
|
|
|
|
Describe("Successful RayJob in K8sJobMode with a maximum name", Ordered, func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate(strings.Repeat("j", utils.MaxRayJobNameLength), namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
It("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
It("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
var svc corev1.Service
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
// Also check its service name.
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName + "-head-svc", Namespace: namespace}, &svc),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
})
|
|
|
|
Describe("Invalid RayJob in K8sJobMode", Ordered, func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-invalid-test", namespace)
|
|
rayCluster := &rayv1.RayCluster{Spec: *rayJob.Spec.RayClusterSpec}
|
|
template := common.GetSubmitterTemplate(&rayJob.Spec, &rayCluster.Spec)
|
|
template.Spec.RestartPolicy = "" // Make it invalid to create a submitter. Ref: https://github.com/ray-project/kuberay/pull/2389#issuecomment-2359564334
|
|
rayJob.Spec.SubmitterPodTemplate = &template
|
|
|
|
It("Verify RayJob spec", func() {
|
|
Expect(rayJob.Spec.SubmissionMode).To(Equal(rayv1.K8sJobMode))
|
|
})
|
|
|
|
It("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
It("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
})
|
|
|
|
It("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
func() ([]corev1.Event, error) {
|
|
events := &corev1.EventList{}
|
|
if err := k8sClient.List(ctx, events, client.InNamespace(rayJob.Namespace)); err != nil {
|
|
return nil, err
|
|
}
|
|
return events.Items, nil
|
|
},
|
|
time.Second*3, time.Millisecond*500).Should(ContainElement(HaveField("Message", ContainSubstring("Failed to create new Kubernetes Job default/rayjob-invalid-test"))))
|
|
|
|
_ = k8sClient.Delete(ctx, rayJob)
|
|
})
|
|
})
|
|
|
|
Describe("Successful RayJob in K8sjobMode with DELETE_RAYJOB_CR_AFTER_JOB_FINISHES", Ordered, func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-delete", namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
It("Verify RayJob spec", func() {
|
|
// This test case simulates the most common scenario in the RayJob code path.
|
|
// (1) The submission mode is K8sJobMode.
|
|
// (2) `shutdownAfterJobFinishes` is true.
|
|
// In this test, RayJob passes through the following states: New -> Initializing -> Running -> Complete
|
|
Expect(rayJob.Spec.SubmissionMode).To(Equal(rayv1.K8sJobMode))
|
|
Expect(rayJob.Spec.ShutdownAfterJobFinishes).To(BeTrue())
|
|
|
|
// This test assumes that there is only one worker group.
|
|
Expect(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs).To(HaveLen(1))
|
|
})
|
|
|
|
It("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
It("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
It("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Succeeded" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusSucceeded, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusComplete), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
It("If DELETE_RAYJOB_CR_AFTER_JOB_FINISHES environement variable is set, RayJob should be deleted.", func() {
|
|
os.Setenv(utils.DELETE_RAYJOB_CR_AFTER_JOB_FINISHES, "true")
|
|
defer os.Unsetenv(utils.DELETE_RAYJOB_CR_AFTER_JOB_FINISHES)
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob)())
|
|
}, time.Second*3, time.Millisecond*500).Should(BeTrue())
|
|
})
|
|
})
|
|
|
|
Describe("RayJob has passed the ActiveDeadlineSeconds", Ordered, func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
activeDeadlineSeconds := int32(3)
|
|
rayJob := rayJobTemplate("rayjob-deadline", namespace)
|
|
rayJob.Spec.ActiveDeadlineSeconds = ptr.To(activeDeadlineSeconds)
|
|
|
|
It("Verify RayJob spec", func() {
|
|
// In this test, RayJob passes through the following states: New -> Initializing -> Complete (because of ActiveDeadlineSeconds).
|
|
Expect(rayJob.Spec.ActiveDeadlineSeconds).NotTo(BeNil())
|
|
|
|
// This test assumes that there is only one worker group.
|
|
Expect(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs).To(HaveLen(1))
|
|
})
|
|
|
|
It("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
It("RayJobs has passed the activeDeadlineSeconds, and the JobDeploymentStatus transitions from Initializing to Complete.", func() {
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusFailed), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
Expect(rayJob.Status.Reason).To(Equal(rayv1.DeadlineExceeded))
|
|
})
|
|
})
|
|
|
|
Describe("Retrying RayJob in K8sJobMode", Ordered, func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-retry-test", namespace)
|
|
rayJob.Spec.BackoffLimit = ptr.To[int32](1)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
It("Verify RayJob spec", func() {
|
|
// This test case simulates a retry scenario in the RayJob when:
|
|
// (1) The submission mode is K8sJobMode.
|
|
// (2) backoffLimit > 0
|
|
// In this test, RayJob passes through the following states: New -> Initializing -> Running -> Retrying -> New
|
|
Expect(rayJob.Spec.SubmissionMode).To(Equal(rayv1.K8sJobMode))
|
|
Expect(*rayJob.Spec.BackoffLimit).To(Equal(int32(1)))
|
|
})
|
|
|
|
It("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
It("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
It("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from Running -> Retrying -> New -> Initializing", func() {
|
|
// Update fake dashboard client to return job info with "Failed" status.
|
|
//nolint:unparam // this is a mock and the function signature cannot change
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) {
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusFailed, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// record the current cluster name
|
|
oldClusterName := rayJob.Status.RayClusterName
|
|
|
|
// RayJob transitions from Running -> Retrying -> New -> Initializing
|
|
// We only check the final state "Initializing" because it's difficult to test transient states like "Retrying" and "New"
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// validate the RayCluster is deleted on retry
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(getResourceFunc(ctx, client.ObjectKey{Name: oldClusterName, Namespace: namespace}, rayCluster)())
|
|
},
|
|
time.Second*3, time.Millisecond*500).Should(BeTrue())
|
|
|
|
// validate the submitter Job is deleted on retry
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(getResourceFunc(ctx, common.RayJobK8sJobNamespacedName(rayJob), job)())
|
|
},
|
|
time.Second*3, time.Millisecond*500).Should(BeTrue())
|
|
})
|
|
|
|
It("In Initializing state, the RayCluster should eventually be created (attempt 2)", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
It("Make RayCluster.Status.State to be rayv1.Ready (attempt 2)", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from Initializing to Running (attempt 2)", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from Running -> Complete (attempt 2)", func() {
|
|
// Update fake dashboard client to return job info with "Failed" status.
|
|
//nolint:unparam // this is a mock and the function signature cannot change
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) {
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusSucceeded, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions from Running -> Complete
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusComplete), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
It("Validate RayJob succeeded and failed status", func() {
|
|
Eventually(
|
|
getRayJobSucceededStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(int32(1)), "succeeded = %v", rayJob.Status.Succeeded)
|
|
|
|
Eventually(
|
|
getRayJobFailedStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(int32(1)), "failed = %v", rayJob.Status.Failed)
|
|
})
|
|
})
|
|
})
|
|
|
|
Context("RayJob in InteractiveMode", func() {
|
|
Describe("Successful RayJob", Ordered, func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-none-mode", namespace)
|
|
rayJob.Spec.SubmissionMode = rayv1.InteractiveMode
|
|
rayCluster := &rayv1.RayCluster{}
|
|
testRayJobId := "fake-id"
|
|
|
|
It("Verify RayJob spec", func() {
|
|
Expect(rayJob.Spec.SubmissionMode).To(Equal(rayv1.InteractiveMode))
|
|
Expect(rayJob.Spec.ShutdownAfterJobFinishes).To(BeTrue())
|
|
Expect(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs).To(HaveLen(1))
|
|
})
|
|
|
|
It("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).To(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
It("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
It("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from Initializing to Waiting.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusWaiting), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
It("sets jobId in RayJob", func() {
|
|
err := setJobIdOnRayJob(ctx, rayJob, testRayJobId)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
})
|
|
|
|
It("RayJobs's JobDeploymentStatus transitions from Waiting to Running if annotation is set.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
It("should set RayJob's JobId to the value of the annotation", func() {
|
|
Expect(rayJob.Status.JobId).To(Equal(testRayJobId))
|
|
})
|
|
})
|
|
})
|
|
|
|
Describe("RayJob with DeletionStrategy", Ordered, func() {
|
|
JustBeforeEach(func() {
|
|
features.SetFeatureGateDuringTest(GinkgoTB(), features.RayJobDeletionPolicy, true)
|
|
})
|
|
|
|
It("Delete cluster on success", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-deletecluster-on-success", namespace)
|
|
|
|
onSuccessPolicy := rayv1.DeleteCluster
|
|
onFailurePolicy := rayv1.DeleteNone
|
|
deletionStrategy := &rayv1.DeletionStrategy{
|
|
OnSuccess: &rayv1.DeletionPolicy{
|
|
Policy: &onSuccessPolicy,
|
|
},
|
|
OnFailure: &rayv1.DeletionPolicy{
|
|
Policy: &onFailurePolicy,
|
|
},
|
|
}
|
|
rayJob.Spec.DeletionStrategy = deletionStrategy
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
OnSuccess: &rayv1.DeletionPolicy{
|
|
Policy: &onSuccessPolicy,
|
|
},
|
|
OnFailure: &rayv1.DeletionPolicy{
|
|
Policy: &onFailurePolicy,
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Succeeded" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusSucceeded, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusComplete), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteCluster, RayCluster should be deleted, but not the submitter Job.", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster)())
|
|
},
|
|
time.Second*3, time.Millisecond*500).Should(BeTrue())
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
})
|
|
|
|
It("Delete cluster on failure", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-deletecluster-on-failure", namespace)
|
|
|
|
onSuccessPolicy := rayv1.DeleteNone
|
|
onFailurePolicy := rayv1.DeleteCluster
|
|
deletionStrategy := &rayv1.DeletionStrategy{
|
|
OnSuccess: &rayv1.DeletionPolicy{
|
|
Policy: &onSuccessPolicy,
|
|
},
|
|
OnFailure: &rayv1.DeletionPolicy{
|
|
Policy: &onFailurePolicy,
|
|
},
|
|
}
|
|
rayJob.Spec.DeletionStrategy = deletionStrategy
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
OnSuccess: &rayv1.DeletionPolicy{
|
|
Policy: &onSuccessPolicy,
|
|
},
|
|
OnFailure: &rayv1.DeletionPolicy{
|
|
Policy: &onFailurePolicy,
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Failed" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusFailed, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusFailed), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteCluster, RayCluster should be deleted, but not the submitter Job.", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster)())
|
|
},
|
|
time.Second*3, time.Millisecond*500).Should(BeTrue())
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
})
|
|
|
|
It("Delete workers on success", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-deleteworkers-on-success", namespace)
|
|
|
|
onSuccessPolicy := rayv1.DeleteWorkers
|
|
onFailurePolicy := rayv1.DeleteNone
|
|
deletionStrategy := rayv1.DeletionStrategy{
|
|
OnSuccess: &rayv1.DeletionPolicy{
|
|
Policy: &onSuccessPolicy,
|
|
},
|
|
OnFailure: &rayv1.DeletionPolicy{
|
|
Policy: &onFailurePolicy,
|
|
},
|
|
}
|
|
rayJob.Spec.DeletionStrategy = &deletionStrategy
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
OnSuccess: &rayv1.DeletionPolicy{
|
|
Policy: &onSuccessPolicy,
|
|
},
|
|
OnFailure: &rayv1.DeletionPolicy{
|
|
Policy: &onFailurePolicy,
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Succeeded" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusSucceeded, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusComplete), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteWorkers, all workers should be deleted, but not the Head pod and submitter Job", func() {
|
|
// RayCluster exists
|
|
Consistently(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check worker group is suspended
|
|
Expect(*rayCluster.Spec.WorkerGroupSpecs[0].Suspend).To(BeTrue())
|
|
|
|
// 0 worker Pods exist
|
|
workerPods := corev1.PodList{}
|
|
workerLabels := common.RayClusterWorkerPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Eventually(
|
|
listResourceFunc(ctx, &workerPods, workerLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(0), "expected 0 workers")
|
|
|
|
// Head Pod is still running
|
|
headPods := corev1.PodList{}
|
|
headLabels := common.RayClusterHeadPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Consistently(
|
|
listResourceFunc(ctx, &headPods, headLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(1), "Head pod list should have only 1 Pod = %v", headPods.Items)
|
|
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
})
|
|
|
|
It("Delete workers on failure", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-deleteworkers-on-failure", namespace)
|
|
|
|
onSuccessPolicy := rayv1.DeleteWorkers
|
|
onFailurePolicy := rayv1.DeleteWorkers
|
|
deletionStrategy := rayv1.DeletionStrategy{
|
|
OnSuccess: &rayv1.DeletionPolicy{
|
|
Policy: &onSuccessPolicy,
|
|
},
|
|
OnFailure: &rayv1.DeletionPolicy{
|
|
Policy: &onFailurePolicy,
|
|
},
|
|
}
|
|
rayJob.Spec.DeletionStrategy = &deletionStrategy
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
OnSuccess: &rayv1.DeletionPolicy{
|
|
Policy: &onSuccessPolicy,
|
|
},
|
|
OnFailure: &rayv1.DeletionPolicy{
|
|
Policy: &onFailurePolicy,
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Failed" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusFailed, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusFailed), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteWorkers, all workers should be deleted, but not the Head pod and submitter Job", func() {
|
|
// RayCluster exists
|
|
Consistently(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check worker group is suspended
|
|
Expect(*rayCluster.Spec.WorkerGroupSpecs[0].Suspend).To(BeTrue())
|
|
|
|
// 0 worker Pods exist
|
|
workerPods := corev1.PodList{}
|
|
workerLabels := common.RayClusterWorkerPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Eventually(
|
|
listResourceFunc(ctx, &workerPods, workerLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(0), "expected 0 workers")
|
|
|
|
// Head Pod is still running
|
|
headPods := corev1.PodList{}
|
|
headLabels := common.RayClusterHeadPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Consistently(
|
|
listResourceFunc(ctx, &headPods, headLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(1), "Head pod list should have only 1 Pod = %v", headPods.Items)
|
|
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
})
|
|
|
|
It("Delete self on success", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-deleteself-on-success", namespace)
|
|
|
|
onSuccessPolicy := rayv1.DeleteSelf
|
|
onFailurePolicy := rayv1.DeleteNone
|
|
deletionStrategy := rayv1.DeletionStrategy{
|
|
OnSuccess: &rayv1.DeletionPolicy{
|
|
Policy: &onSuccessPolicy,
|
|
},
|
|
OnFailure: &rayv1.DeletionPolicy{
|
|
Policy: &onFailurePolicy,
|
|
},
|
|
}
|
|
rayJob.Spec.DeletionStrategy = &deletionStrategy
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Succeeded" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusSucceeded, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteSelf, the RayJob is deleted", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob))
|
|
}, time.Second*5, time.Millisecond*500).Should(BeTrue())
|
|
})
|
|
})
|
|
|
|
It("Delete self on failure", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-deleteself-on-failure", namespace)
|
|
|
|
onSuccessPolicy := rayv1.DeleteNone
|
|
onFailurePolicy := rayv1.DeleteSelf
|
|
deletionStrategy := rayv1.DeletionStrategy{
|
|
OnSuccess: &rayv1.DeletionPolicy{
|
|
Policy: &onSuccessPolicy,
|
|
},
|
|
OnFailure: &rayv1.DeletionPolicy{
|
|
Policy: &onFailurePolicy,
|
|
},
|
|
}
|
|
rayJob.Spec.DeletionStrategy = &deletionStrategy
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Failed" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusFailed, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteSelf, the RayJob is deleted", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob))
|
|
}, time.Second*5, time.Millisecond*500).Should(BeTrue())
|
|
})
|
|
})
|
|
|
|
It("Delete none on success", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-deletenone-on-success", namespace)
|
|
|
|
onSuccessPolicy := rayv1.DeleteNone
|
|
onFailurePolicy := rayv1.DeleteNone
|
|
deletionStrategy := rayv1.DeletionStrategy{
|
|
OnSuccess: &rayv1.DeletionPolicy{
|
|
Policy: &onSuccessPolicy,
|
|
},
|
|
OnFailure: &rayv1.DeletionPolicy{
|
|
Policy: &onFailurePolicy,
|
|
},
|
|
}
|
|
rayJob.Spec.DeletionStrategy = &deletionStrategy
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
OnSuccess: &rayv1.DeletionPolicy{
|
|
Policy: &onSuccessPolicy,
|
|
},
|
|
OnFailure: &rayv1.DeletionPolicy{
|
|
Policy: &onFailurePolicy,
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Succeeded" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusSucceeded, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusComplete), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteNone, no resources are deleted", func() {
|
|
// RayJob exists
|
|
Consistently(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayJob %v not found", rayJob)
|
|
|
|
// RayCluster exists
|
|
Consistently(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Worker replicas set to 3
|
|
Expect(*rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(int32(3)))
|
|
|
|
// 3 worker Pods exist
|
|
workerPods := corev1.PodList{}
|
|
workerLabels := common.RayClusterWorkerPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Consistently(
|
|
listResourceFunc(ctx, &workerPods, workerLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(3), "expected 3 workers")
|
|
|
|
// Head Pod is still running
|
|
headPods := corev1.PodList{}
|
|
headLabels := common.RayClusterHeadPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Consistently(
|
|
listResourceFunc(ctx, &headPods, headLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(1), "Head pod list should have only 1 Pod = %v", headPods.Items)
|
|
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
})
|
|
|
|
It("Delete none on failure", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-deletenone-on-failure", namespace)
|
|
|
|
onSuccessPolicy := rayv1.DeleteCluster
|
|
onFailurePolicy := rayv1.DeleteNone
|
|
deletionStrategy := rayv1.DeletionStrategy{
|
|
OnSuccess: &rayv1.DeletionPolicy{
|
|
Policy: &onSuccessPolicy,
|
|
},
|
|
OnFailure: &rayv1.DeletionPolicy{
|
|
Policy: &onFailurePolicy,
|
|
},
|
|
}
|
|
rayJob.Spec.DeletionStrategy = &deletionStrategy
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
OnSuccess: &rayv1.DeletionPolicy{
|
|
Policy: &onSuccessPolicy,
|
|
},
|
|
OnFailure: &rayv1.DeletionPolicy{
|
|
Policy: &onFailurePolicy,
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Failed" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusFailed, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusFailed), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteNone, no resources are deleted", func() {
|
|
// RayJob exists
|
|
Consistently(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayJob %v not found", rayJob)
|
|
|
|
// RayCluster exists
|
|
Consistently(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Worker replicas set to 3
|
|
Expect(*rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(int32(3)))
|
|
|
|
// 3 worker Pods exist
|
|
workerPods := corev1.PodList{}
|
|
workerLabels := common.RayClusterWorkerPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Consistently(
|
|
listResourceFunc(ctx, &workerPods, workerLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(3), "expected 3 workers")
|
|
|
|
// Head Pod is still running
|
|
headPods := corev1.PodList{}
|
|
headLabels := common.RayClusterHeadPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Consistently(
|
|
listResourceFunc(ctx, &headPods, headLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(1), "Head pod list should have only 1 Pod = %v", headPods.Items)
|
|
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
})
|
|
|
|
It("Should delete workers on success when a single 'DeleteWorkers' rule is set", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-rule-deleteworkers-on-success", namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
rayJob.Spec.DeletionStrategy = &rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteWorkers,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteWorkers,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
},
|
|
},
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Succeeded" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusSucceeded, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusComplete), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteWorkers, all workers should be deleted, but not the Head pod and submitter Job", func() {
|
|
// RayCluster exists
|
|
Consistently(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check worker group is suspended
|
|
Expect(*rayCluster.Spec.WorkerGroupSpecs[0].Suspend).To(BeTrue())
|
|
|
|
// 0 worker Pods exist
|
|
workerPods := corev1.PodList{}
|
|
workerLabels := common.RayClusterWorkerPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Eventually(
|
|
listResourceFunc(ctx, &workerPods, workerLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(0), "expected 0 workers")
|
|
|
|
// Head Pod is still running
|
|
headPods := corev1.PodList{}
|
|
headLabels := common.RayClusterHeadPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Consistently(
|
|
listResourceFunc(ctx, &headPods, headLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(1), "Head pod list should have only 1 Pod = %v", headPods.Items)
|
|
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
})
|
|
|
|
It("Should delete workers on failure when a single 'DeleteWorkers' rule is set", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-rule-deleteworkers-on-failure", namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
rayJob.Spec.DeletionStrategy = &rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteWorkers,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteWorkers,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
},
|
|
},
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Failed" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusFailed, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Failed.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusFailed), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteWorkers, all workers should be deleted, but not the Head pod and submitter Job", func() {
|
|
// RayCluster exists
|
|
Consistently(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check worker group is suspended
|
|
Expect(*rayCluster.Spec.WorkerGroupSpecs[0].Suspend).To(BeTrue())
|
|
|
|
// 0 worker Pods exist
|
|
workerPods := corev1.PodList{}
|
|
workerLabels := common.RayClusterWorkerPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Eventually(
|
|
listResourceFunc(ctx, &workerPods, workerLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(0), "expected 0 workers")
|
|
|
|
// Head Pod is still running
|
|
headPods := corev1.PodList{}
|
|
headLabels := common.RayClusterHeadPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Consistently(
|
|
listResourceFunc(ctx, &headPods, headLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(1), "Head pod list should have only 1 Pod = %v", headPods.Items)
|
|
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
})
|
|
|
|
It("Should delete cluster on success when a single 'DeleteCluster' rule is set", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-rule-deletecluster-on-success", namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
rayJob.Spec.DeletionStrategy = &rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteCluster,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteCluster,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
},
|
|
},
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Succeeded" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusSucceeded, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusComplete), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteCluster, RayCluster should be deleted, but not the submitter Job.", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster)())
|
|
},
|
|
time.Second*3, time.Millisecond*500).Should(BeTrue())
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
})
|
|
|
|
It("Should delete cluster on failure when a single 'DeleteCluster' rule is set", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-rule-deletecluster-on-failure", namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
rayJob.Spec.DeletionStrategy = &rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteCluster,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteCluster,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
},
|
|
},
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Failed" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusFailed, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusFailed), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteCluster, RayCluster should be deleted, but not the submitter Job.", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster)())
|
|
},
|
|
time.Second*3, time.Millisecond*500).Should(BeTrue())
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
})
|
|
|
|
It("Should delete self on success when a single 'DeleteSelf' rule is set", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-rule-deleteself-on-success", namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
rayJob.Spec.DeletionStrategy = &rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteSelf,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Succeeded" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusSucceeded, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteSelf, the RayJob is deleted", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob))
|
|
}, time.Second*5, time.Millisecond*500).Should(BeTrue())
|
|
})
|
|
})
|
|
|
|
It("Should delete self on failure when a single 'DeleteSelf' rule is set", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-rule-deleteself-on-failure", namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
rayJob.Spec.DeletionStrategy = &rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteSelf,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Failed" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusFailed, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteSelf, the RayJob is deleted", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob))
|
|
}, time.Second*5, time.Millisecond*500).Should(BeTrue())
|
|
})
|
|
})
|
|
|
|
It("Should delete none on success when a single 'DeleteNone' rule is set", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-rule-deletenone-on-success", namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
rayJob.Spec.DeletionStrategy = &rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteNone,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteNone,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
},
|
|
},
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Succeeded" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusSucceeded, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusComplete), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteNone, no resources are deleted", func() {
|
|
// RayJob exists
|
|
Consistently(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayJob %v not found", rayJob)
|
|
|
|
// RayCluster exists
|
|
Consistently(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Worker replicas set to 3
|
|
Expect(*rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(int32(3)))
|
|
|
|
// 3 worker Pods exist
|
|
workerPods := corev1.PodList{}
|
|
workerLabels := common.RayClusterWorkerPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Consistently(
|
|
listResourceFunc(ctx, &workerPods, workerLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(3), "expected 3 workers")
|
|
|
|
// Head Pod is still running
|
|
headPods := corev1.PodList{}
|
|
headLabels := common.RayClusterHeadPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Consistently(
|
|
listResourceFunc(ctx, &headPods, headLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(1), "Head pod list should have only 1 Pod = %v", headPods.Items)
|
|
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
})
|
|
|
|
It("Should delete none on failure when a single 'DeleteNone' rule is set", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-rule-deletenone-on-failure", namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
rayJob.Spec.DeletionStrategy = &rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteNone,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteNone,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
},
|
|
},
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Failed" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusFailed, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusFailed), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
By("If DeletionStrategy=DeleteNone, no resources are deleted", func() {
|
|
// RayJob exists
|
|
Consistently(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayJob %v not found", rayJob)
|
|
|
|
// RayCluster exists
|
|
Consistently(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Worker replicas set to 3
|
|
Expect(*rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(int32(3)))
|
|
|
|
// 3 worker Pods exist
|
|
workerPods := corev1.PodList{}
|
|
workerLabels := common.RayClusterWorkerPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Consistently(
|
|
listResourceFunc(ctx, &workerPods, workerLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(3), "expected 3 workers")
|
|
|
|
// Head Pod is still running
|
|
headPods := corev1.PodList{}
|
|
headLabels := common.RayClusterHeadPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Consistently(
|
|
listResourceFunc(ctx, &headPods, headLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(1), "Head pod list should have only 1 Pod = %v", headPods.Items)
|
|
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
})
|
|
|
|
It("Should execute MOST impactful rule (DeleteSelf) when all rules are overdue on success", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-impactful-rule-override-on-success", namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
// Define the multi-stage DeletionStrategy
|
|
rayJob.Spec.DeletionStrategy = &rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteWorkers,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
TTLSeconds: 0,
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteCluster,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
TTLSeconds: 0,
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteSelf,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
TTLSeconds: 0,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteWorkers,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
TTLSeconds: 0,
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteCluster,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
TTLSeconds: 0,
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteSelf,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
TTLSeconds: 0,
|
|
},
|
|
},
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource with multi-stage deletion rules", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Succeeded" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusSucceeded, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
})
|
|
|
|
By("Verify RayJob itself is deleted", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob))
|
|
}, time.Second*5, time.Millisecond*500).Should(BeTrue())
|
|
})
|
|
})
|
|
|
|
It("Should execute MOST impactful rule (DeleteSelf) when all rules are overdue on failure", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-impactful-rule-override-on-failure", namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
// Define the multi-stage DeletionStrategy
|
|
rayJob.Spec.DeletionStrategy = &rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteWorkers,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
TTLSeconds: 0,
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteCluster,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
TTLSeconds: 0,
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteSelf,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
TTLSeconds: 0,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteWorkers,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
TTLSeconds: 0,
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteCluster,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
TTLSeconds: 0,
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteSelf,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
TTLSeconds: 0,
|
|
},
|
|
},
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource with multi-stage deletion rules", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Failed" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusFailed, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
})
|
|
|
|
By("Verify RayJob itself is deleted", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob))
|
|
}, time.Second*5, time.Millisecond*500).Should(BeTrue())
|
|
})
|
|
})
|
|
|
|
It("Should process multi-stage deletions in order on success: Workers, then Cluster, then Self", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-multistage-deletion-on-success", namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
// Define the multi-stage DeletionStrategy
|
|
rayJob.Spec.DeletionStrategy = &rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteWorkers,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
TTLSeconds: 0, // Stage 1: Delete workers after 0 seconds
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteCluster,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
TTLSeconds: 5, // Stage 2: Delete cluster after 5 seconds
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteSelf,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
TTLSeconds: 10, // Stage 3: Delete self after 10 seconds
|
|
},
|
|
},
|
|
},
|
|
}
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteWorkers,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
TTLSeconds: 0,
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteCluster,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
TTLSeconds: 5,
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteSelf,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusSucceeded,
|
|
TTLSeconds: 10,
|
|
},
|
|
},
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource with multi-stage deletion rules", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Succeeded" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusSucceeded, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Complete.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusComplete), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
By("Stage 1: Verify workers are deleted, but cluster and job still exist", func() {
|
|
// RayCluster exists
|
|
Consistently(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check worker group is suspended
|
|
Expect(*rayCluster.Spec.WorkerGroupSpecs[0].Suspend).To(BeTrue())
|
|
|
|
// 0 worker Pods exist
|
|
workerPods := corev1.PodList{}
|
|
workerLabels := common.RayClusterWorkerPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Eventually(
|
|
listResourceFunc(ctx, &workerPods, workerLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(0), "expected 0 workers")
|
|
|
|
// Head Pod is still running
|
|
headPods := corev1.PodList{}
|
|
headLabels := common.RayClusterHeadPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Consistently(
|
|
listResourceFunc(ctx, &headPods, headLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(1), "Head pod list should have only 1 Pod = %v", headPods.Items)
|
|
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
|
|
By("Stage 2 (after 5s): Verify RayCluster is deleted, but job still exists", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster)())
|
|
},
|
|
time.Second*3, time.Millisecond*500).Should(BeTrue())
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
|
|
By("Stage 3 (after 10s): Verify RayJob itself is deleted", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob))
|
|
}, time.Second*5, time.Millisecond*500).Should(BeTrue())
|
|
})
|
|
})
|
|
|
|
It("Should process multi-stage deletions in order on failure: Workers, then Cluster, then Self", func() {
|
|
ctx := context.Background()
|
|
namespace := "default"
|
|
rayJob := rayJobTemplate("rayjob-test-multistage-deletion-on-failure", namespace)
|
|
rayCluster := &rayv1.RayCluster{}
|
|
|
|
// Define the multi-stage DeletionStrategy
|
|
rayJob.Spec.DeletionStrategy = &rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteWorkers,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
TTLSeconds: 0, // Stage 1: Delete workers after 0 seconds
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteCluster,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
TTLSeconds: 5, // Stage 2: Delete cluster after 5 seconds
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteSelf,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
TTLSeconds: 10, // Stage 3: Delete self after 10 seconds
|
|
},
|
|
},
|
|
},
|
|
}
|
|
rayJob.Spec.ShutdownAfterJobFinishes = false
|
|
|
|
By("Verify RayJob spec", func() {
|
|
Expect(*rayJob.Spec.DeletionStrategy).To(Equal(rayv1.DeletionStrategy{
|
|
DeletionRules: []rayv1.DeletionRule{
|
|
{
|
|
Policy: rayv1.DeleteWorkers,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
TTLSeconds: 0,
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteCluster,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
TTLSeconds: 5,
|
|
},
|
|
},
|
|
{
|
|
Policy: rayv1.DeleteSelf,
|
|
Condition: rayv1.DeletionCondition{
|
|
JobStatus: rayv1.JobStatusFailed,
|
|
TTLSeconds: 10,
|
|
},
|
|
},
|
|
},
|
|
}))
|
|
})
|
|
|
|
By("Create a RayJob custom resource with multi-stage deletion rules", func() {
|
|
err := k8sClient.Create(ctx, rayJob)
|
|
Expect(err).NotTo(HaveOccurred(), "Failed to create RayJob")
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "Should be able to see RayJob: %v", rayJob.Name)
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from New to Initializing.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusInitializing), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Initializing state, Status.RayClusterName, Status.JobId, and Status.StartTime must be set.
|
|
Expect(rayJob.Status.RayClusterName).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.JobId).NotTo(BeEmpty())
|
|
Expect(rayJob.Status.StartTime).NotTo(BeNil())
|
|
})
|
|
|
|
By("In Initializing state, the RayCluster should eventually be created.", func() {
|
|
Eventually(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check whether RayCluster is consistent with RayJob's RayClusterSpec.
|
|
Expect(rayCluster.Spec.WorkerGroupSpecs[0].Replicas).To(Equal(rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Replicas))
|
|
Expect(rayCluster.Spec.RayVersion).To(Equal(rayJob.Spec.RayClusterSpec.RayVersion))
|
|
|
|
// TODO (kevin85421): Check the RayCluster labels.
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRNameLabelKey, rayJob.Name))
|
|
Expect(rayCluster.Labels).Should(HaveKeyWithValue(utils.RayOriginatedFromCRDLabelKey, utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD)))
|
|
|
|
Expect(rayCluster.Annotations).Should(Equal(rayJob.Annotations))
|
|
})
|
|
|
|
By("Make RayCluster.Status.State to be rayv1.Ready", func() {
|
|
// The RayCluster is not 'Ready' yet because Pods are not running and ready.
|
|
Expect(rayCluster.Status.State).NotTo(Equal(rayv1.Ready))
|
|
|
|
updateHeadPodToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
updateWorkerPodsToRunningAndReady(ctx, rayJob.Status.RayClusterName, namespace)
|
|
|
|
// The RayCluster.Status.State should be Ready.
|
|
Eventually(
|
|
getClusterState(ctx, namespace, rayCluster.Name),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.Ready))
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Initializing to Running.", func() {
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// In Running state, the RayJob's Status.DashboardURL must be set.
|
|
Expect(rayJob.Status.DashboardURL).NotTo(BeEmpty())
|
|
|
|
// In Running state, the submitter Kubernetes Job must be created if this RayJob is in K8sJobMode.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
})
|
|
|
|
By("RayJobs's JobDeploymentStatus transitions from Running to Complete.", func() {
|
|
// Update fake dashboard client to return job info with "Failed" status.
|
|
getJobInfo := func(context.Context, string) (*utiltypes.RayJobInfo, error) { //nolint:unparam // This is a mock function so parameters are required
|
|
return &utiltypes.RayJobInfo{JobStatus: rayv1.JobStatusFailed, EndTime: uint64(time.Now().UnixMilli())}, nil
|
|
}
|
|
fakeRayDashboardClient.GetJobInfoMock.Store(&getJobInfo)
|
|
defer fakeRayDashboardClient.GetJobInfoMock.Store(nil)
|
|
|
|
// RayJob transitions to Complete if and only if the corresponding submitter Kubernetes Job is Complete or Failed.
|
|
Consistently(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusRunning), "JobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
err := k8sClient.Get(ctx, namespacedName, job)
|
|
Expect(err).NotTo(HaveOccurred(), "failed to get Kubernetes Job")
|
|
|
|
// Update the submitter Kubernetes Job to Complete.
|
|
conditions := []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Now()},
|
|
}
|
|
job.Status.Conditions = conditions
|
|
Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())
|
|
|
|
// RayJob transitions to Failed.
|
|
Eventually(
|
|
getRayJobDeploymentStatus(ctx, rayJob),
|
|
time.Second*5, time.Millisecond*500).Should(Equal(rayv1.JobDeploymentStatusFailed), "jobDeploymentStatus = %v", rayJob.Status.JobDeploymentStatus)
|
|
})
|
|
|
|
By("Stage 1: Verify workers are deleted, but cluster and job still exist", func() {
|
|
// RayCluster exists
|
|
Consistently(
|
|
getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed(), "RayCluster %v not found", rayJob.Status.RayClusterName)
|
|
|
|
// Check worker group is suspended
|
|
Expect(*rayCluster.Spec.WorkerGroupSpecs[0].Suspend).To(BeTrue())
|
|
|
|
// 0 worker Pods exist
|
|
workerPods := corev1.PodList{}
|
|
workerLabels := common.RayClusterWorkerPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Eventually(
|
|
listResourceFunc(ctx, &workerPods, workerLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(0), "expected 0 workers")
|
|
|
|
// Head Pod is still running
|
|
headPods := corev1.PodList{}
|
|
headLabels := common.RayClusterHeadPodsAssociationOptions(rayCluster).ToListOptions()
|
|
Consistently(
|
|
listResourceFunc(ctx, &headPods, headLabels...),
|
|
time.Second*3, time.Millisecond*500).Should(Equal(1), "Head pod list should have only 1 Pod = %v", headPods.Items)
|
|
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
|
|
By("Stage 2 (after 5s): Verify RayCluster is deleted, but job still exists", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(getResourceFunc(ctx, client.ObjectKey{Name: rayJob.Status.RayClusterName, Namespace: namespace}, rayCluster)())
|
|
},
|
|
time.Second*3, time.Millisecond*500).Should(BeTrue())
|
|
namespacedName := common.RayJobK8sJobNamespacedName(rayJob)
|
|
job := &batchv1.Job{}
|
|
Consistently(
|
|
getResourceFunc(ctx, namespacedName, job),
|
|
time.Second*3, time.Millisecond*500).Should(Succeed())
|
|
})
|
|
|
|
By("Stage 3 (after 10s): Verify RayJob itself is deleted", func() {
|
|
Eventually(
|
|
func() bool {
|
|
return apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKey{Name: rayJob.Name, Namespace: namespace}, rayJob))
|
|
}, time.Second*5, time.Millisecond*500).Should(BeTrue())
|
|
})
|
|
})
|
|
})
|
|
})
|