mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
Native Kubernetes Jobs have a suspend flag that allows to temporarily suspend a Job execution and resume it later, or start Jobs in a suspended state and have a custom controller, such as Kueue, decide later when to start them. So adding it to RayJob spec for consistency. Moreover, some frameworks like Kubeflow are adding it, so it becomes a standard functionality. An example implementation for MPIJob: kubeflow/mpi-operator#511 Implementation details If a RayJob is created with a spec.suspend == true, then RayCluster instance (with corresponding Kubernetes resources) is not created and the Ray job is not submitted to the cluster. The JobDeploymentStatus is set to Suspended and the corresponding event is issued. The RayJob remains in this state until somebody unsuspends the job. If suspend flips from true to false, then the RayJob controller immediately creates a RayCluster instance and submits the job. If suspend flips from false to true while Job is running, then the RayJob controller tries to gracefully stop the job and deletes the RayCluster instance (with underlying Kubernetes resources). The JobDeploymentStatus is set to Suspended; JobStatus is set to STOPPED and the corresponding event is issued. Edge case: suspend flag is ignored if a RayJob is submitted against an existing RayCluster instance (matched with ClusterSelector) since we can't delete a RayCluster created by somebody else. No Kueue-specific code leaked to Kuberay implementation Contributors from Kueue/Kubernetes cc'ed: @alculquicondor @mwielgus
142 lines
4.9 KiB
Go
142 lines
4.9 KiB
Go
package utils
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"net/http"
|
|
|
|
"github.com/jarcoal/httpmock"
|
|
. "github.com/onsi/ginkgo"
|
|
. "github.com/onsi/gomega"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
|
|
rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1"
|
|
)
|
|
|
|
const (
|
|
runtimeEnvStr = "{\n \"working_dir\": \"./\",\n \"pip\": [\n \"requests==2.26.0\",\n \"pendulum==2.1.2\"\n ],\n \"conda\": {\n \"dependencies\": [\n \"pytorch\",\n \"torchvision\",\n \"pip\",\n {\n \"pip\": [\n \"pendulum\"\n ]\n }\n ]\n },\n \"eager_install\": false\n}"
|
|
)
|
|
|
|
var _ = Describe("RayFrameworkGenerator", func() {
|
|
var (
|
|
rayJob *rayv1alpha1.RayJob
|
|
expectJobId string
|
|
errorJobId string
|
|
rayDashboardClient *RayDashboardClient
|
|
)
|
|
|
|
BeforeEach(func() {
|
|
expectJobId = "raysubmit_test001"
|
|
encodedRuntimeEnv := base64.StdEncoding.EncodeToString([]byte(runtimeEnvStr))
|
|
rayJob = &rayv1alpha1.RayJob{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "rayjob-sample",
|
|
Namespace: "default",
|
|
},
|
|
Spec: rayv1alpha1.RayJobSpec{
|
|
Entrypoint: "python samply.py",
|
|
Metadata: map[string]string{
|
|
"owner": "test1",
|
|
},
|
|
RuntimeEnv: encodedRuntimeEnv,
|
|
},
|
|
}
|
|
rayDashboardClient = &RayDashboardClient{}
|
|
rayDashboardClient.InitClient("127.0.0.1:8090")
|
|
})
|
|
|
|
It("Test ConvertRayJobToReq", func() {
|
|
rayJobRequest, err := ConvertRayJobToReq(rayJob)
|
|
Expect(err).To(BeNil())
|
|
Expect(len(rayJobRequest.RuntimeEnv)).To(Equal(4))
|
|
Expect(rayJobRequest.RuntimeEnv["working_dir"]).To(Equal("./"))
|
|
})
|
|
|
|
It("Test submitting/getting rayJob", func() {
|
|
httpmock.Activate()
|
|
defer httpmock.DeactivateAndReset()
|
|
httpmock.RegisterResponder("POST", rayDashboardClient.dashboardURL+JobPath,
|
|
func(req *http.Request) (*http.Response, error) {
|
|
body := &RayJobResponse{
|
|
JobId: expectJobId,
|
|
}
|
|
bodyBytes, _ := json.Marshal(body)
|
|
return httpmock.NewBytesResponse(200, bodyBytes), nil
|
|
})
|
|
httpmock.RegisterResponder("GET", rayDashboardClient.dashboardURL+JobPath+expectJobId,
|
|
func(req *http.Request) (*http.Response, error) {
|
|
body := &RayJobInfo{
|
|
JobStatus: rayv1alpha1.JobStatusRunning,
|
|
Entrypoint: rayJob.Spec.Entrypoint,
|
|
Metadata: rayJob.Spec.Metadata,
|
|
}
|
|
bodyBytes, _ := json.Marshal(body)
|
|
return httpmock.NewBytesResponse(200, bodyBytes), nil
|
|
})
|
|
httpmock.RegisterResponder("GET", rayDashboardClient.dashboardURL+JobPath+errorJobId,
|
|
func(req *http.Request) (*http.Response, error) {
|
|
// return a string in the body
|
|
return httpmock.NewStringResponse(200, "Ray misbehaved and sent string, not JSON"), nil
|
|
})
|
|
|
|
jobId, err := rayDashboardClient.SubmitJob(context.TODO(), rayJob, &ctrl.Log)
|
|
Expect(err).To(BeNil())
|
|
Expect(jobId).To(Equal(expectJobId))
|
|
|
|
rayJobInfo, err := rayDashboardClient.GetJobInfo(context.TODO(), jobId)
|
|
Expect(err).To(BeNil())
|
|
Expect(rayJobInfo.Entrypoint).To(Equal(rayJob.Spec.Entrypoint))
|
|
Expect(rayJobInfo.JobStatus).To(Equal(rayv1alpha1.JobStatusRunning))
|
|
|
|
_, err = rayDashboardClient.GetJobInfo(context.TODO(), errorJobId)
|
|
Expect(err).NotTo(BeNil())
|
|
Expect(err.Error()).To(ContainSubstring("GetJobInfo fail"))
|
|
Expect(err.Error()).To(ContainSubstring("Ray misbehaved"))
|
|
})
|
|
|
|
It("Test stop job", func() {
|
|
httpmock.Activate()
|
|
defer httpmock.DeactivateAndReset()
|
|
httpmock.RegisterResponder("POST", rayDashboardClient.dashboardURL+JobPath+"stop-job-1/stop",
|
|
func(req *http.Request) (*http.Response, error) {
|
|
body := &RayJobStopResponse{
|
|
Stopped: true,
|
|
}
|
|
bodyBytes, _ := json.Marshal(body)
|
|
return httpmock.NewBytesResponse(200, bodyBytes), nil
|
|
})
|
|
|
|
err := rayDashboardClient.StopJob(context.TODO(), "stop-job-1", &ctrl.Log)
|
|
Expect(err).To(BeNil())
|
|
})
|
|
|
|
It("Test stop succeeded job", func() {
|
|
// StopJob only returns an error when JobStatus is not in terminated states (STOPPED / SUCCEEDED / FAILED)
|
|
httpmock.Activate()
|
|
defer httpmock.DeactivateAndReset()
|
|
httpmock.RegisterResponder("POST", rayDashboardClient.dashboardURL+JobPath+"stop-job-1/stop",
|
|
func(req *http.Request) (*http.Response, error) {
|
|
body := &RayJobStopResponse{
|
|
Stopped: false,
|
|
}
|
|
bodyBytes, _ := json.Marshal(body)
|
|
return httpmock.NewBytesResponse(200, bodyBytes), nil
|
|
})
|
|
httpmock.RegisterResponder("GET", rayDashboardClient.dashboardURL+JobPath+"stop-job-1",
|
|
func(req *http.Request) (*http.Response, error) {
|
|
body := &RayJobInfo{
|
|
JobStatus: rayv1alpha1.JobStatusSucceeded,
|
|
Entrypoint: rayJob.Spec.Entrypoint,
|
|
Metadata: rayJob.Spec.Metadata,
|
|
}
|
|
bodyBytes, _ := json.Marshal(body)
|
|
return httpmock.NewBytesResponse(200, bodyBytes), nil
|
|
})
|
|
|
|
err := rayDashboardClient.StopJob(context.TODO(), "stop-job-1", &ctrl.Log)
|
|
Expect(err).To(BeNil())
|
|
})
|
|
})
|