mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
+10









Andrew Sy Kim
GitHub
Ping
Han-Ju Chen
Ryan O'Leary
Kai-Hsun Chen
Kavish
Neo Chien
cchung100m
JustinYeh
Jun-Hao Wan
Divyam Raj
Nary Yeh
Alima Azamat
Spencer Peterson
dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Rueian
Cheng-Yeh Chung
fscnick
Copilot
f68857e9ff
* [Bug] Sidecar mode shouldn't restart head pod when head pod is deleted (#4141) * [Bug] Sidecar mode shouldn't restart head pod when head pod is deleted Signed-off-by: 400Ping <[email protected]> * [Fix] Fix e2e error Signed-off-by: 400Ping <[email protected]> * [Fix] fix according to rueian's comment Signed-off-by: 400Ping <[email protected]> * [Chore] fix ci error Signed-off-by: 400Ping <[email protected]> * Update ray-operator/controllers/ray/raycluster_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ping <[email protected]> * Update ray-operator/controllers/ray/rayjob_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ping <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * Trigger CI Signed-off-by: Future-Outlier <[email protected]> --------- Signed-off-by: 400Ping <[email protected]> Signed-off-by: Ping <[email protected]> Signed-off-by: Future-Outlier <[email protected]> Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> * fix: dashboard build for kuberay 1.5.0 (#4161) Signed-off-by: Future-Outlier <[email protected]> * [Feature Enhancement] Set ordered replica index label to support multi-slice (#4163) * [Feature Enhancement] Set ordered replica index label to support multi-slice Signed-off-by: Ryan O'Leary <[email protected]> * rename replica-id -> replica-name Signed-off-by: Ryan O'Leary <[email protected]> * Separate replica index feature gate logic Signed-off-by: Ryan O'Leary <[email protected]> * remove index arg in createWorkerPod Signed-off-by: Ryan O'Leary <[email protected]> --------- Signed-off-by: Ryan O'Leary <[email protected]> * update stale feature gate comments (#4174) Signed-off-by: Andrew Sy Kim <[email protected]> * [RayCluster] Add more context why we don't recreate head Pod for RayJob (#4175) Signed-off-by: Kai-Hsun Chen <[email protected]> * feature: Remove empty resource list initialization. (#4168) Fixes #4142. * [Dockerfile] [KubeRay Dashboard]: Fix Dockerfile warnings (ENV format, CMD JSON args) (#4167) * [#4166] improvement: Fix Dockerfile warnings (ENV format, CMD JSON args) * extract the hostname from CMD Signed-off-by: Neo Chien <[email protected]> --------- Signed-off-by: Neo Chien <[email protected]> Co-authored-by: cchung100m <[email protected]> * [Fix] Resolve int32 overflow by having the calculation in int64 and c… (#4158) * [Fix] Resolve int32 overflow by having the calculation in int64 and cap it if the count is over math.MaxInt32 Signed-off-by: justinyeh1995 <[email protected]> * [Test] Add unit tests for CalculateReadyReplicas Signed-off-by: justinyeh1995 <[email protected]> * [Fix] Add a nosec comment to pass the Lint (pre-commit) test Signed-off-by: justinyeh1995 <[email protected]> * [Refactor] Add CapInt64ToInt32 to replace #nosec directives Signed-off-by: justinyeh1995 <[email protected]> * [Refactor] Rename function to SafeInt64ToInt32 and add a underflowing prevention (it also help pass the lint test) Signed-off-by: justinyeh1995 <[email protected]> * [Refactor] Remove the early return as SafeInt64ToInt32 handles the int32 overflow and underflow checking. Signed-off-by: justinyeh1995 <[email protected]> --------- Signed-off-by: justinyeh1995 <[email protected]> * Add RayService incremental upgrade sample for guide (#4164) Signed-off-by: Ryan O'Leary <[email protected]> * Edit RayCluster example config for label selectors (#4151) Signed-off-by: Ryan O'Leary <[email protected]> * [RayJob] update light weight submitter image from quay.io (#4181) Signed-off-by: Future-Outlier <[email protected]> * [flaky] RayJob fails when head Pod is deleted when job is running (#4182) Signed-off-by: Future-Outlier <[email protected]> * [CI] Pin Docker api version to avoid API version mismatch (#4188) Signed-off-by: win5923 <[email protected]> * Make replicas configurable for kuberay-operator #4180 (#4195) * Make replicas configurable for kuberay-operator #4180 * Make replicas configurable for kuberay-operator #4180 * [Fix] rayjob update raycluster status (#4192) * feat: check if raycluster status update in rayjob * test: e2e test to check the rayjob raycluster status update * fix: dashboard http client tests discovered and passing (#4173) Signed-off-by: alimaazamat <[email protected]> * [RayJob] Lift cluster status while initializing (#4191) Signed-off-by: Spencer Peterson <[email protected]> * [RayJob] Remove updateJobStatus call (#4198) Fast follow to #4191 Signed-off-by: Spencer Peterson <[email protected]> * Add support for Ray token auth (#4179) * Add support for Ray token auth Signed-off-by: Andrew Sy Kim <[email protected]> * add e2e test for Ray cluster auth Signed-off-by: Andrew Sy Kim <[email protected]> * address nits from Ruiean Signed-off-by: Andrew Sy Kim <[email protected]> * update RAY_auth_mode -> RAY_AUTH_MODE Signed-off-by: Andrew Sy Kim <[email protected]> * configure auth for Ray autoscaler Signed-off-by: Andrew Sy Kim <[email protected]> --------- Signed-off-by: Andrew Sy Kim <[email protected]> * Bump js-yaml from 4.1.0 to 4.1.1 in /dashboard (#4194) Bumps [js-yaml](https://github.com/nodeca/js-yaml) from 4.1.0 to 4.1.1. - [Changelog](https://github.com/nodeca/js-yaml/blob/master/CHANGELOG.md) - [Commits](https://github.com/nodeca/js-yaml/compare/4.1.0...4.1.1) --- updated-dependencies: - dependency-name: js-yaml dependency-version: 4.1.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * update minimum Ray version required for token authentication to 2.52.0 (#4201) * update minimum Ray version required for token authentication to 2.52.0 Signed-off-by: Andrew Sy Kim <[email protected]> * update RayCluster auth e2e test to use Ray v2.52 Signed-off-by: Andrew Sy Kim <[email protected]> --------- Signed-off-by: Andrew Sy Kim <[email protected]> * add samples for RayCluster token auth (#4200) Signed-off-by: Andrew Sy Kim <[email protected]> * update (#4208) Signed-off-by: Future-Outlier <[email protected]> * [RayJob] Add token authentication support for All mode (#4210) * dashboard client authentication support Signed-off-by: Future-Outlier <[email protected]> * support rayjob Signed-off-by: Future-Outlier <[email protected]> * update to fix api serverr err Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * updarte Signed-off-by: Future-Outlier <[email protected]> * Rayjob sidecar mode auth token mode support Signed-off-by: Future-Outlier <[email protected]> * RayJob support k8s job mode Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * Address Andrew's advice Signed-off-by: Future-Outlier <[email protected]> * add todo x-ray-authorization comments Signed-off-by: Future-Outlier <[email protected]> --------- Signed-off-by: Future-Outlier <[email protected]> * [RayCluster] Enable Secret informer watch/list and remove unused RBAC verbs (#4202) * Add authentication secret reconciliation support Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * fix flaky test Signed-off-by: Future-Outlier <[email protected]> * remove test fix Signed-off-by: Rueian <[email protected]> --------- Signed-off-by: Future-Outlier <[email protected]> Signed-off-by: Rueian <[email protected]> Co-authored-by: Rueian <[email protected]> * [APIServer][Docs] Add user guide for retry behavior & configuration (#4144) * [Docs] Add the draft description about feature intro, configurations, and usecases Signed-off-by: justinyeh1995 <[email protected]> * [Fix] Update the retry walk-through Signed-off-by: justinyeh1995 <[email protected]> * [Doc] rewrite the first 2 sections Signed-off-by: justinyeh1995 <[email protected]> * [Doc] Revise documentation wording and add Observing Retry Behavior section Signed-off-by: justinyeh1995 <[email protected]> * [Fix] fix linting issue by running pre-commit run berfore commiting Signed-off-by: justinyeh1995 <[email protected]> * [Fix] fix linting errors in the Markdown linting Signed-off-by: justinyeh1995 <[email protected]> * [Fix] Clean up the math equation Signed-off-by: justinyeh1995 <[email protected]> * Update the math formula of Backoff calculation. Co-authored-by: Nary Yeh <[email protected]> Signed-off-by: JustinYeh <[email protected]> * [Fix] Explicitly mentioned exponential backoff and removed the customization parts Signed-off-by: justinyeh1995 <[email protected]> * [Docs] Clarify naming by replacing “APIServer” with “KubeRay APIServer” Co-authored-by: Cheng-Yeh Chung <[email protected]> Signed-off-by: JustinYeh <[email protected]> * [Docs] Rename retry-configuration.md to retry-behavior.md for accuracy Signed-off-by: justinyeh1995 <[email protected]> * Update Title to KubeRay APIServer Retry Behavior Co-authored-by: Cheng-Yeh Chung <[email protected]> Signed-off-by: JustinYeh <[email protected]> * [Docs] Add a note about the limitation of retry configuration Signed-off-by: justinyeh1995 <[email protected]> --------- Signed-off-by: justinyeh1995 <[email protected]> Signed-off-by: JustinYeh <[email protected]> Co-authored-by: Nary Yeh <[email protected]> Co-authored-by: Cheng-Yeh Chung <[email protected]> * Support X-Ray-Authorization fallback header for accepting auth token via proxy (#4213) * Support X-Ray-Authorization fallback header for accepting auth token in dashboard Signed-off-by: Future-Outlier <[email protected]> * remove todo comment Signed-off-by: Future-Outlier <[email protected]> --------- Signed-off-by: Future-Outlier <[email protected]> * [RayCluster] make auth token secret name consistency (#4216) Signed-off-by: fscnick <[email protected]> * [RayCluster] Status includes head containter status message (#4196) * [RayCluster] Status includes head containter status message Signed-off-by: Spencer Peterson <[email protected]> * lint Signed-off-by: Spencer Peterson <[email protected]> * [RayCluster] Containers not ready status reflects structured reason Signed-off-by: Spencer Peterson <[email protected]> * nit Signed-off-by: Spencer Peterson <[email protected]> --------- Signed-off-by: Spencer Peterson <[email protected]> * Remove erroneous call in applyServeTargetCapacity (#4212) Signed-off-by: Ryan O'Leary <[email protected]> * [RayJob] Add token authentication support for light weight job submitter (#4215) * [RayJob] light weight job submitter auth token support Signed-off-by: Future-Outlier <[email protected]> * X-Ray-Authorization Signed-off-by: Rueian <[email protected]> --------- Signed-off-by: Future-Outlier <[email protected]> Signed-off-by: Rueian <[email protected]> Co-authored-by: Rueian <[email protected]> * feat: kubectl ray get token command (#4218) * feat: kubectl ray get token command Signed-off-by: Rueian <[email protected]> * Update kubectl-plugin/pkg/cmd/get/get_token_test.go Co-authored-by: Copilot <[email protected]> Signed-off-by: Rueian <[email protected]> * Update kubectl-plugin/pkg/cmd/get/get_token.go Co-authored-by: Copilot <[email protected]> Signed-off-by: Rueian <[email protected]> * make sure the raycluster exists before getting the secret Signed-off-by: Rueian <[email protected]> * better ux Signed-off-by: Rueian <[email protected]> * Update kubectl-plugin/pkg/cmd/get/get_token.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Rueian <[email protected]> --------- Signed-off-by: Rueian <[email protected]> Co-authored-by: Copilot <[email protected]> Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> --------- Signed-off-by: 400Ping <[email protected]> Signed-off-by: Ping <[email protected]> Signed-off-by: Future-Outlier <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> Signed-off-by: Andrew Sy Kim <[email protected]> Signed-off-by: Kai-Hsun Chen <[email protected]> Signed-off-by: Neo Chien <[email protected]> Signed-off-by: justinyeh1995 <[email protected]> Signed-off-by: win5923 <[email protected]> Signed-off-by: alimaazamat <[email protected]> Signed-off-by: Spencer Peterson <[email protected]> Signed-off-by: dependabot[bot] <[email protected]> Signed-off-by: Rueian <[email protected]> Signed-off-by: JustinYeh <[email protected]> Signed-off-by: fscnick <[email protected]> Co-authored-by: Ping <[email protected]> Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Co-authored-by: Ryan O'Leary <[email protected]> Co-authored-by: Kai-Hsun Chen <[email protected]> Co-authored-by: Kavish <[email protected]> Co-authored-by: Neo Chien <[email protected]> Co-authored-by: cchung100m <[email protected]> Co-authored-by: JustinYeh <[email protected]> Co-authored-by: Jun-Hao Wan <[email protected]> Co-authored-by: Divyam Raj <[email protected]> Co-authored-by: Nary Yeh <[email protected]> Co-authored-by: Alima Azamat <[email protected]> Co-authored-by: Spencer Peterson <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Rueian <[email protected]> Co-authored-by: Cheng-Yeh Chung <[email protected]> Co-authored-by: fscnick <[email protected]> Co-authored-by: Copilot <[email protected]>
390 lines
12 KiB
Go
390 lines
12 KiB
Go
package dashboardclient
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/util/json"
|
|
"k8s.io/apimachinery/pkg/util/yaml"
|
|
|
|
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
|
|
utiltypes "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils/types"
|
|
)
|
|
|
|
var (
|
|
// Multi-application URL paths
|
|
ServeDetailsPath = "/api/serve/applications/"
|
|
APITypeParam = "declarative"
|
|
DeployPathV2 = "/api/serve/applications/"
|
|
// Job URL paths
|
|
JobPath = "/api/jobs/"
|
|
)
|
|
|
|
type RayDashboardClientInterface interface {
|
|
InitClient(client *http.Client, dashboardURL string, authToken string)
|
|
UpdateDeployments(ctx context.Context, configJson []byte) error
|
|
// V2/multi-app Rest API
|
|
GetServeDetails(ctx context.Context) (*utiltypes.ServeDetails, error)
|
|
GetMultiApplicationStatus(context.Context) (map[string]*utiltypes.ServeApplicationStatus, error)
|
|
GetJobInfo(ctx context.Context, jobId string) (*utiltypes.RayJobInfo, error)
|
|
ListJobs(ctx context.Context) (*[]utiltypes.RayJobInfo, error)
|
|
SubmitJob(ctx context.Context, rayJob *rayv1.RayJob) (string, error)
|
|
SubmitJobReq(ctx context.Context, request *utiltypes.RayJobRequest) (string, error)
|
|
GetJobLog(ctx context.Context, jobName string) (*string, error)
|
|
StopJob(ctx context.Context, jobName string) error
|
|
DeleteJob(ctx context.Context, jobName string) error
|
|
}
|
|
|
|
type RayDashboardClient struct {
|
|
client *http.Client
|
|
dashboardURL string
|
|
authToken string
|
|
}
|
|
|
|
func (r *RayDashboardClient) InitClient(client *http.Client, dashboardURL string, authToken string) {
|
|
r.client = client
|
|
r.dashboardURL = dashboardURL
|
|
r.authToken = authToken
|
|
}
|
|
|
|
func (r *RayDashboardClient) setAuthHeader(req *http.Request) {
|
|
if r.authToken != "" {
|
|
req.Header.Set("x-ray-authorization", fmt.Sprintf("Bearer %s", r.authToken))
|
|
}
|
|
}
|
|
|
|
// UpdateDeployments update the deployments in the Ray cluster.
|
|
func (r *RayDashboardClient) UpdateDeployments(ctx context.Context, configJson []byte) error {
|
|
var req *http.Request
|
|
var err error
|
|
if req, err = http.NewRequestWithContext(ctx, http.MethodPut, r.dashboardURL+DeployPathV2, bytes.NewBuffer(configJson)); err != nil {
|
|
return err
|
|
}
|
|
|
|
req.Header.Set("Content-Type", "application/json")
|
|
r.setAuthHeader(req)
|
|
|
|
resp, err := r.client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to read response when updating deployments: %w", err)
|
|
}
|
|
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
|
return fmt.Errorf("UpdateDeployments fail: %s %s", resp.Status, string(body))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (r *RayDashboardClient) GetMultiApplicationStatus(ctx context.Context) (map[string]*utiltypes.ServeApplicationStatus, error) {
|
|
serveDetails, err := r.GetServeDetails(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get serve details: %w", err)
|
|
}
|
|
|
|
return r.ConvertServeDetailsToApplicationStatuses(serveDetails)
|
|
}
|
|
|
|
// GetServeDetails gets details on all declarative applications on the Ray cluster.
|
|
func (r *RayDashboardClient) GetServeDetails(ctx context.Context) (*utiltypes.ServeDetails, error) {
|
|
serveDetailsURL, err := url.Parse(r.dashboardURL + ServeDetailsPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse dashboard URL: %w", err)
|
|
}
|
|
q := serveDetailsURL.Query()
|
|
q.Set("api_type", APITypeParam)
|
|
serveDetailsURL.RawQuery = q.Encode()
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "GET", serveDetailsURL.String(), nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
r.setAuthHeader(req)
|
|
|
|
resp, err := r.client.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read response when getting serve details: %w", err)
|
|
}
|
|
|
|
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
|
return nil, fmt.Errorf("GetServeDetails fail: %s %s", resp.Status, string(body))
|
|
}
|
|
|
|
var serveDetails utiltypes.ServeDetails
|
|
if err = json.Unmarshal(body, &serveDetails); err != nil {
|
|
return nil, fmt.Errorf("GetServeDetails failed. Failed to unmarshal bytes: %s", string(body))
|
|
}
|
|
|
|
return &serveDetails, nil
|
|
}
|
|
|
|
func (r *RayDashboardClient) ConvertServeDetailsToApplicationStatuses(serveDetails *utiltypes.ServeDetails) (map[string]*utiltypes.ServeApplicationStatus, error) {
|
|
detailsJson, err := json.Marshal(serveDetails.Applications)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to marshal serve details: %v", serveDetails.Applications)
|
|
}
|
|
|
|
applicationStatuses := map[string]*utiltypes.ServeApplicationStatus{}
|
|
if err = json.Unmarshal(detailsJson, &applicationStatuses); err != nil {
|
|
return nil, fmt.Errorf("failed to unmarshal serve details bytes into map of application statuses: %w. Bytes: %s", err, string(detailsJson))
|
|
}
|
|
|
|
return applicationStatuses, nil
|
|
}
|
|
|
|
// Note that RayJobInfo and error can't be nil at the same time.
|
|
// Please make sure if the Ray job with JobId can't be found. Return a BadRequest error.
|
|
func (r *RayDashboardClient) GetJobInfo(ctx context.Context, jobId string) (*utiltypes.RayJobInfo, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.dashboardURL+JobPath+jobId, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
r.setAuthHeader(req)
|
|
|
|
resp, err := r.client.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode == http.StatusNotFound {
|
|
return nil, errors.NewBadRequest("Job " + jobId + " does not exist on the cluster")
|
|
}
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read response when getting job info: %w", err)
|
|
}
|
|
|
|
var jobInfo utiltypes.RayJobInfo
|
|
if err = json.Unmarshal(body, &jobInfo); err != nil {
|
|
// Maybe body is not valid json, raise an error with the body.
|
|
return nil, fmt.Errorf("GetJobInfo fail: %s", string(body))
|
|
}
|
|
|
|
return &jobInfo, nil
|
|
}
|
|
|
|
func (r *RayDashboardClient) ListJobs(ctx context.Context) (*[]utiltypes.RayJobInfo, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.dashboardURL+JobPath, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
r.setAuthHeader(req)
|
|
|
|
resp, err := r.client.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode == http.StatusNotFound {
|
|
return nil, nil
|
|
}
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read response when listing jobs: %w", err)
|
|
}
|
|
|
|
var jobInfo []utiltypes.RayJobInfo
|
|
if err = json.Unmarshal(body, &jobInfo); err != nil {
|
|
// Maybe body is not valid json, raise an error with the body.
|
|
return nil, fmt.Errorf("GetJobInfo fail: %s", string(body))
|
|
}
|
|
|
|
return &jobInfo, nil
|
|
}
|
|
|
|
func (r *RayDashboardClient) SubmitJob(ctx context.Context, rayJob *rayv1.RayJob) (jobId string, err error) {
|
|
request, err := ConvertRayJobToReq(rayJob)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return r.SubmitJobReq(ctx, request)
|
|
}
|
|
|
|
func (r *RayDashboardClient) SubmitJobReq(ctx context.Context, request *utiltypes.RayJobRequest) (jobId string, err error) {
|
|
rayJobJson, err := json.Marshal(request)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.dashboardURL+JobPath, bytes.NewBuffer(rayJobJson))
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
req.Header.Set("Content-Type", "application/json")
|
|
r.setAuthHeader(req)
|
|
|
|
resp, err := r.client.Do(req)
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to read response when submitting job: %w", err)
|
|
}
|
|
|
|
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
|
// If the submission_id is already used, the dashboard will return status code 500; we return the submission_id directly.
|
|
if resp.StatusCode == http.StatusInternalServerError && strings.Contains(string(body), "Please use a different submission_id") {
|
|
return request.SubmissionId, fmt.Errorf("submission ID '%s' already used, Please use a different submission_id", request.SubmissionId)
|
|
}
|
|
return "", fmt.Errorf("SubmitJob fail: %s %s", resp.Status, string(body))
|
|
}
|
|
|
|
var jobResp utiltypes.RayJobResponse
|
|
if err = json.Unmarshal(body, &jobResp); err != nil {
|
|
// Maybe body is not valid json, raise an error with the body.
|
|
return "", fmt.Errorf("SubmitJob fail: %s", string(body))
|
|
}
|
|
|
|
return jobResp.JobId, nil
|
|
}
|
|
|
|
// Get Job Log
|
|
func (r *RayDashboardClient) GetJobLog(ctx context.Context, jobName string) (*string, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.dashboardURL+JobPath+jobName+"/logs", nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
r.setAuthHeader(req)
|
|
|
|
resp, err := r.client.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode == http.StatusNotFound {
|
|
// This does the right thing, but breaks E2E test
|
|
// return nil, errors.NewBadRequest("Job " + jobId + " does not exist on the cluster")
|
|
return nil, nil
|
|
}
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read response when getting job log: %w", err)
|
|
}
|
|
|
|
var jobLog utiltypes.RayJobLogsResponse
|
|
if err = json.Unmarshal(body, &jobLog); err != nil {
|
|
// Maybe body is not valid json, raise an error with the body.
|
|
return nil, fmt.Errorf("GetJobLog fail: %s", string(body))
|
|
}
|
|
|
|
return &jobLog.Logs, nil
|
|
}
|
|
|
|
func (r *RayDashboardClient) StopJob(ctx context.Context, jobName string) (err error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.dashboardURL+JobPath+jobName+"/stop", nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
req.Header.Set("Content-Type", "application/json")
|
|
r.setAuthHeader(req)
|
|
|
|
resp, err := r.client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to read response when stopping job: %w", err)
|
|
}
|
|
|
|
var jobStopResp utiltypes.RayJobStopResponse
|
|
if err = json.Unmarshal(body, &jobStopResp); err != nil {
|
|
return err
|
|
}
|
|
|
|
if !jobStopResp.Stopped {
|
|
jobInfo, err := r.GetJobInfo(ctx, jobName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// StopJob only returns an error when JobStatus is not in terminal states (STOPPED / SUCCEEDED / FAILED)
|
|
if !rayv1.IsJobTerminal(jobInfo.JobStatus) {
|
|
return fmt.Errorf("failed to stop job: %v", jobInfo)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *RayDashboardClient) DeleteJob(ctx context.Context, jobName string) error {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, r.dashboardURL+JobPath+jobName, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
req.Header.Set("Content-Type", "application/json")
|
|
r.setAuthHeader(req)
|
|
|
|
resp, err := r.client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
return nil
|
|
}
|
|
|
|
func ConvertRayJobToReq(rayJob *rayv1.RayJob) (*utiltypes.RayJobRequest, error) {
|
|
req := &utiltypes.RayJobRequest{
|
|
Entrypoint: rayJob.Spec.Entrypoint,
|
|
SubmissionId: rayJob.Status.JobId,
|
|
Metadata: rayJob.Spec.Metadata,
|
|
}
|
|
if len(rayJob.Spec.RuntimeEnvYAML) != 0 {
|
|
runtimeEnv, err := UnmarshalRuntimeEnvYAML(rayJob.Spec.RuntimeEnvYAML)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.RuntimeEnv = runtimeEnv
|
|
}
|
|
req.NumCpus = rayJob.Spec.EntrypointNumCpus
|
|
req.NumGpus = rayJob.Spec.EntrypointNumGpus
|
|
if rayJob.Spec.EntrypointResources != "" {
|
|
if err := json.Unmarshal([]byte(rayJob.Spec.EntrypointResources), &req.Resources); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return req, nil
|
|
}
|
|
|
|
func UnmarshalRuntimeEnvYAML(runtimeEnvYAML string) (utiltypes.RuntimeEnvType, error) {
|
|
var runtimeEnv utiltypes.RuntimeEnvType
|
|
if err := yaml.Unmarshal([]byte(runtimeEnvYAML), &runtimeEnv); err != nil {
|
|
return nil, fmt.Errorf("failed to unmarshal RuntimeEnvYAML: %v: %w", runtimeEnvYAML, err)
|
|
}
|
|
return runtimeEnv, nil
|
|
}
|