Files

389 lines
11 KiB
Go

package dashboardclient
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/apimachinery/pkg/util/yaml"
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
utiltypes "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils/types"
)
var (
// Multi-application URL paths
ServeDetailsPath = "/api/serve/applications/"
APITypeParam = "declarative"
DeployPathV2 = "/api/serve/applications/"
// Job URL paths
JobPath = "/api/jobs/"
)
type RayDashboardClientInterface interface {
UpdateDeployments(ctx context.Context, configJson []byte) error
// V2/multi-app Rest API
GetServeDetails(ctx context.Context) (*utiltypes.ServeDetails, error)
GetMultiApplicationStatus(context.Context) (map[string]*utiltypes.ServeApplicationStatus, error)
GetJobInfo(ctx context.Context, jobId string) (*utiltypes.RayJobInfo, error)
ListJobs(ctx context.Context) (*[]utiltypes.RayJobInfo, error)
SubmitJob(ctx context.Context, rayJob *rayv1.RayJob) (string, error)
SubmitJobReq(ctx context.Context, request *utiltypes.RayJobRequest) (string, error)
GetJobLog(ctx context.Context, jobName string) (*string, error)
StopJob(ctx context.Context, jobName string) error
DeleteJob(ctx context.Context, jobName string) error
}
type RayDashboardClient struct {
client *http.Client
dashboardURL string
authToken string
}
func (r *RayDashboardClient) InitClient(client *http.Client, dashboardURL string, authToken string) {
r.client = client
r.dashboardURL = dashboardURL
r.authToken = authToken
}
func (r *RayDashboardClient) setAuthHeader(req *http.Request) {
if r.authToken != "" {
req.Header.Set("x-ray-authorization", fmt.Sprintf("Bearer %s", r.authToken))
}
}
// UpdateDeployments update the deployments in the Ray cluster.
func (r *RayDashboardClient) UpdateDeployments(ctx context.Context, configJson []byte) error {
var req *http.Request
var err error
if req, err = http.NewRequestWithContext(ctx, http.MethodPut, r.dashboardURL+DeployPathV2, bytes.NewBuffer(configJson)); err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
r.setAuthHeader(req)
resp, err := r.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("failed to read response when updating deployments: %w", err)
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return fmt.Errorf("UpdateDeployments fail: %s %s", resp.Status, string(body))
}
return nil
}
func (r *RayDashboardClient) GetMultiApplicationStatus(ctx context.Context) (map[string]*utiltypes.ServeApplicationStatus, error) {
serveDetails, err := r.GetServeDetails(ctx)
if err != nil {
return nil, fmt.Errorf("failed to get serve details: %w", err)
}
return r.ConvertServeDetailsToApplicationStatuses(serveDetails)
}
// GetServeDetails gets details on all declarative applications on the Ray cluster.
func (r *RayDashboardClient) GetServeDetails(ctx context.Context) (*utiltypes.ServeDetails, error) {
serveDetailsURL, err := url.Parse(r.dashboardURL + ServeDetailsPath)
if err != nil {
return nil, fmt.Errorf("failed to parse dashboard URL: %w", err)
}
q := serveDetailsURL.Query()
q.Set("api_type", APITypeParam)
serveDetailsURL.RawQuery = q.Encode()
req, err := http.NewRequestWithContext(ctx, "GET", serveDetailsURL.String(), nil)
if err != nil {
return nil, err
}
r.setAuthHeader(req)
resp, err := r.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response when getting serve details: %w", err)
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return nil, fmt.Errorf("GetServeDetails fail: %s %s", resp.Status, string(body))
}
var serveDetails utiltypes.ServeDetails
if err = json.Unmarshal(body, &serveDetails); err != nil {
return nil, fmt.Errorf("GetServeDetails failed. Failed to unmarshal bytes: %s", string(body))
}
return &serveDetails, nil
}
func (r *RayDashboardClient) ConvertServeDetailsToApplicationStatuses(serveDetails *utiltypes.ServeDetails) (map[string]*utiltypes.ServeApplicationStatus, error) {
detailsJson, err := json.Marshal(serveDetails.Applications)
if err != nil {
return nil, fmt.Errorf("failed to marshal serve details: %v", serveDetails.Applications)
}
applicationStatuses := map[string]*utiltypes.ServeApplicationStatus{}
if err = json.Unmarshal(detailsJson, &applicationStatuses); err != nil {
return nil, fmt.Errorf("failed to unmarshal serve details bytes into map of application statuses: %w. Bytes: %s", err, string(detailsJson))
}
return applicationStatuses, nil
}
// Note that RayJobInfo and error can't be nil at the same time.
// Please make sure if the Ray job with JobId can't be found. Return a BadRequest error.
func (r *RayDashboardClient) GetJobInfo(ctx context.Context, jobId string) (*utiltypes.RayJobInfo, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.dashboardURL+JobPath+jobId, nil)
if err != nil {
return nil, err
}
r.setAuthHeader(req)
resp, err := r.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return nil, errors.NewBadRequest("Job " + jobId + " does not exist on the cluster")
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response when getting job info: %w", err)
}
var jobInfo utiltypes.RayJobInfo
if err = json.Unmarshal(body, &jobInfo); err != nil {
// Maybe body is not valid json, raise an error with the body.
return nil, fmt.Errorf("GetJobInfo fail: %s", string(body))
}
return &jobInfo, nil
}
func (r *RayDashboardClient) ListJobs(ctx context.Context) (*[]utiltypes.RayJobInfo, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.dashboardURL+JobPath, nil)
if err != nil {
return nil, err
}
r.setAuthHeader(req)
resp, err := r.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return nil, nil
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response when listing jobs: %w", err)
}
var jobInfo []utiltypes.RayJobInfo
if err = json.Unmarshal(body, &jobInfo); err != nil {
// Maybe body is not valid json, raise an error with the body.
return nil, fmt.Errorf("GetJobInfo fail: %s", string(body))
}
return &jobInfo, nil
}
func (r *RayDashboardClient) SubmitJob(ctx context.Context, rayJob *rayv1.RayJob) (jobId string, err error) {
request, err := ConvertRayJobToReq(rayJob)
if err != nil {
return "", err
}
return r.SubmitJobReq(ctx, request)
}
func (r *RayDashboardClient) SubmitJobReq(ctx context.Context, request *utiltypes.RayJobRequest) (jobId string, err error) {
rayJobJson, err := json.Marshal(request)
if err != nil {
return
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.dashboardURL+JobPath, bytes.NewBuffer(rayJobJson))
if err != nil {
return
}
req.Header.Set("Content-Type", "application/json")
r.setAuthHeader(req)
resp, err := r.client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("failed to read response when submitting job: %w", err)
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
// If the submission_id is already used, the dashboard will return status code 500; we return the submission_id directly.
if resp.StatusCode == http.StatusInternalServerError && strings.Contains(string(body), "Please use a different submission_id") {
return request.SubmissionId, fmt.Errorf("submission ID '%s' already used, Please use a different submission_id", request.SubmissionId)
}
return "", fmt.Errorf("SubmitJob fail: %s %s", resp.Status, string(body))
}
var jobResp utiltypes.RayJobResponse
if err = json.Unmarshal(body, &jobResp); err != nil {
// Maybe body is not valid json, raise an error with the body.
return "", fmt.Errorf("SubmitJob fail: %s", string(body))
}
return jobResp.JobId, nil
}
// Get Job Log
func (r *RayDashboardClient) GetJobLog(ctx context.Context, jobName string) (*string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.dashboardURL+JobPath+jobName+"/logs", nil)
if err != nil {
return nil, err
}
r.setAuthHeader(req)
resp, err := r.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
// This does the right thing, but breaks E2E test
// return nil, errors.NewBadRequest("Job " + jobId + " does not exist on the cluster")
return nil, nil
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response when getting job log: %w", err)
}
var jobLog utiltypes.RayJobLogsResponse
if err = json.Unmarshal(body, &jobLog); err != nil {
// Maybe body is not valid json, raise an error with the body.
return nil, fmt.Errorf("GetJobLog fail: %s", string(body))
}
return &jobLog.Logs, nil
}
func (r *RayDashboardClient) StopJob(ctx context.Context, jobName string) (err error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.dashboardURL+JobPath+jobName+"/stop", nil)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
r.setAuthHeader(req)
resp, err := r.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("failed to read response when stopping job: %w", err)
}
var jobStopResp utiltypes.RayJobStopResponse
if err = json.Unmarshal(body, &jobStopResp); err != nil {
return err
}
if !jobStopResp.Stopped {
jobInfo, err := r.GetJobInfo(ctx, jobName)
if err != nil {
return err
}
// StopJob only returns an error when JobStatus is not in terminal states (STOPPED / SUCCEEDED / FAILED)
if !rayv1.IsJobTerminal(jobInfo.JobStatus) {
return fmt.Errorf("failed to stop job: %v", jobInfo)
}
}
return nil
}
func (r *RayDashboardClient) DeleteJob(ctx context.Context, jobName string) error {
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, r.dashboardURL+JobPath+jobName, nil)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
r.setAuthHeader(req)
resp, err := r.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func ConvertRayJobToReq(rayJob *rayv1.RayJob) (*utiltypes.RayJobRequest, error) {
req := &utiltypes.RayJobRequest{
Entrypoint: rayJob.Spec.Entrypoint,
SubmissionId: rayJob.Status.JobId,
Metadata: rayJob.Spec.Metadata,
}
if len(rayJob.Spec.RuntimeEnvYAML) != 0 {
runtimeEnv, err := UnmarshalRuntimeEnvYAML(rayJob.Spec.RuntimeEnvYAML)
if err != nil {
return nil, err
}
req.RuntimeEnv = runtimeEnv
}
req.NumCpus = rayJob.Spec.EntrypointNumCpus
req.NumGpus = rayJob.Spec.EntrypointNumGpus
if rayJob.Spec.EntrypointResources != "" {
if err := json.Unmarshal([]byte(rayJob.Spec.EntrypointResources), &req.Resources); err != nil {
return nil, err
}
}
return req, nil
}
func UnmarshalRuntimeEnvYAML(runtimeEnvYAML string) (utiltypes.RuntimeEnvType, error) {
var runtimeEnv utiltypes.RuntimeEnvType
if err := yaml.Unmarshal([]byte(runtimeEnvYAML), &runtimeEnv); err != nil {
return nil, fmt.Errorf("failed to unmarshal RuntimeEnvYAML: %v: %w", runtimeEnvYAML, err)
}
return runtimeEnv, nil
}