Files

730 lines
28 KiB
Go

package job
import (
"bufio"
"context"
"crypto/rand"
"fmt"
"log"
"math/big"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/google/shlex"
"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/api/meta"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/cli-runtime/pkg/genericclioptions"
"k8s.io/cli-runtime/pkg/genericiooptions"
"k8s.io/kubectl/pkg/cmd/portforward"
cmdutil "k8s.io/kubectl/pkg/cmd/util"
"k8s.io/kubectl/pkg/util/templates"
"sigs.k8s.io/yaml"
"github.com/ray-project/kuberay/kubectl-plugin/pkg/util"
"github.com/ray-project/kuberay/kubectl-plugin/pkg/util/client"
"github.com/ray-project/kuberay/kubectl-plugin/pkg/util/generation"
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
rayscheme "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme"
)
const (
dashboardAddr = "http://localhost:8265"
clusterTimeout = 120.0
portForwardTimeout = 60.0
)
type SubmitJobOptions struct {
cmdFactory cmdutil.Factory
ioStreams *genericiooptions.IOStreams
RayJob *rayv1.RayJob
workerNodeSelectors map[string]string
headNodeSelectors map[string]string
logColor string
address string
image string
fileName string
workingDir string
runtimeEnv string
headers string
verify string
cluster string
runtimeEnvJson string
entryPointResource string
metadataJson string
logStyle string
submissionID string
rayjobName string
rayVersion string
entryPoint string
headCPU string
headMemory string
headGPU string
workerCPU string
workerMemory string
workerGPU string
namespace string
entryPointMemory int
entryPointGPU float32
workerReplicas int32
entryPointCPU float32
noWait bool
dryRun bool
verbose bool
shutdownAfterJobFinishes bool
ttlSecondsAfterFinished int32
}
type JobInfo struct {
SubmissionID string `json:"submission_id"`
Entrypoint string `json:"entrypoint"`
Type string `json:"type"`
}
var (
jobSubmitLong = templates.LongDesc(`
Submit Ray job to Ray cluster as one would using Ray CLI e.g. 'ray job submit ENTRYPOINT'.
If Ray cluster is already setup, use 'kubectl ray session' instead. If '--address' is set, we connect directly
without port-forwarding; if empty, we port-forward to localhost:8265.
If no RayJob YAML file is specified, the command will create a default RayJob for the user.
Command will apply RayJob CR and also submit the Ray job. RayJob CR is required.
`)
jobSubmitExample = templates.Examples(fmt.Sprintf(`
# Submit Ray job with working-directory
kubectl ray job submit -f rayjob.yaml --working-dir /path/to/working-dir/ -- python my_script.py
# Submit Ray job with runtime Env file and working directory
kubectl ray job submit -f rayjob.yaml --working-dir /path/to/working-dir/ --runtime-env /runtimeEnv.yaml -- python my_script.py
# Submit Ray job with runtime Env file assuming runtime-env has working_dir set
kubectl ray job submit -f rayjob.yaml --runtime-env path/to/runtimeEnv.yaml -- python my_script.py
# Submit generated Ray job with default values and with runtime Env file and working directory
kubectl ray job submit --name rayjob-sample --working-dir /path/to/working-dir/ --runtime-env /runtimeEnv.yaml -- python my_script.py
# Submit a Ray job with specific head-node-selectors and worker-node-selectors using kubectl ray job submit
kubectl ray job submit --name rayjob-sample --working-dir /path/to/working-dir/ --head-node-selectors kubernetes.io/os=linux --worker-node-selectors kubernetes.io/os=linux -- python my_script.py
# Generate Ray job with specifications and submit Ray job with runtime Env file and working directory
kubectl ray job submit --name rayjob-sample --ray-version %s --image %s --head-cpu 1 --head-memory 5Gi --head-gpu 1 --worker-replicas 3 --worker-cpu 1 --work-gpu 1 --worker-memory 5Gi --runtime-env path/to/runtimeEnv.yaml -- python my_script.py
# Generate Ray job with specifications and print out the generated RayJob YAML
kubectl ray job submit --dry-run --name rayjob-sample --ray-version %s --image %s --head-cpu 1 --head-memory 5Gi --worker-replicas 3 --worker-cpu 1 --worker-memory 5Gi --runtime-env path/to/runtimeEnv.yaml -- python my_script.py
`, util.RayVersion, util.RayImage, util.RayVersion, util.RayImage))
)
func NewJobSubmitOptions(cmdFactory cmdutil.Factory, streams genericiooptions.IOStreams) *SubmitJobOptions {
return &SubmitJobOptions{
cmdFactory: cmdFactory,
ioStreams: &streams,
}
}
func NewJobSubmitCommand(cmdFactory cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
options := NewJobSubmitOptions(cmdFactory, streams)
cmd := &cobra.Command{
Use: "submit [OPTIONS] -f/--filename RAYJOB_YAML -- ENTRYPOINT",
Short: "Submit Ray job to Ray cluster",
Long: jobSubmitLong,
Example: jobSubmitExample,
RunE: func(cmd *cobra.Command, args []string) error {
entryPointStart := cmd.ArgsLenAtDash()
if entryPointStart == -1 || len(args[entryPointStart:]) == 0 {
return cmdutil.UsageErrorf(cmd, "%s", cmd.Use)
}
options.entryPoint = strings.Join(args[entryPointStart:], " ")
if err := options.Complete(); err != nil {
return err
}
if err := options.Validate(cmd); err != nil {
return err
}
return options.Run(cmd.Context(), cmdFactory)
},
}
cmd.Flags().StringVarP(&options.fileName, "filename", "f", "", "Path and name of the Ray Job YAML file")
cmd.Flags().StringVar(&options.address, "address", "", "Ray Dashboard base URL (e.g., https://ray.example.com). If set, skips port-forwarding.")
cmd.Flags().StringVar(&options.submissionID, "submission-id", "", "ID to specify for the Ray job. If not provided, one will be generated")
cmd.Flags().StringVar(&options.runtimeEnv, "runtime-env", "", "Path and name to the runtime env YAML file.")
cmd.Flags().StringVar(&options.workingDir, "working-dir", "", "Directory containing files that your job will run in")
cmd.Flags().StringVar(&options.headers, "headers", "", "Used to pass headers through http/s to Ray Cluster. Must be JSON formatting")
cmd.Flags().StringVar(&options.runtimeEnvJson, "runtime-env-json", "", "JSON-serialized runtime_env dictionary. Precedence over Ray job CR.")
cmd.Flags().StringVar(&options.verify, "verify", "", "Boolean indication to verify the server's TLS certificate or a path to a file or directory of trusted certificates.")
cmd.Flags().StringVar(&options.entryPointResource, "entrypoint-resources", "", "JSON-serialized dictionary mapping resource name to resource quantity")
cmd.Flags().StringVar(&options.metadataJson, "metadata-json", "", "JSON-serialized dictionary of metadata to attach to the job.")
cmd.Flags().StringVar(&options.logStyle, "log-style", "", "Specific to 'ray job submit'. Options are 'auto | record | pretty'")
cmd.Flags().StringVar(&options.logColor, "log-color", "", "Specific to 'ray job submit'. Options are 'auto | false | true'")
cmd.Flags().Float32Var(&options.entryPointCPU, "entrypoint-num-cpus", 0, "Number of CPUs reserved for the entrypoint command")
cmd.Flags().Float32Var(&options.entryPointGPU, "entrypoint-num-gpus", 0, "Number of GPUs reserved for the entrypoint command")
cmd.Flags().IntVar(&options.entryPointMemory, "entrypoint-memory", 0, "Amount of memory reserved for the entrypoint command")
cmd.Flags().BoolVar(&options.noWait, "no-wait", false, "If present, will not stream logs and wait for job to finish")
cmd.Flags().StringVar(&options.rayjobName, "name", "", "Ray job name")
cmd.Flags().StringVar(&options.rayVersion, "ray-version", util.RayVersion, "Ray version to use")
cmd.Flags().StringVar(&options.image, "image", fmt.Sprintf("rayproject/ray:%s", options.rayVersion), "container image to use")
cmd.Flags().StringVar(&options.headCPU, "head-cpu", "2", "number of CPUs in the Ray head")
cmd.Flags().StringVar(&options.headMemory, "head-memory", "4Gi", "amount of memory in the Ray head")
cmd.Flags().StringVar(&options.headGPU, "head-gpu", "0", "number of GPUs in the Ray head")
cmd.Flags().Int32Var(&options.workerReplicas, "worker-replicas", 1, "desired worker group replicas")
cmd.Flags().StringVar(&options.workerCPU, "worker-cpu", "2", "number of CPUs in each worker group replica")
cmd.Flags().StringVar(&options.workerMemory, "worker-memory", "4Gi", "amount of memory in each worker group replica")
cmd.Flags().StringVar(&options.workerGPU, "worker-gpu", "0", "number of GPUs in each worker group replica")
cmd.Flags().BoolVar(&options.dryRun, "dry-run", false, "print the generated YAML instead of creating the cluster. Only works when filename is not provided")
cmd.Flags().BoolVarP(&options.verbose, "verbose", "v", false, "Passing the '--verbose' flag to the 'ray job submit' command")
cmd.Flags().StringToStringVar(&options.headNodeSelectors, "head-node-selectors", nil, "Node selectors to apply to the head pod in the cluster (e.g. --head-node-selectors topology.kubernetes.io/zone=us-east-1c)")
cmd.Flags().StringToStringVar(&options.workerNodeSelectors, "worker-node-selectors", nil, "Node selectors to apply to all worker pods in the cluster (e.g. --worker-node-selectors topology.kubernetes.io/zone=us-east-1c)")
cmd.Flags().Int32Var(&options.ttlSecondsAfterFinished, "ttl-seconds-after-finished", 0, "TTL seconds after finished.")
return cmd
}
func (options *SubmitJobOptions) Complete() error {
namespace, _, err := options.cmdFactory.ToRawKubeConfigLoader().Namespace()
if err != nil {
return fmt.Errorf("failed to get namespace: %w", err)
}
options.namespace = namespace
if len(options.runtimeEnv) > 0 {
options.runtimeEnv = filepath.Clean(options.runtimeEnv)
}
if options.fileName != "" {
options.fileName = filepath.Clean(options.fileName)
}
return nil
}
func (options *SubmitJobOptions) Validate(cmd *cobra.Command) error {
if len(options.runtimeEnv) > 0 {
info, err := os.Stat(options.runtimeEnv)
if os.IsNotExist(err) {
return fmt.Errorf("Runtime Env file does not exist. Failed with: %w", err)
} else if err != nil {
return fmt.Errorf("Error occurred when checking runtime env file: %w", err)
} else if !info.Mode().IsRegular() {
return fmt.Errorf("Filename given is not a regular file. Failed with: %w", err)
}
runtimeEnvWorkingDir, err := runtimeEnvHasWorkingDir(options.runtimeEnv)
if err != nil {
return fmt.Errorf("Error while checking runtime env: %w", err)
}
if len(runtimeEnvWorkingDir) > 0 && options.workingDir == "" {
options.workingDir = runtimeEnvWorkingDir
}
}
if cmd.Flags().Changed("ttl-seconds-after-finished") {
options.shutdownAfterJobFinishes = true
}
if options.ttlSecondsAfterFinished < 0 {
return fmt.Errorf("--ttl-seconds-after-finished must be greater than or equal to 0")
}
// Take care of case where there is a filename input
if options.fileName != "" {
info, err := os.Stat(options.fileName)
if os.IsNotExist(err) {
return fmt.Errorf("Ray Job file does not exist. Failed with: %w", err)
} else if err != nil {
return fmt.Errorf("Error occurred when checking Ray job file: %w", err)
} else if !info.Mode().IsRegular() {
return fmt.Errorf("Filename given is not a regular file. Failed with: %w", err)
}
options.RayJob, err = decodeRayJobYaml(options.fileName)
if err != nil {
return fmt.Errorf("Failed to decode RayJob Yaml: %w", err)
}
submissionMode := options.RayJob.Spec.SubmissionMode
if submissionMode != rayv1.InteractiveMode {
return fmt.Errorf("Submission mode of the Ray Job must be set to 'InteractiveMode'")
}
// InteractiveMode does not support backoffLimit > 1.
// When a RayJob fails (e.g., due to a missing script) and retries,
// spec.JobId remains set, causing the new job to incorrectly transition
// to Running instead of Waiting or Failed.
// After discussion, we decided to disallow retries in InteractiveMode
// to avoid ambiguous state handling and unintended behavior.
// https://github.com/ray-project/kuberay/issues/3525
if submissionMode == rayv1.InteractiveMode && options.RayJob.Spec.BackoffLimit != nil && *options.RayJob.Spec.BackoffLimit > 0 {
return fmt.Errorf("BackoffLimit is incompatible with InteractiveMode")
}
runtimeEnvYaml := options.RayJob.Spec.RuntimeEnvYAML
if options.runtimeEnv == "" && options.runtimeEnvJson == "" && runtimeEnvYaml != "" {
runtimeJson, err := yaml.YAMLToJSON([]byte(runtimeEnvYaml))
if err != nil {
return fmt.Errorf("Failed to convert runtime env to json: %w", err)
}
options.runtimeEnvJson = string(runtimeJson)
}
if cmd.Flags().Changed("ttl-seconds-after-finished") {
options.RayJob.Spec.TTLSecondsAfterFinished = options.ttlSecondsAfterFinished
options.RayJob.Spec.ShutdownAfterJobFinishes = options.shutdownAfterJobFinishes
}
if options.RayJob.Spec.TTLSecondsAfterFinished < 0 {
return fmt.Errorf("ttlSecondsAfterFinished must be greater than or equal to 0")
}
if !options.RayJob.Spec.ShutdownAfterJobFinishes && options.RayJob.Spec.TTLSecondsAfterFinished > 0 {
return fmt.Errorf("ttlSecondsAfterFinished is only supported when shutdownAfterJobFinishes is set to true")
}
} else if strings.TrimSpace(options.rayjobName) == "" {
return fmt.Errorf("Must set either yaml file (--filename) or set Ray job name (--name)")
}
if options.workingDir == "" {
return fmt.Errorf("working directory is required, use --working-dir or set with runtime env")
}
resourceFields := map[string]string{
"head-cpu": options.headCPU,
"head-gpu": options.headGPU,
"head-memory": options.headMemory,
"worker-cpu": options.workerCPU,
"worker-gpu": options.workerGPU,
"worker-memory": options.workerMemory,
}
for name, value := range resourceFields {
if value != "" || cmd.Flags().Changed(name) {
if err := util.ValidateResourceQuantity(value, name); err != nil {
return fmt.Errorf("%w", err)
}
}
}
if cmd.Flags().Changed("address") {
if strings.TrimSpace(options.address) == "" {
return fmt.Errorf("--address was provided but is empty")
}
}
return nil
}
func (options *SubmitJobOptions) Run(ctx context.Context, factory cmdutil.Factory) error {
k8sClients, err := client.NewClient(factory)
if err != nil {
return fmt.Errorf("failed to initialize clientset: %w", err)
}
// If submission ID is not provided by the user, generate one.
if options.submissionID == "" {
generatedID, err := generateSubmissionID()
if err != nil {
return fmt.Errorf("failed to generate submission ID: %w", err)
}
options.submissionID = generatedID
fmt.Printf("Generated submission ID for Ray job: %s\n", options.submissionID)
}
if options.fileName == "" {
// Genarate the Ray job.
rayJobObject := generation.RayJobYamlObject{
RayJobName: options.rayjobName,
Namespace: options.namespace,
ShutdownAfterJobFinishes: options.shutdownAfterJobFinishes,
TTLSecondsAfterFinished: options.ttlSecondsAfterFinished,
SubmissionMode: "InteractiveMode",
// Prior to kuberay 1.2.2, the entry point is required. To maintain
// backwards compatibility with 1.2.x, we submit the entry point
// here, even though it will be ignored.
// See https://github.com/ray-project/kuberay/issues/3126.
Entrypoint: options.entryPoint,
RayClusterConfig: generation.RayClusterConfig{
RayVersion: &options.rayVersion,
Image: &options.image,
Head: &generation.Head{
CPU: &options.headCPU,
Memory: &options.headMemory,
GPU: &options.headGPU,
NodeSelectors: options.headNodeSelectors,
},
WorkerGroups: []generation.WorkerGroup{
{
CPU: &options.workerCPU,
Memory: &options.workerMemory,
GPU: &options.workerGPU,
Replicas: options.workerReplicas,
NodeSelectors: options.workerNodeSelectors,
},
},
},
}
rayJobApplyConfig := rayJobObject.GenerateRayJobApplyConfig()
rayJobApplyConfig.Spec.JobId = &options.submissionID
// Print out the yaml if it is a dry run
if options.dryRun {
resultYaml, err := generation.ConvertRayJobApplyConfigToYaml(rayJobApplyConfig)
if err != nil {
return fmt.Errorf("Failed to convert RayJob into yaml format: %w", err)
}
fmt.Printf("%s\n", resultYaml)
return nil
}
// Apply the generated yaml
rayJobApplyConfigResult, err := k8sClients.RayClient().RayV1().RayJobs(options.namespace).Apply(ctx, rayJobApplyConfig, v1.ApplyOptions{FieldManager: util.FieldManager})
if err != nil {
return fmt.Errorf("Failed to apply generated YAML: %w", err)
}
options.RayJob = &rayv1.RayJob{}
options.RayJob.SetName(rayJobApplyConfigResult.Name)
} else {
options.RayJob.Spec.JobId = options.submissionID
options.RayJob, err = k8sClients.RayClient().RayV1().RayJobs(options.namespace).Create(ctx, options.RayJob, v1.CreateOptions{FieldManager: util.FieldManager})
if err != nil {
return fmt.Errorf("Error when creating RayJob CR: %w", err)
}
}
fmt.Printf("Submitted RayJob %s.\n", options.RayJob.GetName())
if len(options.RayJob.GetName()) > 0 {
// Add timeout?
for len(options.RayJob.Status.RayClusterName) == 0 {
options.RayJob, err = k8sClients.RayClient().RayV1().RayJobs(options.namespace).Get(ctx, options.RayJob.GetName(), v1.GetOptions{})
if err != nil {
return fmt.Errorf("Failed to get Ray Job status")
}
time.Sleep(2 * time.Second)
}
options.cluster = options.RayJob.Status.RayClusterName
} else {
return fmt.Errorf("Unknown cluster and did not provide Ray Job. One of the fields must be set")
}
// Wait til the cluster is ready
var clusterReady bool
clusterWaitStartTime := time.Now()
currTime := clusterWaitStartTime
fmt.Printf("Waiting for RayCluster\n")
fmt.Printf("Checking Cluster Status for cluster %s...\n", options.cluster)
for !clusterReady && currTime.Sub(clusterWaitStartTime).Seconds() <= clusterTimeout {
time.Sleep(2 * time.Second)
currCluster, err := k8sClients.RayClient().RayV1().RayClusters(options.namespace).Get(ctx, options.cluster, v1.GetOptions{})
if err != nil {
return fmt.Errorf("Failed to get cluster information with error: %w", err)
}
clusterReady = isRayClusterReady(currCluster)
if !clusterReady {
fmt.Println("Cluster is not ready")
}
currTime = time.Now()
}
if !clusterReady {
fmt.Printf("Deleting RayJob...\n")
err = k8sClients.RayClient().RayV1().RayJobs(options.namespace).Delete(ctx, options.RayJob.GetName(), v1.DeleteOptions{})
if err != nil {
return fmt.Errorf("Failed to clean up Ray job after time out.: %w", err)
}
fmt.Printf("Cleaned Up RayJob: %s\n", options.RayJob.GetName())
return fmt.Errorf("Timed out waiting for cluster")
}
if options.address == "" {
svcName, err := k8sClients.GetRayHeadSvcName(ctx, options.namespace, util.RayCluster, options.cluster)
if err != nil {
return fmt.Errorf("Failed to find service name: %w", err)
}
// start port forward section
portForwardCmd := portforward.NewCmdPortForward(factory, *options.ioStreams)
portForwardCmd.SetArgs([]string{"service/" + svcName, fmt.Sprintf("%d:%d", 8265, 8265)})
// create new context for port-forwarding so we can cancel the context to stop the port forwarding only
portForwardCtx, cancel := context.WithCancel(ctx)
defer cancel()
go func() {
fmt.Printf("Port forwarding service %s\n", svcName)
if err := portForwardCmd.ExecuteContext(portForwardCtx); err != nil {
log.Fatalf("Error occurred while port-forwarding Ray dashboard: %v", err)
}
}()
// Wait for port forward to be ready
var portForwardReady bool
portForwardWaitStartTime := time.Now()
currTime = portForwardWaitStartTime
portforwardCheckRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, dashboardAddr, nil)
if err != nil {
return fmt.Errorf("Error occurred when trying to create request to probe cluster endpoint: %w", err)
}
httpClient := http.Client{
Timeout: 5 * time.Second,
}
fmt.Printf("Waiting for port forwarding...")
for !portForwardReady && currTime.Sub(portForwardWaitStartTime).Seconds() <= portForwardTimeout {
time.Sleep(2 * time.Second)
rayDashboardResponse, err := httpClient.Do(portforwardCheckRequest)
if err != nil {
err = fmt.Errorf("Error occurred when waiting for port forwarding: %w", err)
fmt.Println(err)
currTime = time.Now()
continue
}
if rayDashboardResponse.StatusCode >= 200 && rayDashboardResponse.StatusCode < 300 {
portForwardReady = true
}
rayDashboardResponse.Body.Close()
currTime = time.Now()
}
if !portForwardReady {
return fmt.Errorf("Timed out waiting for port forwarding")
}
options.address = dashboardAddr
fmt.Printf("Port forwarding started on %s\n", options.address)
} else {
fmt.Printf("Using address %s (no port-forwarding)\n", options.address)
}
// Submitting ray job to cluster
raySubmitCmd, err := options.raySubmitCmd()
if err != nil {
return fmt.Errorf("failed to create Ray submit command with error: %w", err)
}
fmt.Printf("Ray command: %v\n", raySubmitCmd)
cmd := exec.CommandContext(ctx, raySubmitCmd[0], raySubmitCmd[1:]...) //nolint:gosec // command is sanitized in raySubmitCmd() and file paths are cleaned in Complete()
// Get the outputs/pipes for `ray job submit` outputs
rayCmdStdOut, err := cmd.StdoutPipe()
if err != nil {
return fmt.Errorf("Error while setting up `ray job submit` stdout: %w", err)
}
rayCmdStdErr, err := cmd.StderrPipe()
if err != nil {
return fmt.Errorf("Error while setting up `ray job submit` stderr: %w", err)
}
fmt.Printf("Running Ray submit job command...\n")
if err := cmd.Start(); err != nil {
log.Fatalf("error occurred while running command %s: %v", fmt.Sprint(raySubmitCmd), err)
}
rayJobID := options.submissionID
rayCmdStdOutScanner := bufio.NewScanner(rayCmdStdOut)
rayCmdStdErrScanner := bufio.NewScanner(rayCmdStdErr)
go func() {
for {
currStdToken := rayCmdStdOutScanner.Text()
if currStdToken != "" {
fmt.Println(currStdToken)
}
scanNotDone := rayCmdStdOutScanner.Scan()
if !scanNotDone {
break
}
}
}()
go func() {
for {
currErrToken := rayCmdStdErrScanner.Text()
if currErrToken != "" {
fmt.Fprintf(options.ioStreams.ErrOut, "%s\n", currErrToken)
}
scanNotDone := rayCmdStdErrScanner.Scan()
if !scanNotDone {
break
}
}
}()
// Wait for Ray job submit to finish.
err = cmd.Wait()
if err != nil {
return fmt.Errorf("Error occurred with Ray job submit: %w", err)
}
if options.noWait {
fmt.Printf("Ray job submitted with ID %s\n", rayJobID)
return nil
}
// Wait for the Ray job to finish
watcher, err := k8sClients.RayClient().RayV1().
RayJobs(options.namespace).
Watch(ctx, v1.ListOptions{
FieldSelector: "metadata.name=" + options.RayJob.GetName(),
})
if err != nil {
return fmt.Errorf("failed to watch RayJob: %w", err)
}
defer watcher.Stop()
fmt.Println("Waiting for job to finish...")
for evt := range watcher.ResultChan() {
job, ok := evt.Object.(*rayv1.RayJob)
if !ok {
fmt.Fprintf(options.ioStreams.ErrOut, "unexpected watch event type %T\n", evt.Object)
continue
}
status := job.Status.JobStatus
jobID := job.Status.JobId
if jobID == "" {
jobID = "unknown"
}
fmt.Printf("Current status: %s (RayJob: %s, JobID: %s)\n",
status, job.GetName(), jobID)
if rayv1.IsJobTerminal(status) {
switch status {
case rayv1.JobStatusSucceeded, rayv1.JobStatusStopped:
fmt.Printf("Job %s finished with status %s.\n", jobID, status)
return nil
case rayv1.JobStatusFailed:
if msg := job.Status.Message; msg != "" {
return fmt.Errorf("job %s failed: %s", jobID, msg)
}
return fmt.Errorf("job %s failed with status %s", jobID, status)
default:
return fmt.Errorf("job %s in unexpected terminal state %s", jobID, status)
}
}
}
fmt.Fprintf(options.ioStreams.ErrOut,
"rayjob %s watch ended without a clear terminal state\n", options.RayJob.GetName())
return nil
}
func (options *SubmitJobOptions) raySubmitCmd() ([]string, error) {
addr := options.address
if addr == "" {
addr = dashboardAddr
}
raySubmitCmd := []string{"ray", "job", "submit", "--address", addr}
if len(options.runtimeEnv) > 0 {
raySubmitCmd = append(raySubmitCmd, "--runtime-env", options.runtimeEnv)
}
if len(options.runtimeEnvJson) > 0 {
raySubmitCmd = append(raySubmitCmd, "--runtime-env-json", options.runtimeEnvJson)
}
if len(options.submissionID) > 0 {
raySubmitCmd = append(raySubmitCmd, "--submission-id", options.submissionID)
}
if options.entryPointCPU > 0 {
raySubmitCmd = append(raySubmitCmd, "--entrypoint-num-cpus", fmt.Sprintf("%f", options.entryPointCPU))
}
if options.entryPointGPU > 0 {
raySubmitCmd = append(raySubmitCmd, "--entrypoint-num-gpus", fmt.Sprintf("%f", options.entryPointGPU))
}
if options.entryPointMemory > 0 {
raySubmitCmd = append(raySubmitCmd, "--entrypoint-memory", fmt.Sprintf("%d", options.entryPointMemory))
}
if len(options.entryPointResource) > 0 {
raySubmitCmd = append(raySubmitCmd, "--entrypoint-resource", options.entryPointResource)
}
if len(options.metadataJson) > 0 {
raySubmitCmd = append(raySubmitCmd, "--metadata-json", options.metadataJson)
}
if options.noWait {
raySubmitCmd = append(raySubmitCmd, "--no-wait")
}
if len(options.headers) > 0 {
raySubmitCmd = append(raySubmitCmd, "--headers", options.headers)
}
if len(options.verify) > 0 {
raySubmitCmd = append(raySubmitCmd, "--verify", options.verify)
}
if len(options.logStyle) > 0 {
raySubmitCmd = append(raySubmitCmd, "--log-style", options.logStyle)
}
if len(options.logColor) > 0 {
raySubmitCmd = append(raySubmitCmd, "--log-color", options.logColor)
}
if options.verbose {
raySubmitCmd = append(raySubmitCmd, "--verbose")
}
raySubmitCmd = append(raySubmitCmd, "--working-dir", options.workingDir)
raySubmitCmd = append(raySubmitCmd, "--")
// Sanitize entrypoint
entryPointSanitized, err := shlex.Split(options.entryPoint)
if err != nil {
return nil, err
}
raySubmitCmd = append(raySubmitCmd, entryPointSanitized...)
return raySubmitCmd, nil
}
// Decode RayJob YAML if we decide to submit job using kube client
func decodeRayJobYaml(rayJobFilePath string) (*rayv1.RayJob, error) {
decodedRayJob := &rayv1.RayJob{}
rayJobYamlContent, err := os.ReadFile(rayJobFilePath)
if err != nil {
return nil, err
}
decoder := rayscheme.Codecs.UniversalDecoder()
_, _, err = decoder.Decode(rayJobYamlContent, nil, decodedRayJob)
if err != nil {
return nil, err
}
return decodedRayJob, nil
}
func runtimeEnvHasWorkingDir(runtimePath string) (string, error) {
runtimeEnvFileContent, err := os.ReadFile(runtimePath)
if err != nil {
return "", err
}
var runtimeEnvYaml map[string]any
err = yaml.Unmarshal(runtimeEnvFileContent, &runtimeEnvYaml)
if err != nil {
return "", err
}
if workingDir, ok := runtimeEnvYaml["working_dir"].(string); ok {
return workingDir, nil
}
return "", nil
}
func isRayClusterReady(rayCluster *rayv1.RayCluster) bool {
return meta.IsStatusConditionTrue(rayCluster.Status.Conditions, "Ready") || rayCluster.Status.State == rayv1.Ready
}
// Generates a 16-character random ID with a prefix, mimicking Ray Job submission_id.
// ref: ray/python/ray/dashboard/modules/job/job_manager.py
func generateSubmissionID() (string, error) {
// ASCII letters and digits, excluding confusing characters I, l, O, 0, o.
const possibleChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ123456789"
idRunes := make([]rune, 16)
for i := range idRunes {
// Securely generate a random index.
idx, err := rand.Int(rand.Reader, big.NewInt(int64(len(possibleChars))))
if err != nil {
return "", err
}
idRunes[i] = rune(possibleChars[idx.Int64()])
}
return fmt.Sprintf("raysubmit_%s", string(idRunes)), nil
}