Files
a284b829f8 [Fix][RayJob SidecarMode] Prevent premature job termination during transient head node spikes (#4399)
* [Fix][1/N] defer sidecar submitter failure until dashboard check

Signed-off-by: justinyeh1995 <[email protected]>

* [Feature][SidecarMode] Add SidecarSubmitterRestart feature gate for per-container restart

Signed-off-by: justinyeh1995 <[email protected]>

* [Feature][SidecarMode] Add SidecarSubmitterRestart feature gate unit test

Signed-off-by: justinyeh1995 <[email protected]>

* [Feature][SidecarMode] Add K8s version check if SidecarSubmitterRestart feature gate is enabled at operator startup

Signed-off-by: justinyeh1995 <[email protected]>

* [Chore] better explainations

Signed-off-by: justinyeh1995 <[email protected]>

* [Chore] remove incorrect nil check

Signed-off-by: justinyeh1995 <[email protected]>

* [Fix] SidecarMode should not fail job based on container exit status

Signed-off-by: justinyeh1995 <[email protected]>

* Trigger CI

Signed-off-by: justinyeh1995 <[email protected]>

* [Fix][SidecarMode] add e2e tests

Signed-off-by: justinyeh1995 <[email protected]>

* [Fix][SidecarMode] clarify RestartPolicyRules exit code rationale

Signed-off-by: justinyeh1995 <[email protected]>

* [Fix][Sidecar Mode] Use ParseGeneric + GitVersion, and add patch version for more robust k8s version comparison

Signed-off-by: justinyeh1995 <[email protected]>

* Apply suggestions

Co-authored-by: Jia-Wei Jiang <[email protected]>
Signed-off-by: JustinYeh <[email protected]>

* [Fix] use ContainerRestartPolicyOnFailure to express the intention better; update the comment to explain the intention better

Signed-off-by: justinyeh1995 <[email protected]>

* [Fix] bump version check to v1.35 and update sidecar mode test to check onFailure restart policy

Signed-off-by: justinyeh1995 <[email protected]>

* [Fix] check the existence of the feature gate instead of relying on the derivate result

Signed-off-by: justinyeh1995 <[email protected]>

* improve the why for using K8s v1.35

Signed-off-by: justinyeh1995 <[email protected]>

* [Fix] Revert checkSidecarContainerStatus removal; scope PR to K8s v1.35+ use case

Signed-off-by: justinyeh1995 <[email protected]>

* [Fix] use correct health command constant and argument order in the feature gate test.

Signed-off-by: justinyeh1995 <[email protected]>

* [Fix] remove leftover changes

Signed-off-by: justinyeh1995 <[email protected]>

* [Fix] prevent panicing due to failing to fetch head pod

Signed-off-by: justinyeh1995 <[email protected]>

* add warning about the risk of having kubelet version skew

Signed-off-by: justinyeh1995 <[email protected]>

* document ray v2.54.0+ is needed

Co-authored-by: Jun-Hao Wan <[email protected]>
Signed-off-by: JustinYeh <[email protected]>

* add max restart count annotaion, envVar, and default max restart count

Signed-off-by: justinyeh1995 <[email protected]>

* do not fail the job if the submitter container exited successfully

Signed-off-by: justinyeh1995 <[email protected]>

* fix BasePythonHealthCommand argument

Signed-off-by: justinyeh1995 <[email protected]>

* trigger ci

Signed-off-by: justinyeh1995 <[email protected]>

* address suggestion part 1

Signed-off-by: justinyeh1995 <[email protected]>

* k8s 1.35 version

Signed-off-by: Future-Outlier <[email protected]>

* add override

Signed-off-by: Future-Outlier <[email protected]>

* update

Signed-off-by: Future-Outlier <[email protected]>

* update

Signed-off-by: Future-Outlier <[email protected]>

* upd for helm and kustomization

Signed-off-by: Future-Outlier <[email protected]>

* update

Signed-off-by: Future-Outlier <[email protected]>

* fix typo

Signed-off-by: justinyeh1995 <[email protected]>

* address comment, assert no resubmission

Signed-off-by: justinyeh1995 <[email protected]>

* bump kind version to the latest version which supports v1.35.0 by default

Signed-off-by: justinyeh1995 <[email protected]>

* incremental upgrade use kind-config-buildkite.yml (v1.35)

Signed-off-by: justinyeh1995 <[email protected]>

* replace hardcoded value to use rayJob.Spec.SubmitterConfig.BackoffLimit

Signed-off-by: justinyeh1995 <[email protected]>

---------

Signed-off-by: justinyeh1995 <[email protected]>
Signed-off-by: JustinYeh <[email protected]>
Signed-off-by: Future-Outlier <[email protected]>
Co-authored-by: Jia-Wei Jiang <[email protected]>
Co-authored-by: Jun-Hao Wan <[email protected]>
Co-authored-by: Future-Outlier <[email protected]>
2026-06-01 21:20:44 -07:00

247 lines
9.8 KiB
Go

package common
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
semver "github.com/Masterminds/semver/v3"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"sigs.k8s.io/yaml"
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
"github.com/ray-project/kuberay/ray-operator/pkg/features"
pkgutils "github.com/ray-project/kuberay/ray-operator/pkg/utils"
)
// BuildHeadServiceForRayJob builds the service for a pod. Currently, there is only one service that allows
// the worker nodes to connect to the head node.
// RayJob controller updates the service whenever a new RayCluster serves the traffic.
func BuildHeadServiceForRayJob(ctx context.Context, rayJob rayv1.RayJob, rayCluster rayv1.RayCluster) (*corev1.Service, error) {
service, err := BuildServiceForHeadPod(ctx, rayCluster, nil, nil)
if err != nil {
return nil, err
}
headSvcName, err := utils.GenerateHeadServiceName(utils.RayJobCRD, rayv1.RayClusterSpec{}, rayJob.Name)
if err != nil {
return nil, err
}
service.ObjectMeta.Name = headSvcName
service.ObjectMeta.Namespace = rayJob.Namespace
service.ObjectMeta.Labels = map[string]string{
utils.RayOriginatedFromCRNameLabelKey: rayJob.Name,
utils.RayOriginatedFromCRDLabelKey: utils.RayOriginatedFromCRDLabelValue(utils.RayJobCRD),
utils.RayNodeTypeLabelKey: string(rayv1.HeadNode),
utils.RayIDLabelKey: utils.CheckLabel(utils.GenerateIdentifier(rayJob.Name, rayv1.HeadNode)),
}
return service, nil
}
// GetRuntimeEnvJson returns the JSON string of the runtime environment for the Ray job.
func getRuntimeEnvJson(rayJobInstance *rayv1.RayJob) (string, error) {
runtimeEnvYAML := rayJobInstance.Spec.RuntimeEnvYAML
if len(runtimeEnvYAML) > 0 {
// Convert YAML to JSON
jsonData, err := yaml.YAMLToJSON(pkgutils.ConvertStringToByteSlice(runtimeEnvYAML))
if err != nil {
return "", err
}
// We return the JSON as a string
return pkgutils.ConvertByteSliceToString(jsonData), nil
}
return "", nil
}
// getMetadataJSONForSubmitCommand serializes job metadata for `ray job submit --metadata-json`.
// Ray added --metadata-json in 2.6.0, so the only rejected case is when
// RayJob.Spec.RayClusterSpec.RayVersion is explicitly set below 2.6.0. If RayClusterSpec is
// absent (clusterSelector) or RayVersion is unset, we assume the cluster is >= 2.6.0.
func getMetadataJSONForSubmitCommand(rayJobInstance *rayv1.RayJob, metadata map[string]string) (string, error) {
if rayJobInstance.Spec.RayClusterSpec != nil {
rv := rayJobInstance.Spec.RayClusterSpec.RayVersion
if len(rv) > 0 {
constraint, _ := semver.NewConstraint(">= 2.6.0")
v, err := semver.NewVersion(rv)
if err != nil {
return "", fmt.Errorf("failed to parse Ray version: %v: %w", rv, err)
}
if !constraint.Check(v) {
return "", fmt.Errorf("the Ray version must be at least 2.6.0 to use the metadata field")
}
}
}
metadataBytes, err := json.Marshal(metadata)
if err != nil {
return "", fmt.Errorf("failed to marshal metadata: %v: %w", metadata, err)
}
return pkgutils.ConvertByteSliceToString(metadataBytes), nil
}
// BuildJobSubmitCommand builds the `ray job submit` command based on submission mode.
func BuildJobSubmitCommand(rayJobInstance *rayv1.RayJob, submissionMode rayv1.JobSubmissionMode) ([]string, error) {
var address string
port := utils.DefaultDashboardPort
switch submissionMode {
case rayv1.SidecarMode:
// The sidecar submitter shares the same network namespace as the Ray dashboard,
// so it uses 127.0.0.1 to connect to the Ray dashboard.
rayHeadContainer := rayJobInstance.Spec.RayClusterSpec.HeadGroupSpec.Template.Spec.Containers[utils.RayContainerIndex]
port = int(utils.FindContainerPort(&rayHeadContainer, utils.DashboardPortName, utils.DefaultDashboardPort))
address = "http://127.0.0.1:" + strconv.Itoa(port)
case rayv1.K8sJobMode:
// Submitter is a separate K8s Job; use cluster dashboard address.
address = rayJobInstance.Status.DashboardURL
if !strings.HasPrefix(address, "http://") {
address = "http://" + address
}
default:
return nil, fmt.Errorf("unsupported submission mode for job submit command: %s", submissionMode)
}
var cmd []string
metadata := rayJobInstance.Spec.Metadata
jobId := rayJobInstance.Status.JobId
entrypoint := strings.TrimSpace(rayJobInstance.Spec.Entrypoint)
entrypointNumCpus := rayJobInstance.Spec.EntrypointNumCpus
entrypointNumGpus := rayJobInstance.Spec.EntrypointNumGpus
entrypointResources := rayJobInstance.Spec.EntrypointResources
// In K8sJobMode, we need to avoid submitting the job twice, since the job submitter might retry.
// `ray job submit` alone doesn't handle duplicated submission gracefully. See https://github.com/ray-project/kuberay/issues/2154.
// In order to deal with that, we use `ray job status` first to check if the jobId has been submitted.
// If the jobId has been submitted, we use `ray job logs` to follow the logs.
// Otherwise, we submit the job with `ray job submit --no-wait` + `ray job logs`. The full shell command looks like this:
// if ! ray job status --address http://$RAY_ADDRESS $RAY_JOB_SUBMISSION_ID >/dev/null 2>&1 ;
// then ray job submit --address http://$RAY_ADDRESS --submission-id $RAY_JOB_SUBMISSION_ID --no-wait -- ... ;
// fi ; ray job logs --address http://$RAY_ADDRESS --follow $RAY_JOB_SUBMISSION_ID
// In Sidecar mode, the sidecar container's restart policy is set to Never, so duplicated submission won't happen.
jobStatusCommand := []string{"ray", "job", "status", "--address", address, jobId, ">/dev/null", "2>&1"}
jobSubmitCommand := []string{"ray", "job", "submit", "--address", address}
jobFollowCommand := []string{"ray", "job", "logs", "--address", address, "--follow", jobId}
// Wait until Ray Dashboard GCS is healthy before proceeding.
// In SidecarMode the submitter shares the head Pod's network namespace, so we
// probe localhost. In K8sJobMode the submitter runs in a separate Pod and must
// reach the dashboard through the head Service.
var healthURL string
if submissionMode == rayv1.SidecarMode {
healthURL = fmt.Sprintf("http://localhost:%d/%s", port, utils.RayDashboardGCSHealthPath)
} else {
healthURL = address + "/" + utils.RayDashboardGCSHealthPath
}
rayDashboardGCSHealthCommand := fmt.Sprintf(
utils.BasePythonHealthCommand,
healthURL,
utils.RayDashboardGCSHealthCheckTimeoutSeconds,
)
waitLoop := []string{
"until", rayDashboardGCSHealthCommand, ">/dev/null", "2>&1", ";",
"do", "echo", strconv.Quote("Waiting for Ray Dashboard GCS to become healthy at " + address + " ..."), ";", "sleep", "2", ";", "done", ";",
}
cmd = append(cmd, waitLoop...)
// In Sidecar mode without SidecarSubmitterRestart feature gate enabled, we only support RayJob level retry, which means that the submitter retry won't happen,
// so we won't have to check if the job has been submitted.
// In K8sJobMode (submitter Job may retry) or Sidecar mode with SidecarSubmitterRestart feature gate enabled (submitter container may restart on failure).
// we check job status before submitting to handle duplicated submission gracefully.
needsStatusCheck := submissionMode == rayv1.K8sJobMode || (submissionMode == rayv1.SidecarMode && features.Enabled(features.SidecarSubmitterRestart))
if needsStatusCheck {
cmd = append(cmd, "if", "!")
cmd = append(cmd, jobStatusCommand...)
cmd = append(cmd, ";", "then")
}
cmd = append(cmd, jobSubmitCommand...)
if needsStatusCheck {
cmd = append(cmd, "--no-wait")
}
runtimeEnvJson, err := getRuntimeEnvJson(rayJobInstance)
if err != nil {
return nil, err
}
if len(runtimeEnvJson) > 0 {
cmd = append(cmd, "--runtime-env-json", strconv.Quote(runtimeEnvJson))
}
if len(metadata) > 0 {
metadataJson, err := getMetadataJSONForSubmitCommand(rayJobInstance, metadata)
if err != nil {
return nil, err
}
cmd = append(cmd, "--metadata-json", strconv.Quote(metadataJson))
}
if len(jobId) > 0 {
cmd = append(cmd, "--submission-id", jobId)
}
if entrypointNumCpus > 0 {
cmd = append(cmd, "--entrypoint-num-cpus", fmt.Sprintf("%f", entrypointNumCpus))
}
if entrypointNumGpus > 0 {
cmd = append(cmd, "--entrypoint-num-gpus", fmt.Sprintf("%f", entrypointNumGpus))
}
if len(entrypointResources) > 0 {
cmd = append(cmd, "--entrypoint-resources", strconv.Quote(entrypointResources))
}
// "--" is used to separate the entrypoint from the Ray Job CLI command and its arguments.
cmd = append(cmd, "--", entrypoint, ";")
if needsStatusCheck {
cmd = append(cmd, "fi", ";")
cmd = append(cmd, jobFollowCommand...)
}
return cmd, nil
}
// GetSubmitterTemplate creates a default submitter template for the Ray job.
func GetSubmitterTemplate(rayJobSpec *rayv1.RayJobSpec, rayClusterSpec *rayv1.RayClusterSpec) corev1.PodTemplateSpec {
if rayJobSpec.SubmitterPodTemplate != nil {
return *rayJobSpec.SubmitterPodTemplate.DeepCopy()
}
return corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
GetDefaultSubmitterContainer(rayClusterSpec),
},
RestartPolicy: corev1.RestartPolicyNever,
},
}
}
// GetDefaultSubmitterContainer creates a default submitter container for the Ray job.
func GetDefaultSubmitterContainer(rayClusterSpec *rayv1.RayClusterSpec) corev1.Container {
return corev1.Container{
Name: utils.SubmitterContainerName,
// Use the image of the Ray head to be defensive against version mismatch issues
Image: rayClusterSpec.HeadGroupSpec.Template.Spec.Containers[utils.RayContainerIndex].Image,
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("500m"),
corev1.ResourceMemory: resource.MustParse("200Mi"),
},
},
}
}