Files
AndySung320andGitHub c24536d3e4 Update observedGeneration during suspend lifecycle in RayService (#4897)
* Bump observedGeneration during suspend lifecycle in RayService

Signed-off-by: AndySung320 <[email protected]>

* align variable naming with the current tests

Signed-off-by: AndySung320 <[email protected]>

* remove function:bumpObservedGeneration

Signed-off-by: AndySung320 <[email protected]>

* Avoid premature observedGeneration updates during suspend

Signed-off-by: AndySung320 <[email protected]>

---------

Signed-off-by: AndySung320 <[email protected]>
2026-06-18 10:31:07 -07:00

2360 lines
114 KiB
Go

package ray
import (
"context"
errstd "errors"
"fmt"
"maps"
"math"
"os"
"strconv"
"strings"
"time"
"github.com/go-logr/logr"
cmap "github.com/orcaman/concurrent-map/v2"
corev1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/client-go/tools/record"
"k8s.io/utils/lru"
"k8s.io/utils/ptr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
gwv1 "sigs.k8s.io/gateway-api/apis/v1"
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
"github.com/ray-project/kuberay/ray-operator/controllers/ray/common"
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils/dashboardclient"
utiltypes "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils/types"
"github.com/ray-project/kuberay/ray-operator/pkg/features"
)
const (
ServiceDefaultRequeueDuration = 2 * time.Second
RayClusterDeletionDelayDuration = 60 * time.Second
ENABLE_ZERO_DOWNTIME = "ENABLE_ZERO_DOWNTIME"
)
// RayServiceReconciler reconciles a RayService object
type RayServiceReconciler struct {
client.Client
Scheme *runtime.Scheme
Recorder record.EventRecorder
// Currently, the Ray dashboard doesn't cache the Serve application config.
// To avoid reapplying the same config repeatedly, cache the config in this map.
// Cache key is the combination of RayService namespace and name.
// Cache value is map of RayCluster name to Serve application config.
ServeConfigs *lru.Cache
RayClusterDeletionTimestamps cmap.ConcurrentMap[string, time.Time]
dashboardClientFunc func(rayCluster *rayv1.RayCluster, url string) (dashboardclient.RayDashboardClientInterface, error)
httpProxyClientFunc func(hostIp, podNamespace, podName string, port int) utils.RayHttpProxyClientInterface
}
// NewRayServiceReconciler returns a new reconcile.Reconciler
func NewRayServiceReconciler(ctx context.Context, mgr manager.Manager, provider utils.ClientProvider) *RayServiceReconciler {
dashboardClientFunc := provider.GetDashboardClient(ctx, mgr)
httpProxyClientFunc := provider.GetHttpProxyClient(mgr)
return &RayServiceReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("rayservice-controller"),
ServeConfigs: lru.New(utils.ServeConfigLRUSize),
RayClusterDeletionTimestamps: cmap.New[time.Time](),
dashboardClientFunc: dashboardClientFunc,
httpProxyClientFunc: httpProxyClientFunc,
}
}
// +kubebuilder:rbac:groups=ray.io,resources=rayservices,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=ray.io,resources=rayservices/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=ray.io,resources=rayservices/finalizers,verbs=update
// +kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=ray.io,resources=rayclusters/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=ray.io,resources=rayclusters/finalizers,verbs=update
// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=pods/proxy,verbs=get;update;patch
// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=services/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=core,resources=services/proxy,verbs=get;update;patch
// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update
// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create;delete
// +kubebuilder:rbac:groups="gateway.networking.k8s.io",resources=gateways,verbs=get;list;watch;create;update;delete
// +kubebuilder:rbac:groups="gateway.networking.k8s.io",resources=httproutes,verbs=get;list;watch;create;update;delete
// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;delete;update
// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;delete
// +kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch
// [WARNING]: There MUST be a newline after kubebuilder markers.
// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
//
// This the top level reconciliation flow for RayService.
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
func (r *RayServiceReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
logger := ctrl.LoggerFrom(ctx)
rayServiceInstance, err := r.getRayServiceInstance(ctx, request)
if err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
originalRayServiceInstance := rayServiceInstance.DeepCopy()
if manager := utils.ManagedByExternalController(rayServiceInstance.Spec.ManagedBy); manager != nil {
logger.Info("Skipping RayService managed by a custom controller", "managed-by", manager)
return ctrl.Result{}, nil
}
r.cleanUpServeConfigCache(ctx, rayServiceInstance)
hasRayClustersToClean, err := r.cleanUpRayClusterInstance(ctx, rayServiceInstance)
if err != nil {
return ctrl.Result{}, err
}
if !rayServiceInstance.DeletionTimestamp.IsZero() {
if !hasRayClustersToClean {
logger.Info("No RayClusters exist, removing finalizer")
controllerutil.RemoveFinalizer(rayServiceInstance, utils.RayServiceFinalizer)
if err := r.Update(ctx, rayServiceInstance); err != nil {
logger.Error(err, "Failed to remove finalizer for RayService")
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, err
}
return ctrl.Result{}, nil
}
logger.Info("Requeue RayService, waiting for RayClusters to be deleted")
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, err
}
if !controllerutil.ContainsFinalizer(rayServiceInstance, utils.RayServiceFinalizer) {
logger.Info("Adding finalizer to RayService", "finalizer", utils.RayServiceFinalizer)
controllerutil.AddFinalizer(rayServiceInstance, utils.RayServiceFinalizer)
if err := r.Update(ctx, rayServiceInstance); err != nil {
logger.Error(err, "Failed to add finalizer to RayService")
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, err
}
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, nil
}
// Perform all validations and directly fail the RayService if any of the validation fails
errType, err := validateRayService(ctx, rayServiceInstance)
// Immediately update the status after validation
if err != nil {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(errType),
"%s/%s: %v", rayServiceInstance.Namespace, rayServiceInstance.Name, err)
setCondition(rayServiceInstance, rayv1.RayServiceReady, metav1.ConditionFalse, rayv1.RayServiceValidationFailed, err.Error())
rayServiceInstance.Status.LastUpdateTime = &metav1.Time{Time: time.Now()}
if updateErr := r.Status().Update(ctx, rayServiceInstance); updateErr != nil {
logger.Info("Failed to update RayService status", "error", updateErr)
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, updateErr
}
return ctrl.Result{}, nil
}
result, err := r.handleSuspend(ctx, rayServiceInstance)
if updateErr := r.updateStatusIfChanged(ctx, originalRayServiceInstance, rayServiceInstance); updateErr != nil && err == nil {
err = updateErr
if (result == ctrl.Result{}) {
result = ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}
}
}
if err != nil || suspendIsOperative(rayServiceInstance) {
return result, err
}
// If the RayService has timed out during initialization, skip the rest of the reconciliation.
// The service is in a terminal failure state - only cleanup (above) is needed.
// The user must delete and recreate the RayService to recover.
if isInitializingTimeout(rayServiceInstance) {
// Requeue only if there are still RayClusters to clean up
// This avoids unnecessary reconciliations after all resources are deleted
if hasRayClustersToClean {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, nil
}
logger.Info("RayService in terminal failure state, all RayClusters cleaned up")
return ctrl.Result{}, nil
}
// Find active and pending ray cluster objects given current service name.
var activeRayClusterInstance, pendingRayClusterInstance *rayv1.RayCluster
if activeRayClusterInstance, pendingRayClusterInstance, err = r.reconcileRayCluster(ctx, rayServiceInstance); err != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, client.IgnoreNotFound(err)
}
// Determine the rollback state immediately before making serving decisions.
if utils.IsIncrementalUpgradeEnabled(&rayServiceInstance.Spec) {
// If an upgrade is in progress, check if rollback is necessary.
isUpgradeInProgress := meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.UpgradeInProgress))
if isUpgradeInProgress {
if activeRayClusterInstance == nil {
logger.Info("Cannot initiate rollback: active cluster not found")
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.RollbackImpossible), "Active cluster not found, rollback cannot be initiated")
} else if pendingRayClusterInstance != nil {
if err := r.reconcileRollbackState(ctx, rayServiceInstance, activeRayClusterInstance, pendingRayClusterInstance); err != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, err
}
}
}
}
// Check both active and pending Ray clusters to see if the head Pod is ready to serve requests.
// This is important to ensure the reliability of the serve service because the head Pod cannot
// rely on readiness probes to determine serve readiness.
if err := r.updateHeadPodServeLabel(ctx, rayServiceInstance, activeRayClusterInstance, rayServiceInstance.Spec.ExcludeHeadPodFromServeSvc); err != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, err
}
if err := r.updateHeadPodServeLabel(ctx, rayServiceInstance, pendingRayClusterInstance, rayServiceInstance.Spec.ExcludeHeadPodFromServeSvc); err != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, err
}
// Reconcile serve applications for active and/or pending clusters
// 1. If there is a pending cluster, reconcile serve applications for the pending cluster.
// 2. If there are both active and pending clusters, reconcile serve applications for the pending cluster only.
// 3. If there is no pending cluster, reconcile serve applications for the active cluster.
// 4. During NewClusterWithIncrementalUpgrade, reconcileServe will reconcile either the pending or active cluster
// based on total TargetCapacity.
isActiveClusterReady, isPendingClusterReady := false, false
var activeClusterServeApplications, pendingClusterServeApplications map[string]rayv1.AppStatus = nil, nil
if pendingRayClusterInstance != nil {
logger.Info("Reconciling the Serve applications for pending cluster", "clusterName", pendingRayClusterInstance.Name)
if isPendingClusterReady, pendingClusterServeApplications, err = r.reconcileServe(ctx, rayServiceInstance, pendingRayClusterInstance); err != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, err
}
}
if activeRayClusterInstance != nil && pendingRayClusterInstance == nil &&
!shouldPrepareNewCluster(ctx, rayServiceInstance, activeRayClusterInstance, nil, false) {
// Only reconcile serve applications for the active cluster when there is no pending cluster. That is, during the upgrade process,
// in-place update and updating the serve application status for the active cluster will not work.
logger.Info("Reconciling the Serve applications for active cluster", "clusterName", activeRayClusterInstance.Name)
if isActiveClusterReady, activeClusterServeApplications, err = r.reconcileServe(ctx, rayServiceInstance, activeRayClusterInstance); err != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, err
}
} else if activeRayClusterInstance != nil && pendingRayClusterInstance != nil && utils.IsIncrementalUpgradeEnabled(&rayServiceInstance.Spec) {
logger.Info("Reconciling the Serve applications for active cluster during NewClusterWithIncrementalUpgrade", "clusterName", activeRayClusterInstance.Name)
if isActiveClusterReady, activeClusterServeApplications, err = r.reconcileServe(ctx, rayServiceInstance, activeRayClusterInstance); err != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, err
}
}
// Check if NewClusterWithIncrementalUpgrade is enabled, if so reconcile Gateway objects.
var httpRouteInstance *gwv1.HTTPRoute
if utils.IsIncrementalUpgradeEnabled(&rayServiceInstance.Spec) {
// Ensure per-cluster Serve service exists for the active and pending RayClusters.
if err = r.reconcilePerClusterServeService(ctx, rayServiceInstance, activeRayClusterInstance); err != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, err
}
if err = r.reconcilePerClusterServeService(ctx, rayServiceInstance, pendingRayClusterInstance); err != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, err
}
// Creates or updates a Gateway CR that points to the Serve services of
// the active and pending (if it exists) RayClusters. For incremental upgrades,
// the Gateway endpoint is used rather than the Serve service.
err = r.reconcileGateway(ctx, rayServiceInstance)
if err != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, client.IgnoreNotFound(err)
}
// Create or update the HTTPRoute for the Gateway, passing in the pending cluster readiness status.
httpRouteInstance, err = r.reconcileHTTPRoute(ctx, rayServiceInstance, isPendingClusterReady)
if err != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, client.IgnoreNotFound(err)
}
}
// Reconcile K8s services and make sure it points to the correct RayCluster.
var headSvc, serveSvc *corev1.Service
if isPendingClusterReady || isActiveClusterReady {
targetCluster := activeRayClusterInstance
logMsg := "Reconciling K8s services to point to the active Ray cluster."
isIncrementalUpgradeInProgress := utils.IsIncrementalUpgradeEnabled(&rayServiceInstance.Spec) && meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.UpgradeInProgress))
if isPendingClusterReady && !isIncrementalUpgradeInProgress {
// This step is skipped for incremental upgrade, because the pending cluster is ready during the upgrade
// and creates its own per-cluster Serve service.
targetCluster = pendingRayClusterInstance
logMsg = "Reconciling K8s services to point to the pending Ray cluster to switch traffic because it is ready."
}
logger.Info(logMsg)
headSvc, serveSvc, err = r.reconcileServicesToReadyCluster(ctx, rayServiceInstance, targetCluster)
if err != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, err
}
if headSvc == nil || serveSvc == nil {
panic(fmt.Sprintf("Either head service (%v) or serve service (%v) is nil before calculating RayService status. "+
"This should never happen. Please open a GitHub issue in the KubeRay repository.", headSvc, serveSvc))
}
}
// Calculate the status of the RayService based on K8s resources.
if err := r.calculateStatus(
ctx,
rayServiceInstance,
headSvc,
serveSvc,
activeRayClusterInstance,
pendingRayClusterInstance,
activeClusterServeApplications,
pendingClusterServeApplications,
httpRouteInstance,
); err != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, err
}
// Final status update for any CR modification.
if utils.InconsistentRayServiceStatuses(originalRayServiceInstance.Status, rayServiceInstance.Status) {
rayServiceInstance.Status.LastUpdateTime = &metav1.Time{Time: time.Now()}
if errStatus := r.Status().Update(ctx, rayServiceInstance); errStatus != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, errStatus
}
}
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, nil
}
func validateRayService(ctx context.Context, rayServiceInstance *rayv1.RayService) (utils.K8sEventType, error) {
logger := ctrl.LoggerFrom(ctx)
validationRules := []struct {
validate func() error
errType utils.K8sEventType
}{
{func() error { return utils.ValidateRayServiceMetadata(rayServiceInstance.ObjectMeta) }, utils.InvalidRayServiceMetadata},
{func() error { return utils.ValidateRayServiceSpec(rayServiceInstance) }, utils.InvalidRayServiceSpec},
}
for _, validation := range validationRules {
if err := validation.validate(); err != nil {
logger.Error(err, err.Error())
return validation.errType, err
}
}
return "", nil
}
// suspendIsOperative reports whether the suspend state machine currently
// owns this reconcile pass. When true, Reconcile must short-circuit after
// persisting status: running the rest of the loop would either re-create
// resources that handleSuspend tore down or overwrite the conditions it
// staged.
//
// Reading Suspending / Suspended is the canonical "is suspend driving?"
// check because those two conditions must only be mutated by handleSuspend
// (and are the source of truth for the suspend state machine after it
// returns). Do not write to them from anywhere else.
func suspendIsOperative(rayServiceInstance *rayv1.RayService) bool {
return meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.RayServiceSuspending)) ||
meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.RayServiceSuspended))
}
// handleSuspend implements the Spec.Suspend lifecycle. It mutates
// rayServiceInstance.Status in-place; the caller persists the changes via a
// single Status().Update.
//
// State machine:
//
// (no suspend) --Spec.Suspend=true--> Suspending --owned resources deleted--> Suspended --Spec.Suspend=false--> (no suspend)
//
// Atomicity comes from a persisted Suspending condition as the commit point.
// The first reconcile that observes Spec.Suspend=true only stages the status
// transition (Suspending=True + reset of ActiveServiceStatus,
// PendingServiceStatus, NumServeEndpoints, ServiceStatus) in the same status
// update; deletion runs on the next reconcile, once Suspending is durable. If
// a later deletion attempt errors out or Spec.Suspend is flipped back to
// false, Suspending stays True in storage and subsequent reconciles continue
// the cleanup.
func (r *RayServiceReconciler) handleSuspend(ctx context.Context, rayServiceInstance *rayv1.RayService) (ctrl.Result, error) {
logger := ctrl.LoggerFrom(ctx)
isSuspending := meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.RayServiceSuspending))
isSuspended := meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.RayServiceSuspended))
// Case 1: already fully suspended.
if isSuspended {
if !rayServiceInstance.Spec.Suspend {
logger.Info("Spec.Suspend is false; exiting Suspended state and resuming reconcile")
rayServiceInstance.Status.ObservedGeneration = rayServiceInstance.ObjectMeta.Generation
setCondition(rayServiceInstance, rayv1.RayServiceSuspended, metav1.ConditionFalse, rayv1.RayServiceResumed,
"Spec.Suspend is false; RayService has resumed.")
return ctrl.Result{}, nil
}
// Stay suspended; nothing to reconcile.
return ctrl.Result{}, nil
}
// Case 2: Spec.Suspend just transitioned to true. Stage the status
// transition (Suspending=True + reset status fields) and return so the
// caller can persist it. Deletion runs on the next reconcile once
// Suspending is durable.
if !isSuspending {
if !rayServiceInstance.Spec.Suspend {
return ctrl.Result{}, nil
}
logger.Info("Spec.Suspend is true; committing transition to Suspending state")
rayServiceInstance.Status.ObservedGeneration = rayServiceInstance.ObjectMeta.Generation
setCondition(rayServiceInstance, rayv1.RayServiceSuspending, metav1.ConditionTrue, rayv1.SuspendRequested,
"Spec.Suspend is true; will delete all Kubernetes resources owned by this RayService.")
setCondition(rayServiceInstance, rayv1.RayServiceReady, metav1.ConditionFalse, rayv1.SuspendInProgress, "RayService is suspending.")
setCondition(rayServiceInstance, rayv1.UpgradeInProgress, metav1.ConditionFalse, rayv1.SuspendInProgress,
"No upgrade in progress.")
setCondition(rayServiceInstance, rayv1.RollbackInProgress, metav1.ConditionFalse, rayv1.SuspendInProgress,
"No rollback in progress.")
rayServiceInstance.Status.ActiveServiceStatus = rayv1.RayServiceStatus{}
rayServiceInstance.Status.PendingServiceStatus = rayv1.RayServiceStatus{}
rayServiceInstance.Status.NumServeEndpoints = 0
rayServiceInstance.Status.ServiceStatus = rayv1.NotRunning
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, nil
}
// Case 3: Suspending is committed in storage. Delete owned resources.
// Atomic: ignore Spec.Suspend here — once Suspending is persisted, the
// deletion always runs to completion.
allDeleted, err := r.deleteRayServiceOwnedResources(ctx, rayServiceInstance)
if err != nil {
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, err
}
if !allDeleted {
setCondition(rayServiceInstance, rayv1.RayServiceSuspending, metav1.ConditionTrue, rayv1.SuspendInProgress,
"Waiting for all Kubernetes resources owned by this RayService to be deleted.")
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, nil
}
// All resources deleted: transition to Suspended. Requeue so that the
// next reconcile observes Spec.Suspend; this matters when the user
// flipped Spec.Suspend back to false mid-suspend (atomic completion put
// us here despite Spec.Suspend=false), since the status-only update
// below would not otherwise wake the controller up.
logger.Info("All RayService-owned resources deleted; transitioning to Suspended")
setCondition(rayServiceInstance, rayv1.RayServiceSuspending, metav1.ConditionFalse, rayv1.SuspendComplete,
"All owned resources have been deleted.")
setCondition(rayServiceInstance, rayv1.RayServiceSuspended, metav1.ConditionTrue, rayv1.SuspendComplete, "All owned resources have been deleted.")
setCondition(rayServiceInstance, rayv1.RayServiceReady, metav1.ConditionFalse, rayv1.SuspendComplete, "RayService is suspended.")
setCondition(rayServiceInstance, rayv1.UpgradeInProgress, metav1.ConditionFalse, rayv1.SuspendComplete,
"No upgrade in progress.")
setCondition(rayServiceInstance, rayv1.RollbackInProgress, metav1.ConditionFalse, rayv1.SuspendComplete,
"No rollback in progress.")
return ctrl.Result{RequeueAfter: ServiceDefaultRequeueDuration}, nil
}
// deleteRayServiceOwnedResources deletes every Kubernetes resource that the
// RayService controller owns: RayClusters, head/serve Services, and — when
// RayServiceIncrementalUpgrade is enabled — the Gateway and HTTPRoute.
// Per-cluster serve Services used by incremental upgrade are owned by their
// RayCluster, so they are garbage-collected when the cluster is deleted above.
// Returns true when nothing remained to be deleted.
func (r *RayServiceReconciler) deleteRayServiceOwnedResources(ctx context.Context, rayServiceInstance *rayv1.RayService) (bool, error) {
logger := ctrl.LoggerFrom(ctx)
allDeleted := true
// RayClusters.
rayClusterList := rayv1.RayClusterList{}
if err := r.List(ctx, &rayClusterList, common.RayServiceRayClustersAssociationOptions(rayServiceInstance).ToListOptions()...); err != nil {
return false, err
}
for i := range rayClusterList.Items {
cluster := &rayClusterList.Items[i]
allDeleted = false
if !cluster.DeletionTimestamp.IsZero() {
continue
}
logger.Info("Deleting RayCluster for suspend", "name", cluster.Name)
if err := r.Delete(ctx, cluster, client.PropagationPolicy(metav1.DeletePropagationBackground)); err != nil && !errors.IsNotFound(err) {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToDeleteRayCluster),
"Failed to delete the RayCluster %s/%s during suspend: %v", cluster.Namespace, cluster.Name, err)
return false, err
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.DeletedRayCluster),
"Deleted the RayCluster %s/%s during suspend", cluster.Namespace, cluster.Name)
}
// Kubernetes Services (head + serve).
svcList := corev1.ServiceList{}
if err := r.List(ctx, &svcList, common.RayServiceRayClustersAssociationOptions(rayServiceInstance).ToListOptions()...); err != nil {
return false, err
}
for i := range svcList.Items {
svc := &svcList.Items[i]
allDeleted = false
if !svc.DeletionTimestamp.IsZero() {
continue
}
logger.Info("Deleting Service for suspend", "name", svc.Name)
if err := r.Delete(ctx, svc, client.PropagationPolicy(metav1.DeletePropagationBackground)); err != nil && !errors.IsNotFound(err) {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToDeleteService),
"Failed to delete the Service %s/%s during suspend: %v", svc.Namespace, svc.Name, err)
return false, err
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.DeletedService),
"Deleted the Service %s/%s during suspend", svc.Namespace, svc.Name)
}
if utils.IsIncrementalUpgradeEnabled(&rayServiceInstance.Spec) {
gateway := &gwv1.Gateway{}
if err := r.Get(ctx, common.RayServiceGatewayNamespacedName(rayServiceInstance), gateway); err == nil {
allDeleted = false
if gateway.DeletionTimestamp.IsZero() {
logger.Info("Deleting Gateway for suspend", "name", gateway.Name)
if err := r.Delete(ctx, gateway, client.PropagationPolicy(metav1.DeletePropagationBackground)); err != nil && !errors.IsNotFound(err) {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToDeleteGateway),
"Failed to delete the Gateway %s/%s during suspend: %v", gateway.Namespace, gateway.Name, err)
return false, err
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.DeletedGateway),
"Deleted the Gateway %s/%s during suspend", gateway.Namespace, gateway.Name)
}
} else if !errors.IsNotFound(err) {
return false, err
}
httpRoute := &gwv1.HTTPRoute{}
if err := r.Get(ctx, common.RayServiceHTTPRouteNamespacedName(rayServiceInstance), httpRoute); err == nil {
allDeleted = false
if httpRoute.DeletionTimestamp.IsZero() {
logger.Info("Deleting HTTPRoute for suspend", "name", httpRoute.Name)
if err := r.Delete(ctx, httpRoute, client.PropagationPolicy(metav1.DeletePropagationBackground)); err != nil && !errors.IsNotFound(err) {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToDeleteHTTPRoute),
"Failed to delete the HTTPRoute %s/%s during suspend: %v", httpRoute.Namespace, httpRoute.Name, err)
return false, err
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.DeletedHTTPRoute),
"Deleted the HTTPRoute %s/%s during suspend", httpRoute.Namespace, httpRoute.Name)
}
} else if !errors.IsNotFound(err) {
return false, err
}
}
return allDeleted, nil
}
// updateStatusIfChanged persists rayServiceInstance.Status if it differs from
// originalRayService.Status. It mirrors the status-update path used at the end
// of Reconcile so that handleSuspend can finalize a reconcile early.
func (r *RayServiceReconciler) updateStatusIfChanged(ctx context.Context, originalRayService, rayServiceInstance *rayv1.RayService) error {
if !utils.InconsistentRayServiceStatuses(originalRayService.Status, rayServiceInstance.Status) {
return nil
}
rayServiceInstance.Status.LastUpdateTime = &metav1.Time{Time: time.Now()}
return r.Status().Update(ctx, rayServiceInstance)
}
func (r *RayServiceReconciler) reconcileServicesToReadyCluster(ctx context.Context, rayServiceInstance *rayv1.RayService, rayClusterInstance *rayv1.RayCluster) (*corev1.Service, *corev1.Service, error) {
// Create K8s services if they don't exist. If they do exist, update the services to point to the RayCluster passed in.
headSvc, err := r.reconcileServices(ctx, rayServiceInstance, rayClusterInstance, utils.HeadService)
if err != nil {
return headSvc, nil, err
}
serveSvc, err := r.reconcileServices(ctx, rayServiceInstance, rayClusterInstance, utils.ServingService)
if err != nil {
return headSvc, serveSvc, err
}
return headSvc, serveSvc, nil
}
// reconcilePromotionAndServingStatus handles the promotion logic after an upgrade, returning
// isPendingClusterServing: True if the main Kubernetes services are pointing to the pending cluster.
func reconcilePromotionAndServingStatus(ctx context.Context, headSvc, serveSvc *corev1.Service, rayServiceInstance *rayv1.RayService, pendingCluster *rayv1.RayCluster) (isPendingClusterServing bool) {
logger := ctrl.LoggerFrom(ctx)
// Step 1: Service Consistency Check. Ensure head and serve services point to the
// same cluster (active or pending).
clusterSvcPointsTo := utils.GetRayClusterNameFromService(headSvc)
if clusterSvcPointsTo != utils.GetRayClusterNameFromService(serveSvc) {
// This indicates a broken state that the controller cannot recover from automatically.
panic("headSvc and serveSvc are not pointing to the same cluster")
}
// Step 2: Cluster Switching Logic. Determine which cluster the services are currently pointing to and
// determine if promotion should occur.
pendingClusterName := rayServiceInstance.Status.PendingServiceStatus.RayClusterName
activeClusterName := rayServiceInstance.Status.ActiveServiceStatus.RayClusterName
// Verify that the service points to a known cluster (either active or pending).
if clusterSvcPointsTo != pendingClusterName && clusterSvcPointsTo != activeClusterName {
panic("clusterName from services is not equal to pendingCluster or activeCluster")
}
var shouldPromote bool
if utils.IsIncrementalUpgradeEnabled(&rayServiceInstance.Spec) {
// An incremental upgrade is complete when the active cluster has 0% capacity and the pending cluster has
// 100% of the traffic. We can't promote the pending cluster until traffic has been fully migrated.
if meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.UpgradeInProgress)) {
isRollbackInProgress := meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.RollbackInProgress))
// Do not promote the pending cluster during a rollback. Traffic is being
// shifted back to the active cluster, so the pending cluster must not
// become the new active even if its TargetCapacity and TrafficRoutedPercent
// reach the promotion thresholds.
if !isRollbackInProgress && utils.IsIncrementalUpgradeComplete(rayServiceInstance, pendingCluster) {
shouldPromote = true
logger.Info("Incremental upgrade completed, triggering promotion.", "rayService", rayServiceInstance.Name)
}
} else if activeClusterName == "" && pendingClusterName != "" {
// The Active cluster is empty when the RayCluster is first scaling up.
shouldPromote = true
}
} else {
// For traditional blue/green upgrade, promotion is complete when the Service selector has switched.
if activeClusterName != clusterSvcPointsTo {
shouldPromote = true
}
}
// Step 3: Promote the pending cluster if prior conditions are met.
if shouldPromote {
logger.Info("Promoting pending cluster to active.",
"oldCluster", rayServiceInstance.Status.ActiveServiceStatus.RayClusterName,
"newCluster", rayServiceInstance.Status.PendingServiceStatus.RayClusterName)
rayServiceInstance.Status.ActiveServiceStatus = rayServiceInstance.Status.PendingServiceStatus
rayServiceInstance.Status.PendingServiceStatus = rayv1.RayServiceStatus{}
}
return (clusterSvcPointsTo == pendingClusterName)
}
func (r *RayServiceReconciler) calculateStatus(
ctx context.Context,
rayServiceInstance *rayv1.RayService,
headSvc, serveSvc *corev1.Service,
activeCluster, pendingCluster *rayv1.RayCluster,
activeClusterServeApplications, pendingClusterServeApplications map[string]rayv1.AppStatus,
httpRoute *gwv1.HTTPRoute,
) error {
logger := ctrl.LoggerFrom(ctx)
rayServiceInstance.Status.ObservedGeneration = rayServiceInstance.ObjectMeta.Generation
// Update RayClusterStatus in RayService status.
var activeClusterStatus, pendingClusterStatus rayv1.RayClusterStatus
if activeCluster != nil {
activeClusterStatus = activeCluster.Status
}
if pendingCluster != nil {
pendingClusterStatus = pendingCluster.Status
}
rayServiceInstance.Status.ActiveServiceStatus.RayClusterStatus = activeClusterStatus
rayServiceInstance.Status.PendingServiceStatus.RayClusterStatus = pendingClusterStatus
// Update serve application status in RayService status.
rayServiceInstance.Status.ActiveServiceStatus.Applications = activeClusterServeApplications
rayServiceInstance.Status.PendingServiceStatus.Applications = pendingClusterServeApplications
var isPendingClusterServing bool
if headSvc != nil && serveSvc != nil {
if utils.IsIncrementalUpgradeEnabled(&rayServiceInstance.Spec) {
logger.Info("Processing NewClusterWithIncrementalUpgrade strategy.", "rayService", rayServiceInstance.Name)
oldActivePercent := ptr.Deref(rayServiceInstance.Status.ActiveServiceStatus.TrafficRoutedPercent, -1)
oldPendingPercent := ptr.Deref(rayServiceInstance.Status.PendingServiceStatus.TrafficRoutedPercent, -1)
// Update TrafficRoutedPercent to each RayService based on current weights from HTTPRoute.
activeWeight, pendingWeight := utils.GetWeightsFromHTTPRoute(httpRoute, rayServiceInstance)
now := metav1.Time{Time: time.Now()}
if activeWeight >= 0 {
rayServiceInstance.Status.ActiveServiceStatus.TrafficRoutedPercent = new(activeWeight)
logger.Info("Updated active TrafficRoutedPercent from HTTPRoute", "activeClusterWeight", activeWeight)
if activeWeight != oldActivePercent {
rayServiceInstance.Status.ActiveServiceStatus.LastTrafficMigratedTime = &now
logger.Info("Updated LastTrafficMigratedTime of Active Service.")
}
}
if pendingWeight >= 0 && pendingCluster != nil {
rayServiceInstance.Status.PendingServiceStatus.TrafficRoutedPercent = new(pendingWeight)
logger.Info("Updated pending TrafficRoutedPercent from HTTPRoute", "pendingClusterWeight", pendingWeight)
if pendingWeight != oldPendingPercent {
rayServiceInstance.Status.PendingServiceStatus.LastTrafficMigratedTime = &now
logger.Info("Updated LastTrafficMigratedTime of Pending Service.")
}
}
}
// Reconcile serving status and promotion logic for all upgrade strategies.
isPendingClusterServing = reconcilePromotionAndServingStatus(ctx, headSvc, serveSvc, rayServiceInstance, pendingCluster)
}
if meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.RollbackInProgress)) {
activeStatus := &rayServiceInstance.Status.ActiveServiceStatus
pendingStatus := &rayServiceInstance.Status.PendingServiceStatus
if shouldCompleteIncrementalRollback(activeStatus, pendingStatus, pendingCluster) {
logger.Info("Rollback to original cluster is complete. Cleaning up pending cluster from prior upgrade.")
// Clear the RayService pending service status to clean up the pending cluster.
rayServiceInstance.Status.PendingServiceStatus = rayv1.RayServiceStatus{}
pendingCluster = nil
meta.RemoveStatusCondition(&rayServiceInstance.Status.Conditions, string(rayv1.RollbackInProgress))
// Ensure the upgrade state machine resets after a successful rollback.
setCondition(rayServiceInstance, rayv1.UpgradeInProgress, metav1.ConditionFalse, rayv1.NoPendingCluster, "Rollback complete, active Ray cluster exists and no pending Ray cluster")
}
}
if shouldPrepareNewCluster(ctx, rayServiceInstance, activeCluster, pendingCluster, isPendingClusterServing) {
rayServiceInstance.Status.PendingServiceStatus = rayv1.RayServiceStatus{
RayClusterName: utils.GenerateRayClusterName(rayServiceInstance.Name),
}
logger.Info("Preparing a new pending RayCluster instance by setting RayClusterName",
"clusterName", rayServiceInstance.Status.PendingServiceStatus.RayClusterName)
if utils.IsIncrementalUpgradeEnabled(&rayServiceInstance.Spec) {
// Set IncrementalUpgrade related Status fields for new pending RayCluster if enabled.
if rayServiceInstance.Status.ActiveServiceStatus.RayClusterName == "" {
// If no Active RayCluster exists - default to starting with 100% TargetCapacity.
// This is the case when a RayCluster is first starting for a RayService, so we should
// immediately scale it to full target capacity.
if rayServiceInstance.Status.ActiveServiceStatus.TargetCapacity == nil {
rayServiceInstance.Status.PendingServiceStatus.TargetCapacity = new(int32(100))
}
} else {
// Pending RayCluster during an upgrade should start with 0% TargetCapacity, since
// traffic will be gradually migrated to the new cluster.
if rayServiceInstance.Status.PendingServiceStatus.TargetCapacity == nil {
rayServiceInstance.Status.PendingServiceStatus.TargetCapacity = new(int32(0))
}
}
}
}
// Calculate the number of ready serve endpoints for the active cluster.
serveServiceNamespacedName := common.RayServiceServeServiceNamespacedName(rayServiceInstance)
if utils.IsIncrementalUpgradeEnabled(&rayServiceInstance.Spec) && activeCluster != nil {
// The Serve service name is based on the unique RayCluster name, since we use the
// per-cluster Serve services for traffic routing during an incremental upgrade.
serveServiceNamespacedName.Name = utils.GenerateServeServiceName(activeCluster.Name)
}
numServeEndpoints, err := r.calculateNumServeEndpointsFromSlices(ctx, serveServiceNamespacedName)
if err != nil {
return err
}
// During NewClusterWithIncrementalUpgrade, the pending RayCluster is also serving.
if utils.IsIncrementalUpgradeEnabled(&rayServiceInstance.Spec) && pendingCluster != nil {
pendingServeServiceNamespacedName := common.RayClusterServeServiceNamespacedName(pendingCluster)
pendingEndpoints, err := r.calculateNumServeEndpointsFromSlices(ctx, pendingServeServiceNamespacedName)
if err != nil {
return err
}
numServeEndpoints += pendingEndpoints
}
if numServeEndpoints > math.MaxInt32 {
return errstd.New("numServeEndpoints exceeds math.MaxInt32")
}
rayServiceInstance.Status.NumServeEndpoints = int32(numServeEndpoints) //nolint:gosec // This is a false positive from gosec. See https://github.com/securego/gosec/issues/1212 for more details.
// Calculate conditions based on current state (endpoints, clusters, timeout, etc.)
calculateConditions(ctx, r, rayServiceInstance)
// The definition of `ServiceStatus` is equivalent to the `RayServiceReady` condition
rayServiceInstance.Status.ServiceStatus = rayv1.NotRunning
if meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.RayServiceReady)) {
rayServiceInstance.Status.ServiceStatus = rayv1.Running
}
return nil
}
func calculateConditions(ctx context.Context, r *RayServiceReconciler, rayServiceInstance *rayv1.RayService) {
if rayServiceInstance.Status.Conditions == nil {
rayServiceInstance.Status.Conditions = []metav1.Condition{}
}
if len(rayServiceInstance.Status.Conditions) == 0 {
message := "RayService is initializing"
setCondition(rayServiceInstance, rayv1.RayServiceReady, metav1.ConditionFalse, rayv1.RayServiceInitializing, message)
setCondition(rayServiceInstance, rayv1.UpgradeInProgress, metav1.ConditionFalse, rayv1.RayServiceInitializing, message)
}
if rayServiceInstance.Status.NumServeEndpoints > 0 {
setCondition(rayServiceInstance, rayv1.RayServiceReady, metav1.ConditionTrue, rayv1.NonZeroServeEndpoints, "Number of serve endpoints is greater than 0")
} else if meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.RayServiceReady)) {
setCondition(rayServiceInstance, rayv1.RayServiceReady, metav1.ConditionFalse, rayv1.ZeroServeEndpoints, "Number of serve endpoints dropped to 0")
} else {
// Check if initializing timeout has been exceeded
// This runs after endpoint check, so if endpoints appear, they take priority
markFailedOnInitializingTimeout(ctx, r, rayServiceInstance)
}
activeClusterName := rayServiceInstance.Status.ActiveServiceStatus.RayClusterName
pendingClusterName := rayServiceInstance.Status.PendingServiceStatus.RayClusterName
if activeClusterName != "" && pendingClusterName != "" {
setCondition(rayServiceInstance, rayv1.UpgradeInProgress, metav1.ConditionTrue, rayv1.BothActivePendingClustersExist, "Both active and pending Ray clusters exist")
} else if activeClusterName != "" {
setCondition(rayServiceInstance, rayv1.UpgradeInProgress, metav1.ConditionFalse, rayv1.NoPendingCluster, "Active Ray cluster exists and no pending Ray cluster")
} else {
cond := meta.FindStatusCondition(rayServiceInstance.Status.Conditions, string(rayv1.UpgradeInProgress))
// Don't override the condition if RayService is initializing or has timed out
if cond == nil || (cond.Reason != string(rayv1.RayServiceInitializing) && cond.Reason != string(rayv1.RayServiceInitializingTimeout)) {
setCondition(rayServiceInstance, rayv1.UpgradeInProgress, metav1.ConditionUnknown, rayv1.NoActiveCluster, "No active Ray cluster exists, and the RayService is not initializing. Please open a GitHub issue in the KubeRay repository.")
}
}
}
func setCondition(rayServiceInstance *rayv1.RayService, conditionType rayv1.RayServiceConditionType, status metav1.ConditionStatus, reason rayv1.RayServiceConditionReason, message string) {
condition := metav1.Condition{
Type: string(conditionType),
Status: status,
Reason: string(reason),
Message: message,
ObservedGeneration: rayServiceInstance.Status.ObservedGeneration,
}
meta.SetStatusCondition(&rayServiceInstance.Status.Conditions, condition)
}
// SetupWithManager sets up the controller with the Manager.
func (r *RayServiceReconciler) SetupWithManager(mgr ctrl.Manager, reconcileConcurrency int) error {
return ctrl.NewControllerManagedBy(mgr).
For(&rayv1.RayService{}, builder.WithPredicates(predicate.Or(
predicate.GenerationChangedPredicate{},
predicate.LabelChangedPredicate{},
predicate.AnnotationChangedPredicate{},
))).
Owns(&rayv1.RayCluster{}).
Owns(&corev1.Service{}).
WithOptions(controller.Options{
MaxConcurrentReconciles: reconcileConcurrency,
LogConstructor: func(request *reconcile.Request) logr.Logger {
logger := ctrl.Log.WithName("controllers").WithName("RayService")
if request != nil {
logger = logger.WithValues("RayService", request.NamespacedName)
}
return logger
},
}).
Complete(r)
}
func (r *RayServiceReconciler) getRayServiceInstance(ctx context.Context, request ctrl.Request) (*rayv1.RayService, error) {
logger := ctrl.LoggerFrom(ctx)
rayServiceInstance := &rayv1.RayService{}
if err := r.Get(ctx, request.NamespacedName, rayServiceInstance); err != nil {
if errors.IsNotFound(err) {
logger.Info("Read request instance not found error!")
} else {
logger.Error(err, "Read request instance error!")
}
return nil, err
}
return rayServiceInstance, nil
}
func isZeroDowntimeUpgradeEnabled(ctx context.Context, upgradeStrategy *rayv1.RayServiceUpgradeStrategy) bool {
// For LLM serving, some users might not have sufficient GPU resources to run two RayClusters simultaneously.
// Therefore, KubeRay offers ENABLE_ZERO_DOWNTIME as a feature flag for zero-downtime upgrades.
// There are two ways to enable zero downtime upgrade. Through ENABLE_ZERO_DOWNTIME env var or setting Spec.UpgradeStrategy.Type.
// If no fields are set, zero downtime upgrade by default is enabled.
// Spec.UpgradeStrategy.Type takes precedence over ENABLE_ZERO_DOWNTIME.
logger := ctrl.LoggerFrom(ctx)
if upgradeStrategy != nil {
upgradeType := upgradeStrategy.Type
if upgradeType != nil {
if features.Enabled(features.RayServiceIncrementalUpgrade) {
if *upgradeType != rayv1.RayServiceNewCluster && *upgradeType != rayv1.RayServiceNewClusterWithIncrementalUpgrade {
logger.Info("Zero-downtime upgrade is disabled because UpgradeStrategy.Type is not set to %s or %s.", string(rayv1.RayServiceNewCluster), string(rayv1.RayServiceNewClusterWithIncrementalUpgrade))
return false
}
} else if *upgradeType != rayv1.RayServiceNewCluster {
logger.Info("Zero-downtime upgrade is disabled because UpgradeStrategy.Type is not set to NewCluster.")
return false
}
return true
}
}
zeroDowntimeEnvVar := os.Getenv(ENABLE_ZERO_DOWNTIME)
if strings.ToLower(zeroDowntimeEnvVar) == "false" {
logger.Info("Zero-downtime upgrade is disabled because ENABLE_ZERO_DOWNTIME is set to false.")
return false
}
return true
}
// `createGateway` creates a Gateway for a RayService or updates an existing Gateway.
func (r *RayServiceReconciler) createGateway(rayServiceInstance *rayv1.RayService) (*gwv1.Gateway, error) {
options := utils.GetRayServiceClusterUpgradeOptions(&rayServiceInstance.Spec)
if options == nil {
return nil, errstd.New("Missing RayService ClusterUpgradeOptions during upgrade.")
}
gatewayName := rayServiceInstance.Name + "-gateway"
// Define the desired Gateway object
rayServiceGateway := &gwv1.Gateway{
ObjectMeta: metav1.ObjectMeta{
Name: gatewayName,
Namespace: rayServiceInstance.Namespace,
},
Spec: gwv1.GatewaySpec{
GatewayClassName: gwv1.ObjectName(options.GatewayClassName),
Listeners: []gwv1.Listener{
{
Name: gwv1.SectionName(utils.GatewayListenerPortName),
Protocol: gwv1.HTTPProtocolType,
Port: utils.DefaultGatewayListenerPort,
},
},
},
}
return rayServiceGateway, nil
}
// `reconcileGateway` reconciles a Gateway resource for a RayService. The possible cases are:
// (1) Create a new Gateway instance. (2) Update the Gateway instance if RayService has updated. (3) Do nothing.
func (r *RayServiceReconciler) reconcileGateway(ctx context.Context, rayServiceInstance *rayv1.RayService) error {
logger := ctrl.LoggerFrom(ctx)
var err error
// Construct desired Gateway object for RayService
desiredGateway, err := r.createGateway(rayServiceInstance)
if err != nil {
logger.Error(err, "Failed to build Gateway object for Rayservice")
return err
}
if desiredGateway == nil {
logger.Info("Skipping Gateway reconciliation: desired Gateway is nil")
return nil
}
// Check for existing RayService Gateway, create the desired Gateway if none is found
existingGateway := &gwv1.Gateway{}
if err := r.Get(ctx, common.RayServiceGatewayNamespacedName(rayServiceInstance), existingGateway); err != nil {
if errors.IsNotFound(err) {
// Set the ownership in order to do the garbage collection by k8s.
if err := ctrl.SetControllerReference(rayServiceInstance, desiredGateway, r.Scheme); err != nil {
return err
}
logger.Info("Creating a new Gateway instance", "Gateway Listeners", desiredGateway.Spec.Listeners)
if err := r.Create(ctx, desiredGateway); err != nil {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToCreateGateway), "Failed to create Gateway for RayService %s/%s: %v", desiredGateway.Namespace, desiredGateway.Name, err)
return err
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.CreatedGateway), "Created Gateway for RayService %s/%s", desiredGateway.Namespace, desiredGateway.Name)
return nil
}
return err
}
// If Gateway already exists, check if update is needed to reach desired state
if !utils.IsGatewayEqual(existingGateway, desiredGateway) {
logger.Info("Updating existing Gateway", "name", existingGateway.Name)
existingGateway.Spec = desiredGateway.Spec
if err := r.Update(ctx, existingGateway); err != nil {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToUpdateGateway), "Failed to update the Gateway %s/%s: %v", existingGateway.Namespace, existingGateway.Name, err)
return err
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.UpdatedGateway), "Updated the Gateway %s/%s", existingGateway.Namespace, existingGateway.Name)
}
return nil
}
// calculateTrafficRoutedPercent determines the HTTPRoute traffic split between the active and pending RayClusters.
//
// The new weights are calculated using:
// - Current TrafficRoutedPercent values
// - Time-based migration using StepSizePercent and IntervalSeconds
// - TargetCapacity constraints
//
// Returns the active cluster traffic weight, pending cluster traffic weight, and an error if any.
func (r *RayServiceReconciler) calculateTrafficRoutedPercent(ctx context.Context, rayServiceInstance *rayv1.RayService, isPendingClusterReady bool) (activeClusterWeight, pendingClusterWeight int32, err error) {
logger := ctrl.LoggerFrom(ctx)
activeServiceStatus := &rayServiceInstance.Status.ActiveServiceStatus
pendingServiceStatus := &rayServiceInstance.Status.PendingServiceStatus
// Default to 100% traffic on the active cluster.
activeClusterWeight = ptr.Deref(activeServiceStatus.TrafficRoutedPercent, 100)
pendingClusterWeight = ptr.Deref(pendingServiceStatus.TrafficRoutedPercent, 0)
// Zero-downtime upgrade in progress.
options := utils.GetRayServiceClusterUpgradeOptions(&rayServiceInstance.Spec)
if options == nil {
return 0, 0, errstd.New("ClusterUpgradeOptions are not set during upgrade.")
}
// Check that target_capacity has been updated before migrating traffic.
pendingClusterTargetCapacity := ptr.Deref(pendingServiceStatus.TargetCapacity, 0)
activeClusterTargetCapacity := ptr.Deref(activeServiceStatus.TargetCapacity, 100)
isRollbackInProgress := meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.RollbackInProgress))
if (pendingClusterWeight == pendingClusterTargetCapacity && !isRollbackInProgress) || (isRollbackInProgress && activeClusterWeight == activeClusterTargetCapacity) {
// Stop traffic migration because the cluster being migrated to has reached its target capacity limit.
return activeClusterWeight, pendingClusterWeight, nil
}
// If IntervalSeconds has passed since LastTrafficMigratedTime, migrate StepSizePercent traffic
// from the active RayCluster to the pending RayCluster.
interval := time.Duration(*options.IntervalSeconds) * time.Second
// Determine which timestamp to use based on the direction of traffic flow.
// We use the LastTrafficMigratedTime of the cluster to which traffic is increasing.
var lastTrafficMigratedTime *metav1.Time
if isRollbackInProgress {
lastTrafficMigratedTime = activeServiceStatus.LastTrafficMigratedTime
} else {
lastTrafficMigratedTime = pendingServiceStatus.LastTrafficMigratedTime
}
if lastTrafficMigratedTime == nil || time.Since(lastTrafficMigratedTime.Time) >= interval {
if isRollbackInProgress {
// Gradually shift traffic from the pending to the active cluster.
// Rollback traffic migration occurs regardless of pending cluster readiness.
logger.Info("Rollback in progress. Shifting traffic back to active cluster.", "stepSize", *options.StepSizePercent)
proposedActiveWeight := activeClusterWeight + *options.StepSizePercent
activeClusterWeight = min(100, proposedActiveWeight, activeClusterTargetCapacity)
pendingClusterWeight = 100 - activeClusterWeight
} else if isPendingClusterReady {
// Gradually shift traffic from the active to the pending cluster if it's ready.
logger.Info("Upgrade in progress. Migrating traffic by StepSizePercent.", "stepSize", *options.StepSizePercent)
proposedPendingWeight := pendingClusterWeight + *options.StepSizePercent
pendingClusterWeight = min(100, proposedPendingWeight, pendingClusterTargetCapacity)
activeClusterWeight = 100 - pendingClusterWeight
} else {
logger.Info("Upgrade in progress, but pending cluster is not ready. Pausing traffic migration.")
}
}
return activeClusterWeight, pendingClusterWeight, nil
}
// createHTTPRoute creates a desired HTTPRoute object for RayService incremental upgrade.
//
// The function performs the following operations:
// 1. Retrieves Gateway instance to attach the HTTPRoute
// 2. Gets active and pending RayCluster instances and their Serve services
// 3. Calls `calculateTrafficRoutedPercent` to calculate the new traffic weights
// 4. Configures HTTPRoute with appropriate backend references and weights
//
// Returns the configured HTTPRoute object or error if any step fails.
func (r *RayServiceReconciler) createHTTPRoute(ctx context.Context, rayServiceInstance *rayv1.RayService, isPendingClusterReady bool) (*gwv1.HTTPRoute, error) {
logger := ctrl.LoggerFrom(ctx)
// Retrieve Gateway instance to attach this HTTPRoute to.
gatewayInstance := &gwv1.Gateway{}
if err := r.Get(ctx, common.RayServiceGatewayNamespacedName(rayServiceInstance), gatewayInstance); err != nil {
return nil, err
}
// Retrieve the active RayCluster
activeRayCluster, err := r.getRayClusterByNamespacedName(ctx, common.RayServiceActiveRayClusterNamespacedName(rayServiceInstance))
if err != nil && !errors.IsNotFound(err) {
logger.Error(err, "Failed to retrieve active RayCluster")
return nil, err
}
if activeRayCluster == nil {
logger.Info("Active RayCluster not found, skipping HTTPRoute creation.")
return nil, nil
}
// Attempt to retrieve pending RayCluster
pendingRayCluster, err := r.getRayClusterByNamespacedName(ctx, common.RayServicePendingRayClusterNamespacedName(rayServiceInstance))
if err != nil && !errors.IsNotFound(err) {
logger.Error(err, "Failed to retrieve pending RayCluster.")
return nil, err
}
activeClusterWeight, pendingClusterWeight, err := r.calculateTrafficRoutedPercent(ctx, rayServiceInstance, isPendingClusterReady)
if err != nil {
logger.Info("Failed to reconcile TrafficRoutedPercent for active and pending clusters.")
return nil, err
}
activeClusterServeSvcName := utils.GenerateServeServiceName(activeRayCluster.Name)
activeServePort := common.GetServePort(activeRayCluster)
backendRefs := []gwv1.HTTPBackendRef{
{
BackendRef: gwv1.BackendRef{
BackendObjectReference: gwv1.BackendObjectReference{
Name: gwv1.ObjectName(activeClusterServeSvcName),
Namespace: new(gwv1.Namespace(gatewayInstance.Namespace)),
Port: new(activeServePort),
},
Weight: new(activeClusterWeight),
},
},
}
if pendingRayCluster != nil {
logger.Info("Pending RayCluster exists. Including it in HTTPRoute.", "RayCluster", pendingRayCluster.Name)
pendingClusterServeSvcName := utils.GenerateServeServiceName(pendingRayCluster.Name)
pendingServePort := common.GetServePort(pendingRayCluster)
backendRefs = append(backendRefs, gwv1.HTTPBackendRef{
BackendRef: gwv1.BackendRef{
BackendObjectReference: gwv1.BackendObjectReference{
Name: gwv1.ObjectName(pendingClusterServeSvcName),
Namespace: new(gwv1.Namespace(gatewayInstance.Namespace)),
Port: new(pendingServePort),
},
Weight: new(pendingClusterWeight),
},
})
}
httpRouteName := rayServiceInstance.Name + "-httproute"
desiredHTTPRoute := &gwv1.HTTPRoute{
ObjectMeta: metav1.ObjectMeta{Name: httpRouteName, Namespace: gatewayInstance.Namespace},
Spec: gwv1.HTTPRouteSpec{
CommonRouteSpec: gwv1.CommonRouteSpec{
ParentRefs: []gwv1.ParentReference{
{
Name: gwv1.ObjectName(gatewayInstance.Name),
Namespace: new(gwv1.Namespace(gatewayInstance.Namespace)),
},
},
},
Rules: []gwv1.HTTPRouteRule{
{
Matches: []gwv1.HTTPRouteMatch{
{
Path: &gwv1.HTTPPathMatch{
Type: ptr.To(gwv1.PathMatchPathPrefix),
Value: new("/"),
},
},
},
BackendRefs: backendRefs,
},
},
},
}
return desiredHTTPRoute, nil
}
// reconcileHTTPRoute reconciles a HTTPRoute resource for a RayService to route traffic during a NewClusterWithIncrementalUpgrade.
func (r *RayServiceReconciler) reconcileHTTPRoute(ctx context.Context, rayServiceInstance *rayv1.RayService, isPendingClusterReady bool) (*gwv1.HTTPRoute, error) {
logger := ctrl.LoggerFrom(ctx)
var err error
desiredHTTPRoute, err := r.createHTTPRoute(ctx, rayServiceInstance, isPendingClusterReady)
if err != nil {
logger.Error(err, "Failed to build HTTPRoute for RayService upgrade")
return nil, err
}
if desiredHTTPRoute == nil {
logger.Info("Skipping HTTPRoute reconciliation: desired HTTPRoute is nil")
return nil, nil
}
// Check for existing HTTPRoute for RayService
existingHTTPRoute := &gwv1.HTTPRoute{}
if err := r.Get(ctx, common.RayServiceHTTPRouteNamespacedName(rayServiceInstance), existingHTTPRoute); err != nil {
if errors.IsNotFound(err) {
// Set the ownership in order to do the garbage collection by k8s.
if err := ctrl.SetControllerReference(rayServiceInstance, desiredHTTPRoute, r.Scheme); err != nil {
return nil, err
}
if err = r.Create(ctx, desiredHTTPRoute); err != nil {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToCreateHTTPRoute), "Failed to create the HTTPRoute for RayService %s/%s: %v", desiredHTTPRoute.Namespace, desiredHTTPRoute.Name, err)
return nil, err
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.CreatedHTTPRoute), "Created HTTPRoute for RayService %s/%s", desiredHTTPRoute.Namespace, desiredHTTPRoute.Name)
return desiredHTTPRoute, nil
}
return nil, err
}
// If HTTPRoute already exists, check if update is needed
if !utils.IsHTTPRouteEqual(existingHTTPRoute, desiredHTTPRoute) {
logger.Info("Updating existing HTTPRoute", "name", desiredHTTPRoute.Name)
existingHTTPRoute.Spec = desiredHTTPRoute.Spec
if err := r.Update(ctx, existingHTTPRoute); err != nil {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToUpdateHTTPRoute), "Failed to update the HTTPRoute %s/%s: %v", existingHTTPRoute.Namespace, existingHTTPRoute.Name, err)
return nil, err
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.UpdatedHTTPRoute), "Updated the HTTPRoute %s/%s", existingHTTPRoute.Namespace, existingHTTPRoute.Name)
}
return existingHTTPRoute, nil
}
// `reconcileRayCluster` reconciles the active and pending Ray clusters. There are 4 possible cases:
// (1) Create a new pending cluster. (2) Update the active cluster. (3) Update the pending cluster. (4) Do nothing.
func (r *RayServiceReconciler) reconcileRayCluster(ctx context.Context, rayServiceInstance *rayv1.RayService) (*rayv1.RayCluster, *rayv1.RayCluster, error) {
logger := ctrl.LoggerFrom(ctx)
var err error
// Get active cluster and pending cluster instances.
activeRayCluster, err := r.getRayClusterByNamespacedName(ctx, common.RayServiceActiveRayClusterNamespacedName(rayServiceInstance))
if err != nil {
return nil, nil, err
}
pendingRayCluster, err := r.getRayClusterByNamespacedName(ctx, common.RayServicePendingRayClusterNamespacedName(rayServiceInstance))
if err != nil {
return nil, nil, err
}
if rayServiceInstance.Status.PendingServiceStatus.RayClusterName != "" && pendingRayCluster == nil {
logger.Info("Creating a new pending RayCluster instance")
pendingRayCluster, err = r.createRayClusterInstance(ctx, rayServiceInstance)
return activeRayCluster, pendingRayCluster, err
}
if shouldUpdateCluster(rayServiceInstance, activeRayCluster, true) {
// TODO(kevin85421): We should not reconstruct the cluster to update it. This will cause issues if autoscaler is enabled.
logger.Info("Updating the active RayCluster instance", "clusterName", activeRayCluster.Name)
goalCluster, err := constructRayClusterForRayService(rayServiceInstance, activeRayCluster.Name, r.Scheme)
if err != nil {
return nil, nil, err
}
modifyRayCluster(ctx, activeRayCluster, goalCluster)
if err = r.Update(ctx, activeRayCluster); err != nil {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToUpdateRayCluster), "Failed to update the active RayCluster %s/%s: %v", activeRayCluster.Namespace, activeRayCluster.Name, err)
return activeRayCluster, pendingRayCluster, err
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.UpdatedRayCluster), "Updated the active RayCluster %s/%s", activeRayCluster.Namespace, activeRayCluster.Name)
}
if shouldUpdateCluster(rayServiceInstance, pendingRayCluster, false) {
// TODO(kevin85421): We should not reconstruct the cluster to update it. This will cause issues if autoscaler is enabled.
logger.Info("Updating the pending RayCluster instance", "clusterName", pendingRayCluster.Name)
goalCluster, err := constructRayClusterForRayService(rayServiceInstance, pendingRayCluster.Name, r.Scheme)
if err != nil {
return nil, nil, err
}
modifyRayCluster(ctx, pendingRayCluster, goalCluster)
if err = r.Update(ctx, pendingRayCluster); err != nil {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToUpdateRayCluster), "Failed to update the pending RayCluster %s/%s: %v", pendingRayCluster.Namespace, pendingRayCluster.Name, err)
return activeRayCluster, pendingRayCluster, err
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.UpdatedRayCluster), "Updated the pending RayCluster %s/%s", pendingRayCluster.Namespace, pendingRayCluster.Name)
}
return activeRayCluster, pendingRayCluster, nil
}
// cleanUpRayClusterInstance cleans up all the dangling RayCluster instances that are owned by the RayService instance.
// Returns true if there are still RayCluster instances that need to be cleaned up (either scheduled for deletion or waiting for deletion delay).
func (r *RayServiceReconciler) cleanUpRayClusterInstance(ctx context.Context, rayServiceInstance *rayv1.RayService) (bool, error) {
logger := ctrl.LoggerFrom(ctx)
rayClusterList := rayv1.RayClusterList{}
rayServiceIsDeleted := !rayServiceInstance.ObjectMeta.DeletionTimestamp.IsZero()
var err error
if err = r.List(ctx, &rayClusterList, common.RayServiceRayClustersAssociationOptions(rayServiceInstance).ToListOptions()...); err != nil {
return false, err
}
// Determine the ray cluster deletion delay seconds
deletionDelay := RayClusterDeletionDelayDuration
if rayServiceInstance.Spec.RayClusterDeletionDelaySeconds != nil {
deletionDelay = time.Duration(*rayServiceInstance.Spec.RayClusterDeletionDelaySeconds) * time.Second
}
if rayServiceIsDeleted {
deletionDelay = 0 * time.Second
}
hasRayClustersToClean := false
// Clean up RayCluster instances. Each instance is deleted after the configured deletion delay.
for _, rayClusterInstance := range rayClusterList.Items {
if rayServiceIsDeleted || (rayClusterInstance.Name != rayServiceInstance.Status.ActiveServiceStatus.RayClusterName && rayClusterInstance.Name != rayServiceInstance.Status.PendingServiceStatus.RayClusterName) {
hasRayClustersToClean = true
cachedTimestamp, exists := r.RayClusterDeletionTimestamps.Get(rayClusterInstance.Name)
if exists || rayServiceIsDeleted {
reasonForDeletion := ""
if rayServiceIsDeleted {
reasonForDeletion = "RayService is being deleted. Deleting RayCluster"
} else if time.Now().After(cachedTimestamp) {
reasonForDeletion = fmt.Sprintf("Deletion timestamp %s "+
"for RayCluster %s has passed. Deleting cluster "+
"immediately.", cachedTimestamp, rayClusterInstance.Name)
}
if reasonForDeletion != "" {
logger.Info("reconcileRayCluster", "delete Ray cluster", rayClusterInstance.Name, "reason", reasonForDeletion)
if err := r.Delete(ctx, &rayClusterInstance, client.PropagationPolicy(metav1.DeletePropagationBackground)); err != nil {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToDeleteRayCluster), "Failed to delete the RayCluster %s/%s: %v", rayClusterInstance.Namespace, rayClusterInstance.Name, err)
return false, err
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.DeletedRayCluster), "Deleted the RayCluster %s/%s", rayClusterInstance.Namespace, rayClusterInstance.Name)
}
} else {
deletionTimestamp := metav1.Now().Add(deletionDelay)
r.RayClusterDeletionTimestamps.Set(rayClusterInstance.Name, deletionTimestamp)
logger.Info(
"Scheduled dangling RayCluster for deletion",
"rayClusterName", rayClusterInstance.Name,
"deletionDelay", deletionDelay.String(),
"deletionTimestamp", deletionTimestamp,
)
}
}
}
return hasRayClustersToClean, nil
}
func (r *RayServiceReconciler) getRayClusterByNamespacedName(ctx context.Context, clusterKey client.ObjectKey) (*rayv1.RayCluster, error) {
if clusterKey.Name == "" {
return nil, nil
}
rayCluster := &rayv1.RayCluster{}
if err := r.Get(ctx, clusterKey, rayCluster); err != nil {
if errors.IsNotFound(err) {
return nil, nil
}
return nil, err
}
return rayCluster, nil
}
// cleanUpServeConfigCache cleans up the unused serve applications config in the cached map.
func (r *RayServiceReconciler) cleanUpServeConfigCache(ctx context.Context, rayServiceInstance *rayv1.RayService) {
logger := ctrl.LoggerFrom(ctx)
activeRayClusterName := rayServiceInstance.Status.ActiveServiceStatus.RayClusterName
pendingRayClusterName := rayServiceInstance.Status.PendingServiceStatus.RayClusterName
cacheKey := rayServiceInstance.Namespace + "/" + rayServiceInstance.Name
cacheValue, exist := r.ServeConfigs.Get(cacheKey)
if !exist {
return
}
clusterNameToServeConfig := cacheValue.(cmap.ConcurrentMap[string, string])
for key := range clusterNameToServeConfig.Items() {
if key == activeRayClusterName || key == pendingRayClusterName {
continue
}
logger.Info("Remove stale serve application config", "remove key", key, "activeRayClusterName", activeRayClusterName, "pendingRayClusterName", pendingRayClusterName)
clusterNameToServeConfig.Remove(key)
}
}
func shouldUpdateCluster(rayServiceInstance *rayv1.RayService, cluster *rayv1.RayCluster, isActiveCluster bool) bool {
// Check whether to update the RayCluster or not.
if cluster == nil {
return false
}
if isActiveCluster {
if meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.UpgradeInProgress)) {
// If the RayService is upgrading, the `RayService.Spec.RayClusterSpec` should only be compared with the
// pending cluster. The active cluster should not be updated.
return false
}
// If the KubeRay version has changed, update the RayCluster to get the cluster hash and new KubeRay version.
version := cluster.ObjectMeta.Annotations[utils.KubeRayVersion]
if version != utils.KUBERAY_VERSION {
return true
}
}
if isClusterSpecHashEqual(rayServiceInstance, cluster, false) {
// The RayCluster spec matches the cluster spec in the RayService. No need to update the cluster.
return false
}
// Update the RayCluster when the cluster spec in the RayService differs from the RayCluster, but only if the difference
// is in the number of worker groups and the new worker groups are added at the end.
return isClusterSpecHashEqual(rayServiceInstance, cluster, true)
}
func isClusterSpecHashEqual(rayServiceInstance *rayv1.RayService, cluster *rayv1.RayCluster, partial bool) bool {
// If `partial` is true, only compare the first `len(cluster.Spec.WorkerGroupSpecs)` worker groups in the CR spec.
clusterHash := cluster.ObjectMeta.Annotations[utils.HashWithoutReplicasAndWorkersToDeleteKey]
goalClusterSpec := rayClusterSpecForHashing(rayServiceInstance)
goalClusterHash := ""
if !partial {
goalClusterHash, _ = utils.GenerateHashWithoutReplicasAndWorkersToDelete(*goalClusterSpec)
} else {
// If everything is identical except for the Replicas and WorkersToDelete of
// the existing workergroups, and one or more new workergroups are added at the end, then update the cluster.
clusterNumWorkerGroups, err := strconv.Atoi(cluster.ObjectMeta.Annotations[utils.NumWorkerGroupsKey])
if err != nil {
return true
}
goalNumWorkerGroups := len(goalClusterSpec.WorkerGroupSpecs)
if goalNumWorkerGroups >= clusterNumWorkerGroups {
// Remove the new workergroup(s) from the end before calculating the hash.
goalClusterSpec.WorkerGroupSpecs = goalClusterSpec.WorkerGroupSpecs[:clusterNumWorkerGroups]
// Generate the hash of the old worker group specs.
goalClusterHash, err = utils.GenerateHashWithoutReplicasAndWorkersToDelete(*goalClusterSpec)
if err != nil {
return true
}
}
}
return clusterHash == goalClusterHash
}
func shouldPrepareNewCluster(ctx context.Context, rayServiceInstance *rayv1.RayService, activeRayCluster, pendingRayCluster *rayv1.RayCluster, isPendingClusterServing bool) bool {
if isPendingClusterServing {
return false
}
// Do not prepare a new cluster while a rollback is actively in progress.
if meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.RollbackInProgress)) {
return false
}
if activeRayCluster == nil && pendingRayCluster == nil {
// Both active and pending clusters are nil, which means the RayService has just been created.
// Create a new pending cluster.
return true
}
cluster := pendingRayCluster
if cluster == nil {
cluster = activeRayCluster
}
if isClusterSpecHashEqual(rayServiceInstance, cluster, false) {
// The RayCluster spec matches the cluster spec in the RayService. No need to create a new pending cluster.
return false
}
if isClusterSpecHashEqual(rayServiceInstance, cluster, true) {
// KubeRay should update the RayCluster instead of creating a new one.
return false
}
return isZeroDowntimeUpgradeEnabled(ctx, rayServiceInstance.Spec.UpgradeStrategy)
}
// `modifyRayCluster` updates `currentCluster` in place based on `goalCluster`. `currentCluster` is the
// current RayCluster retrieved from the informer cache, and `goalCluster` is the target state of the
// RayCluster derived from the RayService spec.
func modifyRayCluster(ctx context.Context, currentCluster, goalCluster *rayv1.RayCluster) {
logger := ctrl.LoggerFrom(ctx)
if currentCluster.Name != goalCluster.Name || currentCluster.Namespace != goalCluster.Namespace {
panic(fmt.Sprintf(
"currentCluster and goalCluster have different names or namespaces: "+
"%s/%s != %s/%s",
currentCluster.Namespace,
currentCluster.Name,
goalCluster.Namespace,
goalCluster.Name,
))
}
logger.Info("updateRayClusterInstance", "Name", goalCluster.Name, "Namespace", goalCluster.Namespace)
// Update the fetched RayCluster with new changes. Suspend is propagated
// from the RayService to the RayCluster only at creation time; afterwards
// the RayCluster's Suspend is delegated to Kueue, so we preserve the
// existing cluster's Suspend here instead of letting the goal spec
// overwrite it.
existingSuspend := currentCluster.Spec.Suspend
currentCluster.Spec = goalCluster.Spec
currentCluster.Spec.Suspend = existingSuspend
// Update the labels and annotations
currentCluster.Labels = goalCluster.Labels
currentCluster.Annotations = goalCluster.Annotations
}
func (r *RayServiceReconciler) createRayClusterInstance(ctx context.Context, rayServiceInstance *rayv1.RayService) (*rayv1.RayCluster, error) {
logger := ctrl.LoggerFrom(ctx)
rayClusterKey := common.RayServicePendingRayClusterNamespacedName(rayServiceInstance)
logger.Info("createRayClusterInstance", "clusterName", rayClusterKey.Name)
rayClusterInstance, err := constructRayClusterForRayService(rayServiceInstance, rayClusterKey.Name, r.Scheme)
if err != nil {
return nil, err
}
if err = r.Create(ctx, rayClusterInstance); err != nil {
logger.Error(err, "Failed to create the RayCluster")
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToCreateRayCluster), "Failed to create the RayCluster %s/%s: %v", rayClusterInstance.Namespace, rayClusterInstance.Name, err)
return nil, err
}
logger.Info("Created RayCluster for RayService", "clusterName", rayClusterInstance.Name)
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.CreatedRayCluster), "Created the RayCluster %s/%s", rayClusterInstance.Namespace, rayClusterInstance.Name)
return rayClusterInstance, nil
}
// rayClusterSpecForHashing returns a copy of the RayService's RayClusterSpec
// to use for hash comparisons. Fields that should not trigger reconciliation
// of the underlying RayCluster are cleared here. Suspend is excluded because
// the RayService controller propagates Suspend to the RayCluster only at
// creation time; once the RayCluster exists, Suspend is delegated to Kueue
// (or whichever external controller owns the RayCluster's queueing), and
// later changes to RayService.Spec.RayClusterSpec.Suspend must not trigger
// an in-place update or a new cluster preparation.
func rayClusterSpecForHashing(rayService *rayv1.RayService) *rayv1.RayClusterSpec {
spec := rayService.Spec.RayClusterSpec.DeepCopy()
spec.Suspend = nil
return spec
}
func constructRayClusterForRayService(rayService *rayv1.RayService, rayClusterName string, scheme *runtime.Scheme) (*rayv1.RayCluster, error) {
var err error
rayClusterLabel := make(map[string]string)
maps.Copy(rayClusterLabel, rayService.Labels)
rayClusterLabel[utils.RayOriginatedFromCRNameLabelKey] = rayService.Name
rayClusterLabel[utils.RayOriginatedFromCRDLabelKey] = utils.RayOriginatedFromCRDLabelValue(utils.RayServiceCRD)
rayClusterAnnotations := make(map[string]string)
maps.Copy(rayClusterAnnotations, rayService.Annotations)
rayClusterAnnotations[utils.HashWithoutReplicasAndWorkersToDeleteKey], err = utils.GenerateHashWithoutReplicasAndWorkersToDelete(*rayClusterSpecForHashing(rayService))
if err != nil {
return nil, err
}
rayClusterAnnotations[utils.NumWorkerGroupsKey] = strconv.Itoa(len(rayService.Spec.RayClusterSpec.WorkerGroupSpecs))
// set the KubeRay version used to create the RayCluster
rayClusterAnnotations[utils.KubeRayVersion] = utils.KUBERAY_VERSION
clusterSpec := rayService.Spec.RayClusterSpec.DeepCopy()
isPendingClusterForUpgrade := utils.IsIncrementalUpgradeEnabled(&rayService.Spec) &&
rayService.Status.ActiveServiceStatus.RayClusterName != ""
if isPendingClusterForUpgrade {
// For incremental upgrade, start the pending cluster without a replicas value so
// that it autoscales based on the value of target_capacity from MinReplicas.
for i := range clusterSpec.WorkerGroupSpecs {
clusterSpec.WorkerGroupSpecs[i].Replicas = nil
}
}
rayCluster := &rayv1.RayCluster{
ObjectMeta: metav1.ObjectMeta{
Labels: rayClusterLabel,
Annotations: rayClusterAnnotations,
Name: rayClusterName,
Namespace: rayService.Namespace,
},
Spec: *clusterSpec,
}
// Set the ownership in order to do the garbage collection by k8s.
if err := ctrl.SetControllerReference(rayService, rayCluster, scheme); err != nil {
return nil, err
}
return rayCluster, nil
}
func checkIfNeedSubmitServeApplications(cachedServeConfigV2 string, serveConfigV2 string, serveApplications map[string]rayv1.AppStatus) (bool, string) {
// If the Serve config has not been cached, update the Serve config.
if cachedServeConfigV2 == "" {
return true, "Nothing has been cached for the cluster."
}
// Handle the case that the head Pod has crashed and GCS FT is not enabled.
if len(serveApplications) == 0 {
reason := "No Serve application found in the RayCluster. " +
"A possible reason is that the head Pod crashed and GCS FT was not enabled."
return true, reason
}
// If the Serve config has been cached, check if it needs to be updated.
if cachedServeConfigV2 != serveConfigV2 {
return true, "Current V2 Serve config doesn't match cached Serve config."
}
return false, "Current V2 Serve config matches cached Serve config."
}
func (r *RayServiceReconciler) updateServeDeployment(ctx context.Context, rayServiceInstance *rayv1.RayService, rayDashboardClient dashboardclient.RayDashboardClientInterface, clusterName string) error {
logger := ctrl.LoggerFrom(ctx)
logger.Info("updateServeDeployment", "V2 config", rayServiceInstance.Spec.ServeConfigV2)
serveConfig := make(map[string]any)
if err := yaml.Unmarshal([]byte(rayServiceInstance.Spec.ServeConfigV2), &serveConfig); err != nil {
return err
}
if utils.IsIncrementalUpgradeEnabled(&rayServiceInstance.Spec) {
// For incremental upgrades, set target_capacity if specified to avoid
// scaling initial Serve deployment to 100% immediately.
var targetCapacity *int32
activeStatus := rayServiceInstance.Status.ActiveServiceStatus
pendingStatus := rayServiceInstance.Status.PendingServiceStatus
if clusterName == activeStatus.RayClusterName && activeStatus.TargetCapacity != nil {
targetCapacity = activeStatus.TargetCapacity
} else if clusterName == pendingStatus.RayClusterName && pendingStatus.TargetCapacity != nil {
targetCapacity = pendingStatus.TargetCapacity
}
if targetCapacity != nil {
logger.Info("Setting target_capacity from status in Serve config.", "target_capacity", *targetCapacity)
serveConfig["target_capacity"] = *targetCapacity
}
}
configJson, err := json.Marshal(serveConfig)
if err != nil {
return fmt.Errorf("failed to marshal converted serve config into bytes: %w", err)
}
logger.Info("updateServeDeployment", "MULTI_APP json config", string(configJson))
if err := rayDashboardClient.UpdateDeployments(ctx, configJson); err != nil {
err = fmt.Errorf(
"fail to create / update Serve applications. If you observe this error consistently, "+
"please check \"Issue 5: Fail to create / update Serve applications.\" in "+
"https://docs.ray.io/en/master/cluster/kubernetes/troubleshooting/rayservice-troubleshooting.html#kuberay-raysvc-troubleshoot for more details. "+
"err: %v", err)
return err
}
r.cacheServeConfig(rayServiceInstance, clusterName)
logger.Info("updateServeDeployment", "message", "Cached Serve config for Ray cluster with the key", "rayClusterName", clusterName)
return nil
}
// checkIfNeedTargetCapacityUpdate returns whether the controller should adjust the target_capacity
// of the Serve config associated with a RayCluster during NewClusterWithIncrementalUpgrade.
//
// This function implements the incremental upgrade state machine as defined in the design document:
// https://github.com/ray-project/enhancements/blob/main/reps/2024-12-4-ray-service-incr-upgrade.md
//
// The upgrade process follows these phases:
// 1. Phase 1 (Steps 7-8): New cluster scales up to target capacity
// - pendingTargetCapacity: 0% → 100%
// - Returns true: "Pending RayCluster has not finished scaling up."
//
// 2. Phase 2 (Step 9): Traffic gradually migrates to new cluster
// - pendingTrafficRoutedPercent: 0% → 100%
// - Returns true: "Pending RayCluster has not finished scaling up."
//
// 3. Phase 3 (Step 10): Old cluster scales down after new cluster is ready
// - activeTargetCapacity: 100% → 0%
// - Returns true: "Active RayCluster TargetCapacity has not finished scaling down."
//
// 4. Phase 4 (Step 11): Upgrade completion
// - Both clusters reach final state: active=0%, pending=100%
// - Returns false: "All traffic has migrated to the upgraded cluster and NewClusterWithIncrementalUpgrade migration
// is complete."
//
// The function ensures that traffic migration only proceeds when the target cluster has reached
// its capacity limit, preventing resource conflicts and ensuring upgrade stability.
func (r *RayServiceReconciler) checkIfNeedTargetCapacityUpdate(ctx context.Context, rayServiceInstance *rayv1.RayService) (bool, string) {
activeRayServiceStatus := rayServiceInstance.Status.ActiveServiceStatus
pendingRayServiceStatus := rayServiceInstance.Status.PendingServiceStatus
if activeRayServiceStatus.RayClusterName == "" || pendingRayServiceStatus.RayClusterName == "" {
return false, "Both active and pending RayCluster instances are required for NewClusterWithIncrementalUpgrade."
}
// Validate Gateway and HTTPRoute objects are ready
gatewayInstance := &gwv1.Gateway{}
if err := r.Get(ctx, common.RayServiceGatewayNamespacedName(rayServiceInstance), gatewayInstance); err != nil {
return false, fmt.Sprintf("Failed to retrieve Gateway for RayService: %v", err)
}
if !utils.IsGatewayReady(gatewayInstance) {
return false, "Gateway for RayService NewClusterWithIncrementalUpgrade is not ready."
}
httpRouteInstance := &gwv1.HTTPRoute{}
if err := r.Get(ctx, common.RayServiceHTTPRouteNamespacedName(rayServiceInstance), httpRouteInstance); err != nil {
return false, fmt.Sprintf("Failed to retrieve HTTPRoute for RayService: %v", err)
}
if !utils.IsHTTPRouteReady(gatewayInstance, httpRouteInstance) {
return false, "HTTPRoute for RayService NewClusterWithIncrementalUpgrade is not ready."
}
// Retrieve the current observed NewClusterWithIncrementalUpgrade Status fields for each RayService.
if activeRayServiceStatus.TargetCapacity == nil || activeRayServiceStatus.TrafficRoutedPercent == nil {
return true, "Active RayServiceStatus missing TargetCapacity or TrafficRoutedPercent."
}
if pendingRayServiceStatus.TargetCapacity == nil || pendingRayServiceStatus.TrafficRoutedPercent == nil {
return true, "Pending RayServiceStatus missing TargetCapacity or TrafficRoutedPercent."
}
activeTargetCapacity := int(*activeRayServiceStatus.TargetCapacity)
pendingTargetCapacity := int(*pendingRayServiceStatus.TargetCapacity)
pendingTrafficRoutedPercent := int(*pendingRayServiceStatus.TrafficRoutedPercent)
// During rollback, the goal is activeTargetCapacity → 100, pendingTargetCapacity → 0.
if meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.RollbackInProgress)) {
if activeTargetCapacity == 100 && pendingTargetCapacity == 0 {
return false, "Rollback complete: all traffic has been restored to the active cluster."
}
return true, "Rollback in progress: traffic is being restored to the active cluster."
}
if activeTargetCapacity == 0 && pendingTargetCapacity == 100 {
return false, "All traffic has migrated to the upgraded cluster and NewClusterWithIncrementalUpgrade is complete."
} else if pendingTargetCapacity < 100 || pendingTrafficRoutedPercent < 100 {
return true, "Pending RayCluster has not finished scaling up."
}
return true, "Active RayCluster TargetCapacity has not finished scaling down."
}
// applyServeTargetCapacity updates the target_capacity for a given RayCluster's Serve applications.
func (r *RayServiceReconciler) applyServeTargetCapacity(ctx context.Context, rayServiceInstance *rayv1.RayService, rayClusterInstance *rayv1.RayCluster, rayDashboardClient dashboardclient.RayDashboardClientInterface, goalTargetCapacity int32) error {
logger := ctrl.LoggerFrom(ctx).WithValues("RayCluster", rayClusterInstance.Name)
// Retrieve cached ServeConfig from last reconciliation for cluster to update
cachedConfig := r.getServeConfigFromCache(rayServiceInstance, rayClusterInstance.Name)
if cachedConfig == "" {
cachedConfig = rayServiceInstance.Spec.ServeConfigV2
}
serveConfig := make(map[string]any)
if err := yaml.Unmarshal([]byte(cachedConfig), &serveConfig); err != nil {
return err
}
// Check if ServeConfig requires update
if currentTargetCapacity, ok := serveConfig["target_capacity"].(float64); ok {
if int32(currentTargetCapacity) == goalTargetCapacity {
logger.Info("target_capacity already updated on RayCluster", "target_capacity", currentTargetCapacity)
// No update required, return early
return nil
}
}
serveConfig["target_capacity"] = goalTargetCapacity
configJson, err := json.Marshal(serveConfig)
if err != nil {
return fmt.Errorf("failed to marshal serve config: %w", err)
}
logger.Info("Applying new target_capacity to Ray cluster.", "goal", goalTargetCapacity)
if err := rayDashboardClient.UpdateDeployments(ctx, configJson); err != nil {
err = fmt.Errorf(
"fail to create / update Serve applications. If you observe this error consistently, "+
"please check \"Issue 5: Fail to create / update Serve applications.\" in "+
"https://docs.ray.io/en/master/cluster/kubernetes/troubleshooting/rayservice-troubleshooting.html#kuberay-raysvc-troubleshoot for more details. "+
"err: %v", err)
return err
}
// Update the TargetCapacity status fields.
switch rayClusterInstance.Name {
case rayServiceInstance.Status.ActiveServiceStatus.RayClusterName:
rayServiceInstance.Status.ActiveServiceStatus.TargetCapacity = new(goalTargetCapacity)
case rayServiceInstance.Status.PendingServiceStatus.RayClusterName:
rayServiceInstance.Status.PendingServiceStatus.TargetCapacity = new(goalTargetCapacity)
}
return nil
}
// reconcileServeTargetCapacity reconciles the target_capacity of the ServeConfig for a given RayCluster during
// a NewClusterWithIncrementalUpgrade while also updating the Status.TargetCapacity of the Active and Pending RayServices.
func (r *RayServiceReconciler) reconcileServeTargetCapacity(ctx context.Context, rayServiceInstance *rayv1.RayService, rayClusterInstance *rayv1.RayCluster, rayDashboardClient dashboardclient.RayDashboardClientInterface) error {
logger := ctrl.LoggerFrom(ctx)
logger.Info("reconcileServeTargetCapacity", "RayService", rayServiceInstance.Name)
activeRayServiceStatus := &rayServiceInstance.Status.ActiveServiceStatus
pendingRayServiceStatus := &rayServiceInstance.Status.PendingServiceStatus
// Set initial TargetCapacity values if unset
if activeRayServiceStatus.TargetCapacity == nil {
activeRayServiceStatus.TargetCapacity = new(int32(100))
}
if pendingRayServiceStatus.TargetCapacity == nil {
pendingRayServiceStatus.TargetCapacity = new(int32(0))
}
// Retrieve the current observed Status fields for NewClusterWithIncrementalUpgrade
activeTargetCapacity := *activeRayServiceStatus.TargetCapacity
pendingTargetCapacity := *pendingRayServiceStatus.TargetCapacity
pendingTrafficRoutedPercent := ptr.Deref(pendingRayServiceStatus.TrafficRoutedPercent, 0)
// Retrieve MaxSurgePercent - the maximum amount to change TargetCapacity by
options := utils.GetRayServiceClusterUpgradeOptions(&rayServiceInstance.Spec)
if options == nil {
return errstd.New("Missing RayService ClusterUpgradeOptions during upgrade")
}
maxSurgePercent := ptr.Deref(options.MaxSurgePercent, 100)
if meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.RollbackInProgress)) {
// Rollback the upgrade. The active RayCluster should be scaled back to 100% target_capacity,
// while the pending RayCluster is scaled to 0%. This is the inverse of the regular upgrade path.
activeTrafficRoutedPercent := ptr.Deref(activeRayServiceStatus.TrafficRoutedPercent, 0)
if activeTargetCapacity != activeTrafficRoutedPercent {
logger.Info("Traffic is rolling back to active cluster, deferring capacity update.", "ActiveTargetCapacity", activeTargetCapacity, "ActiveTrafficRoutedPercent", activeTrafficRoutedPercent)
return nil
}
if activeTargetCapacity+pendingTargetCapacity > 100 {
if rayClusterInstance.Name == pendingRayServiceStatus.RayClusterName {
goalTargetCapacity := max(int32(0), pendingTargetCapacity-maxSurgePercent)
logger.Info("Rollback: Scaling down pending cluster `target_capacity`.", "goal", goalTargetCapacity)
return r.applyServeTargetCapacity(ctx, rayServiceInstance, rayClusterInstance, rayDashboardClient, goalTargetCapacity)
}
} else {
if rayClusterInstance.Name == activeRayServiceStatus.RayClusterName {
goalTargetCapacity := min(int32(100), activeTargetCapacity+maxSurgePercent)
logger.Info("Rollback: Scaling up active cluster `target_capacity`.", "goal", goalTargetCapacity)
return r.applyServeTargetCapacity(ctx, rayServiceInstance, rayClusterInstance, rayDashboardClient, goalTargetCapacity)
}
}
return nil
}
// Defer updating the target_capacity until traffic weights are updated
if pendingTargetCapacity != pendingTrafficRoutedPercent {
logger.Info("Traffic is currently being migrated to pending cluster", "RayCluster", pendingRayServiceStatus.RayClusterName, "TargetCapacity", pendingTargetCapacity, "TrafficRoutedPercent", pendingTrafficRoutedPercent)
return nil
}
// There are two cases:
// 1. The total target_capacity is greater than 100. This means the pending RayCluster has
// scaled up traffic and the active RayCluster can be scaled down by MaxSurgePercent.
// 2. The total target_capacity is equal to 100. This means the pending RayCluster can
// increase its target_capacity by MaxSurgePercent.
// If the rayClusterInstance passed into this function is not the cluster to update based
// on the above conditions, we return without doing anything.
var goalTargetCapacity int32
shouldUpdate := false
switch rayClusterInstance.Name {
case activeRayServiceStatus.RayClusterName:
if activeTargetCapacity+pendingTargetCapacity > 100 {
// Scale down the Active RayCluster TargetCapacity on this iteration.
goalTargetCapacity = max(int32(0), activeTargetCapacity-maxSurgePercent)
shouldUpdate = true
logger.Info("Setting target_capacity for active Raycluster", "Raycluster", rayClusterInstance.Name, "target_capacity", goalTargetCapacity)
}
case pendingRayServiceStatus.RayClusterName:
if activeTargetCapacity+pendingTargetCapacity <= 100 {
// Scale up the Pending RayCluster TargetCapacity on this iteration.
goalTargetCapacity = min(int32(100), pendingTargetCapacity+maxSurgePercent)
shouldUpdate = true
logger.Info("Setting target_capacity for pending Raycluster", "Raycluster", rayClusterInstance.Name, "target_capacity", goalTargetCapacity)
}
}
if !shouldUpdate {
return nil
}
return r.applyServeTargetCapacity(ctx, rayServiceInstance, rayClusterInstance, rayDashboardClient, goalTargetCapacity)
}
// `getAndCheckServeStatus` gets Serve applications' and deployments' statuses and check whether the
// Serve applications are ready to serve incoming traffic or not. It returns three values:
//
// (1) `isReady`: Whether the Serve applications are ready to serve incoming traffic or not.
// (2) `newApplications`: The Serve applications' statuses.
// (3) `err`: If `err` is not nil, it means that KubeRay failed to get Serve application statuses from the dashboard.
func getAndCheckServeStatus(ctx context.Context, dashboardClient dashboardclient.RayDashboardClientInterface) (bool, map[string]rayv1.AppStatus, error) {
logger := ctrl.LoggerFrom(ctx)
var serveAppStatuses map[string]*utiltypes.ServeApplicationStatus
var err error
if serveAppStatuses, err = dashboardClient.GetMultiApplicationStatus(ctx); err != nil {
err = fmt.Errorf(
"failed to get Serve application statuses from the dashboard. "+
"If you observe this error consistently, please check https://docs.ray.io/en/latest/cluster/kubernetes/troubleshooting/rayservice-troubleshooting.html for more details. "+
"err: %v", err)
return false, nil, err
}
isReady := true
newApplications := make(map[string]rayv1.AppStatus)
for appName, app := range serveAppStatuses {
if appName == "" {
appName = utils.DefaultServeAppName
}
applicationStatus := rayv1.AppStatus{
Message: app.Message,
Status: app.Status,
Deployments: make(map[string]rayv1.ServeDeploymentStatus),
}
// `isReady` is used to determine whether the Serve application is ready or not. The cluster switchover only happens when all Serve
// applications in this RayCluster are ready so that the incoming traffic will not be dropped.
if app.Status != rayv1.ApplicationStatusEnum.RUNNING {
isReady = false
}
// Copy deployment statuses
for deploymentName, deployment := range app.Deployments {
deploymentStatus := rayv1.ServeDeploymentStatus{
Status: deployment.Status,
Message: deployment.Message,
}
applicationStatus.Deployments[deploymentName] = deploymentStatus
}
newApplications[appName] = applicationStatus
}
if len(newApplications) == 0 {
logger.Info("No Serve application found. The RayCluster is not ready to serve requests. Set 'isReady' to false")
isReady = false
}
return isReady, newApplications, nil
}
func (r *RayServiceReconciler) getServeConfigFromCache(rayServiceInstance *rayv1.RayService, clusterName string) string {
cacheKey := rayServiceInstance.Namespace + "/" + rayServiceInstance.Name
cacheValue, exist := r.ServeConfigs.Get(cacheKey)
if !exist {
return ""
}
serveConfigs := cacheValue.(cmap.ConcurrentMap[string, string])
serveConfig, exist := serveConfigs.Get(clusterName)
if !exist {
return ""
}
return serveConfig
}
func (r *RayServiceReconciler) cacheServeConfig(rayServiceInstance *rayv1.RayService, clusterName string) {
serveConfig := rayServiceInstance.Spec.ServeConfigV2
if serveConfig == "" {
return
}
cacheKey := rayServiceInstance.Namespace + "/" + rayServiceInstance.Name
cacheValue, exist := r.ServeConfigs.Get(cacheKey)
var rayServiceServeConfigs cmap.ConcurrentMap[string, string]
if !exist {
rayServiceServeConfigs = cmap.New[string]()
r.ServeConfigs.Add(cacheKey, rayServiceServeConfigs)
} else {
rayServiceServeConfigs = cacheValue.(cmap.ConcurrentMap[string, string])
}
rayServiceServeConfigs.Set(clusterName, serveConfig)
}
func (r *RayServiceReconciler) reconcileServices(ctx context.Context, rayServiceInstance *rayv1.RayService, rayClusterInstance *rayv1.RayCluster, serviceType utils.ServiceType) (*corev1.Service, error) {
logger := ctrl.LoggerFrom(ctx)
var newSvc *corev1.Service
var err error
switch serviceType {
case utils.HeadService:
newSvc, err = common.BuildHeadServiceForRayService(ctx, *rayServiceInstance, *rayClusterInstance)
case utils.ServingService:
newSvc, err = common.BuildServeServiceForRayService(ctx, *rayServiceInstance, *rayClusterInstance)
default:
panic(fmt.Sprintf("unknown service type %v. This should never happen. Please open an issue in the KubeRay repository.", serviceType))
}
if err != nil {
return nil, err
}
// Retrieve the Service from the Kubernetes cluster with the name and namespace.
oldSvc := &corev1.Service{}
err = r.Get(ctx, client.ObjectKey{Name: newSvc.Name, Namespace: rayServiceInstance.Namespace}, oldSvc)
if err == nil {
// Only update the service if the RayCluster switches.
if newSvc.Spec.Selector[utils.RayClusterLabelKey] == oldSvc.Spec.Selector[utils.RayClusterLabelKey] {
logger.Info("Service already exists in the RayCluster, skipping Update", "rayCluster", newSvc.Spec.Selector[utils.RayClusterLabelKey], "serviceType", serviceType)
return oldSvc, nil
}
// ClusterIP is immutable. Starting from Kubernetes v1.21.5, if the new service does not specify a ClusterIP,
// Kubernetes will assign the ClusterIP of the old service to the new one. However, to maintain compatibility
// with older versions of Kubernetes, we need to assign the ClusterIP here.
newSvc.Spec.ClusterIP = oldSvc.Spec.ClusterIP
oldSvc.Spec = *newSvc.Spec.DeepCopy()
logger.Info("Update Kubernetes Service", "serviceType", serviceType)
if updateErr := r.Update(ctx, oldSvc); updateErr != nil {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToUpdateService), "Failed to update the service %s/%s, %v", oldSvc.Namespace, oldSvc.Name, updateErr)
return nil, updateErr
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.UpdatedService), "Updated the service %s/%s", oldSvc.Namespace, oldSvc.Name)
// Return the updated service.
return oldSvc, nil
} else if errors.IsNotFound(err) {
logger.Info("Create a Kubernetes Service", "serviceType", serviceType)
if err := ctrl.SetControllerReference(rayServiceInstance, newSvc, r.Scheme); err != nil {
return nil, err
}
if err := r.Create(ctx, newSvc); err != nil {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToCreateService), "Failed to create the service %s/%s, %v", newSvc.Namespace, newSvc.Name, err)
return nil, err
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.CreatedService), "Created the service %s/%s", newSvc.Namespace, newSvc.Name)
return newSvc, nil
}
return nil, err
}
// Reconciles the Serve applications on the RayCluster. Returns (isReady, serveApplicationStatus, error).
// The `isReady` flag indicates whether the RayCluster is ready to handle incoming traffic.
func (r *RayServiceReconciler) reconcileServe(ctx context.Context, rayServiceInstance *rayv1.RayService, rayClusterInstance *rayv1.RayCluster) (bool, map[string]rayv1.AppStatus, error) {
logger := ctrl.LoggerFrom(ctx)
var err error
var serveApplications map[string]rayv1.AppStatus
// Check if head pod is running and ready. If not, requeue the resource event to avoid
// redundant custom resource status updates.
//
// TODO (kevin85421): Note that the Dashboard and GCS may take a few seconds to start up
// after the head pod is running and ready. Hence, some requests to the Dashboard (e.g. `UpdateDeployments`) may fail.
// This is not an issue since `UpdateDeployments` is an idempotent operation.
if features.Enabled(features.RayClusterStatusConditions) {
if !meta.IsStatusConditionTrue(rayClusterInstance.Status.Conditions, string(rayv1.HeadPodReady)) {
logger.Info("The head Pod is not ready, requeue the resource event to avoid redundant custom resource status updates.")
return false, serveApplications, nil
}
} else {
if isRunningAndReady, err := r.isHeadPodRunningAndReady(ctx, rayClusterInstance); err != nil || !isRunningAndReady {
if err != nil {
logger.Error(err, "Failed to check if head Pod is running and ready!")
} else {
logger.Info("Skipping the update of Serve applications because the Ray head Pod is not ready.")
}
return false, serveApplications, nil
}
}
var clientURL string
if clientURL, err = utils.FetchHeadServiceURL(ctx, r.Client, rayClusterInstance, utils.DashboardPortName); err != nil || clientURL == "" {
return false, serveApplications, err
}
rayDashboardClient, err := r.dashboardClientFunc(rayClusterInstance, clientURL)
if err != nil {
return false, serveApplications, err
}
skipConfigUpdate := false
isActiveCluster := rayClusterInstance.Name == rayServiceInstance.Status.ActiveServiceStatus.RayClusterName
isIncrementalUpgradeInProgress := utils.IsIncrementalUpgradeEnabled(&rayServiceInstance.Spec) &&
meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.UpgradeInProgress))
isRollbackInProgress := utils.IsIncrementalUpgradeEnabled(&rayServiceInstance.Spec) &&
meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.RollbackInProgress))
if isActiveCluster && isIncrementalUpgradeInProgress && !isRollbackInProgress {
// Skip updating the Serve config for the Active cluster during NewClusterWithIncrementalUpgrade. The updated
// Serve config is applied to the pending RayService's RayCluster.
skipConfigUpdate = true
logger.Info("Blocking new Serve config submission for Active cluster during upgrade.", "clusterName", rayClusterInstance.Name)
} else if !isActiveCluster && isRollbackInProgress {
// During rollback, the pending cluster is being scaled down and phased out.
// We skip updating its Serve config to prevent unnecessary disruptive redeployments,
// since its only goal is to drain traffic while the active cluster takes over.
skipConfigUpdate = true
logger.Info("Blocking new Serve config submission for Pending cluster during rollback.", "clusterName", rayClusterInstance.Name)
}
cachedServeConfigV2 := r.getServeConfigFromCache(rayServiceInstance, rayClusterInstance.Name)
isReady, serveApplications, err := getAndCheckServeStatus(ctx, rayDashboardClient)
if err != nil {
return false, serveApplications, err
}
shouldUpdate, reason := checkIfNeedSubmitServeApplications(cachedServeConfigV2, rayServiceInstance.Spec.ServeConfigV2, serveApplications)
logger.Info("checkIfNeedSubmitServeApplications", "shouldUpdate", shouldUpdate, "reason", reason)
if shouldUpdate && !skipConfigUpdate {
if err = r.updateServeDeployment(ctx, rayServiceInstance, rayDashboardClient, rayClusterInstance.Name); err != nil {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToUpdateServeApplications), "Failed to update serve applications to the RayCluster %s/%s: %v", rayClusterInstance.Namespace, rayClusterInstance.Name, err)
return false, serveApplications, err
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.UpdatedServeApplications), "Updated serve applications to the RayCluster %s/%s", rayClusterInstance.Namespace, rayClusterInstance.Name)
}
if isIncrementalUpgradeInProgress {
incrementalUpgradeUpdate, reason := r.checkIfNeedTargetCapacityUpdate(ctx, rayServiceInstance)
logger.Info("checkIfNeedTargetCapacityUpdate", "incrementalUpgradeUpdate", incrementalUpgradeUpdate, "reason", reason)
if incrementalUpgradeUpdate {
if err := r.reconcileServeTargetCapacity(ctx, rayServiceInstance, rayClusterInstance, rayDashboardClient); err != nil {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToUpdateTargetCapacity), "Failed to update target_capacity of serve applications to the RayCluster %s/%s: %v", rayClusterInstance.Namespace, rayClusterInstance.Name, err)
return false, serveApplications, err
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.UpdatedServeTargetCapacity),
"Updated target_capacity of serve applications to to the RayCluster %s/%s", rayClusterInstance.Namespace, rayClusterInstance.Name)
}
}
return isReady, serveApplications, nil
}
func (r *RayServiceReconciler) updateHeadPodServeLabel(ctx context.Context, rayServiceInstance *rayv1.RayService, rayClusterInstance *rayv1.RayCluster, excludeHeadPodFromServeSvc bool) error {
// `updateHeadPodServeLabel` updates the head Pod's serve label based on the health status of the proxy actor.
// If `excludeHeadPodFromServeSvc` is true, the head Pod will not be used to serve requests, regardless of proxy actor health.
// If `excludeHeadPodFromServeSvc` is false, the head Pod's serve label will be set based on the health check result.
// The label is used by the Kubernetes serve service to determine whether to include the head Pod in the service endpoints.
if rayClusterInstance == nil {
return nil
}
headPod, err := common.GetRayClusterHeadPod(ctx, r, rayClusterInstance)
if err != nil {
return err
}
if headPod == nil {
return fmt.Errorf("found 0 head. cluster name %s, namespace %v", rayClusterInstance.Name, rayClusterInstance.Namespace)
}
rayContainer := headPod.Spec.Containers[utils.RayContainerIndex]
servingPort := int(utils.FindContainerPort(&rayContainer, utils.ServingPortName, utils.DefaultServingPort))
client := r.httpProxyClientFunc(headPod.Status.PodIP, headPod.Namespace, headPod.Name, servingPort)
if headPod.Labels == nil {
headPod.Labels = make(map[string]string)
}
oldLabel := headPod.Labels[utils.RayClusterServingServiceLabelKey]
newLabel := utils.EnableRayClusterServingServiceFalse
// If excludeHeadPodFromServeSvc is true, head Pod will not be used to serve requests
// no matter whether the proxy actor is healthy or not. Therefore, only send the health
// check request if excludeHeadPodFromServeSvc is false.
if !excludeHeadPodFromServeSvc {
isHealthy := client.CheckProxyActorHealth(ctx) == nil
newLabel = strconv.FormatBool(isHealthy)
}
if oldLabel != newLabel {
headPod.Labels[utils.RayClusterServingServiceLabelKey] = newLabel
if updateErr := r.Update(ctx, headPod); updateErr != nil {
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeWarning, string(utils.FailedToUpdateHeadPodServeLabel), "Failed to update the serve label to %q for the Head Pod %s/%s: %v", newLabel, headPod.Namespace, headPod.Name, updateErr)
return updateErr
}
r.Recorder.Eventf(rayServiceInstance, corev1.EventTypeNormal, string(utils.UpdatedHeadPodServeLabel), "Updated the serve label to %q for the Head Pod %s/%s", newLabel, headPod.Namespace, headPod.Name)
}
return nil
}
// calculateNumServeEndpointsFromSlices calculates the number of unique ready Pods
// from EndpointSlices associated with a given service.
//
// In dual-stack environments (IPv4 + IPv6), the same Pod may appear in multiple
// EndpointSlices with different IP addresses. This function deduplicates by TargetRef.UID
// to ensure each Pod is counted only once, regardless of how many IP families it serves.
//
// An endpoint is considered ready if it has the `Ready` condition set to true.
// This replaces the legacy Endpoints API approach.
func (r *RayServiceReconciler) calculateNumServeEndpointsFromSlices(ctx context.Context, serviceNamespacedName client.ObjectKey) (int, error) {
logger := ctrl.LoggerFrom(ctx)
// List all EndpointSlices for the service.
// EndpointSlices are automatically created by Kubernetes and labeled with
// kubernetes.io/service-name to associate them with their parent Service.
endpointSliceList := &discoveryv1.EndpointSliceList{}
listOpts := []client.ListOption{
client.InNamespace(serviceNamespacedName.Namespace),
client.MatchingLabels{discoveryv1.LabelServiceName: serviceNamespacedName.Name},
}
if err := r.List(ctx, endpointSliceList, listOpts...); err != nil {
logger.Error(err, "Failed to list EndpointSlices", "serviceName", serviceNamespacedName.Name, "serviceNamespace", serviceNamespacedName.Namespace)
return 0, err
}
uniqueNumReadyPods := make(map[types.UID]struct{})
for _, endpointSlice := range endpointSliceList.Items {
for _, endpoint := range endpointSlice.Endpoints {
// Only count endpoints that are ready to serve traffic
if ptr.Deref(endpoint.Conditions.Ready, false) {
if endpoint.TargetRef != nil && endpoint.TargetRef.UID != "" {
uniqueNumReadyPods[endpoint.TargetRef.UID] = struct{}{}
}
}
}
}
numPods := len(uniqueNumReadyPods)
logger.V(1).Info("Counted serve-ready pods via EndpointSlices",
"serviceName", serviceNamespacedName.Name,
"serviceNamespace", serviceNamespacedName.Namespace,
"numSlices", len(endpointSliceList.Items),
"numReadyPods", numPods,
)
return numPods, nil
}
// isHeadPodRunningAndReady checks if the head pod of the RayCluster is running and ready.
func (r *RayServiceReconciler) isHeadPodRunningAndReady(ctx context.Context, instance *rayv1.RayCluster) (bool, error) {
headPod, err := common.GetRayClusterHeadPod(ctx, r, instance)
if err != nil {
return false, err
}
if headPod == nil {
return false, fmt.Errorf("found 0 head. cluster name %s, namespace %v", instance.Name, instance.Namespace)
}
return utils.IsRunningAndReady(headPod), nil
}
// getInitializingTimeout parses the initializing timeout annotation from RayService.
// Returns (timeout, true) if valid. Accepts Go duration format (e.g., "5m", "1h") or integer seconds.
// The annotation is assumed to be already validated by ValidateRayServiceMetadata.
// If the annotation is absent, returns (0, false).
func getInitializingTimeout(rs *rayv1.RayService) (time.Duration, bool) {
if rs.Annotations == nil {
return 0, false
}
timeoutStr, exists := rs.Annotations[utils.RayServiceInitializingTimeoutAnnotation]
if !exists || timeoutStr == "" {
return 0, false
}
// Try parsing as Go duration first (e.g., "30m", "1h")
// Validation already ensures this is valid, so we can ignore errors
if timeout, err := time.ParseDuration(timeoutStr); err == nil {
return timeout, true
}
// Try parsing as integer seconds
// Validation already ensures this is valid if ParseDuration failed
if seconds, err := strconv.Atoi(timeoutStr); err == nil {
return time.Duration(seconds) * time.Second, true
}
// This should never happen since validation ensures correctness,
// but we handle it gracefully by returning false
return 0, false
}
// isInitializingTimeout returns true if RayServiceReady is False with Reason=InitializingTimeout.
// Once a RayService has timed out, it remains in a terminal failure state regardless of generation changes.
func isInitializingTimeout(rs *rayv1.RayService) bool {
readyCond := meta.FindStatusCondition(rs.Status.Conditions, string(rayv1.RayServiceReady))
if readyCond == nil {
return false
}
return readyCond.Status == metav1.ConditionFalse &&
readyCond.Reason == string(rayv1.RayServiceInitializingTimeout)
}
// markFailedOnInitializingTimeout checks if the RayService has been initializing for too long.
// If timeout is configured and exceeded, it marks the service as failed and triggers cleanup.
func markFailedOnInitializingTimeout(ctx context.Context, r *RayServiceReconciler, rs *rayv1.RayService) {
logger := ctrl.LoggerFrom(ctx)
// Skip if no timeout is configured
timeout, ok := getInitializingTimeout(rs)
if !ok {
return
}
// Check if currently in Initializing state
readyCond := meta.FindStatusCondition(rs.Status.Conditions, string(rayv1.RayServiceReady))
if readyCond == nil {
return
}
if readyCond.Status != metav1.ConditionFalse || readyCond.Reason != string(rayv1.RayServiceInitializing) {
// Not in Initializing state
return
}
// Check if timeout has been exceeded
timeInInitializing := time.Since(readyCond.LastTransitionTime.Time)
if timeInInitializing < timeout {
// Still within timeout
return
}
// Timeout exceeded - mark as failed
logger.Info("RayService initializing timeout exceeded",
"timeout", timeout,
"timeInInitializing", timeInInitializing,
"generation", rs.Generation)
// Clear cluster names to trigger cleanup
rs.Status.ActiveServiceStatus.RayClusterName = ""
rs.Status.PendingServiceStatus.RayClusterName = ""
// Set condition to Failed with InitializingTimeout reason
message := fmt.Sprintf("RayService failed to become ready within the configured timeout of %s. Time spent initializing: %s",
timeout, timeInInitializing)
setCondition(rs, rayv1.RayServiceReady, metav1.ConditionFalse, rayv1.RayServiceInitializingTimeout, message)
// Emit warning event
r.Recorder.Eventf(rs, corev1.EventTypeWarning, string(utils.RayServiceInitializingTimeout),
"RayService initializing timeout exceeded after %s (configured timeout: %s)",
timeInInitializing, timeout)
}
// reconcilePerClusterServeService reconciles a load-balancing serve service for a given RayCluster.
func (r *RayServiceReconciler) reconcilePerClusterServeService(ctx context.Context, rayServiceInstance *rayv1.RayService, rayClusterInstance *rayv1.RayCluster) error {
if rayClusterInstance == nil {
return nil
}
logger := ctrl.LoggerFrom(ctx).WithValues("RayCluster", rayClusterInstance.Name)
logger.Info("Building per-cluster serve service")
// Create a serve service for the RayCluster associated with this RayService. During an incremental
// upgrade, this will be called for both the active and pending RayCluster instances.
desiredSvc, err := common.BuildServeService(ctx, *rayServiceInstance, *rayClusterInstance, true)
if err != nil {
logger.Error(err, "Failed to build per-cluster serve service spec")
return err
}
if err := ctrl.SetControllerReference(rayClusterInstance, desiredSvc, r.Scheme); err != nil {
return err
}
existingSvc := &corev1.Service{}
err = r.Get(ctx, client.ObjectKey{Name: desiredSvc.Name, Namespace: desiredSvc.Namespace}, existingSvc)
if errors.IsNotFound(err) {
logger.Info("Creating new per-cluster serve service for incremental upgrade.", "Service", desiredSvc.Name)
return r.Create(ctx, desiredSvc)
}
return err
}
func shouldCompleteIncrementalRollback(
activeStatus *rayv1.RayServiceStatus,
pendingStatus *rayv1.RayServiceStatus,
pendingCluster *rayv1.RayCluster,
) bool {
// Check if the pending cluster is healthy enough to receive scale-down API calls.
// If the pending cluster is unhealthy and the active cluster has 100% of traffic
// routed to it, we should complete the rollback.
pendingClusterHealthy := pendingCluster != nil &&
meta.IsStatusConditionTrue(pendingCluster.Status.Conditions, string(rayv1.HeadPodReady))
// A rollback is complete when the active cluster is back at 100% TargetCapacity and TrafficRoutedPercent,
// and either:
// - The pending cluster has been fully scaled down (TargetCapacity and TrafficRoutedPercent are both 0), or
// - The pending cluster is unhealthy (head Pod not ready)
return ptr.Deref(activeStatus.TargetCapacity, -1) == 100 &&
ptr.Deref(activeStatus.TrafficRoutedPercent, -1) == 100 &&
(!pendingClusterHealthy || (ptr.Deref(pendingStatus.TargetCapacity, -1) == 0 &&
ptr.Deref(pendingStatus.TrafficRoutedPercent, -1) == 0))
}
// reconcileRollbackState determines whether to initiate a rollback by setting the RollbackInProgress condition.
func (r *RayServiceReconciler) reconcileRollbackState(ctx context.Context, rayServiceInstance *rayv1.RayService, activeCluster, pendingCluster *rayv1.RayCluster) error {
logger := ctrl.LoggerFrom(ctx)
targetHash, err := utils.GenerateHashWithoutReplicasAndWorkersToDelete(*rayClusterSpecForHashing(rayServiceInstance))
if err != nil {
return fmt.Errorf("failed to generate hash for goal cluster spec: %w", err)
}
originalHash := activeCluster.Annotations[utils.HashWithoutReplicasAndWorkersToDeleteKey]
pendingHash := pendingCluster.Annotations[utils.HashWithoutReplicasAndWorkersToDeleteKey]
isRollbackInProgress := meta.IsStatusConditionTrue(rayServiceInstance.Status.Conditions, string(rayv1.RollbackInProgress))
// Case 1: The goal spec matches the pending cluster's spec.
// The upgrade is on track. We should revert any accidental rollback attempt and continue.
if targetHash == pendingHash {
if isRollbackInProgress {
logger.Info("Goal state matches pending cluster. Canceling rollback and resuming upgrade.")
meta.RemoveStatusCondition(&rayServiceInstance.Status.Conditions, string(rayv1.RollbackInProgress))
}
return nil
}
// Case 2: The goal spec diverges from the pending cluster.
// This covers two sub-cases:
// 2.1: The user reverted to the original spec (targetHash == originalHash).
// The pending cluster is no longer needed, so we roll back to the active cluster.
// 2.2: The user submitted a 3rd entirely new spec mid-upgrade (targetHash != originalHash && targetHash != pendingHash).
// The pending cluster doesn't match the new goal either, so we must first roll back
// to the active cluster, clean up the pending cluster, and then start a fresh upgrade.
// In both sub-cases, we must safely route all traffic back to the original cluster before
// allowing a new cluster to be spun up.
if !isRollbackInProgress {
logger.Info("Goal state has changed during upgrade. Initiating safe rollback to the original cluster.", "targetHash", targetHash, "originalHash", originalHash, "pendingHash", pendingHash)
setCondition(rayServiceInstance, rayv1.RollbackInProgress, metav1.ConditionTrue, rayv1.TargetClusterChanged, "Goal state changed mid-upgrade, rolling back to original cluster.")
}
return nil
}