Files
2acc21964d [RayService] Support Incremental Zero-Downtime Upgrades (#3166)
* Add incremental upgrade API changes to KubeRay

Signed-off-by: Ryan O'Leary <[email protected]>

Update go mod dependencies for gateway v1

Signed-off-by: Ryan O'Leary <[email protected]>

Add reconcile Gateway and HTTPRoute

Signed-off-by: Ryan O'Leary <[email protected]>

Add TargetCapacity and TrafficRoutedPercent to RayServiceStatus

Signed-off-by: Ryan O'Leary <[email protected]>

Add controller logic initial commit

Signed-off-by: Ryan O'Leary <[email protected]>

Add IncrementalUpgrade check to ShouldUpdate

Signed-off-by: Ryan O'Leary <[email protected]>

Update controller logic to reconcile incremental upgrade

Signed-off-by: Ryan O'Leary <[email protected]>

TrafficRoutedPercent should not set default value

Signed-off-by: Ryan O'Leary <[email protected]>

Remove test changes to TPU manifest

Signed-off-by: Ryan O'Leary <[email protected]>

Move helper function to utils

Signed-off-by: Ryan O'Leary <[email protected]>

Fix lint

Signed-off-by: Ryan O'Leary <[email protected]>

Fix field alignment

Signed-off-by: Ryan O'Leary <[email protected]>

Fix bad merge

Signed-off-by: Ryan O'Leary <[email protected]>

Fix CRDs and add validation test case

Signed-off-by: Ryan O'Leary <[email protected]>

Test create HTTPRoute and create Gateway

Signed-off-by: Ryan O'Leary <[email protected]>

Add reconcile tests for Gateway and HTTPRoute

Signed-off-by: Ryan O'Leary <[email protected]>

Fix lint

Signed-off-by: Ryan O'Leary <[email protected]>

Add tests for util functions and fix golangci-lint

Signed-off-by: Ryan O'Leary <[email protected]>

Add basic e2e test case

Signed-off-by: Ryan O'Leary <[email protected]>

Fix GetGatewayListeners logic and test

Signed-off-by: Ryan O'Leary <[email protected]>

Add gatewayv1 scheme to util runtime

Signed-off-by: Ryan O'Leary <[email protected]>

Check if IncrementalUpgrade is enabled before checking Gateway

Signed-off-by: Ryan O'Leary <[email protected]>

Fix reconcile logic for Gateway and HTTPRoute

Signed-off-by: Ryan O'Leary <[email protected]>

Add feature gate

Signed-off-by: Ryan O'Leary <[email protected]>

Always create Gateway and HTTPRoute for IncrementalUpgrade

Signed-off-by: Ryan O'Leary <[email protected]>

Fix target_capacity reonciliation logic

Signed-off-by: Ryan O'Leary <[email protected]>

Add additional unit tests

Signed-off-by: Ryan O'Leary <[email protected]>

Move e2e test and add another unit test

Signed-off-by: Ryan O'Leary <[email protected]>

* Fix some tests and create Gateway for pending cluster

Signed-off-by: Ryan O'Leary <[email protected]>

* Fix merge errors

Signed-off-by: Ryan O'Leary <[email protected]>

* Manually sync rbac for gateway

Signed-off-by: Ryan O'Leary <[email protected]>

* Fix bugs and e2e test

Signed-off-by: Ryan O'Leary <[email protected]>

* Add Makefile command

Signed-off-by: Ryan O'Leary <[email protected]>

* Run 'make sync'

Signed-off-by: Ryan O'Leary <[email protected]>

* Run 'make generate'

Signed-off-by: Ryan O'Leary <[email protected]>

* Fix comments

Signed-off-by: Ryan O'Leary <[email protected]>

* Run 'make api-docs'

Signed-off-by: Ryan O'Leary <[email protected]>

* Fix tests after merge conflicts

Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/rayservice_controller.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/rayservice_controller.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/rayservice_controller.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Fix error return

Signed-off-by: Ryan O'Leary <[email protected]>

* Add RayServiceIncrementalUpgrade feature gate option to helm

Signed-off-by: Ryan O'Leary <[email protected]>

* Remove unnecessary perms

Signed-off-by: Ryan O'Leary <[email protected]>

* Remove delete perm and run lint

Signed-off-by: Ryan O'Leary <[email protected]>

* Fix helm roles

Signed-off-by: Ryan O'Leary <[email protected]>

* add back required perms

Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/utils/validation.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/utils/util.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/rayservice_controller.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Change controller to use two serve services during upgrade

Signed-off-by: Ryan O'Leary <[email protected]>

* Remove Gateway and HTTPRoute API fields

Signed-off-by: Ryan O'Leary <[email protected]>

* Fix port errors

Signed-off-by: Ryan O'Leary <[email protected]>

* Fix comments and build issues

Signed-off-by: Ryan O'Leary <[email protected]>

* fix helm-chart-verify-rbac

Signed-off-by: Ryan O'Leary <[email protected]>

* Refactor tests and create HTTPRoute to be clearer

Signed-off-by: Ryan O'Leary <[email protected]>

* Use time &now

Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/rayservice_controller.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Add function comments

Signed-off-by: Ryan O'Leary <[email protected]>

* Fix bad merge

Signed-off-by: Ryan O'Leary <[email protected]>

* Add more comments

Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/rayservice_controller.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Add Ray Serve hostname and serve port logic

Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/rayservice_controller.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/common/service.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/common/service.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/rayservice_controller.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/rayservice_controller.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/rayservice_controller.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/rayservice_controller.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Fix dropped requests and old cluster config not being served

Signed-off-by: Ryan O'Leary <[email protected]>

* Resolve readability comments and improve structure

Signed-off-by: Ryan O'Leary <[email protected]>

* Refactor based on comments

Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/common/service.go

Co-authored-by: Rueian <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Remove hostname from listener

Signed-off-by: Ryan O'Leary <[email protected]>

* ensure pending cluster scales from 0 target_capacity

Signed-off-by: Ryan O'Leary <[email protected]>

* Run make generate after rebase

Signed-off-by: Ryan O'Leary <[email protected]>

* rename upgrade type

Signed-off-by: Ryan O'Leary <[email protected]>

* Clean up utils and add more comments

Signed-off-by: Ryan O'Leary <[email protected]>

* reconcileHTTPRoute should pass created object to calculate status

Signed-off-by: Ryan O'Leary <[email protected]>

* lint

Signed-off-by: Ryan O'Leary <[email protected]>

* Update ray-operator/controllers/ray/rayservice_controller.go

Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>

* Fix test after suggested fix

Signed-off-by: Ryan O'Leary <[email protected]>

---------

Signed-off-by: Ryan O'Leary <[email protected]>
Signed-off-by: Ryan O'Leary <[email protected]>
Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]>
Co-authored-by: Rueian <[email protected]>
2025-10-25 15:55:12 -07:00

114 lines
3.7 KiB
Go

package utils
import (
"reflect"
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
"github.com/ray-project/kuberay/ray-operator/pkg/features"
)
// Checks whether the old and new RayClusterStatus are inconsistent by comparing different fields. If the only
// differences between the old and new status are the `LastUpdateTime` and `ObservedGeneration` fields, the
// status update will not be triggered.
//
// TODO (kevin85421): The field `ObservedGeneration` is not being well-maintained at the moment. In the future,
// this field should be used to determine whether to update this CR or not.
func InconsistentRayClusterStatus(oldStatus rayv1.RayClusterStatus, newStatus rayv1.RayClusterStatus) bool {
if oldStatus.State != newStatus.State || oldStatus.Reason != newStatus.Reason {
return true
}
if oldStatus.ReadyWorkerReplicas != newStatus.ReadyWorkerReplicas ||
oldStatus.AvailableWorkerReplicas != newStatus.AvailableWorkerReplicas ||
oldStatus.DesiredWorkerReplicas != newStatus.DesiredWorkerReplicas ||
oldStatus.MinWorkerReplicas != newStatus.MinWorkerReplicas ||
oldStatus.MaxWorkerReplicas != newStatus.MaxWorkerReplicas {
return true
}
if !reflect.DeepEqual(oldStatus.Endpoints, newStatus.Endpoints) || !reflect.DeepEqual(oldStatus.Head, newStatus.Head) {
return true
}
if !reflect.DeepEqual(oldStatus.Conditions, newStatus.Conditions) {
return true
}
return false
}
// Checks whether the old and new RayServiceStatus are inconsistent by comparing different fields.
// The RayClusterStatus field is only for observability in RayService CR, and changes to it will not trigger the status update.
func inconsistentRayServiceStatus(oldStatus rayv1.RayServiceStatus, newStatus rayv1.RayServiceStatus) bool {
if oldStatus.RayClusterName != newStatus.RayClusterName {
return true
}
if len(oldStatus.Applications) != len(newStatus.Applications) {
return true
}
var ok bool
for appName, newAppStatus := range newStatus.Applications {
var oldAppStatus rayv1.AppStatus
if oldAppStatus, ok = oldStatus.Applications[appName]; !ok {
return true
}
if oldAppStatus.Status != newAppStatus.Status {
return true
} else if oldAppStatus.Message != newAppStatus.Message {
return true
}
if len(oldAppStatus.Deployments) != len(newAppStatus.Deployments) {
return true
}
for deploymentName, newDeploymentStatus := range newAppStatus.Deployments {
var oldDeploymentStatus rayv1.ServeDeploymentStatus
if oldDeploymentStatus, ok = oldAppStatus.Deployments[deploymentName]; !ok {
return true
}
if oldDeploymentStatus.Status != newDeploymentStatus.Status {
return true
} else if oldDeploymentStatus.Message != newDeploymentStatus.Message {
return true
}
}
}
if features.Enabled(features.RayServiceIncrementalUpgrade) {
// Also check for changes in IncrementalUpgrade related Status fields.
if oldStatus.TrafficRoutedPercent != newStatus.TrafficRoutedPercent ||
oldStatus.TargetCapacity != newStatus.TargetCapacity ||
oldStatus.LastTrafficMigratedTime != newStatus.LastTrafficMigratedTime {
return true
}
}
return false
}
// Determine whether to update the status of the RayService instance.
func InconsistentRayServiceStatuses(oldStatus rayv1.RayServiceStatuses, newStatus rayv1.RayServiceStatuses) bool {
if oldStatus.ServiceStatus != newStatus.ServiceStatus {
return true
}
if oldStatus.NumServeEndpoints != newStatus.NumServeEndpoints {
return true
}
if !reflect.DeepEqual(oldStatus.Conditions, newStatus.Conditions) {
return true
}
if inconsistentRayServiceStatus(oldStatus.ActiveServiceStatus, newStatus.ActiveServiceStatus) {
return true
}
if inconsistentRayServiceStatus(oldStatus.PendingServiceStatus, newStatus.PendingServiceStatus) {
return true
}
return false
}