mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
* Add incremental upgrade API changes to KubeRay Signed-off-by: Ryan O'Leary <[email protected]> Update go mod dependencies for gateway v1 Signed-off-by: Ryan O'Leary <[email protected]> Add reconcile Gateway and HTTPRoute Signed-off-by: Ryan O'Leary <[email protected]> Add TargetCapacity and TrafficRoutedPercent to RayServiceStatus Signed-off-by: Ryan O'Leary <[email protected]> Add controller logic initial commit Signed-off-by: Ryan O'Leary <[email protected]> Add IncrementalUpgrade check to ShouldUpdate Signed-off-by: Ryan O'Leary <[email protected]> Update controller logic to reconcile incremental upgrade Signed-off-by: Ryan O'Leary <[email protected]> TrafficRoutedPercent should not set default value Signed-off-by: Ryan O'Leary <[email protected]> Remove test changes to TPU manifest Signed-off-by: Ryan O'Leary <[email protected]> Move helper function to utils Signed-off-by: Ryan O'Leary <[email protected]> Fix lint Signed-off-by: Ryan O'Leary <[email protected]> Fix field alignment Signed-off-by: Ryan O'Leary <[email protected]> Fix bad merge Signed-off-by: Ryan O'Leary <[email protected]> Fix CRDs and add validation test case Signed-off-by: Ryan O'Leary <[email protected]> Test create HTTPRoute and create Gateway Signed-off-by: Ryan O'Leary <[email protected]> Add reconcile tests for Gateway and HTTPRoute Signed-off-by: Ryan O'Leary <[email protected]> Fix lint Signed-off-by: Ryan O'Leary <[email protected]> Add tests for util functions and fix golangci-lint Signed-off-by: Ryan O'Leary <[email protected]> Add basic e2e test case Signed-off-by: Ryan O'Leary <[email protected]> Fix GetGatewayListeners logic and test Signed-off-by: Ryan O'Leary <[email protected]> Add gatewayv1 scheme to util runtime Signed-off-by: Ryan O'Leary <[email protected]> Check if IncrementalUpgrade is enabled before checking Gateway Signed-off-by: Ryan O'Leary <[email protected]> Fix reconcile logic for Gateway and HTTPRoute Signed-off-by: Ryan O'Leary <[email protected]> Add feature gate Signed-off-by: Ryan O'Leary <[email protected]> Always create Gateway and HTTPRoute for IncrementalUpgrade Signed-off-by: Ryan O'Leary <[email protected]> Fix target_capacity reonciliation logic Signed-off-by: Ryan O'Leary <[email protected]> Add additional unit tests Signed-off-by: Ryan O'Leary <[email protected]> Move e2e test and add another unit test Signed-off-by: Ryan O'Leary <[email protected]> * Fix some tests and create Gateway for pending cluster Signed-off-by: Ryan O'Leary <[email protected]> * Fix merge errors Signed-off-by: Ryan O'Leary <[email protected]> * Manually sync rbac for gateway Signed-off-by: Ryan O'Leary <[email protected]> * Fix bugs and e2e test Signed-off-by: Ryan O'Leary <[email protected]> * Add Makefile command Signed-off-by: Ryan O'Leary <[email protected]> * Run 'make sync' Signed-off-by: Ryan O'Leary <[email protected]> * Run 'make generate' Signed-off-by: Ryan O'Leary <[email protected]> * Fix comments Signed-off-by: Ryan O'Leary <[email protected]> * Run 'make api-docs' Signed-off-by: Ryan O'Leary <[email protected]> * Fix tests after merge conflicts Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Fix error return Signed-off-by: Ryan O'Leary <[email protected]> * Add RayServiceIncrementalUpgrade feature gate option to helm Signed-off-by: Ryan O'Leary <[email protected]> * Remove unnecessary perms Signed-off-by: Ryan O'Leary <[email protected]> * Remove delete perm and run lint Signed-off-by: Ryan O'Leary <[email protected]> * Fix helm roles Signed-off-by: Ryan O'Leary <[email protected]> * add back required perms Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/utils/validation.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/utils/util.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Change controller to use two serve services during upgrade Signed-off-by: Ryan O'Leary <[email protected]> * Remove Gateway and HTTPRoute API fields Signed-off-by: Ryan O'Leary <[email protected]> * Fix port errors Signed-off-by: Ryan O'Leary <[email protected]> * Fix comments and build issues Signed-off-by: Ryan O'Leary <[email protected]> * fix helm-chart-verify-rbac Signed-off-by: Ryan O'Leary <[email protected]> * Refactor tests and create HTTPRoute to be clearer Signed-off-by: Ryan O'Leary <[email protected]> * Use time &now Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Add function comments Signed-off-by: Ryan O'Leary <[email protected]> * Fix bad merge Signed-off-by: Ryan O'Leary <[email protected]> * Add more comments Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Add Ray Serve hostname and serve port logic Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/common/service.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/common/service.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Fix dropped requests and old cluster config not being served Signed-off-by: Ryan O'Leary <[email protected]> * Resolve readability comments and improve structure Signed-off-by: Ryan O'Leary <[email protected]> * Refactor based on comments Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/common/service.go Co-authored-by: Rueian <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Remove hostname from listener Signed-off-by: Ryan O'Leary <[email protected]> * ensure pending cluster scales from 0 target_capacity Signed-off-by: Ryan O'Leary <[email protected]> * Run make generate after rebase Signed-off-by: Ryan O'Leary <[email protected]> * rename upgrade type Signed-off-by: Ryan O'Leary <[email protected]> * Clean up utils and add more comments Signed-off-by: Ryan O'Leary <[email protected]> * reconcileHTTPRoute should pass created object to calculate status Signed-off-by: Ryan O'Leary <[email protected]> * lint Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Fix test after suggested fix Signed-off-by: Ryan O'Leary <[email protected]> --------- Signed-off-by: Ryan O'Leary <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Co-authored-by: Rueian <[email protected]>
114 lines
3.7 KiB
Go
114 lines
3.7 KiB
Go
package utils
|
|
|
|
import (
|
|
"reflect"
|
|
|
|
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
|
|
"github.com/ray-project/kuberay/ray-operator/pkg/features"
|
|
)
|
|
|
|
// Checks whether the old and new RayClusterStatus are inconsistent by comparing different fields. If the only
|
|
// differences between the old and new status are the `LastUpdateTime` and `ObservedGeneration` fields, the
|
|
// status update will not be triggered.
|
|
//
|
|
// TODO (kevin85421): The field `ObservedGeneration` is not being well-maintained at the moment. In the future,
|
|
// this field should be used to determine whether to update this CR or not.
|
|
func InconsistentRayClusterStatus(oldStatus rayv1.RayClusterStatus, newStatus rayv1.RayClusterStatus) bool {
|
|
if oldStatus.State != newStatus.State || oldStatus.Reason != newStatus.Reason {
|
|
return true
|
|
}
|
|
if oldStatus.ReadyWorkerReplicas != newStatus.ReadyWorkerReplicas ||
|
|
oldStatus.AvailableWorkerReplicas != newStatus.AvailableWorkerReplicas ||
|
|
oldStatus.DesiredWorkerReplicas != newStatus.DesiredWorkerReplicas ||
|
|
oldStatus.MinWorkerReplicas != newStatus.MinWorkerReplicas ||
|
|
oldStatus.MaxWorkerReplicas != newStatus.MaxWorkerReplicas {
|
|
return true
|
|
}
|
|
if !reflect.DeepEqual(oldStatus.Endpoints, newStatus.Endpoints) || !reflect.DeepEqual(oldStatus.Head, newStatus.Head) {
|
|
return true
|
|
}
|
|
if !reflect.DeepEqual(oldStatus.Conditions, newStatus.Conditions) {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Checks whether the old and new RayServiceStatus are inconsistent by comparing different fields.
|
|
// The RayClusterStatus field is only for observability in RayService CR, and changes to it will not trigger the status update.
|
|
func inconsistentRayServiceStatus(oldStatus rayv1.RayServiceStatus, newStatus rayv1.RayServiceStatus) bool {
|
|
if oldStatus.RayClusterName != newStatus.RayClusterName {
|
|
return true
|
|
}
|
|
|
|
if len(oldStatus.Applications) != len(newStatus.Applications) {
|
|
return true
|
|
}
|
|
|
|
var ok bool
|
|
for appName, newAppStatus := range newStatus.Applications {
|
|
var oldAppStatus rayv1.AppStatus
|
|
if oldAppStatus, ok = oldStatus.Applications[appName]; !ok {
|
|
return true
|
|
}
|
|
|
|
if oldAppStatus.Status != newAppStatus.Status {
|
|
return true
|
|
} else if oldAppStatus.Message != newAppStatus.Message {
|
|
return true
|
|
}
|
|
|
|
if len(oldAppStatus.Deployments) != len(newAppStatus.Deployments) {
|
|
return true
|
|
}
|
|
|
|
for deploymentName, newDeploymentStatus := range newAppStatus.Deployments {
|
|
var oldDeploymentStatus rayv1.ServeDeploymentStatus
|
|
if oldDeploymentStatus, ok = oldAppStatus.Deployments[deploymentName]; !ok {
|
|
return true
|
|
}
|
|
|
|
if oldDeploymentStatus.Status != newDeploymentStatus.Status {
|
|
return true
|
|
} else if oldDeploymentStatus.Message != newDeploymentStatus.Message {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
|
|
if features.Enabled(features.RayServiceIncrementalUpgrade) {
|
|
// Also check for changes in IncrementalUpgrade related Status fields.
|
|
if oldStatus.TrafficRoutedPercent != newStatus.TrafficRoutedPercent ||
|
|
oldStatus.TargetCapacity != newStatus.TargetCapacity ||
|
|
oldStatus.LastTrafficMigratedTime != newStatus.LastTrafficMigratedTime {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// Determine whether to update the status of the RayService instance.
|
|
func InconsistentRayServiceStatuses(oldStatus rayv1.RayServiceStatuses, newStatus rayv1.RayServiceStatuses) bool {
|
|
if oldStatus.ServiceStatus != newStatus.ServiceStatus {
|
|
return true
|
|
}
|
|
|
|
if oldStatus.NumServeEndpoints != newStatus.NumServeEndpoints {
|
|
return true
|
|
}
|
|
|
|
if !reflect.DeepEqual(oldStatus.Conditions, newStatus.Conditions) {
|
|
return true
|
|
}
|
|
|
|
if inconsistentRayServiceStatus(oldStatus.ActiveServiceStatus, newStatus.ActiveServiceStatus) {
|
|
return true
|
|
}
|
|
|
|
if inconsistentRayServiceStatus(oldStatus.PendingServiceStatus, newStatus.PendingServiceStatus) {
|
|
return true
|
|
}
|
|
|
|
return false
|
|
}
|