mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
* Add incremental upgrade API changes to KubeRay Signed-off-by: Ryan O'Leary <[email protected]> Update go mod dependencies for gateway v1 Signed-off-by: Ryan O'Leary <[email protected]> Add reconcile Gateway and HTTPRoute Signed-off-by: Ryan O'Leary <[email protected]> Add TargetCapacity and TrafficRoutedPercent to RayServiceStatus Signed-off-by: Ryan O'Leary <[email protected]> Add controller logic initial commit Signed-off-by: Ryan O'Leary <[email protected]> Add IncrementalUpgrade check to ShouldUpdate Signed-off-by: Ryan O'Leary <[email protected]> Update controller logic to reconcile incremental upgrade Signed-off-by: Ryan O'Leary <[email protected]> TrafficRoutedPercent should not set default value Signed-off-by: Ryan O'Leary <[email protected]> Remove test changes to TPU manifest Signed-off-by: Ryan O'Leary <[email protected]> Move helper function to utils Signed-off-by: Ryan O'Leary <[email protected]> Fix lint Signed-off-by: Ryan O'Leary <[email protected]> Fix field alignment Signed-off-by: Ryan O'Leary <[email protected]> Fix bad merge Signed-off-by: Ryan O'Leary <[email protected]> Fix CRDs and add validation test case Signed-off-by: Ryan O'Leary <[email protected]> Test create HTTPRoute and create Gateway Signed-off-by: Ryan O'Leary <[email protected]> Add reconcile tests for Gateway and HTTPRoute Signed-off-by: Ryan O'Leary <[email protected]> Fix lint Signed-off-by: Ryan O'Leary <[email protected]> Add tests for util functions and fix golangci-lint Signed-off-by: Ryan O'Leary <[email protected]> Add basic e2e test case Signed-off-by: Ryan O'Leary <[email protected]> Fix GetGatewayListeners logic and test Signed-off-by: Ryan O'Leary <[email protected]> Add gatewayv1 scheme to util runtime Signed-off-by: Ryan O'Leary <[email protected]> Check if IncrementalUpgrade is enabled before checking Gateway Signed-off-by: Ryan O'Leary <[email protected]> Fix reconcile logic for Gateway and HTTPRoute Signed-off-by: Ryan O'Leary <[email protected]> Add feature gate Signed-off-by: Ryan O'Leary <[email protected]> Always create Gateway and HTTPRoute for IncrementalUpgrade Signed-off-by: Ryan O'Leary <[email protected]> Fix target_capacity reonciliation logic Signed-off-by: Ryan O'Leary <[email protected]> Add additional unit tests Signed-off-by: Ryan O'Leary <[email protected]> Move e2e test and add another unit test Signed-off-by: Ryan O'Leary <[email protected]> * Fix some tests and create Gateway for pending cluster Signed-off-by: Ryan O'Leary <[email protected]> * Fix merge errors Signed-off-by: Ryan O'Leary <[email protected]> * Manually sync rbac for gateway Signed-off-by: Ryan O'Leary <[email protected]> * Fix bugs and e2e test Signed-off-by: Ryan O'Leary <[email protected]> * Add Makefile command Signed-off-by: Ryan O'Leary <[email protected]> * Run 'make sync' Signed-off-by: Ryan O'Leary <[email protected]> * Run 'make generate' Signed-off-by: Ryan O'Leary <[email protected]> * Fix comments Signed-off-by: Ryan O'Leary <[email protected]> * Run 'make api-docs' Signed-off-by: Ryan O'Leary <[email protected]> * Fix tests after merge conflicts Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Fix error return Signed-off-by: Ryan O'Leary <[email protected]> * Add RayServiceIncrementalUpgrade feature gate option to helm Signed-off-by: Ryan O'Leary <[email protected]> * Remove unnecessary perms Signed-off-by: Ryan O'Leary <[email protected]> * Remove delete perm and run lint Signed-off-by: Ryan O'Leary <[email protected]> * Fix helm roles Signed-off-by: Ryan O'Leary <[email protected]> * add back required perms Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/utils/validation.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/utils/util.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Change controller to use two serve services during upgrade Signed-off-by: Ryan O'Leary <[email protected]> * Remove Gateway and HTTPRoute API fields Signed-off-by: Ryan O'Leary <[email protected]> * Fix port errors Signed-off-by: Ryan O'Leary <[email protected]> * Fix comments and build issues Signed-off-by: Ryan O'Leary <[email protected]> * fix helm-chart-verify-rbac Signed-off-by: Ryan O'Leary <[email protected]> * Refactor tests and create HTTPRoute to be clearer Signed-off-by: Ryan O'Leary <[email protected]> * Use time &now Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Add function comments Signed-off-by: Ryan O'Leary <[email protected]> * Fix bad merge Signed-off-by: Ryan O'Leary <[email protected]> * Add more comments Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Add Ray Serve hostname and serve port logic Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/common/service.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/common/service.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Fix dropped requests and old cluster config not being served Signed-off-by: Ryan O'Leary <[email protected]> * Resolve readability comments and improve structure Signed-off-by: Ryan O'Leary <[email protected]> * Refactor based on comments Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/common/service.go Co-authored-by: Rueian <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Remove hostname from listener Signed-off-by: Ryan O'Leary <[email protected]> * ensure pending cluster scales from 0 target_capacity Signed-off-by: Ryan O'Leary <[email protected]> * Run make generate after rebase Signed-off-by: Ryan O'Leary <[email protected]> * rename upgrade type Signed-off-by: Ryan O'Leary <[email protected]> * Clean up utils and add more comments Signed-off-by: Ryan O'Leary <[email protected]> * reconcileHTTPRoute should pass created object to calculate status Signed-off-by: Ryan O'Leary <[email protected]> * lint Signed-off-by: Ryan O'Leary <[email protected]> * Update ray-operator/controllers/ray/rayservice_controller.go Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> * Fix test after suggested fix Signed-off-by: Ryan O'Leary <[email protected]> --------- Signed-off-by: Ryan O'Leary <[email protected]> Signed-off-by: Ryan O'Leary <[email protected]> Co-authored-by: Han-Ju Chen (Future-Outlier) <[email protected]> Co-authored-by: Rueian <[email protected]>
220 lines
7.4 KiB
Go
220 lines
7.4 KiB
Go
package common
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
|
|
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
|
|
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
|
|
)
|
|
|
|
func RayClusterServeServiceNamespacedName(instance *rayv1.RayCluster) types.NamespacedName {
|
|
return types.NamespacedName{
|
|
Namespace: instance.Namespace,
|
|
Name: utils.GenerateServeServiceName(instance.Name),
|
|
}
|
|
}
|
|
|
|
func RayClusterAutoscalerRoleNamespacedName(instance *rayv1.RayCluster) types.NamespacedName {
|
|
return types.NamespacedName{Namespace: instance.Namespace, Name: instance.Name}
|
|
}
|
|
|
|
func RayClusterAutoscalerRoleBindingNamespacedName(instance *rayv1.RayCluster) types.NamespacedName {
|
|
return types.NamespacedName{Namespace: instance.Namespace, Name: instance.Name}
|
|
}
|
|
|
|
func RayClusterAutoscalerServiceAccountNamespacedName(instance *rayv1.RayCluster) types.NamespacedName {
|
|
return types.NamespacedName{Namespace: instance.Namespace, Name: utils.GetHeadGroupServiceAccountName(instance)}
|
|
}
|
|
|
|
func RayClusterHeadlessServiceListOptions(instance *rayv1.RayCluster) []client.ListOption {
|
|
return []client.ListOption{
|
|
client.InNamespace(instance.Namespace),
|
|
client.MatchingLabels(map[string]string{utils.RayClusterHeadlessServiceLabelKey: instance.Name}),
|
|
}
|
|
}
|
|
|
|
func RayClusterHeadServiceListOptions(instance *rayv1.RayCluster) []client.ListOption {
|
|
return []client.ListOption{
|
|
client.InNamespace(instance.Namespace),
|
|
client.MatchingLabels(map[string]string{
|
|
utils.RayClusterLabelKey: instance.Name,
|
|
utils.RayNodeTypeLabelKey: string(rayv1.HeadNode),
|
|
utils.RayIDLabelKey: utils.CheckLabel(utils.GenerateIdentifier(instance.Name, rayv1.HeadNode)),
|
|
}),
|
|
}
|
|
}
|
|
|
|
type AssociationOption interface {
|
|
client.ListOption
|
|
client.DeleteAllOfOption
|
|
}
|
|
|
|
type AssociationOptions []AssociationOption
|
|
|
|
func (list AssociationOptions) ToListOptions() (options []client.ListOption) {
|
|
for _, option := range list {
|
|
options = append(options, option.(client.ListOption))
|
|
}
|
|
return options
|
|
}
|
|
|
|
func (list AssociationOptions) ToDeleteOptions() (options []client.DeleteAllOfOption) {
|
|
for _, option := range list {
|
|
options = append(options, option.(client.DeleteAllOfOption))
|
|
}
|
|
return options
|
|
}
|
|
|
|
func (list AssociationOptions) ToMetaV1ListOptions() (options metav1.ListOptions) {
|
|
listOptions := client.ListOptions{}
|
|
for _, option := range list {
|
|
option.(client.ListOption).ApplyToList(&listOptions)
|
|
}
|
|
return *listOptions.AsListOptions()
|
|
}
|
|
|
|
func RayClusterHeadPodsAssociationOptions(instance *rayv1.RayCluster) AssociationOptions {
|
|
return AssociationOptions{
|
|
client.InNamespace(instance.Namespace),
|
|
client.MatchingLabels{
|
|
utils.RayClusterLabelKey: instance.Name,
|
|
utils.RayNodeTypeLabelKey: string(rayv1.HeadNode),
|
|
},
|
|
}
|
|
}
|
|
|
|
func RayClusterWorkerPodsAssociationOptions(instance *rayv1.RayCluster) AssociationOptions {
|
|
return AssociationOptions{
|
|
client.InNamespace(instance.Namespace),
|
|
client.MatchingLabels{
|
|
utils.RayClusterLabelKey: instance.Name,
|
|
utils.RayNodeTypeLabelKey: string(rayv1.WorkerNode),
|
|
},
|
|
}
|
|
}
|
|
|
|
func RayClusterRedisCleanupJobAssociationOptions(instance *rayv1.RayCluster) AssociationOptions {
|
|
return AssociationOptions{
|
|
client.InNamespace(instance.Namespace),
|
|
client.MatchingLabels{
|
|
utils.RayClusterLabelKey: instance.Name,
|
|
utils.RayNodeTypeLabelKey: string(rayv1.RedisCleanupNode),
|
|
},
|
|
}
|
|
}
|
|
|
|
func RayClusterGroupPodsAssociationOptions(instance *rayv1.RayCluster, group string) AssociationOptions {
|
|
return AssociationOptions{
|
|
client.InNamespace(instance.Namespace),
|
|
client.MatchingLabels{
|
|
utils.RayClusterLabelKey: instance.Name,
|
|
utils.RayNodeGroupLabelKey: group,
|
|
},
|
|
}
|
|
}
|
|
|
|
func RayClusterAllPodsAssociationOptions(instance *rayv1.RayCluster) AssociationOptions {
|
|
return AssociationOptions{
|
|
client.InNamespace(instance.Namespace),
|
|
client.MatchingLabels{
|
|
utils.RayClusterLabelKey: instance.Name,
|
|
},
|
|
}
|
|
}
|
|
|
|
func RayServiceRayClustersAssociationOptions(rayService *rayv1.RayService) AssociationOptions {
|
|
return AssociationOptions{
|
|
client.InNamespace(rayService.Namespace),
|
|
client.MatchingLabels{
|
|
utils.RayOriginatedFromCRNameLabelKey: rayService.Name,
|
|
utils.RayOriginatedFromCRDLabelKey: utils.RayOriginatedFromCRDLabelValue(utils.RayServiceCRD),
|
|
},
|
|
}
|
|
}
|
|
|
|
func RayServiceServeServiceNamespacedName(rayService *rayv1.RayService) types.NamespacedName {
|
|
if rayService.Spec.ServeService != nil && rayService.Spec.ServeService.Name != "" {
|
|
return types.NamespacedName{
|
|
Namespace: rayService.Namespace, // We do not respect s.Spec.ServeService.Namespace intentionally. Ref: https://github.com/ray-project/kuberay/blob/f6b4c3126654d1ef42965abc781846624b8e5dfc/ray-operator/controllers/ray/common/service.go#L298
|
|
Name: rayService.Spec.ServeService.Name,
|
|
}
|
|
}
|
|
return types.NamespacedName{
|
|
Namespace: rayService.Namespace,
|
|
Name: utils.GenerateServeServiceName(rayService.Name),
|
|
}
|
|
}
|
|
|
|
func RayServiceActiveRayClusterNamespacedName(rayService *rayv1.RayService) types.NamespacedName {
|
|
return types.NamespacedName{Name: rayService.Status.ActiveServiceStatus.RayClusterName, Namespace: rayService.Namespace}
|
|
}
|
|
|
|
func RayServicePendingRayClusterNamespacedName(rayService *rayv1.RayService) types.NamespacedName {
|
|
return types.NamespacedName{Name: rayService.Status.PendingServiceStatus.RayClusterName, Namespace: rayService.Namespace}
|
|
}
|
|
|
|
// RayJobK8sJobNamespacedName is the only place to associate the RayJob with the submitter Kubernetes Job.
|
|
func RayJobK8sJobNamespacedName(rayJob *rayv1.RayJob) types.NamespacedName {
|
|
return types.NamespacedName{
|
|
Namespace: rayJob.Namespace,
|
|
Name: rayJob.Name,
|
|
}
|
|
}
|
|
|
|
func RayJobRayClusterNamespacedName(rayJob *rayv1.RayJob) types.NamespacedName {
|
|
return types.NamespacedName{
|
|
Name: rayJob.Status.RayClusterName,
|
|
Namespace: rayJob.Namespace,
|
|
}
|
|
}
|
|
|
|
// GetRayClusterHeadPod gets a *corev1.Pod from a *rayv1.RayCluster. Note that it returns (nil, nil) in the case of no head pod exists.
|
|
func GetRayClusterHeadPod(ctx context.Context, reader client.Reader, instance *rayv1.RayCluster) (*corev1.Pod, error) {
|
|
logger := ctrl.LoggerFrom(ctx)
|
|
|
|
runtimePods := corev1.PodList{}
|
|
filterLabels := RayClusterHeadPodsAssociationOptions(instance)
|
|
if err := reader.List(ctx, &runtimePods, filterLabels.ToListOptions()...); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(runtimePods.Items) == 0 {
|
|
logger.Info("Found 0 head pod", "filter labels", filterLabels)
|
|
return nil, nil
|
|
}
|
|
if len(runtimePods.Items) > 1 {
|
|
logger.Info("Found multiple head pods", "count", len(runtimePods.Items), "filter labels", filterLabels)
|
|
return nil, fmt.Errorf("found multiple heads. filter labels %v", filterLabels)
|
|
}
|
|
return &runtimePods.Items[0], nil
|
|
}
|
|
|
|
func RayClusterNetworkResourcesOptions(instance *rayv1.RayCluster) AssociationOptions {
|
|
return AssociationOptions{
|
|
client.InNamespace(instance.Namespace),
|
|
client.MatchingLabels{
|
|
utils.RayClusterLabelKey: instance.Name,
|
|
},
|
|
}
|
|
}
|
|
|
|
func RayServiceGatewayNamespacedName(rayService *rayv1.RayService) types.NamespacedName {
|
|
return types.NamespacedName{
|
|
Name: fmt.Sprintf("%s-gateway", rayService.Name),
|
|
Namespace: rayService.Namespace,
|
|
}
|
|
}
|
|
|
|
func RayServiceHTTPRouteNamespacedName(rayService *rayv1.RayService) types.NamespacedName {
|
|
return types.NamespacedName{
|
|
Name: fmt.Sprintf("%s-httproute", rayService.Name),
|
|
Namespace: rayService.Namespace,
|
|
}
|
|
}
|