mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
188 lines
6.4 KiB
Go
188 lines
6.4 KiB
Go
package client
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
dockerparser "github.com/novln/docker-parser"
|
|
"k8s.io/apimachinery/pkg/api/meta"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/fields"
|
|
"k8s.io/apimachinery/pkg/watch"
|
|
"k8s.io/client-go/kubernetes"
|
|
cmdutil "k8s.io/kubectl/pkg/cmd/util"
|
|
|
|
"github.com/ray-project/kuberay/kubectl-plugin/pkg/util"
|
|
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
|
|
rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned"
|
|
)
|
|
|
|
type Client interface {
|
|
KubernetesClient() kubernetes.Interface
|
|
RayClient() rayclient.Interface
|
|
// GetRayHeadSvcName retrieves the name of RayHead service for the given RayCluster, RayJob, or RayService.
|
|
GetRayHeadSvcName(ctx context.Context, namespace string, resourceType util.ResourceType, name string) (string, error)
|
|
GetKubeRayOperatorVersion(ctx context.Context) (string, error)
|
|
WaitRayClusterProvisioned(ctx context.Context, namespace, name string, timeout time.Duration) error
|
|
}
|
|
|
|
type k8sClient struct {
|
|
kubeClient kubernetes.Interface
|
|
rayClient rayclient.Interface
|
|
}
|
|
|
|
func NewClient(factory cmdutil.Factory) (Client, error) {
|
|
kubeClient, err := factory.KubernetesClientSet()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
restConfig, err := factory.ToRESTConfig()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rayClient, err := rayclient.NewForConfig(restConfig)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &k8sClient{
|
|
kubeClient: kubeClient,
|
|
rayClient: rayClient,
|
|
}, nil
|
|
}
|
|
|
|
func NewClientForTesting(kubeClient kubernetes.Interface, rayClient rayclient.Interface) Client {
|
|
return &k8sClient{
|
|
kubeClient: kubeClient,
|
|
rayClient: rayClient,
|
|
}
|
|
}
|
|
|
|
func (c *k8sClient) KubernetesClient() kubernetes.Interface {
|
|
return c.kubeClient
|
|
}
|
|
|
|
func (c *k8sClient) RayClient() rayclient.Interface {
|
|
return c.rayClient
|
|
}
|
|
|
|
func (c *k8sClient) GetKubeRayOperatorVersion(ctx context.Context) (string, error) {
|
|
deployment, err := c.kubeClient.AppsV1().Deployments("").List(ctx, metav1.ListOptions{
|
|
LabelSelector: "app.kubernetes.io/name in (kuberay-operator,kuberay)",
|
|
})
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to get KubeRay operator deployment: %w", err)
|
|
}
|
|
|
|
if len(deployment.Items) == 0 {
|
|
return "", fmt.Errorf("no KubeRay operator deployments found in any namespace")
|
|
}
|
|
|
|
containers := deployment.Items[0].Spec.Template.Spec.Containers
|
|
if len(containers) == 0 {
|
|
return "", fmt.Errorf("no containers found in KubeRay operator deployment")
|
|
}
|
|
|
|
image := containers[0].Image
|
|
ref, err := dockerparser.Parse(image)
|
|
if err != nil {
|
|
return "", fmt.Errorf("unable to parse KubeRay operator version from image: %w", err)
|
|
}
|
|
|
|
// If image reference contains both digest and tag, return both
|
|
if strings.Contains(image, "@sha256:") && strings.Count(image, ":") == 2 {
|
|
parts := strings.SplitN(image, ":", 2)
|
|
return parts[len(parts)-1], nil
|
|
}
|
|
|
|
return ref.Tag(), nil
|
|
}
|
|
|
|
func (c *k8sClient) GetRayHeadSvcName(ctx context.Context, namespace string, resourceType util.ResourceType, name string) (string, error) {
|
|
switch resourceType {
|
|
case util.RayCluster:
|
|
return c.getRayHeadSvcNameByRayCluster(ctx, namespace, name)
|
|
case util.RayJob:
|
|
return c.getRayHeadSvcNameByRayJob(ctx, namespace, name)
|
|
case util.RayService:
|
|
return c.getRayHeadSvcNameByRayService(ctx, namespace, name)
|
|
default:
|
|
return "", fmt.Errorf("unsupported resource type: %s", resourceType)
|
|
}
|
|
}
|
|
|
|
// WaitRayClusterProvisioned blocks until the RayCluster has a status condition with type=RayClusterProvisioned and status=true.
|
|
// It returns an error if the timeout is reached.
|
|
func (c *k8sClient) WaitRayClusterProvisioned(ctx context.Context, namespace, name string, timeout time.Duration) error {
|
|
timeoutCtx, cancel := context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
|
|
watcher, err := c.RayClient().RayV1().RayClusters(namespace).Watch(ctx, metav1.ListOptions{
|
|
FieldSelector: fields.OneTermEqualSelector("metadata.name", name).String(),
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to watch Ray cluster %s in namespace %s: %w", name, namespace, err)
|
|
}
|
|
defer watcher.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-timeoutCtx.Done():
|
|
if timeoutCtx.Err() == context.DeadlineExceeded {
|
|
return fmt.Errorf("timed out waiting for Ray cluster %s in namespace %s to be provisioned", name, namespace)
|
|
}
|
|
return ctx.Err()
|
|
case event := <-watcher.ResultChan():
|
|
if event.Type == watch.Error {
|
|
return fmt.Errorf("error watching Ray cluster: %v", event.Object)
|
|
}
|
|
|
|
cluster, ok := event.Object.(*rayv1.RayCluster)
|
|
if !ok {
|
|
return fmt.Errorf("unexpected type %T", event.Object)
|
|
}
|
|
|
|
if meta.IsStatusConditionTrue(cluster.Status.Conditions, string(rayv1.RayClusterProvisioned)) || cluster.Status.State == rayv1.Ready {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *k8sClient) getRayHeadSvcNameByRayCluster(ctx context.Context, namespace string, name string) (string, error) {
|
|
rayCluster, err := c.RayClient().RayV1().RayClusters(namespace).Get(ctx, name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return "", fmt.Errorf("unable to find RayCluster %s: %w", name, err)
|
|
}
|
|
svcName := rayCluster.Status.Head.ServiceName
|
|
return svcName, nil
|
|
}
|
|
|
|
func (c *k8sClient) getRayHeadSvcNameByRayJob(ctx context.Context, namespace string, name string) (string, error) {
|
|
rayJob, err := c.RayClient().RayV1().RayJobs(namespace).Get(ctx, name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return "", fmt.Errorf("unable to find RayJob %s: %w", name, err)
|
|
}
|
|
svcName := rayJob.Status.RayClusterStatus.Head.ServiceName
|
|
return svcName, nil
|
|
}
|
|
|
|
// There are 3 services associated with a RayService:
|
|
// - <rayservice-name>-head-svc
|
|
// - <rayservice-name>-serve-svc
|
|
// - <raycluster-name>-head-svc
|
|
// This function retrieves the name of the <raycluster-name>-head-svc service.
|
|
// Actually there is no difference between which service to use, because kubectl port-forward source code first tries to find the underlying pod.
|
|
// See https://github.com/kubernetes/kubectl/blob/262825a8a665c7cae467dfaa42b63be5a5b8e5a2/pkg/cmd/portforward/portforward.go#L345 for details.
|
|
func (c *k8sClient) getRayHeadSvcNameByRayService(ctx context.Context, namespace string, name string) (string, error) {
|
|
rayService, err := c.RayClient().RayV1().RayServices(namespace).Get(ctx, name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return "", fmt.Errorf("unable to find RayService %s: %w", name, err)
|
|
}
|
|
svcName := rayService.Status.ActiveServiceStatus.RayClusterStatus.Head.ServiceName
|
|
return svcName, nil
|
|
}
|