mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
642 lines
20 KiB
Go
642 lines
20 KiB
Go
package model
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"maps"
|
|
"slices"
|
|
"strconv"
|
|
|
|
"google.golang.org/protobuf/types/known/timestamppb"
|
|
corev1 "k8s.io/api/core/v1"
|
|
klog "k8s.io/klog/v2"
|
|
|
|
"github.com/ray-project/kuberay/apiserver/pkg/util"
|
|
api "github.com/ray-project/kuberay/proto/go_client"
|
|
rayv1api "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
|
|
pkgutils "github.com/ray-project/kuberay/ray-operator/pkg/utils"
|
|
)
|
|
|
|
// Default annotations used by Ray nodes
|
|
func getNodeDefaultAnnotations() []string {
|
|
return []string{
|
|
"ray.io/compute-image",
|
|
"openshift.io/scc",
|
|
"cni.projectcalico.org/podIP",
|
|
"ray.io/health-state",
|
|
"ray.io/ft-enabled",
|
|
"cni.projectcalico.org/podIPs",
|
|
"cni.projectcalico.org/containerID",
|
|
"ray.io/compute-template",
|
|
"k8s.v1.cni.cncf.io/network-status",
|
|
"k8s.v1.cni.cncf.io/networks-status",
|
|
}
|
|
}
|
|
|
|
// Default labels used by Ray nodes
|
|
func getNodeDefaultLabels() []string {
|
|
return []string{
|
|
"app.kubernetes.io/created-by",
|
|
"app.kubernetes.io/name",
|
|
"ray.io/cluster",
|
|
"ray.io/cluster-dashboard",
|
|
"ray.io/group",
|
|
"ray.io/identifier",
|
|
"ray.io/is-ray-node",
|
|
"ray.io/node-type",
|
|
}
|
|
}
|
|
|
|
// Default env used by Ray head nodes
|
|
func getHeadNodeEnv() []string {
|
|
return []string{
|
|
"MY_POD_IP",
|
|
"RAY_CLUSTER_NAME",
|
|
"RAY_PORT",
|
|
"RAY_ADDRESS",
|
|
"RAY_USAGE_STATS_KUBERAY_IN_USE",
|
|
}
|
|
}
|
|
|
|
// Default env used by Ray worker nodes
|
|
func getWorkNodeEnv() []string {
|
|
return []string{
|
|
"RAY_DISABLE_DOCKER_CPU_WARNING",
|
|
"TYPE",
|
|
"CPU_REQUEST",
|
|
"CPU_LIMITS",
|
|
"MEMORY_REQUESTS",
|
|
"MEMORY_LIMITS",
|
|
"MY_POD_NAME",
|
|
"MY_POD_IP",
|
|
"FQ_RAY_IP",
|
|
"RAY_IP",
|
|
"RAY_CLUSTER_NAME",
|
|
"RAY_PORT",
|
|
"RAY_ADDRESS",
|
|
"RAY_USAGE_STATS_KUBERAY_IN_USE",
|
|
"REDIS_PASSWORD",
|
|
}
|
|
}
|
|
|
|
// Check if an array contains string
|
|
func contains(s []string, str string) bool {
|
|
return slices.Contains(s, str)
|
|
}
|
|
|
|
func FromCrdToAPIClusters(clusters []*rayv1api.RayCluster, clusterEventsMap map[string][]corev1.Event) []*api.Cluster {
|
|
apiClusters := make([]*api.Cluster, 0, len(clusters))
|
|
for _, cluster := range clusters {
|
|
apiClusters = append(apiClusters, FromCrdToAPICluster(cluster, clusterEventsMap[cluster.Name]))
|
|
}
|
|
return apiClusters
|
|
}
|
|
|
|
func FromCrdToAPICluster(cluster *rayv1api.RayCluster, events []corev1.Event) *api.Cluster {
|
|
pbCluster := &api.Cluster{
|
|
Name: cluster.Name,
|
|
Namespace: cluster.Namespace,
|
|
Version: cluster.Labels[util.RayClusterVersionLabelKey],
|
|
User: cluster.Labels[util.RayClusterUserLabelKey],
|
|
Environment: api.Cluster_Environment(
|
|
api.Cluster_Environment_value[cluster.Labels[util.RayClusterEnvironmentLabelKey]],
|
|
),
|
|
CreatedAt: ×tamppb.Timestamp{Seconds: cluster.CreationTimestamp.Unix()},
|
|
ClusterState: string(cluster.Status.State),
|
|
}
|
|
|
|
if len(cluster.ObjectMeta.Annotations) > 0 {
|
|
pbCluster.Annotations = cluster.ObjectMeta.Annotations
|
|
}
|
|
|
|
// loop container and find the resource
|
|
pbCluster.ClusterSpec = PopulateRayClusterSpec(cluster.Spec)
|
|
|
|
// parse events
|
|
for _, event := range events {
|
|
clusterEvent := &api.ClusterEvent{
|
|
Id: event.Name,
|
|
Name: fmt.Sprintf("%s-%s", cluster.Labels[util.RayClusterNameLabelKey], event.Name),
|
|
CreatedAt: ×tamppb.Timestamp{Seconds: event.ObjectMeta.CreationTimestamp.Unix()},
|
|
FirstTimestamp: ×tamppb.Timestamp{Seconds: event.FirstTimestamp.Unix()},
|
|
LastTimestamp: ×tamppb.Timestamp{Seconds: event.LastTimestamp.Unix()},
|
|
Reason: event.Reason,
|
|
Message: event.Message,
|
|
Type: event.Type,
|
|
Count: event.Count,
|
|
}
|
|
pbCluster.Events = append(pbCluster.Events, clusterEvent)
|
|
}
|
|
|
|
pbCluster.ServiceEndpoint = map[string]string{}
|
|
maps.Copy(pbCluster.ServiceEndpoint, cluster.Status.Endpoints)
|
|
return pbCluster
|
|
}
|
|
|
|
func PopulateRayClusterSpec(spec rayv1api.RayClusterSpec) *api.ClusterSpec {
|
|
clusterSpec := &api.ClusterSpec{}
|
|
clusterSpec.HeadGroupSpec = PopulateHeadNodeSpec(spec.HeadGroupSpec)
|
|
clusterSpec.WorkerGroupSpec = PopulateWorkerNodeSpec(spec.WorkerGroupSpecs)
|
|
if spec.EnableInTreeAutoscaling != nil && *spec.EnableInTreeAutoscaling {
|
|
clusterSpec.EnableInTreeAutoscaling = true
|
|
clusterSpec.AutoscalerOptions = convertAutoscalingOptions(spec.AutoscalerOptions)
|
|
}
|
|
clusterSpec.HeadServiceAnnotations = spec.HeadServiceAnnotations
|
|
return clusterSpec
|
|
}
|
|
|
|
func convertAutoscalingOptions(opts *rayv1api.AutoscalerOptions) *api.AutoscalerOptions {
|
|
if opts == nil {
|
|
return nil
|
|
}
|
|
options := api.AutoscalerOptions{}
|
|
if opts.IdleTimeoutSeconds != nil {
|
|
options.IdleTimeoutSeconds = *opts.IdleTimeoutSeconds
|
|
}
|
|
if opts.UpscalingMode != nil {
|
|
options.UpscalingMode = string(*opts.UpscalingMode)
|
|
}
|
|
if opts.Image != nil {
|
|
options.Image = *opts.Image
|
|
}
|
|
if opts.ImagePullPolicy != nil {
|
|
options.ImagePullPolicy = string(*opts.ImagePullPolicy)
|
|
}
|
|
if len(opts.Env) > 0 || len(opts.EnvFrom) > 0 {
|
|
options.Envs = &api.EnvironmentVariables{}
|
|
if len(opts.Env) > 0 {
|
|
options.Envs.Values = map[string]string{}
|
|
for _, elem := range opts.Env {
|
|
options.Envs.Values[elem.Name] = elem.Value
|
|
}
|
|
}
|
|
if len(opts.EnvFrom) > 0 {
|
|
options.Envs.ValuesFrom = map[string]*api.EnvValueFrom{}
|
|
for i, elem := range opts.EnvFrom {
|
|
if elem.ConfigMapRef != nil {
|
|
options.Envs.ValuesFrom[strconv.Itoa(i)] = &api.EnvValueFrom{
|
|
Source: api.EnvValueFrom_CONFIGMAP,
|
|
Name: elem.ConfigMapRef.Name,
|
|
}
|
|
} else {
|
|
options.Envs.ValuesFrom[strconv.Itoa(i)] = &api.EnvValueFrom{
|
|
Source: api.EnvValueFrom_SECRET,
|
|
Name: elem.SecretRef.Name,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if len(opts.VolumeMounts) > 0 {
|
|
options.Volumes = make([]*api.Volume, len(opts.VolumeMounts))
|
|
for i, v := range opts.VolumeMounts {
|
|
options.Volumes[i] = &api.Volume{
|
|
Name: v.Name,
|
|
MountPath: v.MountPath,
|
|
ReadOnly: v.ReadOnly,
|
|
MountPropagationMode: GetVolumeMountPropagation(&v),
|
|
}
|
|
}
|
|
}
|
|
if opts.Resources != nil {
|
|
rlist := opts.Resources.Limits
|
|
if len(rlist) == 0 {
|
|
rlist = opts.Resources.Requests
|
|
}
|
|
if cc := rlist.Cpu().String(); cc != "" {
|
|
options.Cpu = cc
|
|
}
|
|
if cm := rlist.Memory().String(); cm != "" {
|
|
options.Memory = cm
|
|
}
|
|
}
|
|
|
|
return &options
|
|
}
|
|
|
|
func PopulateHeadNodeSpec(spec rayv1api.HeadGroupSpec) *api.HeadGroupSpec {
|
|
headNodeSpec := &api.HeadGroupSpec{
|
|
RayStartParams: spec.RayStartParams,
|
|
ServiceType: string(spec.ServiceType),
|
|
Image: spec.Template.Annotations[util.RayClusterImageAnnotationKey],
|
|
ComputeTemplate: spec.Template.Annotations[util.RayClusterComputeTemplateAnnotationKey],
|
|
Volumes: PopulateVolumes(&spec.Template),
|
|
}
|
|
|
|
for _, annotation := range getNodeDefaultAnnotations() {
|
|
delete(spec.Template.Annotations, annotation)
|
|
}
|
|
if len(spec.Template.Annotations) > 0 {
|
|
headNodeSpec.Annotations = spec.Template.Annotations
|
|
}
|
|
|
|
for _, label := range getNodeDefaultLabels() {
|
|
delete(spec.Template.Labels, label)
|
|
}
|
|
if len(spec.Template.Labels) > 0 {
|
|
headNodeSpec.Labels = spec.Template.Labels
|
|
}
|
|
|
|
if spec.EnableIngress != nil && *spec.EnableIngress {
|
|
headNodeSpec.EnableIngress = true
|
|
}
|
|
|
|
// Here we update environment and security context only for a container named 'ray-head'
|
|
if container, _, ok := util.GetContainerByName(spec.Template.Spec.Containers, "ray-head"); ok {
|
|
if len(container.Env) > 0 {
|
|
headNodeSpec.Environment = convertEnvVariables(container.Env, true)
|
|
}
|
|
headNodeSpec.SecurityContext = convertSecurityContext(container.SecurityContext)
|
|
}
|
|
|
|
if len(spec.Template.Spec.ServiceAccountName) > 1 {
|
|
headNodeSpec.ServiceAccount = spec.Template.Spec.ServiceAccountName
|
|
}
|
|
|
|
if len(spec.Template.Spec.ImagePullSecrets) > 0 {
|
|
headNodeSpec.ImagePullSecret = spec.Template.Spec.ImagePullSecrets[0].Name
|
|
}
|
|
if spec.Template.Spec.Containers[0].ImagePullPolicy == corev1.PullAlways {
|
|
headNodeSpec.ImagePullPolicy = "Always"
|
|
}
|
|
|
|
return headNodeSpec
|
|
}
|
|
|
|
func PopulateWorkerNodeSpec(specs []rayv1api.WorkerGroupSpec) []*api.WorkerGroupSpec {
|
|
var workerNodeSpecs []*api.WorkerGroupSpec
|
|
|
|
for _, spec := range specs {
|
|
workerNodeSpec := &api.WorkerGroupSpec{
|
|
RayStartParams: spec.RayStartParams,
|
|
MaxReplicas: *spec.MaxReplicas,
|
|
MinReplicas: *spec.MinReplicas,
|
|
Replicas: *spec.Replicas,
|
|
GroupName: spec.GroupName,
|
|
Image: spec.Template.Annotations[util.RayClusterImageAnnotationKey],
|
|
ComputeTemplate: spec.Template.Annotations[util.RayClusterComputeTemplateAnnotationKey],
|
|
Volumes: PopulateVolumes(&spec.Template),
|
|
}
|
|
|
|
for _, annotation := range getNodeDefaultAnnotations() {
|
|
delete(spec.Template.Annotations, annotation)
|
|
}
|
|
if len(spec.Template.Annotations) > 0 {
|
|
workerNodeSpec.Annotations = spec.Template.Annotations
|
|
}
|
|
|
|
for _, label := range getNodeDefaultLabels() {
|
|
delete(spec.Template.Labels, label)
|
|
}
|
|
if len(spec.Template.Labels) > 0 {
|
|
workerNodeSpec.Labels = spec.Template.Labels
|
|
}
|
|
|
|
// Here we update environment and security context only for a container named 'ray-worker'
|
|
if container, _, ok := util.GetContainerByName(spec.Template.Spec.Containers, "ray-worker"); ok {
|
|
if len(container.Env) > 0 {
|
|
workerNodeSpec.Environment = convertEnvVariables(container.Env, false)
|
|
}
|
|
workerNodeSpec.SecurityContext = convertSecurityContext(container.SecurityContext)
|
|
}
|
|
|
|
if len(spec.Template.Spec.ServiceAccountName) > 1 {
|
|
workerNodeSpec.ServiceAccount = spec.Template.Spec.ServiceAccountName
|
|
}
|
|
|
|
if len(spec.Template.Spec.ImagePullSecrets) > 0 {
|
|
workerNodeSpec.ImagePullSecret = spec.Template.Spec.ImagePullSecrets[0].Name
|
|
}
|
|
if spec.Template.Spec.Containers[0].ImagePullPolicy == corev1.PullAlways {
|
|
workerNodeSpec.ImagePullPolicy = "Always"
|
|
}
|
|
|
|
workerNodeSpecs = append(workerNodeSpecs, workerNodeSpec)
|
|
}
|
|
|
|
return workerNodeSpecs
|
|
}
|
|
|
|
func convertSecurityContext(securityCtx *corev1.SecurityContext) *api.SecurityContext {
|
|
if securityCtx == nil {
|
|
return nil
|
|
}
|
|
result := &api.SecurityContext{
|
|
Privileged: securityCtx.Privileged,
|
|
Capabilities: &api.Capabilities{},
|
|
}
|
|
if securityCtx.Capabilities != nil {
|
|
for _, cap := range securityCtx.Capabilities.Add {
|
|
result.Capabilities.Add = append(result.Capabilities.Add, string(cap))
|
|
}
|
|
for _, cap := range securityCtx.Capabilities.Drop {
|
|
result.Capabilities.Drop = append(result.Capabilities.Drop, string(cap))
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func convertEnvVariables(cenv []corev1.EnvVar, header bool) *api.EnvironmentVariables {
|
|
env := api.EnvironmentVariables{
|
|
Values: make(map[string]string),
|
|
ValuesFrom: make(map[string]*api.EnvValueFrom),
|
|
}
|
|
for _, kv := range cenv {
|
|
if header {
|
|
if contains(getHeadNodeEnv(), kv.Name) {
|
|
continue
|
|
}
|
|
} else {
|
|
if contains(getWorkNodeEnv(), kv.Name) {
|
|
// Skip reserved names
|
|
continue
|
|
}
|
|
}
|
|
if kv.ValueFrom != nil {
|
|
// this is value from
|
|
if kv.ValueFrom.ConfigMapKeyRef != nil {
|
|
// This is config map
|
|
env.ValuesFrom[kv.Name] = &api.EnvValueFrom{
|
|
Source: api.EnvValueFrom_CONFIGMAP,
|
|
Name: kv.ValueFrom.ConfigMapKeyRef.Name,
|
|
Key: kv.ValueFrom.ConfigMapKeyRef.Key,
|
|
}
|
|
continue
|
|
}
|
|
if kv.ValueFrom.SecretKeyRef != nil {
|
|
// This is Secret
|
|
env.ValuesFrom[kv.Name] = &api.EnvValueFrom{
|
|
Source: api.EnvValueFrom_SECRET,
|
|
Name: kv.ValueFrom.SecretKeyRef.Name,
|
|
Key: kv.ValueFrom.SecretKeyRef.Key,
|
|
}
|
|
continue
|
|
}
|
|
if kv.ValueFrom.ResourceFieldRef != nil {
|
|
// This resource ref
|
|
env.ValuesFrom[kv.Name] = &api.EnvValueFrom{
|
|
Source: api.EnvValueFrom_RESOURCEFIELD,
|
|
Name: kv.ValueFrom.ResourceFieldRef.ContainerName,
|
|
Key: kv.ValueFrom.ResourceFieldRef.Resource,
|
|
}
|
|
continue
|
|
}
|
|
if kv.ValueFrom.FieldRef != nil {
|
|
// This resource ref
|
|
env.ValuesFrom[kv.Name] = &api.EnvValueFrom{
|
|
Source: api.EnvValueFrom_FIELD,
|
|
Key: kv.ValueFrom.FieldRef.FieldPath,
|
|
}
|
|
continue
|
|
}
|
|
} else {
|
|
// This is value
|
|
env.Values[kv.Name] = kv.Value
|
|
}
|
|
}
|
|
return &env
|
|
}
|
|
|
|
func FromKubeToAPIComputeTemplate(configMap *corev1.ConfigMap) *api.ComputeTemplate {
|
|
cpu, _ := strconv.ParseUint(configMap.Data["cpu"], 10, 32)
|
|
memory, _ := strconv.ParseUint(configMap.Data["memory"], 10, 32)
|
|
gpu, _ := strconv.ParseUint(configMap.Data["gpu"], 10, 32)
|
|
|
|
runtime := &api.ComputeTemplate{}
|
|
runtime.Name = configMap.Name
|
|
runtime.Namespace = configMap.Namespace
|
|
runtime.Cpu = uint32(cpu)
|
|
runtime.Memory = uint32(memory)
|
|
runtime.MemoryUnit = configMap.Data["memory_unit"]
|
|
runtime.Gpu = uint32(gpu)
|
|
runtime.GpuAccelerator = configMap.Data["gpu_accelerator"]
|
|
|
|
val, ok := configMap.Data["extended_resources"]
|
|
if ok {
|
|
err := json.Unmarshal(pkgutils.ConvertStringToByteSlice(val), &runtime.ExtendedResources)
|
|
if err != nil {
|
|
klog.Error("failed to unmarshall extended resources for compute template ", runtime.Name, " value ",
|
|
runtime.ExtendedResources, " error ", err)
|
|
}
|
|
}
|
|
|
|
val, ok = configMap.Data["tolerations"]
|
|
if ok {
|
|
err := json.Unmarshal(pkgutils.ConvertStringToByteSlice(val), &runtime.Tolerations)
|
|
if err != nil {
|
|
klog.Error("failed to unmarshall tolerations for compute template ", runtime.Name, " value ",
|
|
runtime.Tolerations, " error ", err)
|
|
}
|
|
}
|
|
return runtime
|
|
}
|
|
|
|
func FromKubeToAPIComputeTemplates(configMaps []*corev1.ConfigMap) []*api.ComputeTemplate {
|
|
apiComputeTemplates := make([]*api.ComputeTemplate, 0, len(configMaps))
|
|
for _, configMap := range configMaps {
|
|
apiComputeTemplates = append(apiComputeTemplates, FromKubeToAPIComputeTemplate(configMap))
|
|
}
|
|
return apiComputeTemplates
|
|
}
|
|
|
|
func FromCrdToAPIJobs(jobs []*rayv1api.RayJob) []*api.RayJob {
|
|
apiJobs := make([]*api.RayJob, 0, len(jobs))
|
|
for _, job := range jobs {
|
|
apiJobs = append(apiJobs, FromCrdToAPIJob(job))
|
|
}
|
|
return apiJobs
|
|
}
|
|
|
|
func FromCrdToAPIJob(job *rayv1api.RayJob) (pbJob *api.RayJob) {
|
|
defer func() {
|
|
err := recover()
|
|
if err != nil {
|
|
klog.Errorf("failed to transfer job crd to job protobuf, err: %v, crd: %+v", err, job)
|
|
}
|
|
}()
|
|
|
|
var version string
|
|
if job.Spec.RayClusterSpec != nil {
|
|
version = job.Spec.RayClusterSpec.RayVersion
|
|
}
|
|
pbJob = &api.RayJob{
|
|
Name: job.Name,
|
|
Namespace: job.Namespace,
|
|
User: job.Labels[util.RayClusterUserLabelKey],
|
|
Version: version,
|
|
Entrypoint: job.Spec.Entrypoint,
|
|
Metadata: job.Spec.Metadata,
|
|
RuntimeEnv: job.Spec.RuntimeEnvYAML,
|
|
JobId: job.Status.JobId,
|
|
ShutdownAfterJobFinishes: job.Spec.ShutdownAfterJobFinishes,
|
|
CreatedAt: ×tamppb.Timestamp{Seconds: job.CreationTimestamp.Unix()},
|
|
JobStatus: string(job.Status.JobStatus),
|
|
JobDeploymentStatus: string(job.Status.JobDeploymentStatus),
|
|
Message: job.Status.Message,
|
|
}
|
|
|
|
// Add optional params
|
|
if job.Spec.ClusterSelector != nil {
|
|
pbJob.ClusterSelector = job.Spec.ClusterSelector
|
|
}
|
|
|
|
if job.Spec.RayClusterSpec != nil {
|
|
pbJob.ClusterSpec = PopulateRayClusterSpec(*job.Spec.RayClusterSpec)
|
|
}
|
|
|
|
pbJob.TtlSecondsAfterFinished = job.Spec.TTLSecondsAfterFinished
|
|
|
|
if job.DeletionTimestamp != nil {
|
|
pbJob.DeleteAt = ×tamppb.Timestamp{Seconds: job.DeletionTimestamp.Unix()}
|
|
}
|
|
|
|
if job.Spec.SubmitterPodTemplate != nil {
|
|
pbJob.JobSubmitter = &api.RayJobSubmitter{
|
|
Image: job.Spec.SubmitterPodTemplate.Spec.Containers[0].Image,
|
|
}
|
|
if cpu := job.Spec.SubmitterPodTemplate.Spec.Containers[0].Resources.Limits.Cpu().String(); cpu != "1" {
|
|
pbJob.JobSubmitter.Cpu = cpu
|
|
}
|
|
if mem := job.Spec.SubmitterPodTemplate.Spec.Containers[0].Resources.Limits.Memory().String(); mem != "1Gi" {
|
|
pbJob.JobSubmitter.Memory = mem
|
|
}
|
|
}
|
|
if jcpus := job.Spec.EntrypointNumCpus; jcpus > 0 {
|
|
pbJob.EntrypointNumCpus = jcpus
|
|
}
|
|
if jgpus := job.Spec.EntrypointNumGpus; jgpus > 0 {
|
|
pbJob.EntrypointNumGpus = jgpus
|
|
}
|
|
if jres := job.Spec.EntrypointResources; jres != "" {
|
|
pbJob.EntrypointResources = jres
|
|
}
|
|
|
|
if jstarttime := job.Status.StartTime; jstarttime != nil {
|
|
pbJob.StartTime = timestamppb.New(job.Status.StartTime.Time)
|
|
}
|
|
if jendtime := job.Status.EndTime; jendtime != nil {
|
|
pbJob.EndTime = timestamppb.New(job.Status.EndTime.Time)
|
|
}
|
|
if jclustername := job.Status.RayClusterName; jclustername != "" {
|
|
pbJob.RayClusterName = jclustername
|
|
}
|
|
|
|
return pbJob
|
|
}
|
|
|
|
func FromCrdToAPIServices(
|
|
services []*rayv1api.RayService,
|
|
serviceEventsMap map[string][]corev1.Event,
|
|
) []*api.RayService {
|
|
apiServices := make([]*api.RayService, 0, len(services))
|
|
for _, service := range services {
|
|
apiServices = append(apiServices, FromCrdToAPIService(service, serviceEventsMap[service.Name]))
|
|
}
|
|
return apiServices
|
|
}
|
|
|
|
func FromCrdToAPIService(service *rayv1api.RayService, events []corev1.Event) *api.RayService {
|
|
defer func() {
|
|
err := recover()
|
|
if err != nil {
|
|
klog.Errorf("failed to transfer ray service, err: %v, item: %v", err, service)
|
|
}
|
|
}()
|
|
|
|
var deleteTime int64 = -1
|
|
if service.DeletionTimestamp != nil {
|
|
deleteTime = service.DeletionTimestamp.Unix()
|
|
}
|
|
pbService := &api.RayService{
|
|
Name: service.Name,
|
|
Namespace: service.Namespace,
|
|
User: service.Labels[util.RayClusterUserLabelKey],
|
|
Version: service.Spec.RayClusterSpec.RayVersion,
|
|
ServeConfig_V2: service.Spec.ServeConfigV2,
|
|
ClusterSpec: PopulateRayClusterSpec(service.Spec.RayClusterSpec),
|
|
ServiceUnhealthySecondThreshold: PoplulateUnhealthySecondThreshold(
|
|
service.Spec.ServiceUnhealthySecondThreshold,
|
|
),
|
|
DeploymentUnhealthySecondThreshold: PoplulateUnhealthySecondThreshold(
|
|
service.Spec.DeploymentUnhealthySecondThreshold,
|
|
),
|
|
RayServiceStatus: PoplulateRayServiceStatus(service.Name, service.Status, events),
|
|
CreatedAt: ×tamppb.Timestamp{Seconds: service.CreationTimestamp.Unix()},
|
|
DeleteAt: ×tamppb.Timestamp{Seconds: deleteTime},
|
|
}
|
|
return pbService
|
|
}
|
|
|
|
func PoplulateUnhealthySecondThreshold(value *int32) int32 {
|
|
if value == nil {
|
|
return 0
|
|
}
|
|
return *value
|
|
}
|
|
|
|
func PoplulateRayServiceStatus(
|
|
serviceName string,
|
|
serviceStatus rayv1api.RayServiceStatuses,
|
|
events []corev1.Event,
|
|
) *api.RayServiceStatus {
|
|
status := &api.RayServiceStatus{
|
|
RayServiceEvents: PopulateRayServiceEvent(serviceName, events),
|
|
RayClusterName: serviceStatus.ActiveServiceStatus.RayClusterName,
|
|
RayClusterState: string(serviceStatus.ActiveServiceStatus.RayClusterStatus.State),
|
|
ServeApplicationStatus: PopulateServeApplicationStatus(serviceStatus.ActiveServiceStatus.Applications),
|
|
}
|
|
status.ServiceEndpoint = map[string]string{}
|
|
maps.Copy(status.ServiceEndpoint, serviceStatus.ActiveServiceStatus.RayClusterStatus.Endpoints)
|
|
return status
|
|
}
|
|
|
|
func PopulateServeApplicationStatus(
|
|
serveApplicationStatuses map[string]rayv1api.AppStatus,
|
|
) []*api.ServeApplicationStatus {
|
|
appStatuses := make([]*api.ServeApplicationStatus, 0, len(serveApplicationStatuses))
|
|
for appName, appStatus := range serveApplicationStatuses {
|
|
ds := &api.ServeApplicationStatus{
|
|
Name: appName,
|
|
Status: appStatus.Status,
|
|
Message: appStatus.Message,
|
|
ServeDeploymentStatus: PopulateServeDeploymentStatus(appStatus.Deployments),
|
|
}
|
|
appStatuses = append(appStatuses, ds)
|
|
}
|
|
return appStatuses
|
|
}
|
|
|
|
func PopulateServeDeploymentStatus(
|
|
serveDeploymentStatuses map[string]rayv1api.ServeDeploymentStatus,
|
|
) []*api.ServeDeploymentStatus {
|
|
deploymentStatuses := make([]*api.ServeDeploymentStatus, 0, len(serveDeploymentStatuses))
|
|
for deploymentName, deploymentStatus := range serveDeploymentStatuses {
|
|
ds := &api.ServeDeploymentStatus{
|
|
DeploymentName: deploymentName,
|
|
Status: deploymentStatus.Status,
|
|
Message: deploymentStatus.Message,
|
|
}
|
|
deploymentStatuses = append(deploymentStatuses, ds)
|
|
}
|
|
return deploymentStatuses
|
|
}
|
|
|
|
func PopulateRayServiceEvent(serviceName string, events []corev1.Event) []*api.RayServiceEvent {
|
|
serviceEvents := make([]*api.RayServiceEvent, 0, len(events))
|
|
for _, event := range events {
|
|
serviceEvent := &api.RayServiceEvent{
|
|
Id: event.Name,
|
|
Name: fmt.Sprintf("%s-%s", serviceName, event.Name),
|
|
CreatedAt: ×tamppb.Timestamp{Seconds: event.ObjectMeta.CreationTimestamp.Unix()},
|
|
FirstTimestamp: ×tamppb.Timestamp{Seconds: event.FirstTimestamp.Unix()},
|
|
LastTimestamp: ×tamppb.Timestamp{Seconds: event.LastTimestamp.Unix()},
|
|
Reason: event.Reason,
|
|
Message: event.Message,
|
|
Type: event.Type,
|
|
Count: event.Count,
|
|
}
|
|
serviceEvents = append(serviceEvents, serviceEvent)
|
|
}
|
|
return serviceEvents
|
|
}
|