mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
456 lines
13 KiB
Go
456 lines
13 KiB
Go
package utils
|
|
|
|
import (
|
|
"crypto/sha1"
|
|
"encoding/base32"
|
|
"fmt"
|
|
"math"
|
|
"os"
|
|
"reflect"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
|
|
"k8s.io/apimachinery/pkg/util/json"
|
|
|
|
"k8s.io/apimachinery/pkg/util/rand"
|
|
|
|
"github.com/sirupsen/logrus"
|
|
|
|
rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
)
|
|
|
|
const (
|
|
RayClusterSuffix = "-raycluster-"
|
|
DashboardName = "dashboard"
|
|
ServeName = "serve"
|
|
ClusterDomainEnvKey = "CLUSTER_DOMAIN"
|
|
DefaultDomainName = "cluster.local"
|
|
)
|
|
|
|
// GetClusterDomainName returns cluster's domain name
|
|
func GetClusterDomainName() string {
|
|
if domain := os.Getenv(ClusterDomainEnvKey); len(domain) > 0 {
|
|
return domain
|
|
}
|
|
|
|
// Return default domain name.
|
|
return DefaultDomainName
|
|
}
|
|
|
|
// IsCreated returns true if pod has been created and is maintained by the API server
|
|
func IsCreated(pod *corev1.Pod) bool {
|
|
return pod.Status.Phase != ""
|
|
}
|
|
|
|
// IsRunningAndReady returns true if pod is in the PodRunning Phase, if it has a condition of PodReady.
|
|
func IsRunningAndReady(pod *corev1.Pod) bool {
|
|
if pod.Status.Phase != corev1.PodRunning {
|
|
return false
|
|
}
|
|
for _, cond := range pod.Status.Conditions {
|
|
if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// CheckName makes sure the name does not start with a numeric value and the total length is < 63 char
|
|
func CheckName(s string) string {
|
|
maxLength := 50 // 63 - (max(8,6) + 5 ) // 6 to 8 char are consumed at the end with "-head-" or -worker- + 5 generated.
|
|
|
|
if len(s) > maxLength {
|
|
// shorten the name
|
|
offset := int(math.Abs(float64(maxLength) - float64(len(s))))
|
|
fmt.Printf("pod name is too long: len = %v, we will shorten it by offset = %v\n", len(s), offset)
|
|
s = s[offset:]
|
|
}
|
|
|
|
// cannot start with a numeric value
|
|
if unicode.IsDigit(rune(s[0])) {
|
|
s = "r" + s[1:]
|
|
}
|
|
|
|
// cannot start with a punctuation
|
|
if unicode.IsPunct(rune(s[0])) {
|
|
fmt.Println(s)
|
|
s = "r" + s[1:]
|
|
}
|
|
|
|
return s
|
|
}
|
|
|
|
// CheckLabel makes sure the label value does not start with a punctuation and the total length is < 63 char
|
|
func CheckLabel(s string) string {
|
|
maxLenght := 63
|
|
|
|
if len(s) > maxLenght {
|
|
// shorten the name
|
|
offset := int(math.Abs(float64(maxLenght) - float64(len(s))))
|
|
fmt.Printf("label value is too long: len = %v, we will shorten it by offset = %v\n", len(s), offset)
|
|
s = s[offset:]
|
|
}
|
|
|
|
// cannot start with a punctuation
|
|
if unicode.IsPunct(rune(s[0])) {
|
|
fmt.Println(s)
|
|
s = "r" + s[1:]
|
|
}
|
|
|
|
return s
|
|
}
|
|
|
|
// Before Get substring before a string.
|
|
func Before(value string, a string) string {
|
|
pos := strings.Index(value, a)
|
|
if pos == -1 {
|
|
return ""
|
|
}
|
|
return value[0:pos]
|
|
}
|
|
|
|
// FormatInt returns the string representation of i in the given base,
|
|
// for 2 <= base <= 36. The result uses the lower-case letters 'a' to 'z'
|
|
// for digit values >= 10.
|
|
func FormatInt32(n int32) string {
|
|
return strconv.FormatInt(int64(n), 10)
|
|
}
|
|
|
|
// GetNamespace return namespace
|
|
func GetNamespace(metaData metav1.ObjectMeta) string {
|
|
if metaData.Namespace == "" {
|
|
return "default"
|
|
}
|
|
return metaData.Namespace
|
|
}
|
|
|
|
// GenerateServiceName generates a ray head service name from cluster name
|
|
func GenerateServiceName(clusterName string) string {
|
|
return CheckName(fmt.Sprintf("%s-%s-%s", clusterName, rayiov1alpha1.HeadNode, "svc"))
|
|
}
|
|
|
|
// GenerateFQDNServiceName generates a Fully Qualified Domain Name.
|
|
func GenerateFQDNServiceName(clusterName string, namespace string) string {
|
|
return fmt.Sprintf("%s.%s.svc.%s", GenerateServiceName(clusterName), namespace, GetClusterDomainName())
|
|
}
|
|
|
|
// ExtractRayIPFromFQDN extracts the head service name (i.e., RAY_IP, deprecated) from a fully qualified
|
|
// domain name (FQDN). This function is provided for backward compatibility purposes only.
|
|
func ExtractRayIPFromFQDN(fqdnRayIP string) string {
|
|
return strings.Split(fqdnRayIP, ".")[0]
|
|
}
|
|
|
|
// GenerateDashboardServiceName generates a ray head service name from cluster name
|
|
func GenerateDashboardServiceName(clusterName string) string {
|
|
return fmt.Sprintf("%s-%s-%s", clusterName, DashboardName, "svc")
|
|
}
|
|
|
|
// GenerateDashboardAgentLabel generates label value for agent service selector.
|
|
func GenerateDashboardAgentLabel(clusterName string) string {
|
|
return fmt.Sprintf("%s-%s", clusterName, DashboardName)
|
|
}
|
|
|
|
// GenerateServeServiceName generates name for serve service.
|
|
func GenerateServeServiceName(serviceName string) string {
|
|
return fmt.Sprintf("%s-%s-%s", serviceName, ServeName, "svc")
|
|
}
|
|
|
|
// GenerateServeServiceLabel generates label value for serve service selector.
|
|
func GenerateServeServiceLabel(serviceName string) string {
|
|
return fmt.Sprintf("%s-%s", serviceName, ServeName)
|
|
}
|
|
|
|
// GenerateIngressName generates an ingress name from cluster name
|
|
func GenerateIngressName(clusterName string) string {
|
|
return fmt.Sprintf("%s-%s-%s", clusterName, rayiov1alpha1.HeadNode, "ingress")
|
|
}
|
|
|
|
// GenerateRayClusterName generates a ray cluster name from ray service name
|
|
func GenerateRayClusterName(serviceName string) string {
|
|
return fmt.Sprintf("%s%s%s", serviceName, RayClusterSuffix, rand.String(5))
|
|
}
|
|
|
|
// GenerateRayJobId generates a ray job id for submission
|
|
func GenerateRayJobId(rayjob string) string {
|
|
return fmt.Sprintf("%s-%s", rayjob, rand.String(5))
|
|
}
|
|
|
|
// GenerateIdentifier generates identifier of same group pods
|
|
func GenerateIdentifier(clusterName string, nodeType rayiov1alpha1.RayNodeType) string {
|
|
return fmt.Sprintf("%s-%s", clusterName, nodeType)
|
|
}
|
|
|
|
// TODO: find target container through name instead of using index 0.
|
|
// FindRayContainerIndex finds the ray head/worker container's index in the pod
|
|
func FindRayContainerIndex(spec corev1.PodSpec) (index int) {
|
|
// We only support one container at this moment. We definitely need a better way to filter out sidecar containers.
|
|
if len(spec.Containers) > 1 {
|
|
logrus.Warnf("Pod has multiple containers, we choose index=0 as Ray container")
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// CalculateDesiredReplicas calculate desired worker replicas at the cluster level
|
|
func CalculateDesiredReplicas(cluster *rayiov1alpha1.RayCluster) int32 {
|
|
count := int32(0)
|
|
for _, nodeGroup := range cluster.Spec.WorkerGroupSpecs {
|
|
count += *nodeGroup.Replicas
|
|
}
|
|
|
|
return count
|
|
}
|
|
|
|
// CalculateMinReplicas calculates min worker replicas at the cluster level
|
|
func CalculateMinReplicas(cluster *rayiov1alpha1.RayCluster) int32 {
|
|
count := int32(0)
|
|
for _, nodeGroup := range cluster.Spec.WorkerGroupSpecs {
|
|
count += *nodeGroup.MinReplicas
|
|
}
|
|
|
|
return count
|
|
}
|
|
|
|
// CalculateMaxReplicas calculates max worker replicas at the cluster level
|
|
func CalculateMaxReplicas(cluster *rayiov1alpha1.RayCluster) int32 {
|
|
count := int32(0)
|
|
for _, nodeGroup := range cluster.Spec.WorkerGroupSpecs {
|
|
count += *nodeGroup.MaxReplicas
|
|
}
|
|
|
|
return count
|
|
}
|
|
|
|
// CalculateAvailableReplicas calculates available worker replicas at the cluster level
|
|
// A worker is available if its Pod is running
|
|
func CalculateAvailableReplicas(pods corev1.PodList) int32 {
|
|
count := int32(0)
|
|
for _, pod := range pods.Items {
|
|
if val, ok := pod.Labels["ray.io/node-type"]; !ok || val != string(rayiov1alpha1.WorkerNode) {
|
|
continue
|
|
}
|
|
if pod.Status.Phase == corev1.PodRunning {
|
|
count++
|
|
}
|
|
}
|
|
|
|
return count
|
|
}
|
|
|
|
func CalculateDesiredResources(cluster *rayiov1alpha1.RayCluster) corev1.ResourceList {
|
|
desiredResourcesList := []corev1.ResourceList{{}}
|
|
headPodResource := calculatePodResource(cluster.Spec.HeadGroupSpec.Template.Spec)
|
|
for i := int32(0); i < *cluster.Spec.HeadGroupSpec.Replicas; i++ {
|
|
desiredResourcesList = append(desiredResourcesList, headPodResource)
|
|
}
|
|
for _, nodeGroup := range cluster.Spec.WorkerGroupSpecs {
|
|
podResource := calculatePodResource(nodeGroup.Template.Spec)
|
|
for i := int32(0); i < *nodeGroup.Replicas; i++ {
|
|
desiredResourcesList = append(desiredResourcesList, podResource)
|
|
}
|
|
}
|
|
return sumResourceList(desiredResourcesList)
|
|
}
|
|
|
|
func CalculateMinResources(cluster *rayiov1alpha1.RayCluster) corev1.ResourceList {
|
|
minResourcesList := []corev1.ResourceList{{}}
|
|
headPodResource := calculatePodResource(cluster.Spec.HeadGroupSpec.Template.Spec)
|
|
for i := int32(0); i < *cluster.Spec.HeadGroupSpec.Replicas; i++ {
|
|
minResourcesList = append(minResourcesList, headPodResource)
|
|
}
|
|
for _, nodeGroup := range cluster.Spec.WorkerGroupSpecs {
|
|
podResource := calculatePodResource(nodeGroup.Template.Spec)
|
|
for i := int32(0); i < *nodeGroup.MinReplicas; i++ {
|
|
minResourcesList = append(minResourcesList, podResource)
|
|
}
|
|
}
|
|
return sumResourceList(minResourcesList)
|
|
}
|
|
|
|
func calculatePodResource(podSpec corev1.PodSpec) corev1.ResourceList {
|
|
podResource := corev1.ResourceList{}
|
|
for _, container := range podSpec.Containers {
|
|
containerResource := container.Resources.Requests
|
|
for name, quantity := range container.Resources.Limits {
|
|
if _, ok := containerResource[name]; !ok {
|
|
containerResource[name] = quantity
|
|
}
|
|
}
|
|
for name, quantity := range containerResource {
|
|
if totalQuantity, ok := podResource[name]; ok {
|
|
totalQuantity.Add(quantity)
|
|
podResource[name] = totalQuantity
|
|
} else {
|
|
podResource[name] = quantity
|
|
}
|
|
}
|
|
}
|
|
return podResource
|
|
}
|
|
|
|
func sumResourceList(list []corev1.ResourceList) corev1.ResourceList {
|
|
totalResource := corev1.ResourceList{}
|
|
for _, l := range list {
|
|
for name, quantity := range l {
|
|
if value, ok := totalResource[name]; !ok {
|
|
totalResource[name] = quantity.DeepCopy()
|
|
} else {
|
|
value.Add(quantity)
|
|
totalResource[name] = value
|
|
}
|
|
}
|
|
}
|
|
return totalResource
|
|
}
|
|
|
|
func Contains(elems []string, searchTerm string) bool {
|
|
for _, s := range elems {
|
|
if searchTerm == s {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func FilterContainerByName(containers []corev1.Container, name string) (corev1.Container, error) {
|
|
for _, container := range containers {
|
|
if strings.Compare(container.Name, name) == 0 {
|
|
return container, nil
|
|
}
|
|
}
|
|
|
|
return corev1.Container{}, fmt.Errorf("can not find container %s", name)
|
|
}
|
|
|
|
// GetHeadGroupServiceAccountName returns the head group service account if it exists.
|
|
// Otherwise, it returns the name of the cluster itself.
|
|
func GetHeadGroupServiceAccountName(cluster *rayiov1alpha1.RayCluster) string {
|
|
headGroupServiceAccountName := cluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName
|
|
if headGroupServiceAccountName != "" {
|
|
return headGroupServiceAccountName
|
|
}
|
|
return cluster.Name
|
|
}
|
|
|
|
// CheckAllPodsRunnning check if all pod in a list is running
|
|
func CheckAllPodsRunnning(runningPods corev1.PodList) bool {
|
|
// check if there is no pods.
|
|
if len(runningPods.Items) == 0 {
|
|
return false
|
|
}
|
|
for _, pod := range runningPods.Items {
|
|
if pod.Status.Phase != corev1.PodRunning {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func PodNotMatchingTemplate(pod corev1.Pod, template corev1.PodTemplateSpec) bool {
|
|
if pod.Status.Phase == corev1.PodRunning && pod.ObjectMeta.DeletionTimestamp == nil {
|
|
if len(template.Spec.Containers) != len(pod.Spec.Containers) {
|
|
return true
|
|
}
|
|
cmap := map[string]*corev1.Container{}
|
|
for _, container := range pod.Spec.Containers {
|
|
cmap[container.Name] = &container
|
|
}
|
|
for _, container1 := range template.Spec.Containers {
|
|
if container2, ok := cmap[container1.Name]; ok {
|
|
if container1.Image != container2.Image {
|
|
// image name do not match
|
|
return true
|
|
}
|
|
if len(container1.Resources.Requests) != len(container2.Resources.Requests) ||
|
|
len(container1.Resources.Limits) != len(container2.Resources.Limits) {
|
|
// resource entries do not match
|
|
return true
|
|
}
|
|
|
|
resources1 := []corev1.ResourceList{
|
|
container1.Resources.Requests,
|
|
container1.Resources.Limits,
|
|
}
|
|
resources2 := []corev1.ResourceList{
|
|
container2.Resources.Requests,
|
|
container2.Resources.Limits,
|
|
}
|
|
for i := range resources1 {
|
|
// we need to make sure all fields match
|
|
for name, quantity1 := range resources1[i] {
|
|
if quantity2, ok := resources2[i][name]; ok {
|
|
if quantity1.Cmp(quantity2) != 0 {
|
|
// request amount does not match
|
|
return true
|
|
}
|
|
} else {
|
|
// no such request
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
|
|
// now we consider them equal
|
|
delete(cmap, container1.Name)
|
|
} else {
|
|
// container name do not match
|
|
return true
|
|
}
|
|
}
|
|
if len(cmap) != 0 {
|
|
// one or more containers do not match
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// CompareJsonStruct This is a way to better compare if two objects are the same when they are json/yaml structs. reflect.DeepEqual will fail in some cases.
|
|
func CompareJsonStruct(objA interface{}, objB interface{}) bool {
|
|
a, err := json.Marshal(objA)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
b, err := json.Marshal(objB)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
var v1, v2 interface{}
|
|
err = json.Unmarshal(a, &v1)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
err = json.Unmarshal(b, &v2)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return reflect.DeepEqual(v1, v2)
|
|
}
|
|
|
|
func ConvertUnixTimeToMetav1Time(unixTime int64) *metav1.Time {
|
|
// The Ray jobInfo returns the start_time, which is a unix timestamp in milliseconds.
|
|
// https://docs.ray.io/en/latest/cluster/jobs-package-ref.html#jobinfo
|
|
t := time.Unix(unixTime/1000, unixTime%1000*1000000)
|
|
kt := metav1.NewTime(t)
|
|
return &kt
|
|
}
|
|
|
|
// Json-serializes obj and returns its hash string
|
|
func GenerateJsonHash(obj interface{}) (string, error) {
|
|
serialObj, err := json.Marshal(obj)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
hashBytes := sha1.Sum(serialObj)
|
|
|
|
// Convert to an ASCII string
|
|
hashStr := string(base32.HexEncoding.EncodeToString(hashBytes[:]))
|
|
|
|
return hashStr, nil
|
|
}
|