mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
* Updating client-go to v1.36.0-beta.0 (and updating golang to 1.26) Signed-off-by: Mark Rossetti <[email protected]> * Bump golangci-lint to v2.11.4 and fix lint issues for Go 1.26 Signed-off-by: Mark Rossetti <[email protected]> * bump go to 1.26 in ray-operator docker file Signed-off-by: Mark Rossetti <[email protected]> * Migrate webhooks from deprecated CustomValidator to admission.Validation[T] Signed-off-by: Mark Rossetti <[email protected]> * Target K8s 1.36 GA deps Signed-off-by: Mark Rossetti <[email protected]> * Merge branch 'master' into go-client-go-updates Signed-off-by: Rueian <[email protected]> --------- Signed-off-by: Mark Rossetti <[email protected]> Signed-off-by: Rueian <[email protected]> Co-authored-by: Rueian <[email protected]>
1007 lines
30 KiB
Go
1007 lines
30 KiB
Go
package util
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"maps"
|
|
"net"
|
|
"strconv"
|
|
"strings"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
|
|
api "github.com/ray-project/kuberay/proto/go_client"
|
|
rayv1api "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
|
|
)
|
|
|
|
type RayCluster struct {
|
|
*rayv1api.RayCluster
|
|
}
|
|
|
|
// NewRayCluster creates a RayCluster.
|
|
// func NewRayCluster(apiCluster *api.Cluster, clusterRuntime *api.ClusterRuntime, computeRuntime *api.ComputeRuntime) *RayCluster {
|
|
func NewRayCluster(apiCluster *api.Cluster, computeTemplateMap map[string]*api.ComputeTemplate) (*RayCluster, error) {
|
|
// Check for "ray.io/enable-serve-service=true"
|
|
enableServeService := false
|
|
if enableServeServiceValue, exist := apiCluster.Annotations["ray.io/enable-serve-service"]; exist && enableServeServiceValue == "true" {
|
|
enableServeService = true
|
|
}
|
|
|
|
// Build cluster spec
|
|
spec, err := buildRayClusterSpec(apiCluster.Version, apiCluster.Envs, apiCluster.ClusterSpec, computeTemplateMap, enableServeService)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// Build cluster
|
|
rayCluster := &rayv1api.RayCluster{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: apiCluster.Name,
|
|
Namespace: apiCluster.Namespace,
|
|
Labels: buildRayClusterLabels(apiCluster),
|
|
Annotations: buildRayClusterAnnotations(apiCluster),
|
|
},
|
|
Spec: *spec,
|
|
}
|
|
|
|
return &RayCluster{rayCluster}, nil
|
|
}
|
|
|
|
// Build cluster labels
|
|
func buildRayClusterLabels(cluster *api.Cluster) map[string]string {
|
|
labels := map[string]string{}
|
|
labels[RayClusterNameLabelKey] = cluster.Name
|
|
labels[RayClusterUserLabelKey] = cluster.User
|
|
labels[RayClusterVersionLabelKey] = cluster.Version
|
|
labels[RayClusterEnvironmentLabelKey] = cluster.Environment.String()
|
|
labels[KubernetesApplicationNameLabelKey] = ApplicationName
|
|
labels[KubernetesManagedByLabelKey] = ComponentName
|
|
return labels
|
|
}
|
|
|
|
// Build cluster annotations
|
|
func buildRayClusterAnnotations(cluster *api.Cluster) map[string]string {
|
|
if cluster.Annotations == nil {
|
|
return map[string]string{}
|
|
}
|
|
return cluster.Annotations
|
|
}
|
|
|
|
// TODO(Basasuya & MissionToMars): The job spec depends on ClusterSpec which not all cluster-related configs are included,
|
|
// such as `metadata` and `envs`. We just put `imageVersion` and `envs` in the arguments list, and should be refactored later.
|
|
func buildRayClusterSpec(imageVersion string, envs *api.EnvironmentVariables, clusterSpec *api.ClusterSpec, computeTemplateMap map[string]*api.ComputeTemplate, enableServeService bool) (*rayv1api.RayClusterSpec, error) {
|
|
computeTemplate := computeTemplateMap[clusterSpec.HeadGroupSpec.ComputeTemplate]
|
|
headPodTemplate, err := buildHeadPodTemplate(imageVersion, envs, clusterSpec.HeadGroupSpec, computeTemplate, enableServeService)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rayClusterSpec := &rayv1api.RayClusterSpec{
|
|
RayVersion: imageVersion,
|
|
HeadGroupSpec: rayv1api.HeadGroupSpec{
|
|
ServiceType: corev1.ServiceType(clusterSpec.HeadGroupSpec.ServiceType),
|
|
Template: *headPodTemplate,
|
|
RayStartParams: clusterSpec.HeadGroupSpec.RayStartParams,
|
|
},
|
|
WorkerGroupSpecs: []rayv1api.WorkerGroupSpec{},
|
|
HeadServiceAnnotations: clusterSpec.HeadServiceAnnotations,
|
|
}
|
|
|
|
// If enable ingress is specified, add it to the head node spec.
|
|
if clusterSpec.HeadGroupSpec.EnableIngress {
|
|
rayClusterSpec.HeadGroupSpec.EnableIngress = &clusterSpec.HeadGroupSpec.EnableIngress
|
|
}
|
|
|
|
// Build worker groups
|
|
for _, spec := range clusterSpec.WorkerGroupSpec {
|
|
computeTemplate = computeTemplateMap[spec.ComputeTemplate]
|
|
workerPodTemplate, err := buildWorkerPodTemplate(imageVersion, envs, spec, computeTemplate)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
minReplicas := spec.Replicas
|
|
maxReplicas := spec.Replicas
|
|
if spec.MinReplicas != 0 {
|
|
minReplicas = spec.MinReplicas
|
|
}
|
|
if spec.MaxReplicas != 0 {
|
|
maxReplicas = spec.MaxReplicas
|
|
}
|
|
|
|
workerNodeSpec := rayv1api.WorkerGroupSpec{
|
|
GroupName: spec.GroupName,
|
|
MinReplicas: new(minReplicas),
|
|
MaxReplicas: new(maxReplicas),
|
|
Replicas: new(spec.Replicas),
|
|
RayStartParams: spec.RayStartParams,
|
|
Template: *workerPodTemplate,
|
|
}
|
|
|
|
rayClusterSpec.WorkerGroupSpecs = append(rayClusterSpec.WorkerGroupSpecs, workerNodeSpec)
|
|
}
|
|
|
|
if clusterSpec.EnableInTreeAutoscaling {
|
|
// This is a cluster with auto scaler
|
|
rayClusterSpec.EnableInTreeAutoscaling = &clusterSpec.EnableInTreeAutoscaling
|
|
options, err := buildAutoscalerOptions(clusterSpec.AutoscalerOptions)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if options != nil {
|
|
rayClusterSpec.AutoscalerOptions = options
|
|
}
|
|
}
|
|
|
|
return rayClusterSpec, nil
|
|
}
|
|
|
|
// Annotations common to both head and worker nodes
|
|
func buildNodeGroupAnnotations(computeTemplate *api.ComputeTemplate, image string) map[string]string {
|
|
annotations := map[string]string{}
|
|
annotations[RayClusterComputeTemplateAnnotationKey] = computeTemplate.Name
|
|
annotations[RayClusterImageAnnotationKey] = image
|
|
return annotations
|
|
}
|
|
|
|
// Add resource to container
|
|
func addResourceToContainer(container *corev1.Container, resourceName string, quantity uint32) {
|
|
if quantity == 0 {
|
|
return
|
|
}
|
|
quantityStr := fmt.Sprint(quantity)
|
|
resourceQuantity := resource.MustParse(quantityStr)
|
|
|
|
if container.Resources.Requests == nil {
|
|
container.Resources.Requests = make(corev1.ResourceList)
|
|
}
|
|
if container.Resources.Limits == nil {
|
|
container.Resources.Limits = make(corev1.ResourceList)
|
|
}
|
|
|
|
container.Resources.Requests[corev1.ResourceName(resourceName)] = resourceQuantity
|
|
container.Resources.Limits[corev1.ResourceName(resourceName)] = resourceQuantity
|
|
}
|
|
|
|
// Build head node template
|
|
func buildHeadPodTemplate(imageVersion string, envs *api.EnvironmentVariables, spec *api.HeadGroupSpec, computeRuntime *api.ComputeTemplate, enableServeService bool) (*corev1.PodTemplateSpec, error) {
|
|
image := constructRayImage(RayClusterDefaultImageRepository, imageVersion)
|
|
if len(spec.Image) != 0 {
|
|
image = spec.Image
|
|
}
|
|
|
|
// Image pull policy. Kubernetes default image pull policy IfNotPresent, so we here only
|
|
// Overwrite it if it is Always
|
|
imagePullPolicy := corev1.PullIfNotPresent
|
|
if len(spec.ImagePullPolicy) > 0 && strings.ToLower(spec.ImagePullPolicy) == "always" {
|
|
imagePullPolicy = corev1.PullAlways
|
|
}
|
|
|
|
// calculate resources
|
|
cpu := fmt.Sprint(computeRuntime.GetCpu())
|
|
memoryUnit := computeRuntime.GetMemoryUnit()
|
|
memory := fmt.Sprintf("%d%s", computeRuntime.GetMemory(), memoryUnit)
|
|
|
|
// build volume and volumeMounts
|
|
volMounts := buildVolumeMounts(spec.Volumes)
|
|
vols, err := buildVols(spec.Volumes)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
podTemplateSpec := corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Annotations: buildNodeGroupAnnotations(computeRuntime, spec.Image),
|
|
Labels: map[string]string{},
|
|
},
|
|
Spec: corev1.PodSpec{
|
|
Tolerations: []corev1.Toleration{},
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "ray-head",
|
|
Image: image,
|
|
ImagePullPolicy: imagePullPolicy,
|
|
Env: []corev1.EnvVar{
|
|
{
|
|
Name: "MY_POD_IP",
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
FieldRef: &corev1.ObjectFieldSelector{
|
|
FieldPath: "status.podIP",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
// Customization is not allowed here. We should consider whether to make this part smart.
|
|
// For now we use serve 8000 port for rayservice and added at util/service.go, do not use the 8000 port here for other purpose.
|
|
Ports: []corev1.ContainerPort{
|
|
{
|
|
Name: "redis",
|
|
ContainerPort: 6379,
|
|
},
|
|
{
|
|
Name: "head",
|
|
ContainerPort: 10001,
|
|
},
|
|
{
|
|
Name: "dashboard",
|
|
ContainerPort: 8265,
|
|
},
|
|
{
|
|
Name: "metrics",
|
|
ContainerPort: 8080,
|
|
},
|
|
},
|
|
Resources: corev1.ResourceRequirements{
|
|
Limits: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse(cpu),
|
|
corev1.ResourceMemory: resource.MustParse(memory),
|
|
},
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse(cpu),
|
|
corev1.ResourceMemory: resource.MustParse(memory),
|
|
},
|
|
},
|
|
VolumeMounts: volMounts,
|
|
SecurityContext: buildSecurityContext(spec.SecurityContext),
|
|
},
|
|
},
|
|
Volumes: vols,
|
|
},
|
|
}
|
|
|
|
// We are filtering container by name `ray-head`. If container with this name does not exist
|
|
// (should never happen) we are not adding container specific parameters
|
|
if container, index, ok := GetContainerByName(podTemplateSpec.Spec.Containers, "ray-head"); ok {
|
|
if gpu := computeRuntime.GetGpu(); gpu != 0 {
|
|
accelerator := "nvidia.com/gpu"
|
|
if len(computeRuntime.GetGpuAccelerator()) != 0 {
|
|
accelerator = computeRuntime.GetGpuAccelerator()
|
|
}
|
|
addResourceToContainer(&container, accelerator, gpu)
|
|
}
|
|
|
|
for k, v := range computeRuntime.GetExtendedResources() {
|
|
addResourceToContainer(&container, k, v)
|
|
}
|
|
|
|
globalEnv := convertEnvironmentVariables(envs)
|
|
if len(globalEnv) > 0 {
|
|
container.Env = append(container.Env, globalEnv...)
|
|
}
|
|
|
|
// Add specific environments
|
|
specEnv := convertEnvironmentVariables(spec.Environment)
|
|
if len(specEnv) > 0 {
|
|
container.Env = append(container.Env, specEnv...)
|
|
}
|
|
|
|
// If enableServeService add port
|
|
if enableServeService {
|
|
container.Ports = append(container.Ports, corev1.ContainerPort{Name: "dashboard-agent", ContainerPort: 52365})
|
|
container.Ports = append(container.Ports, corev1.ContainerPort{Name: "serve", ContainerPort: 8000})
|
|
}
|
|
|
|
// Replace container
|
|
podTemplateSpec.Spec.Containers[index] = container
|
|
}
|
|
|
|
// Add specific annotations
|
|
if spec.Annotations != nil {
|
|
maps.Copy(podTemplateSpec.ObjectMeta.Annotations, spec.Annotations)
|
|
}
|
|
|
|
// Add specific labels
|
|
if spec.Labels != nil {
|
|
maps.Copy(podTemplateSpec.ObjectMeta.Labels, spec.Labels)
|
|
}
|
|
|
|
// Add specific tollerations
|
|
if computeRuntime.Tolerations != nil {
|
|
for _, t := range computeRuntime.Tolerations {
|
|
podTemplateSpec.Spec.Tolerations = append(podTemplateSpec.Spec.Tolerations, corev1.Toleration{
|
|
Key: t.Key, Operator: convertTolerationOperator(t.Operator), Value: t.Value, Effect: convertTaintEffect(t.Effect),
|
|
})
|
|
}
|
|
}
|
|
|
|
// If service account is specified, add it to the pod spec.
|
|
if len(spec.ServiceAccount) > 1 {
|
|
podTemplateSpec.Spec.ServiceAccountName = spec.ServiceAccount
|
|
}
|
|
|
|
// If image pull secret is specified, add it to the pod spec.
|
|
if len(spec.ImagePullSecret) > 1 {
|
|
podTemplateSpec.Spec.ImagePullSecrets = []corev1.LocalObjectReference{
|
|
{
|
|
Name: spec.ImagePullSecret,
|
|
},
|
|
}
|
|
}
|
|
|
|
return &podTemplateSpec, nil
|
|
}
|
|
|
|
// Convert environment variables
|
|
func convertEnvironmentVariables(envs *api.EnvironmentVariables) []corev1.EnvVar {
|
|
converted := []corev1.EnvVar{}
|
|
if envs == nil {
|
|
return converted
|
|
}
|
|
if len(envs.Values) > 0 {
|
|
// Add values
|
|
for key, value := range envs.Values {
|
|
converted = append(converted, corev1.EnvVar{
|
|
Name: key, Value: value,
|
|
})
|
|
}
|
|
}
|
|
if len(envs.ValuesFrom) > 0 {
|
|
// Add values ref
|
|
for key, value := range envs.ValuesFrom {
|
|
switch value.Source {
|
|
case api.EnvValueFrom_CONFIGMAP:
|
|
converted = append(converted, corev1.EnvVar{
|
|
Name: key,
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
|
|
LocalObjectReference: corev1.LocalObjectReference{
|
|
Name: value.Name,
|
|
},
|
|
Key: value.Key,
|
|
},
|
|
},
|
|
})
|
|
case api.EnvValueFrom_SECRET:
|
|
converted = append(converted, corev1.EnvVar{
|
|
Name: key,
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
SecretKeyRef: &corev1.SecretKeySelector{
|
|
LocalObjectReference: corev1.LocalObjectReference{
|
|
Name: value.Name,
|
|
},
|
|
Key: value.Key,
|
|
},
|
|
},
|
|
})
|
|
case api.EnvValueFrom_RESOURCEFIELD:
|
|
converted = append(converted, corev1.EnvVar{
|
|
Name: key,
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
ResourceFieldRef: &corev1.ResourceFieldSelector{
|
|
ContainerName: value.Name,
|
|
Resource: value.Key,
|
|
},
|
|
},
|
|
})
|
|
default:
|
|
converted = append(converted, corev1.EnvVar{
|
|
Name: key,
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
FieldRef: &corev1.ObjectFieldSelector{
|
|
FieldPath: value.Key,
|
|
},
|
|
},
|
|
})
|
|
}
|
|
}
|
|
}
|
|
return converted
|
|
}
|
|
|
|
// Convert Toleration operator from string
|
|
func convertTolerationOperator(val string) corev1.TolerationOperator {
|
|
if val == "Exists" {
|
|
return corev1.TolerationOpExists
|
|
}
|
|
return corev1.TolerationOpEqual
|
|
}
|
|
|
|
// Convert taint effect from string
|
|
func convertTaintEffect(val string) corev1.TaintEffect {
|
|
if val == "NoExecute" {
|
|
return corev1.TaintEffectNoExecute
|
|
}
|
|
if val == "NoSchedule" {
|
|
return corev1.TaintEffectNoSchedule
|
|
}
|
|
return corev1.TaintEffectPreferNoSchedule
|
|
}
|
|
|
|
// Construct Ray image
|
|
func constructRayImage(containerImage string, version string) string {
|
|
return fmt.Sprintf("%s:%s", containerImage, version)
|
|
}
|
|
|
|
// Build worker pod template
|
|
func buildWorkerPodTemplate(imageVersion string, envs *api.EnvironmentVariables, spec *api.WorkerGroupSpec, computeRuntime *api.ComputeTemplate) (*corev1.PodTemplateSpec, error) {
|
|
// If user doesn't provide the image, let's use the default image instead.
|
|
// TODO: verify the versions in the range
|
|
image := constructRayImage(RayClusterDefaultImageRepository, imageVersion)
|
|
if len(spec.Image) != 0 {
|
|
image = spec.Image
|
|
}
|
|
|
|
// Image pull policy. Kubernetes default image pull policy IfNotPresent, so we here only
|
|
// Overwrite it if it is Always
|
|
imagePullPolicy := corev1.PullIfNotPresent
|
|
if len(spec.ImagePullPolicy) > 0 && strings.ToLower(spec.ImagePullPolicy) == "always" {
|
|
imagePullPolicy = corev1.PullAlways
|
|
}
|
|
|
|
// calculate resources
|
|
cpu := fmt.Sprint(computeRuntime.GetCpu())
|
|
memoryUnit := computeRuntime.GetMemoryUnit()
|
|
memory := fmt.Sprintf("%d%s", computeRuntime.GetMemory(), memoryUnit)
|
|
|
|
// build volume and volumeMounts
|
|
volMounts := buildVolumeMounts(spec.Volumes)
|
|
vols, err := buildVols(spec.Volumes)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
podTemplateSpec := corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Annotations: buildNodeGroupAnnotations(computeRuntime, spec.Image),
|
|
Labels: map[string]string{},
|
|
},
|
|
Spec: corev1.PodSpec{
|
|
Tolerations: []corev1.Toleration{},
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "ray-worker",
|
|
Image: image,
|
|
ImagePullPolicy: imagePullPolicy,
|
|
Env: []corev1.EnvVar{
|
|
{
|
|
Name: "RAY_DISABLE_DOCKER_CPU_WARNING",
|
|
Value: "1",
|
|
},
|
|
{
|
|
Name: "TYPE",
|
|
Value: "worker",
|
|
},
|
|
{
|
|
Name: "CPU_REQUEST",
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
ResourceFieldRef: &corev1.ResourceFieldSelector{
|
|
ContainerName: "ray-worker",
|
|
Resource: "requests.cpu",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "CPU_LIMITS",
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
ResourceFieldRef: &corev1.ResourceFieldSelector{
|
|
ContainerName: "ray-worker",
|
|
Resource: "limits.cpu",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "MEMORY_REQUESTS",
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
ResourceFieldRef: &corev1.ResourceFieldSelector{
|
|
ContainerName: "ray-worker",
|
|
Resource: "requests.memory",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "MEMORY_LIMITS",
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
ResourceFieldRef: &corev1.ResourceFieldSelector{
|
|
ContainerName: "ray-worker",
|
|
Resource: "limits.memory",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "MY_POD_NAME",
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
FieldRef: &corev1.ObjectFieldSelector{
|
|
FieldPath: "metadata.name",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "MY_POD_IP",
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
FieldRef: &corev1.ObjectFieldSelector{
|
|
FieldPath: "status.podIP",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Ports: []corev1.ContainerPort{
|
|
{
|
|
ContainerPort: 80,
|
|
},
|
|
},
|
|
Lifecycle: &corev1.Lifecycle{
|
|
PreStop: &corev1.LifecycleHandler{
|
|
Exec: &corev1.ExecAction{
|
|
Command: []string{
|
|
"/bin/sh", "-c", "ray stop",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Resources: corev1.ResourceRequirements{
|
|
Limits: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse(cpu),
|
|
corev1.ResourceMemory: resource.MustParse(memory),
|
|
},
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse(cpu),
|
|
corev1.ResourceMemory: resource.MustParse(memory),
|
|
},
|
|
},
|
|
VolumeMounts: volMounts,
|
|
SecurityContext: buildSecurityContext(spec.SecurityContext),
|
|
},
|
|
},
|
|
Volumes: vols,
|
|
},
|
|
}
|
|
|
|
// We are filtering container by name `ray-worker`. If container with this name does not exist
|
|
// (should never happen) we are not adding container specific parameters
|
|
if container, index, ok := GetContainerByName(podTemplateSpec.Spec.Containers, "ray-worker"); ok {
|
|
if gpu := computeRuntime.GetGpu(); gpu != 0 {
|
|
accelerator := "nvidia.com/gpu"
|
|
if len(computeRuntime.GetGpuAccelerator()) != 0 {
|
|
accelerator = computeRuntime.GetGpuAccelerator()
|
|
}
|
|
addResourceToContainer(&container, accelerator, gpu)
|
|
}
|
|
|
|
for k, v := range computeRuntime.GetExtendedResources() {
|
|
addResourceToContainer(&container, k, v)
|
|
}
|
|
|
|
globalEnv := convertEnvironmentVariables(envs)
|
|
if len(globalEnv) > 0 {
|
|
container.Env = append(container.Env, globalEnv...)
|
|
}
|
|
|
|
// Add specific environments
|
|
specEnv := convertEnvironmentVariables(spec.Environment)
|
|
if len(specEnv) > 0 {
|
|
container.Env = append(container.Env, specEnv...)
|
|
}
|
|
|
|
// Replace container
|
|
podTemplateSpec.Spec.Containers[index] = container
|
|
}
|
|
|
|
// Add specific annotations
|
|
if spec.Annotations != nil {
|
|
maps.Copy(podTemplateSpec.ObjectMeta.Annotations, spec.Annotations)
|
|
}
|
|
|
|
// Add specific labels
|
|
if spec.Labels != nil {
|
|
maps.Copy(podTemplateSpec.ObjectMeta.Labels, spec.Labels)
|
|
}
|
|
|
|
// Add specific tollerations
|
|
if computeRuntime.Tolerations != nil {
|
|
for _, t := range computeRuntime.Tolerations {
|
|
podTemplateSpec.Spec.Tolerations = append(podTemplateSpec.Spec.Tolerations, corev1.Toleration{
|
|
Key: t.Key, Operator: convertTolerationOperator(t.Operator), Value: t.Value, Effect: convertTaintEffect(t.Effect),
|
|
})
|
|
}
|
|
}
|
|
|
|
// If service account is specified, add it to the pod spec.
|
|
if len(spec.ServiceAccount) > 1 {
|
|
podTemplateSpec.Spec.ServiceAccountName = spec.ServiceAccount
|
|
}
|
|
|
|
// If image pull secret is specified, add it to the pod spec.
|
|
if len(spec.ImagePullSecret) > 1 {
|
|
podTemplateSpec.Spec.ImagePullSecrets = []corev1.LocalObjectReference{
|
|
{
|
|
Name: spec.ImagePullSecret,
|
|
},
|
|
}
|
|
}
|
|
|
|
return &podTemplateSpec, nil
|
|
}
|
|
|
|
// Build Volume mounts
|
|
func buildVolumeMounts(apiVolumes []*api.Volume) []corev1.VolumeMount {
|
|
var (
|
|
volMounts []corev1.VolumeMount
|
|
hostToContainer = corev1.MountPropagationHostToContainer
|
|
bidirectonal = corev1.MountPropagationBidirectional
|
|
)
|
|
for _, vol := range apiVolumes {
|
|
volMount := corev1.VolumeMount{
|
|
Name: vol.Name,
|
|
ReadOnly: vol.ReadOnly,
|
|
MountPath: vol.MountPath,
|
|
}
|
|
switch vol.MountPropagationMode {
|
|
case api.Volume_HOSTTOCONTAINER:
|
|
volMount.MountPropagation = &hostToContainer
|
|
case api.Volume_BIDIRECTIONAL:
|
|
volMount.MountPropagation = &bidirectonal
|
|
}
|
|
volMounts = append(volMounts, volMount)
|
|
}
|
|
return volMounts
|
|
}
|
|
|
|
// Build host path
|
|
func newHostPathType(pathType string) *corev1.HostPathType {
|
|
hostPathType := new(corev1.HostPathType)
|
|
*hostPathType = corev1.HostPathType(pathType)
|
|
return hostPathType
|
|
}
|
|
|
|
// Build volumes
|
|
func buildVols(apiVolumes []*api.Volume) ([]corev1.Volume, error) {
|
|
var vols []corev1.Volume
|
|
for _, rayVol := range apiVolumes {
|
|
if rayVol.VolumeType == api.Volume_CONFIGMAP {
|
|
vol := corev1.Volume{
|
|
Name: rayVol.Name,
|
|
VolumeSource: corev1.VolumeSource{
|
|
ConfigMap: &corev1.ConfigMapVolumeSource{
|
|
LocalObjectReference: corev1.LocalObjectReference{
|
|
Name: rayVol.Source,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
if len(rayVol.Items) > 0 {
|
|
// Add items
|
|
items := []corev1.KeyToPath{}
|
|
for key, value := range rayVol.Items {
|
|
items = append(items, corev1.KeyToPath{Key: key, Path: value})
|
|
}
|
|
vol.ConfigMap.Items = items
|
|
}
|
|
vols = append(vols, vol)
|
|
}
|
|
if rayVol.VolumeType == api.Volume_SECRET {
|
|
vol := corev1.Volume{
|
|
Name: rayVol.Name,
|
|
VolumeSource: corev1.VolumeSource{
|
|
Secret: &corev1.SecretVolumeSource{
|
|
SecretName: rayVol.Source,
|
|
},
|
|
},
|
|
}
|
|
if len(rayVol.Items) > 0 {
|
|
// Add items
|
|
items := []corev1.KeyToPath{}
|
|
for key, value := range rayVol.Items {
|
|
items = append(items, corev1.KeyToPath{Key: key, Path: value})
|
|
}
|
|
vol.Secret.Items = items
|
|
}
|
|
vols = append(vols, vol)
|
|
}
|
|
if rayVol.VolumeType == api.Volume_EMPTY_DIR {
|
|
vol := corev1.Volume{
|
|
Name: rayVol.Name,
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{},
|
|
},
|
|
}
|
|
if rayVol.Storage != "" {
|
|
// Max Storage size is defined
|
|
// Ensure that storage size is formatted correctly
|
|
_, err := resource.ParseQuantity(rayVol.Storage)
|
|
if err != nil {
|
|
return nil, errors.New("storage for empty dir volume is not specified correctly")
|
|
}
|
|
limit := resource.MustParse(rayVol.Storage)
|
|
vol.EmptyDir.SizeLimit = &limit
|
|
}
|
|
vols = append(vols, vol)
|
|
}
|
|
if rayVol.VolumeType == api.Volume_HOST_PATH {
|
|
vol := corev1.Volume{
|
|
Name: rayVol.Name,
|
|
VolumeSource: corev1.VolumeSource{
|
|
HostPath: &corev1.HostPathVolumeSource{
|
|
Path: rayVol.Source,
|
|
},
|
|
},
|
|
}
|
|
switch rayVol.HostPathType {
|
|
case api.Volume_DIRECTORY:
|
|
vol.VolumeSource.HostPath.Type = newHostPathType(string(corev1.HostPathDirectory))
|
|
case api.Volume_FILE:
|
|
vol.VolumeSource.HostPath.Type = newHostPathType(string(corev1.HostPathFile))
|
|
default:
|
|
vol.VolumeSource.HostPath.Type = newHostPathType(string(corev1.HostPathDirectory))
|
|
}
|
|
vols = append(vols, vol)
|
|
}
|
|
if rayVol.VolumeType == api.Volume_PERSISTENT_VOLUME_CLAIM {
|
|
vol := corev1.Volume{
|
|
Name: rayVol.Name,
|
|
VolumeSource: corev1.VolumeSource{
|
|
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
|
|
ClaimName: rayVol.Source,
|
|
ReadOnly: rayVol.ReadOnly,
|
|
},
|
|
},
|
|
}
|
|
vols = append(vols, vol)
|
|
}
|
|
if rayVol.VolumeType == api.Volume_EPHEMERAL {
|
|
// Make sure that at least the storage size is defined
|
|
if rayVol.Storage == "" {
|
|
// Storage size is not defined
|
|
return nil, errors.New("storage for ephemeral volume is empty")
|
|
}
|
|
// Ensure that storage size is formatted correctly
|
|
_, err := resource.ParseQuantity(rayVol.Storage)
|
|
if err != nil {
|
|
return nil, errors.New("storage for ephemeral volume is not specified correctly")
|
|
}
|
|
vol := corev1.Volume{
|
|
Name: rayVol.Name,
|
|
VolumeSource: corev1.VolumeSource{
|
|
Ephemeral: &corev1.EphemeralVolumeSource{
|
|
VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{
|
|
"app.kubernetes.io/managed-by": "kuberay-apiserver",
|
|
},
|
|
},
|
|
Spec: corev1.PersistentVolumeClaimSpec{
|
|
Resources: corev1.VolumeResourceRequirements{
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceStorage: resource.MustParse(rayVol.Storage),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
if len(rayVol.StorageClassName) > 0 {
|
|
// Populate storage class, if defined
|
|
vol.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.StorageClassName = &rayVol.StorageClassName
|
|
}
|
|
|
|
// Populate access mode if defined
|
|
switch rayVol.AccessMode {
|
|
case api.Volume_RWO:
|
|
vol.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{
|
|
corev1.ReadWriteOnce,
|
|
}
|
|
case api.Volume_RWX:
|
|
vol.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{
|
|
corev1.ReadWriteMany,
|
|
}
|
|
case api.Volume_ROX:
|
|
vol.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{
|
|
corev1.ReadOnlyMany,
|
|
}
|
|
default:
|
|
vol.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{
|
|
corev1.ReadWriteOnce,
|
|
}
|
|
}
|
|
vols = append(vols, vol)
|
|
}
|
|
}
|
|
|
|
return vols, nil
|
|
}
|
|
|
|
// Build security context
|
|
func buildSecurityContext(securityCtx *api.SecurityContext) *corev1.SecurityContext {
|
|
if securityCtx == nil {
|
|
return nil
|
|
}
|
|
result := &corev1.SecurityContext{
|
|
Privileged: securityCtx.Privileged,
|
|
Capabilities: &corev1.Capabilities{},
|
|
}
|
|
if securityCtx.Capabilities != nil {
|
|
for _, cap := range securityCtx.Capabilities.Add {
|
|
result.Capabilities.Add = append(result.Capabilities.Add, corev1.Capability(cap))
|
|
}
|
|
for _, cap := range securityCtx.Capabilities.Drop {
|
|
result.Capabilities.Drop = append(result.Capabilities.Drop, corev1.Capability(cap))
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// Get converts this object to a rayv1api.Workflow.
|
|
func (c *RayCluster) Get() *rayv1api.RayCluster {
|
|
return c.RayCluster
|
|
}
|
|
|
|
// SetAnnotations sets annotations on all templates in a RayCluster
|
|
func (c *RayCluster) SetAnnotationsToAllTemplates(_ string, _ string) {
|
|
// TODO: reserved for common parameters.
|
|
}
|
|
|
|
// Build compute template
|
|
func NewComputeTemplate(runtime *api.ComputeTemplate) (*corev1.ConfigMap, error) {
|
|
extendedResourcesJSON, err := json.Marshal(runtime.ExtendedResources)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to marshal extended resources: %w", err)
|
|
}
|
|
memoryUnit := "Gi"
|
|
if runtime.MemoryUnit != "" {
|
|
memoryUnit = runtime.MemoryUnit
|
|
}
|
|
|
|
memory := strconv.FormatUint(uint64(runtime.Memory), 10)
|
|
quantity := memory + memoryUnit
|
|
|
|
if _, err := resource.ParseQuantity(quantity); err != nil {
|
|
return nil, fmt.Errorf("invalid memory quantity %q: %w", quantity, err)
|
|
}
|
|
|
|
// Create data map
|
|
dmap := map[string]string{
|
|
"name": runtime.Name,
|
|
"namespace": runtime.Namespace,
|
|
"cpu": strconv.FormatUint(uint64(runtime.Cpu), 10),
|
|
"memory": memory,
|
|
"memory_unit": memoryUnit,
|
|
"gpu": strconv.FormatUint(uint64(runtime.Gpu), 10),
|
|
"gpu_accelerator": runtime.GpuAccelerator,
|
|
"extended_resources": string(extendedResourcesJSON),
|
|
}
|
|
// Add tolerations in defined
|
|
if len(runtime.Tolerations) > 0 {
|
|
t, err := json.Marshal(runtime.Tolerations)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to marshal tolerations for compute template %s: %w", runtime.Name, err)
|
|
}
|
|
dmap["tolerations"] = string(t)
|
|
}
|
|
|
|
config := &corev1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: runtime.Name,
|
|
Namespace: runtime.Namespace,
|
|
Labels: map[string]string{
|
|
"ray.io/config-type": "compute-template",
|
|
"ray.io/compute-template": runtime.Name,
|
|
},
|
|
},
|
|
Data: dmap,
|
|
}
|
|
|
|
return config, nil
|
|
}
|
|
|
|
// GetNodeHostIP returns the provided node's IP, based on the priority:
|
|
// 1. NodeInternalIP
|
|
// 2. NodeExternalIP
|
|
func GetNodeHostIP(node *corev1.Node) (net.IP, error) {
|
|
addresses := node.Status.Addresses
|
|
addressMap := make(map[corev1.NodeAddressType][]corev1.NodeAddress)
|
|
for _, nodeAddress := range addresses {
|
|
addressMap[nodeAddress.Type] = append(addressMap[nodeAddress.Type], nodeAddress)
|
|
}
|
|
if addresses, ok := addressMap[corev1.NodeInternalIP]; ok {
|
|
return net.ParseIP(addresses[0].Address), nil
|
|
}
|
|
if addresses, ok := addressMap[corev1.NodeExternalIP]; ok {
|
|
return net.ParseIP(addresses[0].Address), nil
|
|
}
|
|
return nil, fmt.Errorf("host IP unknown; known addresses: %v", addresses)
|
|
}
|
|
|
|
func GetContainerByName(containers []corev1.Container, name string) (corev1.Container, int, bool) {
|
|
for index, container := range containers {
|
|
if container.Name == name {
|
|
return container, index, true
|
|
}
|
|
}
|
|
return corev1.Container{}, 0, false
|
|
}
|
|
|
|
func buildAutoscalerOptions(autoscalerOptions *api.AutoscalerOptions) (*rayv1api.AutoscalerOptions, error) {
|
|
if autoscalerOptions == nil {
|
|
return nil, nil
|
|
}
|
|
options := rayv1api.AutoscalerOptions{}
|
|
if autoscalerOptions.IdleTimeoutSeconds > 0 {
|
|
options.IdleTimeoutSeconds = &autoscalerOptions.IdleTimeoutSeconds
|
|
}
|
|
if len(autoscalerOptions.UpscalingMode) > 0 {
|
|
options.UpscalingMode = (*rayv1api.UpscalingMode)(&autoscalerOptions.UpscalingMode)
|
|
}
|
|
if len(autoscalerOptions.Image) > 0 {
|
|
options.Image = &autoscalerOptions.Image
|
|
}
|
|
if len(autoscalerOptions.ImagePullPolicy) > 0 && strings.ToLower(autoscalerOptions.ImagePullPolicy) == "always" {
|
|
policy := corev1.PullAlways
|
|
options.ImagePullPolicy = &policy
|
|
}
|
|
|
|
if autoscalerOptions.Envs != nil {
|
|
if len(autoscalerOptions.Envs.Values) > 0 {
|
|
options.Env = make([]corev1.EnvVar, len(autoscalerOptions.Envs.Values))
|
|
evCount := 0
|
|
for key, value := range autoscalerOptions.Envs.Values {
|
|
options.Env[evCount] = corev1.EnvVar{Name: key, Value: value}
|
|
evCount++
|
|
}
|
|
}
|
|
if len(autoscalerOptions.Envs.ValuesFrom) > 0 {
|
|
options.EnvFrom = make([]corev1.EnvFromSource, 0)
|
|
for _, value := range autoscalerOptions.Envs.ValuesFrom {
|
|
if evfrom := convertEnvFrom(value); evfrom != nil {
|
|
options.EnvFrom = append(options.EnvFrom, *evfrom)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if len(autoscalerOptions.Volumes) > 0 {
|
|
options.VolumeMounts = buildVolumeMounts(autoscalerOptions.Volumes)
|
|
}
|
|
if len(autoscalerOptions.Cpu) > 0 || len(autoscalerOptions.Memory) > 0 {
|
|
rcpu := "500m"
|
|
rmemory := "512Mi"
|
|
if len(autoscalerOptions.Cpu) > 0 {
|
|
rcpu = autoscalerOptions.Cpu
|
|
}
|
|
if len(autoscalerOptions.Memory) > 0 {
|
|
rmemory = autoscalerOptions.Memory
|
|
}
|
|
_, err := resource.ParseQuantity(rcpu)
|
|
if err != nil {
|
|
return nil, errors.New("cpu for autoscaler is not specified correctly")
|
|
}
|
|
_, err = resource.ParseQuantity(rmemory)
|
|
if err != nil {
|
|
return nil, errors.New("memory for autoscaler is not specified correctly")
|
|
}
|
|
options.Resources = &corev1.ResourceRequirements{
|
|
Limits: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse(rcpu),
|
|
corev1.ResourceMemory: resource.MustParse(rmemory),
|
|
},
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse(rcpu),
|
|
corev1.ResourceMemory: resource.MustParse(rmemory),
|
|
},
|
|
}
|
|
}
|
|
return &options, nil
|
|
}
|
|
|
|
func convertEnvFrom(from *api.EnvValueFrom) *corev1.EnvFromSource {
|
|
switch from.Source {
|
|
case api.EnvValueFrom_CONFIGMAP:
|
|
return &corev1.EnvFromSource{
|
|
ConfigMapRef: &corev1.ConfigMapEnvSource{
|
|
LocalObjectReference: corev1.LocalObjectReference{
|
|
Name: from.Name,
|
|
},
|
|
},
|
|
}
|
|
case api.EnvValueFrom_SECRET:
|
|
return &corev1.EnvFromSource{
|
|
SecretRef: &corev1.SecretEnvSource{
|
|
LocalObjectReference: corev1.LocalObjectReference{
|
|
Name: from.Name,
|
|
},
|
|
},
|
|
}
|
|
default:
|
|
return nil
|
|
}
|
|
}
|