mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
350 lines
11 KiB
Go
350 lines
11 KiB
Go
package common
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
|
|
rayiov1alpha1 "github.com/ray-project/kuberay/ray-operator/api/v1alpha1"
|
|
"github.com/ray-project/kuberay/ray-operator/controllers/utils"
|
|
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
|
|
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
|
|
|
v1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
)
|
|
|
|
const (
|
|
SharedMemoryVolumeName = "shared-mem"
|
|
SharedMemoryVolumeMountPath = "/dev/shm"
|
|
)
|
|
|
|
var (
|
|
log = logf.Log.WithName("RayCluster-Controller")
|
|
)
|
|
|
|
// DefaultHeadPodTemplate sets the config values
|
|
func DefaultHeadPodTemplate(instance rayiov1alpha1.RayCluster, headSpec rayiov1alpha1.HeadGroupSpec, podName string, svcName string) v1.PodTemplateSpec {
|
|
podTemplate := headSpec.Template
|
|
podTemplate.GenerateName = podName
|
|
if podTemplate.ObjectMeta.Namespace == "" {
|
|
podTemplate.ObjectMeta.Namespace = instance.Namespace
|
|
log.Info("Setting pod namespaces", "namespace", instance.Namespace)
|
|
}
|
|
|
|
if podTemplate.Labels == nil {
|
|
podTemplate.Labels = make(map[string]string)
|
|
}
|
|
podTemplate.Labels = labelPod(rayiov1alpha1.HeadNode, instance.Name, "headgroup", instance.Spec.HeadGroupSpec.Template.ObjectMeta.Labels)
|
|
headSpec.RayStartParams = setMissingRayStartParams(headSpec.RayStartParams, rayiov1alpha1.HeadNode, svcName)
|
|
return podTemplate
|
|
}
|
|
|
|
// DefaultWorkerPodTemplate sets the config values
|
|
func DefaultWorkerPodTemplate(instance rayiov1alpha1.RayCluster, workerSpec rayiov1alpha1.WorkerGroupSpec, podName string, svcName string) v1.PodTemplateSpec {
|
|
podTemplate := workerSpec.Template
|
|
podTemplate.GenerateName = podName
|
|
if podTemplate.ObjectMeta.Namespace == "" {
|
|
podTemplate.ObjectMeta.Namespace = instance.Namespace
|
|
log.Info("Setting pod namespaces", "namespace", instance.Namespace)
|
|
}
|
|
|
|
if podTemplate.Labels == nil {
|
|
podTemplate.Labels = make(map[string]string)
|
|
}
|
|
podTemplate.Labels = labelPod(rayiov1alpha1.WorkerNode, instance.Name, workerSpec.GroupName, workerSpec.Template.ObjectMeta.Labels)
|
|
workerSpec.RayStartParams = setMissingRayStartParams(workerSpec.RayStartParams, rayiov1alpha1.WorkerNode, svcName)
|
|
|
|
return podTemplate
|
|
}
|
|
|
|
// BuildPod a pod config
|
|
func BuildPod(podTemplateSpec v1.PodTemplateSpec, rayNodeType rayiov1alpha1.RayNodeType, rayStartParams map[string]string, svcName string) (aPod v1.Pod) {
|
|
pod := v1.Pod{
|
|
TypeMeta: metav1.TypeMeta{
|
|
APIVersion: "v1",
|
|
Kind: "Pod",
|
|
},
|
|
ObjectMeta: podTemplateSpec.ObjectMeta,
|
|
Spec: podTemplateSpec.Spec,
|
|
}
|
|
index := getRayContainerIndex(pod)
|
|
cont := concatenateContainerCommand(rayNodeType, rayStartParams)
|
|
|
|
addEmptyDir(&pod.Spec.Containers[index], &pod)
|
|
cleanupInvalidVolumeMounts(&pod.Spec.Containers[index], &pod)
|
|
if len(pod.Spec.InitContainers) > index {
|
|
cleanupInvalidVolumeMounts(&pod.Spec.InitContainers[index], &pod)
|
|
}
|
|
|
|
// saving temporarily the old command and args
|
|
var cmd, args string
|
|
if len(pod.Spec.Containers[index].Command) > 0 {
|
|
cmd = convertCmdToString(pod.Spec.Containers[index].Command)
|
|
}
|
|
if len(pod.Spec.Containers[index].Args) > 0 {
|
|
cmd += convertCmdToString(pod.Spec.Containers[index].Args)
|
|
}
|
|
if !strings.Contains(cmd, "ray start") {
|
|
// replacing the old command
|
|
pod.Spec.Containers[index].Command = []string{"/bin/bash", "-c", "--"}
|
|
if cmd != "" {
|
|
// sleep infinity is used to keep the pod `running` after the last command exits, and not go into `completed` state
|
|
args = fmt.Sprintf("%s && %s && %s", cont, cmd, "sleep infinity")
|
|
} else {
|
|
args = fmt.Sprintf("%s && %s", cont, "sleep infinity")
|
|
}
|
|
|
|
pod.Spec.Containers[index].Args = []string{args}
|
|
}
|
|
for index := range pod.Spec.InitContainers {
|
|
setInitContainerEnvVars(&pod.Spec.InitContainers[index], svcName)
|
|
}
|
|
|
|
setContainerEnvVars(&pod.Spec.Containers[index], rayNodeType, rayStartParams, svcName)
|
|
|
|
return pod
|
|
}
|
|
|
|
func convertCmdToString(cmdArr []string) (cmd string) {
|
|
cmdAggr := new(bytes.Buffer)
|
|
for _, v := range cmdArr {
|
|
fmt.Fprintf(cmdAggr, " %s ", v)
|
|
}
|
|
return cmdAggr.String()
|
|
|
|
}
|
|
|
|
func getRayContainerIndex(pod v1.Pod) (index int) {
|
|
// theoretically, a ray pod can have multiple containers.
|
|
// we identify the ray container based on env var: RAY=true
|
|
// if the env var is missing, we choose containers[0].
|
|
for i, container := range pod.Spec.Containers {
|
|
for _, env := range container.Env {
|
|
if env.Name == strings.ToLower("ray") && env.Value == strings.ToLower("true") {
|
|
return i
|
|
}
|
|
}
|
|
}
|
|
//not found, use first container
|
|
return 0
|
|
}
|
|
|
|
// labelPod returns the labels for selecting the resources
|
|
// belonging to the given RayCluster CR name.
|
|
func labelPod(rayNodeType rayiov1alpha1.RayNodeType, rayClusterName string, groupName string, labels map[string]string) (ret map[string]string) {
|
|
if labels == nil {
|
|
labels = make(map[string]string)
|
|
}
|
|
|
|
ret = map[string]string{
|
|
RayNodeLabelKey: "yes",
|
|
RayClusterLabelKey: rayClusterName,
|
|
RayNodeTypeLabelKey: string(rayNodeType),
|
|
RayNodeGroupLabelKey: groupName,
|
|
RayIDLabelKey: utils.CheckLabel(utils.GenerateIdentifier(rayClusterName, rayNodeType)),
|
|
}
|
|
|
|
for k, v := range ret {
|
|
if k == string(rayNodeType) {
|
|
// overriding invalide values for this label
|
|
if v != string(rayiov1alpha1.HeadNode) && v != string(rayiov1alpha1.WorkerNode) {
|
|
labels[k] = v
|
|
}
|
|
}
|
|
if k == RayNodeGroupLabelKey {
|
|
// overriding invalide values for this label
|
|
if v != groupName {
|
|
labels[k] = v
|
|
}
|
|
}
|
|
if _, ok := labels[k]; !ok {
|
|
labels[k] = v
|
|
}
|
|
}
|
|
return labels
|
|
}
|
|
|
|
func setInitContainerEnvVars(container *v1.Container, svcName string) {
|
|
//RAY_IP can be used in the DNS lookup
|
|
if container.Env == nil || len(container.Env) == 0 {
|
|
container.Env = []v1.EnvVar{}
|
|
}
|
|
if !envVarExists("RAY_IP", container.Env) {
|
|
ip := v1.EnvVar{Name: "RAY_IP"}
|
|
ip.Value = fmt.Sprintf("%s", svcName)
|
|
container.Env = append(container.Env, ip)
|
|
}
|
|
}
|
|
|
|
func setContainerEnvVars(container *v1.Container, rayNodeType rayiov1alpha1.RayNodeType, rayStartParams map[string]string, svcName string) {
|
|
// set IP to local host if head, or the the svc otherwise RAY_IP
|
|
// set the port RAY_PORT
|
|
// set the password?
|
|
if container.Env == nil || len(container.Env) == 0 {
|
|
container.Env = []v1.EnvVar{}
|
|
}
|
|
if !envVarExists(RAY_IP, container.Env) {
|
|
ip := v1.EnvVar{Name: RAY_IP}
|
|
if rayNodeType == rayiov1alpha1.HeadNode {
|
|
// if head, use localhost
|
|
ip.Value = "127.0.0.1"
|
|
} else {
|
|
// if worker, use the service name of the head
|
|
ip.Value = fmt.Sprintf("%s", svcName)
|
|
}
|
|
container.Env = append(container.Env, ip)
|
|
}
|
|
if !envVarExists(RAY_PORT, container.Env) {
|
|
port := v1.EnvVar{Name: RAY_PORT}
|
|
if value, ok := rayStartParams["port"]; !ok {
|
|
// using default port
|
|
port.Value = strconv.Itoa(DefaultRedisPort)
|
|
} else {
|
|
// setting the RAY_PORT env var from the params
|
|
port.Value = value
|
|
}
|
|
container.Env = append(container.Env, port)
|
|
}
|
|
if !envVarExists(REDIS_PASSWORD, container.Env) {
|
|
// setting the REDIS_PASSWORD env var from the params
|
|
port := v1.EnvVar{Name: REDIS_PASSWORD}
|
|
if value, ok := rayStartParams["redis-password"]; ok {
|
|
port.Value = value
|
|
}
|
|
container.Env = append(container.Env, port)
|
|
}
|
|
}
|
|
|
|
func envVarExists(envName string, envVars []v1.EnvVar) bool {
|
|
if envVars == nil || len(envVars) == 0 {
|
|
return false
|
|
}
|
|
|
|
for _, env := range envVars {
|
|
if env.Name == envName {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
//TODO auto complete params
|
|
func setMissingRayStartParams(rayStartParams map[string]string, nodeType rayiov1alpha1.RayNodeType, svcName string) (completeStartParams map[string]string) {
|
|
if nodeType == rayiov1alpha1.WorkerNode {
|
|
if _, ok := rayStartParams["address"]; !ok {
|
|
address := fmt.Sprintf("%s", svcName)
|
|
if _, okPort := rayStartParams["port"]; !okPort {
|
|
address = fmt.Sprintf("%s:%s", address, "6379")
|
|
} else {
|
|
address = fmt.Sprintf("%s:%s", address, rayStartParams["port"])
|
|
}
|
|
rayStartParams["address"] = address
|
|
}
|
|
}
|
|
return rayStartParams
|
|
}
|
|
|
|
// concatenateContainerCommand with ray start
|
|
func concatenateContainerCommand(nodeType rayiov1alpha1.RayNodeType, rayStartParams map[string]string) (fullCmd string) {
|
|
switch nodeType {
|
|
case rayiov1alpha1.HeadNode:
|
|
return fmt.Sprintf("ulimit -n 65536; ray start --head %s", convertParamMap(rayStartParams))
|
|
case rayiov1alpha1.WorkerNode:
|
|
return fmt.Sprintf("ulimit -n 65536; ray start %s", convertParamMap(rayStartParams))
|
|
default:
|
|
log.Error(fmt.Errorf("missing node type"), "a node must be either head or worker")
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func convertParamMap(rayStartParams map[string]string) (s string) {
|
|
flags := new(bytes.Buffer)
|
|
for k, v := range rayStartParams {
|
|
fmt.Fprintf(flags, " --%s=%s ", k, v)
|
|
}
|
|
return flags.String()
|
|
}
|
|
|
|
// addEmptyDir add an emptyDir to the shared memory mount point /dev/shm
|
|
// this is to avoid: "The object store is using /tmp instead of /dev/shm because /dev/shm has only 67108864 bytes available. This may slow down performance!...""
|
|
func addEmptyDir(container *v1.Container, pod *v1.Pod) {
|
|
if checkIfVolumeMounted(container, pod) {
|
|
return
|
|
}
|
|
//1) create a Volume of type emptyDir and add it to Volumes
|
|
emptyDirVolume := v1.Volume{
|
|
Name: SharedMemoryVolumeName,
|
|
VolumeSource: v1.VolumeSource{
|
|
EmptyDir: &v1.EmptyDirVolumeSource{
|
|
Medium: v1.StorageMediumMemory,
|
|
SizeLimit: findMemoryReqOrLimit(*container),
|
|
},
|
|
},
|
|
}
|
|
if !checkIfVolumeMounted(container, pod) {
|
|
pod.Spec.Volumes = append(pod.Spec.Volumes, emptyDirVolume)
|
|
}
|
|
|
|
//2) create a VolumeMount that uses the emptyDir
|
|
mountedVolume := v1.VolumeMount{
|
|
MountPath: SharedMemoryVolumeMountPath,
|
|
Name: SharedMemoryVolumeName,
|
|
ReadOnly: false,
|
|
}
|
|
if !checkIfVolumeMounted(container, pod) {
|
|
container.VolumeMounts = append(container.VolumeMounts, mountedVolume)
|
|
}
|
|
}
|
|
|
|
func checkIfVolumeMounted(container *v1.Container, pod *v1.Pod) bool {
|
|
for _, mountedVol := range container.VolumeMounts {
|
|
if mountedVol.MountPath == SharedMemoryVolumeMountPath {
|
|
for _, podVolume := range pod.Spec.Volumes {
|
|
if mountedVol.Name == podVolume.Name {
|
|
// already mounted, nothing to do
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func cleanupInvalidVolumeMounts(container *v1.Container, pod *v1.Pod) {
|
|
// if a volumeMount is specified in the container,
|
|
// but has no corresponding pod volume, it is removed
|
|
for index, mountedVol := range container.VolumeMounts {
|
|
valid := false
|
|
for _, podVolume := range pod.Spec.Volumes {
|
|
if mountedVol.Name == podVolume.Name {
|
|
// valid mount, moving on...
|
|
valid = true
|
|
break
|
|
}
|
|
}
|
|
if !valid {
|
|
//remove the VolumeMount
|
|
container.VolumeMounts[index] = container.VolumeMounts[len(container.VolumeMounts)-1]
|
|
container.VolumeMounts = container.VolumeMounts[:len(container.VolumeMounts)-1]
|
|
}
|
|
}
|
|
}
|
|
|
|
func findMemoryReqOrLimit(container v1.Container) (res *resource.Quantity) {
|
|
mem := &resource.Quantity{}
|
|
//check the requests, if they are not set, check the limits.
|
|
if q, ok := container.Resources.Requests[v1.ResourceMemory]; ok {
|
|
mem = &q
|
|
return mem
|
|
}
|
|
if q, ok := container.Resources.Limits[v1.ResourceMemory]; ok {
|
|
mem = &q
|
|
return mem
|
|
}
|
|
return nil
|
|
}
|