kserve/charts/kserve-runtime-configs/files/runtimes/clusterservingruntimes-patch.yaml

{{- if .Values.kserve.servingruntime.enabled }}
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
  name: kserve-lgbserver
  annotations:
    serving.kserve.io/server-type: lgbserver
spec:
  disabled: {{ .Values.kserve.servingruntime.lgbserver.disabled }}
  containers:
    - name: kserve-container
      image: "{{ .Values.kserve.servingruntime.lgbserver.image }}:{{ .Values.kserve.servingruntime.lgbserver.tag | default .Values.kserve.version }}"
      imagePullPolicy: {{ .Values.kserve.servingruntime.lgbserver.imagePullPolicy }}
      args:
        - --model_name={{ .Values.kserve.servingruntime.modelNamePlaceholder }}
        - --model_dir=/mnt/models
        - --http_port=8080
        - --nthread=1
      securityContext:
          {{- with .Values.kserve.servingruntime.lgbserver.securityContext}}
          {{- toYaml . | nindent 10 }}
          {{- end }}
      resources:
        requests:
          cpu: "1"
          memory: 2Gi
        limits:
          cpu: "1"
          memory: 2Gi
  {{- with .Values.kserve.servingruntime.lgbserver.imagePullSecrets }}
  imagePullSecrets:
    {{- toYaml . | nindent 4 }}
  {{- end }}

---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
  name: kserve-predictiveserver
spec:
  disabled: {{ .Values.kserve.servingruntime.predictiveserver.disabled }}
  containers:
    - name: kserve-container
      image: "{{ .Values.kserve.servingruntime.predictiveserver.image }}:{{ .Values.kserve.servingruntime.predictiveserver.tag | default .Values.kserve.version }}"
      imagePullPolicy: {{ .Values.kserve.servingruntime.predictiveserver.imagePullPolicy }}
      args:
        - --model_name={{ .Values.kserve.servingruntime.modelNamePlaceholder }}
        - --model_dir=/mnt/models
        - --http_port=8080
        - --framework={{ "{{.Annotations.modelFormat}}" }}
        - --nthread=1
      securityContext:
          {{- with .Values.kserve.servingruntime.predictiveserver.securityContext}}
          {{- toYaml . | nindent 10 }}
          {{- end }}
      resources:
        requests:
          cpu: "1"
          memory: 2Gi
        limits:
          cpu: "1"
          memory: 2Gi
  {{- with .Values.kserve.servingruntime.predictiveserver.imagePullSecrets }}
  imagePullSecrets:
    {{- toYaml . | nindent 4 }}
  {{- end }}

---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
  name: kserve-mlserver
  annotations:
    serving.kserve.io/server-type: mlserver
spec:
  disabled: {{ .Values.kserve.servingruntime.mlserver.disabled }}
  containers:
    - name: kserve-container
      image: "{{ .Values.kserve.servingruntime.mlserver.image }}:{{ .Values.kserve.servingruntime.mlserver.tag }}"
      imagePullPolicy: {{ .Values.kserve.servingruntime.mlserver.imagePullPolicy }}
      env:
        - name: "MLSERVER_MODEL_IMPLEMENTATION"
          value: "{{ .Values.kserve.servingruntime.mlserver.modelClassPlaceholder }}"
        - name: "MLSERVER_HTTP_PORT"
          value: "8080"
        - name: "MLSERVER_GRPC_PORT"
          value: "9000"
        - name: "MODELS_DIR"
          value: "/mnt/models"
      securityContext:
          {{- with .Values.kserve.servingruntime.mlserver.securityContext}}
          {{- toYaml . | nindent 10 }}
          {{- end }}
      resources:
        requests:
          cpu: "1"
          memory: 2Gi
        limits:
          cpu: "1"
          memory: 2Gi
  {{- with .Values.kserve.servingruntime.mlserver.imagePullSecrets }}
  imagePullSecrets:
    {{- toYaml . | nindent 4 }}
  {{- end }}

---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
  name: kserve-paddleserver
  annotations:
    serving.kserve.io/server-type: paddleserver
spec:
  disabled: {{ .Values.kserve.servingruntime.paddleserver.disabled }}
  containers:
    - name: kserve-container
      image: "{{ .Values.kserve.servingruntime.paddleserver.image }}:{{ .Values.kserve.servingruntime.paddleserver.tag | default .Values.kserve.version }}"
      imagePullPolicy: {{ .Values.kserve.servingruntime.paddleserver.imagePullPolicy }}
      args:
        - --model_name={{ .Values.kserve.servingruntime.modelNamePlaceholder }}
        - --model_dir=/mnt/models
        - --http_port=8080
      securityContext:
          {{- with .Values.kserve.servingruntime.paddleserver.securityContext}}
          {{- toYaml . | nindent 10 }}
          {{- end }}
      resources:
        requests:
          cpu: "1"
          memory: 2Gi
        limits:
          cpu: "1"
          memory: 2Gi
  {{- with .Values.kserve.servingruntime.paddleserver.imagePullSecrets }}
  imagePullSecrets:
    {{- toYaml . | nindent 4 }}
  {{- end }}

---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
  name: kserve-pmmlserver
  annotations:
    serving.kserve.io/server-type: pmmlserver
spec:
  disabled: {{ .Values.kserve.servingruntime.pmmlserver.disabled }}
  containers:
    - name: kserve-container
      image: "{{ .Values.kserve.servingruntime.pmmlserver.image }}:{{ .Values.kserve.servingruntime.pmmlserver.tag | default .Values.kserve.version }}"
      imagePullPolicy: {{ .Values.kserve.servingruntime.pmmlserver.imagePullPolicy }}
      args:
        - --model_name={{ .Values.kserve.servingruntime.modelNamePlaceholder }}
        - --model_dir=/mnt/models
        - --http_port=8080
      securityContext:
          {{- with .Values.kserve.servingruntime.pmmlserver.securityContext}}
          {{- toYaml . | nindent 10 }}
          {{- end }}
      resources:
        requests:
          cpu: "1"
          memory: 2Gi
        limits:
          cpu: "1"
          memory: 2Gi
  {{- with .Values.kserve.servingruntime.pmmlserver.imagePullSecrets }}
  imagePullSecrets:
    {{- toYaml . | nindent 4 }}
  {{- end }}

---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
  name: kserve-sklearnserver
  annotations:
    serving.kserve.io/server-type: sklearnserver
spec:
  disabled: {{ .Values.kserve.servingruntime.sklearnserver.disabled }}
  containers:
    - name: kserve-container
      image: "{{ .Values.kserve.servingruntime.sklearnserver.image }}:{{ .Values.kserve.servingruntime.sklearnserver.tag | default .Values.kserve.version }}"
      imagePullPolicy: {{ .Values.kserve.servingruntime.sklearnserver.imagePullPolicy }}
      args:
        - --model_name={{ .Values.kserve.servingruntime.modelNamePlaceholder }}
        - --model_dir=/mnt/models
        - --http_port=8080
      securityContext:
          {{- with .Values.kserve.servingruntime.sklearnserver.securityContext}}
          {{- toYaml . | nindent 10 }}
          {{- end }}
      resources:
        requests:
          cpu: "1"
          memory: 2Gi
        limits:
          cpu: "1"
          memory: 2Gi
  {{- with .Values.kserve.servingruntime.sklearnserver.imagePullSecrets }}
  imagePullSecrets:
    {{- toYaml . | nindent 4 }}
  {{- end }}

---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
  name: kserve-tensorflow-serving
  annotations:
    serving.kserve.io/server-type: tensorflow-serving
spec:
  disabled: {{ .Values.kserve.servingruntime.tensorflow.disabled }}
  containers:
    - name: kserve-container
      image: "{{ .Values.kserve.servingruntime.tensorflow.image }}:{{ .Values.kserve.servingruntime.tensorflow.tag }}"
      imagePullPolicy: {{ .Values.kserve.servingruntime.tensorflow.imagePullPolicy }}
      command: [/usr/bin/tensorflow_model_server]
      args:
        - --model_name={{ .Values.kserve.servingruntime.modelNamePlaceholder }}
        - --port=9000
        - --rest_api_port=8080
        - --model_base_path=/mnt/models
        - --rest_api_timeout_in_ms=60000
      securityContext:
          {{- with .Values.kserve.servingruntime.tensorflow.securityContext}}
          {{- toYaml . | nindent 10 }}
          {{- end }}
      resources:
        requests:
          cpu: "1"
          memory: 2Gi
        limits:
          cpu: "1"
          memory: 2Gi
  {{- with .Values.kserve.servingruntime.tensorflow.imagePullSecrets }}
  imagePullSecrets:
    {{- toYaml . | nindent 4 }}
  {{- end }}

---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
  name: kserve-torchserve
  annotations:
    serving.kserve.io/server-type: torchserve
spec:
  disabled: {{ .Values.kserve.servingruntime.torchserve.disabled }}
  containers:
    - name: kserve-container
      image: "{{ .Values.kserve.servingruntime.torchserve.image }}:{{ .Values.kserve.servingruntime.torchserve.tag }}"
      imagePullPolicy: {{ .Values.kserve.servingruntime.torchserve.imagePullPolicy }}
      args:
        - torchserve
        - --start
        - --model-store=/mnt/models/model-store
        - --ts-config=/mnt/models/config/config.properties
      env:
        - name: "TS_SERVICE_ENVELOPE"
          value: "{{ .Values.kserve.servingruntime.torchserve.serviceEnvelopePlaceholder }}"
      securityContext:
          {{- with .Values.kserve.servingruntime.torchserve.securityContext}}
          {{- toYaml . | nindent 10 }}
          {{- end }}
      resources:
        requests:
          cpu: "1"
          memory: 2Gi
        limits:
          cpu: "1"
          memory: 2Gi
  {{- with .Values.kserve.servingruntime.torchserve.imagePullSecrets }}
  imagePullSecrets:
    {{- toYaml . | nindent 4 }}
  {{- end }}

---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
  name: kserve-tritonserver
  annotations:
    serving.kserve.io/server-type: tritonserver
spec:
  disabled: {{ .Values.kserve.servingruntime.tritonserver.disabled }}
  containers:
    - name: kserve-container
      image: "{{ .Values.kserve.servingruntime.tritonserver.image }}:{{ .Values.kserve.servingruntime.tritonserver.tag }}"
      imagePullPolicy: {{ .Values.kserve.servingruntime.tritonserver.imagePullPolicy }}
      args:
        - tritonserver
        - --model-store=/mnt/models
        - --grpc-port=9000
        - --http-port=8080
        - --allow-grpc=true
        - --allow-http=true
      securityContext:
          {{- with .Values.kserve.servingruntime.tritonserver.securityContext}}
          {{- toYaml . | nindent 10 }}
          {{- end }}
      resources:
        requests:
          cpu: "1"
          memory: 2Gi
        limits:
          cpu: "1"
          memory: 2Gi
  {{- with .Values.kserve.servingruntime.tritonserver.imagePullSecrets }}
  imagePullSecrets:
    {{- toYaml . | nindent 4 }}
  {{- end }}

---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
  name: kserve-xgbserver
  annotations:
    serving.kserve.io/server-type: xgbserver
spec:
  disabled: {{ .Values.kserve.servingruntime.xgbserver.disabled }}
  containers:
    - name: kserve-container
      image: "{{ .Values.kserve.servingruntime.xgbserver.image }}:{{ .Values.kserve.servingruntime.xgbserver.tag | default .Values.kserve.version }}"
      imagePullPolicy: {{ .Values.kserve.servingruntime.xgbserver.imagePullPolicy }}
      args:
        - --model_name={{ .Values.kserve.servingruntime.modelNamePlaceholder }}
        - --model_dir=/mnt/models
        - --http_port=8080
        - --nthread=1
      securityContext:
          {{- with .Values.kserve.servingruntime.xgbserver.securityContext}}
          {{- toYaml . | nindent 10 }}
          {{- end }}
      resources:
        requests:
          cpu: "1"
          memory: 2Gi
        limits:
          cpu: "1"
          memory: 2Gi
  {{- with .Values.kserve.servingruntime.xgbserver.imagePullSecrets }}
  imagePullSecrets:
    {{- toYaml . | nindent 4 }}
  {{- end }}
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
  name: kserve-huggingfaceserver
  annotations:
    serving.kserve.io/server-type: huggingfaceserver
spec:
  disabled: {{ .Values.kserve.servingruntime.huggingfaceserver.disabled }}
  containers:
    - name: kserve-container
      image: "{{ .Values.kserve.servingruntime.huggingfaceserver.image }}:{{ .Values.kserve.servingruntime.huggingfaceserver.tag | default .Values.kserve.version }}"
      imagePullPolicy: {{ .Values.kserve.servingruntime.huggingfaceserver.imagePullPolicy }}
      args:
        - --model_name={{ .Values.kserve.servingruntime.modelNamePlaceholder }}
      env:
        - name: LMCACHE_USE_EXPERIMENTAL
          value: "{{ .Values.kserve.servingruntime.huggingfaceserver.lmcacheUseExperimental }}"
      securityContext:
          {{- with .Values.kserve.servingruntime.huggingfaceserver.securityContext}}
          {{- toYaml . | nindent 10 }}
          {{- end }}
      resources:
        requests:
          cpu: "1"
          memory: 2Gi
        limits:
          cpu: "1"
          memory: 2Gi
      {{- if .Values.kserve.servingruntime.huggingfaceserver.devShm.enabled }}
      volumeMounts:
        - name: devshm
          mountPath: /dev/shm
      {{- end }}
  {{- if .Values.kserve.servingruntime.huggingfaceserver.devShm.enabled }}
  volumes:
    - name: devshm
      emptyDir:
        medium: Memory
        {{- with .Values.kserve.servingruntime.huggingfaceserver.devShm.sizeLimit }}
        sizeLimit: {{ . }}
        {{- end }}
  {{- end }}
  {{- with .Values.kserve.servingruntime.huggingfaceserver.imagePullSecrets }}
  imagePullSecrets:
    {{- toYaml . | nindent 4 }}
  {{- end }}
  {{- if .Values.kserve.servingruntime.huggingfaceserver.hostIPC.enabled }}
  hostIPC: true
  {{- end }}
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
  name: kserve-huggingfaceserver-multinode
  annotations:
    serving.kserve.io/server-type: huggingfaceserver
spec:
  disabled: {{ .Values.kserve.servingruntime.huggingfaceserver.disabled }}
  containers:
    - name: kserve-container
      image: "{{ .Values.kserve.servingruntime.huggingfaceserver.image }}:{{ .Values.kserve.servingruntime.huggingfaceserver.tag | default .Values.kserve.version }}-gpu"
      imagePullPolicy: {{ .Values.kserve.servingruntime.huggingfaceserver.imagePullPolicy }}
      args:
        - --model_name={{ .Values.kserve.servingruntime.modelNamePlaceholder }}
      command:
        - "bash"
        - "-c"
        - |
          export MODEL=${MODEL_ID}
          if [[ ! -z ${MODEL_DIR} ]]
          then
            export MODEL=${MODEL_DIR}
          fi

          export RAY_ADDRESS=${POD_IP}:${RAY_PORT}
          ray start --head --disable-usage-stats --include-dashboard false 
          python ./huggingfaceserver/health_check.py registered_nodes --retries 200  --probe_name runtime_start

          python -m huggingfaceserver --model_dir=${MODEL} --tensor-parallel-size=${TENSOR_PARALLEL_SIZE} --pipeline-parallel-size=${PIPELINE_PARALLEL_SIZE} $0 $@        
      securityContext:
          {{- with .Values.kserve.servingruntime.huggingfaceserver_multinode.securityContext}}
          {{- toYaml . | nindent 10 }}
          {{- end }}
      env:
        - name: RAY_PORT
          value: "6379"        
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP  
        - name: VLLM_CONFIG_ROOT
          value: /tmp      
        - name: HF_HUB_CACHE
          value: /tmp          
      resources:
        requests:
          cpu: "2"
          memory: 6Gi
        limits:
          cpu: "4"
          memory: 12Gi
      livenessProbe:
        failureThreshold: 2
        periodSeconds: 5
        successThreshold: 1
        timeoutSeconds: 15
        exec:
          command:
            - bash
            - -c
            - |
              python ./huggingfaceserver/health_check.py registered_node_and_runtime_health --health_check_url http://localhost:8080 --probe_name head_liveness
      readinessProbe:
        failureThreshold: 2
        periodSeconds: 5
        successThreshold: 1
        timeoutSeconds: 15
        exec:
          command:
            - bash
            - -c
            - |
              python ./huggingfaceserver/health_check.py runtime_health --health_check_url http://localhost:8080 --probe_name head_readiness
      startupProbe:
        failureThreshold: 40
        periodSeconds: 30
        successThreshold: 1
        timeoutSeconds: 30
        initialDelaySeconds: 60
        exec:
          command:
            - bash
            - -c
            - |
              python ./huggingfaceserver/health_check.py registered_node_and_runtime_health --health_check_url http://localhost:8080 --probe_name head_startup          
      {{- if .Values.kserve.servingruntime.huggingfaceserver_multinode.shm.enabled }}
      volumeMounts:
        - name: shm
          mountPath: /dev/shm
      {{- end }}
  {{- if .Values.kserve.servingruntime.huggingfaceserver_multinode.shm.enabled }}
  volumes:
    - name: shm
      emptyDir:
        medium: Memory
        {{- with .Values.kserve.servingruntime.huggingfaceserver_multinode.shm.sizeLimit }}
        sizeLimit: {{ . }}
        {{- end }}
  {{- end }}
  {{- with .Values.kserve.servingruntime.huggingfaceserver_multinode.imagePullSecrets }}
  imagePullSecrets:
    {{- toYaml . | nindent 4 }}
  {{- end }}
  workerSpec:
    pipelineParallelSize: 2
    tensorParallelSize: 1
    containers:
      - name: worker-container
        image: "{{ .Values.kserve.servingruntime.huggingfaceserver.image }}:{{ .Values.kserve.servingruntime.huggingfaceserver.tag | default .Values.kserve.version }}-gpu"
        imagePullPolicy: {{ .Values.kserve.servingruntime.huggingfaceserver.imagePullPolicy }}
        command:
        - "bash"
        - "-c"
        - |
          export RAY_HEAD_ADDRESS=${HEAD_SVC}.${POD_NAMESPACE}.svc.cluster.local:6379
          SECONDS=0

          while true; do              
            if (( SECONDS <= 240 )); then
              if ray health-check --address "${RAY_HEAD_ADDRESS}" > /dev/null 2>&1; then
                echo "Ray Global Control Service(GCS) is ready."
                break
              fi
              echo "$SECONDS seconds elapsed: Waiting for Ray Global Control Service(GCS) to be ready."
            else
              if ray health-check --address "${RAY_HEAD_ADDRESS}"; then
                echo "Ray Global Control Service(GCS) is ready. Any error messages above can be safely ignored."
                break
              fi
              echo "$SECONDS seconds elapsed: Still waiting for Ray Global Control Service(GCS) to be ready."
            fi

            sleep 5
          done

          echo "Attempting to connect to Ray cluster at $RAY_HEAD_ADDRESS ..."
          ray start --address="${RAY_HEAD_ADDRESS}" --block
        securityContext:
            {{- with .Values.kserve.servingruntime.huggingfaceserver_multinode.securityContext}}
            {{- toYaml . | nindent 10 }}
            {{- end }}          
        env:
          - name: POD_NAME
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
          - name: POD_NAMESPACE
            valueFrom:
              fieldRef:
                fieldPath: metadata.namespace
        resources:
          requests:
            cpu: "2"
            memory: 6Gi
          limits:
            cpu: "4"
            memory: 12Gi
        {{- if .Values.kserve.servingruntime.huggingfaceserver_multinode.shm.enabled }}
        volumeMounts:
          - name: shm
            mountPath: /dev/shm
        {{- end }}
        livenessProbe:
          failureThreshold: 2
          periodSeconds: 5
          successThreshold: 1
          timeoutSeconds: 15
          exec:
            command:
              - bash
              - -c
              - |
                export RAY_ADDRESS=${HEAD_SVC}.${POD_NAMESPACE}.svc.cluster.local:6379
                python ./huggingfaceserver/health_check.py registered_nodes --probe_name worker_liveness
        startupProbe:
          failureThreshold: 40
          periodSeconds: 30
          successThreshold: 1
          timeoutSeconds: 30
          initialDelaySeconds: 60
          exec:
            command:
              - bash
              - -c 
              - |
                export RAY_HEAD_NODE=${HEAD_SVC}.${POD_NAMESPACE}.svc.cluster.local
                export RAY_ADDRESS=${RAY_HEAD_NODE}:6379
                python ./huggingfaceserver/health_check.py registered_node_and_runtime_models --runtime_url http://${RAY_HEAD_NODE}:8080/v1/models --probe_name worker_startup    
    {{- if .Values.kserve.servingruntime.huggingfaceserver_multinode.shm.enabled }}
    volumes:
      - name: shm
        emptyDir:
          medium: Memory
          {{- with .Values.kserve.servingruntime.huggingfaceserver_multinode.shm.sizeLimit }}
          sizeLimit: {{ . }}
          {{- end }}
    {{- end }}
    {{- with .Values.kserve.servingruntime.huggingfaceserver_multinode.imagePullSecrets }}
    imagePullSecrets:
      {{- toYaml . | nindent 4 }}
    {{- end }}
{{- end }}