kserve/config/llmisvcconfig/config-llm-decode-worker-data-parallel.yaml

apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
  name: kserve-config-llm-decode-worker-data-parallel
spec:
  template:
    initContainers:
      - name: llm-d-routing-sidecar
        imagePullPolicy: IfNotPresent
        image: ghcr.io/llm-d/llm-d-routing-sidecar:v0.7.1
        restartPolicy: Always
        ports:
          - containerPort: 8000
            protocol: TCP
        securityContext:
          allowPrivilegeEscalation: false
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          capabilities:
            drop:
              - ALL
          seccompProfile:
            type: RuntimeDefault
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: FallbackToLogsOnError
        livenessProbe:
          httpGet:
            path: /health
            port: 8000
            scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
          initialDelaySeconds: 10
          periodSeconds: 10
          timeoutSeconds: 10
          failureThreshold: 3
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
            scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
          initialDelaySeconds: 10
          periodSeconds: 10
          timeoutSeconds: 5
          failureThreshold: 10
        command:
          - "/app/pd-sidecar"
          - "--port=8000"
          - "--vllm-port=8001"
          - "--connector=nixlv2"
          - "--enable-ssrf-protection=true"
          - "--pool-group=inference.networking.x-k8s.io"
          - '{{ if .GlobalConfig.EnableTLS }}--secure-proxy=true{{else}}--secure-proxy=false{{- end }}'
          - '{{ if .GlobalConfig.EnableTLS }}--cert-path=/var/run/kserve/tls{{- end }}'
          - '{{ if .GlobalConfig.EnableTLS }}--decoder-use-tls=true{{- end }}'
          - '{{ if .GlobalConfig.EnableTLS }}--prefiller-use-tls=true{{- end }}'
        volumeMounts:
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
        env:
          - name: INFERENCE_POOL_NAMESPACE
            valueFrom:
              fieldRef:
                fieldPath: metadata.namespace
          - name: SSL_CERT_DIR
            value: "/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs"
    containers:
      - image: ghcr.io/llm-d/llm-d-cuda:v0.6.0
        imagePullPolicy: IfNotPresent
        name: main
        ports:
          - containerPort: 8001
            protocol: TCP
        command:
          - "/bin/bash"
          - "-c"
          - |-
            # In some versions, ZMQ bind doesn't resolve the address through DNS
            # Retry DP_ADDRESS resolution (configurable attempts, default 30)
            RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
            for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
              DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
              if [ -n "$DP_ADDRESS" ]; then
                echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
                break
              else
                echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
                sleep 1
              fi
            done

            if [ -z "$DP_ADDRESS" ]; then
              echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
              DP_ADDRESS=${LWS_LEADER_ADDRESS}
              echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
            fi

            if [ "$KSERVE_INFER_ROCE" = "true" ]; then
              echo "Trying to infer RoCE configs ... "
              grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
              grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null

              cat /proc/driver/nvidia/params

              KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}

              echo "[Infer RoCE] Discovering active HCAs ..."
              active_hcas=()
              # Loop through all mlx5 devices found in sysfs
              for hca_dir in /sys/class/infiniband/mlx5_*; do
                  # Ensure it's a directory before proceeding
                  if [ -d "$hca_dir" ]; then
                      hca_name=$(basename "$hca_dir")
                      port_state_file="$hca_dir/ports/1/state" # Assume port 1
                      type_file="$hca_dir/ports/1/gid_attrs/types/*"

                      echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
                      if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
                          echo "[Infer RoCE] Found active HCA: $hca_name"
                          active_hcas+=("$hca_name")
                      else
                          echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
                      fi
                  fi
              done

              ucx_hcas=()
              for hca in "${active_hcas[@]}"; do
                ucx_hcas+=("${hca}:1")
              done

              # Check if we found any active HCAs
              if [ ${#active_hcas[@]} -gt 0 ]; then
                  # Join the array elements with a comma
                  hcas=$(IFS=,; echo "${active_hcas[*]}")
                  echo "[Infer RoCE] Setting active HCAs: ${hcas}"
                  export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
                  export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}
                  export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}

                  echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
                  echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
              else
                  echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
              fi

              if [ ${#active_hcas[@]} -gt 0 ]; then
                  echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."

                  # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
                  declare -A gid_index_count
                  declare -A hca_gid_index

                  for hca_name in "${active_hcas[@]}"; do
                      echo "[Infer RoCE] Processing HCA: ${hca_name}"

                      # Find all RoCE v2 IPv4 GIDs for this HCA and count by index
                      for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
                          if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
                              idx=$(basename "$tpath")
                              gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
                              # Check for IPv4 GID (contains ffff:)
                              if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
                                  gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
                                  echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
                                  hca_gid_index["${hca_name}"]="${idx}"
                                  gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
                                  break  # Use first found IPv4 GID per HCA
                              fi
                          fi
                      done
                  done

                  # Find the most common GID index (most likely to be consistent across nodes)
                  best_gid_index=""
                  max_count=0
                  for idx in "${!gid_index_count[@]}"; do
                      count=${gid_index_count["${idx}"]}
                      echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
                      if [ $count -gt $max_count ]; then
                          max_count=$count
                          best_gid_index="$idx"
                      fi
                  done

                  # Use deterministic fallback if counts are equal - prefer lower index number
                  if [ ${#gid_index_count[@]} -gt 1 ]; then
                      echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
                      # If there's a tie, prefer index 3 as it's most common in SR-IOV setups
                      if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
                          best_gid_index="3"
                          echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
                      fi
                  fi

                  # Check if GID_INDEX is already set via environment variables
                  if [ -n "${NCCL_IB_GID_INDEX}" ]; then
                      echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
                  elif [ -n "$best_gid_index" ]; then
                      echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"

                      export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}

                      echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
                  else
                      echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
                  fi
              else
                  echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
              fi
            fi

            START_RANK=0
            eval "vllm serve \
              /mnt/models \
              --served-model-name "{{ .Spec.Model.Name }}" \
              --port 8001 \
              --api-server-count ${VLLM_API_SERVER_COUNT:-8} \
              {{- if .Spec.Parallelism.Expert -}}--enable-expert-parallel{{- end }} \
              {{- if .Spec.Parallelism.Tensor -}}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
              --data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
              --data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
              --data-parallel-address ${DP_ADDRESS} \
              --data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
              --data-parallel-start-rank $START_RANK \
              --disable-uvicorn-access-log \
              {{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
              {{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
              {{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
              ${VLLM_ADDITIONAL_ARGS} \
              $@"
          # "--" ends bash options; subsequent args become positional params ($@) forwarded to vllm serve
          - "--"

        env:
          - name: HOME
            value: /home
          - name: VLLM_LOGGING_LEVEL
            value: INFO
          - name: HF_HUB_CACHE
            value: /models
        securityContext:
          allowPrivilegeEscalation: false
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          capabilities:
            add:
              - "IPC_LOCK"
              - "SYS_RAWIO"
              - "NET_RAW"
            drop:
              - ALL
          seccompProfile:
            type: RuntimeDefault
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: FallbackToLogsOnError
        livenessProbe:
          httpGet:
            path: /health
            port: 8001
            scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
          periodSeconds: 10
          timeoutSeconds: 10
          failureThreshold: 3
        readinessProbe:
          httpGet:
            path: /health
            port: 8001
            scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
          periodSeconds: 30
          timeoutSeconds: 5
          failureThreshold: 60
        startupProbe:
          httpGet:
            path: /health
            port: 8001
            scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
          failureThreshold: 60
          periodSeconds: 10
        volumeMounts:
          - mountPath: /home
            name: home
          - mountPath: /tmp
            name: tmp-dir
          - mountPath: /dev/shm
            name: dshm
          - mountPath: /models
            name: model-cache
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
    terminationGracePeriodSeconds: 30
    volumes:
      - emptyDir: { }
        name: home
      - emptyDir: { }
        name: tmp-dir
      - emptyDir:
          medium: Memory
          sizeLimit: 8Gi
        name: dshm
      - emptyDir: { }
        name: model-cache
      - name: tls-certs
        secret:
          secretName: "{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}"
  worker:
    containers:
      - image: ghcr.io/llm-d/llm-d-cuda:v0.6.0
        imagePullPolicy: IfNotPresent
        name: main
        ports:
          - containerPort: 8001
            protocol: TCP
        command:
          - "/bin/bash"
          - "-c"
          - |-
            # In some versions, ZMQ bind doesn't resolve the address through DNS
            # Retry DP_ADDRESS resolution (configurable attempts, default 30)
            RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
            for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
              DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
              if [ -n "$DP_ADDRESS" ]; then
                echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
                break
              else
                echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
                sleep 1
              fi
            done

            if [ -z "$DP_ADDRESS" ]; then
              echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
              DP_ADDRESS=${LWS_LEADER_ADDRESS}
              echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
            fi

            if [ "$KSERVE_INFER_ROCE" = "true" ]; then
              echo "Trying to infer RoCE configs ... "
              grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
              grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null

              cat /proc/driver/nvidia/params

              KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}

              echo "[Infer RoCE] Discovering active HCAs ..."
              active_hcas=()
              # Loop through all mlx5 devices found in sysfs
              for hca_dir in /sys/class/infiniband/mlx5_*; do
                  # Ensure it's a directory before proceeding
                  if [ -d "$hca_dir" ]; then
                      hca_name=$(basename "$hca_dir")
                      port_state_file="$hca_dir/ports/1/state" # Assume port 1
                      type_file="$hca_dir/ports/1/gid_attrs/types/*"

                      echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
                      if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
                          echo "[Infer RoCE] Found active HCA: $hca_name"
                          active_hcas+=("$hca_name")
                      else
                          echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
                      fi
                  fi
              done

              ucx_hcas=()
              for hca in "${active_hcas[@]}"; do
                ucx_hcas+=("${hca}:1")
              done

              # Check if we found any active HCAs
              if [ ${#active_hcas[@]} -gt 0 ]; then
                  # Join the array elements with a comma
                  hcas=$(IFS=,; echo "${active_hcas[*]}")
                  echo "[Infer RoCE] Setting active HCAs: ${hcas}"
                  export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
                  export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}
                  export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}

                  echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
                  echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
              else
                  echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
              fi

              if [ ${#active_hcas[@]} -gt 0 ]; then
                  echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."

                  # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
                  declare -A gid_index_count
                  declare -A hca_gid_index

                  for hca_name in "${active_hcas[@]}"; do
                      echo "[Infer RoCE] Processing HCA: ${hca_name}"

                      # Find all RoCE v2 IPv4 GIDs for this HCA and count by index
                      for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
                          if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
                              idx=$(basename "$tpath")
                              gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
                              # Check for IPv4 GID (contains ffff:)
                              if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
                                  gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
                                  echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
                                  hca_gid_index["${hca_name}"]="${idx}"
                                  gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
                                  break  # Use first found IPv4 GID per HCA
                              fi
                          fi
                      done
                  done

                  # Find the most common GID index (most likely to be consistent across nodes)
                  best_gid_index=""
                  max_count=0
                  for idx in "${!gid_index_count[@]}"; do
                      count=${gid_index_count["${idx}"]}
                      echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
                      if [ $count -gt $max_count ]; then
                          max_count=$count
                          best_gid_index="$idx"
                      fi
                  done

                  # Use deterministic fallback if counts are equal - prefer lower index number
                  if [ ${#gid_index_count[@]} -gt 1 ]; then
                      echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
                      # If there's a tie, prefer index 3 as it's most common in SR-IOV setups
                      if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
                          best_gid_index="3"
                          echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
                      fi
                  fi

                  # Check if GID_INDEX is already set via environment variables
                  if [ -n "${NCCL_IB_GID_INDEX}" ]; then
                      echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
                      echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
                  elif [ -n "$best_gid_index" ]; then
                      echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"

                      export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
                      export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
                      export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}

                      echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
                  else
                      echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
                  fi
              else
                  echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
              fi
            fi

            START_RANK=$(( ${LWS_WORKER_INDEX:-0} * {{ or .Spec.Parallelism.DataLocal 1 }} ))
            eval "vllm serve \
              /mnt/models \
              --served-model-name "{{ .Spec.Model.Name }}" \
              --port 8001 \
              {{- if .Spec.Parallelism.Expert }}--enable-expert-parallel{{- end }} \
              {{- if .Spec.Parallelism.Tensor }}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
              --data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
              --data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
              --data-parallel-address ${DP_ADDRESS} \
              --data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
              --data-parallel-start-rank $START_RANK \
              --headless \
              --disable-uvicorn-access-log \
              {{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
              {{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
              {{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
              ${VLLM_ADDITIONAL_ARGS} \
              $@"
          # "--" ends bash options; subsequent args become positional params ($@) forwarded to vllm serve
          - "--"

        env:
          - name: HOME
            value: /home
          - name: VLLM_LOGGING_LEVEL
            value: INFO
          - name: HF_HUB_CACHE
            value: /models
          - name: VLLM_RANDOMIZE_DP_DUMMY_INPUTS
            value: "1"
        securityContext:
          allowPrivilegeEscalation: false
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          capabilities:
            add:
              - "IPC_LOCK"
              - "SYS_RAWIO"
              - "NET_RAW"
            drop:
              - ALL
          seccompProfile:
            type: RuntimeDefault
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: FallbackToLogsOnError
        volumeMounts:
          - mountPath: /home
            name: home
          - mountPath: /tmp
            name: tmp-dir
          - mountPath: /dev/shm
            name: dshm
          - mountPath: /models
            name: model-cache
          - mountPath: /var/run/kserve/tls
            name: tls-certs
            readOnly: true
    terminationGracePeriodSeconds: 30
    volumes:
      - emptyDir: { }
        name: home
      - emptyDir: { }
        name: tmp-dir
      - emptyDir:
          medium: Memory
          sizeLimit: 8Gi
        name: dshm
      - emptyDir: { }
        name: model-cache
      - name: tls-certs
        secret:
          secretName: "{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}"