hanzo-ml/kserve
2
0
Fork 0
mirror of https://github.com/hanzo-ml/kserve.git synced 2026-10-30 20:53:49 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
master
kserve/charts/kserve-runtime-configs/files/llmisvcconfigs/resources.yaml
T
Raúl SevillaandGitHub 94963c327c fix: set limits and increase requests in llmisvc (#5637) 
Signed-off-by: Raul Sevilla <[email protected]>
2026-07-02 12:24:24 +08:00

3135 lines
132 KiB
YAML
Generated
Raw Permalink Blame History

apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-decode-template
namespace: kserve
spec:
annotations:
serving.kserve.io/model-based-routing-enabled: "true"
template:
containers:
- command:
- /bin/bash
- -c
- |-
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hca_port_pairs=()
for hca in "${active_hcas[@]}"; do
hca_port_pairs+=("${hca}:1")
done
active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
# --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
# Older versions still need the blanket --disable-uvicorn-access-log.
ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
fi
echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"
# --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
SHUTDOWN_TIMEOUT_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout {{ shutdownTimeout .Spec.Template 15 }}"
fi
# --kv-transfer-config with OffloadingConnector requires vLLM 0.22.0+ (vllm-project/vllm#40020).
KV_TRANSFER_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.22.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.22.0" ]; then
if [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv-transfer-config"* ]] && [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv_transfer_config"* ]] && [[ "$*" != *"--kv-transfer-config"* ]] && [[ "$*" != *"--kv_transfer_config"* ]]; then
KV_TRANSFER_ARGS="{{ kvTransferConfig .Spec.KVCacheOffloading }}"
fi
fi
eval "exec vllm serve /mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" "publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}" \
--port 8001 \
${ACCESS_LOG_ARGS} \
${SHUTDOWN_TIMEOUT_ARGS} \
${KV_TRANSFER_ARGS} \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.7.0
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /bin/sleep
- "15"
livenessProbe:
failureThreshold: 10
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 1
name: main
ports:
- containerPort: 8001
protocol: TCP
readinessProbe:
failureThreshold: 2
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 1
timeoutSeconds: 1
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
initContainers:
- command:
- /app/pd-sidecar
- --port=8000
- --vllm-port=8001
- --kv-connector=nixlv2
- --enable-ssrf-protection=true
- --pool-group=inference.networking.x-k8s.io
- --inference-pool={{ .GlobalConfig.InferencePoolNamespacedName }}
- '{{ if .GlobalConfig.EnableTLS }}--secure-proxy=true{{else}}--secure-proxy=false{{-
end }}'
- '{{ if .GlobalConfig.EnableTLS }}--cert-path=/var/run/kserve/tls{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--decoder-use-tls=true{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--prefiller-use-tls=true{{- end }}'
env:
- name: INFERENCE_POOL_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: SSL_CERT_DIR
value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
image: ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0-rc.2
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 10
name: llm-d-routing-sidecar
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 10
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
resources: {}
restartPolicy: Always
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 60
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-decode-worker-data-parallel
namespace: kserve
spec:
annotations:
serving.kserve.io/model-based-routing-enabled: "true"
template:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
# Retry DP_ADDRESS resolution (configurable attempts, default 30)
RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
if [ -n "$DP_ADDRESS" ]; then
echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
break
else
echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
sleep 1
fi
done
if [ -z "$DP_ADDRESS" ]; then
echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
DP_ADDRESS=${LWS_LEADER_ADDRESS}
echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
fi
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hca_port_pairs=()
for hca in "${active_hcas[@]}"; do
hca_port_pairs+=("${hca}:1")
done
active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=0
# --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
# Older versions still need the blanket --disable-uvicorn-access-log.
ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
fi
echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"
# --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
SHUTDOWN_TIMEOUT_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout {{ shutdownTimeout .Spec.Template 15 }}"
fi
# --kv-transfer-config with OffloadingConnector requires vLLM 0.22.0+ (vllm-project/vllm#40020).
KV_TRANSFER_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.22.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.22.0" ]; then
if [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv-transfer-config"* ]] && [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv_transfer_config"* ]] && [[ "$*" != *"--kv-transfer-config"* ]] && [[ "$*" != *"--kv_transfer_config"* ]]; then
KV_TRANSFER_ARGS="{{ kvTransferConfig .Spec.KVCacheOffloading }}"
fi
fi
eval "exec vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" "publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}" \
--port 8001 \
--api-server-count ${VLLM_API_SERVER_COUNT:-8} \
{{- if .Spec.Parallelism.Expert -}}--enable-expert-parallel{{- end }} \
{{- if .Spec.Parallelism.Tensor -}}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
${ACCESS_LOG_ARGS} \
${SHUTDOWN_TIMEOUT_ARGS} \
${KV_TRANSFER_ARGS} \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.7.0
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /bin/sleep
- "15"
livenessProbe:
failureThreshold: 10
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 1
name: main
ports:
- containerPort: 8001
protocol: TCP
readinessProbe:
failureThreshold: 2
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 1
timeoutSeconds: 1
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
initContainers:
- command:
- /app/pd-sidecar
- --port=8000
- --vllm-port=8001
- --kv-connector=nixlv2
- --enable-ssrf-protection=true
- --pool-group=inference.networking.x-k8s.io
- --inference-pool={{ .GlobalConfig.InferencePoolNamespacedName }}
- '{{ if .GlobalConfig.EnableTLS }}--secure-proxy=true{{else}}--secure-proxy=false{{-
end }}'
- '{{ if .GlobalConfig.EnableTLS }}--cert-path=/var/run/kserve/tls{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--decoder-use-tls=true{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--prefiller-use-tls=true{{- end }}'
env:
- name: INFERENCE_POOL_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: SSL_CERT_DIR
value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
image: ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0-rc.2
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 10
name: llm-d-routing-sidecar
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 10
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
restartPolicy: Always
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 60
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
worker:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
# Retry DP_ADDRESS resolution (configurable attempts, default 30)
RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
if [ -n "$DP_ADDRESS" ]; then
echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
break
else
echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
sleep 1
fi
done
if [ -z "$DP_ADDRESS" ]; then
echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
DP_ADDRESS=${LWS_LEADER_ADDRESS}
echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
fi
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hca_port_pairs=()
for hca in "${active_hcas[@]}"; do
hca_port_pairs+=("${hca}:1")
done
active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=$(( ${LWS_WORKER_INDEX:-0} * {{ or .Spec.Parallelism.DataLocal 1 }} ))
# --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
# Older versions still need the blanket --disable-uvicorn-access-log.
ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
fi
echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"
# --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
SHUTDOWN_TIMEOUT_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout {{ shutdownTimeout .Spec.Worker 15 }}"
fi
# --kv-transfer-config with OffloadingConnector requires vLLM 0.22.0+ (vllm-project/vllm#40020).
KV_TRANSFER_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.22.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.22.0" ]; then
if [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv-transfer-config"* ]] && [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv_transfer_config"* ]] && [[ "$*" != *"--kv-transfer-config"* ]] && [[ "$*" != *"--kv_transfer_config"* ]]; then
KV_TRANSFER_ARGS="{{ kvTransferConfig .Spec.KVCacheOffloading }}"
fi
fi
eval "exec vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" "publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}" \
--port 8001 \
{{- if .Spec.Parallelism.Expert }}--enable-expert-parallel{{- end }} \
{{- if .Spec.Parallelism.Tensor }}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--headless \
${ACCESS_LOG_ARGS} \
${SHUTDOWN_TIMEOUT_ARGS} \
${KV_TRANSFER_ARGS} \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
- name: VLLM_RANDOMIZE_DP_DUMMY_INPUTS
value: "1"
image: ghcr.io/llm-d/llm-d-cuda:v0.7.0
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /bin/sleep
- "15"
name: main
ports:
- containerPort: 8001
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 60
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-prefill-template
namespace: kserve
spec:
prefill:
annotations:
serving.kserve.io/model-based-routing-enabled: "true"
template:
containers:
- command:
- /bin/bash
- -c
- |-
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hca_port_pairs=()
for hca in "${active_hcas[@]}"; do
hca_port_pairs+=("${hca}:1")
done
active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
# --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
# Older versions still need the blanket --disable-uvicorn-access-log.
ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
fi
echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"
# --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
SHUTDOWN_TIMEOUT_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout {{ if .Spec.Prefill }}{{ shutdownTimeout .Spec.Prefill.Template 15 }}{{ else }}{{ shutdownTimeout nil 15 }}{{ end }}"
fi
# --kv-transfer-config with OffloadingConnector requires vLLM 0.22.0+ (vllm-project/vllm#40020).
KV_TRANSFER_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.22.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.22.0" ]; then
if [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv-transfer-config"* ]] && [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv_transfer_config"* ]] && [[ "$*" != *"--kv-transfer-config"* ]] && [[ "$*" != *"--kv_transfer_config"* ]]; then
KV_TRANSFER_ARGS="{{ if .Spec.Prefill }}{{ kvTransferConfig .Spec.Prefill.KVCacheOffloading }}{{ end }}"
fi
fi
eval "exec vllm serve /mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8000 \
${ACCESS_LOG_ARGS} \
${SHUTDOWN_TIMEOUT_ARGS} \
${KV_TRANSFER_ARGS} \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.7.0
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /bin/sleep
- "15"
livenessProbe:
failureThreshold: 10
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 1
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 2
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 1
timeoutSeconds: 1
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 60
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-prefill-worker-data-parallel
namespace: kserve
spec:
prefill:
annotations:
serving.kserve.io/model-based-routing-enabled: "true"
template:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
# Retry DP_ADDRESS resolution (configurable attempts, default 30)
RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
if [ -n "$DP_ADDRESS" ]; then
echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
break
else
echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
sleep 1
fi
done
if [ -z "$DP_ADDRESS" ]; then
echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
DP_ADDRESS=${LWS_LEADER_ADDRESS}
echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
fi
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hca_port_pairs=()
for hca in "${active_hcas[@]}"; do
hca_port_pairs+=("${hca}:1")
done
active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=0
# --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
# Older versions still need the blanket --disable-uvicorn-access-log.
ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
fi
echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"
# --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
SHUTDOWN_TIMEOUT_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout {{ if .Spec.Prefill }}{{ shutdownTimeout .Spec.Prefill.Template 15 }}{{ else }}{{ shutdownTimeout nil 15 }}{{ end }}"
fi
# --kv-transfer-config with OffloadingConnector requires vLLM 0.22.0+ (vllm-project/vllm#40020).
KV_TRANSFER_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.22.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.22.0" ]; then
if [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv-transfer-config"* ]] && [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv_transfer_config"* ]] && [[ "$*" != *"--kv-transfer-config"* ]] && [[ "$*" != *"--kv_transfer_config"* ]]; then
KV_TRANSFER_ARGS="{{ if .Spec.Prefill }}{{ kvTransferConfig .Spec.Prefill.KVCacheOffloading }}{{ end }}"
fi
fi
eval "exec vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" "publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}" \
--port 8000 \
--api-server-count ${VLLM_API_SERVER_COUNT:-8} \
{{- if .Spec.Prefill.Parallelism.Expert -}}--enable-expert-parallel{{- end }} \
{{- if .Spec.Prefill.Parallelism.Tensor -}}--tensor-parallel-size {{ .Spec.Prefill.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Prefill.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Prefill.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Prefill.Parallelism.DataRPCPort }}{{ .Spec.Prefill.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
${ACCESS_LOG_ARGS} \
${SHUTDOWN_TIMEOUT_ARGS} \
${KV_TRANSFER_ARGS} \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.7.0
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /bin/sleep
- "15"
livenessProbe:
failureThreshold: 10
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 1
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 2
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 1
timeoutSeconds: 1
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 60
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
worker:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
# Retry DP_ADDRESS resolution (configurable attempts, default 30)
RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
if [ -n "$DP_ADDRESS" ]; then
echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
break
else
echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
sleep 1
fi
done
if [ -z "$DP_ADDRESS" ]; then
echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
DP_ADDRESS=${LWS_LEADER_ADDRESS}
echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
fi
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hca_port_pairs=()
for hca in "${active_hcas[@]}"; do
hca_port_pairs+=("${hca}:1")
done
active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=$(( ${LWS_WORKER_INDEX:-0} * {{ or .Spec.Prefill.Parallelism.DataLocal 1 }} ))
# --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
# Older versions still need the blanket --disable-uvicorn-access-log.
ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
fi
echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"
# --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
SHUTDOWN_TIMEOUT_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout {{ if .Spec.Prefill }}{{ shutdownTimeout .Spec.Prefill.Worker 15 }}{{ else }}{{ shutdownTimeout nil 15 }}{{ end }}"
fi
# --kv-transfer-config with OffloadingConnector requires vLLM 0.22.0+ (vllm-project/vllm#40020).
KV_TRANSFER_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.22.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.22.0" ]; then
if [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv-transfer-config"* ]] && [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv_transfer_config"* ]] && [[ "$*" != *"--kv-transfer-config"* ]] && [[ "$*" != *"--kv_transfer_config"* ]]; then
KV_TRANSFER_ARGS="{{ if .Spec.Prefill }}{{ kvTransferConfig .Spec.Prefill.KVCacheOffloading }}{{ end }}"
fi
fi
eval "exec vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" "publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}" \
--port 8000 \
{{- if .Spec.Prefill.Parallelism.Expert }}--enable-expert-parallel{{- end }} \
{{- if .Spec.Prefill.Parallelism.Tensor }}--tensor-parallel-size {{ .Spec.Prefill.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Prefill.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Prefill.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Prefill.Parallelism.DataRPCPort }}{{ .Spec.Prefill.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--headless \
${ACCESS_LOG_ARGS} \
${SHUTDOWN_TIMEOUT_ARGS} \
${KV_TRANSFER_ARGS} \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.7.0
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /bin/sleep
- "15"
name: main
ports:
- containerPort: 8000
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 60
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-router-route
namespace: kserve
spec:
router:
route:
http:
spec:
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: '{{ .GlobalConfig.IngressGatewayName }}'
namespace: '{{ .GlobalConfig.IngressGatewayNamespace }}'
rules:
- backendRefs:
- group: inference.networking.k8s.io
kind: InferencePool
name: '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
port: 8000
weight: 1
filters:
- type: URLRewrite
urlRewrite:
path:
replacePrefixMatch: /v1/completions
type: ReplacePrefixMatch
matches:
- path:
type: PathPrefix
value: /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}/v1/completions
name: v1-completions-path
timeouts:
backendRequest: 0s
request: 0s
- backendRefs:
- group: inference.networking.k8s.io
kind: InferencePool
name: '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
port: 8000
weight: 1
matches:
- headers:
- name: '{{ .GlobalConfig.ModelBasedRoutingHeaderName }}'
type: Exact
value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name
}}
path:
type: Exact
value: /v1/completions
- headers:
- name: '{{ .GlobalConfig.ModelBasedRoutingHeaderName }}'
type: Exact
value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name
}}
path:
type: Exact
value: /v1/completions/
name: v1-completions-model-routing
timeouts:
backendRequest: 0s
request: 0s
- backendRefs:
- group: inference.networking.k8s.io
kind: InferencePool
name: '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
port: 8000
weight: 1
filters:
- type: URLRewrite
urlRewrite:
path:
replacePrefixMatch: /v1/chat/completions
type: ReplacePrefixMatch
matches:
- path:
type: PathPrefix
value: /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}/v1/chat/completions
name: v1-chat-completions-path
timeouts:
backendRequest: 0s
request: 0s
- backendRefs:
- group: inference.networking.k8s.io
kind: InferencePool
name: '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
port: 8000
weight: 1
matches:
- headers:
- name: '{{ .GlobalConfig.ModelBasedRoutingHeaderName }}'
type: Exact
value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name
}}
path:
type: Exact
value: /v1/chat/completions
- headers:
- name: '{{ .GlobalConfig.ModelBasedRoutingHeaderName }}'
type: Exact
value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name
}}
path:
type: Exact
value: /v1/chat/completions/
name: v1-chat-completions-model-routing
timeouts:
backendRequest: 0s
request: 0s
- backendRefs:
- group: inference.networking.k8s.io
kind: InferencePool
name: '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
port: 8000
weight: 1
filters:
- type: URLRewrite
urlRewrite:
path:
replacePrefixMatch: /v1/responses
type: ReplacePrefixMatch
matches:
- path:
type: PathPrefix
value: /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}/v1/responses
name: v1-responses-path
timeouts:
backendRequest: 0s
request: 0s
- backendRefs:
- group: inference.networking.k8s.io
kind: InferencePool
name: '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
port: 8000
weight: 1
matches:
- headers:
- name: '{{ .GlobalConfig.ModelBasedRoutingHeaderName }}'
type: Exact
value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name
}}
path:
type: Exact
value: /v1/responses
- headers:
- name: '{{ .GlobalConfig.ModelBasedRoutingHeaderName }}'
type: Exact
value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name
}}
path:
type: Exact
value: /v1/responses/
name: v1-responses-model-routing
timeouts:
backendRequest: 0s
request: 0s
- backendRefs:
- group: inference.networking.k8s.io
kind: InferencePool
name: '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
port: 8000
weight: 1
filters:
- type: URLRewrite
urlRewrite:
path:
replacePrefixMatch: /v1/messages
type: ReplacePrefixMatch
matches:
- path:
type: PathPrefix
value: /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}/v1/messages
name: v1-messages-path
timeouts:
backendRequest: 0s
request: 0s
- backendRefs:
- group: inference.networking.k8s.io
kind: InferencePool
name: '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
port: 8000
weight: 1
matches:
- headers:
- name: '{{ .GlobalConfig.ModelBasedRoutingHeaderName }}'
type: Exact
value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name
}}
path:
type: Exact
value: /v1/messages
- headers:
- name: '{{ .GlobalConfig.ModelBasedRoutingHeaderName }}'
type: Exact
value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name
}}
path:
type: Exact
value: /v1/messages/
name: v1-messages-model-routing
timeouts:
backendRequest: 0s
request: 0s
- backendRefs:
- kind: Service
name: '{{ ChildName .ObjectMeta.Name `-kserve-workload-svc` }}'
port: 8000
weight: 1
filters:
- type: URLRewrite
urlRewrite:
path:
replacePrefixMatch: /
type: ReplacePrefixMatch
matches:
- path:
type: PathPrefix
value: /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}
name: v1-catch-all-path
timeouts:
backendRequest: 0s
request: 0s
- backendRefs:
- kind: Service
name: '{{ ChildName .ObjectMeta.Name `-kserve-workload-svc` }}'
port: 8000
weight: 1
matches:
- headers:
- name: '{{ .GlobalConfig.ModelBasedRoutingHeaderName }}'
type: Exact
value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name
}}
name: v1-catch-all-model-routing
timeouts:
backendRequest: 0s
request: 0s
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-scheduler
namespace: kserve
spec:
router:
scheduler:
annotations:
app.kubernetes.io/version: 0.9.0
pool:
spec:
endpointPickerRef:
failureMode: FailOpen
kind: Service
name: '{{ ChildName .ObjectMeta.Name `-epp-service` }}'
port:
number: 9002
selector:
matchLabels:
app.kubernetes.io/name: '{{ .ObjectMeta.Name }}'
app.kubernetes.io/part-of: llminferenceservice
kserve.io/component: workload
targetPorts:
- number: 8000
template:
containers:
- command:
- /app/epp
- --pool-name
- '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
- --pool-namespace
- '{{ .ObjectMeta.Namespace }}'
- --zap-encoder
- json
- --grpc-port
- "9002"
- --grpc-health-port
- "9003"
- '{{ if .GlobalConfig.EnableTLS }}--enable-cert-reload=true{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--secure-serving=true{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--model-server-metrics-scheme=https{{-
end }}'
- '{{ if .GlobalConfig.EnableTLS }}--cert-path=/var/run/kserve/tls{{- end
}}'
env:
- name: SSL_CERT_DIR
value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
image: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0-rc.2
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /bin/sleep
- "15"
livenessProbe:
failureThreshold: 3
grpc:
port: 9003
service: liveness
initialDelaySeconds: 5
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: main
ports:
- containerPort: 9002
name: grpc
protocol: TCP
- containerPort: 9003
name: grpc-health
protocol: TCP
- containerPort: 9090
name: metrics
protocol: TCP
- containerPort: 5557
name: zmq
protocol: TCP
readinessProbe:
failureThreshold: 3
grpc:
port: 9003
service: readiness
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources:
limits:
cpu: 6
memory: 16Gi
requests:
cpu: 1
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
- mountPath: /tmp/tokenizer
name: tokenizer-uds
- env:
- name: TOKENIZERS_DIR
value: /mnt/models
image: ghcr.io/llm-d/llm-d-uds-tokenizer:vllm-v0.19.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 8082
periodSeconds: 15
timeoutSeconds: 5
name: tokenizer
ports:
- containerPort: 8082
name: health
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 8082
periodSeconds: 10
timeoutSeconds: 5
resources:
requests:
cpu: 256m
memory: 500Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /healthz
port: 8082
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /tmp
name: tokenizer-tmp
- mountPath: /.cache
name: tokenizer-cache
- mountPath: /tmp/tokenizer
name: tokenizer-uds
workingDir: /mnt/models
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 60
volumes:
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs`
}}'
- emptyDir: {}
name: tokenizer-uds
- emptyDir: {}
name: tokenizer-tmp
- emptyDir: {}
name: tokenizer-cache
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-scheduler-latency-predictor
namespace: kserve
spec:
router:
scheduler:
template:
containers:
- env:
- name: PREDICTION_SERVER_URL
value: http://localhost:8001
- name: TRAINING_SERVER_URL
value: http://localhost:8000
- name: LATENCY_MAX_SAMPLE_SIZE
value: "10000"
- name: LATENCY_MAX_CONCURRENT_DISPATCHES
value: "36"
- name: LATENCY_COALESCE_WINDOW_MS
value: "1"
name: main
- env:
- name: LATENCY_RETRAINING_INTERVAL_SEC
value: "10"
- name: LATENCY_MIN_SAMPLES_FOR_RETRAIN
value: "100"
- name: LATENCY_TTFT_MODEL_PATH
value: /models/ttft.joblib
- name: LATENCY_TPOT_MODEL_PATH
value: /models/tpot.joblib
- name: LATENCY_TTFT_SCALER_PATH
value: /models/ttft_scaler.joblib
- name: LATENCY_TPOT_SCALER_PATH
value: /models/tpot_scaler.joblib
- name: LATENCY_TTFT_GATED_MODEL_PATH
value: /models/ttft_gated.joblib
- name: LATENCY_TPOT_GATED_MODEL_PATH
value: /models/tpot_gated.joblib
- name: LATENCY_MODEL_TYPE
value: xgboost
- name: LATENCY_MAX_TRAINING_DATA_SIZE_PER_BUCKET
value: "500"
- name: LATENCY_OBJECTIVE_TYPE
value: mean
image: ghcr.io/llm-d/llm-d-latency-predictor-training-server:v0.8.0
imagePullPolicy: IfNotPresent
livenessProbe:
httpGet:
path: /healthz
port: 8000
initialDelaySeconds: 30
periodSeconds: 20
name: training-server
ports:
- containerPort: 8000
name: training-port
readinessProbe:
httpGet:
path: /readyz
port: 8000
initialDelaySeconds: 45
periodSeconds: 10
resources:
limits:
cpu: 4000m
memory: 8Gi
requests:
cpu: 2000m
memory: 4Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 30
httpGet:
path: /healthz
port: 8000
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /models
name: training-server-storage
- mountPath: /tmp
name: training-server-tmp
- env:
- name: TRAINING_SERVER_URL
value: http://localhost:8000
- name: LATENCY_MODEL_TYPE
value: xgboost
- name: PREDICT_HOST
value: 0.0.0.0
- name: PREDICT_PORT
value: "8001"
- name: LOCAL_TTFT_MODEL_PATH
value: /server_models/ttft.joblib
- name: LOCAL_TPOT_MODEL_PATH
value: /server_models/tpot.joblib
- name: LOCAL_TTFT_SCALER_PATH
value: /server_models/ttft_scaler.joblib
- name: LOCAL_TPOT_SCALER_PATH
value: /server_models/tpot_scaler.joblib
- name: LOCAL_TTFT_GATED_MODEL_PATH
value: /server_models/ttft_gated.joblib
- name: LOCAL_TPOT_GATED_MODEL_PATH
value: /server_models/tpot_gated.joblib
- name: UVICORN_WORKERS
value: "28"
- name: OMP_NUM_THREADS
value: "1"
- name: MODEL_SYNC_INTERVAL_SEC
value: "30"
- name: LATENCY_OBJECTIVE_TYPE
value: mean
image: ghcr.io/llm-d/llm-d-latency-predictor-prediction-server:v0.8.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: 8001
initialDelaySeconds: 15
periodSeconds: 15
timeoutSeconds: 5
name: prediction-server
ports:
- containerPort: 8001
name: predict-port
readinessProbe:
failureThreshold: 3
httpGet:
path: /readyz
port: 8001
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
resources:
limits:
cpu: 28000m
memory: 8Gi
requests:
cpu: 8000m
memory: 4Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /readyz
port: 8001
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /server_models
name: prediction-server-storage
- mountPath: /tmp
name: prediction-server-tmp
restartPolicy: Always
terminationGracePeriodSeconds: 60
volumes:
- emptyDir:
sizeLimit: 20Gi
name: training-server-storage
- emptyDir:
sizeLimit: 10Gi
name: prediction-server-storage
- emptyDir: {}
name: training-server-tmp
- emptyDir: {}
name: prediction-server-tmp
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-template
namespace: kserve
spec:
annotations:
serving.kserve.io/model-based-routing-enabled: "true"
template:
containers:
- command:
- /bin/bash
- -c
- |-
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hca_port_pairs=()
for hca in "${active_hcas[@]}"; do
hca_port_pairs+=("${hca}:1")
done
active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
# --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
# Older versions still need the blanket --disable-uvicorn-access-log.
ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
fi
echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"
# --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
SHUTDOWN_TIMEOUT_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout {{ shutdownTimeout .Spec.Template 15 }}"
fi
# --kv-transfer-config with OffloadingConnector requires vLLM 0.22.0+ (vllm-project/vllm#40020).
KV_TRANSFER_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.22.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.22.0" ]; then
if [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv-transfer-config"* ]] && [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv_transfer_config"* ]] && [[ "$*" != *"--kv-transfer-config"* ]] && [[ "$*" != *"--kv_transfer_config"* ]]; then
KV_TRANSFER_ARGS="{{ kvTransferConfig .Spec.KVCacheOffloading }}"
fi
fi
eval "exec vllm serve /mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" "publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}" \
--port 8000 \
${ACCESS_LOG_ARGS} \
${SHUTDOWN_TIMEOUT_ARGS} \
${KV_TRANSFER_ARGS} \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.7.0
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /bin/sleep
- "15"
livenessProbe:
failureThreshold: 10
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 1
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 2
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 1
timeoutSeconds: 1
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 60
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- emptyDir: {}
name: tmp-dir
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-tracing
namespace: kserve
spec:
tracing:
exporter: otlp
exporterEndpoint: http://otel-collector:4317
sampler: parentbased_traceidratio
samplerArg: "0.05"
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-worker-data-parallel
namespace: kserve
spec:
annotations:
serving.kserve.io/model-based-routing-enabled: "true"
template:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
# Retry DP_ADDRESS resolution (configurable attempts, default 30)
RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
if [ -n "$DP_ADDRESS" ]; then
echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
break
else
echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
sleep 1
fi
done
if [ -z "$DP_ADDRESS" ]; then
echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
DP_ADDRESS=${LWS_LEADER_ADDRESS}
echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
fi
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hca_port_pairs=()
for hca in "${active_hcas[@]}"; do
hca_port_pairs+=("${hca}:1")
done
active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=0
# --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
# Older versions still need the blanket --disable-uvicorn-access-log.
ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
fi
echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"
# --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
SHUTDOWN_TIMEOUT_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout {{ shutdownTimeout .Spec.Template 15 }}"
fi
# --kv-transfer-config with OffloadingConnector requires vLLM 0.22.0+ (vllm-project/vllm#40020).
KV_TRANSFER_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.22.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.22.0" ]; then
if [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv-transfer-config"* ]] && [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv_transfer_config"* ]] && [[ "$*" != *"--kv-transfer-config"* ]] && [[ "$*" != *"--kv_transfer_config"* ]]; then
KV_TRANSFER_ARGS="{{ kvTransferConfig .Spec.KVCacheOffloading }}"
fi
fi
eval "exec vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" "publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}" \
--port 8000 \
--api-server-count ${VLLM_API_SERVER_COUNT:-8} \
{{- if .Spec.Parallelism.Expert -}}--enable-expert-parallel{{- end }} \
{{- if .Spec.Parallelism.Tensor -}}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
${ACCESS_LOG_ARGS} \
${SHUTDOWN_TIMEOUT_ARGS} \
${KV_TRANSFER_ARGS} \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.7.0
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /bin/sleep
- "15"
livenessProbe:
failureThreshold: 10
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 1
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 2
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 1
timeoutSeconds: 1
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 60
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
worker:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
# Retry DP_ADDRESS resolution (configurable attempts, default 30)
RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
if [ -n "$DP_ADDRESS" ]; then
echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
break
else
echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
sleep 1
fi
done
if [ -z "$DP_ADDRESS" ]; then
echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
DP_ADDRESS=${LWS_LEADER_ADDRESS}
echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
fi
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hca_port_pairs=()
for hca in "${active_hcas[@]}"; do
hca_port_pairs+=("${hca}:1")
done
active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=$(( ${LWS_WORKER_INDEX:-0} * {{ or .Spec.Parallelism.DataLocal 1 }} ))
# --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
# Older versions still need the blanket --disable-uvicorn-access-log.
ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
fi
echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"
# --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
SHUTDOWN_TIMEOUT_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout {{ shutdownTimeout .Spec.Worker 15 }}"
fi
# --kv-transfer-config with OffloadingConnector requires vLLM 0.22.0+ (vllm-project/vllm#40020).
KV_TRANSFER_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.22.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.22.0" ]; then
if [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv-transfer-config"* ]] && [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv_transfer_config"* ]] && [[ "$*" != *"--kv-transfer-config"* ]] && [[ "$*" != *"--kv_transfer_config"* ]]; then
KV_TRANSFER_ARGS="{{ kvTransferConfig .Spec.KVCacheOffloading }}"
fi
fi
eval "exec vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" "publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}" \
--port 8000 \
{{- if .Spec.Parallelism.Expert }}--enable-expert-parallel{{- end }} \
{{- if .Spec.Parallelism.Tensor }}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--headless \
${ACCESS_LOG_ARGS} \
${SHUTDOWN_TIMEOUT_ARGS} \
${KV_TRANSFER_ARGS} \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.7.0
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /bin/sleep
- "15"
name: main
ports:
- containerPort: 8000
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 60
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
Reference in New Issue View Git Blame Copy Permalink