hanzo-ml/kserve
2
0
Fork 0
mirror of https://github.com/hanzo-ml/kserve.git synced 2026-10-30 20:53:49 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
release-0.18
kserve/charts/kserve-runtime-configs/files/llmisvcconfigs/resources.yaml
T

2453 lines
100 KiB
YAML
Generated
Raw Permalink Blame History

apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-decode-template
namespace: kserve
spec:
template:
containers:
- command:
- /bin/bash
- -c
- |-
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
eval "vllm serve /mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8001 \
--disable-uvicorn-access-log \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.6.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8001
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
initContainers:
- command:
- /app/pd-sidecar
- --port=8000
- --vllm-port=8001
- --connector=nixlv2
- --enable-ssrf-protection=true
- --pool-group=inference.networking.x-k8s.io
- '{{ if .GlobalConfig.EnableTLS }}--secure-proxy=true{{else}}--secure-proxy=false{{-
end }}'
- '{{ if .GlobalConfig.EnableTLS }}--cert-path=/var/run/kserve/tls{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--decoder-use-tls=true{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--prefiller-use-tls=true{{- end }}'
env:
- name: INFERENCE_POOL_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: SSL_CERT_DIR
value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
image: ghcr.io/llm-d/llm-d-routing-sidecar:v0.7.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 10
name: llm-d-routing-sidecar
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 10
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
resources: {}
restartPolicy: Always
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-decode-worker-data-parallel
namespace: kserve
spec:
template:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
# Retry DP_ADDRESS resolution (configurable attempts, default 30)
RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
if [ -n "$DP_ADDRESS" ]; then
echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
break
else
echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
sleep 1
fi
done
if [ -z "$DP_ADDRESS" ]; then
echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
DP_ADDRESS=${LWS_LEADER_ADDRESS}
echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
fi
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=0
eval "vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8001 \
--api-server-count ${VLLM_API_SERVER_COUNT:-8} \
{{- if .Spec.Parallelism.Expert -}}--enable-expert-parallel{{- end }} \
{{- if .Spec.Parallelism.Tensor -}}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--disable-uvicorn-access-log \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.6.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8001
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 30
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
initContainers:
- command:
- /app/pd-sidecar
- --port=8000
- --vllm-port=8001
- --connector=nixlv2
- --enable-ssrf-protection=true
- --pool-group=inference.networking.x-k8s.io
- '{{ if .GlobalConfig.EnableTLS }}--secure-proxy=true{{else}}--secure-proxy=false{{-
end }}'
- '{{ if .GlobalConfig.EnableTLS }}--cert-path=/var/run/kserve/tls{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--decoder-use-tls=true{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--prefiller-use-tls=true{{- end }}'
env:
- name: INFERENCE_POOL_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: SSL_CERT_DIR
value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
image: ghcr.io/llm-d/llm-d-routing-sidecar:v0.7.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 10
name: llm-d-routing-sidecar
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 10
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
restartPolicy: Always
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
worker:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
# Retry DP_ADDRESS resolution (configurable attempts, default 30)
RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
if [ -n "$DP_ADDRESS" ]; then
echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
break
else
echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
sleep 1
fi
done
if [ -z "$DP_ADDRESS" ]; then
echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
DP_ADDRESS=${LWS_LEADER_ADDRESS}
echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
fi
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=$(( ${LWS_WORKER_INDEX:-0} * {{ or .Spec.Parallelism.DataLocal 1 }} ))
eval "vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8001 \
{{- if .Spec.Parallelism.Expert }}--enable-expert-parallel{{- end }} \
{{- if .Spec.Parallelism.Tensor }}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--headless \
--disable-uvicorn-access-log \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
- name: VLLM_RANDOMIZE_DP_DUMMY_INPUTS
value: "1"
image: ghcr.io/llm-d/llm-d-cuda:v0.6.0
imagePullPolicy: IfNotPresent
name: main
ports:
- containerPort: 8001
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-prefill-template
namespace: kserve
spec:
prefill:
template:
containers:
- command:
- /bin/bash
- -c
- |-
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
eval "vllm serve /mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8000 \
--disable-uvicorn-access-log \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.6.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-prefill-worker-data-parallel
namespace: kserve
spec:
prefill:
template:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
# Retry DP_ADDRESS resolution (configurable attempts, default 30)
RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
if [ -n "$DP_ADDRESS" ]; then
echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
break
else
echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
sleep 1
fi
done
if [ -z "$DP_ADDRESS" ]; then
echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
DP_ADDRESS=${LWS_LEADER_ADDRESS}
echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
fi
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=0
eval "vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8000 \
--api-server-count ${VLLM_API_SERVER_COUNT:-8} \
{{- if .Spec.Prefill.Parallelism.Expert -}}--enable-expert-parallel{{- end }} \
{{- if .Spec.Prefill.Parallelism.Tensor -}}--tensor-parallel-size {{ .Spec.Prefill.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Prefill.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Prefill.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Prefill.Parallelism.DataRPCPort }}{{ .Spec.Prefill.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--disable-uvicorn-access-log \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.6.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 30
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
worker:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
# Retry DP_ADDRESS resolution (configurable attempts, default 30)
RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
if [ -n "$DP_ADDRESS" ]; then
echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
break
else
echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
sleep 1
fi
done
if [ -z "$DP_ADDRESS" ]; then
echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
DP_ADDRESS=${LWS_LEADER_ADDRESS}
echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
fi
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=$(( ${LWS_WORKER_INDEX:-0} * {{ or .Spec.Prefill.Parallelism.DataLocal 1 }} ))
eval "vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8000 \
{{- if .Spec.Prefill.Parallelism.Expert }}--enable-expert-parallel{{- end }} \
{{- if .Spec.Prefill.Parallelism.Tensor }}--tensor-parallel-size {{ .Spec.Prefill.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Prefill.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Prefill.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Prefill.Parallelism.DataRPCPort }}{{ .Spec.Prefill.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--headless \
--disable-uvicorn-access-log \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.6.0
imagePullPolicy: IfNotPresent
name: main
ports:
- containerPort: 8000
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-router-route
namespace: kserve
spec:
router:
route:
http:
spec:
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: '{{ .GlobalConfig.IngressGatewayName }}'
namespace: '{{ .GlobalConfig.IngressGatewayNamespace }}'
rules:
- backendRefs:
- group: inference.networking.k8s.io
kind: InferencePool
name: '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
port: 8000
weight: 1
filters:
- type: URLRewrite
urlRewrite:
path:
replacePrefixMatch: /v1/completions
type: ReplacePrefixMatch
matches:
- path:
type: PathPrefix
value: /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}/v1/completions
timeouts:
backendRequest: 0s
request: 0s
- backendRefs:
- group: inference.networking.k8s.io
kind: InferencePool
name: '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
port: 8000
weight: 1
filters:
- type: URLRewrite
urlRewrite:
path:
replacePrefixMatch: /v1/chat/completions
type: ReplacePrefixMatch
matches:
- path:
type: PathPrefix
value: /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}/v1/chat/completions
timeouts:
backendRequest: 0s
request: 0s
- backendRefs:
- group: inference.networking.k8s.io
kind: InferencePool
name: '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
port: 8000
weight: 1
filters:
- type: URLRewrite
urlRewrite:
path:
replacePrefixMatch: /v1/responses
type: ReplacePrefixMatch
matches:
- path:
type: PathPrefix
value: /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}/v1/responses
timeouts:
backendRequest: 0s
request: 0s
- backendRefs:
- kind: Service
name: '{{ ChildName .ObjectMeta.Name `-kserve-workload-svc` }}'
port: 8000
weight: 1
filters:
- type: URLRewrite
urlRewrite:
path:
replacePrefixMatch: /
type: ReplacePrefixMatch
matches:
- path:
type: PathPrefix
value: /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}
timeouts:
backendRequest: 0s
request: 0s
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-scheduler
namespace: kserve
spec:
router:
scheduler:
pool:
spec:
endpointPickerRef:
failureMode: FailOpen
kind: Service
name: '{{ ChildName .ObjectMeta.Name `-epp-service` }}'
port:
number: 9002
selector:
matchLabels:
app.kubernetes.io/name: '{{ .ObjectMeta.Name }}'
app.kubernetes.io/part-of: llminferenceservice
kserve.io/component: workload
targetPorts:
- number: 8000
template:
containers:
- command:
- /app/epp
- --pool-name
- '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
- --pool-namespace
- '{{ .ObjectMeta.Namespace }}'
- --zap-encoder
- json
- --grpc-port
- "9002"
- --grpc-health-port
- "9003"
- '{{ if .GlobalConfig.EnableTLS }}--enable-cert-reload=true{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--secure-serving=true{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--model-server-metrics-scheme=https{{-
end }}'
- '{{ if .GlobalConfig.EnableTLS }}--cert-path=/var/run/kserve/tls{{- end
}}'
env:
- name: SSL_CERT_DIR
value: /var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs
image: ghcr.io/llm-d/llm-d-inference-scheduler:v0.7.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
grpc:
port: 9003
service: liveness
initialDelaySeconds: 5
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: main
ports:
- containerPort: 9002
name: grpc
protocol: TCP
- containerPort: 9003
name: grpc-health
protocol: TCP
- containerPort: 9090
name: metrics
protocol: TCP
- containerPort: 5557
name: zmq
protocol: TCP
readinessProbe:
failureThreshold: 3
grpc:
port: 9003
service: readiness
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources:
requests:
cpu: 256m
memory: 500Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
- mountPath: /tmp/tokenizer
name: tokenizer-uds
- env:
- name: TOKENIZERS_DIR
value: /mnt/models
image: ghcr.io/llm-d/llm-d-uds-tokenizer:v0.7.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 8082
periodSeconds: 15
timeoutSeconds: 5
name: tokenizer
ports:
- containerPort: 8082
name: health
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 8082
periodSeconds: 10
timeoutSeconds: 5
resources:
requests:
cpu: 256m
memory: 500Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /healthz
port: 8082
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /tmp
name: tokenizer-tmp
- mountPath: /.cache
name: tokenizer-cache
- mountPath: /tmp/tokenizer
name: tokenizer-uds
workingDir: /mnt/models
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 30
volumes:
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs`
}}'
- emptyDir: {}
name: tokenizer-uds
- emptyDir: {}
name: tokenizer-tmp
- emptyDir: {}
name: tokenizer-cache
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-template
namespace: kserve
spec:
template:
containers:
- command:
- /bin/bash
- -c
- |-
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
eval "vllm serve /mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8000 \
--disable-uvicorn-access-log \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.6.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- emptyDir: {}
name: tmp-dir
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-worker-data-parallel
namespace: kserve
spec:
template:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
# Retry DP_ADDRESS resolution (configurable attempts, default 30)
RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
if [ -n "$DP_ADDRESS" ]; then
echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
break
else
echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
sleep 1
fi
done
if [ -z "$DP_ADDRESS" ]; then
echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
DP_ADDRESS=${LWS_LEADER_ADDRESS}
echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
fi
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=0
eval "vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8000 \
--api-server-count ${VLLM_API_SERVER_COUNT:-8} \
{{- if .Spec.Parallelism.Expert -}}--enable-expert-parallel{{- end }} \
{{- if .Spec.Parallelism.Tensor -}}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--disable-uvicorn-access-log \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.6.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 30
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
worker:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
# Retry DP_ADDRESS resolution (configurable attempts, default 30)
RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
if [ -n "$DP_ADDRESS" ]; then
echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
break
else
echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
sleep 1
fi
done
if [ -z "$DP_ADDRESS" ]; then
echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
DP_ADDRESS=${LWS_LEADER_ADDRESS}
echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
fi
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${ucx_hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=$(( ${LWS_WORKER_INDEX:-0} * {{ or .Spec.Parallelism.DataLocal 1 }} ))
eval "vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8000 \
{{- if .Spec.Parallelism.Expert }}--enable-expert-parallel{{- end }} \
{{- if .Spec.Parallelism.Tensor }}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--headless \
--disable-uvicorn-access-log \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.6.0
imagePullPolicy: IfNotPresent
name: main
ports:
- containerPort: 8000
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir: {}
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
Reference in New Issue View Git Blame Copy Permalink