hanzo-ml/kserve
2
0
Fork 0
mirror of https://github.com/hanzo-ml/kserve.git synced 2026-10-30 20:53:49 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
master
kserve/config/llmisvcconfig/config-llm-decode-worker-data-parallel.yaml
T
Andres LlausasGitHubClaude Sonnet 4.6
 15a92ece5c feat(llmisvc): add kvCacheOffloading spec for CPU KV cache tiering (#5599) 
Signed-off-by: Andres Llausas <[email protected]>
Co-authored-by: Claude Sonnet 4.6 <[email protected]>
2026-06-24 09:21:34 -07:00

612 lines
29 KiB
YAML
Raw Permalink Blame History

apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-decode-worker-data-parallel
spec:
annotations:
serving.kserve.io/model-based-routing-enabled: "true"
template:
initContainers:
- name: llm-d-routing-sidecar
imagePullPolicy: IfNotPresent
image: ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0-rc.2
restartPolicy: Always
ports:
- containerPort: 8000
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
# nvidia-cdi-hook requires a writable root filesystem
readOnlyRootFilesystem: false
runAsNonRoot: true
capabilities:
drop:
- ALL
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
livenessProbe:
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 10
failureThreshold: 3
readinessProbe:
httpGet:
path: /health
port: 8000
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 10
command:
- "/app/pd-sidecar"
- "--port=8000"
- "--vllm-port=8001"
- "--kv-connector=nixlv2"
- "--enable-ssrf-protection=true"
- "--pool-group=inference.networking.x-k8s.io"
- "--inference-pool={{ .GlobalConfig.InferencePoolNamespacedName }}"
- '{{ if .GlobalConfig.EnableTLS }}--secure-proxy=true{{else}}--secure-proxy=false{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--cert-path=/var/run/kserve/tls{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--decoder-use-tls=true{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--prefiller-use-tls=true{{- end }}'
volumeMounts:
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
env:
- name: INFERENCE_POOL_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: SSL_CERT_DIR
value: "/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs"
containers:
- image: ghcr.io/llm-d/llm-d-cuda:v0.7.0
imagePullPolicy: IfNotPresent
name: main
ports:
- containerPort: 8001
protocol: TCP
command:
- "/bin/bash"
- "-c"
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
# Retry DP_ADDRESS resolution (configurable attempts, default 30)
RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
if [ -n "$DP_ADDRESS" ]; then
echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
break
else
echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
sleep 1
fi
done
if [ -z "$DP_ADDRESS" ]; then
echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
DP_ADDRESS=${LWS_LEADER_ADDRESS}
echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
fi
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hca_port_pairs=()
for hca in "${active_hcas[@]}"; do
hca_port_pairs+=("${hca}:1")
done
active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=0
# --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
# Older versions still need the blanket --disable-uvicorn-access-log.
ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
fi
echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"
# --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
SHUTDOWN_TIMEOUT_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout {{ shutdownTimeout .Spec.Template 15 }}"
fi
# --kv-transfer-config with OffloadingConnector requires vLLM 0.22.0+ (vllm-project/vllm#40020).
KV_TRANSFER_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.22.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.22.0" ]; then
if [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv-transfer-config"* ]] && [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv_transfer_config"* ]] && [[ "$*" != *"--kv-transfer-config"* ]] && [[ "$*" != *"--kv_transfer_config"* ]]; then
KV_TRANSFER_ARGS="{{ kvTransferConfig .Spec.KVCacheOffloading }}"
fi
fi
eval "exec vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" "publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}" \
--port 8001 \
--api-server-count ${VLLM_API_SERVER_COUNT:-8} \
{{- if .Spec.Parallelism.Expert -}}--enable-expert-parallel{{- end }} \
{{- if .Spec.Parallelism.Tensor -}}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
${ACCESS_LOG_ARGS} \
${SHUTDOWN_TIMEOUT_ARGS} \
${KV_TRANSFER_ARGS} \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
# "--" ends bash options; subsequent args become positional params ($@) forwarded to vllm serve
- "--"
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
securityContext:
allowPrivilegeEscalation: false
# nvidia-cdi-hook requires a writable root filesystem
readOnlyRootFilesystem: false
runAsNonRoot: true
capabilities:
add:
- "IPC_LOCK"
- "SYS_RAWIO"
- "NET_RAW"
drop:
- ALL
seccompProfile:
type: RuntimeDefault
lifecycle:
preStop:
exec:
command:
- /bin/sleep
- "15"
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
livenessProbe:
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 10
timeoutSeconds: 1
failureThreshold: 10
readinessProbe:
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
periodSeconds: 1
timeoutSeconds: 1
failureThreshold: 2
startupProbe:
httpGet:
path: /health
port: 8001
scheme: '{{ if .GlobalConfig.EnableTLS }}HTTPS{{else}}HTTP{{- end }}'
failureThreshold: 60
periodSeconds: 10
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 60
volumes:
- emptyDir: { }
name: home
- emptyDir: { }
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: { }
name: model-cache
- name: tls-certs
secret:
secretName: "{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}"
worker:
containers:
- image: ghcr.io/llm-d/llm-d-cuda:v0.7.0
imagePullPolicy: IfNotPresent
name: main
ports:
- containerPort: 8001
protocol: TCP
command:
- "/bin/bash"
- "-c"
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
# Retry DP_ADDRESS resolution (configurable attempts, default 30)
RESOLVE_ATTEMPTS=${DP_ADDRESS_RESOLVE_ATTEMPTS:-30}
for ((i=1; i<=RESOLVE_ATTEMPTS; i++)); do
DP_ADDRESS=$(getent hosts ${LWS_LEADER_ADDRESS} | cut -d' ' -f1)
if [ -n "$DP_ADDRESS" ]; then
echo "DP_ADDRESS=${DP_ADDRESS} (resolved on attempt $i)"
break
else
echo "DP_ADDRESS resolution failed on attempt $i, retrying..."
sleep 1
fi
done
if [ -z "$DP_ADDRESS" ]; then
echo "WARNING: Failed to resolve DP_ADDRESS after ${RESOLVE_ATTEMPTS} attempts, falling back to LWS_LEADER_ADDRESS"
DP_ADDRESS=${LWS_LEADER_ADDRESS}
echo "DP_ADDRESS=${DP_ADDRESS} (fallback)"
fi
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
cat /proc/driver/nvidia/params
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hca_port_pairs=()
for hca in "${active_hcas[@]}"; do
hca_port_pairs+=("${hca}:1")
done
active_hca_list=$(IFS=,; echo "${active_hcas[*]}")
hca_port_pairs_list=$(IFS=,; echo "${hca_port_pairs[*]}")
echo "[Infer RoCE] Setting active HCAs: ${active_hca_list}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${active_hca_list}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hca_port_pairs_list}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${hca_port_pairs_list}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
echo "[Infer RoCE] UCX_NET_DEVICES=${UCX_NET_DEVICES}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if tied - prefer index 3 (SR-IOV standard)
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using pre-configured GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=$(( ${LWS_WORKER_INDEX:-0} * {{ or .Spec.Parallelism.DataLocal 1 }} ))
# --disable-access-log-for-endpoints landed in vLLM 0.16.0 (vllm-project/vllm#30011).
# Older versions still need the blanket --disable-uvicorn-access-log.
ACCESS_LOG_ARGS="--disable-uvicorn-access-log"
VLLM_VERSION=$(vllm --version 2>/dev/null | tail -1 | awk '{print $NF}')
echo "[access-log-detect] vllm version='${VLLM_VERSION}'"
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.16.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.16.0" ]; then
ACCESS_LOG_ARGS="--disable-access-log-for-endpoints /health,/metrics,/ping"
fi
echo "[access-log-detect] selected ACCESS_LOG_ARGS='${ACCESS_LOG_ARGS}'"
# --shutdown-timeout landed in vLLM 0.18.0 (vllm-project/vllm#36666).
SHUTDOWN_TIMEOUT_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.18.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.18.0" ]; then
SHUTDOWN_TIMEOUT_ARGS="--shutdown-timeout {{ shutdownTimeout .Spec.Worker 15 }}"
fi
# --kv-transfer-config with OffloadingConnector requires vLLM 0.22.0+ (vllm-project/vllm#40020).
KV_TRANSFER_ARGS=""
if [[ "$VLLM_VERSION" =~ ^[0-9]+\.[0-9]+ ]] && [ "$(printf '%s\n%s\n' "0.22.0" "${VLLM_VERSION}" | sort -V | head -1)" = "0.22.0" ]; then
if [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv-transfer-config"* ]] && [[ "${VLLM_ADDITIONAL_ARGS:-}" != *"--kv_transfer_config"* ]] && [[ "$*" != *"--kv-transfer-config"* ]] && [[ "$*" != *"--kv_transfer_config"* ]]; then
KV_TRANSFER_ARGS="{{ kvTransferConfig .Spec.KVCacheOffloading }}"
fi
fi
eval "exec vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" "publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}" \
--port 8001 \
{{- if .Spec.Parallelism.Expert }}--enable-expert-parallel{{- end }} \
{{- if .Spec.Parallelism.Tensor }}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--headless \
${ACCESS_LOG_ARGS} \
${SHUTDOWN_TIMEOUT_ARGS} \
${KV_TRANSFER_ARGS} \
{{ if .GlobalConfig.EnableTLS }}--enable-ssl-refresh{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-certfile /var/run/kserve/tls/tls.crt{{- end }} \
{{ if .GlobalConfig.EnableTLS }}--ssl-keyfile /var/run/kserve/tls/tls.key{{- end }} \
${VLLM_ADDITIONAL_ARGS} \
$@"
# "--" ends bash options; subsequent args become positional params ($@) forwarded to vllm serve
- "--"
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
- name: VLLM_RANDOMIZE_DP_DUMMY_INPUTS
value: "1"
securityContext:
allowPrivilegeEscalation: false
# nvidia-cdi-hook requires a writable root filesystem
readOnlyRootFilesystem: false
runAsNonRoot: true
capabilities:
add:
- "IPC_LOCK"
- "SYS_RAWIO"
- "NET_RAW"
drop:
- ALL
seccompProfile:
type: RuntimeDefault
lifecycle:
preStop:
exec:
command:
- /bin/sleep
- "15"
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /tmp
name: tmp-dir
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 60
volumes:
- emptyDir: { }
name: home
- emptyDir: { }
name: tmp-dir
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: { }
name: model-cache
- name: tls-certs
secret:
secretName: "{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}"
Reference in New Issue View Git Blame Copy Permalink