hanzo-ml/kserve
2
0
Fork 0
mirror of https://github.com/hanzo-ml/kserve.git synced 2026-10-30 20:53:49 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
master
kserve/install/v0.17.0/kserve-cluster-resources.yaml
T
Jooho LeeandGitHub e002cec10d fix: include 0.17.0 installation scripts (#5256) 
Signed-off-by: Jooho Lee <[email protected]>
2026-03-30 11:10:01 -04:00

2630 lines
89 KiB
YAML
Raw Permalink Blame History

apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
annotations:
serving.kserve.io/server-type: huggingfaceserver
name: kserve-huggingfaceserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
env:
- name: LMCACHE_USE_EXPERIMENTAL
value: "True"
image: kserve/huggingfaceserver:v0.17.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
volumeMounts:
- mountPath: /dev/shm
name: devshm
hostIPC: false
protocolVersions:
- v2
- v1
supportedModelFormats:
- autoSelect: true
name: huggingface
priority: 1
version: "1"
volumes:
- emptyDir:
medium: Memory
name: devshm
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
annotations:
serving.kserve.io/server-type: huggingfaceserver
name: kserve-huggingfaceserver-multinode
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
command:
- bash
- -c
- "export MODEL_DIR_ARG=\"\"\nif [[ ! -z ${MODEL_ID} ]]\nthen\n export MODEL_DIR_ARG=\"--model_dir=${MODEL_ID}\"\nfi\n\nif
[[ ! -z ${MODEL_DIR} ]]\nthen\n export MODEL_DIR_ARG=\"--model_dir=${MODEL_DIR}\"\nfi\n\nexport
RAY_ADDRESS=${POD_IP}:${RAY_PORT}\nray start --head --disable-usage-stats --include-dashboard
false \npython ./huggingfaceserver/health_check.py registered_nodes --retries
200 --probe_name runtime_start\n\npython -m huggingfaceserver ${MODEL_DIR_ARG}
--tensor-parallel-size=${TENSOR_PARALLEL_SIZE} --pipeline-parallel-size=${PIPELINE_PARALLEL_SIZE}
$0 $@\n"
env:
- name: RAY_PORT
value: "6379"
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: VLLM_CONFIG_ROOT
value: /tmp
- name: HF_HUB_CACHE
value: /tmp
image: kserve/huggingfaceserver:v0.17.0-gpu
livenessProbe:
exec:
command:
- bash
- -c
- |
python ./huggingfaceserver/health_check.py registered_node_and_runtime_health --health_check_url http://localhost:8080 --probe_name head_liveness
failureThreshold: 2
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 15
name: kserve-container
readinessProbe:
exec:
command:
- bash
- -c
- |
python ./huggingfaceserver/health_check.py runtime_health --health_check_url http://localhost:8080 --probe_name head_readiness
failureThreshold: 2
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 15
resources:
limits:
cpu: "4"
memory: 12Gi
requests:
cpu: "2"
memory: 6Gi
startupProbe:
exec:
command:
- bash
- -c
- |
python ./huggingfaceserver/health_check.py registered_node_and_runtime_health --health_check_url http://localhost:8080 --probe_name head_startup
failureThreshold: 40
initialDelaySeconds: 60
periodSeconds: 30
successThreshold: 1
timeoutSeconds: 30
volumeMounts:
- mountPath: /dev/shm
name: shm
protocolVersions:
- v2
- v1
supportedModelFormats:
- autoSelect: true
name: huggingface
priority: 2
version: "1"
volumes:
- emptyDir:
medium: Memory
sizeLimit: 3Gi
name: shm
workerSpec:
containers:
- command:
- bash
- -c
- "export RAY_HEAD_ADDRESS=${HEAD_SVC}.${POD_NAMESPACE}.svc.cluster.local:6379\nSECONDS=0\n\nwhile
true; do \n if (( SECONDS <= 240 )); then\n if ray health-check
--address \"${RAY_HEAD_ADDRESS}\" > /dev/null 2>&1; then\n echo \"Ray
Global Control Service(GCS) is ready.\"\n break\n fi\n echo \"$SECONDS
seconds elapsed: Waiting for Ray Global Control Service(GCS) to be ready.\"\n
\ else\n if ray health-check --address \"${RAY_HEAD_ADDRESS}\"; then\n
\ echo \"Ray Global Control Service(GCS) is ready. Any error messages
above can be safely ignored.\"\n break\n fi\n echo \"$SECONDS seconds
elapsed: Still waiting for Ray Global Control Service(GCS) to be ready.\"\n
\ fi\n\n sleep 5\ndone\n\necho \"Attempting to connect to Ray cluster at
$RAY_HEAD_ADDRESS ...\"\nray start --address=\"${RAY_HEAD_ADDRESS}\" --block\n"
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: kserve/huggingfaceserver:v0.17.0-gpu
livenessProbe:
exec:
command:
- bash
- -c
- |
export RAY_ADDRESS=${HEAD_SVC}.${POD_NAMESPACE}.svc.cluster.local:6379
python ./huggingfaceserver/health_check.py registered_nodes --probe_name worker_liveness
failureThreshold: 2
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 15
name: worker-container
resources:
limits:
cpu: "4"
memory: 12Gi
requests:
cpu: "2"
memory: 6Gi
startupProbe:
exec:
command:
- bash
- -c
- |
export RAY_HEAD_NODE=${HEAD_SVC}.${POD_NAMESPACE}.svc.cluster.local
export RAY_ADDRESS=${RAY_HEAD_NODE}:6379
python ./huggingfaceserver/health_check.py registered_node_and_runtime_models --runtime_url http://${RAY_HEAD_NODE}:8080/v1/models --probe_name worker_startup
failureThreshold: 40
initialDelaySeconds: 60
periodSeconds: 30
successThreshold: 1
timeoutSeconds: 30
volumeMounts:
- mountPath: /dev/shm
name: shm
pipelineParallelSize: 1
tensorParallelSize: 1
volumes:
- emptyDir:
medium: Memory
sizeLimit: 3Gi
name: shm
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
annotations:
serving.kserve.io/server-type: lgbserver
name: kserve-lgbserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
- --model_dir=/mnt/models
- --http_port=8080
- --nthread=1
image: kserve/lgbserver:v0.17.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
protocolVersions:
- v1
- v2
supportedModelFormats:
- autoSelect: true
name: lightgbm
priority: 1
version: "4"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
annotations:
serving.kserve.io/server-type: mlserver
name: kserve-mlserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- env:
- name: MLSERVER_MODEL_IMPLEMENTATION
value: '{{.Labels.modelClass}}'
- name: MLSERVER_HTTP_PORT
value: "8080"
- name: MLSERVER_GRPC_PORT
value: "9000"
image: docker.io/seldonio/mlserver:1.5.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
protocolVersions:
- v2
supportedModelFormats:
- autoSelect: true
name: sklearn
priority: 2
version: "0"
- autoSelect: true
name: sklearn
priority: 2
version: "1"
- autoSelect: true
name: xgboost
priority: 2
version: "1"
- autoSelect: true
name: xgboost
priority: 2
version: "2"
- autoSelect: true
name: lightgbm
priority: 2
version: "3"
- autoSelect: true
name: lightgbm
priority: 2
version: "4"
- autoSelect: true
name: mlflow
priority: 1
version: "1"
- autoSelect: true
name: mlflow
priority: 1
version: "2"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
annotations:
serving.kserve.io/server-type: paddleserver
name: kserve-paddleserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
- --model_dir=/mnt/models
- --http_port=8080
image: kserve/paddleserver:v0.17.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
protocolVersions:
- v1
- v2
supportedModelFormats:
- autoSelect: true
name: paddle
priority: 1
version: "2"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
annotations:
serving.kserve.io/server-type: pmmlserver
name: kserve-pmmlserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
- --model_dir=/mnt/models
- --http_port=8080
image: kserve/pmmlserver:v0.17.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
protocolVersions:
- v1
- v2
supportedModelFormats:
- autoSelect: true
name: pmml
priority: 1
version: "3"
- autoSelect: true
name: pmml
priority: 1
version: "4"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
annotations:
serving.kserve.io/server-type: predictiveserver
name: kserve-predictiveserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
- --model_dir=/mnt/models
- --http_port=8080
- --framework={{.Annotations.modelFormat}}
- --nthread=1
image: kserve/predictiveserver:v0.17.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
multiModel: false
protocolVersions:
- v1
- v2
supportedModelFormats:
- autoSelect: false
name: sklearn
priority: 3
version: "1"
- autoSelect: false
name: xgboost
priority: 3
version: "2"
- autoSelect: false
name: lightgbm
priority: 3
version: "4"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
annotations:
serving.kserve.io/server-type: sklearnserver
name: kserve-sklearnserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
- --model_dir=/mnt/models
- --http_port=8080
image: kserve/sklearnserver:v0.17.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
protocolVersions:
- v1
- v2
supportedModelFormats:
- autoSelect: true
name: sklearn
priority: 1
version: "1"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
annotations:
serving.kserve.io/server-type: tensorflow-serving
name: kserve-tensorflow-serving
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
- --port=9000
- --rest_api_port=8080
- --model_base_path=/mnt/models
- --rest_api_timeout_in_ms=60000
command:
- /usr/bin/tensorflow_model_server
image: tensorflow/serving:2.6.2
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
runAsUser: 1000
protocolVersions:
- v1
- grpc-v1
supportedModelFormats:
- autoSelect: true
name: tensorflow
priority: 2
version: "1"
- autoSelect: true
name: tensorflow
priority: 2
version: "2"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
annotations:
serving.kserve.io/server-type: torchserve
name: kserve-torchserve
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8082"
containers:
- args:
- torchserve
- --start
- --model-store=/mnt/models/model-store
- --ts-config=/mnt/models/config/config.properties
env:
- name: TS_SERVICE_ENVELOPE
value: '{{.Labels.serviceEnvelope}}'
image: pytorch/torchserve-kfs:0.9.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
runAsUser: 1000
protocolVersions:
- v1
- v2
- grpc-v2
supportedModelFormats:
- autoSelect: true
name: pytorch
priority: 2
version: "1"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
annotations:
serving.kserve.io/server-type: tritonserver
name: kserve-tritonserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8002"
containers:
- args:
- tritonserver
- --model-store=/mnt/models
- --grpc-port=9000
- --http-port=8080
- --allow-grpc=true
- --allow-http=true
image: nvcr.io/nvidia/tritonserver:23.05-py3
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
runAsUser: 1000
protocolVersions:
- v2
- grpc-v2
supportedModelFormats:
- autoSelect: true
name: tensorrt
priority: 1
version: "8"
- autoSelect: true
name: tensorflow
priority: 1
version: "1"
- autoSelect: true
name: tensorflow
priority: 1
version: "2"
- autoSelect: true
name: onnx
priority: 1
version: "1"
- name: pytorch
version: "1"
- autoSelect: true
name: triton
priority: 1
version: "2"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
annotations:
serving.kserve.io/server-type: xgbserver
name: kserve-xgbserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
- --model_dir=/mnt/models
- --http_port=8080
- --nthread=1
image: kserve/xgbserver:v0.17.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
protocolVersions:
- v1
- v2
supportedModelFormats:
- autoSelect: true
name: xgboost
priority: 1
version: "2"
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-decode-template
namespace: kserve
spec:
template:
containers:
- command:
- vllm
- serve
- /mnt/models
- --served-model-name
- '{{ .Spec.Model.Name }}'
- --port
- "8001"
- --disable-uvicorn-access-log
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.5.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8001
scheme: HTTP
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8001
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8001
scheme: HTTP
periodSeconds: 10
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8001
scheme: HTTP
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
initContainers:
- args:
- --port=8000
- --vllm-port=8001
- --connector=nixlv2
- --secure-proxy=false
env:
- name: INFERENCE_POOL_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: ghcr.io/llm-d/llm-d-routing-sidecar:v0.6.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 10
name: llm-d-routing-sidecar
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 10
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
resources: {}
restartPolicy: Always
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-decode-worker-data-parallel
namespace: kserve
spec:
template:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
DP_ADDRESS=$(getent hosts "${LWS_LEADER_ADDRESS}" | cut -d' ' -f1)
if [ -z "${DP_ADDRESS}" ]; then
echo "ERROR: Failed to resolve LWS_LEADER_ADDRESS='${LWS_LEADER_ADDRESS}'" >&2
exit 1
fi
echo "DP_ADDRESS=${DP_ADDRESS}"
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=0
eval "vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8001 \
--api-server-count ${VLLM_API_SERVER_COUNT:-8} \
{{- if .Spec.Parallelism.Expert -}}--enable-expert-parallel{{- end }} \
{{- if .Spec.Parallelism.Tensor -}}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--disable-uvicorn-access-log \
${VLLM_ADDITIONAL_ARGS} \
$@ \
--trust-remote-code"
# BackendTLSPolicy is not implemented yet so disable SSL for now
# --enable-ssl-refresh \
# --ssl-certfile \
# /var/run/kserve/tls/tls.crt \
# --ssl-keyfile \
# /var/run/kserve/tls/tls.key"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.5.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8001
scheme: HTTP
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8001
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8001
scheme: HTTP
periodSeconds: 30
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8001
scheme: HTTP
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
initContainers:
- args:
- --port=8000
- --vllm-port=8001
- --connector=nixlv2
- --secure-proxy=false
env:
- name: INFERENCE_POOL_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: ghcr.io/llm-d/llm-d-routing-sidecar:v0.6.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 10
name: llm-d-routing-sidecar
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 10
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
restartPolicy: Always
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
worker:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
DP_ADDRESS=$(getent hosts "${LWS_LEADER_ADDRESS}" | cut -d' ' -f1)
if [ -z "${DP_ADDRESS}" ]; then
echo "ERROR: Failed to resolve LWS_LEADER_ADDRESS='${LWS_LEADER_ADDRESS}'" >&2
exit 1
fi
echo "DP_ADDRESS=${DP_ADDRESS}"
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=$(( ${LWS_WORKER_INDEX:-0} * {{ or .Spec.Parallelism.DataLocal 1 }} ))
eval "vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8001 \
{{- if .Spec.Parallelism.Expert }}--enable-expert-parallel{{- end }} \
{{- if .Spec.Parallelism.Tensor }}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--disable-uvicorn-access-log \
${VLLM_ADDITIONAL_ARGS} \
$@ \
--trust-remote-code \
--headless"
# BackendTLSPolicy is not implemented yet so disable SSL for now
# --enable-ssl-refresh \
# --ssl-certfile \
# /var/run/kserve/tls/tls.crt \
# --ssl-keyfile \
# /var/run/kserve/tls/tls.key"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
- name: VLLM_RANDOMIZE_DP_DUMMY_INPUTS
value: "1"
image: ghcr.io/llm-d/llm-d-cuda:v0.5.1
imagePullPolicy: IfNotPresent
name: main
ports:
- containerPort: 8001
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-prefill-template
namespace: kserve
spec:
prefill:
template:
containers:
- command:
- vllm
- serve
- /mnt/models
- --served-model-name
- '{{ .Spec.Model.Name }}'
- --port
- "8000"
- --disable-uvicorn-access-log
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.5.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: HTTP
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: HTTP
periodSeconds: 10
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: HTTP
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-prefill-worker-data-parallel
namespace: kserve
spec:
prefill:
template:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
DP_ADDRESS=$(getent hosts "${LWS_LEADER_ADDRESS}" | cut -d' ' -f1)
if [ -z "${DP_ADDRESS}" ]; then
echo "ERROR: Failed to resolve LWS_LEADER_ADDRESS='${LWS_LEADER_ADDRESS}'" >&2
exit 1
fi
echo "DP_ADDRESS=${DP_ADDRESS}"
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=0
eval "vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8000 \
--api-server-count ${VLLM_API_SERVER_COUNT:-8} \
{{- if .Spec.Prefill.Parallelism.Expert -}}--enable-expert-parallel{{- end }} \
{{- if .Spec.Prefill.Parallelism.Tensor -}}--tensor-parallel-size {{ .Spec.Prefill.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Prefill.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Prefill.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Prefill.Parallelism.DataRPCPort }}{{ .Spec.Prefill.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--disable-uvicorn-access-log \
${VLLM_ADDITIONAL_ARGS} \
$@ \
--trust-remote-code"
# BackendTLSPolicy is not implemented yet so disable SSL for now
# --enable-ssl-refresh \
# --ssl-certfile \
# /var/run/kserve/tls/tls.crt \
# --ssl-keyfile \
# /var/run/kserve/tls/tls.key"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.5.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: HTTP
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: HTTP
periodSeconds: 30
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: HTTP
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
worker:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
DP_ADDRESS=$(getent hosts "${LWS_LEADER_ADDRESS}" | cut -d' ' -f1)
if [ -z "${DP_ADDRESS}" ]; then
echo "ERROR: Failed to resolve LWS_LEADER_ADDRESS='${LWS_LEADER_ADDRESS}'" >&2
exit 1
fi
echo "DP_ADDRESS=${DP_ADDRESS}"
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=$(( ${LWS_WORKER_INDEX:-0} * {{ or .Spec.Prefill.Parallelism.DataLocal 1 }} ))
eval "vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8000 \
{{- if .Spec.Prefill.Parallelism.Expert }}--enable-expert-parallel{{- end }} \
{{- if .Spec.Prefill.Parallelism.Tensor }}--tensor-parallel-size {{ .Spec.Prefill.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Prefill.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Prefill.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Prefill.Parallelism.DataRPCPort }}{{ .Spec.Prefill.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--disable-uvicorn-access-log \
${VLLM_ADDITIONAL_ARGS} \
$@ \
--trust-remote-code \
--headless"
# BackendTLSPolicy is not implemented yet so disable SSL for now
# --enable-ssl-refresh \
# --ssl-certfile \
# /var/run/kserve/tls/tls.crt \
# --ssl-keyfile \
# /var/run/kserve/tls/tls.key"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.5.1
imagePullPolicy: IfNotPresent
name: main
ports:
- containerPort: 8000
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-router-route
namespace: kserve
spec:
router:
route:
http:
spec:
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: '{{ .GlobalConfig.IngressGatewayName }}'
namespace: '{{ .GlobalConfig.IngressGatewayNamespace }}'
rules:
- backendRefs:
- group: inference.networking.k8s.io
kind: InferencePool
name: '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
port: 8000
weight: 1
filters:
- type: URLRewrite
urlRewrite:
path:
replacePrefixMatch: /v1/completions
type: ReplacePrefixMatch
matches:
- path:
type: PathPrefix
value: /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}/v1/completions
timeouts:
backendRequest: 0s
request: 0s
- backendRefs:
- group: inference.networking.k8s.io
kind: InferencePool
name: '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
port: 8000
weight: 1
filters:
- type: URLRewrite
urlRewrite:
path:
replacePrefixMatch: /v1/chat/completions
type: ReplacePrefixMatch
matches:
- path:
type: PathPrefix
value: /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}/v1/chat/completions
timeouts:
backendRequest: 0s
request: 0s
- backendRefs:
- kind: Service
name: '{{ ChildName .ObjectMeta.Name `-kserve-workload-svc` }}'
port: 8000
weight: 1
filters:
- type: URLRewrite
urlRewrite:
path:
replacePrefixMatch: /
type: ReplacePrefixMatch
matches:
- path:
type: PathPrefix
value: /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}
timeouts:
backendRequest: 0s
request: 0s
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-scheduler
namespace: kserve
spec:
router:
scheduler:
pool:
spec:
endpointPickerRef:
failureMode: FailOpen
kind: Service
name: '{{ ChildName .ObjectMeta.Name `-epp-service` }}'
port:
number: 9002
selector:
matchLabels:
app.kubernetes.io/name: '{{ .ObjectMeta.Name }}'
app.kubernetes.io/part-of: llminferenceservice
kserve.io/component: workload
targetPorts:
- number: 8000
template:
containers:
- args:
- --pool-name
- '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
- --pool-namespace
- '{{ .ObjectMeta.Namespace }}'
- --zap-encoder
- json
- --grpc-port
- "9002"
- --grpc-health-port
- "9003"
- --kv-cache-usage-percentage-metric
- vllm:kv_cache_usage_perc
image: ghcr.io/llm-d/llm-d-inference-scheduler:v0.6.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
grpc:
port: 9003
service: liveness
initialDelaySeconds: 5
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: main
ports:
- containerPort: 9002
name: grpc
protocol: TCP
- containerPort: 9003
name: grpc-health
protocol: TCP
- containerPort: 9090
name: metrics
protocol: TCP
- containerPort: 5557
name: zmq
protocol: TCP
readinessProbe:
failureThreshold: 3
grpc:
port: 9003
service: readiness
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources:
requests:
cpu: 256m
memory: 500Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
- mountPath: /tmp/tokenizer
name: tokenizer-uds
- env:
- name: TOKENIZERS_DIR
value: /mnt/models
image: ghcr.io/llm-d/llm-d-uds-tokenizer:v0.6.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8082
periodSeconds: 15
timeoutSeconds: 5
name: tokenizer
ports:
- containerPort: 8082
name: health
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8082
periodSeconds: 10
timeoutSeconds: 5
resources:
requests:
cpu: 256m
memory: 500Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8082
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /tmp/tokenizer
name: tokenizer-uds
workingDir: /mnt/models
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 30
volumes:
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs`
}}'
- emptyDir: {}
name: tokenizer-uds
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-template
namespace: kserve
spec:
template:
containers:
- command:
- vllm
- serve
- /mnt/models
- --served-model-name
- '{{ .Spec.Model.Name }}'
- --port
- "8000"
- --disable-uvicorn-access-log
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.5.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: HTTP
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: HTTP
periodSeconds: 10
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: HTTP
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-worker-data-parallel
namespace: kserve
spec:
template:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
DP_ADDRESS=$(getent hosts "${LWS_LEADER_ADDRESS}" | cut -d' ' -f1)
if [ -z "${DP_ADDRESS}" ]; then
echo "ERROR: Failed to resolve LWS_LEADER_ADDRESS='${LWS_LEADER_ADDRESS}'" >&2
exit 1
fi
echo "DP_ADDRESS=${DP_ADDRESS}"
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=0
eval "vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8000 \
--api-server-count ${VLLM_API_SERVER_COUNT:-8} \
{{- if .Spec.Parallelism.Expert -}}--enable-expert-parallel{{- end }} \
{{- if .Spec.Parallelism.Tensor -}}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--disable-uvicorn-access-log \
${VLLM_ADDITIONAL_ARGS} \
$@ \
--trust-remote-code"
# BackendTLSPolicy is not implemented yet so disable SSL for now
# --enable-ssl-refresh \
# --ssl-certfile \
# /var/run/kserve/tls/tls.crt \
# --ssl-keyfile \
# /var/run/kserve/tls/tls.key"
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.5.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: HTTP
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: HTTP
periodSeconds: 30
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: HTTP
periodSeconds: 10
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
worker:
containers:
- command:
- /bin/bash
- -c
- |-
# In some versions, ZMQ bind doesn't resolve the address through DNS
DP_ADDRESS=$(getent hosts "${LWS_LEADER_ADDRESS}" | cut -d' ' -f1)
if [ -z "${DP_ADDRESS}" ]; then
echo "ERROR: Failed to resolve LWS_LEADER_ADDRESS='${LWS_LEADER_ADDRESS}'" >&2
exit 1
fi
echo "DP_ADDRESS=${DP_ADDRESS}"
if [ "$KSERVE_INFER_ROCE" = "true" ]; then
echo "Trying to infer RoCE configs ... "
grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null
grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null
KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-"RoCE v2"}
echo "[Infer RoCE] Discovering active HCAs ..."
active_hcas=()
# Loop through all mlx5 devices found in sysfs
for hca_dir in /sys/class/infiniband/mlx5_*; do
# Ensure it's a directory before proceeding
if [ -d "$hca_dir" ]; then
hca_name=$(basename "$hca_dir")
port_state_file="$hca_dir/ports/1/state" # Assume port 1
type_file="$hca_dir/ports/1/gid_attrs/types/*"
echo "[Infer RoCE] Check if the port state file ${port_state_file} exists and contains 'ACTIVE'"
if [ -f "$port_state_file" ] && grep -q "ACTIVE" "$port_state_file" && grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" ${type_file} 2>/dev/null; then
echo "[Infer RoCE] Found active HCA: $hca_name"
active_hcas+=("$hca_name")
else
echo "[Infer RoCE] Skipping inactive or down HCA: $hca_name"
fi
fi
done
ucx_hcas=()
for hca in "${active_hcas[@]}"; do
ucx_hcas+=("${hca}:1")
done
# Check if we found any active HCAs
if [ ${#active_hcas[@]} -gt 0 ]; then
# Join the array elements with a comma
hcas=$(IFS=,; echo "${active_hcas[*]}")
echo "[Infer RoCE] Setting active HCAs: ${hcas}"
export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}
export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hcas}}
export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}
echo "[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}"
echo "[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}"
else
echo "[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set."
fi
if [ ${#active_hcas[@]} -gt 0 ]; then
echo "[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)..."
# For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across all HCAs
declare -A gid_index_count
declare -A hca_gid_index
for hca_name in "${active_hcas[@]}"; do
echo "[Infer RoCE] Processing HCA: ${hca_name}"
# Find all RoCE v2 IPv4 GIDs for this HCA and count by index
for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*; do
if grep -q "${KSERVE_INFER_IB_GID_INDEX_GREP}" "$tpath" 2>/dev/null; then
idx=$(basename "$tpath")
gid_file="/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}"
# Check for IPv4 GID (contains ffff:)
if [ -f "$gid_file" ] && grep -q "ffff:" "$gid_file"; then
gid_value=$(cat "$gid_file" 2>/dev/null || echo "")
echo "[Infer RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}"
hca_gid_index["${hca_name}"]="${idx}"
gid_index_count["${idx}"]=$((${gid_index_count["${idx}"]} + 1))
break # Use first found IPv4 GID per HCA
fi
fi
done
done
# Find the most common GID index (most likely to be consistent across nodes)
best_gid_index=""
max_count=0
for idx in "${!gid_index_count[@]}"; do
count=${gid_index_count["${idx}"]}
echo "[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs"
if [ $count -gt $max_count ]; then
max_count=$count
best_gid_index="$idx"
fi
done
# Use deterministic fallback if counts are equal - prefer lower index number
if [ ${#gid_index_count[@]} -gt 1 ]; then
echo "[Infer RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}"
# If there's a tie, prefer index 3 as it's most common in SR-IOV setups
if [ -n "${gid_index_count['3']}" ] && [ "${gid_index_count['3']}" -eq "$max_count" ]; then
best_gid_index="3"
echo "[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)"
fi
fi
# Check if GID_INDEX is already set via environment variables
if [ -n "${NCCL_IB_GID_INDEX}" ]; then
echo "[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX} from environment"
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}
echo "[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX} for NCCL, NVSHMEM, and UCX"
elif [ -n "$best_gid_index" ]; then
echo "[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found on ${max_count} HCAs)"
export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}
export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}
export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}
echo "[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL, NVSHMEM, and UCX"
else
echo "[Infer RoCE] ERROR: No valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA."
fi
else
echo "[Infer RoCE] No active HCAs found, skipping GID_INDEX inference."
fi
fi
START_RANK=$(( ${LWS_WORKER_INDEX:-0} * {{ or .Spec.Parallelism.DataLocal 1 }} ))
eval "vllm serve \
/mnt/models \
--served-model-name "{{ .Spec.Model.Name }}" \
--port 8000 \
{{- if .Spec.Parallelism.Expert }}--enable-expert-parallel{{- end }} \
{{- if .Spec.Parallelism.Tensor }}--tensor-parallel-size {{ .Spec.Parallelism.Tensor }}{{- end }} \
--data-parallel-size {{ or .Spec.Parallelism.Data 1 }} \
--data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \
--data-parallel-address ${DP_ADDRESS} \
--data-parallel-rpc-port {{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \
--data-parallel-start-rank $START_RANK \
--disable-uvicorn-access-log \
${VLLM_ADDITIONAL_ARGS} \
$@ \
--trust-remote-code \
--headless"
# BackendTLSPolicy is not implemented yet so disable SSL for now
# --enable-ssl-refresh \
# --ssl-certfile \
# /var/run/kserve/tls/tls.crt \
# --ssl-keyfile \
# /var/run/kserve/tls/tls.key
- --
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-cuda:v0.5.1
imagePullPolicy: IfNotPresent
name: main
ports:
- containerPort: 8000
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /var/run/kserve/tls
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 8Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
Reference in New Issue View Git Blame Copy Permalink