hanzo-ml/kserve
2
0
Fork 0
mirror of https://github.com/hanzo-ml/kserve.git synced 2026-10-30 20:53:49 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
master
kserve/install/v0.16.0/kserve-cluster-resources.yaml
T
VincentandGitHub 42ea11b7c9 Prepare for 0.16.0 release (#4786) 
Signed-off-by: Vincent Hou <[email protected]>
2025-10-31 19:00:45 -04:00

2036 lines
81 KiB
YAML
Raw Permalink Blame History

apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
name: kserve-huggingfaceserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
env:
- name: LMCACHE_USE_EXPERIMENTAL
value: "True"
image: kserve/huggingfaceserver:v0.16.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
volumeMounts:
- mountPath: /dev/shm
name: devshm
hostIPC: false
protocolVersions:
- v2
- v1
supportedModelFormats:
- autoSelect: true
name: huggingface
priority: 1
version: "1"
volumes:
- emptyDir:
medium: Memory
name: devshm
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
name: kserve-huggingfaceserver-multinode
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
command:
- bash
- -c
- "export MODEL_DIR_ARG=\"\"\nif [[ ! -z ${MODEL_ID} ]]\nthen\n export MODEL_DIR_ARG=\"--model_dir=${MODEL_ID}\"\nfi\n\nif
[[ ! -z ${MODEL_DIR} ]]\nthen\n export MODEL_DIR_ARG=\"--model_dir=${MODEL_DIR}\"\nfi\n\nexport
RAY_ADDRESS=${POD_IP}:${RAY_PORT}\nray start --head --disable-usage-stats --include-dashboard
false \npython ./huggingfaceserver/health_check.py registered_nodes --retries
200 --probe_name runtime_start\n\npython -m huggingfaceserver ${MODEL_DIR_ARG}
--tensor-parallel-size=${TENSOR_PARALLEL_SIZE} --pipeline-parallel-size=${PIPELINE_PARALLEL_SIZE}
$0 $@\n"
env:
- name: RAY_PORT
value: "6379"
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: VLLM_CONFIG_ROOT
value: /tmp
- name: HF_HUB_CACHE
value: /tmp
image: kserve/huggingfaceserver:v0.16.0-gpu
livenessProbe:
exec:
command:
- bash
- -c
- |
python ./huggingfaceserver/health_check.py registered_node_and_runtime_health --health_check_url http://localhost:8080 --probe_name head_liveness
failureThreshold: 2
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 15
name: kserve-container
readinessProbe:
exec:
command:
- bash
- -c
- |
python ./huggingfaceserver/health_check.py runtime_health --health_check_url http://localhost:8080 --probe_name head_readiness
failureThreshold: 2
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 15
resources:
limits:
cpu: "4"
memory: 12Gi
requests:
cpu: "2"
memory: 6Gi
startupProbe:
exec:
command:
- bash
- -c
- |
python ./huggingfaceserver/health_check.py registered_node_and_runtime_health --health_check_url http://localhost:8080 --probe_name head_startup
failureThreshold: 40
initialDelaySeconds: 60
periodSeconds: 30
successThreshold: 1
timeoutSeconds: 30
volumeMounts:
- mountPath: /dev/shm
name: shm
protocolVersions:
- v2
- v1
supportedModelFormats:
- autoSelect: true
name: huggingface
priority: 2
version: "1"
volumes:
- emptyDir:
medium: Memory
sizeLimit: 3Gi
name: shm
workerSpec:
containers:
- command:
- bash
- -c
- "export RAY_HEAD_ADDRESS=${HEAD_SVC}.${POD_NAMESPACE}.svc.cluster.local:6379\nSECONDS=0\n\nwhile
true; do \n if (( SECONDS <= 240 )); then\n if ray health-check
--address \"${RAY_HEAD_ADDRESS}\" > /dev/null 2>&1; then\n echo \"Ray
Global Control Service(GCS) is ready.\"\n break\n fi\n echo \"$SECONDS
seconds elapsed: Waiting for Ray Global Control Service(GCS) to be ready.\"\n
\ else\n if ray health-check --address \"${RAY_HEAD_ADDRESS}\"; then\n
\ echo \"Ray Global Control Service(GCS) is ready. Any error messages
above can be safely ignored.\"\n break\n fi\n echo \"$SECONDS seconds
elapsed: Still waiting for Ray Global Control Service(GCS) to be ready.\"\n
\ fi\n\n sleep 5\ndone\n\necho \"Attempting to connect to Ray cluster at
$RAY_HEAD_ADDRESS ...\"\nray start --address=\"${RAY_HEAD_ADDRESS}\" --block\n"
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: kserve/huggingfaceserver:v0.16.0-gpu
livenessProbe:
exec:
command:
- bash
- -c
- |
export RAY_ADDRESS=${HEAD_SVC}.${POD_NAMESPACE}.svc.cluster.local:6379
python ./huggingfaceserver/health_check.py registered_nodes --probe_name worker_liveness
failureThreshold: 2
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 15
name: worker-container
resources:
limits:
cpu: "4"
memory: 12Gi
requests:
cpu: "2"
memory: 6Gi
startupProbe:
exec:
command:
- bash
- -c
- |
export RAY_HEAD_NODE=${HEAD_SVC}.${POD_NAMESPACE}.svc.cluster.local
export RAY_ADDRESS=${RAY_HEAD_NODE}:6379
python ./huggingfaceserver/health_check.py registered_node_and_runtime_models --runtime_url http://${RAY_HEAD_NODE}:8080/v1/models --probe_name worker_startup
failureThreshold: 40
initialDelaySeconds: 60
periodSeconds: 30
successThreshold: 1
timeoutSeconds: 30
volumeMounts:
- mountPath: /dev/shm
name: shm
pipelineParallelSize: 1
tensorParallelSize: 1
volumes:
- emptyDir:
medium: Memory
sizeLimit: 3Gi
name: shm
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
name: kserve-lgbserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
- --model_dir=/mnt/models
- --http_port=8080
- --nthread=1
image: kserve/lgbserver:v0.16.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
protocolVersions:
- v1
- v2
supportedModelFormats:
- autoSelect: true
name: lightgbm
priority: 1
version: "3"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
name: kserve-mlserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- env:
- name: MLSERVER_MODEL_IMPLEMENTATION
value: '{{.Labels.modelClass}}'
- name: MLSERVER_HTTP_PORT
value: "8080"
- name: MLSERVER_GRPC_PORT
value: "9000"
- name: MODELS_DIR
value: /mnt/models
image: docker.io/seldonio/mlserver:1.5.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
protocolVersions:
- v2
supportedModelFormats:
- autoSelect: true
name: sklearn
priority: 2
version: "0"
- autoSelect: true
name: sklearn
priority: 2
version: "1"
- autoSelect: true
name: xgboost
priority: 2
version: "1"
- autoSelect: true
name: xgboost
priority: 2
version: "2"
- autoSelect: true
name: lightgbm
priority: 2
version: "3"
- autoSelect: true
name: lightgbm
priority: 2
version: "4"
- autoSelect: true
name: mlflow
priority: 1
version: "1"
- autoSelect: true
name: mlflow
priority: 1
version: "2"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
name: kserve-paddleserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
- --model_dir=/mnt/models
- --http_port=8080
image: kserve/paddleserver:v0.16.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
protocolVersions:
- v1
- v2
supportedModelFormats:
- autoSelect: true
name: paddle
priority: 1
version: "2"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
name: kserve-pmmlserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
- --model_dir=/mnt/models
- --http_port=8080
image: kserve/pmmlserver:v0.16.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
protocolVersions:
- v1
- v2
supportedModelFormats:
- autoSelect: true
name: pmml
priority: 1
version: "3"
- autoSelect: true
name: pmml
priority: 1
version: "4"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
name: kserve-sklearnserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
- --model_dir=/mnt/models
- --http_port=8080
image: kserve/sklearnserver:v0.16.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
protocolVersions:
- v1
- v2
supportedModelFormats:
- autoSelect: true
name: sklearn
priority: 1
version: "1"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
name: kserve-tensorflow-serving
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
- --port=9000
- --rest_api_port=8080
- --model_base_path=/mnt/models
- --rest_api_timeout_in_ms=60000
command:
- /usr/bin/tensorflow_model_server
image: tensorflow/serving:2.6.2
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
runAsUser: 1000
protocolVersions:
- v1
- grpc-v1
supportedModelFormats:
- autoSelect: true
name: tensorflow
priority: 2
version: "1"
- autoSelect: true
name: tensorflow
priority: 2
version: "2"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
name: kserve-torchserve
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8082"
containers:
- args:
- torchserve
- --start
- --model-store=/mnt/models/model-store
- --ts-config=/mnt/models/config/config.properties
env:
- name: TS_SERVICE_ENVELOPE
value: '{{.Labels.serviceEnvelope}}'
image: pytorch/torchserve-kfs:0.9.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
runAsUser: 1000
protocolVersions:
- v1
- v2
- grpc-v2
supportedModelFormats:
- autoSelect: true
name: pytorch
priority: 2
version: "1"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
name: kserve-tritonserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8002"
containers:
- args:
- tritonserver
- --model-store=/mnt/models
- --grpc-port=9000
- --http-port=8080
- --allow-grpc=true
- --allow-http=true
image: nvcr.io/nvidia/tritonserver:23.05-py3
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
runAsUser: 1000
protocolVersions:
- v2
- grpc-v2
supportedModelFormats:
- autoSelect: true
name: tensorrt
priority: 1
version: "8"
- autoSelect: true
name: tensorflow
priority: 1
version: "1"
- autoSelect: true
name: tensorflow
priority: 1
version: "2"
- autoSelect: true
name: onnx
priority: 1
version: "1"
- name: pytorch
version: "1"
- autoSelect: true
name: triton
priority: 1
version: "2"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterServingRuntime
metadata:
name: kserve-xgbserver
spec:
annotations:
prometheus.kserve.io/path: /metrics
prometheus.kserve.io/port: "8080"
containers:
- args:
- --model_name={{.Name}}
- --model_dir=/mnt/models
- --http_port=8080
- --nthread=1
image: kserve/xgbserver:v0.16.0
name: kserve-container
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: "1"
memory: 2Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
runAsNonRoot: true
protocolVersions:
- v1
- v2
supportedModelFormats:
- autoSelect: true
name: xgboost
priority: 1
version: "1"
---
apiVersion: serving.kserve.io/v1alpha1
kind: ClusterStorageContainer
metadata:
name: default
spec:
container:
image: kserve/storage-initializer:v0.16.0
name: storage-initializer
resources:
limits:
cpu: "1"
memory: 1Gi
requests:
cpu: 100m
memory: 100Mi
supportedUriFormats:
- prefix: gs://
- prefix: s3://
- prefix: hdfs://
- prefix: hf://
- prefix: webhdfs://
- regex: https://(.+?).blob.core.windows.net/(.+)
- regex: https://(.+?).file.core.windows.net/(.+)
- regex: https?://(.+)/(.+)
workloadType: initContainer
---
apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-decode-template
namespace: kserve
spec:
template:
containers:
- args:
- --served-model-name
- '{{ .Spec.Model.Name }}'
- --port
- "8001"
- --disable-log-requests
command:
- vllm
- serve
- /mnt/models
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-dev:v0.2.2
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8001
scheme: HTTP
initialDelaySeconds: 120
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8001
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8001
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /etc/ssl/certs
name: tls-certs
readOnly: true
initContainers:
- args:
- --port=8000
- --vllm-port=8001
- --connector=nixlv2
env:
- name: INFERENCE_POOL_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: ghcr.io/llm-d/llm-d-routing-sidecar:v0.2.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 10
name: llm-d-routing-sidecar
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 10
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
resources: {}
restartPolicy: Always
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /etc/ssl/certs
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-decode-worker-data-parallel
namespace: kserve
spec:
template:
containers:
- args:
- "\nif [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer
RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n
\ grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n
\ KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE
v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n
\ # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*;
do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\"
]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\"
# Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n
\ echo \"[Infer RoCE] Check if the port state file ${port_state_file}
exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] &&
grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\"
${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active
HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n
\ echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n
\ fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\";
do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active
HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements
with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo
\"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n
\ export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n
\ echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer
RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer
RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n
\ fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE]
Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n #
For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across
all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n
\ for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE]
Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for
this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*;
do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\"
2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n
\ # Check for IPv4 GID (contains ffff:)\n if
[ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat
\"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer
RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n
\ hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]}
+ 1))\n break # Use first found IPv4 GID per HCA\n fi\n
\ fi\n done\n done\n\n # Find the most common
GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n
\ max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n
\ echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n
\ if [ $count -gt $max_count ]; then\n max_count=$count\n
\ best_gid_index=\"$idx\"\n fi\n done\n\n #
Use deterministic fallback if counts are equal - prefer lower index number
\ \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer
RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n
\ # If there's a tie, prefer index 3 as it's most common in SR-IOV
setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\"
-eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo
\"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n
\ fi\n fi\n\n # Check if GID_INDEX is already set via environment
variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo
\"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX}
from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n
\ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n
\ echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX}
for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n
\ echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found
on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n
\ export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n
\ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n
\ echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL,
NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid
IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n
\ else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX
inference.\"\n fi\nfi\n\nSTART_RANK=0\neval \"vllm serve \\\n /mnt/models
\\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8001 \\\n
\ --api-server-count ${VLLM_API_SERVER_COUNT:-8} \\\n --disable-log-requests
\\\n {{- if .Spec.Parallelism.Expert -}}--enable-expert-parallel{{- end }}
\\\n {{- if .Spec.Parallelism.Tensor -}}--tensor-parallel-size {{ .Spec.Parallelism.Tensor
}}{{- end }} \\\n --data-parallel-size {{ or .Spec.Parallelism.Data 1 }}
\\\n --data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \\\n
\ --data-parallel-address $(LWS_LEADER_ADDRESS) \\\n --data-parallel-rpc-port
{{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{
else }}5555{{- end }} \\\n --data-parallel-start-rank $START_RANK \\\n ${VLLM_ADDITIONAL_ARGS}
\\\n --trust-remote-code\"\n # BackendTLSPolicy is not implemented yet so
disable SSL for now\n # --enable-ssl-refresh \\\n # --ssl-certfile \\\n
\ # /etc/ssl/certs/tls.crt \\\n # --ssl-keyfile \\\n # /etc/ssl/certs/tls.key\""
command:
- /bin/bash
- -c
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-dev:v0.2.2
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8001
scheme: HTTP
initialDelaySeconds: 300
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8001
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8001
scheme: HTTP
initialDelaySeconds: 200
periodSeconds: 30
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /etc/ssl/certs
name: tls-certs
readOnly: true
initContainers:
- args:
- --port=8000
- --vllm-port=8001
- --connector=nixlv2
- --secure-proxy=true
env:
- name: INFERENCE_POOL_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: ghcr.io/llm-d/llm-d-routing-sidecar:v0.2.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 10
name: llm-d-routing-sidecar
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 10
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
restartPolicy: Always
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /etc/ssl/certs
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
worker:
containers:
- args:
- "\nif [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer
RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n
\ grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n
\ KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE
v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n
\ # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*;
do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\"
]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\"
# Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n
\ echo \"[Infer RoCE] Check if the port state file ${port_state_file}
exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] &&
grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\"
${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active
HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n
\ echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n
\ fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\";
do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active
HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements
with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo
\"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n
\ export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n
\ echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer
RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer
RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n
\ fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE]
Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n #
For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across
all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n
\ for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE]
Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for
this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*;
do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\"
2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n
\ # Check for IPv4 GID (contains ffff:)\n if
[ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat
\"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer
RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n
\ hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]}
+ 1))\n break # Use first found IPv4 GID per HCA\n fi\n
\ fi\n done\n done\n\n # Find the most common
GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n
\ max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n
\ echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n
\ if [ $count -gt $max_count ]; then\n max_count=$count\n
\ best_gid_index=\"$idx\"\n fi\n done\n\n #
Use deterministic fallback if counts are equal - prefer lower index number
\ \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer
RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n
\ # If there's a tie, prefer index 3 as it's most common in SR-IOV
setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\"
-eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo
\"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n
\ fi\n fi\n\n # Check if GID_INDEX is already set via environment
variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo
\"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX}
from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n
\ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n
\ echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX}
for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n
\ echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found
on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n
\ export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n
\ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n
\ echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL,
NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid
IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n
\ else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX
inference.\"\n fi\nfi\n\nSTART_RANK=$(( ${LWS_WORKER_INDEX:-0} * {{ or .Spec.Parallelism.DataLocal
1 }} ))\neval \"vllm serve \\\n /mnt/models \\\n --served-model-name \"{{
.Spec.Model.Name }}\" \\\n --port 8001 \\\n --disable-log-requests \\\n
\ {{- if .Spec.Parallelism.Expert }}--enable-expert-parallel{{- end }} \\\n
\ {{- if .Spec.Parallelism.Tensor }}--tensor-parallel-size {{ .Spec.Parallelism.Tensor
}}{{- end }} \\\n --data-parallel-size {{ or .Spec.Parallelism.Data 1 }}
\\\n --data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \\\n
\ --data-parallel-address $(LWS_LEADER_ADDRESS) \\\n --data-parallel-rpc-port
{{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{
else }}5555{{- end }} \\\n --data-parallel-start-rank $START_RANK \\\n ${VLLM_ADDITIONAL_ARGS}
\\\n --trust-remote-code \\\n --headless\"\n # BackendTLSPolicy is not
implemented yet so disable SSL for now\n # --enable-ssl-refresh \\\n # --ssl-certfile
\\\n # /etc/ssl/certs/tls.crt \\\n # --ssl-keyfile \\\n # /etc/ssl/certs/tls.key\""
command:
- /bin/bash
- -c
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
- name: VLLM_RANDOMIZE_DP_DUMMY_INPUTS
value: "1"
image: ghcr.io/llm-d/llm-d-dev:v0.2.2
imagePullPolicy: IfNotPresent
name: main
ports:
- containerPort: 8001
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /etc/ssl/certs
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-prefill-template
namespace: kserve
spec:
prefill:
template:
containers:
- args:
- --served-model-name
- '{{ .Spec.Model.Name }}'
- --port
- "8000"
- --disable-log-requests
command:
- vllm
- serve
- /mnt/models
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-dev:v0.2.2
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 120
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /etc/ssl/certs
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-prefill-worker-data-parallel
namespace: kserve
spec:
prefill:
template:
containers:
- args:
- "\nif [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer
RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/*
2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/*
2>/dev/null\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE
v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n
\ # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*;
do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\"
]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\"
# Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n
\ echo \"[Infer RoCE] Check if the port state file ${port_state_file}
exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ]
&& grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\"
${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found
active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n
\ echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n
\ fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\";
do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active
HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements
with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo
\"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n
\ export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hcas}}\n export
UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE]
NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n
\ else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA
will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo
\"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n
\ # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index
across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n
\ for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer
RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4
GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*;
do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\"
2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n
\ # Check for IPv4 GID (contains ffff:)\n if
[ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat
\"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer
RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n
\ hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]}
+ 1))\n break # Use first found IPv4 GID per HCA\n
\ fi\n fi\n done\n done\n\n #
Find the most common GID index (most likely to be consistent across nodes)\n
\ best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\";
do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer
RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt
$max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n
\ fi\n done\n\n # Use deterministic fallback if counts
are equal - prefer lower index number \n if [ ${#gid_index_count[@]}
-gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found,
selecting most common: ${best_gid_index}\"\n # If there's a tie,
prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\"
] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n
\ echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3
(SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX
is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\"
]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX}
from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n
\ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n
\ echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX}
for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n
\ echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found
on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n
\ export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n
\ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n
\ echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL,
NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No
valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n
\ fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping
GID_INDEX inference.\"\n fi\nfi\n\nSTART_RANK=0\neval \"vllm serve \\\n
\ /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n
\ --port 8000 \\\n --api-server-count ${VLLM_API_SERVER_COUNT:-8} \\\n
\ --disable-log-requests \\\n {{- if .Spec.Prefill.Parallelism.Expert -}}--enable-expert-parallel{{-
end }} \\\n {{- if .Spec.Prefill.Parallelism.Tensor -}}--tensor-parallel-size
{{ .Spec.Prefill.Parallelism.Tensor }}{{- end }} \\\n --data-parallel-size
{{ or .Spec.Prefill.Parallelism.Data 1 }} \\\n --data-parallel-size-local
{{ or .Spec.Prefill.Parallelism.DataLocal 1 }} \\\n --data-parallel-address
$(LWS_LEADER_ADDRESS) \\\n --data-parallel-rpc-port {{ if .Spec.Prefill.Parallelism.DataRPCPort
}}{{ .Spec.Prefill.Parallelism.DataRPCPort }}{{ else }}5555{{- end }} \\\n
\ --data-parallel-start-rank $START_RANK \\\n ${VLLM_ADDITIONAL_ARGS} \\\n
\ --trust-remote-code\"\n # BackendTLSPolicy is not implemented yet so
disable SSL for now\n # --enable-ssl-refresh \\\n # --ssl-certfile \\\n
\ # /etc/ssl/certs/tls.crt \\\n # --ssl-keyfile \\\n # /etc/ssl/certs/tls.key\""
command:
- /bin/bash
- -c
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-dev:v0.2.2
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 300
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 200
periodSeconds: 30
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /etc/ssl/certs
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
worker:
containers:
- args:
- "\nif [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer
RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/*
2>/dev/null\n grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/*
2>/dev/null\n\n KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE
v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n
\ # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*;
do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\"
]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\"
# Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n
\ echo \"[Infer RoCE] Check if the port state file ${port_state_file}
exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ]
&& grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\"
${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found
active HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n
\ echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n
\ fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\";
do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active
HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements
with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo
\"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n
\ export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hcas}}\n export
UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n echo \"[Infer RoCE]
NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n
\ else\n echo \"[Infer RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA
will not be set.\"\n fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo
\"[Infer RoCE] Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n
\ # For SR-IOV environments, find the most common IPv4 RoCE v2 GID index
across all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n
\ for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer
RoCE] Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4
GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*;
do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\"
2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n
\ # Check for IPv4 GID (contains ffff:)\n if
[ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat
\"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer
RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n
\ hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]}
+ 1))\n break # Use first found IPv4 GID per HCA\n
\ fi\n fi\n done\n done\n\n #
Find the most common GID index (most likely to be consistent across nodes)\n
\ best_gid_index=\"\"\n max_count=0\n for idx in \"${!gid_index_count[@]}\";
do\n count=${gid_index_count[\"${idx}\"]}\n echo \"[Infer
RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n if [ $count -gt
$max_count ]; then\n max_count=$count\n best_gid_index=\"$idx\"\n
\ fi\n done\n\n # Use deterministic fallback if counts
are equal - prefer lower index number \n if [ ${#gid_index_count[@]}
-gt 1 ]; then\n echo \"[Infer RoCE] Multiple GID indices found,
selecting most common: ${best_gid_index}\"\n # If there's a tie,
prefer index 3 as it's most common in SR-IOV setups\n if [ -n \"${gid_index_count['3']}\"
] && [ \"${gid_index_count['3']}\" -eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n
\ echo \"[Infer RoCE] Using deterministic fallback: GID_INDEX=3
(SR-IOV standard)\"\n fi\n fi\n\n # Check if GID_INDEX
is already set via environment variables\n if [ -n \"${NCCL_IB_GID_INDEX}\"
]; then\n echo \"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX}
from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n
\ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n
\ echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX}
for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n
\ echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found
on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n
\ export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n
\ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n
\ echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL,
NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No
valid IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n
\ fi\n else\n echo \"[Infer RoCE] No active HCAs found, skipping
GID_INDEX inference.\"\n fi\nfi\n\nSTART_RANK=$(( ${LWS_WORKER_INDEX:-0}
* {{ or .Spec.Prefill.Parallelism.DataLocal 1 }} ))\neval \"vllm serve \\\n
\ /mnt/models \\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n
\ --port 8000 \\\n --disable-log-requests \\\n {{- if .Spec.Prefill.Parallelism.Expert
}}--enable-expert-parallel{{- end }} \\\n {{- if .Spec.Prefill.Parallelism.Tensor
}}--tensor-parallel-size {{ .Spec.Prefill.Parallelism.Tensor }}{{- end }}
\\\n --data-parallel-size {{ or .Spec.Prefill.Parallelism.Data 1 }} \\\n
\ --data-parallel-size-local {{ or .Spec.Prefill.Parallelism.DataLocal 1
}} \\\n --data-parallel-address $(LWS_LEADER_ADDRESS) \\\n --data-parallel-rpc-port
{{ if .Spec.Prefill.Parallelism.DataRPCPort }}{{ .Spec.Prefill.Parallelism.DataRPCPort
}}{{ else }}5555{{- end }} \\\n --data-parallel-start-rank $START_RANK
\\\n ${VLLM_ADDITIONAL_ARGS} \\\n --trust-remote-code \\\n --headless\"
\ \n # BackendTLSPolicy is not implemented yet so disable
SSL for now\n # --enable-ssl-refresh \\\n # --ssl-certfile \\\n # /etc/ssl/certs/tls.crt
\\\n # --ssl-keyfile \\\n # /etc/ssl/certs/tls.key\""
command:
- /bin/bash
- -c
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-dev:v0.2.2
imagePullPolicy: IfNotPresent
name: main
ports:
- containerPort: 8000
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /etc/ssl/certs
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-router-route
namespace: kserve
spec:
router:
route:
http:
spec:
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: '{{ .GlobalConfig.IngressGatewayName }}'
namespace: '{{ .GlobalConfig.IngressGatewayNamespace }}'
rules:
- backendRefs:
- group: inference.networking.x-k8s.io
kind: InferencePool
name: '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
port: 8000
weight: 1
filters:
- type: URLRewrite
urlRewrite:
path:
replacePrefixMatch: /
type: ReplacePrefixMatch
matches:
- path:
type: PathPrefix
value: /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}
timeouts:
backendRequest: 0s
request: 0s
---
apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-scheduler
namespace: kserve
spec:
router:
scheduler:
pool:
spec:
extensionRef:
failureMode: FailOpen
kind: Service
name: '{{ ChildName .ObjectMeta.Name `-epp-service` }}'
selector: {}
targetPortNumber: 8000
template:
containers:
- args:
- --poolName
- '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
- --poolNamespace
- '{{ .ObjectMeta.Namespace }}'
- --zap-encoder
- json
- --grpcPort
- "9002"
- --grpcHealthPort
- "9003"
image: ghcr.io/llm-d/llm-d-inference-scheduler:v0.2.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
grpc:
port: 9003
service: envoy.service.ext_proc.v3.ExternalProcessor
initialDelaySeconds: 5
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: main
ports:
- containerPort: 9002
name: grpc
protocol: TCP
- containerPort: 9003
name: grpc-health
protocol: TCP
- containerPort: 9090
name: metrics
protocol: TCP
readinessProbe:
failureThreshold: 3
grpc:
port: 9003
service: envoy.service.ext_proc.v3.ExternalProcessor
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources:
requests:
cpu: 256m
memory: 500Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /etc/ssl/certs
name: tls-certs
readOnly: true
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 30
volumes:
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs`
}}'
---
apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-template
namespace: kserve
spec:
template:
containers:
- args:
- --served-model-name
- '{{ .Spec.Model.Name }}'
- --port
- "8000"
- --disable-log-requests
command:
- vllm
- serve
- /mnt/models
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-dev:v0.2.2
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 120
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /etc/ssl/certs
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
---
apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceServiceConfig
metadata:
name: kserve-config-llm-worker-data-parallel
namespace: kserve
spec:
template:
containers:
- args:
- "\nif [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer
RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n
\ grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n
\ KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE
v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n
\ # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*;
do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\"
]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\"
# Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n
\ echo \"[Infer RoCE] Check if the port state file ${port_state_file}
exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] &&
grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\"
${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active
HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n
\ echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n
\ fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\";
do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active
HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements
with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo
\"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n
\ export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n
\ echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer
RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer
RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n
\ fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE]
Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n #
For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across
all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n\n
\ for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer RoCE]
Processing HCA: ${hca_name}\"\n\n # Find all RoCE v2 IPv4 GIDs for
this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*;
do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\"
2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n
\ # Check for IPv4 GID (contains ffff:)\n if
[ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat
\"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer
RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n
\ hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]}
+ 1))\n break # Use first found IPv4 GID per HCA\n fi\n
\ fi\n done\n done\n\n # Find the most common
GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n
\ max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n
\ echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n
\ if [ $count -gt $max_count ]; then\n max_count=$count\n
\ best_gid_index=\"$idx\"\n fi\n done\n\n #
Use deterministic fallback if counts are equal - prefer lower index number
\ \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer
RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n
\ # If there's a tie, prefer index 3 as it's most common in SR-IOV
setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\"
-eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo
\"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n
\ fi\n fi\n\n # Check if GID_INDEX is already set via environment
variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo
\"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX}
from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n
\ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n
\ echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX}
for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n
\ echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found
on ${max_count} HCAs)\"\n\n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n
\ export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n
\ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n\n
\ echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL,
NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid
IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n
\ else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX
inference.\"\n fi\nfi\n\nSTART_RANK=0\neval \"vllm serve \\\n /mnt/models
\\\n --served-model-name \"{{ .Spec.Model.Name }}\" \\\n --port 8000 \\\n
\ --api-server-count ${VLLM_API_SERVER_COUNT:-8} \\\n --disable-log-requests
\\\n {{- if .Spec.Parallelism.Expert -}}--enable-expert-parallel{{- end }}
\\\n {{- if .Spec.Parallelism.Tensor -}}--tensor-parallel-size {{ .Spec.Parallelism.Tensor
}}{{- end }} \\\n --data-parallel-size {{ or .Spec.Parallelism.Data 1 }}
\\\n --data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \\\n
\ --data-parallel-address $(LWS_LEADER_ADDRESS) \\\n --data-parallel-rpc-port
{{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{
else }}5555{{- end }} \\\n --data-parallel-start-rank $START_RANK \\\n ${VLLM_ADDITIONAL_ARGS}
\\\n --trust-remote-code\"\n # BackendTLSPolicy is not implemented yet so
disable SSL for now\n # --enable-ssl-refresh \\\n # --ssl-certfile \\\n
\ # /etc/ssl/certs/tls.crt \\\n # --ssl-keyfile \\\n # /etc/ssl/certs/tls.key\""
command:
- /bin/bash
- -c
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-dev:v0.2.2
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 300
periodSeconds: 10
timeoutSeconds: 10
name: main
ports:
- containerPort: 8000
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /health
port: 8000
scheme: HTTP
initialDelaySeconds: 200
periodSeconds: 30
timeoutSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /etc/ssl/certs
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
worker:
containers:
- args:
- "\nif [ \"$KSERVE_INFER_ROCE\" = \"true\" ]; then\n echo \"Trying to infer
RoCE configs ... \"\n grep -H . /sys/class/infiniband/*/ports/*/gids/* 2>/dev/null\n
\ grep -H . /sys/class/infiniband/*/ports/*/gid_attrs/types/* 2>/dev/null\n\n
\ KSERVE_INFER_IB_GID_INDEX_GREP=${KSERVE_INFER_IB_GID_INDEX_GREP:-\"RoCE
v2\"}\n\n echo \"[Infer RoCE] Discovering active HCAs ...\"\n active_hcas=()\n
\ # Loop through all mlx5 devices found in sysfs\n for hca_dir in /sys/class/infiniband/mlx5_*;
do\n # Ensure it's a directory before proceeding\n if [ -d \"$hca_dir\"
]; then\n hca_name=$(basename \"$hca_dir\")\n port_state_file=\"$hca_dir/ports/1/state\"
# Assume port 1\n type_file=\"$hca_dir/ports/1/gid_attrs/types/*\"\n\n
\ echo \"[Infer RoCE] Check if the port state file ${port_state_file}
exists and contains 'ACTIVE'\"\n if [ -f \"$port_state_file\" ] &&
grep -q \"ACTIVE\" \"$port_state_file\" && grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\"
${type_file} 2>/dev/null; then\n echo \"[Infer RoCE] Found active
HCA: $hca_name\"\n active_hcas+=(\"$hca_name\")\n else\n
\ echo \"[Infer RoCE] Skipping inactive or down HCA: $hca_name\"\n
\ fi\n fi\n done\n\n ucx_hcas=()\n for hca in \"${active_hcas[@]}\";
do\n ucx_hcas+=(\"${hca}:1\")\n done\n\n # Check if we found any active
HCAs\n if [ ${#active_hcas[@]} -gt 0 ]; then\n # Join the array elements
with a comma\n hcas=$(IFS=,; echo \"${active_hcas[*]}\")\n echo
\"[Infer RoCE] Setting active HCAs: ${hcas}\"\n export NCCL_IB_HCA=${NCCL_IB_HCA:-${hcas}}\n
\ export NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST:-${hcas}}\n export UCX_NET_DEVICES=${UCX_NET_DEVICES:-${ucx_hcas}}\n\n
\ echo \"[Infer RoCE] NCCL_IB_HCA=${NCCL_IB_HCA}\"\n echo \"[Infer
RoCE] NVSHMEM_HCA_LIST=${NVSHMEM_HCA_LIST}\"\n else\n echo \"[Infer
RoCE] WARNING: No active RoCE HCAs found. NCCL_IB_HCA will not be set.\"\n
\ fi\n\n if [ ${#active_hcas[@]} -gt 0 ]; then\n echo \"[Infer RoCE]
Finding GID_INDEX for each active HCA (SR-IOV compatible)...\"\n\n #
For SR-IOV environments, find the most common IPv4 RoCE v2 GID index across
all HCAs\n declare -A gid_index_count\n declare -A hca_gid_index\n
\ \n for hca_name in \"${active_hcas[@]}\"; do\n echo \"[Infer
RoCE] Processing HCA: ${hca_name}\"\n \n # Find all RoCE
v2 IPv4 GIDs for this HCA and count by index\n for tpath in /sys/class/infiniband/${hca_name}/ports/1/gid_attrs/types/*;
do\n if grep -q \"${KSERVE_INFER_IB_GID_INDEX_GREP}\" \"$tpath\"
2>/dev/null; then\n idx=$(basename \"$tpath\")\n gid_file=\"/sys/class/infiniband/${hca_name}/ports/1/gids/${idx}\"\n
\ # Check for IPv4 GID (contains ffff:)\n if
[ -f \"$gid_file\" ] && grep -q \"ffff:\" \"$gid_file\"; then\n gid_value=$(cat
\"$gid_file\" 2>/dev/null || echo \"\")\n echo \"[Infer
RoCE] Found IPv4 RoCE v2 GID for ${hca_name}: index=${idx}, gid=${gid_value}\"\n
\ hca_gid_index[\"${hca_name}\"]=\"${idx}\"\n gid_index_count[\"${idx}\"]=$((${gid_index_count[\"${idx}\"]}
+ 1))\n break # Use first found IPv4 GID per HCA\n fi\n
\ fi\n done\n done\n\n # Find the most common
GID index (most likely to be consistent across nodes)\n best_gid_index=\"\"\n
\ max_count=0\n for idx in \"${!gid_index_count[@]}\"; do\n count=${gid_index_count[\"${idx}\"]}\n
\ echo \"[Infer RoCE] GID_INDEX ${idx} found on ${count} HCAs\"\n
\ if [ $count -gt $max_count ]; then\n max_count=$count\n
\ best_gid_index=\"$idx\"\n fi\n done\n\n #
Use deterministic fallback if counts are equal - prefer lower index number
\ \n if [ ${#gid_index_count[@]} -gt 1 ]; then\n echo \"[Infer
RoCE] Multiple GID indices found, selecting most common: ${best_gid_index}\"\n
\ # If there's a tie, prefer index 3 as it's most common in SR-IOV
setups\n if [ -n \"${gid_index_count['3']}\" ] && [ \"${gid_index_count['3']}\"
-eq \"$max_count\" ]; then\n best_gid_index=\"3\"\n echo
\"[Infer RoCE] Using deterministic fallback: GID_INDEX=3 (SR-IOV standard)\"\n
\ fi\n fi\n\n # Check if GID_INDEX is already set via environment
variables\n if [ -n \"${NCCL_IB_GID_INDEX}\" ]; then\n echo
\"[Infer RoCE] Using pre-configured NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX}
from environment\"\n export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n
\ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$NCCL_IB_GID_INDEX}\n
\ echo \"[Infer RoCE] Using hardcoded GID_INDEX=${NCCL_IB_GID_INDEX}
for NCCL, NVSHMEM, and UCX\"\n elif [ -n \"$best_gid_index\" ]; then\n
\ echo \"[Infer RoCE] Selected GID_INDEX: ${best_gid_index} (found
on ${max_count} HCAs)\"\n \n export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-$best_gid_index}\n
\ export NVSHMEM_IB_GID_INDEX=${NVSHMEM_IB_GID_INDEX:-$best_gid_index}\n
\ export UCX_IB_GID_INDEX=${UCX_IB_GID_INDEX:-$best_gid_index}\n \n
\ echo \"[Infer RoCE] Exported GID_INDEX=${best_gid_index} for NCCL,
NVSHMEM, and UCX\"\n else\n echo \"[Infer RoCE] ERROR: No valid
IPv4 ${KSERVE_INFER_IB_GID_INDEX_GREP} GID_INDEX found on any HCA.\"\n fi\n
\ else\n echo \"[Infer RoCE] No active HCAs found, skipping GID_INDEX
inference.\"\n fi\nfi\n\nSTART_RANK=$(( ${LWS_WORKER_INDEX:-0} * {{ or .Spec.Parallelism.DataLocal
1 }} ))\neval \"vllm serve \\\n /mnt/models \\\n --served-model-name \"{{
.Spec.Model.Name }}\" \\\n --port 8000 \\\n --disable-log-requests \\\n
\ {{- if .Spec.Parallelism.Expert }}--enable-expert-parallel{{- end }} \\\n
\ {{- if .Spec.Parallelism.Tensor }}--tensor-parallel-size {{ .Spec.Parallelism.Tensor
}}{{- end }} \\\n --data-parallel-size {{ or .Spec.Parallelism.Data 1 }}
\\\n --data-parallel-size-local {{ or .Spec.Parallelism.DataLocal 1 }} \\\n
\ --data-parallel-address $(LWS_LEADER_ADDRESS) \\\n --data-parallel-rpc-port
{{ if .Spec.Parallelism.DataRPCPort }}{{ .Spec.Parallelism.DataRPCPort }}{{
else }}5555{{- end }} \\\n --data-parallel-start-rank $START_RANK \\\n ${VLLM_ADDITIONAL_ARGS}
\\\n --trust-remote-code \\\n --headless\"\n # BackendTLSPolicy is not
implemented yet so disable SSL for now\n # --enable-ssl-refresh \\\n # --ssl-certfile
\\\n # /etc/ssl/certs/tls.crt \\\n # --ssl-keyfile \\\n # /etc/ssl/certs/tls.key"
command:
- /bin/bash
- -c
env:
- name: HOME
value: /home
- name: VLLM_LOGGING_LEVEL
value: INFO
- name: HF_HUB_CACHE
value: /models
image: ghcr.io/llm-d/llm-d-dev:v0.2.2
imagePullPolicy: IfNotPresent
name: main
ports:
- containerPort: 8000
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- IPC_LOCK
- SYS_RAWIO
- NET_RAW
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: false
seccompProfile:
type: RuntimeDefault
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /home
name: home
- mountPath: /dev/shm
name: dshm
- mountPath: /models
name: model-cache
- mountPath: /etc/ssl/certs
name: tls-certs
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- emptyDir: {}
name: home
- emptyDir:
medium: Memory
sizeLimit: 1Gi
name: dshm
- emptyDir: {}
name: model-cache
- name: tls-certs
secret:
secretName: '{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}'
Reference in New Issue View Git Blame Copy Permalink