mirror of
https://github.com/hanzo-ml/kserve.git
synced 2026-10-30 20:53:49 +00:00
0cc7b33d97
Signed-off-by: ZabinskiMichal <[email protected]> Signed-off-by: Michał Żabiński <[email protected]> Signed-off-by: Spolti <[email protected]> Co-authored-by: Spolti <[email protected]>
813 lines
19 KiB
YAML
Generated
813 lines
19 KiB
YAML
Generated
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/server-type: autogluonserver
|
|
name: kserve-autogluonserver
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8080"
|
|
containers:
|
|
- args:
|
|
- --model_name={{.Name}}
|
|
- --model_dir=/mnt/models
|
|
- --http_port=8080
|
|
image: kserve/autogluonserver:latest
|
|
name: kserve-container
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
privileged: false
|
|
runAsNonRoot: true
|
|
protocolVersions:
|
|
- v1
|
|
- v2
|
|
supportedModelFormats:
|
|
- autoSelect: true
|
|
name: autogluon
|
|
priority: 1
|
|
version: "1"
|
|
---
|
|
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/server-type: huggingfaceserver
|
|
name: kserve-huggingfaceserver
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8080"
|
|
containers:
|
|
- args:
|
|
- --model_name={{.Name}}
|
|
env:
|
|
- name: LMCACHE_USE_EXPERIMENTAL
|
|
value: "True"
|
|
image: kserve/huggingfaceserver:latest
|
|
name: kserve-container
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
privileged: false
|
|
runAsNonRoot: true
|
|
volumeMounts:
|
|
- mountPath: /dev/shm
|
|
name: devshm
|
|
hostIPC: false
|
|
protocolVersions:
|
|
- v2
|
|
- v1
|
|
supportedModelFormats:
|
|
- autoSelect: true
|
|
name: huggingface
|
|
priority: 1
|
|
version: "1"
|
|
volumes:
|
|
- emptyDir:
|
|
medium: Memory
|
|
name: devshm
|
|
---
|
|
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/server-type: huggingfaceserver
|
|
name: kserve-huggingfaceserver-multinode
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8080"
|
|
containers:
|
|
- args:
|
|
- --model_name={{.Name}}
|
|
command:
|
|
- bash
|
|
- -c
|
|
- "export MODEL_DIR_ARG=\"\"\nif [[ ! -z ${MODEL_ID} ]]\nthen\n export MODEL_DIR_ARG=\"--model_dir=${MODEL_ID}\"\nfi\n\nif
|
|
[[ ! -z ${MODEL_DIR} ]]\nthen\n export MODEL_DIR_ARG=\"--model_dir=${MODEL_DIR}\"\nfi\n\nexport
|
|
RAY_ADDRESS=${POD_IP}:${RAY_PORT}\nray start --head --disable-usage-stats --include-dashboard
|
|
false \npython ./huggingfaceserver/health_check.py registered_nodes --retries
|
|
200 --probe_name runtime_start\n\npython -m huggingfaceserver ${MODEL_DIR_ARG}
|
|
--tensor-parallel-size=${TENSOR_PARALLEL_SIZE} --pipeline-parallel-size=${PIPELINE_PARALLEL_SIZE}
|
|
$0 $@\n"
|
|
env:
|
|
- name: RAY_PORT
|
|
value: "6379"
|
|
- name: POD_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
- name: POD_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: status.podIP
|
|
- name: VLLM_CONFIG_ROOT
|
|
value: /tmp
|
|
- name: HF_HUB_CACHE
|
|
value: /tmp
|
|
image: kserve/huggingfaceserver:latest-gpu
|
|
livenessProbe:
|
|
exec:
|
|
command:
|
|
- bash
|
|
- -c
|
|
- |
|
|
python ./huggingfaceserver/health_check.py registered_node_and_runtime_health --health_check_url http://localhost:8080 --probe_name head_liveness
|
|
failureThreshold: 2
|
|
periodSeconds: 5
|
|
successThreshold: 1
|
|
timeoutSeconds: 15
|
|
name: kserve-container
|
|
readinessProbe:
|
|
exec:
|
|
command:
|
|
- bash
|
|
- -c
|
|
- |
|
|
python ./huggingfaceserver/health_check.py runtime_health --health_check_url http://localhost:8080 --probe_name head_readiness
|
|
failureThreshold: 2
|
|
periodSeconds: 5
|
|
successThreshold: 1
|
|
timeoutSeconds: 15
|
|
resources:
|
|
limits:
|
|
cpu: "4"
|
|
memory: 12Gi
|
|
requests:
|
|
cpu: "2"
|
|
memory: 6Gi
|
|
startupProbe:
|
|
exec:
|
|
command:
|
|
- bash
|
|
- -c
|
|
- |
|
|
python ./huggingfaceserver/health_check.py registered_node_and_runtime_health --health_check_url http://localhost:8080 --probe_name head_startup
|
|
failureThreshold: 40
|
|
initialDelaySeconds: 60
|
|
periodSeconds: 30
|
|
successThreshold: 1
|
|
timeoutSeconds: 30
|
|
volumeMounts:
|
|
- mountPath: /dev/shm
|
|
name: shm
|
|
protocolVersions:
|
|
- v2
|
|
- v1
|
|
supportedModelFormats:
|
|
- autoSelect: true
|
|
name: huggingface
|
|
priority: 2
|
|
version: "1"
|
|
volumes:
|
|
- emptyDir:
|
|
medium: Memory
|
|
sizeLimit: 3Gi
|
|
name: shm
|
|
workerSpec:
|
|
containers:
|
|
- command:
|
|
- bash
|
|
- -c
|
|
- "export RAY_HEAD_ADDRESS=${HEAD_SVC}.${POD_NAMESPACE}.svc.cluster.local:6379\nSECONDS=0\n\nwhile
|
|
true; do \n if (( SECONDS <= 240 )); then\n if ray health-check
|
|
--address \"${RAY_HEAD_ADDRESS}\" > /dev/null 2>&1; then\n echo \"Ray
|
|
Global Control Service(GCS) is ready.\"\n break\n fi\n echo \"$SECONDS
|
|
seconds elapsed: Waiting for Ray Global Control Service(GCS) to be ready.\"\n
|
|
\ else\n if ray health-check --address \"${RAY_HEAD_ADDRESS}\"; then\n
|
|
\ echo \"Ray Global Control Service(GCS) is ready. Any error messages
|
|
above can be safely ignored.\"\n break\n fi\n echo \"$SECONDS seconds
|
|
elapsed: Still waiting for Ray Global Control Service(GCS) to be ready.\"\n
|
|
\ fi\n\n sleep 5\ndone\n\necho \"Attempting to connect to Ray cluster at
|
|
$RAY_HEAD_ADDRESS ...\"\nray start --address=\"${RAY_HEAD_ADDRESS}\" --block\n"
|
|
env:
|
|
- name: POD_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.name
|
|
- name: POD_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
image: kserve/huggingfaceserver:latest-gpu
|
|
livenessProbe:
|
|
exec:
|
|
command:
|
|
- bash
|
|
- -c
|
|
- |
|
|
export RAY_ADDRESS=${HEAD_SVC}.${POD_NAMESPACE}.svc.cluster.local:6379
|
|
python ./huggingfaceserver/health_check.py registered_nodes --probe_name worker_liveness
|
|
failureThreshold: 2
|
|
periodSeconds: 5
|
|
successThreshold: 1
|
|
timeoutSeconds: 15
|
|
name: worker-container
|
|
resources:
|
|
limits:
|
|
cpu: "4"
|
|
memory: 12Gi
|
|
requests:
|
|
cpu: "2"
|
|
memory: 6Gi
|
|
startupProbe:
|
|
exec:
|
|
command:
|
|
- bash
|
|
- -c
|
|
- |
|
|
export RAY_HEAD_NODE=${HEAD_SVC}.${POD_NAMESPACE}.svc.cluster.local
|
|
export RAY_ADDRESS=${RAY_HEAD_NODE}:6379
|
|
python ./huggingfaceserver/health_check.py registered_node_and_runtime_models --runtime_url http://${RAY_HEAD_NODE}:8080/v1/models --probe_name worker_startup
|
|
failureThreshold: 40
|
|
initialDelaySeconds: 60
|
|
periodSeconds: 30
|
|
successThreshold: 1
|
|
timeoutSeconds: 30
|
|
volumeMounts:
|
|
- mountPath: /dev/shm
|
|
name: shm
|
|
pipelineParallelSize: 1
|
|
tensorParallelSize: 1
|
|
volumes:
|
|
- emptyDir:
|
|
medium: Memory
|
|
sizeLimit: 3Gi
|
|
name: shm
|
|
---
|
|
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/server-type: lgbserver
|
|
name: kserve-lgbserver
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8080"
|
|
containers:
|
|
- args:
|
|
- --model_name={{.Name}}
|
|
- --model_dir=/mnt/models
|
|
- --http_port=8080
|
|
- --nthread=1
|
|
image: kserve/lgbserver:latest
|
|
name: kserve-container
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
privileged: false
|
|
runAsNonRoot: true
|
|
protocolVersions:
|
|
- v1
|
|
- v2
|
|
supportedModelFormats:
|
|
- autoSelect: true
|
|
name: lightgbm
|
|
priority: 1
|
|
version: "4"
|
|
---
|
|
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/server-type: mlserver
|
|
name: kserve-mlserver
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8080"
|
|
containers:
|
|
- env:
|
|
- name: MLSERVER_MODEL_IMPLEMENTATION
|
|
value: '{{.Labels.modelClass}}'
|
|
- name: MLSERVER_HTTP_PORT
|
|
value: "8080"
|
|
- name: MLSERVER_GRPC_PORT
|
|
value: "9000"
|
|
image: docker.io/seldonio/mlserver:1.7.1
|
|
name: kserve-container
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
privileged: false
|
|
runAsNonRoot: true
|
|
protocolVersions:
|
|
- v2
|
|
supportedModelFormats:
|
|
- autoSelect: true
|
|
name: sklearn
|
|
priority: 2
|
|
version: "0"
|
|
- autoSelect: true
|
|
name: sklearn
|
|
priority: 2
|
|
version: "1"
|
|
- autoSelect: true
|
|
name: xgboost
|
|
priority: 2
|
|
version: "1"
|
|
- autoSelect: true
|
|
name: xgboost
|
|
priority: 2
|
|
version: "2"
|
|
- autoSelect: true
|
|
name: lightgbm
|
|
priority: 2
|
|
version: "3"
|
|
- autoSelect: true
|
|
name: lightgbm
|
|
priority: 2
|
|
version: "4"
|
|
- autoSelect: true
|
|
name: mlflow
|
|
priority: 1
|
|
version: "1"
|
|
- autoSelect: true
|
|
name: mlflow
|
|
priority: 1
|
|
version: "2"
|
|
---
|
|
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/server-type: paddleserver
|
|
name: kserve-paddleserver
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8080"
|
|
containers:
|
|
- args:
|
|
- --model_name={{.Name}}
|
|
- --model_dir=/mnt/models
|
|
- --http_port=8080
|
|
image: kserve/paddleserver:latest
|
|
name: kserve-container
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
privileged: false
|
|
runAsNonRoot: true
|
|
protocolVersions:
|
|
- v1
|
|
- v2
|
|
supportedModelFormats:
|
|
- autoSelect: true
|
|
name: paddle
|
|
priority: 1
|
|
version: "2"
|
|
---
|
|
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/server-type: pmmlserver
|
|
name: kserve-pmmlserver
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8080"
|
|
containers:
|
|
- args:
|
|
- --model_name={{.Name}}
|
|
- --model_dir=/mnt/models
|
|
- --http_port=8080
|
|
image: kserve/pmmlserver:latest
|
|
name: kserve-container
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
privileged: false
|
|
runAsNonRoot: true
|
|
protocolVersions:
|
|
- v1
|
|
- v2
|
|
supportedModelFormats:
|
|
- autoSelect: true
|
|
name: pmml
|
|
priority: 1
|
|
version: "3"
|
|
- autoSelect: true
|
|
name: pmml
|
|
priority: 1
|
|
version: "4"
|
|
---
|
|
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/server-type: predictiveserver
|
|
name: kserve-predictiveserver
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8080"
|
|
containers:
|
|
- args:
|
|
- --model_name={{.Name}}
|
|
- --model_dir=/mnt/models
|
|
- --http_port=8080
|
|
- --framework={{.Annotations.modelFormat}}
|
|
- --nthread=1
|
|
image: kserve/predictiveserver:latest
|
|
name: kserve-container
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
privileged: false
|
|
runAsNonRoot: true
|
|
multiModel: false
|
|
protocolVersions:
|
|
- v1
|
|
- v2
|
|
supportedModelFormats:
|
|
- autoSelect: false
|
|
name: sklearn
|
|
priority: 3
|
|
version: "1"
|
|
- autoSelect: false
|
|
name: xgboost
|
|
priority: 3
|
|
version: "2"
|
|
- autoSelect: false
|
|
name: lightgbm
|
|
priority: 3
|
|
version: "4"
|
|
---
|
|
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/server-type: sklearnserver
|
|
name: kserve-sklearnserver
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8080"
|
|
containers:
|
|
- args:
|
|
- --model_name={{.Name}}
|
|
- --model_dir=/mnt/models
|
|
- --http_port=8080
|
|
image: kserve/sklearnserver:latest
|
|
name: kserve-container
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
privileged: false
|
|
runAsNonRoot: true
|
|
protocolVersions:
|
|
- v1
|
|
- v2
|
|
supportedModelFormats:
|
|
- autoSelect: true
|
|
name: sklearn
|
|
priority: 1
|
|
version: "1"
|
|
---
|
|
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/server-type: tensorflow-serving
|
|
name: kserve-tensorflow-serving
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8080"
|
|
containers:
|
|
- args:
|
|
- --model_name={{.Name}}
|
|
- --port=9000
|
|
- --rest_api_port=8080
|
|
- --model_base_path=/mnt/models
|
|
- --rest_api_timeout_in_ms=60000
|
|
command:
|
|
- /usr/bin/tensorflow_model_server
|
|
image: tensorflow/serving:2.6.2
|
|
name: kserve-container
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
privileged: false
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
protocolVersions:
|
|
- v1
|
|
- grpc-v1
|
|
supportedModelFormats:
|
|
- autoSelect: true
|
|
name: tensorflow
|
|
priority: 2
|
|
version: "1"
|
|
- autoSelect: true
|
|
name: tensorflow
|
|
priority: 2
|
|
version: "2"
|
|
---
|
|
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/server-type: torchserve
|
|
name: kserve-torchserve
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8082"
|
|
containers:
|
|
- args:
|
|
- torchserve
|
|
- --start
|
|
- --model-store=/mnt/models/model-store
|
|
- --ts-config=/mnt/models/config/config.properties
|
|
env:
|
|
- name: TS_SERVICE_ENVELOPE
|
|
value: '{{.Labels.serviceEnvelope}}'
|
|
image: pytorch/torchserve-kfs:0.9.0
|
|
name: kserve-container
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
privileged: false
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
protocolVersions:
|
|
- v1
|
|
- v2
|
|
- grpc-v2
|
|
supportedModelFormats:
|
|
- autoSelect: true
|
|
name: pytorch
|
|
priority: 2
|
|
version: "1"
|
|
---
|
|
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/server-type: tritonserver
|
|
name: kserve-tritonserver
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8002"
|
|
containers:
|
|
- args:
|
|
- tritonserver
|
|
- --model-store=/mnt/models
|
|
- --grpc-port=9000
|
|
- --http-port=8080
|
|
- --allow-grpc=true
|
|
- --allow-http=true
|
|
image: nvcr.io/nvidia/tritonserver:23.05-py3
|
|
name: kserve-container
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
privileged: false
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
protocolVersions:
|
|
- v2
|
|
- grpc-v2
|
|
supportedModelFormats:
|
|
- autoSelect: true
|
|
name: tensorrt
|
|
priority: 1
|
|
version: "8"
|
|
- autoSelect: true
|
|
name: tensorflow
|
|
priority: 1
|
|
version: "1"
|
|
- autoSelect: true
|
|
name: tensorflow
|
|
priority: 1
|
|
version: "2"
|
|
- autoSelect: true
|
|
name: onnx
|
|
priority: 1
|
|
version: "1"
|
|
- name: pytorch
|
|
version: "1"
|
|
- autoSelect: true
|
|
name: triton
|
|
priority: 1
|
|
version: "2"
|
|
---
|
|
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/server-type: vllmserver
|
|
name: kserve-vllmserver
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8080"
|
|
containers:
|
|
- args:
|
|
- --port=8080
|
|
- --served-model-name={{.Name}}
|
|
- --model=/mnt/models
|
|
command:
|
|
- python
|
|
- -m
|
|
- vllm.entrypoints.openai.api_server
|
|
env:
|
|
- name: LMCACHE_USE_EXPERIMENTAL
|
|
value: "True"
|
|
- name: HF_HOME
|
|
value: /tmp
|
|
- name: VLLM_CONFIG_ROOT
|
|
value: /tmp
|
|
image: vllm/vllm-openai:latest
|
|
name: kserve-container
|
|
readinessProbe:
|
|
failureThreshold: 3
|
|
httpGet:
|
|
path: /v1/models
|
|
port: 8080
|
|
periodSeconds: 10
|
|
successThreshold: 1
|
|
timeoutSeconds: 5
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
privileged: false
|
|
startupProbe:
|
|
failureThreshold: 60
|
|
httpGet:
|
|
path: /v1/models
|
|
port: 8080
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 30
|
|
successThreshold: 1
|
|
timeoutSeconds: 10
|
|
volumeMounts:
|
|
- mountPath: /dev/shm
|
|
name: devshm
|
|
hostIPC: false
|
|
supportedModelFormats:
|
|
- autoSelect: true
|
|
name: vLLM
|
|
priority: 1
|
|
version: "1"
|
|
volumes:
|
|
- emptyDir:
|
|
medium: Memory
|
|
name: devshm
|
|
---
|
|
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/server-type: xgbserver
|
|
name: kserve-xgbserver
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8080"
|
|
containers:
|
|
- args:
|
|
- --model_name={{.Name}}
|
|
- --model_dir=/mnt/models
|
|
- --http_port=8080
|
|
- --nthread=1
|
|
image: kserve/xgbserver:latest
|
|
name: kserve-container
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
privileged: false
|
|
runAsNonRoot: true
|
|
protocolVersions:
|
|
- v1
|
|
- v2
|
|
supportedModelFormats:
|
|
- autoSelect: true
|
|
name: xgboost
|
|
priority: 1
|
|
version: "2"
|