mirror of
https://github.com/hanzo-ml/kserve.git
synced 2026-10-30 20:53:49 +00:00
f956207415
Signed-off-by: Vivek Karunai Kiri Ragavan <[email protected]> Signed-off-by: Pierangelo Di Pilato <[email protected]> Co-authored-by: Pierangelo Di Pilato <[email protected]>
67 lines
2.1 KiB
YAML
67 lines
2.1 KiB
YAML
---
|
|
apiVersion: serving.kserve.io/v1alpha2
|
|
kind: LLMInferenceServiceConfig
|
|
metadata:
|
|
name: llmisvc-intelligent-inference-scheduling
|
|
spec:
|
|
template:
|
|
containers:
|
|
# Override image version and command from base config
|
|
- image: ghcr.io/llm-d/llm-d-cuda:v0.7.0
|
|
name: main
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- >
|
|
vllm serve /mnt/models
|
|
--served-model-name "{{ .Spec.Model.Name }}"
|
|
--port 8000
|
|
--disable-access-log-for-endpoints /health,/metrics,/ping
|
|
--disable-log-requests
|
|
$VLLM_ADDITIONAL_ARGS
|
|
# Override security context
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
# Override liveness probe initial delay
|
|
livenessProbe:
|
|
initialDelaySeconds: 1200
|
|
# Override readiness probe settings
|
|
readinessProbe:
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 10
|
|
router:
|
|
scheduler:
|
|
pool:
|
|
spec:
|
|
endpointPickerRef:
|
|
group: ""
|
|
template:
|
|
containers:
|
|
- name: main
|
|
# Override scheduler image version
|
|
image: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0-rc.2
|
|
# Override args to enable secure serving
|
|
args:
|
|
- --pool-name
|
|
- '{{ ChildName .ObjectMeta.Name `-inference-pool` }}'
|
|
- --pool-namespace
|
|
- '{{ .ObjectMeta.Namespace }}'
|
|
- --zap-encoder
|
|
- json
|
|
- --grpc-port
|
|
- "9002"
|
|
- --grpc-health-port
|
|
- "9003"
|
|
- --metrics-endpoint-auth=false
|
|
- --secure-serving
|
|
- --cert-path
|
|
- /etc/ssl/certs
|
|
# Override grpc service names and ports for probes
|
|
livenessProbe:
|
|
grpc:
|
|
port: 9003
|
|
service: envoy.service.ext_proc.v3.ExternalProcessor
|
|
readinessProbe:
|
|
grpc:
|
|
port: 9003
|
|
service: envoy.service.ext_proc.v3.ExternalProcessor
|