mirror of
https://github.com/hanzo-ml/kserve.git
synced 2026-10-30 20:53:49 +00:00
7d3d0c9ee6
Signed-off-by: greg pereira <[email protected]> Signed-off-by: Andres Llausas <[email protected]> Co-authored-by: Andres Llausas <[email protected]> Co-authored-by: Claude Sonnet 4.5 <[email protected]>
175 lines
6.0 KiB
YAML
175 lines
6.0 KiB
YAML
apiVersion: serving.kserve.io/v1alpha2
|
|
kind: LLMInferenceServiceConfig
|
|
metadata:
|
|
name: kserve-config-llm-scheduler
|
|
spec:
|
|
router:
|
|
scheduler:
|
|
pool:
|
|
spec:
|
|
endpointPickerRef:
|
|
failureMode: FailOpen
|
|
kind: Service
|
|
name: |-
|
|
{{ ChildName .ObjectMeta.Name `-epp-service` }}
|
|
port:
|
|
number: 9002
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: |-
|
|
{{ .ObjectMeta.Name }}
|
|
app.kubernetes.io/part-of: llminferenceservice
|
|
kserve.io/component: workload
|
|
targetPorts:
|
|
- number: 8000
|
|
template:
|
|
containers:
|
|
- name: main
|
|
ports:
|
|
- containerPort: 9002
|
|
name: grpc
|
|
protocol: TCP
|
|
- containerPort: 9003
|
|
name: grpc-health
|
|
protocol: TCP
|
|
- containerPort: 9090
|
|
name: metrics
|
|
protocol: TCP
|
|
- containerPort: 5557
|
|
name: zmq
|
|
protocol: TCP
|
|
image: ghcr.io/llm-d/llm-d-inference-scheduler:v0.7.1
|
|
imagePullPolicy: IfNotPresent
|
|
livenessProbe:
|
|
failureThreshold: 3
|
|
grpc:
|
|
port: 9003
|
|
service: liveness
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 10
|
|
successThreshold: 1
|
|
timeoutSeconds: 1
|
|
readinessProbe:
|
|
failureThreshold: 3
|
|
grpc:
|
|
port: 9003
|
|
service: readiness
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 10
|
|
successThreshold: 1
|
|
timeoutSeconds: 1
|
|
command:
|
|
- /app/epp
|
|
- --pool-name
|
|
- "{{ ChildName .ObjectMeta.Name `-inference-pool` }}"
|
|
- --pool-namespace
|
|
- "{{ .ObjectMeta.Namespace }}"
|
|
- --zap-encoder
|
|
- json
|
|
- --grpc-port
|
|
- "9002"
|
|
- --grpc-health-port
|
|
- "9003"
|
|
- '{{ if .GlobalConfig.EnableTLS }}--enable-cert-reload=true{{- end }}'
|
|
- '{{ if .GlobalConfig.EnableTLS }}--secure-serving=true{{- end }}'
|
|
- '{{ if .GlobalConfig.EnableTLS }}--model-server-metrics-scheme=https{{- end }}'
|
|
- '{{ if .GlobalConfig.EnableTLS }}--cert-path=/var/run/kserve/tls{{- end }}'
|
|
env:
|
|
- name: SSL_CERT_DIR
|
|
value: "/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs"
|
|
resources:
|
|
requests:
|
|
cpu: 256m
|
|
memory: 500Mi
|
|
terminationMessagePath: /dev/termination-log
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
runAsNonRoot: true
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
volumeMounts:
|
|
- mountPath: /var/run/kserve/tls
|
|
name: tls-certs
|
|
readOnly: true
|
|
- mountPath: /tmp/tokenizer
|
|
name: tokenizer-uds
|
|
- name: tokenizer
|
|
ports:
|
|
- containerPort: 8082
|
|
name: health
|
|
protocol: TCP
|
|
image: ghcr.io/llm-d/llm-d-uds-tokenizer:v0.7.1
|
|
imagePullPolicy: IfNotPresent
|
|
workingDir: /mnt/models
|
|
env:
|
|
- name: TOKENIZERS_DIR
|
|
value: /mnt/models
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /healthz
|
|
port: 8082
|
|
periodSeconds: 15
|
|
timeoutSeconds: 5
|
|
failureThreshold: 3
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /healthz
|
|
port: 8082
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
failureThreshold: 3
|
|
startupProbe:
|
|
httpGet:
|
|
path: /healthz
|
|
port: 8082
|
|
periodSeconds: 10
|
|
initialDelaySeconds: 5
|
|
timeoutSeconds: 5
|
|
failureThreshold: 60
|
|
resources:
|
|
requests:
|
|
cpu: 256m
|
|
memory: 500Mi
|
|
terminationMessagePath: /dev/termination-log
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
runAsNonRoot: true
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
volumeMounts:
|
|
# The tokenizer image imports torch/dill which requires a writable
|
|
# /tmp at import time (tempfile.gettempdir()). Without this mount,
|
|
# readOnlyRootFilesystem causes the container to crash on startup.
|
|
- mountPath: /tmp
|
|
name: tokenizer-tmp
|
|
# vllm writes model info cache to /.cache/vllm/modelinfos.
|
|
- mountPath: /.cache
|
|
name: tokenizer-cache
|
|
- mountPath: /tmp/tokenizer
|
|
name: tokenizer-uds
|
|
volumes:
|
|
- name: tls-certs
|
|
secret:
|
|
secretName: "{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}"
|
|
- name: tokenizer-uds
|
|
emptyDir: {}
|
|
# Writable /tmp for the tokenizer container (see volumeMounts comment above).
|
|
- name: tokenizer-tmp
|
|
emptyDir: {}
|
|
# Writable /.cache for vllm model info cache.
|
|
- name: tokenizer-cache
|
|
emptyDir: {}
|
|
dnsPolicy: ClusterFirst
|
|
restartPolicy: Always
|
|
terminationGracePeriodSeconds: 30
|