mirror of
https://github.com/hanzo-ml/kserve.git
synced 2026-10-30 20:53:49 +00:00
48 lines
1.3 KiB
YAML
48 lines
1.3 KiB
YAML
apiVersion: serving.kserve.io/v1alpha2
|
|
kind: LLMInferenceService
|
|
metadata:
|
|
name: qwen2-7b-instruct-dra-multi
|
|
# Multi-device claim with CEL selectors and explicit container targeting.
|
|
annotations:
|
|
serving.kserve.io/exp-dra-device-class: gpu.nvidia.com
|
|
serving.kserve.io/exp-dra-device-count: "2"
|
|
serving.kserve.io/exp-dra-container-name: main
|
|
serving.kserve.io/exp-dra-cel-selector: |
|
|
device.attributes['gpu.nvidia.com']['type'] == 'A100'
|
|
device.capacity['gpu.nvidia.com']['memory'].compareTo(quantity('40Gi')) >= 0
|
|
spec:
|
|
model:
|
|
uri: hf://Qwen/Qwen2.5-7B-Instruct
|
|
name: Qwen/Qwen2.5-7B-Instruct
|
|
replicas: 1
|
|
router:
|
|
scheduler: { }
|
|
route: { }
|
|
gateway: { }
|
|
template:
|
|
containers:
|
|
- name: sidecar
|
|
image: busybox:1.36
|
|
command: [ "sh", "-c", "while true; do sleep 3600; done" ]
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
- name: main
|
|
resources:
|
|
limits:
|
|
cpu: '8'
|
|
memory: 64Gi
|
|
requests:
|
|
cpu: '4'
|
|
memory: 32Gi
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8000
|
|
scheme: HTTPS
|
|
initialDelaySeconds: 120
|
|
periodSeconds: 30
|
|
timeoutSeconds: 30
|
|
failureThreshold: 5
|