mirror of
https://github.com/hanzo-ml/kserve.git
synced 2026-10-30 20:53:49 +00:00
7d3d0c9ee6
Signed-off-by: greg pereira <[email protected]> Signed-off-by: Andres Llausas <[email protected]> Co-authored-by: Andres Llausas <[email protected]> Co-authored-by: Claude Sonnet 4.5 <[email protected]>
88 lines
2.5 KiB
YAML
88 lines
2.5 KiB
YAML
# Precise prefix KV cache routing with llm-d simulator (CPU, no GPU required)
|
|
#
|
|
# This sample uses the llm-d-inference-sim simulator instead of a real model server,
|
|
# making it suitable for testing on CPU-only clusters.
|
|
apiVersion: serving.kserve.io/v1alpha2
|
|
kind: LLMInferenceService
|
|
metadata:
|
|
name: precise-prefix-test
|
|
spec:
|
|
model:
|
|
uri: hf://facebook/opt-125m
|
|
name: opt-125m
|
|
replicas: 2
|
|
router:
|
|
scheduler:
|
|
template:
|
|
containers:
|
|
- name: tokenizer
|
|
env:
|
|
- name: LOG_LEVEL
|
|
value: DEBUG
|
|
config:
|
|
inline:
|
|
apiVersion: inference.networking.x-k8s.io/v1alpha1
|
|
kind: EndpointPickerConfig
|
|
plugins:
|
|
- type: single-profile-handler
|
|
- type: precise-prefix-cache-scorer
|
|
parameters:
|
|
tokenProcessorConfig:
|
|
blockSize: 16
|
|
hashSeed: "42"
|
|
kvEventsConfig:
|
|
zmqEndpoint: "tcp://*:5557"
|
|
indexerConfig:
|
|
kvBlockIndexConfig:
|
|
enableMetrics: true
|
|
metricsLoggingInterval: 60000000000
|
|
- type: queue-scorer
|
|
- type: kv-cache-utilization-scorer
|
|
- type: max-score-picker
|
|
schedulingProfiles:
|
|
- name: default
|
|
plugins:
|
|
- pluginRef: queue-scorer
|
|
weight: 2
|
|
- pluginRef: kv-cache-utilization-scorer
|
|
weight: 2
|
|
- pluginRef: precise-prefix-cache-scorer
|
|
weight: 3
|
|
- pluginRef: max-score-picker
|
|
route: { }
|
|
gateway: { }
|
|
template:
|
|
containers:
|
|
- name: main
|
|
image: ghcr.io/llm-d/llm-d-inference-sim:v0.8.2
|
|
command: ["/app/llm-d-inference-sim"]
|
|
args:
|
|
- --port
|
|
- "8000"
|
|
- --model
|
|
- "{{ .Spec.Model.Name }}"
|
|
- --mode
|
|
- random
|
|
- --enable-kvcache
|
|
- --block-size
|
|
- "16"
|
|
- --zmq-endpoint
|
|
- "tcp://{{ ChildName .ObjectMeta.Name `-epp-service` }}:5557"
|
|
- --hash-seed
|
|
- "42"
|
|
- --event-batch-size
|
|
- "1"
|
|
env:
|
|
- name: POD_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
apiVersion: v1
|
|
fieldPath: status.podIP
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: 100m
|
|
memory: 100Mi
|