mirror of
https://github.com/hanzo-ml/kserve.git
synced 2026-10-30 20:53:49 +00:00
Vivek Karunai Kiri RagavanandGitHub
4fbf86a8e4
Signed-off-by: Vivek Karunai Kiri Ragavan <[email protected]>
30 lines
734 B
YAML
30 lines
734 B
YAML
apiVersion: serving.kserve.io/v1beta1
|
|
kind: InferenceService
|
|
metadata:
|
|
name: llama-31-8b-instruct
|
|
spec:
|
|
predictor:
|
|
minReplicas: 0
|
|
nodeSelector:
|
|
node.kubernetes.io/instance-type: g5.8xlarge
|
|
model:
|
|
runtime: custom-kserve-huggingfaceserver
|
|
modelFormat:
|
|
name: huggingface
|
|
storageUri: pvc://s3-data/llama-31-8b-instruct
|
|
args:
|
|
- --gpu-memory-utilization=0.95
|
|
- --max-model-len=1024
|
|
- --tensor-parallel-size=1
|
|
- --enforce-eager
|
|
- --disable-log-stats
|
|
resources:
|
|
limits:
|
|
cpu: "24"
|
|
memory: 48Gi
|
|
nvidia.com/gpu: "1"
|
|
requests:
|
|
cpu: "24"
|
|
memory: 48Gi
|
|
nvidia.com/gpu: "1"
|