mirror of
https://github.com/hanzo-ml/kserve.git
synced 2026-10-30 20:53:49 +00:00
Vivek Karunai Kiri RagavanandGitHub
4fbf86a8e4
Signed-off-by: Vivek Karunai Kiri Ragavan <[email protected]>
32 lines
818 B
YAML
32 lines
818 B
YAML
apiVersion: serving.kserve.io/v1beta1
|
|
kind: InferenceService
|
|
metadata:
|
|
name: llama-31-8b-instruct
|
|
spec:
|
|
predictor:
|
|
annotations:
|
|
traffic.sidecar.istio.io/excludeOutboundPorts: "443"
|
|
minReplicas: 0
|
|
serviceAccountName: s3sa
|
|
nodeSelector:
|
|
node.kubernetes.io/instance-type: g5.8xlarge
|
|
model:
|
|
modelFormat:
|
|
name: huggingface
|
|
args:
|
|
- --gpu-memory-utilization=0.95
|
|
- --max-model-len=1024
|
|
- --tensor-parallel-size=1
|
|
- --enforce-eager
|
|
- --disable-log-stats
|
|
storageUri: s3://${bucket}/models/meta-llama--Meta-Llama-3.1-8B-Instruct
|
|
resources:
|
|
limits:
|
|
cpu: "24"
|
|
memory: 48Gi
|
|
nvidia.com/gpu: "1"
|
|
requests:
|
|
cpu: "24"
|
|
memory: 48Gi
|
|
nvidia.com/gpu: "1"
|