mirror of
https://github.com/hanzo-ml/kserve.git
synced 2026-10-30 20:53:49 +00:00
Lize CaiandGitHub
fab85e09c2
* update fluid kserve demo to use huggingface servingruntime and other model. Signed-off-by: Lize Cai <[email protected]> * fix lint Signed-off-by: Lize Cai <[email protected]> * explicitly set custom servingruntime, update devshm. Signed-off-by: Lize Cai <[email protected]> * update the args in custom kserve hfserver Signed-off-by: Lize Cai <[email protected]> * address comments Signed-off-by: Lize Cai <[email protected]> * add return of line Signed-off-by: Lize Cai <[email protected]> --------- Signed-off-by: Lize Cai <[email protected]>
68 lines
1.7 KiB
YAML
68 lines
1.7 KiB
YAML
apiVersion: data.fluid.io/v1alpha1
|
|
kind: Dataset
|
|
metadata:
|
|
name: s3-data
|
|
spec:
|
|
mounts:
|
|
- mountPoint: "s3://${bucket}/models/meta-llama--Meta-Llama-3.1-8B-Instruct/"
|
|
name: llama-31-8b-instruct
|
|
options:
|
|
fs.s3.region: "eu-central-1"
|
|
fs.s3.endpoint: "s3.eu-central-1.amazonaws.com"
|
|
encryptOptions:
|
|
- name: fs.s3.accessKeyId
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: s3creds
|
|
key: AWS_ACCESS_KEY_ID
|
|
- name: fs.s3.accessKeySecret
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: s3creds
|
|
key: AWS_SECRET_ACCESS_KEY
|
|
accessModes:
|
|
- ReadOnlyMany
|
|
nodeAffinity:
|
|
required:
|
|
nodeSelectorTerms:
|
|
- matchExpressions:
|
|
- key: node.kubernetes.io/instance-type
|
|
operator: In
|
|
values:
|
|
- "m5n.xlarge"
|
|
placement: "Shared"
|
|
---
|
|
apiVersion: data.fluid.io/v1alpha1
|
|
kind: JindoRuntime
|
|
metadata:
|
|
name: s3-data
|
|
spec:
|
|
replicas: 3
|
|
tieredstore:
|
|
levels:
|
|
- mediumtype: SSD
|
|
path: /mnt/ssd0/cache
|
|
quota: 100Gi
|
|
high: "0.95"
|
|
low: "0.7"
|
|
master:
|
|
nodeSelector:
|
|
node.kubernetes.io/instance-type: m5n.xlarge
|
|
worker:
|
|
nodeSelector:
|
|
node.kubernetes.io/instance-type: m5n.xlarge
|
|
fuse:
|
|
properties:
|
|
fs.jindofsx.data.cache.enable: "true"
|
|
# performance optimization
|
|
fs.jindofsx.stream.maxPendingBlocks: "200"
|
|
fs.s3.download.queue.size: "16"
|
|
fs.s3.download.thread.concurrency: "32"
|
|
fs.s3.read.readahead.buffer.count: "16"
|
|
args:
|
|
- -okernel_cache
|
|
- -oro
|
|
- -oattr_timeout=7200
|
|
- -oentry_timeout=7200
|
|
- -ometrics_port=9089
|