mirror of
https://github.com/hanzo-ml/kserve.git
synced 2026-10-30 20:53:49 +00:00
Lize CaiandGitHub
fab85e09c2
* update fluid kserve demo to use huggingface servingruntime and other model. Signed-off-by: Lize Cai <[email protected]> * fix lint Signed-off-by: Lize Cai <[email protected]> * explicitly set custom servingruntime, update devshm. Signed-off-by: Lize Cai <[email protected]> * update the args in custom kserve hfserver Signed-off-by: Lize Cai <[email protected]> * address comments Signed-off-by: Lize Cai <[email protected]> * add return of line Signed-off-by: Lize Cai <[email protected]> --------- Signed-off-by: Lize Cai <[email protected]>
66 lines
1.6 KiB
YAML
66 lines
1.6 KiB
YAML
apiVersion: serving.kserve.io/v1alpha1
|
|
kind: ClusterServingRuntime
|
|
metadata:
|
|
name: custom-kserve-huggingfaceserver
|
|
spec:
|
|
annotations:
|
|
prometheus.kserve.io/path: /metrics
|
|
prometheus.kserve.io/port: "8080"
|
|
containers:
|
|
- image: kserve/huggingfaceserver:latest
|
|
name: kserve-container
|
|
command: ["/bin/bash"]
|
|
args:
|
|
- "-c"
|
|
- |
|
|
set -e
|
|
TARGET_DIR="${TARGET_DIR:-/mnt/models}"
|
|
MAX_JOBS="${MAX_JOBS:-16}"
|
|
BLOCK_SIZE="${BLOCK_SIZE:-4M}"
|
|
echo "Warming up data in directory: $TARGET_DIR with max jobs: $MAX_JOBS and block size: $BLOCK_SIZE"
|
|
start_time=$(date +%s)
|
|
count=0
|
|
for file in $(find "$TARGET_DIR" -type f); do
|
|
dd if="$file" of=/dev/null bs="$BLOCK_SIZE" &
|
|
count=$((count + 1))
|
|
if (( count >= MAX_JOBS )); then
|
|
wait -n
|
|
count=$((count - 1))
|
|
fi
|
|
done
|
|
wait
|
|
end_time=$(date +%s)
|
|
elapsed_time=$((end_time - start_time))
|
|
echo "Data warmup completed in $elapsed_time seconds"
|
|
|
|
# Print all arguments for debugging
|
|
echo "All arguments: $@"
|
|
|
|
# Now run the Python script with all arguments
|
|
python3 -m huggingfaceserver "$@"
|
|
- "--"
|
|
- --model_name={{.Name}}
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
volumeMounts:
|
|
- name: devshm
|
|
mountPath: /dev/shm
|
|
volumes:
|
|
- name: devshm
|
|
emptyDir:
|
|
medium: Memory
|
|
sizeLimit: 1Gi
|
|
protocolVersions:
|
|
- v2
|
|
- v1
|
|
supportedModelFormats:
|
|
- autoSelect: true
|
|
name: huggingface
|
|
priority: 1
|
|
version: "1"
|