mirror of
https://github.com/hanzoai/visor.git
synced 2026-09-24 17:04:14 +00:00
22 lines
704 B
Docker
22 lines
704 B
Docker
# Use the official vLLM TPU base image
|
|
FROM vllm/vllm-tpu:v0.13.2
|
|
|
|
# Set environment variables for TPU runtime
|
|
ENV VLLM_TARGET_DEVICE=tpu
|
|
ENV HF_HOME=/vllm_data/cache
|
|
|
|
# Create a directory for model weights (mapped to host for persistence)
|
|
RUN mkdir -p /vllm_data/cache
|
|
|
|
RUN python3 -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='Qwen/Qwen2.5-1.5B-Instruct')"
|
|
|
|
# Expose the OpenAI-compatible API port
|
|
EXPOSE 8000
|
|
|
|
# Start the vLLM server with Qwen2.5 settings
|
|
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
|
|
CMD ["--model", "Qwen/Qwen2.5-1.5B-Instruct", \
|
|
"--tensor-parallel-size", "1", \
|
|
"--max-model-len", "4096", \
|
|
"--host", "0.0.0.0"]
|