mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
Builders for the fused router bit-exact to vLLM: src/cuda/route.rs, lane.rs, branch.rs and the exact kernels, plus the shared-expert overlap and the goldens. The router top-k moved out of ops.rs, which is why that file loses lines. Worktree checkpoint — the build had not happened when the stream stopped. Co-Authored-By: Claude Code <[email protected]>
76 lines
3.0 KiB
Docker
76 lines
3.0 KiB
Docker
# syntax=docker/dockerfile:1
|
|
|
|
# Stage 1: Build environment
|
|
FROM nvidia/cuda:13.0.3-cudnn-devel-ubuntu24.04 AS builder
|
|
SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]
|
|
|
|
ARG DEBIAN_FRONTEND=noninteractive
|
|
RUN <<HEREDOC
|
|
for i in 1 2 3 4 5; do apt-get -o Acquire::Retries=3 update && break; echo "apt-get update failed (attempt $i/5), mirror may be syncing; retrying in 15s"; sleep 15; done
|
|
apt-get install -y --no-install-recommends \
|
|
curl \
|
|
git \
|
|
libssl-dev \
|
|
pkg-config
|
|
|
|
rm -rf /var/lib/apt/lists/*
|
|
HEREDOC
|
|
|
|
RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
|
|
ENV PATH="/root/.cargo/bin:${PATH}"
|
|
RUN rustup update nightly && rustup default nightly
|
|
|
|
WORKDIR /hanzo
|
|
COPY . .
|
|
|
|
ARG CUDA_COMPUTE_CAP=80
|
|
ARG WITH_FEATURES="cuda,cudnn,flash-attn"
|
|
# Portable, memory-bounded release build. These MUST be ENV, not ARG: cargo and
|
|
# build.rs read the process environment, so an ARG alone is invisible to them.
|
|
# RUSTFLAGS="" overrides the repo .cargo/config.toml `target-cpu=native`,
|
|
# which would otherwise bake the builder pod's CPU features
|
|
# into the image and SIGILL on any other host.
|
|
# CARGO_BUILD_JOBS=2 caps concurrent rustc so the ARC pod does not OOM. The
|
|
# unbounded default (num_cpus) recycled the runner
|
|
# mid-compile: "the runner has received a shutdown signal".
|
|
# CUDA_COMPUTE_CAP is read by build.rs — as ARG-only every matrix leg
|
|
# silently compiled the default cap instead of its own.
|
|
ENV RUSTFLAGS="" \
|
|
CARGO_INCREMENTAL=0 \
|
|
CARGO_NET_RETRY=5 \
|
|
CARGO_BUILD_JOBS=2 \
|
|
RAYON_NUM_THREADS=4 \
|
|
CUDA_COMPUTE_CAP=${CUDA_COMPUTE_CAP}
|
|
# Only the two binaries the runtime stage copies (hanzo-server, hanzo-bench) —
|
|
# not the whole workspace.
|
|
RUN cargo build --release -p hanzo-server -p hanzo-bench --features "${WITH_FEATURES}"
|
|
|
|
|
|
# Stage 2: Minimal runtime environment
|
|
FROM nvidia/cuda:13.0.3-cudnn-runtime-ubuntu24.04 AS runtime
|
|
SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]
|
|
|
|
ARG DEBIAN_FRONTEND=noninteractive
|
|
RUN <<HEREDOC
|
|
for i in 1 2 3 4 5; do apt-get -o Acquire::Retries=3 update && break; echo "apt-get update failed (attempt $i/5), mirror may be syncing; retrying in 15s"; sleep 15; done
|
|
apt-get install -y --no-install-recommends \
|
|
libomp-dev \
|
|
ca-certificates \
|
|
libssl-dev \
|
|
curl \
|
|
pkg-config
|
|
# The MoE gate is held bitwise to vLLM 0.29.0, whose wheel ships cuBLAS 13.1.1.3: cuBLAS
|
|
# picks the gate's GEMM kernel, so another build moves routing (lane.rs CUBLAS).
|
|
apt-get install -y --no-install-recommends --allow-downgrades libcublas-13-0=13.1.1.3-1
|
|
|
|
rm -rf /var/lib/apt/lists/*
|
|
HEREDOC
|
|
|
|
COPY --chmod=755 --from=builder /hanzo/target/release/hanzo-bench /usr/local/bin/hanzo-bench
|
|
COPY --chmod=755 --from=builder /hanzo/target/release/hanzo-server /usr/local/bin/hanzo-server
|
|
# Copy chat templates for users running models which may not include them
|
|
COPY --from=builder /hanzo/chat_templates /chat_templates
|
|
|
|
ENV HUGGINGFACE_HUB_CACHE=/data \
|
|
PORT=80
|