mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
- Demand-sized unified-memory KV cache: size to max_seq_len*max_batch_size (context*concurrency) bounded by the post-model/OS-reserve ceiling, instead of grabbing all free RAM. Fixes the ROCm/APU thrash where an ~88 GB grab on a 128 GB unified box never returned; 30B-A3B now 88 GB -> 24 GB, elastic with concurrency. (hanzo-engine/src/paged_attention/mod.rs) - Vulkan paged-attention wired (is_vulkan branch, opt-in via --paged-attn on): runs and eliminates the naive-Sdpa layout-copy churn, but the v1 scalar kernel regresses decode on this APU, so default OFF until v2 + MoE-glue fusion land. i64/i32 paged-attn index metadata now binds directly (backend coerces to u32). - hanzo-quant: Vulkan/Wgpu storage arms in the distributed pipeline. - Drop the temporary [patch.crates-io] hanzo-ml redirect; pin hanzo-ml 0.11.55. - Manifest hygiene: version on path deps (enso, ffi, server-core, zen3-serving).
34 lines
972 B
TOML
34 lines
972 B
TOML
[package]
|
|
name = "zen3-serving"
|
|
version = "0.1.0"
|
|
edition = "2021"
|
|
license = "MIT"
|
|
description = "Standalone CPU harness exercising the engine's zen3-ASR + zen3-TTS speech pipelines on real weights."
|
|
|
|
[[bin]]
|
|
name = "zen3-serving"
|
|
path = "src/main.rs"
|
|
|
|
[dependencies]
|
|
hanzo-engine = { version = "1.7.41", path = "../../hanzo-engine" }
|
|
hanzo-ml = { path = "../../../ml/hanzo-ml" }
|
|
hanzo-nn = { path = "../../../ml/hanzo-nn" }
|
|
hanzo-quant = { version = "1.7.41", path = "../../hanzo-quant" }
|
|
hanzo-audio = { version = "1.7.41", path = "../../hanzo-audio" }
|
|
|
|
anyhow = "1"
|
|
clap = { version = "4", features = ["derive"] }
|
|
tokenizers = { version = "0.21", default-features = false, features = ["onig"] }
|
|
serde_json = "1"
|
|
ahash = "0.8"
|
|
hound = "3.5"
|
|
|
|
[features]
|
|
cuda = ["hanzo-engine/cuda", "hanzo-ml/cuda", "hanzo-nn/cuda", "hanzo-quant/cuda"]
|
|
metal = ["hanzo-engine/metal", "hanzo-ml/metal", "hanzo-nn/metal", "hanzo-quant/metal"]
|
|
|
|
[profile.release]
|
|
opt-level = 3
|
|
|
|
[workspace]
|