mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
Defaults (`hanzo serve` flags, a router Extension) hold the server-wide values a
request may omit. The flags are --served-model-name, --max-tokens (8192),
--completion-max-tokens (128), --max-tokens-cap (65536), --temperature, --top-p,
--top-k, --min-p, --presence-penalty, --frequency-penalty, --reasoning-effort,
--enable-thinking, --max-thinking-tokens and --thinking-answer-room. They work
for `hanzo serve`, hanzo-server and the serve config file. Defaults::validated
checks them at startup:
- A value out of range, or a default budget over the cap, stops the server with
a message that names the flag. Nothing is clamped.
- --reasoning-effort is stored mapped: minimal is low, high is xhigh. `none` is
refused in favour of --enable-thinking 0.
- --served-model-name becomes the engine's model id; the id the model was loaded
from stays an alias for it.
CliConfig boxes both variants, since the defaults made ServeConfig the large one.
Three modules hold the rules as pure functions, each rule with a test:
- controls: the token budget.
- Its three names fold together, and conflicting values are refused.
- The route default applies, the cap is a 400 (never a clamp), and <= 0 is 1.
- chat_template_kwargs carries exactly three keys, each type-checked.
- The thinking aliases fold together.
- reasoning_effort maps: none turns thinking off, anything unknown is a 400.
- The think budget is capped to leave answer room, never 0.
- Sampling ranges and defaults: a request temperature of 0 is greedy and takes
no default; with neither set, decoding is greedy.
- logit_bias keys are ids below the vocabulary, values are numbers, and a
request carries at most 20480 entries.
- tool_choice is checked against the tools, Responses' flat tool_choice is
nested, and drafter names are validated.
- error: ApiError and OpenAI's envelope, `type` by status, `code` engine_busy for
a 503.
- shape: usage with completion_tokens_details.reasoning_tokens and
prompt_tokens_details.cached_tokens, the `timings` object, include_usage
stream framing, and one tool call when parallel_tool_calls is false.
parse_request applies them in this order: the budget and its aliases, the
template controls and thinking aliases, sampling ranges and defaults,
tool_choice, the effort, the messages, the drafter, logit_bias, the cap.
`tool_choice: none` drops the tools.
- Completions share the budget (default --completion-max-tokens), sampling,
seed, drafter and logit_bias rules.
- Responses folds max_output_tokens/max_tokens/max_completion_tokens (0 is the
default) and folds `reasoning` as chat does.
- Messages maps thinking.type to the template and budget_tokens to the think
budget, with --enable-thinking as its default.
Errors:
- Every error body is OpenAI's envelope.
- A body that does not parse, a refused request and an engine ValidationError
are 400s (they were 422 or 500). An unknown model is a 404, and a declined
forced tool call is a 502.
- An error after a stream started is a `data: {"error":...}` frame, then
[DONE].
- /v1/messages keeps Anthropic's error shape, with the same statuses.
Streams:
- stream_options.include_usage puts "usage": null on every chunk and sends a
choices:[] usage chunk before [DONE].
- --served-model-name is the `model` of every response, and the one id
/v1/models lists.
- SSE keepalives are the `: keepalive` comment every 10 s.
Tests:
- The Anthropic tests build Usage with its new fields.
- test_parse_image_url sets HANZO_ALLOW_LOCAL_MEDIA, so the server-core suite
passes.
Co-authored-by: Hanzo Dev <[email protected]>
71 lines
2.1 KiB
TOML
71 lines
2.1 KiB
TOML
[package]
|
|
name = "hanzo-server-core"
|
|
readme = "../README.md"
|
|
authors = ["Hanzo AI"]
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
rust-version.workspace = true
|
|
description.workspace = true
|
|
repository.workspace = true
|
|
keywords.workspace = true
|
|
categories.workspace = true
|
|
license.workspace = true
|
|
homepage.workspace = true
|
|
|
|
[dependencies]
|
|
anyhow.workspace = true
|
|
axum = { workspace = true, features = ["tokio", "multipart"] }
|
|
clap.workspace = true
|
|
hanzo-ml.workspace = true
|
|
hanzo-router = { version = "0.1", path = "../hanzo-router" }
|
|
hanzo-3d = "0.1"
|
|
data-url.workspace = true
|
|
either.workspace = true
|
|
futures.workspace = true
|
|
image.workspace = true
|
|
indexmap.workspace = true
|
|
itertools.workspace = true
|
|
base64.workspace = true
|
|
hanzo-engine = { workspace = true, features = [
|
|
"utoipa",
|
|
] }
|
|
hanzo-train = { workspace = true, features = ["utoipa"] }
|
|
dirs.workspace = true
|
|
reqwest.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
tokio.workspace = true
|
|
tower-http = { workspace = true, features = ["cors"] }
|
|
tracing.workspace = true
|
|
url.workspace = true
|
|
utoipa = { workspace = true, features = ["axum_extras"] }
|
|
utoipa-swagger-ui = { workspace = true, features = ["axum"], optional = true }
|
|
uuid = { workspace = true, features = ["v4"] }
|
|
|
|
accelerate-src = { workspace = true, optional = true }
|
|
intel-mkl-src = { workspace = true, optional = true }
|
|
|
|
[dev-dependencies]
|
|
tempfile.workspace = true
|
|
|
|
[features]
|
|
default = ["swagger-ui"]
|
|
swagger-ui = ["dep:utoipa-swagger-ui"]
|
|
cuda = ["hanzo-engine/cuda"]
|
|
vulkan = ["hanzo-engine/vulkan"]
|
|
wgpu = ["hanzo-engine/wgpu"]
|
|
rocm = ["hanzo-engine/rocm"]
|
|
cudnn = ["hanzo-engine/cudnn"]
|
|
metal = ["hanzo-engine/metal"]
|
|
flash-attn = ["cuda", "hanzo-engine/flash-attn"]
|
|
flash-attn-v3 = ["cuda", "hanzo-engine/flash-attn-v3"]
|
|
accelerate = ["hanzo-engine/accelerate"]
|
|
mkl = ["hanzo-engine/mkl"]
|
|
nccl = ["hanzo-engine/nccl"]
|
|
ring = ["hanzo-engine/ring"]
|
|
code-execution = ["hanzo-engine/code-execution"]
|
|
license-enforce = ["hanzo-engine/license-enforce"]
|
|
license-app-hanzo = ["hanzo-engine/license-app-hanzo"]
|
|
license-app-lux = ["hanzo-engine/license-app-lux"]
|
|
license-app-zoo = ["hanzo-engine/license-app-zoo"]
|