mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
Every mention of another inference engine is rewritten to say what our code does, or the math it computes. Reference values are "the reference value from the published config" or "the recorded reference output", line citations into another engine's source become the rule itself, comparisons with other engines are dropped, and asides about other engines' bugs go. Identifiers follow: - CUDA: hanzo-paged-attn's `namespace vllm` is `pagedattn`, hanzo-engine's MoE helpers `moe_dtype` and `moe_detail`, the rotary kernels' `hanzo_rotary`; the shuffle/ldg macros are HANZO_*. - GPTQ: the shuffled-layout path (`reconstruct_exllama`, `use_exllama`) is `reconstruct_shuffled` / `is_shuffled`, on the kernel and the FFI side. - Tests: `*_matches_vllm` and friends name what they compare against (the reference fixture, the tile rule). - MiniCPM-o's vision-LLM embedding is `vlm_embedding`; the rotary custom op is `rotary-embedding`. - hanzo-bench's board scores ours against a "rival" whose name comes from the manifest: `rival_<tag>.json`, `--rival*` manifest flags, `rival t/s` columns; dossier.sh and spec_decode.sh take `--rival-bench`. - install.ps1 installs this repository's hanzo-cli, as install.sh does, with HANZO_INSTALL_NCCL / HANZO_INSTALL_NO_NCCL. - bench_http.py reads only the hanzo: metric families. Kept, because they are what they are: LICENSE, LICENSE-APACHE, NOTICE and tools/upstream-map.sh; each ported file's copyright/licence/provenance header; Cargo.toml dependency names; data identifiers (Hugging Face repo IDs, fixture file names and fixture metadata keys the tests read, on-disk paths, executable names the GPU and engine guards match on a host); model and format names. LLM.md says the rule once, under "One name: Hanzo Engine". Co-authored-by: Hanzo Dev <[email protected]>
110 lines
3.9 KiB
Python
110 lines
3.9 KiB
Python
## Qwen 3 30B A3B
|
|
# cargo run --release --features metal --' --port 1234 --paged-attn --max-seqs 128 plain -m mlx-community/Qwen3-30B-A3B-4bit--max-seq-len 1024 --max-batch-size 128
|
|
|
|
## Llama 3.2 3b
|
|
# cargo run --release --features metal '--' --port 1234 --isq 8 --paged-attn --max-seqs 128 plain -m ../hf_models/llama3.2_3b --max-seq-len 1024 --max-batch-size 128
|
|
|
|
## Mistral 7b
|
|
# cargo run --release --features metal '--' --port 1234 --paged-attn --max-seqs 128 plain -m mlx-community/Mistral-7B-Instruct-v0.3-4bit --max-seq-len 1024 --max-batch-size 128
|
|
|
|
import asyncio
|
|
from openai import AsyncOpenAI
|
|
import httpx
|
|
import textwrap
|
|
import json
|
|
import time
|
|
|
|
NUM_USERS = 8
|
|
REQUESTS_PER_USER = 8
|
|
PORT = 1234
|
|
|
|
|
|
def log_response(response: httpx.Response):
|
|
request = response.request
|
|
print(f"Request: {request.method} {request.url}")
|
|
print(" Headers:")
|
|
for key, value in request.headers.items():
|
|
if key.lower() == "authorization":
|
|
value = "[...]"
|
|
if key.lower() == "cookie":
|
|
value = value.split("=")[0] + "=..."
|
|
print(f" {key}: {value}")
|
|
print(" Body:")
|
|
try:
|
|
request_body = json.loads(request.content)
|
|
print(textwrap.indent(json.dumps(request_body, indent=2), " "))
|
|
except json.JSONDecodeError:
|
|
print(textwrap.indent(request.content.decode(), " "))
|
|
print(f"Response: status_code={response.status_code}")
|
|
print(" Headers:")
|
|
for key, value in response.headers.items():
|
|
if key.lower() == "set-cookie":
|
|
value = value.split("=")[0] + "=..."
|
|
print(f" {key}: {value}")
|
|
|
|
|
|
# Use the async-capable client
|
|
client = AsyncOpenAI(api_key="foobar", base_url=f"http://localhost:{PORT}/v1/")
|
|
|
|
|
|
async def timed_chat(client: AsyncOpenAI, messages):
|
|
"""
|
|
Send one chat completion request and return (completion, elapsed_seconds, completion_tokens).
|
|
"""
|
|
start = time.perf_counter()
|
|
completion = await client.chat.completions.create(
|
|
model="default",
|
|
messages=messages,
|
|
max_tokens=256,
|
|
frequency_penalty=1.0,
|
|
top_p=0.1,
|
|
temperature=0,
|
|
)
|
|
elapsed = time.perf_counter() - start
|
|
# Safely get number of completion tokens, default to 0 if missing
|
|
completion_tokens = getattr(completion.usage, "completion_tokens", 0)
|
|
return completion, elapsed, completion_tokens
|
|
|
|
|
|
async def user_task(client: AsyncOpenAI, system_prompt: str, user_message: str):
|
|
"""
|
|
Returns list of (completion, elapsed_seconds, completion_tokens).
|
|
"""
|
|
results = []
|
|
base_messages = []
|
|
if system_prompt:
|
|
base_messages.append({"role": "system", "content": system_prompt})
|
|
|
|
for _ in range(REQUESTS_PER_USER):
|
|
messages = base_messages + [{"role": "user", "content": user_message}]
|
|
completion, elapsed, completion_tokens = await timed_chat(client, messages)
|
|
results.append((completion, elapsed, completion_tokens))
|
|
return results
|
|
|
|
|
|
async def main() -> None:
|
|
"""
|
|
Computes and prints overall average request time, total requests, and average T/s.
|
|
"""
|
|
system_prompt = None # "You are a helpful assistant."
|
|
user_message = "Say hello!"
|
|
|
|
tasks = [user_task(client, system_prompt, user_message) for _ in range(NUM_USERS)]
|
|
all_results_nested = await asyncio.gather(*tasks)
|
|
all_results = [item for sublist in all_results_nested for item in sublist]
|
|
|
|
total_requests = len(all_results)
|
|
total_time = sum(elapsed for _, elapsed, _ in all_results)
|
|
total_tokens = sum(tokens for _, _, tokens in all_results)
|
|
avg_time = total_time / total_requests if total_requests else 0.0
|
|
avg_tps = total_tokens / total_time if total_time > 0 else 0.0
|
|
|
|
print(f"Total requests: {total_requests}")
|
|
print(f"Average request time: {avg_time:.2f}s")
|
|
print(f"Total tokens: {total_tokens}")
|
|
print(f"Average tokens per second (T/s): {avg_tps:.2f}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|