mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
Every mention of another inference engine is rewritten to say what our code does, or the math it computes. Reference values are "the reference value from the published config" or "the recorded reference output", line citations into another engine's source become the rule itself, comparisons with other engines are dropped, and asides about other engines' bugs go. Identifiers follow: - CUDA: hanzo-paged-attn's `namespace vllm` is `pagedattn`, hanzo-engine's MoE helpers `moe_dtype` and `moe_detail`, the rotary kernels' `hanzo_rotary`; the shuffle/ldg macros are HANZO_*. - GPTQ: the shuffled-layout path (`reconstruct_exllama`, `use_exllama`) is `reconstruct_shuffled` / `is_shuffled`, on the kernel and the FFI side. - Tests: `*_matches_vllm` and friends name what they compare against (the reference fixture, the tile rule). - MiniCPM-o's vision-LLM embedding is `vlm_embedding`; the rotary custom op is `rotary-embedding`. - hanzo-bench's board scores ours against a "rival" whose name comes from the manifest: `rival_<tag>.json`, `--rival*` manifest flags, `rival t/s` columns; dossier.sh and spec_decode.sh take `--rival-bench`. - install.ps1 installs this repository's hanzo-cli, as install.sh does, with HANZO_INSTALL_NCCL / HANZO_INSTALL_NO_NCCL. - bench_http.py reads only the hanzo: metric families. Kept, because they are what they are: LICENSE, LICENSE-APACHE, NOTICE and tools/upstream-map.sh; each ported file's copyright/licence/provenance header; Cargo.toml dependency names; data identifiers (Hugging Face repo IDs, fixture file names and fixture metadata keys the tests read, on-disk paths, executable names the GPU and engine guards match on a host); model and format names. LLM.md says the rule once, under "One name: Hanzo Engine". Co-authored-by: Hanzo Dev <[email protected]>
182 lines
6.4 KiB
Python
182 lines
6.4 KiB
Python
#!/usr/bin/env python3
|
|
"""bench_http.py against a fake engine in two stream shapes: thinking under `reasoning` with usage
|
|
in a trailing chunk, and hanzo's, thinking under `reasoning_content` with usage on the finish chunk."""
|
|
import http.server
|
|
import io
|
|
import json
|
|
import os
|
|
import socket
|
|
import sys
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from contextlib import redirect_stdout
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import bench_http # noqa: E402
|
|
|
|
GAP = 0.05 # seconds between token chunks
|
|
|
|
|
|
def chunk(delta=None, finish=None, usage=None, choices=True):
|
|
body = {"id": "x", "object": "chat.completion.chunk", "model": "fake"}
|
|
body["choices"] = [{"index": 0, "delta": delta or {}, "finish_reason": finish}] if choices else []
|
|
if usage:
|
|
body["usage"] = usage
|
|
return body
|
|
|
|
|
|
def reasoning_field_stream():
|
|
"""Role chunk, 3 reasoning + 3 content chunks carrying 8 tokens, finish, then usage 2 s later."""
|
|
yield chunk({"role": "assistant", "content": ""}), 0
|
|
for text in ["Let", " me think", " now"]:
|
|
yield chunk({"reasoning": text}), GAP
|
|
for text in ["The", " answer", " is"]:
|
|
yield chunk({"content": text}), GAP
|
|
yield chunk({}, finish="length"), 0
|
|
yield chunk(choices=False, usage={"prompt_tokens": 42, "completion_tokens": 8,
|
|
"total_tokens": 50}), 2.0
|
|
|
|
|
|
def hanzo_stream():
|
|
"""reasoning_content deltas, usage on the finish chunk."""
|
|
yield chunk({"role": "assistant"}), 0
|
|
for text in ["Let", " me think", " now"]:
|
|
yield chunk({"reasoning_content": text}), GAP
|
|
for text in ["The", " answer", " is"]:
|
|
yield chunk({"content": text}), GAP
|
|
yield chunk({}, finish="length", usage={"prompt_tokens": 42, "completion_tokens": 8,
|
|
"total_tokens": 50}), 0
|
|
|
|
|
|
def no_usage_stream():
|
|
yield chunk({"content": "a"}), 0
|
|
yield chunk({"content": "b"}), GAP
|
|
yield chunk({}, finish="stop"), 0
|
|
|
|
|
|
class Fake(http.server.ThreadingHTTPServer):
|
|
daemon_threads = True
|
|
|
|
def __init__(self, shape, hold=0.0):
|
|
super().__init__(("127.0.0.1", 0), Handler)
|
|
self.shape, self.hold = shape, hold
|
|
self.prompts, self.lock, self.inflight, self.peak = [], threading.Lock(), 0, 0
|
|
self.spans = []
|
|
|
|
@property
|
|
def url(self):
|
|
return f"http://127.0.0.1:{self.server_address[1]}/v1/chat/completions"
|
|
|
|
|
|
class Handler(http.server.BaseHTTPRequestHandler):
|
|
def setup(self):
|
|
super().setup()
|
|
# Stream chunks as written, as a real server does; Nagle would add delayed-ACK stalls.
|
|
self.connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
|
|
|
def log_message(self, *args):
|
|
pass
|
|
|
|
def do_GET(self):
|
|
body = json.dumps({"object": "list", "data": [{"id": "fake", "object": "model"}]}).encode()
|
|
self.send_response(200)
|
|
self.send_header("content-type", "application/json")
|
|
self.send_header("content-length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def do_POST(self):
|
|
server = self.server
|
|
req = json.loads(self.rfile.read(int(self.headers["content-length"])))
|
|
assert req["stream_options"] == {"include_usage": True}
|
|
with server.lock:
|
|
server.prompts.append(req["messages"][0]["content"])
|
|
server.inflight += 1
|
|
server.peak = max(server.peak, server.inflight)
|
|
self.send_response(200)
|
|
self.send_header("content-type", "text/event-stream")
|
|
self.end_headers()
|
|
time.sleep(server.hold)
|
|
written = []
|
|
for body, delay in server.shape():
|
|
time.sleep(delay)
|
|
self.wfile.write(f"data: {json.dumps(body)}\n\n".encode())
|
|
self.wfile.flush()
|
|
if bench_http.is_token_chunk(body):
|
|
written.append(time.monotonic())
|
|
with server.lock:
|
|
server.spans.append(written[-1] - written[0] if written else 0.0)
|
|
self.wfile.write(b"data: [DONE]\n\n")
|
|
with server.lock:
|
|
server.inflight -= 1
|
|
|
|
|
|
def serve(shape, hold=0.0):
|
|
server = Fake(shape, hold)
|
|
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
return server
|
|
|
|
|
|
def run(argv):
|
|
out = io.StringIO()
|
|
with redirect_stdout(out):
|
|
code = bench_http.main(argv)
|
|
return code, [json.loads(line) for line in out.getvalue().splitlines() if line]
|
|
|
|
|
|
class Streams(unittest.TestCase):
|
|
def check_shape(self, shape):
|
|
server = serve(shape)
|
|
try:
|
|
code, rows = run([server.url, "--sizes", "4", "--gen", "8"])
|
|
finally:
|
|
server.shutdown()
|
|
self.assertEqual(code, 0)
|
|
(row,) = rows
|
|
self.assertEqual(row["prompt_tokens"], 42)
|
|
self.assertEqual(row["completion_tokens"], 8)
|
|
self.assertEqual(row["chunks"], 6)
|
|
# The window is the server's first-to-last token chunk; the 2 s usage delay and the empty
|
|
# role and finish chunks are outside it.
|
|
(span,) = server.spans
|
|
self.assertAlmostEqual(row["decode_s"], span, delta=0.03)
|
|
self.assertLess(row["decode_s"], 1.5)
|
|
self.assertAlmostEqual(row["decode_tps"], 7 / row["decode_s"], places=6)
|
|
self.assertAlmostEqual(row["itl_p50_s"], span / 5, delta=0.03)
|
|
|
|
def test_reasoning_field_and_trailing_usage(self):
|
|
self.check_shape(reasoning_field_stream)
|
|
|
|
def test_hanzo_shape(self):
|
|
self.check_shape(hanzo_stream)
|
|
|
|
def test_no_usage_fails(self):
|
|
server = serve(no_usage_stream)
|
|
try:
|
|
code, rows = run([server.url, "--sizes", "4"])
|
|
finally:
|
|
server.shutdown()
|
|
self.assertNotEqual(code, 0)
|
|
self.assertIn("no usage", rows[0]["error"])
|
|
|
|
def test_concurrency_overlaps_with_distinct_nonces(self):
|
|
server = serve(hanzo_stream, hold=0.3)
|
|
try:
|
|
code, rows = run([server.url, "--sizes", "4", "--concurrency", "4"])
|
|
finally:
|
|
server.shutdown()
|
|
self.assertEqual(code, 0)
|
|
self.assertEqual(len(server.prompts), 4)
|
|
nonces = {p.split(".")[0] for p in server.prompts}
|
|
self.assertEqual(len(nonces), 4)
|
|
self.assertTrue(all(p.startswith("Session ") for p in server.prompts))
|
|
self.assertGreaterEqual(server.peak, 2)
|
|
aggregate = [r for r in rows if r.get("aggregate")]
|
|
self.assertEqual(len(aggregate), 1)
|
|
self.assertEqual(aggregate[0]["completion_tokens"], 32)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|