hanzoai/python-sdk
2
0
Fork 0
mirror of https://github.com/hanzoai/python-sdk.git synced 2026-10-23 20:23:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

hanzo_kai.jev lists the decision models of data in Jev's shape, and the conformance fixtures name every reach refusal and pin Kai's versioned id

Browse Source
This commit is contained in:
zeekay
2026-09-28 17:37:54 -07:00
parent 70cb092e67
commit 169648c43d
11 changed files with 371 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
LLM.md
+6 -2
View File
@@ -374,8 +374,12 @@ sentence becomes `error.message`. `hanzo/kai` bodies arrive with sorted keys, so
read by name, and score maps are keyed and ordered by int level.
`hanzo_kai.jev` is the Jev-compatibility layer: `Client.system_one` on `POST /v1/systemone`, Jev's
body as `Response`, FastAPI errors as the same classes, `models.list()` on the `models` key. `from
hanzo_kai.jev import Choice, Noul, Score, Client as TypeSafeClient` ports a TypeSafe program.
body as `Response`, FastAPI errors as the same classes, and `models.list()` presenting the decision
models of `data` in Jev's shape, because api.hanzo.ai keeps the `models` key empty. `from
hanzo_kai.jev import Choice, Noul, Score, Client as TypeSafeClient` ports a TypeSafe program. A
refusal over reach is `422` with `error.code` a name on both paths (`state_too_long`,
`question_too_long`, `option_too_long`, `request_too_long`), as hanzoai/decision's
`decision/src/error.rs` writes it.
Tests: `cd pkg/hanzo-kai && uv run pytest`, over `httpx.MockTransport` with no network, each
client test on both flavours; they are outside `hanzo.yml`'s gate, whose scope is the cloud
pkg/hanzo-kai/hanzo_kai/_answers.py
+23 -14
View File
@@ -2,6 +2,7 @@
import json
from typing import Any, Literal, Annotated
from datetime import UTC, datetime
import httpx
from pydantic import Field, BaseModel, ConfigDict, PrivateAttr, ValidationError, field_validator
@@ -181,19 +182,19 @@ class Model(BaseModel):
class ModelMetadata(BaseModel):
"""A model as the `models` key of `GET /v1/models` lists it."""
"""A decision model in Jev's shape: its name, a description and its release date."""
model_config = ConfigDict(extra="allow", frozen=True)
name: str
"""The id to pass as `model`."""
description: str | None = None
description: str = ""
release_date: str | None = None
"""YYYY-MM-DD."""
"""YYYY-MM-DD, the day the listing dates the model."""
class ListModelsResponse(_Read):
"""The `models` key of `GET /v1/models`."""
"""The decision models of `GET /v1/models`, in Jev's shape."""
models: tuple[ModelMetadata, ...] = ()
@@ -254,16 +255,24 @@ def models(response: httpx.Response) -> list[Model]:
def catalog(response: httpx.Response) -> ListModelsResponse:
"""The `models` key of `GET /v1/models`."""
data = parse(response)
if not isinstance(data, dict) or not isinstance(data.get("models"), list):
raise invalid(response, "models")
try:
result = ListModelsResponse.model_validate({"models": data["models"]})
except ValidationError as error:
raise invalid(response, where(error)) from error
result._response = response
return result
"""The decision models `GET /v1/models` lists under `data`, in Jev's shape.
The name is the model's id, the description is empty, and the release date is the day of its
`created` time; the listing's own `models` key stays empty on api.hanzo.ai.
"""
listing = ListModelsResponse(
models=tuple(
ModelMetadata(
name=model.id,
release_date=None
if model.created is None
else datetime.fromtimestamp(model.created, UTC).date().isoformat(),
)
for model in models(response)
)
)
listing._response = response
return listing
def parse(response: httpx.Response) -> Any:
pkg/hanzo-kai/tests/conformance/__init__.py
+28 -19
View File
@@ -9,7 +9,8 @@ Each `*.json` beside this file holds a `rule` and its `cases`. A case names its
`live_conformance.py` runs them against the API.
Generators keep large requests short: `{"$questions": [n, question or [questions]]}`,
`{"$labels": n}`, `{"$levels": n}` and `{"$words": n}`.
`{"$labels": n}`, `{"$levels": n}`, `{"$words": n}` and `{"$bytes": n}`. A `model` of `"$version"`
is the served checkpoint's versioned id, which the runner resolves.
"""
import re
@@ -95,6 +96,9 @@ def expand(value: Any) -> Any:
return [f"level {index}" for index in range(arg)]
if key == "$words":
return " ".join(WORDS[index % len(WORDS)] for index in range(arg))
if key == "$bytes":
line = " ".join(WORDS) + " "
return (line * (arg // len(line) + 1))[:arg]
return {key: expand(item) for key, item in value.items()}
@@ -114,9 +118,14 @@ def client(
return sdk, lambda: len(sent)
def run(case: Case, sdk: Kai | Client, attempts: Callable[[], int]) -> list[str]:
"""What did not hold when the case went through `sdk`; empty when it passed."""
outcomes = [call(case, sdk, body) for body in case.send]
def run(case: Case, sdk: Kai | Client, attempts: Callable[[], int], version: Callable[[], str]) -> list[str]:
"""What did not hold when the case went through `sdk`; empty when it passed.
`version` gives the served checkpoint's versioned id, for a case that sends `"$version"`.
"""
outcomes = [
call(case, sdk, {**body, "model": version()} if body.get("model") == "$version" else body) for body in case.send
]
expect = case.expect
failures = [problem for outcome in outcomes for problem in verdict(expect, outcome)]
if failures:
@@ -307,28 +316,27 @@ def model_kai(case: Case, outcomes: list[Outcome]) -> list[str]:
def catalog(case: Case, outcomes: list[Outcome]) -> list[str]:
"""`GET /v1/models` lists decision models under `models` with name, description and release date."""
"""`hanzo_kai.jev` lists the decision models of `data` in Jev's shape, and the `models` key stays `[]`."""
problems = []
for listing in outcomes:
if not isinstance(listing, ListModelsResponse):
return [f"not a listing: {said(listing)}"]
data = listing.raw_http_response.json()
rows = data.get("models")
if not isinstance(rows, list) or not rows:
return [f"models key {rows!r}"]
for row in rows:
ok = isinstance(row, dict) and all(
isinstance(row.get(k), str) for k in ("name", "description", "release_date")
)
if not ok or not re.fullmatch(r"\d{4}-\d{2}-\d{2}", row["release_date"]):
problems.append(f"row {row!r}")
if not any(isinstance(row, dict) and row.get("name") == "kai" for row in rows):
problems.append("no row named kai")
if "data" not in data:
problems.append("the data key is gone")
if not listing.models or "kai" not in [m.name for m in listing.models]:
problems.append(f"models {listing.models!r}")
for m in listing.models:
if not isinstance(m.description, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}", m.release_date or ""):
problems.append(f"row {m!r}")
raw = listing.raw_http_response.json()
if raw.get("models") != [] or "data" not in raw:
problems.append(f"models key {raw.get('models')!r}, data key {'present' if 'data' in raw else 'gone'}")
return problems
def versioned(case: Case, outcomes: list[Outcome]) -> list[str]:
"""The answer names Kai's versioned id: `kai-` and 12 hex digits of its weights' SHA-256."""
return [f"model {r.model!r}" for r in responses(outcomes) if not re.fullmatch(r"kai-[0-9a-f]{12}", r.model)]
def data(case: Case, outcomes: list[Outcome]) -> list[str]:
"""`GET /v1/models` still lists `kai` under `data`, with decision outputs."""
return [
@@ -382,6 +390,7 @@ CHECKS: dict[str, Check] = {
answered,
model_kai,
catalog,
versioned,
data,
request_id,
fastapi,
pkg/hanzo-kai/tests/conformance/fake.py
+64 -33
View File
@@ -16,42 +16,36 @@ from typing import Any
import httpx
KEY = "sk-test-4b1d9c0e7f"
REACH = 8192
"""Tokens of state and one question the served checkpoint reads."""
VERSION = "kai-a10"
"""The versioned id `kai` resolves to."""
SHA256 = hashlib.sha256(b"the fake checkpoint's weights").hexdigest()
VERSION = f"kai-{SHA256[:12]}"
"""The versioned id: `kai-` and the first 12 hex digits of the weights' SHA-256."""
READS = 1024
"""Tokens of state and one question the served checkpoint reads; a question may take half."""
OPTION = 512
"""Tokens of one option read whole."""
STATE = 131072
"""Tokens of state the wire carries."""
BODY = 16 * 1024 * 1024
"""Bytes of body the service reads."""
NATIVE = frozenset({"kai", "hanzo/kai", VERSION})
COMPAT = frozenset({"kai", VERSION})
FAULTS = {402: "insufficient balance", 429: "rate limited", 529: "overloaded"}
CREATED = 1790629327
LISTING = {
"object": "list",
"data": [
{
"id": "hanzo/kai",
"object": "model",
"owned_by": "hanzo",
"outputs": ["decision"],
"pricing": {"input": 0.021, "output": 0},
},
{
"id": "kai",
"object": "model",
"owned_by": "hanzo",
"outputs": ["decision"],
"pricing": {"input": 0.021, "output": 0},
},
{
"id": "zen5",
"object": "model",
"owned_by": "hanzo",
"outputs": ["text"],
"pricing": {"input": 1, "output": 2},
},
{"id": "hanzo/kai", "object": "model", "created": CREATED, "owned_by": "hanzo", "outputs": ["decision"]},
{"id": "kai", "object": "model", "created": CREATED, "owned_by": "hanzo", "outputs": ["decision"]},
{"id": VERSION, "object": "model", "created": CREATED, "owned_by": "hanzo", "outputs": ["decision"]},
{"id": "zen5", "object": "model", "created": CREATED, "owned_by": "hanzo", "outputs": ["text"]},
],
"models": [{"name": "kai", "description": "Kai, Hanzo's decision model", "release_date": "2026-09-28"}],
"models": [],
}
REACH = frozenset({"state_too_long", "question_too_long", "option_too_long", "request_too_long"})
class Refusal(Exception):
"""A request the contract refuses, and how."""
@@ -66,7 +60,12 @@ class Refusal(Exception):
return {"detail": self.message}
if self.status == 401:
return {"status": "error", "msg": self.message}
return {"error": {"code": self.code or self.status, "message": self.message}}
return {
"error": {
"code": self.code if self.status == 422 and self.code in REACH else self.status,
"message": self.message,
}
}
class Service:
@@ -90,6 +89,8 @@ class Service:
status, extra = self.faults.pop(0)
headers.update(extra)
raise Refusal(status, FAULTS[status])
if len(request.content) > BODY:
raise Refusal(422, f"the body is over {BODY} bytes", "request_too_long")
try:
body = json.loads(request.content)
except ValueError as error:
@@ -110,11 +111,7 @@ def answer(body: dict[str, Any], compat: bool) -> dict[str, Any]:
if not isinstance(questions, dict) or not 1 <= len(questions) <= 100:
raise Refusal(422, "questions holds 1 to 100 questions", loc=("body", "questions"))
asked = {name: question(name, q, compat) for name, q in questions.items()}
need = tokens(state) + max(cost(q) for q in asked.values())
if need > REACH:
raise Refusal(
422, f"state and question need {need} tokens; Kai reads {REACH}", "state_too_long", ("body", "state")
)
reach(state, asked)
answers = {name: decide(state, q, compat) for name, q in asked.items()}
usage = {"input_tokens": tokens(state) + sum(cost(q) for q in asked.values()), "output_tokens": 0}
if compat:
@@ -126,7 +123,7 @@ def answer(body: dict[str, Any], compat: bool) -> dict[str, Any]:
"provider": "Hanzo",
"answers": answers,
"usage": usage,
"routing": {"backend": "kai", "checkpoint": "fake"},
"routing": {"backend": "kai", "checkpoint": "fake", "sha256": SHA256},
"state_hash": f"sha256:{digest}",
"latency_ms": 1.0,
}
@@ -165,6 +162,40 @@ def question(name: str, q: Any, compat: bool) -> dict[str, Any]:
return {"type": kind, "instructions": q.get("instructions"), "options": options, "labels": q.get("labels")}
def reach(state: Any, asked: dict[str, dict[str, Any]]) -> None:
"""Refuses, by name, the part of a request the checkpoint would have to cut."""
held = tokens(state)
if held > STATE:
raise Refusal(
422, f"the state takes {held} tokens; the wire carries {STATE}", "state_too_long", ("body", "state")
)
for name, q in asked.items():
at = ("body", "questions", name, q["type"])
prompt = tokens(q["instructions"]) + 3
if prompt > READS // 2:
raise Refusal(
422,
f"question {name!r} takes {prompt} tokens; Kai reads {READS // 2}",
"question_too_long",
(*at, "instructions"),
)
if prompt + held > READS:
raise Refusal(
422,
f"the state and question {name!r} take {prompt + held} tokens; Kai reads {READS}",
"state_too_long",
("body", "state"),
)
for key, text in q["options"]:
if tokens(key) + tokens(text) > OPTION:
raise Refusal(
422,
f"option {key} of question {name!r} is over {OPTION} tokens",
"option_too_long",
(*at, "criteria", key),
)
def decide(state: Any, q: dict[str, Any], compat: bool) -> dict[str, Any]:
p = distribution(state, q)
top = max(p)
pkg/hanzo-kai/tests/conformance/models.json
+1 -1
View File
@@ -1,5 +1,5 @@
{
"rule": "GET /v1/models keeps `data` and adds `models`, listing each decision model's name, description and release date.",
"rule": "GET /v1/models lists the decision models under `data` and keeps its `models` key empty; hanzo_kai.jev lists `data` in Jev's shape.",
"cases": [
{
"path": "native",
pkg/hanzo-kai/tests/conformance/reach.json
+85 -1
View File
@@ -1,7 +1,8 @@
{
"rule": "A state that does not fit what Kai reads is refused with 422 and code `state_too_long`; nothing is cut.",
"rule": "What the checkpoint would have to cut is refused with 422 and named in code: a state or a question past what Kai reads, an option past 512 tokens, a body past 16 MiB; nothing is cut.",
"cases": [
{
"name": "state",
"path": [
"native",
"compat"
@@ -28,6 +29,89 @@
"error": "UnprocessableEntityError",
"code": "state_too_long"
}
},
{
"name": "question",
"path": [
"native",
"compat"
],
"send": [
{
"state": "I was charged twice for my March invoice. Please refund the duplicate.",
"questions": {
"billing": {
"type": "noul",
"instructions": {
"$words": 2000
}
}
}
}
],
"expect": {
"status": 422,
"error": "UnprocessableEntityError",
"code": "question_too_long"
}
},
{
"name": "option",
"path": [
"native",
"compat"
],
"send": [
{
"state": "I was charged twice for my March invoice. Please refund the duplicate.",
"questions": {
"team": {
"type": "choice",
"instructions": "Which team should handle this?",
"criteria": {
"billing": {
"$words": 1000
},
"tech": "bugs and outages"
}
}
}
}
],
"expect": {
"status": 422,
"error": "UnprocessableEntityError",
"code": "option_too_long"
}
},
{
"name": "request",
"path": [
"native",
"compat"
],
"send": [
{
"state": {
"$bytes": 17825792
},
"questions": {
"billing": {
"type": "noul",
"instructions": "Is this about billing?",
"criteria": {
"true": "it concerns charges, invoices or refunds",
"false": "it concerns something else"
}
}
}
}
],
"expect": {
"status": 422,
"error": "UnprocessableEntityError",
"code": "request_too_long"
}
}
]
}
pkg/hanzo-kai/tests/conformance/versioned.json
+100
View File
@@ -0,0 +1,100 @@
{
"rule": "Kai's versioned id, kai- and the first 12 hex digits of its weights' SHA-256, is accepted on both paths, and /v1/systemone answers with it.",
"cases": [
{
"name": "sent",
"path": [
"native",
"compat"
],
"send": [
{
"state": "I was charged twice for my March invoice. Please refund the duplicate.",
"questions": {
"billing": {
"type": "noul",
"instructions": "Is this about billing?",
"criteria": {
"true": "it concerns charges, invoices or refunds",
"false": "it concerns something else"
}
},
"team": {
"type": "choice",
"instructions": "Which team should handle this?",
"criteria": {
"billing": "charges, invoices, refunds",
"tech": "bugs and outages",
"sales": "plans and upgrades"
}
},
"urgency": {
"type": "score",
"instructions": "How urgent is this?",
"criteria": [
"can wait",
"this week",
"today",
"right now"
]
}
},
"model": "$version"
}
],
"expect": {
"status": 200,
"checks": [
"answered",
"versioned"
]
}
},
{
"name": "answered",
"path": "compat",
"send": [
{
"state": "I was charged twice for my March invoice. Please refund the duplicate.",
"questions": {
"billing": {
"type": "noul",
"instructions": "Is this about billing?",
"criteria": {
"true": "it concerns charges, invoices or refunds",
"false": "it concerns something else"
}
},
"team": {
"type": "choice",
"instructions": "Which team should handle this?",
"criteria": {
"billing": "charges, invoices, refunds",
"tech": "bugs and outages",
"sales": "plans and upgrades"
}
},
"urgency": {
"type": "score",
"instructions": "How urgent is this?",
"criteria": [
"can wait",
"this week",
"today",
"right now"
]
}
},
"model": "kai"
}
],
"expect": {
"status": 200,
"checks": [
"versioned",
"request_id"
]
}
}
]
}
pkg/hanzo-kai/tests/live_conformance.py
+11 -2
View File
@@ -13,12 +13,21 @@ import sys
import time
import datetime
import platform
import functools
import hanzo_kai
from hanzo_kai import RetryPolicy
from hanzo_kai import Kai, Noul, RetryPolicy
from conformance import run, load, client
@functools.cache
def version(base: str) -> str:
"""The served checkpoint's versioned id: `kai-` and the first 12 hex digits of its weights' SHA-256."""
with Kai(base_url=base) as kai:
d = kai.decide("ping", {"q": Noul(instructions="The message is a greeting.")})
return "kai-" + d.routing["sha256"][:12]
def main() -> int:
base = sys.argv[1] if len(sys.argv) > 1 else "https://api.hanzo.ai"
now = datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds")
@@ -36,7 +45,7 @@ def main() -> int:
sdk, attempts = client(case, base_url=base, retry=RetryPolicy(max_retries=10))
started = time.monotonic()
with sdk:
failures = run(case, sdk, attempts)
failures = run(case, sdk, attempts, lambda: version(base))
verdict = "FAIL" if failures else "PASS"
counts[verdict] += 1
took = f"{time.monotonic() - started:.1f} s"
pkg/hanzo-kai/tests/test_conformance.py
+4 -3
View File
@@ -3,7 +3,7 @@
import httpx
import pytest
from conformance import CHECKS, Case, run, load, client
from conformance.fake import KEY, Service
from conformance.fake import KEY, VERSION, Service
CASES = load()
@@ -12,7 +12,7 @@ CASES = load()
def test_contract(case: Case, slept: list[float]) -> None:
sdk, attempts = client(case, key=KEY, transport=httpx.MockTransport(Service(case.fault)))
with sdk:
assert run(case, sdk, attempts) == []
assert run(case, sdk, attempts, lambda: VERSION) == []
def test_fixtures() -> None:
@@ -33,6 +33,7 @@ def test_fixtures() -> None:
"legend",
"models",
"request_id",
"versioned",
):
assert any(name.startswith(f"{rule}/") for name in names), rule
used = {check for case in CASES for check in case.expect.get("checks", [])}
@@ -62,5 +63,5 @@ def test_a_broken_service_fails_the_checks() -> None:
case = next(c for c in CASES if c.name == f"{rule}/native")
sdk, attempts = client(case, key=KEY, transport=httpx.MockTransport(broken))
with sdk:
failures = run(case, sdk, attempts)
failures = run(case, sdk, attempts, lambda: VERSION)
assert failures, check
pkg/hanzo-kai/tests/test_errors.py
+35
View File
@@ -121,6 +121,41 @@ BODIES: list[tuple[int, dict[str, Any] | None, str | None, str, Any]] = [
"questions.q.criteria: at most 255 labels; state: field required",
"too_long",
),
# every reach refusal names what is over, on either path
*(
row
for code, message, loc in (
(
"question_too_long",
"question 'q' takes 900 tokens with its type line",
["body", "questions", "q", "noul", "instructions"],
),
(
"option_too_long",
"option refund of question 'q' takes 600 tokens",
["body", "questions", "q", "choice", "criteria", "refund"],
),
("request_too_long", "the body is over 16777216 bytes", ["body"]),
)
for row in (
(422, {"error": {"code": code, "message": message}}, None, message, code),
(
422,
{"detail": [{"loc": loc, "msg": message, "type": code}]},
None,
": ".join(filter(None, [".".join(loc[1:]), message])),
code,
),
)
),
# a schema refusal keeps the status as its code
(
422,
{"error": {"code": 422, "message": "questions: at least 1 question"}},
None,
"questions: at least 1 question",
422,
),
]
pkg/hanzo-kai/tests/test_jev.py
+14 -6
View File
@@ -35,10 +35,16 @@ QUESTIONS = {
"billing": jev.Noul(instructions="Is this about billing?"),
"urgency": jev.Score(instructions="How urgent?", criteria=["can wait", {"when": "this week"}, ["today", "now"]]),
}
# As api.hanzo.ai lists them: the decision models under `data`, and `models` empty.
LISTING = {
"object": "list",
"data": [{"id": "kai", "object": "model", "outputs": ["decision"], "pricing": {"input": 0.021, "output": 0}}],
"models": [{"name": "kai", "description": "Kai, Hanzo's decision model", "release_date": "2026-09-28"}],
"data": [
{"id": "zen5", "object": "model", "created": 1790629327, "outputs": ["text"]},
{"id": "kai", "object": "model", "created": 1790629327, "owned_by": "hanzo", "outputs": ["decision"]},
{"id": "kai-a211cc701038", "object": "model", "created": 1790629327, "outputs": ["decision"]},
{"id": "no-date", "object": "model", "outputs": ["decision"]},
],
"models": [],
}
@@ -172,18 +178,20 @@ def test_retries_honour_the_headers(compat: Build, slept: list[float]) -> None:
assert [r.headers.get("x-kai-retry-count") for r in wire.requests] == [None, "1", "2"]
def test_models_reads_the_models_key(compat: Build) -> None:
def test_models_reads_the_catalogue(compat: Build) -> None:
wire = Wire(reply(json_body=LISTING, headers={"x-request-id": "req-m"}))
listing = compat(wire).models()
assert isinstance(listing, jev.ListModelsResponse)
assert str(wire.requests[0].url) == "https://api.hanzo.ai/v1/models"
assert listing.models == (
jev.ModelMetadata(name="kai", description="Kai, Hanzo's decision model", release_date="2026-09-28"),
jev.ModelMetadata(name="kai", description="", release_date="2026-09-28"),
jev.ModelMetadata(name="kai-a211cc701038", description="", release_date="2026-09-28"),
jev.ModelMetadata(name="no-date", description="", release_date=None),
)
assert listing.request_id == "req-m"
with pytest.raises(jev.APIResponseValidationError) as caught:
compat(Wire(reply(json_body={"data": []}))).models()
assert caught.value.field_path == "models"
compat(Wire(reply(json_body={"models": []}))).models()
assert caught.value.field_path == "data"
def test_one_line_port(monkeypatch: pytest.MonkeyPatch) -> None: