mirror of
https://github.com/hanzoai/python-sdk.git
synced 2026-10-23 20:23:00 +00:00
research: hanzo-research SDK — ergonomic, zero-config, auto-instrumenting client
The ONE way producers record/query R&D evidence via /v1/research (HIP-0512), generated-shape from openapi/research and wrapped in a tiny verb surface so hand-rolling is the worse choice: exp = research.experiment(kind, subject, task) # get/create → handle exp.record(item, model, result) # attempt, idempotent by stable id exp.snapshot(bytes) / exp.report(text) # diary artifacts (sha256-addressed) exp.finish(value) # seal the run research.query(project=, kind=) # read canonical Zero-config auto-instrumentation (caller supplies no provenance): captures git sha/branch/dirty, lib versions, host, and THREE self-documenting narrative sources — the calling code's docstring (what this experiment is), the commit messages since this experiment's last recorded run (what changed + why), and an optional note. The project self-documents as a side effect of running. Importer-agnostic: ingest() is one idempotent POST; any source (SQLite backfill, git history, kernel-perf campaign) maps to stable ids and re-imports as a no-op. Stdlib only (urllib); per-org key auth; private-by-default.
This commit is contained in:
@@ -0,0 +1,46 @@
|
||||
# hanzo-research
|
||||
|
||||
Record and query Hanzo R&D evidence in one import. The ONE way every producer
|
||||
(enso-bench benchmarks, hanzo-engine/hanzo-ml kernel-perf, any research project) files
|
||||
runs into the unified `/v1/research` plane (HIP-0512) — versioned, append-only,
|
||||
idempotent by content, private by default.
|
||||
|
||||
```python
|
||||
import hanzo_research as research
|
||||
|
||||
# get/create the experiment; provenance is auto-captured (see below)
|
||||
exp = research.experiment("benchmark", "grok-4.5", "gpqa_diamond")
|
||||
exp.record("q1", "grok-4.5", {"answer": "A", "correct": True}) # an attempt, idempotent
|
||||
exp.snapshot(png_bytes) # a board-snapshot artifact
|
||||
exp.report(markdown_text) # a generated-report artifact
|
||||
exp.finish(value=94.3) # seal the run
|
||||
|
||||
rows = research.query(project="enso-bench", kind="benchmark") # read canonical
|
||||
```
|
||||
|
||||
## Zero-config auto-instrumentation
|
||||
|
||||
The caller supplies **no** provenance. On `experiment()` the SDK reads the run's
|
||||
environment and captures three self-documenting narrative sources, plus machine context:
|
||||
|
||||
- **the calling code's docstring** — *what this experiment is* (write a normal module
|
||||
docstring);
|
||||
- **the commit messages since this experiment's last recorded run** — *what changed +
|
||||
why* (commit normally);
|
||||
- an optional **`note=`** you pass.
|
||||
|
||||
Plus `git_sha` / `git_branch` / `git_dirty`, `lib_versions`, and host. The project
|
||||
self-documents as a side effect of running.
|
||||
|
||||
## Importer-agnostic, idempotent
|
||||
|
||||
`ingest(experiments=…, attempts=…)` is one idempotent POST. Any source — a SQLite
|
||||
backfill, git history, a kernel-perf campaign — maps its records to stable ids and calls
|
||||
it; re-import is a no-op. "Load all research in" is just running importers.
|
||||
|
||||
## Config
|
||||
|
||||
`RESEARCH_BASE` (default `https://api.hanzo.ai`), `HANZO_API_KEY` (per-org key →
|
||||
Bearer), `RESEARCH_PROJECT`. Uploads are private; public visibility, training, and
|
||||
commons rights are each a separate grant. Stdlib only (urllib) — importable in any
|
||||
zero-dep harness.
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Hanzo Research SDK — record and query R&D evidence in one import.
|
||||
|
||||
import hanzo_research as research
|
||||
|
||||
exp = research.experiment("benchmark", "grok-4.5", "gpqa_diamond")
|
||||
exp.record("q1", "grok-4.5", {"answer": "A", "correct": True})
|
||||
exp.snapshot(png_bytes)
|
||||
exp.report(markdown_text)
|
||||
exp.finish(value=94.3)
|
||||
|
||||
rows = research.query(project="enso-bench", kind="benchmark")
|
||||
|
||||
Zero-config: provenance (git sha/branch + the commit-message narrative since the last
|
||||
recorded run, lib versions, host) is auto-captured — the caller supplies none. Records
|
||||
mirror to cloud /v1/research (HIP-0512): versioned, append-only, idempotent by content,
|
||||
private by default.
|
||||
"""
|
||||
from .client import Research, Experiment
|
||||
|
||||
__all__ = ["Research", "Experiment", "experiment", "query", "totals", "configure", "client"]
|
||||
|
||||
_default = None
|
||||
|
||||
|
||||
def configure(**kwargs):
|
||||
"""Set the process-wide default client (base, api_key, project, repo, libs)."""
|
||||
global _default
|
||||
_default = Research(**kwargs)
|
||||
return _default
|
||||
|
||||
|
||||
def client():
|
||||
"""The process-wide default client, built from the environment on first use."""
|
||||
global _default
|
||||
if _default is None:
|
||||
_default = Research()
|
||||
return _default
|
||||
|
||||
|
||||
def experiment(kind, subject, task, metric="accuracy", n_total=0, note=""):
|
||||
"""Get/create an experiment handle on the default client."""
|
||||
return client().experiment(kind, subject, task, metric=metric, n_total=n_total, note=note)
|
||||
|
||||
|
||||
def query(project=None, kind=None, canonical=True):
|
||||
"""Read canonical experiments on the default client."""
|
||||
return client().query(project=project, kind=kind, canonical=canonical)
|
||||
|
||||
|
||||
def totals(project=None):
|
||||
"""Read the headline totals (canonical + retained) on the default client."""
|
||||
return client().totals(project=project)
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Hanzo Research SDK — the ONE way every producer records and queries research.
|
||||
|
||||
Generated-shape from the openapi/research contract, wrapped in a tiny, intuitive,
|
||||
ZERO-CONFIG surface so hand-rolling is obviously the worse choice:
|
||||
|
||||
import hanzo_research as research
|
||||
|
||||
exp = research.experiment("benchmark", "grok-4.5", "gpqa_diamond") # get/create → handle
|
||||
exp.record("q1", "grok-4.5", {"answer": "A", "correct": True}) # an attempt, idempotent
|
||||
exp.snapshot(png_bytes) # a board snapshot artifact
|
||||
exp.report(markdown_text) # a generated report artifact
|
||||
exp.finish(value=94.3) # seal the run (or use `with`)
|
||||
|
||||
rows = research.query(project="enso-bench", kind="benchmark") # read canonical
|
||||
|
||||
Auto-instrumentation (the caller supplies NO provenance): on experiment()/finish() the
|
||||
SDK reads git sha/branch/dirty, the commit messages SINCE the last recorded run of this
|
||||
experiment (the narrative of what changed), the installed lib versions, and the host —
|
||||
and weaves them into the record. Research self-documents as a side effect of running.
|
||||
|
||||
Auth is the per-org key (private-by-default is enforced server-side). Records mirror to
|
||||
the cloud /v1/research surface (HIP-0512), which is versioned + append-only + idempotent
|
||||
by content, so re-recording the same result is a no-op.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
from . import provenance
|
||||
|
||||
DEFAULT_LIBS = [
|
||||
"hanzo-kernel", "hanzo-ml", "hanzo-engine", "hanzo-research",
|
||||
"enso-bench", "datasets", "huggingface-hub", "torch", "transformers",
|
||||
]
|
||||
|
||||
|
||||
class Research:
|
||||
"""A configured research client. Base URL, per-org key, and project come from the
|
||||
environment by default (HANZO_API_KEY, RESEARCH_BASE, RESEARCH_PROJECT); the repo the
|
||||
caller runs in is auto-detected for provenance."""
|
||||
|
||||
def __init__(self, base=None, api_key=None, project=None, repo=None, libs=None, timeout=120):
|
||||
self.base = (base or os.environ.get("RESEARCH_BASE", "https://api.hanzo.ai")).rstrip("/")
|
||||
self.api_key = api_key or os.environ.get("HANZO_API_KEY", "")
|
||||
self.project = project or os.environ.get("RESEARCH_PROJECT", "default")
|
||||
self.repo = repo or provenance.find_repo()
|
||||
self.libs = libs if libs is not None else DEFAULT_LIBS
|
||||
self.timeout = timeout
|
||||
|
||||
# ── transport ──────────────────────────────────────────────────────────────
|
||||
def _headers(self):
|
||||
h = {"Content-Type": "application/json", "X-Project-Id": self.project}
|
||||
if self.api_key:
|
||||
h["Authorization"] = "Bearer " + self.api_key
|
||||
elif os.environ.get("RESEARCH_ORG"): # dev/local: raw principal for a gateway-less server
|
||||
org = os.environ["RESEARCH_ORG"]
|
||||
h["X-Org-Id"] = org
|
||||
h["X-User-Id"] = "u-" + org
|
||||
return h
|
||||
|
||||
def _req(self, method, path, payload=None):
|
||||
data = json.dumps(payload).encode() if payload is not None else None
|
||||
req = urllib.request.Request(self.base + path, data=data, headers=self._headers(), method=method)
|
||||
with urllib.request.urlopen(req, timeout=self.timeout) as r:
|
||||
return json.loads(r.read() or b"{}")
|
||||
|
||||
# ── low-level surface (mirrors the contract; used by the bulk uploader) ──────
|
||||
def ingest(self, experiments=None, attempts=None):
|
||||
"""POST a batch of experiments + attempts. Idempotent by content; returns the
|
||||
cloud counts (canonical + retained)."""
|
||||
return self._req("POST", "/v1/research/experiments",
|
||||
{"experiments": experiments or [], "attempts": attempts or []})
|
||||
|
||||
def artifact(self, art):
|
||||
return self._req("POST", "/v1/research/artifacts", art)
|
||||
|
||||
def grant(self, **g):
|
||||
return self._req("POST", "/v1/research/grants", g)
|
||||
|
||||
# ── ergonomic surface ───────────────────────────────────────────────────────
|
||||
def experiment(self, kind, subject, task, metric="accuracy", n_total=0, note=""):
|
||||
"""Get/create the experiment for (kind, subject, task) and return a handle. Auto-
|
||||
captures provenance + the self-documenting narrative (the caller's docstring, the
|
||||
commit messages since this experiment's last run, and an optional note)."""
|
||||
return Experiment(self, kind, subject, task, metric=metric, n_total=n_total, note=note)
|
||||
|
||||
def query(self, project=None, kind=None, canonical=True):
|
||||
"""Read canonical experiments (the answered latest-run view). canonical=False is
|
||||
reserved for the full retained-version history (a follow-up endpoint; the retained
|
||||
COUNTS are already on /v1/research/totals)."""
|
||||
q = []
|
||||
if project or self.project:
|
||||
q.append("project=" + urllib.parse.quote(project or self.project))
|
||||
if kind:
|
||||
q.append("kind=" + urllib.parse.quote(kind))
|
||||
path = "/v1/research/experiments" + ("?" + "&".join(q) if q else "")
|
||||
out = self._req("GET", path)
|
||||
return out.get("data", [])
|
||||
|
||||
def totals(self, project=None):
|
||||
path = "/v1/research/totals" + ("?project=" + urllib.parse.quote(project) if project else "")
|
||||
return self._req("GET", path)
|
||||
|
||||
# ── internal ────────────────────────────────────────────────────────────────
|
||||
def _last_run_sha(self, exp_id):
|
||||
"""The git sha of this experiment's last recorded run, for the since-narrative."""
|
||||
try:
|
||||
for e in self.query():
|
||||
if e.get("id") == exp_id:
|
||||
return e.get("git_sha", "")
|
||||
except Exception:
|
||||
return ""
|
||||
return ""
|
||||
|
||||
|
||||
class Experiment:
|
||||
"""A handle to one experiment (a run of kind:subject:task). record() files attempts;
|
||||
snapshot()/report() file diary artifacts; finish() seals the run with its final
|
||||
number + the auto-captured narrative. Usable as a context manager (finish on exit)."""
|
||||
|
||||
def __init__(self, client, kind, subject, task, metric="accuracy", n_total=0, note=""):
|
||||
self.c = client
|
||||
self.kind, self.subject, self.task, self.metric = kind, subject, task, metric
|
||||
self.id = f"{kind}:{subject}:{task}"
|
||||
self.n_total = n_total
|
||||
self._ok = 0
|
||||
self._n = 0
|
||||
# Zero-config, self-documenting provenance. THREE narrative sources, captured with
|
||||
# nothing asked of the caller: (a) the calling code's docstring — what this
|
||||
# experiment IS; (b) the commit messages since this experiment's last recorded run
|
||||
# — what changed + why; (c) an optional note. Plus git state, lib versions, host.
|
||||
self.prov = provenance.git_state(self.c.repo)
|
||||
self.doc = provenance.caller_doc()
|
||||
self.note = note
|
||||
last = self.c._last_run_sha(self.id)
|
||||
self.commits = provenance.commit_narrative(self.c.repo, since_sha=last)
|
||||
self.libs = provenance.lib_versions(self.c.libs)
|
||||
self.host = provenance.host()
|
||||
# Post the run as in-flight so the ops board sees it immediately.
|
||||
self._post(status="running", value=0.0)
|
||||
|
||||
def _post(self, status, value):
|
||||
e = {
|
||||
"id": self.id, "kind": self.kind, "subject": self.subject, "task": self.task,
|
||||
"metric": self.metric, "value": value, "n": self._n, "n_total": self.n_total,
|
||||
"status": status, "git_sha": self.prov["git_sha"], "git_branch": self.prov["git_branch"],
|
||||
"git_dirty": self.prov["git_dirty"], "lib_versions": self.libs,
|
||||
"meta": {"doc": self.doc, "commits": self.commits, "note": self.note, "host": self.host},
|
||||
}
|
||||
return self.c.ingest(experiments=[e])
|
||||
|
||||
def record(self, item, model, result):
|
||||
"""File one attempt (idempotent by stable id). `result` is a dict:
|
||||
{answer, correct, response?, gold?, source?, status?}."""
|
||||
a = {
|
||||
"benchmark": self.task, "item": item, "model": model,
|
||||
"answer": result.get("answer", ""), "correct": bool(result.get("correct")),
|
||||
"response": result.get("response", ""), "gold": result.get("gold", ""),
|
||||
"source": result.get("source", "hanzo-measured"), "status": result.get("status", "complete"),
|
||||
}
|
||||
if a["status"] not in ("faulted", "failed"):
|
||||
self._n += 1
|
||||
if a["correct"]:
|
||||
self._ok += 1
|
||||
return self.c.ingest(attempts=[a])
|
||||
|
||||
def snapshot(self, data, ref=None, run_id=None):
|
||||
"""File a board-snapshot artifact. `data` is the PNG bytes (or a path to the PNG);
|
||||
the SDK content-addresses it by sha256. `ref` is the blob location (defaults to
|
||||
the content address; the blob upload to object storage is the recorder's job)."""
|
||||
raw = data if isinstance(data, (bytes, bytearray)) else _read_bytes(data)
|
||||
return self._artifact("snapshot", raw, ref, run_id)
|
||||
|
||||
def report(self, data, ref=None, run_id=None):
|
||||
"""File a generated-report artifact — HTML/Markdown TEXT (or bytes)."""
|
||||
raw = data if isinstance(data, (bytes, bytearray)) else str(data).encode()
|
||||
return self._artifact("report", raw, ref, run_id)
|
||||
|
||||
def _artifact(self, kind, raw, ref, run_id):
|
||||
sha = hashlib.sha256(raw).hexdigest()
|
||||
return self.c.artifact({
|
||||
"sha256": sha, "kind": kind, "ref": ref or f"sha256:{sha}",
|
||||
"run_id": run_id or self.id, "git_sha": self.prov["git_sha"],
|
||||
"git_branch": self.prov["git_branch"], "git_dirty": self.prov["git_dirty"],
|
||||
"lib_versions": self.libs,
|
||||
})
|
||||
|
||||
def finish(self, value=None):
|
||||
"""Seal the run: post the final number (computed from recorded attempts if not
|
||||
given) + status complete. A new version supersedes the in-flight one; the prior
|
||||
is retained."""
|
||||
if value is None:
|
||||
value = round(100.0 * self._ok / self._n, 2) if self._n else 0.0
|
||||
return self._post(status="complete", value=value)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
# A run that raised is sealed faulted (retained), else complete.
|
||||
if exc_type is not None:
|
||||
self._post(status="faulted", value=0.0)
|
||||
else:
|
||||
self.finish()
|
||||
return False
|
||||
|
||||
|
||||
def _read_bytes(path):
|
||||
with open(path, "rb") as f:
|
||||
return f.read()
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Zero-config provenance auto-capture — like OpenTelemetry auto-instrument, for
|
||||
experiments. The caller supplies nothing; the SDK reads the run's environment: git
|
||||
sha/branch/dirty, the recent COMMIT MESSAGES (the narrative of what changed), the
|
||||
installed lib versions, and the host. Import and go.
|
||||
"""
|
||||
import inspect
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
from importlib import metadata
|
||||
|
||||
|
||||
def _git(repo, *args):
|
||||
try:
|
||||
return subprocess.check_output(["git", "-C", repo, *args], stderr=subprocess.DEVNULL, text=True).strip()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def git_state(repo):
|
||||
"""The producing repo's commit sha, branch, and whether the tree was dirty."""
|
||||
return {
|
||||
"git_sha": _git(repo, "rev-parse", "HEAD"),
|
||||
"git_branch": _git(repo, "rev-parse", "--abbrev-ref", "HEAD"),
|
||||
"git_dirty": bool(_git(repo, "status", "--porcelain")),
|
||||
}
|
||||
|
||||
|
||||
def commit_narrative(repo, since_sha="", window=10):
|
||||
"""The commit-message narrative — the 'what changed + why' story. Commit subjects
|
||||
SINCE the last recorded run's sha when known (`<since>..HEAD`), else the last
|
||||
`window` commits. Research self-documents as a side effect of running."""
|
||||
rng = f"{since_sha}..HEAD" if since_sha else f"-n{window}"
|
||||
raw = _git(repo, "log", rng, "--format=%s") if since_sha else _git(repo, "log", f"-{window}", "--format=%s")
|
||||
return [ln for ln in raw.splitlines() if ln.strip()]
|
||||
|
||||
|
||||
def lib_versions(names):
|
||||
"""{lib: installed version} for the relevant libs, skipping any not installed."""
|
||||
out = {}
|
||||
for n in names or []:
|
||||
try:
|
||||
out[n] = metadata.version(n)
|
||||
except Exception:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def host():
|
||||
"""The box the run executed on — enough to correlate a result to hardware."""
|
||||
return {"hostname": socket.gethostname(), "platform": os.uname().sysname if hasattr(os, "uname") else os.name}
|
||||
|
||||
|
||||
def find_repo(start=None):
|
||||
"""The git repo the caller runs in (the toplevel), auto-detected from the CWD."""
|
||||
start = start or os.getcwd()
|
||||
top = _git(start, "rev-parse", "--show-toplevel")
|
||||
return top or start
|
||||
|
||||
|
||||
def caller_doc():
|
||||
"""The docstring of the first caller frame OUTSIDE this SDK — 'what this experiment
|
||||
is', captured zero-config from the researcher's normal module docstring. Together
|
||||
with the commit narrative and an optional note, the project self-documents as a side
|
||||
effect of running: write a normal docstring + commit normally, nothing extra."""
|
||||
for fr in inspect.stack():
|
||||
mod = fr.frame.f_globals.get("__name__", "")
|
||||
if not mod.startswith("hanzo_research"):
|
||||
return (fr.frame.f_globals.get("__doc__") or "").strip()
|
||||
return ""
|
||||
@@ -0,0 +1,16 @@
|
||||
[project]
|
||||
name = "hanzo-research"
|
||||
version = "0.1.0"
|
||||
description = "Hanzo Research SDK — record and query R&D evidence (experiments, attempts, diary artifacts) via /v1/research. Zero-config provenance auto-capture."
|
||||
requires-python = ">=3.9"
|
||||
dependencies = [] # stdlib only (urllib) — importable in any zero-dep harness
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://hanzo.ai"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["hanzo_research"]
|
||||
Reference in New Issue
Block a user