hanzo-research 0.2.0: benchmarks, runs, studies, papers and compare

The research client spoke only the experiments wire. The record
hanzo.ai/benchmarks draws from is /v1/research/{benchmarks,runs,studies,
papers,compare}, and producers posted it by hand (Kai's bench/code
post.py). record() posts each non-empty batch under its own key; runs(),
benchmarks(), studies() and papers() read with the API's own filters,
org= included for another org's public runs; compare() reads two runs
side by side. Tested against a stand-in server and read live.

Claude-Session: https://claude.ai/code/session_01M8RoubcPN71KgJuZ8LdBUL

pkg/hanzo-research/README.md

@@ -18,6 +18,24 @@ exp.finish(value=94.3)                                          # seal the run
 rows = research.query(project="enso-bench", kind="benchmark")    # read canonical
 ```
 
+## Benchmarks, runs, studies, papers
+
+The record hanzo.ai/benchmarks draws from. A benchmark defines what is measured; a run
+is one execution with its measures; a study is the question runs answer; a paper cites
+runs. Each write is a batch, versioned and idempotent by content.
+
+```python
+research.record(
+    benchmarks=[{"id": "enso.route", "title": "...", "metric": "route_p50_us"}],
+    runs=[{"id": "enso.route-dgx", "benchmark": "enso.route", "system": "enso",
+           "measures": [{"category": "route", "metric": "route_p50_us", "value": 12.576}],
+           "visibility": "public", "canonical": True}],
+)
+research.runs(benchmark="enso.route")              # yours
+research.runs(benchmark="mcp.injection", org="hanzo")  # another org's public runs
+research.compare("run-a", "run-b")                 # side by side, with what stops it
+```
+
 ## Zero-config auto-instrumentation
 
 The caller supplies **no** provenance. On `experiment()` the SDK reads the run's

pkg/hanzo-research/hanzo_research/__init__.py

@@ -17,7 +17,8 @@ private by default.
 """
 from .client import Research, Experiment
 
-__all__ = ["Research", "Experiment", "experiment", "query", "totals", "configure", "client"]
+__all__ = ["Research", "Experiment", "experiment", "query", "totals", "configure", "client",
+           "record", "benchmarks", "runs", "studies", "papers", "compare"]
 
 _default = None
 
@@ -53,3 +54,29 @@ def query(project=None, kind=None, canonical=True):
 def totals(project=None):
     """Read the headline totals (canonical + retained) on the default client."""
     return client().totals(project=project)
+
+
+def record(benchmarks=(), runs=(), studies=(), papers=()):
+    """Record benchmark definitions, runs, studies and papers on the default client."""
+    return client().record(benchmarks=benchmarks, runs=runs, studies=studies, papers=papers)
+
+
+def benchmarks(**filters):
+    return client().benchmarks(**filters)
+
+
+def runs(**filters):
+    """Runs, e.g. runs(benchmark="coding.own", org="hanzo") for another org's public ones."""
+    return client().runs(**filters)
+
+
+def studies(**filters):
+    return client().studies(**filters)
+
+
+def papers(**filters):
+    return client().papers(**filters)
+
+
+def compare(a, b):
+    return client().compare(a, b)

pkg/hanzo-research/hanzo_research/client.py

@@ -122,6 +122,44 @@ class Research:
         path = "/v1/research/totals" + ("?project=" + urllib.parse.quote(project) if project else "")
         return self._req("GET", path)
 
+    # ── the record: benchmarks, runs, studies, papers ───────────────────────────
+    # A benchmark defines what is measured; a run is one execution of it with its
+    # measures (each {category, metric, value, lo?, hi?, n?, of?}); a study is the
+    # question runs answer; a paper is writing that cites runs. Every write is a
+    # batch, versioned and idempotent by content: re-recording the same row is a
+    # no-op, and a changed one becomes its next revision.
+    def record(self, benchmarks=(), runs=(), studies=(), papers=()):
+        """POST each non-empty batch and return the cloud's answers by kind."""
+        out = {}
+        for kind, rows in (("benchmarks", benchmarks), ("studies", studies), ("runs", runs), ("papers", papers)):
+            if rows:
+                out[kind] = self._req("POST", "/v1/research/" + kind, {kind: list(rows)})
+        return out
+
+    def _list(self, kind, **params):
+        q = urllib.parse.urlencode({k: v for k, v in params.items() if v is not None})
+        return self._req("GET", "/v1/research/" + kind + ("?" + q if q else "")).get("data") or []
+
+    def benchmarks(self, id=None, project=None):
+        """Benchmark definitions, by id or project."""
+        return self._list("benchmarks", id=id, project=project)
+
+    def runs(self, id=None, benchmark=None, system=None, study=None, completion=None, since=None, until=None, org=None):
+        """Runs, filtered as GET /v1/research/runs filters them; `org` reads another
+        org's public runs."""
+        return self._list("runs", id=id, benchmark=benchmark, system=system, study=study,
+                          completion=completion, since=since, until=until, org=org)
+
+    def studies(self, id=None, project=None):
+        return self._list("studies", id=id, project=project)
+
+    def papers(self, id=None, study=None, cites=None):
+        return self._list("papers", id=id, study=study, cites=cites)
+
+    def compare(self, a, b):
+        """Two runs side by side, with what stops the comparison."""
+        return self._req("GET", "/v1/research/compare?" + urllib.parse.urlencode({"a": a, "b": b}))
+
     # ── internal ────────────────────────────────────────────────────────────────
     def _last_run_sha(self, exp_id):
         """The git sha of this experiment's last recorded run, for the since-narrative."""

pkg/hanzo-research/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "hanzo-research"
-version = "0.1.0"
-description = "Hanzo Research SDK — record and query R&D evidence (experiments, attempts, diary artifacts) via /v1/research. Zero-config provenance auto-capture."
+version = "0.2.0"
+description = "Hanzo Research SDK — record and query R&D evidence (benchmarks, runs, studies, papers, experiments, artifacts) via /v1/research. Zero-config provenance auto-capture."
 readme = "README.md"
 license = "MIT"
 authors = [{ name = "Hanzo AI", email = "[email protected]" }]

pkg/hanzo-research/tests/test_record.py

@@ -0,0 +1,71 @@
+"""The record wire against a local stand-in for /v1/research: each batch posts to its
+own path under its own key, a filter reaches the query, and compare names both runs."""
+import json
+import threading
+import unittest
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+import hanzo_research as research
+
+SEEN = []
+
+
+class Fake(BaseHTTPRequestHandler):
+    def _answer(self, body):
+        raw = json.dumps(body).encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(raw)))
+        self.end_headers()
+        self.wfile.write(raw)
+
+    def do_POST(self):
+        n = int(self.headers.get("Content-Length", 0))
+        SEEN.append(("POST", self.path, json.loads(self.rfile.read(n)), self.headers.get("Authorization")))
+        self._answer({"project": "default", "recorded": 1})
+
+    def do_GET(self):
+        SEEN.append(("GET", self.path, None, self.headers.get("Authorization")))
+        self._answer({"data": [{"id": "r1"}], "total": 1} if "/runs" in self.path else {"a": {}, "b": {}})
+
+    def log_message(self, *a):
+        pass
+
+
+class RecordWire(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.srv = HTTPServer(("127.0.0.1", 0), Fake)
+        threading.Thread(target=cls.srv.serve_forever, daemon=True).start()
+        research.configure(base=f"http://127.0.0.1:{cls.srv.server_port}", api_key="k", project="enso")
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.srv.shutdown()
+
+    def setUp(self):
+        SEEN.clear()
+
+    def test_each_batch_posts_to_its_path(self):
+        out = research.record(benchmarks=[{"id": "b"}], runs=[{"id": "r", "benchmark": "b"}])
+        self.assertEqual(sorted(out), ["benchmarks", "runs"])
+        posted = {p: body for m, p, body, _ in SEEN if m == "POST"}
+        self.assertEqual(posted["/v1/research/benchmarks"], {"benchmarks": [{"id": "b"}]})
+        self.assertEqual(posted["/v1/research/runs"], {"runs": [{"id": "r", "benchmark": "b"}]})
+        self.assertTrue(all(auth == "Bearer k" for *_, auth in SEEN))
+
+    def test_an_empty_batch_is_not_sent(self):
+        self.assertEqual(research.record(), {})
+        self.assertEqual(SEEN, [])
+
+    def test_runs_carry_their_filters(self):
+        self.assertEqual(research.runs(benchmark="enso.route", org="hanzo"), [{"id": "r1"}])
+        self.assertEqual(SEEN[0][1], "/v1/research/runs?benchmark=enso.route&org=hanzo")
+
+    def test_compare_names_both_runs(self):
+        research.compare("x", "y")
+        self.assertEqual(SEEN[0][1], "/v1/research/compare?a=x&b=y")
+
+
+if __name__ == "__main__":
+    unittest.main()