mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
hanzo/research: cap JSON parser depth, cap response body, match CPython float repr
- json::Parser bounds nesting at 256 levels: past the cap value() returns Null (a parse-fail) instead of recursing, so a deeply-nested response degrades rather than overflowing the stack. Every server response is parsed here; a 200k-nested body previously SIGSEGV'd the producer. - Curl transport caps the response body at 8 MiB (a short write aborts the transfer), so a hostile server cannot OOM the client. Redirects were already disabled (FOLLOWLOCATION=0). Parity with the Go/Rust ports. - writeReal emits CPython repr(float) notation: fixed for decimal exponent in [-4, 16), scientific otherwise, so 1e15/0.0001 match json.dumps. The realistic value domain is unchanged (ingest byte-identity to Python preserved). - Docs state the accurate contract: cross-language identity is semantic (server keys on project + id); this serializer additionally matches json.dumps bytes. Verified: 200k-nest parse returns without crashing; SHA-256 self-test PASS; ingest body still 1375 bytes / sha256 1914bc1e… (byte-identical to Python).
This commit is contained in:
+5
-3
@@ -85,9 +85,11 @@ DATASTORE_DB=hanzo
|
||||
|
||||
The C++ port of the Hanzo Research SDK: the ONE way a native producer (a kernel, a
|
||||
benchmark, the datastore/luxcpp stack) records + queries R&D evidence on the unified
|
||||
`/v1/research` plane (HIP-0512). It mirrors the Python `hanzo-research` SDK verb-for-verb
|
||||
and is **byte-identical on the wire** (same key order, `ensure_ascii`, float repr), so every
|
||||
language emits uniform records into the one store.
|
||||
`/v1/research` plane (HIP-0512). It mirrors the Python `hanzo-research` SDK verb-for-verb.
|
||||
Records are **semantically identical** across languages — the server keys each experiment on
|
||||
`(project, id = kind:subject:task)`, so every language upserts the same row regardless of
|
||||
serialization. This C++ serializer additionally matches Python's `json.dumps` byte-for-byte
|
||||
(same key order, `ensure_ascii`, float repr).
|
||||
|
||||
```cpp
|
||||
#include "research.hpp"
|
||||
|
||||
+49
-8
@@ -1,5 +1,7 @@
|
||||
// Implementation of the Hanzo Research C++ client. See research.hpp for the API and the
|
||||
// wire contract it mirrors (the Python hanzo-research SDK, byte-for-byte).
|
||||
// wire contract it mirrors (the Python hanzo-research SDK). Records are semantically
|
||||
// identical across languages (the server keys on project + id = kind:subject:task); this
|
||||
// serializer also happens to match Python's json.dumps byte-for-byte.
|
||||
#include "research.hpp"
|
||||
|
||||
#include <array>
|
||||
@@ -182,10 +184,29 @@ void escape(std::string& out, const std::string& s) {
|
||||
}
|
||||
|
||||
void writeReal(std::string& out, double d) {
|
||||
char buf[32];
|
||||
auto r = std::to_chars(buf, buf + sizeof buf, d);
|
||||
char buf[64];
|
||||
// Non-finite has no JSON form; emit the to_chars text ("inf"/"nan") and stop.
|
||||
if (!std::isfinite(d)) {
|
||||
auto r = std::to_chars(buf, buf + sizeof buf, d);
|
||||
out.append(buf, r.ptr);
|
||||
return;
|
||||
}
|
||||
// Match CPython repr(float): shortest round-tripping digits, in FIXED notation when the
|
||||
// decimal exponent E = floor(log10(|d|)) is in [-4, 16) and SCIENTIFIC otherwise (the
|
||||
// json.dumps float rule). std::to_chars yields the shortest digits in either notation;
|
||||
// we only pick the notation CPython would, so the boundary values (1e15, 1e-4) agree.
|
||||
std::chars_format fmt = std::chars_format::fixed;
|
||||
if (d != 0.0) {
|
||||
char sci[64];
|
||||
auto r = std::to_chars(sci, sci + sizeof sci, d, std::chars_format::scientific);
|
||||
*r.ptr = '\0';
|
||||
const char* e = std::strchr(sci, 'e');
|
||||
int E = e ? std::atoi(e + 1) : 0;
|
||||
if (E < -4 || E >= 16) fmt = std::chars_format::scientific;
|
||||
}
|
||||
auto r = std::to_chars(buf, buf + sizeof buf, d, fmt);
|
||||
std::string s(buf, r.ptr);
|
||||
if (s.find_first_of(".eEnN") == std::string::npos) s += ".0"; // a Python float always shows a point
|
||||
if (s.find_first_of(".eE") == std::string::npos) s += ".0"; // a Python float always shows a point
|
||||
out += s;
|
||||
}
|
||||
|
||||
@@ -249,6 +270,11 @@ namespace {
|
||||
struct Parser {
|
||||
const std::string& s;
|
||||
size_t i = 0;
|
||||
int depth = 0;
|
||||
// Nested-container cap. Past it, value() returns Null (a parse-fail) instead of
|
||||
// recursing — a hostile deeply-nested response degrades, it never overflows the stack.
|
||||
// Every server response is parsed here, so the bound is a DoS guard, not a nicety.
|
||||
static constexpr int kMaxDepth = 256;
|
||||
explicit Parser(const std::string& t) : s(t) {}
|
||||
|
||||
void ws() { while (i < s.size() && (s[i] == ' ' || s[i] == '\t' || s[i] == '\n' || s[i] == '\r')) ++i; }
|
||||
@@ -258,8 +284,15 @@ struct Parser {
|
||||
ws();
|
||||
if (i >= s.size()) return Value();
|
||||
char c = s[i];
|
||||
if (c == '{') return object();
|
||||
if (c == '[') return array();
|
||||
if (c == '{' || c == '[') {
|
||||
// Bound the nesting: past the cap, fail the parse (Null) rather than recurse into
|
||||
// a stack overflow. Increment before descending, decrement on the way back up.
|
||||
if (depth >= kMaxDepth) return Value();
|
||||
++depth;
|
||||
Value v = (c == '{') ? object() : array();
|
||||
--depth;
|
||||
return v;
|
||||
}
|
||||
if (c == '"') return Value(string());
|
||||
if (c == 't') { i += 4; return Value(true); }
|
||||
if (c == 'f') { i += 5; return Value(false); }
|
||||
@@ -474,9 +507,17 @@ public:
|
||||
}
|
||||
|
||||
private:
|
||||
// Response-body cap (8 MiB): a hostile or broken server cannot stream an unbounded body
|
||||
// to OOM the client. Parity with the Go/Rust ports.
|
||||
static constexpr size_t kMaxResponseBytes = 8u << 20;
|
||||
static size_t write(char* p, size_t sz, size_t n, void* ud) {
|
||||
static_cast<std::string*>(ud)->append(p, sz * n);
|
||||
return sz * n;
|
||||
auto* body = static_cast<std::string*>(ud);
|
||||
size_t add = sz * n;
|
||||
// Refuse an oversized body: returning a short count aborts the transfer, so
|
||||
// curl_easy_perform returns CURLE_WRITE_ERROR and request() throws — never an OOM.
|
||||
if (add > 0 && body->size() + add > kMaxResponseBytes) return 0;
|
||||
body->append(p, add);
|
||||
return add;
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
+9
-6
@@ -1,7 +1,9 @@
|
||||
// The C++ port of the Hanzo Research SDK — the ONE way a native producer (a kernel, a
|
||||
// benchmark, the datastore/luxcpp stack) records and queries R&D evidence on the unified
|
||||
// /v1/research plane (HIP-0512). It mirrors the Python reference verb-for-verb so every
|
||||
// language emits byte-identical records into the one store.
|
||||
// /v1/research plane (HIP-0512). It mirrors the Python reference verb-for-verb. Records are
|
||||
// semantically identical across languages — the server keys each experiment on (project,
|
||||
// id = kind:subject:task) — so every language upserts the same row regardless of
|
||||
// serialization. This serializer also happens to match Python's json.dumps byte-for-byte.
|
||||
//
|
||||
// #include "research.hpp"
|
||||
// namespace research = hanzo::research;
|
||||
@@ -34,10 +36,11 @@ namespace hanzo::research {
|
||||
|
||||
// ── json ────────────────────────────────────────────────────────────────────────────
|
||||
// A minimal ordered JSON value. Objects preserve insertion order and dump() matches
|
||||
// Python's json.dumps default (", "/": " separators, ensure_ascii, float repr) so a C++
|
||||
// record is byte-identical to the same record from the Python SDK. The record schema is
|
||||
// small and closed, so a focused serializer is simpler than taking a JSON dependency —
|
||||
// and it is the only way to guarantee that byte-identity.
|
||||
// Python's json.dumps default (", "/": " separators, ensure_ascii, float repr), so a C++
|
||||
// record is byte-for-byte the same as the Python SDK's. That match is a convenience, not
|
||||
// the identity guarantee: the server keys records on (project, id = kind:subject:task), so
|
||||
// identity across languages is semantic. The record schema is small and closed, so a
|
||||
// focused serializer is simpler than taking on a JSON dependency.
|
||||
namespace json {
|
||||
|
||||
class Value {
|
||||
|
||||
Reference in New Issue
Block a user