mirror of
https://github.com/hanzoai/ml.git
synced 2026-11-20 23:08:29 +00:00
The Rustfmt job runs `cargo fmt --all -- --check` and reported 653 diffs across
43 files. This is that command's output, nothing else — no logic is touched.
Worth noting why it surfaced only now: the previous run on this repo was a
`schedule` trigger, which fires gpu-tests alone. Rustfmt, Clippy and the test
suite belong to the push-triggered workflow and had not run in that window, so
they were not newly broken — they were newly VISIBLE.
Scope: hanzo-flash-attn-v3, hanzo-kernel, hanzo-kernel-macros,
hanzo-metal-kernels, hanzo-ml, hanzo-nn, hanzo-research. The insertion count is
much larger than the deletion count because rustfmt rewraps long lines rather
than merely reindenting them, which is also why `git diff -w` does not collapse
the change — whitespace-blind diff is not a purity test for a formatter.
Evidence it is safe, given `cargo check --workspace` cannot run here (onig_sys
fails to build its vendored oniguruma against this macOS SDK, unrelated to
formatting):
- rustfmt is idempotent on the result. Pass 1 wrote 43 files; pass 2 re-read
and re-parsed every one and produced zero further change. rustfmt fails
outright on unparseable input, so a clean second pass means the emitted code
is valid Rust.
- `cargo check -p` is clean (0 errors) for hanzo-kernel, hanzo-kernel-macros
and hanzo-research — the changed crates that pull no C dependencies.
- `cargo fmt --all -- --check`, the exact command CI runs, now exits 0.
No rustfmt.toml exists and no toolchain is pinned, so this is default stable
style — the same thing dtolnay/rust-toolchain@stable gives the runner.
Clippy is NOT addressed here. Its findings are in hanzo-ml/src/quantized/ and
are real lint work, not formatting.
Co-authored-by: Hanzo Dev <[email protected]>
171 lines
6.3 KiB
Rust
171 lines
6.3 KiB
Rust
//! End-to-end proof: a kernel-perf run structured as a falsifiable test drives the exact
|
|
//! `/v1/research` wire calls against a throwaway mock server. Asserts the in-flight record
|
|
//! is shape-identical to the Python SDK (frame set, empty log) and the sealed record
|
|
//! carries the refutation verdict + auto-captured provenance.
|
|
|
|
use std::io::{BufRead, BufReader, Read, Write};
|
|
use std::net::{TcpListener, TcpStream};
|
|
use std::sync::mpsc::{self, Receiver};
|
|
use std::thread;
|
|
use std::time::Duration;
|
|
|
|
use hanzo_research::{Outcome, Research, Verdict};
|
|
use serde_json::{json, Value};
|
|
|
|
/// A minimal HTTP/1.1 mock: one request per connection, captures (method, path, body),
|
|
/// answers GETs with an empty listing and POSTs with an ingest ack. std-only, no deps.
|
|
fn mock() -> (String, Receiver<(String, String, String)>) {
|
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
let addr = listener.local_addr().unwrap();
|
|
let (tx, rx) = mpsc::channel();
|
|
thread::spawn(move || {
|
|
for stream in listener.incoming() {
|
|
let mut s = stream.unwrap();
|
|
let (method, path, body) = read_request(&mut s);
|
|
let resp_body = if method == "GET" {
|
|
r#"{"data":[],"total":0}"#.to_string()
|
|
} else {
|
|
r#"{"project":"proj","experiments_ingested":1,"rolled_up":true}"#.to_string()
|
|
};
|
|
let resp = format!(
|
|
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
|
|
resp_body.len(),
|
|
resp_body
|
|
);
|
|
let _ = s.write_all(resp.as_bytes());
|
|
let _ = tx.send((method, path, body));
|
|
}
|
|
});
|
|
(format!("http://{addr}"), rx)
|
|
}
|
|
|
|
fn read_request(s: &mut TcpStream) -> (String, String, String) {
|
|
let mut reader = BufReader::new(s.try_clone().unwrap());
|
|
let mut line = String::new();
|
|
reader.read_line(&mut line).unwrap();
|
|
let mut parts = line.split_whitespace();
|
|
let method = parts.next().unwrap_or("").to_string();
|
|
let path = parts.next().unwrap_or("").to_string();
|
|
let mut len = 0usize;
|
|
loop {
|
|
let mut h = String::new();
|
|
reader.read_line(&mut h).unwrap();
|
|
if h == "\r\n" || h.is_empty() {
|
|
break;
|
|
}
|
|
if let Some(v) = h.to_ascii_lowercase().strip_prefix("content-length:") {
|
|
len = v.trim().parse().unwrap_or(0);
|
|
}
|
|
}
|
|
let mut body = vec![0u8; len];
|
|
if len > 0 {
|
|
reader.read_exact(&mut body).unwrap();
|
|
}
|
|
(method, path, String::from_utf8_lossy(&body).to_string())
|
|
}
|
|
|
|
#[test]
|
|
fn kernel_perf_refutation_drives_the_wire() {
|
|
let (base, rx) = mock();
|
|
let r = Research::new(&base, "test-key", "proj", "", &[]);
|
|
|
|
let mut k = r
|
|
.experiment("kernel-perf", "matvec_q4k_f32_blk", "vulkan/6144x2048")
|
|
.metric("ratio_vs_hand")
|
|
.hypothesis("the DSL f32-direct matvec beats the hand kernel")
|
|
.predict("DSL/hand >= 1.0 cold in-engine at the dominant FFN shape");
|
|
k.log("cold in-engine A/B, evo gfx1151, quiet window, 3 runs, bit-exact 2.3e-6");
|
|
k.conclude(
|
|
Verdict::Refuted,
|
|
"0.79x at 6144 rows — memory-BW wall, not craft",
|
|
Some(0.79),
|
|
)
|
|
.unwrap();
|
|
|
|
// Drain: a since-narrative GET, the running POST, the complete POST.
|
|
let mut got_get = false;
|
|
let mut posts: Vec<Value> = Vec::new();
|
|
for _ in 0..3 {
|
|
let (m, p, b) = rx.recv_timeout(Duration::from_secs(5)).unwrap();
|
|
assert!(p.starts_with("/v1/research/"), "path was {p}");
|
|
if m == "GET" {
|
|
got_get = true;
|
|
} else {
|
|
posts.push(serde_json::from_str(&b).unwrap());
|
|
}
|
|
}
|
|
assert!(got_get, "the since-narrative GET was issued");
|
|
assert_eq!(posts.len(), 2, "a running post then a complete post");
|
|
|
|
// The in-flight record: frame stated, no verdict, empty log — as the Python SDK posts.
|
|
let running = &posts[0]["experiments"][0];
|
|
assert_eq!(running["status"], "running");
|
|
assert_eq!(
|
|
running["meta"]["hypothesis"],
|
|
"the DSL f32-direct matvec beats the hand kernel"
|
|
);
|
|
assert_eq!(running["meta"]["verdict"], "");
|
|
assert_eq!(
|
|
running["meta"]["log"],
|
|
json!([]),
|
|
"running log empty, as in Python"
|
|
);
|
|
|
|
// The sealed record: the refutation is first-class + provenance auto-stamped.
|
|
let done = &posts[1]["experiments"][0];
|
|
assert_eq!(done["status"], "complete");
|
|
assert_eq!(done["value"], 0.79);
|
|
assert_eq!(done["metric"], "ratio_vs_hand");
|
|
assert_eq!(done["meta"]["verdict"], "refuted");
|
|
assert_eq!(
|
|
done["meta"]["because"],
|
|
"0.79x at 6144 rows — memory-BW wall, not craft"
|
|
);
|
|
assert_eq!(
|
|
done["meta"]["log"][0],
|
|
"cold in-engine A/B, evo gfx1151, quiet window, 3 runs, bit-exact 2.3e-6"
|
|
);
|
|
assert_eq!(
|
|
done["git_sha"], running["git_sha"],
|
|
"provenance consistent across versions"
|
|
);
|
|
assert!(done["lib_versions"].is_object());
|
|
assert!(done["meta"]["host"]["hostname"].is_string());
|
|
}
|
|
|
|
#[test]
|
|
fn benchmark_attempts_then_finish_computes_the_score() {
|
|
let (base, rx) = mock();
|
|
let r = Research::new(&base, "test-key", "enso-bench", "", &[]);
|
|
|
|
let mut b = r
|
|
.experiment("benchmark", "grok-4.5", "gpqa_diamond")
|
|
.n_total(2);
|
|
b.record("q1", "grok-4.5", Outcome::correct("A")).unwrap();
|
|
b.record("q2", "grok-4.5", Outcome::wrong("B", "C"))
|
|
.unwrap();
|
|
b.finish(None).unwrap(); // value computed from attempts: 1 of 2 -> 50.0
|
|
|
|
// GET(since) + running POST + 2 attempt POSTs + complete POST.
|
|
let mut posts: Vec<Value> = Vec::new();
|
|
for _ in 0..5 {
|
|
let (m, _p, body) = rx.recv_timeout(Duration::from_secs(5)).unwrap();
|
|
if m == "POST" {
|
|
posts.push(serde_json::from_str(&body).unwrap());
|
|
}
|
|
}
|
|
let finish = posts.last().unwrap();
|
|
let exp = &finish["experiments"][0];
|
|
assert_eq!(exp["status"], "complete");
|
|
assert_eq!(exp["value"], 50.0, "1 correct of 2 scored");
|
|
assert_eq!(exp["n"], 2);
|
|
|
|
// The first attempt post carried the item under the experiment's task/benchmark.
|
|
let first_attempt = posts
|
|
.iter()
|
|
.find(|p| !p["attempts"].as_array().unwrap().is_empty())
|
|
.unwrap();
|
|
assert_eq!(first_attempt["attempts"][0]["benchmark"], "gpqa_diamond");
|
|
assert_eq!(first_attempt["attempts"][0]["source"], "hanzo-measured");
|
|
}
|