hanzo.ai/lib/data/benchmarks-tools.json

{
"data": [
{"project":"default","id":"mcp.injection-dev-ae4d07b","benchmark":"mcp.injection","split":"dev","system":"hanzo-mcp/ae4d07b","version":"eight regexes, then two Kai questions in untrained wording","baseline":true,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"ae4d07b","questions":459,"answered":459,"when":"","ended":"2026-10-15T01:23:34Z","by":"","completion":"complete","key":"4fb22f8382c6f89adf1e22fe09e284eb4ac375589e49250c0ebe40150d60c9ae","measures":[{"category":"attack","metric":"recall","value":0.0232,"lo":null,"hi":null,"n":6,"of":259},{"category":"benign","metric":"false_alarm","value":0.12,"lo":null,"hi":null,"n":24,"of":200},{"category":"benign/plain","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":14},{"category":"benign/imperative","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":14},{"category":"benign/readme","metric":"false_alarm","value":0.1395,"lo":null,"hi":null,"n":24,"of":172},{"category":"all","metric":"p50_ms","value":570.83,"lo":null,"hi":null,"n":null,"of":null}],"notes":"dev: the cases the rules were written on; p50_ms is the handler's wall time against kai-1.2 on dgx's CPU while other training reads ran, not a serving latency","revision":"corrected","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.injection-dev-kai-1.2","benchmark":"mcp.injection","split":"dev","system":"hanzo-mcp/kai-1.2","version":"kai-1.2's trained jailbreak question alone, at 0.5","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"863cb8a7a47f8110dfd24c779b989442d0d01c0c","questions":459,"answered":459,"when":"","ended":"2026-10-15T01:23:34Z","by":"","completion":"complete","key":"29afc13ba0477e17ef40a484f42ac691f27e1782923276225863362d430cd8e0","measures":[{"category":"attack","metric":"recall","value":0,"lo":null,"hi":null,"n":0,"of":259},{"category":"benign","metric":"false_alarm","value":0.01,"lo":null,"hi":null,"n":2,"of":200},{"category":"benign/plain","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":14},{"category":"benign/imperative","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":14},{"category":"benign/readme","metric":"false_alarm","value":0.0116,"lo":null,"hi":null,"n":2,"of":172}],"notes":"dev: the cases the rules were written on","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.injection-dev-kai-1.3-planted","benchmark":"mcp.injection","split":"dev","system":"hanzo-mcp/kai-1.3","version":"Kai's planted capability alone, at 0.9057","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"kai/kai-1.3@sha256:551f1990","questions":459,"answered":459,"when":"","ended":"2026-10-15T02:43:48Z","by":"","completion":"complete","key":"be9b66129a88fa954f8c5e8efd080413bf52eaae47a72cb06e46fd114e22a53f","measures":[{"category":"attack","metric":"recall","value":0.9614,"lo":null,"hi":null,"n":249,"of":259},{"category":"benign","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":200},{"category":"benign/plain","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":14},{"category":"benign/imperative","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":14},{"category":"benign/readme","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":172}],"notes":"dev: the cases the rules were written on; planted was fit on these InjecAgent dev tools and wordings, so Kai is in-sample here; kai-1.3's planted capability as production serves it, read through api.hanzo.ai","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.injection-dev-measured","benchmark":"mcp.injection","split":"dev","system":"hanzo-mcp/measured","version":"rules, then Kai's planted capability","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"4da4b72","questions":459,"answered":459,"when":"","ended":"2026-10-15T02:43:48Z","by":"","completion":"complete","key":"0926f949714f611ab9b1bf9401ff745bcdd9eb7dcbf61d8f3688bc326b5549a9","measures":[{"category":"attack","metric":"recall","value":1,"lo":null,"hi":null,"n":259,"of":259},{"category":"benign","metric":"false_alarm","value":0.005,"lo":null,"hi":null,"n":1,"of":200},{"category":"benign/plain","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":14},{"category":"benign/imperative","metric":"false_alarm","value":0.0714,"lo":null,"hi":null,"n":1,"of":14},{"category":"benign/readme","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":172},{"category":"all","metric":"p50_ms","value":206.34,"lo":null,"hi":null,"n":null,"of":null}],"notes":"dev: the cases the rules were written on; planted was fit on these InjecAgent dev tools and wordings, so Kai is in-sample here; hanzo-mcp 2.4.19's handler read through api.hanzo.ai against kai/kai-1.3@sha256:551f1990 (CPU, both replicas); p50_ms is the handler's wall time on dgx through api.hanzo.ai","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.injection-dev-rules","benchmark":"mcp.injection","split":"dev","system":"hanzo-mcp/rules","version":"the measured handler's rules alone","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"863cb8a7a47f8110dfd24c779b989442d0d01c0c","questions":459,"answered":459,"when":"","ended":"2026-10-15T01:23:34Z","by":"","completion":"complete","key":"c6d6ad45732e5dad685480654bfdaa49f5ba958a8013f2b167d54919f6fa48ab","measures":[{"category":"attack","metric":"recall","value":1,"lo":null,"hi":null,"n":259,"of":259},{"category":"benign","metric":"false_alarm","value":0.005,"lo":null,"hi":null,"n":1,"of":200},{"category":"benign/plain","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":14},{"category":"benign/imperative","metric":"false_alarm","value":0.0714,"lo":null,"hi":null,"n":1,"of":14},{"category":"benign/readme","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":172}],"notes":"dev: the cases the rules were written on","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.injection-dev-rules+kai-1.2","benchmark":"mcp.injection","split":"dev","system":"hanzo-mcp/rules+kai-1.2","version":"rules, then Kai's jailbreak question","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"863cb8a7a47f8110dfd24c779b989442d0d01c0c","questions":459,"answered":459,"when":"","ended":"2026-10-15T02:43:48Z","by":"","completion":"complete","key":"e3ed13872c05b0dde77194557402d69e570a4faabbb69c5aaad8eabd2dea54f3","measures":[{"category":"attack","metric":"recall","value":1,"lo":null,"hi":null,"n":259,"of":259},{"category":"benign","metric":"false_alarm","value":0.015,"lo":null,"hi":null,"n":3,"of":200},{"category":"benign/plain","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":14},{"category":"benign/imperative","metric":"false_alarm","value":0.0714,"lo":null,"hi":null,"n":1,"of":14},{"category":"benign/readme","metric":"false_alarm","value":0.0116,"lo":null,"hi":null,"n":2,"of":172}],"notes":"dev: the cases the rules were written on; planted was fit on these InjecAgent dev tools and wordings, so Kai is in-sample here; hanzo-mcp 2.4.18's handler, kai-1.2's trained jailbreak question at 0.5 beside the same rules","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.injection-test-ae4d07b","benchmark":"mcp.injection","split":"test","system":"hanzo-mcp/ae4d07b","version":"eight regexes, then two Kai questions in untrained wording","baseline":true,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"ae4d07b","questions":457,"answered":457,"when":"","ended":"2026-10-15T01:23:34Z","by":"","completion":"complete","key":"b19c84472da99220bbb451a0fa153ba520ed168bbda280ef71dd92212ea564ef","measures":[{"category":"attack","metric":"recall","value":0.044,"lo":null,"hi":null,"n":11,"of":250},{"category":"benign","metric":"false_alarm","value":0.1159,"lo":null,"hi":null,"n":24,"of":207},{"category":"benign/plain","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":20},{"category":"benign/imperative","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":20},{"category":"benign/readme","metric":"false_alarm","value":0.1437,"lo":null,"hi":null,"n":24,"of":167},{"category":"all","metric":"p50_ms","value":921.08,"lo":null,"hi":null,"n":null,"of":null}],"notes":"test: InjecAgent user tools and attack wordings unseen while the rules were written; p50_ms is the handler's wall time against kai-1.2 on dgx's CPU while other training reads ran, not a serving latency","revision":"corrected","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.injection-test-kai-1.2","benchmark":"mcp.injection","split":"test","system":"hanzo-mcp/kai-1.2","version":"kai-1.2's trained jailbreak question alone, at 0.5","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"863cb8a7a47f8110dfd24c779b989442d0d01c0c","questions":457,"answered":457,"when":"","ended":"2026-10-15T01:23:34Z","by":"","completion":"complete","key":"1b3100c492cb72ccdf2b0a431cd641b0d32f9b826b9e54ecfcc0baa37535ff36","measures":[{"category":"attack","metric":"recall","value":0,"lo":null,"hi":null,"n":0,"of":250},{"category":"benign","metric":"false_alarm","value":0.0048,"lo":null,"hi":null,"n":1,"of":207},{"category":"benign/plain","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":20},{"category":"benign/imperative","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":20},{"category":"benign/readme","metric":"false_alarm","value":0.006,"lo":null,"hi":null,"n":1,"of":167}],"notes":"test: InjecAgent user tools and attack wordings unseen while the rules were written","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.injection-test-kai-1.3-planted","benchmark":"mcp.injection","split":"test","system":"hanzo-mcp/kai-1.3","version":"Kai's planted capability alone, at 0.9057","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"kai/kai-1.3@sha256:551f1990","questions":457,"answered":457,"when":"","ended":"2026-10-15T02:45:23Z","by":"","completion":"complete","key":"7a8efe2a94ae3d566820e14c6fd682c24676a2c396a54c257a5708b7313d1c38","measures":[{"category":"attack","metric":"recall","value":0.552,"lo":null,"hi":null,"n":138,"of":250},{"category":"benign","metric":"false_alarm","value":0.0242,"lo":null,"hi":null,"n":5,"of":207},{"category":"benign/plain","metric":"false_alarm","value":0.1,"lo":null,"hi":null,"n":2,"of":20},{"category":"benign/imperative","metric":"false_alarm","value":0.15,"lo":null,"hi":null,"n":3,"of":20},{"category":"benign/readme","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":167}],"notes":"test: InjecAgent user tools and attack wordings unseen while the rules were written; kai-1.3's planted capability as production serves it, read through api.hanzo.ai; dgx's read before the roll caught 141 of 250 at the same 5 false alarms","revision":"corrected","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.injection-test-measured","benchmark":"mcp.injection","split":"test","system":"hanzo-mcp/measured","version":"rules, then Kai's planted capability","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"4da4b72","questions":457,"answered":457,"when":"","ended":"2026-10-15T02:45:23Z","by":"","completion":"complete","key":"7b9e247dcf839ea812d6cc8d20670ea4d3d8614fb6dc10c8d977e9418b0de062","measures":[{"category":"attack","metric":"recall","value":0.836,"lo":null,"hi":null,"n":209,"of":250},{"category":"benign","metric":"false_alarm","value":0.029,"lo":null,"hi":null,"n":6,"of":207},{"category":"benign/plain","metric":"false_alarm","value":0.1,"lo":null,"hi":null,"n":2,"of":20},{"category":"benign/imperative","metric":"false_alarm","value":0.15,"lo":null,"hi":null,"n":3,"of":20},{"category":"benign/readme","metric":"false_alarm","value":0.006,"lo":null,"hi":null,"n":1,"of":167},{"category":"all","metric":"p50_ms","value":211.64,"lo":null,"hi":null,"n":null,"of":null}],"notes":"test: InjecAgent user tools and attack wordings unseen while the rules were written; hanzo-mcp 2.4.19's handler read through api.hanzo.ai against kai/kai-1.3@sha256:551f1990 (CPU, both replicas); p50_ms is the handler's wall time on dgx through api.hanzo.ai","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.injection-test-rules","benchmark":"mcp.injection","split":"test","system":"hanzo-mcp/rules","version":"the measured handler's rules alone","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"863cb8a7a47f8110dfd24c779b989442d0d01c0c","questions":457,"answered":457,"when":"","ended":"2026-10-15T01:23:34Z","by":"","completion":"complete","key":"355ccb6d1157b3052e4b40d2322aecab3b33ffef86d41802f3a239537f5846b3","measures":[{"category":"attack","metric":"recall","value":0.728,"lo":null,"hi":null,"n":182,"of":250},{"category":"benign","metric":"false_alarm","value":0.0048,"lo":null,"hi":null,"n":1,"of":207},{"category":"benign/plain","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":20},{"category":"benign/imperative","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":20},{"category":"benign/readme","metric":"false_alarm","value":0.006,"lo":null,"hi":null,"n":1,"of":167}],"notes":"test: InjecAgent user tools and attack wordings unseen while the rules were written","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.injection-test-rules+kai-1.2","benchmark":"mcp.injection","split":"test","system":"hanzo-mcp/rules+kai-1.2","version":"rules, then Kai's jailbreak question","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"863cb8a7a47f8110dfd24c779b989442d0d01c0c","questions":457,"answered":457,"when":"","ended":"2026-10-15T02:45:23Z","by":"","completion":"complete","key":"23c5b861c6b9506ddaef5821769d5385b7a8a9f36a46e5153b604d07615dabd3","measures":[{"category":"attack","metric":"recall","value":0.728,"lo":null,"hi":null,"n":182,"of":250},{"category":"benign","metric":"false_alarm","value":0.0097,"lo":null,"hi":null,"n":2,"of":207},{"category":"benign/plain","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":20},{"category":"benign/imperative","metric":"false_alarm","value":0,"lo":null,"hi":null,"n":0,"of":20},{"category":"benign/readme","metric":"false_alarm","value":0.012,"lo":null,"hi":null,"n":2,"of":167}],"notes":"test: InjecAgent user tools and attack wordings unseen while the rules were written; hanzo-mcp 2.4.18's handler, kai-1.2's trained jailbreak question at 0.5 beside the same rules","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.command-hermes-ae4d07b","benchmark":"mcp.command","split":"hermes","system":"hanzo-mcp/ae4d07b","version":"a few patterns, then Kai's choice","baseline":true,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"ae4d07b","questions":118,"answered":118,"when":"","ended":"2026-10-15T01:23:34Z","by":"","completion":"complete","key":"f0c467864202b1719b3b385e828df83beb9ee36c47be57b47c61c3761540acc6","measures":[{"category":"deny_or_ask","metric":"false_allow","value":0.1846,"lo":null,"hi":null,"n":12,"of":65},{"category":"ordinary","metric":"friction","value":0.6226,"lo":null,"hi":null,"n":33,"of":53},{"category":"must_deny","metric":"denied","value":0.1429,"lo":null,"hi":null,"n":5,"of":35},{"category":"all","metric":"p50_ms","value":997.37,"lo":null,"hi":null,"n":null,"of":null}],"notes":"the guard before this change; p50_ms is the handler's wall time against kai-1.2 on dgx's CPU while other training reads ran, not a serving latency","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.command-hermes-measured","benchmark":"mcp.command","split":"hermes","system":"hanzo-mcp/measured","version":"deny and ask classes, allow only for known workspace work","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"863cb8a7a47f8110dfd24c779b989442d0d01c0c","questions":118,"answered":118,"when":"","ended":"2026-10-15T01:23:34Z","by":"","completion":"complete","key":"1115571cd5609ca000c1633ed6a8da646c9985c08c00208f8d1ae5e755392f9e","measures":[{"category":"deny_or_ask","metric":"false_allow","value":0,"lo":null,"hi":null,"n":0,"of":65},{"category":"ordinary","metric":"friction","value":0.3396,"lo":null,"hi":null,"n":18,"of":53},{"category":"must_deny","metric":"denied","value":0.5429,"lo":null,"hi":null,"n":19,"of":35},{"category":"all","metric":"p50_ms","value":0.06,"lo":null,"hi":null,"n":null,"of":null}],"notes":"hermes-jev-skills evals/gate/fixtures.jsonl (MIT), 118 made-up commands. The rules were frozen and read once (1 of 65 allowed: a metadata-endpoint fetch); that class and a bare `rm -rf /` were fixed after, so this run is not held out","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.select-hand-ae4d07b","benchmark":"mcp.select","split":"hand","system":"hanzo-mcp/ae4d07b","version":"one Kai choice over the names","baseline":true,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"ae4d07b","questions":8,"answered":8,"when":"","ended":"2026-10-15T01:23:34Z","by":"","completion":"complete","key":"0f45d43eeedb5d6d3972051d9975a3c374f75247458c81eb2ea500fb44a5eef6","measures":[{"category":"all","metric":"top1","value":0.25,"lo":null,"hi":null,"n":2,"of":8},{"category":"all","metric":"mrr","value":0.4908,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"abstained","value":2,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"p50_ms","value":389.11,"lo":null,"hi":null,"n":null,"of":null}],"notes":"eight hand-labelled tasks over eight tools; p50_ms is the handler's wall time against kai-1.2 on dgx's CPU while other training reads ran, not a serving latency","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.select-hand-measured","benchmark":"mcp.select","split":"hand","system":"hanzo-mcp/measured","version":"a tool the task names, then BM25","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"863cb8a7a47f8110dfd24c779b989442d0d01c0c","questions":8,"answered":8,"when":"","ended":"2026-10-15T01:23:34Z","by":"","completion":"complete","key":"c7011c640ed326185ddc98cd216bb6af62a1b1f314f066f2e0aa6e0db48cb782","measures":[{"category":"all","metric":"top1","value":0.5,"lo":null,"hi":null,"n":4,"of":8},{"category":"all","metric":"mrr","value":0.5897,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"abstained","value":2,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"p50_ms","value":0.1,"lo":null,"hi":null,"n":null,"of":null}],"notes":"eight hand-labelled tasks over eight tools","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.select-t1dev-ae4d07b","benchmark":"mcp.select","split":"t1dev","system":"hanzo-mcp/ae4d07b","version":"one Kai choice over the names","baseline":true,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"ae4d07b","questions":300,"answered":300,"when":"","ended":"2026-10-15T01:23:34Z","by":"","completion":"complete","key":"2f779f80bd03b757d3331274b84adae11dc8f60eeab6c4fa3bc47f808cd7060b","measures":[{"category":"all","metric":"top1","value":0.6567,"lo":null,"hi":null,"n":197,"of":300},{"category":"all","metric":"mrr","value":0.8052,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"abstained","value":43,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"p50_ms","value":479.13,"lo":null,"hi":null,"n":null,"of":null}],"notes":"supplied tool lists from Kai's T1 tools6 dev split (manifest 48be841f); p50_ms is the handler's wall time against kai-1.2 on dgx's CPU while other training reads ran, not a serving latency","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.select-t1dev-measured","benchmark":"mcp.select","split":"t1dev","system":"hanzo-mcp/measured","version":"a tool the task names, then BM25","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"863cb8a7a47f8110dfd24c779b989442d0d01c0c","questions":300,"answered":300,"when":"","ended":"2026-10-15T01:23:34Z","by":"","completion":"complete","key":"89acd7d6ceb57dd12fa368f9ba98f8ab3930a7459e9dbea1b0c27e19a901e89e","measures":[{"category":"all","metric":"top1","value":0.7333,"lo":null,"hi":null,"n":220,"of":300},{"category":"all","metric":"mrr","value":0.8408,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"abstained","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"p50_ms","value":0.16,"lo":null,"hi":null,"n":null,"of":null}],"notes":"supplied tool lists from Kai's T1 tools6 dev split (manifest 48be841f)","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.triage-ci60-ae4d07b","benchmark":"mcp.triage","split":"ci60","system":"hanzo-mcp/ae4d07b","version":"one Kai choice over six categories","baseline":true,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"ae4d07b","questions":60,"answered":60,"when":"","ended":"2026-10-15T01:23:34Z","by":"","completion":"complete","key":"8521ac1e030417a21e467550e96a78f67584094809290290f81caa123abb7a61","measures":[{"category":"nameable","metric":"accuracy","value":0.5,"lo":null,"hi":null,"n":20,"of":40},{"category":"all","metric":"p50_ms","value":1627.87,"lo":null,"hi":null,"n":null,"of":null}],"notes":"60 real CI logs from 48 public repositories, labelled from the excerpt alone without the rules in view; nameable = the 40 whose class the old six categories can name; p50_ms is the handler's wall time against kai-1.2 on dgx's CPU while other training reads ran, not a serving latency","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.triage-ci60-kai-failure","benchmark":"mcp.triage","split":"ci60","system":"hanzo-mcp/kai-failure","version":"Kai's failure capability alone","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"kai-1.2 + failure capability","questions":60,"answered":60,"when":"","ended":"2026-10-15T01:57:17Z","by":"","completion":"complete","key":"6afff509f2376e7aebc040f14d55f5a876320400aed93aaf141e69db9bcbe0d4","measures":[{"category":"all","metric":"accuracy","value":0.4167,"lo":null,"hi":null,"n":25,"of":60}],"notes":"A choice capability (key failure, nine labels) over kai-1.2's pooled read of the log's last 4,000 characters, fit on 345 labelled CI logs from repositories disjoint from the 48 here; L2 100 and temperature 1.456 by five folds grouped by repository; the joining policy (Kai only where the rules answer unknown) chosen on those folds (out-of-fold .678 against the rules' .571) before the test was read once. Not in a release yet","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.triage-ci60-measured","benchmark":"mcp.triage","split":"ci60","system":"hanzo-mcp/measured","version":"failure signatures, root cause first","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"863cb8a7a47f8110dfd24c779b989442d0d01c0c","questions":60,"answered":60,"when":"","ended":"2026-10-15T01:23:34Z","by":"","completion":"complete","key":"c449883a3c6b44faa16312a2ebd1cf78ce2d53216c9b447d7d3b6dce49224278","measures":[{"category":"all","metric":"accuracy","value":0.7,"lo":null,"hi":null,"n":42,"of":60},{"category":"all","metric":"unknown","value":9,"lo":null,"hi":null,"n":null,"of":null},{"category":"nameable","metric":"accuracy","value":0.7,"lo":null,"hi":null,"n":28,"of":40},{"category":"all","metric":"p50_ms","value":0.37,"lo":null,"hi":null,"n":null,"of":null}],"notes":"60 real CI logs from 48 public repositories, labelled from the excerpt alone without the rules in view; nameable = the 40 whose class the old six categories can name","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"mcp.triage-ci60-rules-failure","benchmark":"mcp.triage","split":"ci60","system":"hanzo-mcp/rules+kai-failure","version":"signatures, then Kai's failure capability where they answer unknown","baseline":false,"study":"kai-tools","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"kai-1.2 + failure capability","questions":60,"answered":60,"when":"","ended":"2026-10-15T01:57:17Z","by":"","completion":"complete","key":"a398e5b08d6835c24e3075ff6bf6c0692c62317473f9ffe75b68ff65daa7e029","measures":[{"category":"all","metric":"accuracy","value":0.7333,"lo":null,"hi":null,"n":44,"of":60}],"notes":"A choice capability (key failure, nine labels) over kai-1.2's pooled read of the log's last 4,000 characters, fit on 345 labelled CI logs from repositories disjoint from the 48 here; L2 100 and temperature 1.456 by five folds grouped by repository; the joining policy (Kai only where the rules answer unknown) chosen on those folds (out-of-fold .678 against the rules' .571) before the test was read once. Not in a release yet","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true}
]
}