hanzo.ai/lib/data/benchmarks-coding.json

{
"data": [
{"project":"default","id":"coding.repair-held-devfast-claude-code","benchmark":"coding.repair","split":"held","system":"claude-code","version":"sonnet-5.5 default effort","baseline":true,"study":"devfast","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"34ae9180a85a499c46d309ce4ce00fc1d96b50f6","questions":44,"answered":44,"when":"","ended":"2026-10-14T23:08:50Z","by":"","completion":"complete","key":"4f39092d7dd7a4b91e8685fa17d84fa14d1d2bcacfc82cef9094a58a3c63bdcf","measures":[{"category":"all","metric":"solved","value":44,"lo":null,"hi":null,"n":44,"of":44},{"category":"all","metric":"median_s","value":9.602141905997996,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"model_calls","value":3.7954545454545454,"lo":null,"hi":null,"n":null,"of":null}],"notes":"native Claude Code on the owner's login, one attempt a task","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"coding.repair-held-devfast-hanzo-dev-dc","benchmark":"coding.repair","split":"held","system":"hanzo-dev","version":"dev flow fix, Kai + enso-flash","baseline":false,"study":"devfast","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"34ae9180a85a499c46d309ce4ce00fc1d96b50f6","questions":44,"answered":44,"when":"","ended":"2026-10-14T23:18:05Z","by":"","completion":"complete","key":"94d85fd1293442ca2f500ef4a47652469148d4ca19729581465fa4df3d7cbd4e","measures":[{"category":"all","metric":"solved","value":40,"lo":null,"hi":null,"n":44,"of":44},{"category":"all","metric":"median_s","value":2.2457525420031743,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"model_calls","value":1.0227272727272727,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"kai_decisions","value":10,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"kai_applied","value":0,"lo":null,"hi":null,"n":null,"of":null},{"category":"vs-claude-code","metric":"speedup","value":3.834,"lo":2.763,"hi":5.251,"n":40,"of":null},{"category":"vs-claude-code/search","metric":"speedup","value":8.22,"lo":6.863,"hi":9.82,"n":24,"of":null},{"category":"vs-claude-code/generated","metric":"speedup","value":1.916,"lo":1.835,"hi":1.992,"n":5,"of":null},{"category":"vs-claude-code/recovered","metric":"speedup","value":0.995,"lo":0.765,"hi":1.3,"n":11,"of":null},{"category":"vs-claude-code/model","metric":"speedup","value":1.221,"lo":0.961,"hi":1.535,"n":16,"of":null}],"notes":"arm DC; settled by search 24, generated 6, recovered 11, unsolved 3; the search tries single-token edits of the kinds the faults are drawn from, against the visible test only","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"coding.repair-held-devfix-claude-code","benchmark":"coding.repair","split":"held","system":"claude-code","version":"sonnet-5.5 default effort","baseline":true,"study":"devfix","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"8e966bee4833385abf80603891b760c2e5da1d42","questions":44,"answered":44,"when":"","ended":"2026-10-14T23:08:50Z","by":"","completion":"complete","key":"4f39092d7dd7a4b91e8685fa17d84fa14d1d2bcacfc82cef9094a58a3c63bdcf","measures":[{"category":"all","metric":"solved","value":44,"lo":null,"hi":null,"n":44,"of":44},{"category":"all","metric":"median_s","value":9.523428922999301,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"model_calls","value":3.7954545454545454,"lo":null,"hi":null,"n":null,"of":null}],"notes":"native Claude Code on the owner's login, one attempt a task","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true},
{"project":"default","id":"coding.repair-held-devfix-hanzo-dev-dc","benchmark":"coding.repair","split":"held","system":"hanzo-dev","version":"dev flow fix, Kai + enso-flash","baseline":false,"study":"devfix","embedder":"","reader":"","k":null,"temperature":null,"max_tokens":null,"prompt":"","prompt_digest":"","dataset_digest":"","store_digest":"","facts_digest":"","commit":"8e966bee4833385abf80603891b760c2e5da1d42","questions":44,"answered":44,"when":"","ended":"2026-10-15T01:03:55Z","by":"","completion":"complete","key":"94d85fd1293442ca2f500ef4a47652469148d4ca19729581465fa4df3d7cbd4e","measures":[{"category":"all","metric":"solved","value":42,"lo":null,"hi":null,"n":44,"of":44},{"category":"all","metric":"median_s","value":2.293089587998111,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"model_calls","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"kai_decisions","value":9,"lo":null,"hi":null,"n":null,"of":null},{"category":"all","metric":"kai_applied","value":1,"lo":null,"hi":null,"n":null,"of":null},{"category":"vs-claude-code","metric":"speedup","value":3.02,"lo":2.063,"hi":4.361,"n":42,"of":null},{"category":"vs-claude-code/search","metric":"speedup","value":8.22,"lo":6.863,"hi":9.82,"n":24,"of":null},{"category":"vs-claude-code/search","metric":"model_calls","value":0,"lo":null,"hi":null,"n":24,"of":null},{"category":"vs-claude-code/search","metric":"model_calls_claude_code","value":3.42,"lo":null,"hi":null,"n":24,"of":null},{"category":"vs-claude-code/generated","metric":"speedup","value":1.313,"lo":1.14,"hi":1.48,"n":4,"of":null},{"category":"vs-claude-code/generated","metric":"model_calls","value":1,"lo":null,"hi":null,"n":4,"of":null},{"category":"vs-claude-code/generated","metric":"model_calls_claude_code","value":3,"lo":null,"hi":null,"n":4,"of":null},{"category":"vs-claude-code/recovered","metric":"speedup","value":0.837,"lo":0.726,"hi":0.988,"n":5,"of":null},{"category":"vs-claude-code/recovered","metric":"model_calls","value":2,"lo":null,"hi":null,"n":5,"of":null},{"category":"vs-claude-code/recovered","metric":"model_calls_claude_code","value":4,"lo":null,"hi":null,"n":5,"of":null},{"category":"vs-claude-code/escalated","metric":"speedup","value":0.618,"lo":0.478,"hi":0.78,"n":9,"of":null},{"category":"vs-claude-code/escalated","metric":"model_calls","value":2.89,"lo":null,"hi":null,"n":9,"of":null},{"category":"vs-claude-code/escalated","metric":"model_calls_claude_code","value":4.78,"lo":null,"hi":null,"n":9,"of":null},{"category":"vs-claude-code/model","metric":"speedup","value":0.795,"lo":0.649,"hi":0.953,"n":18,"of":null},{"category":"vs-claude-code/model","metric":"model_calls","value":2.22,"lo":null,"hi":null,"n":18,"of":null},{"category":"vs-claude-code/model","metric":"model_calls_claude_code","value":4.17,"lo":null,"hi":null,"n":18,"of":null}],"notes":"arm DC, passes devfast, devfix; settled by escalated 9, generated 5, recovered 5, search 24, unsolved 1; the search tries single-token edits of the kinds the faults are drawn from, against the visible test only","revision":"original","canonical":true,"visibility":"public","trainable":false,"publishable":true}
]
}