mirror of
https://github.com/hanzoai/site.git
synced 2026-11-20 21:21:13 +00:00
2 lines
80 KiB
JSON
2 lines
80 KiB
JSON
{"source":"https://api.hanzo.ai/v1/research/runs?org=hanzo&benchmark={coding.repair,decision.latency,enso.gpqa_diamond}&view=summary&limit=200","fetched_at":"2026-11-11T00:58:33.098Z","repair":[{"project":"default","id":"coding.repair-held-devfast-claude-code","benchmark":"coding.repair","split":"held","system":"claude-code","version":"sonnet-5.5 default effort","baseline":true,"study":"devfast","commit":"34ae9180a85a499c46d309ce4ce00fc1d96b50f6","questions":44,"answered":44,"ended":"2026-10-14T23:08:50Z","completion":"complete","measures":[{"category":"all","metric":"solved","value":44,"n":44,"of":44},{"category":"all","metric":"median_s","value":9.602141905997996},{"category":"all","metric":"model_calls","value":3.7954545454545454}],"notes":"native Claude Code on the owner's login, one attempt a task","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"coding.repair-held-devfast-hanzo-dev-dc","benchmark":"coding.repair","split":"held","system":"hanzo-dev","version":"dev flow fix, Kai + enso-flash","baseline":false,"study":"devfast","commit":"34ae9180a85a499c46d309ce4ce00fc1d96b50f6","questions":44,"answered":44,"ended":"2026-10-14T23:18:05Z","completion":"complete","measures":[{"category":"all","metric":"solved","value":40,"n":44,"of":44},{"category":"all","metric":"median_s","value":2.2457525420031743},{"category":"all","metric":"model_calls","value":1.0227272727272727},{"category":"all","metric":"kai_decisions","value":10},{"category":"all","metric":"kai_applied","value":0},{"category":"vs-claude-code","metric":"speedup","value":3.834,"lo":2.763,"hi":5.251,"n":40},{"category":"vs-claude-code/search","metric":"speedup","value":8.22,"lo":6.863,"hi":9.82,"n":24},{"category":"vs-claude-code/generated","metric":"speedup","value":1.916,"lo":1.835,"hi":1.992,"n":5},{"category":"vs-claude-code/recovered","metric":"speedup","value":0.995,"lo":0.765,"hi":1.3,"n":11},{"category":"vs-claude-code/model","metric":"speedup","value":1.221,"lo":0.961,"hi":1.535,"n":16}],"notes":"arm DC; settled by search 24, generated 6, recovered 11, unsolved 3; the search tries single-token edits of the kinds the faults are drawn from, against the visible test only","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"coding.repair-held-devfix-claude-code","benchmark":"coding.repair","split":"held","system":"claude-code","version":"sonnet-5.5 default effort","baseline":true,"study":"devfix","commit":"8e966bee4833385abf80603891b760c2e5da1d42","questions":44,"answered":44,"ended":"2026-10-14T23:08:50Z","completion":"complete","measures":[{"category":"all","metric":"solved","value":44,"n":44,"of":44},{"category":"all","metric":"median_s","value":9.523428922999301},{"category":"all","metric":"model_calls","value":3.7954545454545454}],"notes":"native Claude Code on the owner's login, one attempt a task","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"coding.repair-held-devfix-hanzo-dev-dc","benchmark":"coding.repair","split":"held","system":"hanzo-dev","version":"dev flow fix, Kai + enso-flash","baseline":false,"study":"devfix","commit":"8e966bee4833385abf80603891b760c2e5da1d42","questions":44,"answered":44,"ended":"2026-10-15T01:03:55Z","completion":"complete","measures":[{"category":"all","metric":"solved","value":42,"n":44,"of":44},{"category":"all","metric":"median_s","value":2.293089587998111},{"category":"all","metric":"model_calls","value":1},{"category":"all","metric":"kai_decisions","value":9},{"category":"all","metric":"kai_applied","value":1},{"category":"vs-claude-code","metric":"speedup","value":3.02,"lo":2.063,"hi":4.361,"n":42},{"category":"vs-claude-code/search","metric":"speedup","value":8.22,"lo":6.863,"hi":9.82,"n":24},{"category":"vs-claude-code/search","metric":"model_calls","value":0,"n":24},{"category":"vs-claude-code/search","metric":"model_calls_claude_code","value":3.42,"n":24},{"category":"vs-claude-code/generated","metric":"speedup","value":1.313,"lo":1.14,"hi":1.48,"n":4},{"category":"vs-claude-code/generated","metric":"model_calls","value":1,"n":4},{"category":"vs-claude-code/generated","metric":"model_calls_claude_code","value":3,"n":4},{"category":"vs-claude-code/recovered","metric":"speedup","value":0.837,"lo":0.726,"hi":0.988,"n":5},{"category":"vs-claude-code/recovered","metric":"model_calls","value":2,"n":5},{"category":"vs-claude-code/recovered","metric":"model_calls_claude_code","value":4,"n":5},{"category":"vs-claude-code/escalated","metric":"speedup","value":0.618,"lo":0.478,"hi":0.78,"n":9},{"category":"vs-claude-code/escalated","metric":"model_calls","value":2.89,"n":9},{"category":"vs-claude-code/escalated","metric":"model_calls_claude_code","value":4.78,"n":9},{"category":"vs-claude-code/model","metric":"speedup","value":0.795,"lo":0.649,"hi":0.953,"n":18},{"category":"vs-claude-code/model","metric":"model_calls","value":2.22,"n":18},{"category":"vs-claude-code/model","metric":"model_calls_claude_code","value":4.17,"n":18}],"notes":"arm DC, passes devfast, devfix; settled by escalated 9, generated 5, recovered 5, search 24, unsolved 1; the search tries single-token edits of the kinds the faults are drawn from, against the visible test only","revision":"original","canonical":true,"visibility":"public"}],"latency":[{"project":"default","id":"decision.latency-loopback-dgx.cpu-kai","benchmark":"decision.latency","split":"loopback","system":"kai","version":"kai-1.3-guard@417b4949bbff2c6c6fb18755e23bd4abb11712d9f96c6159b288c6f06b350cc4","baseline":false,"study":"decision-suite","commit":"","questions":null,"answered":null,"ended":"2026-10-15T18:33:27Z","completion":"unknown","measures":[{"category":"served/loopback/cpu/same/wall","metric":"p50ms","value":24.148405,"of":60},{"category":"served/loopback/cpu/same/server","metric":"p50ms","value":22.343029,"of":60},{"category":"served/loopback/cpu/distinct/wall","metric":"p50ms","value":69.674438,"of":60},{"category":"served/loopback/cpu/distinct/server","metric":"p50ms","value":67.75554,"of":60}],"notes":"Measured by bench/latency_audit.py (results/latency/audit.json): 60 POST /v1/decisions to http://127.0.0.1:18641 on dgx (aarch64), after 5 discarded, one at a time; the body {\"model\": \"kai\", \"state\": {\"message\": \"Our checkout page returns HTTP 500 for every customer since the 14:02 deploy; orders have stopped.\"}, \"questions\": {\"critical\": {\"type\": \"noul\", \"instructions\": \"Is this a critical incident that needs someone paged now?\"}}}. Folder: ~/.cache/decision-a2/release/kai-1.3-guard (SHA256SUMS 417b4949): kai-1.2 plus guard-1's head on the jailbreak key; the critical noul answers by kai-1.2's reader (3b85b6bd), as production kai-1.3 answers it. Threads: RAYON_NUM_THREADS unset: hanzo-ml's pool sizes to dgx's 20 physical cores. Shapes: same, the body above every time: its sequences are cached after the first; distinct, the body with ' Ticket NNNN.' appended to the state: a fresh encode each. wall: the client's time for the call; server: the answer's latency_ms. Load average 1.2, 6.0, 7.5 at the start.","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"decision.latency-network-n1-jev-1.13","benchmark":"decision.latency","split":"network","system":"jev","version":"typesafe/jev-1.13-20260917","baseline":true,"study":"decision-suite","commit":"f2dd6ea","questions":null,"answered":null,"ended":"2026-09-27T10:47:01Z","completion":"unknown","measures":[{"category":"served/network/n1","metric":"p50ms","value":133.3509,"of":10}],"notes":"Jev 1.13 through OpenRouter's decisions endpoint, one question a call, 10 rounds over the capability pool's 400 states (bench speed, benchmarks/decision/capability/results/questions.json, run 2026-09-27 10:47:01 UTC). Wall time at the client, network included. No p99 recorded.","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"decision.latency-network-n1-kai-1.3","benchmark":"decision.latency","split":"network","system":"kai","version":"kai-1.3@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"decision-suite","commit":"32c6432","questions":null,"answered":null,"ended":"2026-10-16T20:06:44Z","completion":"unknown","measures":[{"category":"served/network/n1","metric":"p50ms","value":240.5709,"of":100},{"category":"served/server/n1","metric":"p50ms","value":70.6327,"of":100}],"notes":"Kai on api.hanzo.ai (POST /v1/decisions, model kai; the answers report kai-1.3, weights 0834a74f, on CPU), timed from one client on the US west coast, about 85 ms from api.hanzo.ai (AWS us-east-1). The calls are `bench speed --who jev --sizes 1 --batch 0 --rounds 100`'s own (hanzo-inc/kai, the bench built 2026-10-07, 32c6432: the capability pool, one typed-decision state and one question a call, states and questions taken in turn), captured as the bench sends them (sha256:9111e582b9b5d1bc97dfbc2ba7028661723a44eeb16f995feb3a82f0c3149158), sent over one kept-alive HTTPS connection, the first discarded and 100 timed; wall time at the client from the request to its answer parsed, network included; nearest-rank percentiles. This is the second of two passes of the same calls, 2026-10-16 20:05:49Z to 20:06:44Z, alternating call by call with Jev through api.hanzo.ai. The first pass, 20:04:43Z to 20:05:29Z, measured p50 277.4 ms, p95 425.6 ms (the answers' own latency_ms p50 146.6 ms). The server keeps the options it has encoded, so a call's time falls as the calls repeat: in this pass the answers' own latency_ms p50 fell from 126.9 ms over the first 20 calls to 26.6 over the last 20; served/server/n1 is their p50 over all 100. Not paired with Jev's 2026-09-27 run: other day, other rounds, other inputs digest.","revision":"corrected","canonical":true,"visibility":"public"},{"project":"default","id":"decision.latency-served-cpu-kai-1.2","benchmark":"decision.latency","split":"served","system":"kai","version":"kai-1.2@e45dd919623b83f01595b6dffbcd63315adeab70bfb7dd5d6a942380b320d575","baseline":false,"study":"decision-suite","commit":"f144cc1","questions":null,"answered":null,"ended":"2026-10-09T02:57:04Z","completion":"unknown","measures":[{"category":"served/http/cpu-f32-q8_0/n1","metric":"p50ms","value":50.68598,"of":50}],"notes":"kai serve --device cpu --reach 32768 over the kai-1.2 folder on dgx's arm64 CPU, 4 threads: one yes/no question, the server's own time, 50 after 5 (capability promote, results/bernoulli/reader12/Z2/served.json). No p99 recorded.","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"decision.latency-served-load-dgx.cpu-kai-1.3","benchmark":"decision.latency","split":"served-load","system":"kai","version":"kai-1.3@551f19901c8c45df680077e36e563ea59974e399cbbf1175b46e15517a41a61a","baseline":false,"study":"decision-suite","commit":"3cd393e","questions":null,"answered":null,"ended":"2026-10-15T05:58:17Z","completion":"unknown","measures":[{"category":"served/http/cpu/jailbreak/server","metric":"p50ms","value":164.754492,"of":99},{"category":"served/http/cpu/jailbreak/wall","metric":"p50ms","value":168.38436,"of":99},{"category":"served/http/cpu/critical/server","metric":"p50ms","value":233.205866,"of":99},{"category":"served/http/cpu/critical/wall","metric":"p50ms","value":236.534837,"of":99}],"notes":"From results/guard2/latency.b.json (train/guard/latency2.py, guard-2's regression, results/latency/audit.json): 99 sequential questions a key through kai serve --device cpu on dgx (threads as the regression's serve.sh set them; the file does not record them), load 10.3 to 10.4; key jailbreak answered by kai-1.2's reader, key critical by kai-1.2's reader. Kai's own times, not Jev's; contention, not a speed result.","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"decision.latency-served-load-dgx.cpu-kai-1.3-guard-2","benchmark":"decision.latency","split":"served-load","system":"kai","version":"kai-1.3@551f19901c8c45df680077e36e563ea59974e399cbbf1175b46e15517a41a61a+guard-2","baseline":false,"study":"decision-suite","commit":"3cd393e","questions":null,"answered":null,"ended":"2026-10-15T05:58:17Z","completion":"unknown","measures":[{"category":"served/http/cpu/jailbreak/server","metric":"p50ms","value":158.971362,"of":99},{"category":"served/http/cpu/jailbreak/wall","metric":"p50ms","value":161.088338,"of":99},{"category":"served/http/cpu/critical/server","metric":"p50ms","value":141.533488,"of":99},{"category":"served/http/cpu/critical/wall","metric":"p50ms","value":144.397975,"of":99}],"notes":"From results/guard2/latency.d.json (train/guard/latency2.py, guard-2's regression, results/latency/audit.json): 99 sequential questions a key through kai serve --device cpu on dgx (threads as the regression's serve.sh set them; the file does not record them), load 10.9 to 10.8; key jailbreak routed to guard-2's reader, key critical by kai-1.2's reader. Kai's own times, not Jev's; contention, not a speed result.","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"decision.latency-turns-claude-code","benchmark":"decision.latency","split":"turns","system":"claude-code","version":"anthropic/claude-sonnet-5.5","baseline":false,"study":"decision-suite","commit":"","questions":null,"answered":null,"ended":"2026-10-12T15:28:11Z","completion":"unknown","measures":[{"category":"turn/latency_ms","metric":"p50ms","value":2307,"of":650},{"category":"attempt/client_seconds_per_call","metric":"p50ms","value":3775.252137,"of":160}],"notes":"From benchmarks/coding/attempts.jsonl (results/latency/audit.json): 160 native Claude Code attempts (bench/code arm N, anthropic/claude-sonnet-5.5, no Hanzo gateway), 2026-10-12T07:12:08Z to 2026-10-12T15:28:11Z. turn: field latency_ms, each model call as the client saw it (bench/code/run.py latencies(): from the event before a reply to the reply), pooled over 650 calls. attempt: client_seconds / calls, one value an attempt, tool time included. Milliseconds.","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"decision.latency-window2-dgx.cpu.f32.q8.t4-kai","benchmark":"decision.latency","split":"window2","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"decision-suite","commit":"c5bf93f","questions":null,"answered":null,"ended":"2026-09-28T16:44:25Z","completion":"unknown","measures":[{"category":"cold/tensor/cpu-f32-q8_0/n1","metric":"p50ms","value":238.082467,"of":10},{"category":"cached/tensor/cpu-f32-q8_0/n1","metric":"p50ms","value":19.567729,"of":10},{"category":"cached/plan-host/cpu-q8_0/n1","metric":"p50ms","value":6.580673,"of":10},{"category":"cached/plan-device/cpu-f32/n1","metric":"p50ms","value":12.968975,"of":10},{"category":"cold/tensor/cpu-f32-q8_0/n10","metric":"p50ms","value":1493.395464,"of":10},{"category":"cached/tensor/cpu-f32-q8_0/n10","metric":"p50ms","value":282.970475,"of":10},{"category":"cached/plan-host/cpu-q8_0/n10","metric":"p50ms","value":200.565545,"of":10},{"category":"cached/plan-device/cpu-f32/n10","metric":"p50ms","value":553.66194,"of":10},{"category":"cold/tensor/cpu-f32-q8_0/n100","metric":"p50ms","value":8153.691463,"of":10},{"category":"cached/tensor/cpu-f32-q8_0/n100","metric":"p50ms","value":2354.882434,"of":10},{"category":"cached/plan-host/cpu-q8_0/n100","metric":"p50ms","value":940.949684,"of":10},{"category":"cached/plan-device/cpu-f32/n100","metric":"p50ms","value":4498.514596,"of":10}],"notes":"bench decide (results/latency/window2/dgx/decide.json, config dgx.cpu.f32.q8.t4); host dgx (GB10, 20 arm64 cores, CUDA sm121; a shared desktop); threads 4; rounds 10 a row after one discarded; load 10.6, 9.4, 10.2; checkpoint a7 (weights 0834a74f, kai-1's base). The release serves on the CPU; GPU rows are not production.","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"decision.latency-window2-dgx.cpu.f32.q8.t8-kai","benchmark":"decision.latency","split":"window2","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"decision-suite","commit":"c5bf93f","questions":null,"answered":null,"ended":"2026-09-28T16:44:25Z","completion":"unknown","measures":[{"category":"cold/tensor/cpu-f32-q8_0/n1","metric":"p50ms","value":221.43052,"of":10},{"category":"cached/tensor/cpu-f32-q8_0/n1","metric":"p50ms","value":15.101596,"of":10},{"category":"cached/plan-host/cpu-q8_0/n1","metric":"p50ms","value":5.553008,"of":10},{"category":"cached/plan-device/cpu-f32/n1","metric":"p50ms","value":8.642862,"of":10},{"category":"cold/tensor/cpu-f32-q8_0/n10","metric":"p50ms","value":1290.120901,"of":10},{"category":"cached/tensor/cpu-f32-q8_0/n10","metric":"p50ms","value":303.776741,"of":10},{"category":"cached/plan-host/cpu-q8_0/n10","metric":"p50ms","value":112.397999,"of":10},{"category":"cached/plan-device/cpu-f32/n10","metric":"p50ms","value":410.8223,"of":10},{"category":"cold/tensor/cpu-f32-q8_0/n100","metric":"p50ms","value":6051.118665,"of":10},{"category":"cached/tensor/cpu-f32-q8_0/n100","metric":"p50ms","value":3152.876922,"of":10},{"category":"cached/plan-host/cpu-q8_0/n100","metric":"p50ms","value":703.561045,"of":10},{"category":"cached/plan-device/cpu-f32/n100","metric":"p50ms","value":8459.669687,"of":10}],"notes":"bench decide (results/latency/window2/dgx/decide.json, config dgx.cpu.f32.q8.t8); host dgx (GB10, 20 arm64 cores, CUDA sm121; a shared desktop); threads 8; rounds 10 a row after one discarded; load 11.7, 11.7, 11.2; checkpoint a7 (weights 0834a74f, kai-1's base). The release serves on the CPU; GPU rows are not production.","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"decision.latency-window2-dgx.cuda.f32.q8.tall-kai","benchmark":"decision.latency","split":"window2","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"decision-suite","commit":"c5bf93f","questions":null,"answered":null,"ended":"2026-09-28T16:44:25Z","completion":"unknown","measures":[{"category":"cold/tensor/cuda-f32/n1","metric":"p50ms","value":23.54333,"of":50},{"category":"cached/tensor/cuda-f32/n1","metric":"p50ms","value":1.919841,"of":50},{"category":"cached/plan-host/cpu-q8_0/n1","metric":"p50ms","value":5.529489,"of":50},{"category":"cached/plan-device/cuda-f32/n1","metric":"p50ms","value":1.089481,"of":50},{"category":"cached/plan-device/cuda-bf16/n1","metric":"p50ms","value":0.269107,"of":50},{"category":"cold/tensor/cuda-f32/n10","metric":"p50ms","value":81.845258,"of":50},{"category":"cached/tensor/cuda-f32/n10","metric":"p50ms","value":12.676913,"of":50},{"category":"cached/plan-host/cpu-q8_0/n10","metric":"p50ms","value":92.362439,"of":50},{"category":"cached/plan-device/cuda-f32/n10","metric":"p50ms","value":12.285997,"of":50},{"category":"cached/plan-device/cuda-bf16/n10","metric":"p50ms","value":4.845355,"of":50},{"category":"cold/tensor/cuda-f32/n100","metric":"p50ms","value":414.557843,"of":50},{"category":"cached/tensor/cuda-f32/n100","metric":"p50ms","value":173.902512,"of":50},{"category":"cached/plan-host/cpu-q8_0/n100","metric":"p50ms","value":812.505222,"of":50},{"category":"cached/plan-device/cuda-f32/n100","metric":"p50ms","value":302.984241,"of":50},{"category":"cached/plan-device/cuda-bf16/n100","metric":"p50ms","value":131.21114,"of":50}],"notes":"bench decide (results/latency/window2/dgx/decide.json, config dgx.cuda.f32.q8.tall); host dgx (GB10, 20 arm64 cores, CUDA sm121; a shared desktop); threads all; rounds 50 a row after one discarded; load 6.7, 11.0, 10.9; checkpoint a7 (weights 0834a74f, kai-1's base). The release serves on the CPU; GPU rows are not production.","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"decision.latency-window2-dgx.cuda.n1.transport-kai","benchmark":"decision.latency","split":"window2","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"decision-suite","commit":"c5bf93f","questions":null,"answered":null,"ended":"2026-09-28T16:32:35Z","completion":"unknown","measures":[{"category":"cached/handle/cuda/n1","metric":"p50ms","value":0.427956,"of":300},{"category":"cached/runtime/cuda/n1","metric":"p50ms","value":0.437236,"of":300},{"category":"cached/uds/cuda/n1","metric":"p50ms","value":0.537637,"of":300},{"category":"cached/http/cuda/n1","metric":"p50ms","value":0.781367,"of":300}],"notes":"bench transport (results/latency/window2/dgx/transport.json); host dgx (GB10, 20 arm64 cores, CUDA sm121; a shared desktop); 300 rounds interleaved; load 5.6, 10.6, 10.8; one cached decision, the state observed once, on the plan the race chose. The Unix socket was retired 2026-10-09.","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"decision.latency-window2-evo.cpu.f32.q8.t4-kai","benchmark":"decision.latency","split":"window2","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"decision-suite","commit":"c5bf93f","questions":null,"answered":null,"ended":"2026-09-28T16:37:40Z","completion":"unknown","measures":[{"category":"cold/tensor/cpu-f32-q8_0/n1","metric":"p50ms","value":84.805857,"of":10},{"category":"cached/tensor/cpu-f32-q8_0/n1","metric":"p50ms","value":9.170072,"of":10},{"category":"cached/plan-host/cpu-q8_0/n1","metric":"p50ms","value":2.820162,"of":10},{"category":"cached/plan-device/cpu-f32/n1","metric":"p50ms","value":5.586366,"of":10},{"category":"cold/tensor/cpu-f32-q8_0/n10","metric":"p50ms","value":599.438052,"of":10},{"category":"cached/tensor/cpu-f32-q8_0/n10","metric":"p50ms","value":162.868345,"of":10},{"category":"cached/plan-host/cpu-q8_0/n10","metric":"p50ms","value":64.692634,"of":10},{"category":"cached/plan-device/cpu-f32/n10","metric":"p50ms","value":233.602168,"of":10},{"category":"cold/tensor/cpu-f32-q8_0/n100","metric":"p50ms","value":4825.16963,"of":10},{"category":"cached/tensor/cpu-f32-q8_0/n100","metric":"p50ms","value":2947.094733,"of":10},{"category":"cached/plan-host/cpu-q8_0/n100","metric":"p50ms","value":766.371197,"of":10},{"category":"cached/plan-device/cpu-f32/n100","metric":"p50ms","value":5602.230622,"of":10}],"notes":"bench decide (results/latency/window2/evo/decide.json, config evo.cpu.f32.q8.t4); host evo (Strix Halo, 16 x86-64 cores, ROCm gfx1151); threads 4; rounds 10 a row after one discarded; load 3.1, 3.3, 2.3; checkpoint a7 (weights 0834a74f, kai-1's base). The release serves on the CPU; GPU rows are not production.","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"decision.latency-window2-evo.cpu.f32.q8.t8-kai","benchmark":"decision.latency","split":"window2","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"decision-suite","commit":"c5bf93f","questions":null,"answered":null,"ended":"2026-09-28T16:37:40Z","completion":"unknown","measures":[{"category":"cold/tensor/cpu-f32-q8_0/n1","metric":"p50ms","value":70.319616,"of":10},{"category":"cached/tensor/cpu-f32-q8_0/n1","metric":"p50ms","value":7.058544,"of":10},{"category":"cached/plan-host/cpu-q8_0/n1","metric":"p50ms","value":1.982388,"of":10},{"category":"cached/plan-device/cpu-f32/n1","metric":"p50ms","value":5.201012,"of":10},{"category":"cold/tensor/cpu-f32-q8_0/n10","metric":"p50ms","value":385.727998,"of":10},{"category":"cached/tensor/cpu-f32-q8_0/n10","metric":"p50ms","value":118.484855,"of":10},{"category":"cached/plan-host/cpu-q8_0/n10","metric":"p50ms","value":34.253221,"of":10},{"category":"cached/plan-device/cpu-f32/n10","metric":"p50ms","value":158.062412,"of":10},{"category":"cold/tensor/cpu-f32-q8_0/n100","metric":"p50ms","value":3291.021344,"of":10},{"category":"cached/tensor/cpu-f32-q8_0/n100","metric":"p50ms","value":2248.221711,"of":10},{"category":"cached/plan-host/cpu-q8_0/n100","metric":"p50ms","value":418.988812,"of":10},{"category":"cached/plan-device/cpu-f32/n100","metric":"p50ms","value":3747.476489,"of":10}],"notes":"bench decide (results/latency/window2/evo/decide.json, config evo.cpu.f32.q8.t8); host evo (Strix Halo, 16 x86-64 cores, ROCm gfx1151); threads 8; rounds 10 a row after one discarded; load 5.3, 3.6, 2.1; checkpoint a7 (weights 0834a74f, kai-1's base). The release serves on the CPU; GPU rows are not production.","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"decision.latency-window2-evo.rocm.f32.q8.tall-kai","benchmark":"decision.latency","split":"window2","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"decision-suite","commit":"c5bf93f","questions":null,"answered":null,"ended":"2026-09-28T16:37:40Z","completion":"unknown","measures":[{"category":"cold/tensor/rocm-f32/n1","metric":"p50ms","value":32.10558,"of":50},{"category":"cached/tensor/rocm-f32/n1","metric":"p50ms","value":2.660234,"of":50},{"category":"cached/plan-host/cpu-q8_0/n1","metric":"p50ms","value":2.028257,"of":50},{"category":"cached/plan-device/rocm-f32/n1","metric":"p50ms","value":1.397709,"of":50},{"category":"cached/plan-device/rocm-bf16/n1","metric":"p50ms","value":0.859189,"of":50},{"category":"cold/tensor/rocm-f32/n10","metric":"p50ms","value":167.214956,"of":50},{"category":"cached/tensor/rocm-f32/n10","metric":"p50ms","value":23.777711,"of":50},{"category":"cached/plan-host/cpu-q8_0/n10","metric":"p50ms","value":22.206117,"of":50},{"category":"cached/plan-device/rocm-f32/n10","metric":"p50ms","value":31.750112,"of":50},{"category":"cached/plan-device/rocm-bf16/n10","metric":"p50ms","value":8.527111,"of":50},{"category":"cold/tensor/rocm-f32/n100","metric":"p50ms","value":856.694591,"of":50},{"category":"cached/tensor/rocm-f32/n100","metric":"p50ms","value":282.462178,"of":50},{"category":"cached/plan-host/cpu-q8_0/n100","metric":"p50ms","value":252.181652,"of":50},{"category":"cached/plan-device/rocm-f32/n100","metric":"p50ms","value":678.284623,"of":50},{"category":"cached/plan-device/rocm-bf16/n100","metric":"p50ms","value":157.548742,"of":50}],"notes":"bench decide (results/latency/window2/evo/decide.json, config evo.rocm.f32.q8.tall); host evo (Strix Halo, 16 x86-64 cores, ROCm gfx1151); threads all; rounds 50 a row after one discarded; load 3.3, 2.0, 1.4; checkpoint a7 (weights 0834a74f, kai-1's base). The release serves on the CPU; GPU rows are not production.","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"decision.latency-window2-evo.rocm.n1.transport-kai","benchmark":"decision.latency","split":"window2","system":"kai","version":"a7@0834a74f2d140642a453373da09e5a128e3d2c8d9e8bb1dcaf86d7210a4ecdfc","baseline":false,"study":"decision-suite","commit":"c5bf93f","questions":null,"answered":null,"ended":"2026-09-28T16:30:52Z","completion":"unknown","measures":[{"category":"cached/handle/rocm/n1","metric":"p50ms","value":0.939986,"of":300},{"category":"cached/runtime/rocm/n1","metric":"p50ms","value":0.874399,"of":300},{"category":"cached/uds/rocm/n1","metric":"p50ms","value":0.913377,"of":300},{"category":"cached/http/rocm/n1","metric":"p50ms","value":1.031082,"of":300}],"notes":"bench transport (results/latency/window2/evo/transport.json); host evo (Strix Halo, 16 x86-64 cores, ROCm gfx1151); 300 rounds interleaved; load 3.1, 2.0, 1.4; one cached decision, the state observed once, on the plan the race chose. The Unix socket was retired 2026-10-09.","revision":"original","canonical":true,"visibility":"public"}],"gpqa":[{"project":"default","id":"enso.gpqa_diamond-heldout-1.6.6-best-single","benchmark":"enso.gpqa_diamond","split":"heldout","system":"best-single","version":"","baseline":true,"study":"enso-bench","commit":"ec4d929f","questions":159,"answered":159,"ended":"2026-10-16T21:02:49Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.930818,"n":148,"of":159},{"category":"all","metric":"stderr","value":0.0201},{"category":"all","metric":"n","value":159},{"category":"all","metric":"correct","value":148},{"category":"all","metric":"usd_per_1k","value":5.1556}],"notes":"","revision":"corrected","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-heldout-1.6.6-enso","benchmark":"enso.gpqa_diamond","split":"heldout","system":"enso","version":"","baseline":false,"study":"enso-bench","commit":"ec4d929f","questions":159,"answered":159,"ended":"2026-10-16T21:02:49Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.937107,"n":149,"of":159},{"category":"all","metric":"stderr","value":0.0193},{"category":"all","metric":"n","value":159},{"category":"all","metric":"correct","value":149},{"category":"all","metric":"usd_per_1k","value":10.0316}],"notes":"","revision":"corrected","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-heldout-1.6.6-enso-flash","benchmark":"enso.gpqa_diamond","split":"heldout","system":"enso-flash","version":"","baseline":false,"study":"enso-bench","commit":"ec4d929f","questions":159,"answered":159,"ended":"2026-10-16T21:02:49Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.90566,"n":144,"of":159},{"category":"all","metric":"stderr","value":0.0232},{"category":"all","metric":"n","value":159},{"category":"all","metric":"correct","value":144},{"category":"all","metric":"usd_per_1k","value":3.2602}],"notes":"","revision":"corrected","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-heldout-1.6.6-enso-ultra","benchmark":"enso.gpqa_diamond","split":"heldout","system":"enso-ultra","version":"","baseline":false,"study":"enso-bench","commit":"ec4d929f","questions":159,"answered":159,"ended":"2026-10-16T21:02:49Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.955975,"n":152,"of":159},{"category":"all","metric":"stderr","value":0.0163},{"category":"all","metric":"n","value":159},{"category":"all","metric":"correct","value":152},{"category":"all","metric":"usd_per_1k","value":14.7631}],"notes":"","revision":"corrected","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-lockbox-1.6.6-enso","benchmark":"enso.gpqa_diamond","split":"lockbox","system":"enso","version":"","baseline":false,"study":"enso-bench","commit":"ec4d929f","questions":39,"answered":39,"ended":"2026-10-16T21:02:49Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.923077,"n":36,"of":39},{"category":"all","metric":"stderr","value":0.0427},{"category":"all","metric":"n","value":39},{"category":"all","metric":"correct","value":36},{"category":"all","metric":"usd_per_1k","value":9.2669}],"notes":"","revision":"corrected","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-lockbox-1.6.6-enso-flash","benchmark":"enso.gpqa_diamond","split":"lockbox","system":"enso-flash","version":"","baseline":false,"study":"enso-bench","commit":"ec4d929f","questions":39,"answered":39,"ended":"2026-10-16T21:02:49Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.948718,"n":37,"of":39},{"category":"all","metric":"stderr","value":0.0353},{"category":"all","metric":"n","value":39},{"category":"all","metric":"correct","value":37},{"category":"all","metric":"usd_per_1k","value":2.5885}],"notes":"","revision":"corrected","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-lockbox-1.6.6-enso-ultra","benchmark":"enso.gpqa_diamond","split":"lockbox","system":"enso-ultra","version":"","baseline":false,"study":"enso-bench","commit":"ec4d929f","questions":39,"answered":39,"ended":"2026-10-16T21:02:49Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.948718,"n":37,"of":39},{"category":"all","metric":"stderr","value":0.0353},{"category":"all","metric":"n","value":39},{"category":"all","metric":"correct","value":37},{"category":"all","metric":"usd_per_1k","value":10.5556}],"notes":"","revision":"corrected","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-20260720-212511-enso-ultra","benchmark":"enso.gpqa_diamond","split":"test","system":"enso-ultra","version":"","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-20T23:56:21Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"stderr","value":0.0182},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":184},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"list_cost_usd","value":343.36963861199996}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-corpus-train-enso-ultra","benchmark":"enso.gpqa_diamond","split":"test","system":"enso-ultra","version":"","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":89,"ended":"2026-07-21T19:37:53Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.414141,"n":82,"of":198},{"category":"all","metric":"stderr","value":0.035},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":82},{"category":"all","metric":"errors","value":109},{"category":"all","metric":"blanks","value":109},{"category":"all","metric":"list_cost_usd","value":159.696502504}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-fill-fable-5.1","benchmark":"enso.gpqa_diamond","split":"test","system":"fable-5.1","version":"anthropic/claude-fable-5.1","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":193,"ended":"2026-10-16T19:53:15Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.878788,"n":174,"of":198},{"category":"all","metric":"stderr","value":0.0232},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":174},{"category":"all","metric":"errors","value":5},{"category":"all","metric":"blanks","value":5},{"category":"all","metric":"prompt_tokens","value":75407},{"category":"all","metric":"completion_tokens","value":149482},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":193},{"category":"all","metric":"cost_usd","value":9.1582}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-fill-haiku-5.5","benchmark":"enso.gpqa_diamond","split":"test","system":"haiku-5.5","version":"anthropic/claude-haiku-5.5","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-10-16T18:51:14Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.833333,"n":165,"of":198},{"category":"all","metric":"stderr","value":0.0265},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":165},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":78517},{"category":"all","metric":"completion_tokens","value":364129},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":0.2114}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-fill-opus-5.5","benchmark":"enso.gpqa_diamond","split":"test","system":"opus-5.5","version":"anthropic/claude-opus-5.5","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":195,"ended":"2026-10-16T19:25:41Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.909091,"n":180,"of":198},{"category":"all","metric":"stderr","value":0.0204},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":180},{"category":"all","metric":"errors","value":3},{"category":"all","metric":"blanks","value":3},{"category":"all","metric":"prompt_tokens","value":76538},{"category":"all","metric":"completion_tokens","value":173489},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":195},{"category":"all","metric":"cost_usd","value":4.2027}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-fill-sonnet-5.5","benchmark":"enso.gpqa_diamond","split":"test","system":"sonnet-5.5","version":"anthropic/claude-sonnet-5.5","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":197,"ended":"2026-10-16T19:25:13Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.90404,"n":179,"of":198},{"category":"all","metric":"stderr","value":0.0209},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":179},{"category":"all","metric":"errors","value":1},{"category":"all","metric":"blanks","value":1},{"category":"all","metric":"prompt_tokens","value":77500},{"category":"all","metric":"completion_tokens","value":169690},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":197},{"category":"all","metric":"cost_usd","value":2.0612}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-frontier-claude-5-sonnet","benchmark":"enso.gpqa_diamond","split":"test","system":"claude-5-sonnet","version":"anthropic-claude-5-sonnet","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-19T10:29:15Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.777778,"n":154,"of":198},{"category":"all","metric":"stderr","value":0.0295},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":154},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":78121},{"category":"all","metric":"completion_tokens","value":550804},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":8.4964}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-frontier-deepseek-v4-pro","benchmark":"enso.gpqa_diamond","split":"test","system":"deepseek-v4-pro","version":"deepseek-v4-pro","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-19T09:02:00Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.742424,"n":147,"of":198},{"category":"all","metric":"stderr","value":0.0311},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":147},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":52523},{"category":"all","metric":"completion_tokens","value":109126},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":0.2089}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-frontier-enso","benchmark":"enso.gpqa_diamond","split":"test","system":"enso","version":"","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-19T07:41:21Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"stderr","value":0.0182},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":184},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"list_cost_usd","value":5.3110161719999995}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-frontier-enso-flash","benchmark":"enso.gpqa_diamond","split":"test","system":"enso-flash","version":"","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-19T07:18:27Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.722222,"n":143,"of":198},{"category":"all","metric":"stderr","value":0.0318},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":143},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"list_cost_usd","value":0.22131779999999998}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-frontier-enso-ultra","benchmark":"enso.gpqa_diamond","split":"test","system":"enso-ultra","version":"","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-19T08:48:43Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"stderr","value":0.0182},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":184},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"list_cost_usd","value":19.232440612}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-frontier-fable-5","benchmark":"enso.gpqa_diamond","split":"test","system":"fable-5","version":"anthropic-claude-fable-5","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-19T09:51:13Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.828283,"n":164,"of":198},{"category":"all","metric":"stderr","value":0.0268},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":164},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":78121},{"category":"all","metric":"completion_tokens","value":181711},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":2.96}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-frontier-gpt-4o","benchmark":"enso.gpqa_diamond","split":"test","system":"gpt-4o","version":"openai-gpt-4o","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-19T09:59:04Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.545455,"n":108,"of":198},{"category":"all","metric":"stderr","value":0.0354},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":108},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":55114},{"category":"all","metric":"completion_tokens","value":144289},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":1.5807}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-frontier-gpt-5.5","benchmark":"enso.gpqa_diamond","split":"test","system":"gpt-5.5","version":"openai-gpt-5.5","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-19T09:36:27Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.89899,"n":178,"of":198},{"category":"all","metric":"stderr","value":0.0214},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":178},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":54916},{"category":"all","metric":"completion_tokens","value":389125},{"category":"all","metric":"reasoning_tokens","value":352545},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":3.9599}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-frontier-gpt-5.6-sol","benchmark":"enso.gpqa_diamond","split":"test","system":"gpt-5.6-sol","version":"openai-gpt-5.6-sol","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-19T06:52:53Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.919192,"n":182,"of":198},{"category":"all","metric":"stderr","value":0.0194},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":182},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":54916},{"category":"all","metric":"completion_tokens","value":192804},{"category":"all","metric":"reasoning_tokens","value":155827},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":1.9967}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-frontier-opus-4.8","benchmark":"enso.gpqa_diamond","split":"test","system":"opus-4.8","version":"anthropic-claude-opus-4.8","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-19T07:03:15Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.863636,"n":171,"of":198},{"category":"all","metric":"stderr","value":0.0244},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":171},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":78121},{"category":"all","metric":"completion_tokens","value":130593},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":10.9663}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-k3fix-enso-ultra","benchmark":"enso.gpqa_diamond","split":"test","system":"enso-ultra","version":"","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":68,"ended":"2026-07-20T11:27:28Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.328283,"n":65,"of":198},{"category":"all","metric":"stderr","value":0.0334},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":65},{"category":"all","metric":"errors","value":130},{"category":"all","metric":"blanks","value":130},{"category":"all","metric":"list_cost_usd","value":32.489882656}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-max-deepseek-v4-pro","benchmark":"enso.gpqa_diamond","split":"test","system":"deepseek-v4-pro","version":"deepseek-v4-pro","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-20T02:54:52Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.752525,"n":149,"of":198},{"category":"all","metric":"stderr","value":0.0307},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":149},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":60695},{"category":"all","metric":"completion_tokens","value":113354},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":0.2204}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-max-fable-5","benchmark":"enso.gpqa_diamond","split":"test","system":"fable-5","version":"claude-fable-5","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":170,"ended":"2026-07-20T00:24:36Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.80303,"n":159,"of":198},{"category":"all","metric":"stderr","value":0.0283},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":159},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":28},{"category":"all","metric":"prompt_tokens","value":60695},{"category":"all","metric":"completion_tokens","value":66379},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":1.1778}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-max-gpt-5.6-sol","benchmark":"enso.gpqa_diamond","split":"test","system":"gpt-5.6-sol","version":"gpt-5.6-sol","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":193,"ended":"2026-07-19T23:54:55Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.924242,"n":183,"of":198},{"category":"all","metric":"stderr","value":0.0188},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":183},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":5},{"category":"all","metric":"prompt_tokens","value":60695},{"category":"all","metric":"completion_tokens","value":34023},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-max-opus-4.8","benchmark":"enso.gpqa_diamond","split":"test","system":"opus-4.8","version":"claude-opus-4.8","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":196,"ended":"2026-07-20T00:07:57Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.873737,"n":173,"of":198},{"category":"all","metric":"stderr","value":0.0236},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":173},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":2},{"category":"all","metric":"prompt_tokens","value":60695},{"category":"all","metric":"completion_tokens","value":106682},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":8.9116}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K1","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K1","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.9293},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9293},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9293},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":1},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K10","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K10","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.994949,"n":197,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.9192},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9178},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9064},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":10},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K11","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K11","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.994949,"n":197,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.9165},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9155},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9064},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":11},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K12","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K12","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.994949,"n":197,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.9125},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9158},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9064},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":12},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K13","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K13","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.994949,"n":197,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.8997},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9118},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9064},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":13},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K14","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K14","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.994949,"n":197,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.8949},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9104},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9064},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":14},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K15","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K15","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.994949,"n":197,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.8939},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9101},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9064},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":15},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K2","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K2","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.974747,"n":193,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.9384},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9279},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9172},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":2},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K3","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K3","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.979798,"n":194,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.9401},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9286},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9064},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":3},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K4","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K4","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.984848,"n":195,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.9448},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9296},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9064},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":4},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K5","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K5","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.989899,"n":196,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.9441},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9296},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9064},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":5},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K6","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K6","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.994949,"n":197,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.93},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9249},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9064},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":6},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K7","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K7","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.994949,"n":197,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.931},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9253},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9064},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":7},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K8","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K8","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.994949,"n":197,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.9266},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9222},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9064},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":8},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-pool-oracle-pool-oracle@K9","benchmark":"enso.gpqa_diamond","split":"test","system":"pool-oracle@K9","version":"","baseline":false,"study":"enso-bench","commit":"e336832","questions":198,"answered":198,"ended":"2026-07-23T16:58:13Z","completion":"complete","measures":[{"category":"all","metric":"oracle_accuracy","value":0.994949,"n":197,"of":198},{"category":"all","metric":"best_single_accuracy","value":0.929293,"n":184,"of":198},{"category":"all","metric":"router_heldout_accuracy","value":0.9269},{"category":"all","metric":"champion_relative_heldout_accuracy","value":0.9232},{"category":"all","metric":"train_selected_single_heldout_accuracy","value":0.9064},{"category":"all","metric":"arms","value":15},{"category":"all","metric":"k","value":9},{"category":"all","metric":"n","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-seen-1.6.6-enso","benchmark":"enso.gpqa_diamond","split":"test","system":"enso","version":"","baseline":false,"study":"enso-bench","commit":"ec4d929f","questions":198,"answered":198,"ended":"2026-10-16T21:02:49Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.964646,"n":191,"of":198},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":191}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-seen-1.6.6-enso-flash","benchmark":"enso.gpqa_diamond","split":"test","system":"enso-flash","version":"","baseline":false,"study":"enso-bench","commit":"ec4d929f","questions":198,"answered":198,"ended":"2026-10-16T21:02:49Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.954545,"n":189,"of":198},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":189}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-seen-1.6.6-enso-ultra","benchmark":"enso.gpqa_diamond","split":"test","system":"enso-ultra","version":"","baseline":false,"study":"enso-bench","commit":"ec4d929f","questions":198,"answered":198,"ended":"2026-10-16T21:02:49Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.979798,"n":194,"of":198},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":194}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v1-deepseek-v4-pro","benchmark":"enso.gpqa_diamond","split":"test","system":"deepseek-v4-pro","version":"deepseek-v4-pro","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-16T02:16:55Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.762626,"n":151,"of":198},{"category":"all","metric":"stderr","value":0.0302},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":151},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":52523},{"category":"all","metric":"completion_tokens","value":108318},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":0.2076}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v1-enso","benchmark":"enso.gpqa_diamond","split":"test","system":"enso","version":"","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-16T02:46:59Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.878788,"n":174,"of":198},{"category":"all","metric":"stderr","value":0.0232},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":174},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"list_cost_usd","value":4.383191736}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v1-enso-ultra","benchmark":"enso.gpqa_diamond","split":"test","system":"enso-ultra","version":"","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-16T04:07:59Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.89899,"n":178,"of":198},{"category":"all","metric":"stderr","value":0.0214},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":178},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"list_cost_usd","value":17.117347940000002}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v1-fable-5","benchmark":"enso.gpqa_diamond","split":"test","system":"fable-5","version":"anthropic-claude-fable-5","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-16T02:33:53Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.813131,"n":161,"of":198},{"category":"all","metric":"stderr","value":0.0277},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":161},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":78121},{"category":"all","metric":"completion_tokens","value":199072},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":3.2204}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v1-gpt-5.5","benchmark":"enso.gpqa_diamond","split":"test","system":"gpt-5.5","version":"openai-gpt-5.5","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-16T01:54:55Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.90404,"n":179,"of":198},{"category":"all","metric":"stderr","value":0.0209},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":179},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":54916},{"category":"all","metric":"completion_tokens","value":362128},{"category":"all","metric":"reasoning_tokens","value":326158},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":3.6899},{"category":"all","metric":"wall_seconds","value":2396.2}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v1-opus-4.8","benchmark":"enso.gpqa_diamond","split":"test","system":"opus-4.8","version":"anthropic-claude-opus-4.8","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-16T01:07:58Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.868687,"n":172,"of":198},{"category":"all","metric":"stderr","value":0.024},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":172},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":78121},{"category":"all","metric":"completion_tokens","value":126069},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":10.627},{"category":"all","metric":"wall_seconds","value":241.3}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v2-deepseek-3.2","benchmark":"enso.gpqa_diamond","split":"test","system":"deepseek-3.2","version":"deepseek-3.2","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-19T20:07:52Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.737374,"n":146,"of":198},{"category":"all","metric":"stderr","value":0.0313},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":146},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":60695},{"category":"all","metric":"completion_tokens","value":367921},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v2-deepseek-4-flash","benchmark":"enso.gpqa_diamond","split":"test","system":"deepseek-4-flash","version":"deepseek-4-flash","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":182,"ended":"2026-07-19T17:04:56Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.707071,"n":140,"of":198},{"category":"all","metric":"stderr","value":0.0323},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":140},{"category":"all","metric":"errors","value":16},{"category":"all","metric":"blanks","value":16},{"category":"all","metric":"prompt_tokens","value":56442},{"category":"all","metric":"completion_tokens","value":101277},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":182}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v2-deepseek-v4-pro","benchmark":"enso.gpqa_diamond","split":"test","system":"deepseek-v4-pro","version":"deepseek-v4-pro","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-19T19:39:37Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.752525,"n":149,"of":198},{"category":"all","metric":"stderr","value":0.0307},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":149},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"prompt_tokens","value":60695},{"category":"all","metric":"completion_tokens","value":107808},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":0.2113}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v2-enso","benchmark":"enso.gpqa_diamond","split":"test","system":"enso","version":"","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":196,"ended":"2026-07-19T20:12:42Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.878788,"n":174,"of":198},{"category":"all","metric":"stderr","value":0.0232},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":174},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":2},{"category":"all","metric":"list_cost_usd","value":3.10335102}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v2-enso-flash","benchmark":"enso.gpqa_diamond","split":"test","system":"enso-flash","version":"","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-19T19:59:36Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.757576,"n":150,"of":198},{"category":"all","metric":"stderr","value":0.0305},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":150},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"list_cost_usd","value":0.61722375}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v2-enso-ultra","benchmark":"enso.gpqa_diamond","split":"test","system":"enso-ultra","version":"","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":198,"ended":"2026-07-17T13:30:33Z","completion":"complete","measures":[{"category":"all","metric":"accuracy","value":0.90404,"n":179,"of":198},{"category":"all","metric":"stderr","value":0.0209},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":179},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":0},{"category":"all","metric":"list_cost_usd","value":16.938323864}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v2-fable-5","benchmark":"enso.gpqa_diamond","split":"test","system":"fable-5","version":"claude-fable-5","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":172,"ended":"2026-07-19T19:24:01Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.813131,"n":161,"of":198},{"category":"all","metric":"stderr","value":0.0277},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":161},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":26},{"category":"all","metric":"prompt_tokens","value":60695},{"category":"all","metric":"completion_tokens","value":66199},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":1.1751}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v2-gpt-5.6-luna","benchmark":"enso.gpqa_diamond","split":"test","system":"gpt-5.6-luna","version":"gpt-5.6-luna","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":183,"ended":"2026-07-19T18:55:16Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.828283,"n":164,"of":198},{"category":"all","metric":"stderr","value":0.0268},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":164},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":15},{"category":"all","metric":"prompt_tokens","value":60695},{"category":"all","metric":"completion_tokens","value":31104},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v2-gpt-5.6-sol","benchmark":"enso.gpqa_diamond","split":"test","system":"gpt-5.6-sol","version":"gpt-5.6-sol","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":189,"ended":"2026-07-19T18:32:24Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.90404,"n":179,"of":198},{"category":"all","metric":"stderr","value":0.0209},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":179},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":9},{"category":"all","metric":"prompt_tokens","value":60695},{"category":"all","metric":"completion_tokens","value":32702},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v2-gpt-5.6-terra","benchmark":"enso.gpqa_diamond","split":"test","system":"gpt-5.6-terra","version":"gpt-5.6-terra","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":191,"ended":"2026-07-19T18:44:24Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.878788,"n":174,"of":198},{"category":"all","metric":"stderr","value":0.0232},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":174},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":7},{"category":"all","metric":"prompt_tokens","value":60695},{"category":"all","metric":"completion_tokens","value":25529},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198}],"notes":"","revision":"original","canonical":true,"visibility":"public"},{"project":"default","id":"enso.gpqa_diamond-test-v2-opus-4.8","benchmark":"enso.gpqa_diamond","split":"test","system":"opus-4.8","version":"claude-opus-4.8","baseline":false,"study":"enso-bench","commit":"","questions":198,"answered":196,"ended":"2026-07-19T19:07:45Z","completion":"partial","measures":[{"category":"all","metric":"accuracy","value":0.853535,"n":169,"of":198},{"category":"all","metric":"stderr","value":0.0251},{"category":"all","metric":"n","value":198},{"category":"all","metric":"correct","value":169},{"category":"all","metric":"errors","value":0},{"category":"all","metric":"blanks","value":2},{"category":"all","metric":"prompt_tokens","value":60695},{"category":"all","metric":"completion_tokens","value":106384},{"category":"all","metric":"reasoning_tokens","value":0},{"category":"all","metric":"calls","value":198},{"category":"all","metric":"cost_usd","value":8.8892}],"notes":"","revision":"original","canonical":true,"visibility":"public"}]}
|