mirror of
https://github.com/hanzoai/cli.git
synced 2026-10-30 15:00:27 +00:00
572 lines
14 KiB
YAML
572 lines
14 KiB
YAML
- dirname: 2024-12-21-18-41-18--polyglot-gpt-4o-mini
|
|
test_cases: 225
|
|
model: gpt-4o-mini-2024-07-18
|
|
edit_format: whole
|
|
commit_hash: a755079-dirty
|
|
pass_rate_1: 0.9
|
|
pass_rate_2: 3.6
|
|
pass_num_1: 2
|
|
pass_num_2: 8
|
|
percent_cases_well_formed: 100.0
|
|
error_outputs: 0
|
|
num_malformed_responses: 0
|
|
num_with_malformed_responses: 0
|
|
user_asks: 36
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 3
|
|
total_tests: 225
|
|
command: dev --model gpt-4o-mini-2024-07-18
|
|
date: 2024-12-21
|
|
versions: 0.69.2.dev
|
|
seconds_per_case: 17.3
|
|
total_cost: 0.3236
|
|
|
|
- dirname: 2025-01-17-19-44-33--sonnet-baseline-jan-17
|
|
test_cases: 225
|
|
model: claude-3-5-sonnet-20241022
|
|
edit_format: diff
|
|
commit_hash: 6451d59
|
|
pass_rate_1: 22.2
|
|
pass_rate_2: 51.6
|
|
pass_num_1: 50
|
|
pass_num_2: 116
|
|
percent_cases_well_formed: 99.6
|
|
error_outputs: 2
|
|
num_malformed_responses: 1
|
|
num_with_malformed_responses: 1
|
|
user_asks: 11
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 1
|
|
test_timeouts: 8
|
|
total_tests: 225
|
|
command: dev --model claude-3-5-sonnet-20241022
|
|
date: 2025-01-17
|
|
versions: 0.71.2.dev
|
|
seconds_per_case: 21.4
|
|
total_cost: 14.4063
|
|
|
|
- dirname: 2024-12-30-20-57-12--gpt-4o-2024-11-20-ex-as-sys
|
|
test_cases: 225
|
|
model: gpt-4o-2024-11-20
|
|
edit_format: diff
|
|
commit_hash: 09ee197-dirty
|
|
pass_rate_1: 4.9
|
|
pass_rate_2: 18.2
|
|
pass_num_1: 11
|
|
pass_num_2: 41
|
|
percent_cases_well_formed: 95.1
|
|
error_outputs: 12
|
|
num_malformed_responses: 12
|
|
num_with_malformed_responses: 11
|
|
user_asks: 53
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 12
|
|
total_tests: 225
|
|
command: dev --model gpt-4o-2024-11-20
|
|
date: 2024-12-30
|
|
versions: 0.70.1.dev
|
|
seconds_per_case: 12.1
|
|
total_cost: 6.7351
|
|
|
|
- dirname: 2024-12-30-20-44-54--gpt4o-ex-as-sys-clean-prompt
|
|
test_cases: 225
|
|
model: gpt-4o-2024-08-06
|
|
edit_format: diff
|
|
commit_hash: 09ee197-dirty
|
|
pass_rate_1: 4.9
|
|
pass_rate_2: 23.1
|
|
pass_num_1: 11
|
|
pass_num_2: 52
|
|
percent_cases_well_formed: 94.2
|
|
error_outputs: 21
|
|
num_malformed_responses: 21
|
|
num_with_malformed_responses: 13
|
|
user_asks: 65
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 3
|
|
total_tests: 225
|
|
command: dev --model gpt-4o-2024-08-06
|
|
date: 2024-12-30
|
|
versions: 0.70.1.dev
|
|
seconds_per_case: 16.0
|
|
total_cost: 7.0286
|
|
|
|
- dirname: 2024-12-21-19-23-03--polyglot-o1-hard-diff
|
|
test_cases: 224
|
|
model: o1-2024-12-17 (high)
|
|
edit_format: diff
|
|
commit_hash: a755079-dirty
|
|
pass_rate_1: 23.7
|
|
pass_rate_2: 61.7
|
|
pass_num_1: 53
|
|
pass_num_2: 139
|
|
percent_cases_well_formed: 91.5
|
|
error_outputs: 25
|
|
num_malformed_responses: 24
|
|
num_with_malformed_responses: 19
|
|
user_asks: 16
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 2
|
|
total_tests: 225
|
|
command: dev --model openrouter/openai/o1
|
|
date: 2024-12-21
|
|
versions: 0.69.2.dev
|
|
seconds_per_case: 133.2
|
|
total_cost: 186.4958
|
|
|
|
- dirname: 2024-12-21-20-56-21--polyglot-deepseek-diff
|
|
test_cases: 225
|
|
model: DeepSeek Chat V2.5
|
|
edit_format: diff
|
|
commit_hash: a755079-dirty
|
|
pass_rate_1: 5.3
|
|
pass_rate_2: 17.8
|
|
pass_num_1: 12
|
|
pass_num_2: 40
|
|
percent_cases_well_formed: 92.9
|
|
error_outputs: 42
|
|
num_malformed_responses: 37
|
|
num_with_malformed_responses: 16
|
|
user_asks: 23
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 5
|
|
test_timeouts: 5
|
|
total_tests: 225
|
|
command: dev --model deepseek/deepseek-chat
|
|
date: 2024-12-21
|
|
versions: 0.69.2.dev
|
|
seconds_per_case: 184.0
|
|
total_cost: 0.5101
|
|
|
|
- dirname: 2024-12-21-21-46-27--polyglot-haiku-diff
|
|
test_cases: 225
|
|
model: claude-3-5-haiku-20241022
|
|
edit_format: diff
|
|
commit_hash: a755079-dirty
|
|
pass_rate_1: 7.1
|
|
pass_rate_2: 28.0
|
|
pass_num_1: 16
|
|
pass_num_2: 63
|
|
percent_cases_well_formed: 91.1
|
|
error_outputs: 31
|
|
num_malformed_responses: 30
|
|
num_with_malformed_responses: 20
|
|
user_asks: 13
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 1
|
|
test_timeouts: 9
|
|
total_tests: 225
|
|
command: dev --model claude-3-5-haiku-20241022
|
|
date: 2024-12-21
|
|
versions: 0.69.2.dev
|
|
seconds_per_case: 31.8
|
|
total_cost: 6.0583
|
|
|
|
- dirname: 2024-12-22-13-22-32--polyglot-qwen-diff
|
|
test_cases: 225
|
|
model: Qwen2.5-Coder-32B-Instruct
|
|
edit_format: diff
|
|
commit_hash: 6d7e8be-dirty
|
|
pass_rate_1: 4.4
|
|
pass_rate_2: 8.0
|
|
pass_num_1: 10
|
|
pass_num_2: 18
|
|
percent_cases_well_formed: 71.6
|
|
error_outputs: 158
|
|
num_malformed_responses: 148
|
|
num_with_malformed_responses: 64
|
|
user_asks: 132
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 1
|
|
test_timeouts: 2
|
|
total_tests: 225
|
|
command: "dev --model openai/Qwen/Qwen2.5-Coder-32B-Instruct # via hyperbolic"
|
|
date: 2024-12-22
|
|
versions: 0.69.2.dev
|
|
seconds_per_case: 84.4
|
|
total_cost: 0.0000
|
|
|
|
- dirname: 2024-12-22-21-26-35--polyglot-o1mini-whole
|
|
test_cases: 225
|
|
model: o1-mini-2024-09-12
|
|
edit_format: whole
|
|
commit_hash: 37df899
|
|
pass_rate_1: 5.8
|
|
pass_rate_2: 32.9
|
|
pass_num_1: 13
|
|
pass_num_2: 74
|
|
percent_cases_well_formed: 96.9
|
|
error_outputs: 8
|
|
num_malformed_responses: 8
|
|
num_with_malformed_responses: 7
|
|
user_asks: 27
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 3
|
|
total_tests: 225
|
|
command: dev --model o1-mini
|
|
date: 2024-12-22
|
|
versions: 0.69.2.dev
|
|
seconds_per_case: 34.7
|
|
total_cost: 18.5770
|
|
|
|
- dirname: 2024-12-22-18-43-25--gemini-exp-1206-polyglot-whole-2
|
|
test_cases: 225
|
|
model: gemini-exp-1206
|
|
edit_format: whole
|
|
commit_hash: b1bc2f8
|
|
pass_rate_1: 19.6
|
|
pass_rate_2: 38.2
|
|
pass_num_1: 44
|
|
pass_num_2: 86
|
|
percent_cases_well_formed: 98.2
|
|
error_outputs: 8
|
|
num_malformed_responses: 8
|
|
num_with_malformed_responses: 4
|
|
user_asks: 32
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 9
|
|
total_tests: 225
|
|
command: dev --model gemini/gemini-exp-1206
|
|
date: 2024-12-22
|
|
versions: 0.69.2.dev
|
|
seconds_per_case: 45.5
|
|
total_cost: 0.0000
|
|
|
|
- dirname: 2024-12-22-20-08-13--gemini-2.0-flash-exp-polyglot-whole
|
|
test_cases: 225
|
|
model: gemini-2.0-flash-exp
|
|
edit_format: whole
|
|
commit_hash: b1bc2f8
|
|
pass_rate_1: 11.6
|
|
pass_rate_2: 22.2
|
|
pass_num_1: 26
|
|
pass_num_2: 50
|
|
percent_cases_well_formed: 100.0
|
|
error_outputs: 1
|
|
num_malformed_responses: 0
|
|
num_with_malformed_responses: 0
|
|
user_asks: 9
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 1
|
|
test_timeouts: 8
|
|
total_tests: 225
|
|
command: dev --model gemini/gemini-2.0-flash-exp
|
|
date: 2024-12-22
|
|
versions: 0.69.2.dev
|
|
seconds_per_case: 12.2
|
|
total_cost: 0.0000
|
|
|
|
- dirname: 2024-12-23-01-11-56--yi-test
|
|
test_cases: 225
|
|
model: yi-lightning
|
|
edit_format: whole
|
|
commit_hash: 2b1625e
|
|
pass_rate_1: 5.8
|
|
pass_rate_2: 12.9
|
|
pass_num_1: 13
|
|
pass_num_2: 29
|
|
percent_cases_well_formed: 92.9
|
|
error_outputs: 87
|
|
num_malformed_responses: 72
|
|
num_with_malformed_responses: 16
|
|
user_asks: 107
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 1
|
|
test_timeouts: 6
|
|
total_tests: 225
|
|
command: dev --model openai/yi-lightning
|
|
date: 2024-12-23
|
|
versions: 0.69.2.dev
|
|
seconds_per_case: 146.7
|
|
total_cost: 0.0000
|
|
|
|
- dirname: 2024-12-25-13-31-51--deepseekv3preview-diff2
|
|
test_cases: 225
|
|
model: DeepSeek Chat V3
|
|
edit_format: diff
|
|
commit_hash: 0a23c4a-dirty
|
|
pass_rate_1: 22.7
|
|
pass_rate_2: 48.4
|
|
pass_num_1: 51
|
|
pass_num_2: 109
|
|
percent_cases_well_formed: 98.7
|
|
error_outputs: 7
|
|
num_malformed_responses: 7
|
|
num_with_malformed_responses: 3
|
|
user_asks: 19
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 8
|
|
total_tests: 225
|
|
command: dev --model deepseek/deepseek-chat
|
|
date: 2024-12-25
|
|
versions: 0.69.2.dev
|
|
seconds_per_case: 34.8
|
|
total_cost: 0.3369
|
|
|
|
- dirname: 2024-12-26-00-55-20--Qwen2.5-Coder-32B-Instruct
|
|
test_cases: 225
|
|
model: Qwen2.5-Coder-32B-Instruct
|
|
edit_format: whole
|
|
commit_hash: b51768b0
|
|
pass_rate_1: 4.9
|
|
pass_rate_2: 16.4
|
|
pass_num_1: 11
|
|
pass_num_2: 37
|
|
percent_cases_well_formed: 99.6
|
|
error_outputs: 1
|
|
num_malformed_responses: 1
|
|
num_with_malformed_responses: 1
|
|
user_asks: 33
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 6
|
|
total_tests: 225
|
|
command: dev --model openai/Qwen2.5-Coder-32B-Instruct
|
|
date: 2024-12-26
|
|
versions: 0.69.2.dev
|
|
seconds_per_case: 42.0
|
|
total_cost: 0.0000
|
|
|
|
- dirname: 2025-01-13-18-17-25--codestral-whole2
|
|
test_cases: 225
|
|
model: Codestral 25.01
|
|
edit_format: whole
|
|
commit_hash: 0cba898-dirty
|
|
pass_rate_1: 4.0
|
|
pass_rate_2: 11.1
|
|
pass_num_1: 9
|
|
pass_num_2: 25
|
|
percent_cases_well_formed: 100.0
|
|
error_outputs: 0
|
|
num_malformed_responses: 0
|
|
num_with_malformed_responses: 0
|
|
user_asks: 47
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 4
|
|
total_tests: 225
|
|
command: dev --model mistral/codestral-latest
|
|
date: 2025-01-13
|
|
versions: 0.71.2.dev
|
|
seconds_per_case: 9.3
|
|
total_cost: 1.9834
|
|
|
|
- dirname: 2025-01-20-19-11-38--ds-turns-upd-cur-msgs-fix-with-summarizer
|
|
test_cases: 225
|
|
model: DeepSeek R1
|
|
edit_format: diff
|
|
commit_hash: 5650697-dirty
|
|
pass_rate_1: 26.7
|
|
pass_rate_2: 56.9
|
|
pass_num_1: 60
|
|
pass_num_2: 128
|
|
percent_cases_well_formed: 96.9
|
|
error_outputs: 8
|
|
num_malformed_responses: 7
|
|
num_with_malformed_responses: 7
|
|
user_asks: 15
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 1
|
|
test_timeouts: 5
|
|
total_tests: 225
|
|
command: dev --model deepseek/deepseek-reasoner
|
|
date: 2025-01-20
|
|
versions: 0.71.2.dev
|
|
seconds_per_case: 113.7
|
|
total_cost: 5.4193
|
|
|
|
- dirname: 2025-01-23-19-14-48--r1-architect-sonnet
|
|
test_cases: 225
|
|
model: DeepSeek R1 + claude-3-5-sonnet-20241022
|
|
edit_format: architect
|
|
commit_hash: 05a77c7
|
|
editor_model: claude-3-5-sonnet-20241022
|
|
editor_edit_format: editor-diff
|
|
pass_rate_1: 27.1
|
|
pass_rate_2: 64.0
|
|
pass_num_1: 61
|
|
pass_num_2: 144
|
|
percent_cases_well_formed: 100.0
|
|
error_outputs: 2
|
|
num_malformed_responses: 0
|
|
num_with_malformed_responses: 0
|
|
user_asks: 392
|
|
lazy_comments: 6
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 5
|
|
total_tests: 225
|
|
command: dev --architect --model r1 --editor-model sonnet
|
|
date: 2025-01-23
|
|
versions: 0.72.3.dev
|
|
seconds_per_case: 251.6
|
|
total_cost: 13.2933
|
|
|
|
- dirname: 2025-01-28-16-00-03--qwen-max-2025-01-25-polyglot-diff
|
|
test_cases: 225
|
|
model: qwen-max-2025-01-25
|
|
edit_format: diff
|
|
commit_hash: ae7d459
|
|
pass_rate_1: 9.3
|
|
pass_rate_2: 21.8
|
|
pass_num_1: 21
|
|
pass_num_2: 49
|
|
percent_cases_well_formed: 90.2
|
|
error_outputs: 46
|
|
num_malformed_responses: 44
|
|
num_with_malformed_responses: 22
|
|
user_asks: 23
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 9
|
|
total_tests: 225
|
|
command: OPENAI_API_BASE=https://dashscope-intl.aliyuncs.com/compatible-mode/v1 dev --model openai/qwen-max-2025-01-25
|
|
date: 2025-01-28
|
|
versions: 0.72.4.dev
|
|
seconds_per_case: 39.5
|
|
|
|
- dirname: 2025-01-31-20-27-46--o3-mini-diff2
|
|
test_cases: 225
|
|
model: o3-mini (medium)
|
|
edit_format: diff
|
|
commit_hash: 2fb517b-dirty
|
|
pass_rate_1: 19.1
|
|
pass_rate_2: 53.8
|
|
pass_num_1: 43
|
|
pass_num_2: 121
|
|
percent_cases_well_formed: 95.1
|
|
error_outputs: 28
|
|
num_malformed_responses: 28
|
|
num_with_malformed_responses: 11
|
|
user_asks: 17
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 2
|
|
total_tests: 225
|
|
command: dev --model o3-mini
|
|
date: 2025-01-31
|
|
versions: 0.72.4.dev
|
|
seconds_per_case: 47.2
|
|
total_cost: 8.8599
|
|
|
|
- dirname: 2025-01-31-20-42-47--o3-mini-diff-high
|
|
test_cases: 224
|
|
model: o3-mini (high)
|
|
edit_format: diff
|
|
commit_hash: b0d58d1-dirty
|
|
pass_rate_1: 21.0
|
|
pass_rate_2: 60.4
|
|
pass_num_1: 47
|
|
pass_num_2: 136
|
|
percent_cases_well_formed: 93.3
|
|
error_outputs: 26
|
|
num_malformed_responses: 24
|
|
num_with_malformed_responses: 15
|
|
user_asks: 19
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 1
|
|
test_timeouts: 7
|
|
total_tests: 225
|
|
command: dev --model o3-mini --reasoning-effort high
|
|
date: 2025-01-31
|
|
versions: 0.72.4.dev
|
|
seconds_per_case: 124.6
|
|
total_cost: 18.1584
|
|
|
|
- dirname: 2025-01-21-22-51-49--gemini-2.0-flash-thinking-exp-01-21-polyglot-diff
|
|
test_cases: 225
|
|
model: gemini-2.0-flash-thinking-exp-01-21
|
|
edit_format: diff
|
|
commit_hash: 843720a
|
|
pass_rate_1: 5.8
|
|
pass_rate_2: 18.2
|
|
pass_num_1: 13
|
|
pass_num_2: 41
|
|
percent_cases_well_formed: 77.8
|
|
error_outputs: 182
|
|
num_malformed_responses: 180
|
|
num_with_malformed_responses: 50
|
|
user_asks: 26
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 2
|
|
test_timeouts: 7
|
|
total_tests: 225
|
|
command: dev --model gemini/gemini-2.0-flash-thinking-exp-01-21
|
|
date: 2025-01-21
|
|
versions: 0.72.2.dev
|
|
seconds_per_case: 24.2
|
|
total_cost: 0.0000
|
|
|
|
- dirname: 2025-02-15-19-51-22--chatgpt4o-feb15-diff
|
|
test_cases: 223
|
|
model: chatgpt-4o-latest (2025-02-15)
|
|
edit_format: diff
|
|
commit_hash: 108ce18-dirty
|
|
pass_rate_1: 9.0
|
|
pass_rate_2: 27.1
|
|
pass_num_1: 20
|
|
pass_num_2: 61
|
|
percent_cases_well_formed: 93.3
|
|
error_outputs: 66
|
|
num_malformed_responses: 21
|
|
num_with_malformed_responses: 15
|
|
user_asks: 57
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 2
|
|
total_tests: 225
|
|
command: dev --model chatgpt-4o-latest
|
|
date: 2025-02-15
|
|
versions: 0.74.3.dev
|
|
seconds_per_case: 12.4
|
|
total_cost: 14.3703 |