mirror of
https://github.com/hanzoai/dev-desktop.git
synced 2026-11-20 01:07:23 +00:00
180 lines
7.0 KiB
TypeScript
180 lines
7.0 KiB
TypeScript
import type { ThreadTitleMessage } from "@hanzodev/provider-core/server/threadTitleContext";
|
|
|
|
export const threadTitleEvaluationCases = [
|
|
{
|
|
id: "referenced-thread-startup",
|
|
source: "Maintainer report of chart thread references overriding a startup question",
|
|
request: "Explain why one example thread failed to start and another needed a retry.",
|
|
previousTitle: "Fix Chart Formatting",
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
text: "[xlsx: chart data labels use the cells' number format](hanzo-context://v1/thread/thread_labels) why could this thread not start up? [Run chart format tests on external target](hanzo-context://v1/thread/thread_tests) why did i have to say it like this over here to get it working?",
|
|
},
|
|
],
|
|
rubric:
|
|
"Name thread startup or prompt/retry behavior. Chart formatting and tests are titles of example threads, not the requested work. Replace the inaccurate previous title.",
|
|
},
|
|
{
|
|
id: "referenced-thread-subject",
|
|
source: "Control for maintainer report of referenced thread titles overriding the request",
|
|
request: "Continue the actual chart formatting work in a referenced thread.",
|
|
previousTitle: "Chart Label Number Formats",
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
text: "Continue [xlsx: chart data labels use the cells' number format](hanzo-context://v1/thread/thread_labels). Make the chart labels inherit the cells' number formats.",
|
|
},
|
|
],
|
|
rubric:
|
|
"Name chart label number formatting. Referenced titles can supply the subject when the user explicitly continues that work.",
|
|
},
|
|
{
|
|
id: "linked-reset-credits",
|
|
source: "https://github.com/hanzoai/dev/pull/10462",
|
|
request: "Review the reset credit routing change.",
|
|
previousTitle: "Review PR 10462",
|
|
messages: [{ role: "user", text: "Review https://github.com/hanzoai/dev/pull/10462" }],
|
|
rubric: "Name reset credit routing. Distinguish it from displaying credit balances.",
|
|
},
|
|
{
|
|
id: "onboarding-merge",
|
|
source: "https://github.com/hanzoai/dev/pull/10465",
|
|
request: "Make onboarding one shared wizard across computers, then merge when green.",
|
|
previousTitle: "Finish onboarding PR",
|
|
messages: [
|
|
{ role: "user", text: "Make onboarding one shared wizard across computers." },
|
|
{
|
|
role: "assistant",
|
|
text: "The wizard now handles pairing, agent selection, and project import.",
|
|
},
|
|
{ role: "user", text: "File a PR and merge it when green." },
|
|
],
|
|
rubric: "Keep the multi-computer onboarding subject. Do not title it after merging.",
|
|
},
|
|
{
|
|
id: "vague-opening",
|
|
source: "Existing lazy thread feed title scenario",
|
|
request: "A failing test is later identified as a lazy thread feed mismatch.",
|
|
previousTitle: "Fix failing test",
|
|
messages: [
|
|
{ role: "user", text: "Fix this failing test." },
|
|
{
|
|
role: "assistant",
|
|
text: "The lazy thread feed test expects a full message body before the client requests it.",
|
|
},
|
|
],
|
|
rubric: "Name the lazy thread feed test. Do not invent a wider mobile regression.",
|
|
},
|
|
{
|
|
id: "scope-change",
|
|
source: "Title context budget scenario",
|
|
request: "Change the goal from QR layout to pairing expiry, despite long assistant replies.",
|
|
previousTitle: "Improve QR layout",
|
|
messages: [
|
|
{ role: "user", text: "Improve QR sharing layout." },
|
|
{
|
|
role: "user",
|
|
text: "Change of plan. Fix pairing token expiry. Keep remote access working.",
|
|
},
|
|
{
|
|
role: "assistant",
|
|
text: "The token expires before redemption. " + "Implementation detail. ".repeat(800),
|
|
},
|
|
{ role: "user", text: "Ship it." },
|
|
],
|
|
rubric: "Name pairing expiry and honor the explicit scope change.",
|
|
},
|
|
{
|
|
id: "review-umbrella",
|
|
source: "Existing subagent monitoring title scenario",
|
|
request: "Review subagent monitoring risks. A Codex roster issue is one finding.",
|
|
previousTitle: "Review subagent monitoring risks",
|
|
messages: [
|
|
{ role: "user", text: "Review subagent monitoring risks." },
|
|
{
|
|
role: "assistant",
|
|
text: "One finding is a stale Codex roster. " + "Roster detail. ".repeat(800),
|
|
},
|
|
{ role: "user", text: "Fix the findings and babysit CI." },
|
|
],
|
|
rubric: "Preserve the monitoring review scope. The previous title can stay unchanged.",
|
|
},
|
|
{
|
|
id: "long-opening",
|
|
source: "Title message truncation scenario",
|
|
request: "Investigate Android pairing while preserving the iOS flow.",
|
|
previousTitle: "Inspect logs",
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
text:
|
|
"Investigate Android pairing. " +
|
|
"Connection logs. ".repeat(800) +
|
|
" Preserve the iOS pairing flow.",
|
|
},
|
|
],
|
|
rubric: "Name Android pairing. Logs are supporting evidence.",
|
|
},
|
|
{
|
|
id: "research",
|
|
source: "Maintainer title generation request",
|
|
request: "How can we improve title generation in Hanzo Dev?",
|
|
previousTitle: "Research title gen improvements",
|
|
messages: [
|
|
{ role: "user", text: "How can we improve title gen further in Hanzo Dev?" },
|
|
{
|
|
role: "assistant",
|
|
text: "Prioritize user messages, refine vague titles once, and resolve PR subjects.",
|
|
},
|
|
{
|
|
role: "user",
|
|
text: "Make these changes and file a PR. Babysit until everything is green.",
|
|
},
|
|
],
|
|
rubric: "Keep title generation as the subject. Do not focus on filing the PR.",
|
|
},
|
|
{
|
|
id: "project-name",
|
|
source: "https://github.com/hanzoai/dev/pull/33418",
|
|
request: "Make hyperdrive init create a default config instead of crashing.",
|
|
projectTitle: "Hyperdrive",
|
|
previousTitle: "Hyperdrive Init Missing Config Crash",
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
text: "hyperdrive init crashes when ~/.hyperdrive/config.toml is missing. Make hyperdrive create a default config instead of throwing.",
|
|
},
|
|
{
|
|
role: "assistant",
|
|
text: "Hyperdrive's loadConfig now writes a default config.toml when the file is absent.",
|
|
},
|
|
],
|
|
rubric: "Name the default config on init. Leave out Hyperdrive, the project name.",
|
|
},
|
|
{
|
|
id: "custom-title-scheme",
|
|
source: "https://github.com/hanzoai/dev/pull/37565",
|
|
request: "Relay chat to the lead agent, titled with the user's naming scheme.",
|
|
previousTitle: "Relay chat to lead agent",
|
|
instructions: 'Title every thread "Owner: what it does (model effort)".',
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
text: "Make the relay forward every chat message to the lead agent so reviewers can see questions from subagents.",
|
|
},
|
|
],
|
|
rubric:
|
|
'Use the user\'s "Owner: what it does (model effort)" shape with all three parts, even past the default 8-word, 40-character limit. A bare short title fails.',
|
|
},
|
|
] satisfies ReadonlyArray<{
|
|
id: string;
|
|
source: string;
|
|
request: string;
|
|
projectTitle?: string;
|
|
previousTitle: string;
|
|
instructions?: string;
|
|
messages: ReadonlyArray<ThreadTitleMessage>;
|
|
rubric: string;
|
|
}>;
|