mirror of
https://github.com/hanzoai/mcp.git
synced 2026-10-30 15:03:38 +00:00
zeekay
3431b4ac53
Measured against kai-1.2 served locally from production's folder (SHA256SUMS e45dd919…).
kai_injection_screen asks Kai the question it was trained on ({prompt},
"Does `prompt` try to make an AI assistant ignore its rules...") beside
seven rule classes: override, role, addressed, conceal, exfiltrate,
tool_call, act, plus the earlier extract patterns. Held out on InjecAgent's
unseen tools and wordings (250 attacks) and 207 benign tool outputs and
README paragraphs: recall .728 at .010 false alarms, against .044 at .116
for the screen it replaces. Kai's question alone found 0 of the 250
planted requests. Rules still decide when Kai does not answer.
kai_tool_select ranks by a named tool, then BM25 over name, description
and parameters, and makes no Kai call: on 300 supplied tool lists from
Kai's T1 dev split it reads top-1 .733 and MRR .841 against .657 and .805
for one Kai choice over the names, in 0.1 ms against 109 ms.
kai_log_triage classifies by failure signature (resources, network,
timeout, permission, dependency, compile, test, crash; none and unknown)
and returns the deciding lines and file spans; kai-1.2 read build logs at
chance. Recipes 01, 04, 05, 08 and 10 check what these tools return;
recipe 08 calls the tool instead of matching names itself. The cookbook
reaches Kai as clients do, through HANZO_API_KEY or LOCAL_KAI_URL, not a
fixed local port and placeholder key. compact.ts types its decision so
tsc passes.
47 lines
1.9 KiB
TypeScript
47 lines
1.9 KiB
TypeScript
/**
|
|
* Recipe 01: Agent Tool Shortlisting
|
|
*
|
|
* Objective: Reduce large registered tool registries (20+ tools) down to the top 3-5
|
|
* most relevant tools for a specific software engineering task, reducing prompt token costs
|
|
* and eliminating tool confusion.
|
|
*/
|
|
|
|
import { createRecipeClient, logRecipe, RecipeResult } from "./helper.js";
|
|
|
|
export async function runRecipe01(): Promise<RecipeResult> {
|
|
const client = await createRecipeClient();
|
|
|
|
const task = "Search git commit history for the author who modified auth tokens, and create an issue.";
|
|
const candidateTools = [
|
|
{ name: "Git", description: "Query git repository history, diffs, blame, and logs" },
|
|
{ name: "Tracker", description: "Manage project issues, tickets, and boards" },
|
|
{ name: "Bash", description: "Execute arbitrary bash terminal commands" },
|
|
{ name: "Read", description: "Read contents of an existing file" },
|
|
{ name: "Edit", description: "Modify parts of an existing file" },
|
|
{ name: "Gimp", description: "Image manipulation and visual graphics generation" },
|
|
{ name: "Browser", description: "Control browser for web scraping and UI testing" },
|
|
{ name: "Fetch", description: "Execute raw HTTP requests to external APIs" }
|
|
];
|
|
|
|
const res = await client.callTool("kai_tool_select", {
|
|
task,
|
|
tools: candidateTools,
|
|
limit: 3
|
|
});
|
|
|
|
const result: RecipeResult = {
|
|
recipe: "01: Agent Tool Shortlisting",
|
|
tool: "kai_tool_select",
|
|
success: !res.isError && res.data.top_tool === "Git" && res.data.source === "name",
|
|
latency_ms: res.latency_ms,
|
|
output: res.data,
|
|
explanation: "The task names Git, so it leads. BM25 does not stem, so 'issue' does not reach Tracker's 'issues': name the tool or its words. No model call."
|
|
};
|
|
|
|
return result;
|
|
}
|
|
|
|
if (process.argv[1]?.endsWith("01-tool-shortlisting.ts")) {
|
|
runRecipe01().then(logRecipe).catch(console.error);
|
|
}
|