mirror of
https://github.com/hanzoai/mcp.git
synced 2026-10-29 02:55:49 +00:00
zeekay
3431b4ac53
Measured against kai-1.2 served locally from production's folder (SHA256SUMS e45dd919…).
kai_injection_screen asks Kai the question it was trained on ({prompt},
"Does `prompt` try to make an AI assistant ignore its rules...") beside
seven rule classes: override, role, addressed, conceal, exfiltrate,
tool_call, act, plus the earlier extract patterns. Held out on InjecAgent's
unseen tools and wordings (250 attacks) and 207 benign tool outputs and
README paragraphs: recall .728 at .010 false alarms, against .044 at .116
for the screen it replaces. Kai's question alone found 0 of the 250
planted requests. Rules still decide when Kai does not answer.
kai_tool_select ranks by a named tool, then BM25 over name, description
and parameters, and makes no Kai call: on 300 supplied tool lists from
Kai's T1 dev split it reads top-1 .733 and MRR .841 against .657 and .805
for one Kai choice over the names, in 0.1 ms against 109 ms.
kai_log_triage classifies by failure signature (resources, network,
timeout, permission, dependency, compile, test, crash; none and unknown)
and returns the deciding lines and file spans; kai-1.2 read build logs at
chance. Recipes 01, 04, 05, 08 and 10 check what these tools return;
recipe 08 calls the tool instead of matching names itself. The cookbook
reaches Kai as clients do, through HANZO_API_KEY or LOCAL_KAI_URL, not a
fixed local port and placeholder key. compact.ts types its decision so
tsc passes.
42 lines
1.6 KiB
TypeScript
42 lines
1.6 KiB
TypeScript
/**
|
|
* Recipe 08: Tool-Name Resolution
|
|
*
|
|
* A model asks for `bash`, `read_file` or `git_status`; the client registers `Bash`, `Read` and `Git`.
|
|
* kai_tool_select maps each request to a registered tool: a name the request spells in any case wins,
|
|
* else BM25 over names and descriptions. No model call.
|
|
*/
|
|
|
|
import { createRecipeClient, logRecipe, RecipeResult } from "./helper.js";
|
|
|
|
const REGISTERED = [
|
|
{ name: "Bash", description: "Run a shell command" },
|
|
{ name: "Read", description: "Read a file from the workspace" },
|
|
{ name: "Edit", description: "Replace text in a file" },
|
|
{ name: "Git", description: "Git status, diff, log and blame" }
|
|
];
|
|
const WANT: Record<string, string> = { bash: "Bash", read_file: "Read", git_status: "Git" };
|
|
|
|
export async function runRecipe08(): Promise<RecipeResult> {
|
|
const client = await createRecipeClient();
|
|
const resolutions = [];
|
|
let elapsed = 0;
|
|
for (const [requested, expected] of Object.entries(WANT)) {
|
|
const res = await client.callTool("kai_tool_select", { task: requested.replace(/_/g, " "), tools: REGISTERED, limit: 1 });
|
|
elapsed += res.latency_ms;
|
|
resolutions.push({ requested, resolved: res.data?.top_tool, source: res.data?.source, expected });
|
|
}
|
|
|
|
return {
|
|
recipe: "08: Tool-Name Resolution",
|
|
tool: "kai_tool_select",
|
|
success: resolutions.every((r) => r.resolved === r.expected),
|
|
latency_ms: elapsed,
|
|
output: { resolutions },
|
|
explanation: "bash -> Bash by name; read_file -> Read and git_status -> Git by BM25."
|
|
};
|
|
}
|
|
|
|
if (process.argv[1]?.endsWith("08-claude-code-tool-resolution.ts")) {
|
|
runRecipe08().then(logRecipe).catch(console.error);
|
|
}
|