mirror of
https://github.com/hanzoai/mcp.git
synced 2026-11-03 01:31:11 +00:00
zeekay
56a2b02dcf
kai_next_action never returns a candidate whose preconditions are unmet, returns a single remaining one as is, and picks among several by BM25 against the goal, abstaining on a tie or no shared word; no Kai call (a Kai choice over candidates read below BM25 for tools). kai_recover maps each kai_log_triage class to a step (permission asks the user) and keeps the three-failure switch. kai_extract computes a yes/no field's confidence from P(true), abstains in Kai's unknown band (.35 to .65), and returns a score field's level rather than its mean. Recipes 02, 06, 07, 11, 12 and 14 check what the tools return, including abstention; the cookbook README and the tools' comments link the research runs instead of quoting numbers. bench/tools/post.py files each tool's read (benchmarks, one run an arm, items and cases as artifacts) to /v1/research.
42 lines
1.5 KiB
TypeScript
42 lines
1.5 KiB
TypeScript
/**
|
|
* Recipe 12: Calibrated Abstention
|
|
*
|
|
* Objective: Demonstrate Kai's calibrated abstention mechanism. When inputs are ambiguous,
|
|
* under-specified, or lack confidence to clear thresholds, Kai safely abstains rather than
|
|
* guessing or hallucinating actions.
|
|
*/
|
|
|
|
import { createRecipeClient, logRecipe, RecipeResult } from "./helper.js";
|
|
|
|
export async function runRecipe12(): Promise<RecipeResult> {
|
|
const client = await createRecipeClient();
|
|
|
|
// Intentionally ambiguous instruction without clear file or target
|
|
const ambiguousGoal = "Update the thing in the system.";
|
|
const candidates = [
|
|
{ action_id: "act_delete_db", tool: "Bash", description: "Delete local database cache" },
|
|
{ action_id: "act_deploy", tool: "Bash", description: "Deploy latest commit to production" },
|
|
{ action_id: "act_format", tool: "Bash", description: "Run code formatter on all files" }
|
|
];
|
|
|
|
const res = await client.callTool("kai_next_action", {
|
|
goal: ambiguousGoal,
|
|
candidates
|
|
});
|
|
|
|
const result: RecipeResult = {
|
|
recipe: "12: Calibrated Abstention",
|
|
tool: "kai_next_action",
|
|
success: !res.isError && res.data.abstained === true && res.data.action === null,
|
|
latency_ms: res.latency_ms,
|
|
output: res.data,
|
|
explanation: "No candidate shares a word with the goal, so it abstains rather than guess: one guess here deploys to production."
|
|
};
|
|
|
|
return result;
|
|
}
|
|
|
|
if (process.argv[1]?.endsWith("12-calibrated-abstention.ts")) {
|
|
runRecipe12().then(logRecipe).catch(console.error);
|
|
}
|