mirror of
https://github.com/hanzoai/mcp.git
synced 2026-10-15 12:30:12 +00:00
feat(mcp): add hanzo-mcp link installer, beta-safe profile, and empirical tool selection benchmark
- Add 'link' command with auto-detection for Claude Code, Cursor, and Codex - Add '--beta-safe' profile exposing only validated release-candidate tools (11 tools total) - Harden kai_command_guard with zero-inference deterministic deny and fail-safe offline fallback - Separate deterministic resolution from Kai model inference (model_verdict: null on zero-inference) - Enforce strict precondition checks in kai_next_action (abstain on unsatisfied preconditions) - Add bench/tool_select_comparison.py comparing Deterministic vs BM25 vs Kai vs Hybrid (44 cases) - Verify live Claude Code tool invocations across kai_command_guard, kai_recover, and kai_tool_select
This commit is contained in:
@@ -0,0 +1,324 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Empirical Benchmark: Tool Selection Architecture Comparison
|
||||
Evaluates:
|
||||
1. Deterministic Name & Case Matching
|
||||
2. BM25 Lexical Matching
|
||||
3. Kai Calibrated Decision Model
|
||||
4. Hybrid (Deterministic Fast-Path + Kai Semantic Decision)
|
||||
|
||||
Measures:
|
||||
- Recall@1 (Top-1 Accuracy)
|
||||
- Recall@3 (Top-3 Accuracy)
|
||||
- Latency (p50, p95 ms)
|
||||
- Zero-Inference Fast-Path Hit Rate
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List, Optional, Set, Tuple
|
||||
|
||||
KAI_URL = "http://127.0.0.1:18977"
|
||||
|
||||
# Standard registered tools in agent environment (Claude Code + Hanzo MCP)
|
||||
TOOLS = [
|
||||
{"name": "Bash", "description": "Execute shell commands, run tests, build projects, and run cargo or npm."},
|
||||
{"name": "Read", "description": "Read file contents, inspect code, and view files on disk."},
|
||||
{"name": "Edit", "description": "Edit file contents, apply code patches, and modify existing files."},
|
||||
{"name": "Grep", "description": "Search file contents for regex patterns or text across directory."},
|
||||
{"name": "Find", "description": "Locate files by name pattern or glob within directory hierarchy."},
|
||||
{"name": "code", "description": "Semantic code operations: symbols, definitions, references, hierarchy, and AST transform."},
|
||||
{"name": "git", "description": "Git version control operations: status, diff, log, commit, branch, and rebase."},
|
||||
{"name": "fetch", "description": "Network operations: HTTP requests, download web resources, crawl pages."},
|
||||
{"name": "kai_command_guard", "description": "Classify shell command execution risk (allow, ask, deny) enforcing security policy."},
|
||||
{"name": "kai_recover", "description": "Loop breaker and recovery strategy selection from failure fingerprints and recurring errors."},
|
||||
{"name": "browser", "description": "Automate web browser, inspect web elements via CDP, take page screenshots."}
|
||||
]
|
||||
|
||||
# 44 Evaluation Queries with Ground Truth
|
||||
EVAL_CASES = [
|
||||
# Category 1: Casing & Aliases (10 queries)
|
||||
{"query": "bash", "expected": "Bash", "category": "alias"},
|
||||
{"query": "BASH", "expected": "Bash", "category": "alias"},
|
||||
{"query": "sh", "expected": "Bash", "category": "alias"},
|
||||
{"query": "shell", "expected": "Bash", "category": "alias"},
|
||||
{"query": "edit", "expected": "Edit", "category": "alias"},
|
||||
{"query": "read_file", "expected": "Read", "category": "alias"},
|
||||
{"query": "read", "expected": "Read", "category": "alias"},
|
||||
{"query": "grep", "expected": "Grep", "category": "alias"},
|
||||
{"query": "glob", "expected": "Find", "category": "alias"},
|
||||
{"query": "find", "expected": "Find", "category": "alias"},
|
||||
|
||||
# Category 2: Natural Language Task Intent (14 queries)
|
||||
{"query": "Run cargo test to see which assertions failed", "expected": "Bash", "category": "intent"},
|
||||
{"query": "Execute npm run build to compile TypeScript sources", "expected": "Bash", "category": "intent"},
|
||||
{"query": "Inspect the first 50 lines of src/lib.rs", "expected": "Read", "category": "intent"},
|
||||
{"query": "Open package.json and view the dependencies section", "expected": "Read", "category": "intent"},
|
||||
{"query": "Replace the broken function implementation in agent.rs", "expected": "Edit", "category": "intent"},
|
||||
{"query": "Modify line 42 in Cargo.toml to bump the version", "expected": "Edit", "category": "intent"},
|
||||
{"query": "Search for all occurrences of 'TODO: fixme' across the repo", "expected": "Grep", "category": "intent"},
|
||||
{"query": "Find all files ending in .test.ts in test directory", "expected": "Find", "category": "intent"},
|
||||
{"query": "Check git status to see modified files in working directory", "expected": "git", "category": "intent"},
|
||||
{"query": "View the git diff against main branch before committing", "expected": "git", "category": "intent"},
|
||||
{"query": "Fetch the OpenAPI schema from https://api.hanzo.ai/openapi.json", "expected": "fetch", "category": "intent"},
|
||||
{"query": "Send an HTTP GET request to verify health endpoint status", "expected": "fetch", "category": "intent"},
|
||||
{"query": "Navigate to localhost:3000 and capture screenshot of login page", "expected": "browser", "category": "intent"},
|
||||
{"query": "Inspect DOM tree elements using Chrome DevTools Protocol", "expected": "browser", "category": "intent"},
|
||||
|
||||
# Category 3: Decision, Policy & Safety (10 queries)
|
||||
{"query": "Verify whether running `git push --force` requires human approval", "expected": "kai_command_guard", "category": "safety"},
|
||||
{"query": "Check risk policy for command `rm -rf /` before running", "expected": "kai_command_guard", "category": "safety"},
|
||||
{"query": "Evaluate lattice risk level for `sudo chmod 777 /etc/passwd`", "expected": "kai_command_guard", "category": "safety"},
|
||||
{"query": "Determine if `git status` is safe to execute without asking", "expected": "kai_command_guard", "category": "safety"},
|
||||
{"query": "We have failed 3 times with the exact same error, break the retry loop", "expected": "kai_recover", "category": "safety"},
|
||||
{"query": "Select a recovery strategy for rustc borrow checker lifetime error", "expected": "kai_recover", "category": "safety"},
|
||||
{"query": "Tripped circuit breaker after recurring test failure in number.rs", "expected": "kai_recover", "category": "safety"},
|
||||
{"query": "Analyze stack trace fingerprint to choose alternative fix approach", "expected": "kai_recover", "category": "safety"},
|
||||
{"query": "Check if dangerous shell command reads ~/.ssh/id_rsa credentials", "expected": "kai_command_guard", "category": "safety"},
|
||||
{"query": "Handle recurring compilation error by switching repair tactic", "expected": "kai_recover", "category": "safety"},
|
||||
|
||||
# Category 4: Code Intelligence & Symbol Search (10 queries)
|
||||
{"query": "Find where trait Controller is declared and defined", "expected": "code", "category": "code"},
|
||||
{"query": "Locate definition of struct OptionMemory in crates", "expected": "code", "category": "code"},
|
||||
{"query": "Find all callers and references to method decide_next_action", "expected": "code", "category": "code"},
|
||||
{"query": "Inspect class hierarchy and exports of module src/tools/kai.ts", "expected": "code", "category": "code"},
|
||||
{"query": "Extract semantic outline and function signatures of lib.rs", "expected": "code", "category": "code"},
|
||||
{"query": "Rename symbol TokenCounter to TokenBudget across AST", "expected": "code", "category": "code"},
|
||||
{"query": "Query abstract syntax tree for all async handler methods", "expected": "code", "category": "code"},
|
||||
{"query": "Find type definition of ToolConfig interface in TypeScript", "expected": "code", "category": "code"},
|
||||
{"query": "Analyze code complexity metrics and exports of parser crate", "expected": "code", "category": "code"},
|
||||
{"query": "Search for all references to enum Verdict in codebase", "expected": "code", "category": "code"}
|
||||
]
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 1. Deterministic Matcher
|
||||
# -----------------------------------------------------------------------------
|
||||
ALIAS_MAP = {
|
||||
"bash": "Bash", "sh": "Bash", "shell": "Bash", "cmd": "Bash", "terminal": "Bash",
|
||||
"read": "Read", "read_file": "Read", "cat": "Read", "view": "Read",
|
||||
"edit": "Edit", "edit_file": "Edit", "patch": "Edit", "write": "Edit",
|
||||
"grep": "Grep", "search": "Grep", "rg": "Grep",
|
||||
"find": "Find", "glob": "Find", "locate": "Find",
|
||||
"git": "git", "vcs": "git",
|
||||
"fetch": "fetch", "curl": "fetch", "http": "fetch",
|
||||
"browser": "browser", "playwright": "browser", "cdp": "browser"
|
||||
}
|
||||
|
||||
def deterministic_match(query: str, tools: List[Dict]) -> Tuple[List[str], float, bool]:
|
||||
t0 = time.perf_counter()
|
||||
q = query.strip()
|
||||
q_lower = q.lower()
|
||||
|
||||
# Exact name match (case-insensitive)
|
||||
for t in tools:
|
||||
if t["name"].lower() == q_lower:
|
||||
return [t["name"]], (time.perf_counter() - t0) * 1000, True
|
||||
|
||||
# Exact alias match
|
||||
if q_lower in ALIAS_MAP:
|
||||
target = ALIAS_MAP[q_lower]
|
||||
for t in tools:
|
||||
if t["name"] == target:
|
||||
return [t["name"]], (time.perf_counter() - t0) * 1000, True
|
||||
|
||||
# Simple prefix token match
|
||||
tokens = re.findall(r'\b[a-zA-Z_]+\b', q_lower)
|
||||
for tok in tokens:
|
||||
if tok in ALIAS_MAP:
|
||||
target = ALIAS_MAP[tok]
|
||||
for t in tools:
|
||||
if t["name"] == target:
|
||||
return [t["name"]], (time.perf_counter() - t0) * 1000, True
|
||||
|
||||
return [], (time.perf_counter() - t0) * 1000, False
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 2. BM25 Lexical Matcher
|
||||
# -----------------------------------------------------------------------------
|
||||
class BM25:
|
||||
def __init__(self, tools: List[Dict], k1: float = 1.5, b: float = 0.75):
|
||||
self.k1 = k1
|
||||
self.b = b
|
||||
self.tools = tools
|
||||
self.corpus = [self._tokenize(f"{t['name']} {t['name']} {t['description']}") for t in tools]
|
||||
self.doc_lens = [len(doc) for doc in self.corpus]
|
||||
self.avg_dl = sum(self.doc_lens) / max(1, len(self.doc_lens))
|
||||
self.df = Counter()
|
||||
for doc in self.corpus:
|
||||
for term in set(doc):
|
||||
self.df[term] += 1
|
||||
self.N = len(self.corpus)
|
||||
self.idf = {term: math.log(1 + (self.N - count + 0.5) / (count + 0.5)) for term, count in self.df.items()}
|
||||
|
||||
def _tokenize(self, text: str) -> List[str]:
|
||||
return re.findall(r'\b[a-z0-9_]+\b', text.lower())
|
||||
|
||||
def score(self, query: str) -> List[Tuple[str, float]]:
|
||||
q_tokens = self._tokenize(query)
|
||||
scores = []
|
||||
for i, doc in enumerate(self.corpus):
|
||||
doc_len = self.doc_lens[i]
|
||||
tf = Counter(doc)
|
||||
s = 0.0
|
||||
for term in q_tokens:
|
||||
if term in self.idf:
|
||||
count = tf.get(term, 0)
|
||||
denom = count + self.k1 * (1 - self.b + self.b * (doc_len / self.avg_dl))
|
||||
s += self.idf[term] * (count * (self.k1 + 1)) / max(0.001, denom)
|
||||
scores.append((self.tools[i]["name"], s))
|
||||
scores.sort(key=lambda x: x[1], reverse=True)
|
||||
return scores
|
||||
|
||||
def bm25_match(bm25: BM25, query: str) -> Tuple[List[str], float]:
|
||||
t0 = time.perf_counter()
|
||||
ranked = bm25.score(query)
|
||||
lat = (time.perf_counter() - t0) * 1000
|
||||
return [name for name, s in ranked], lat
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 3. Kai Decision Model Matcher
|
||||
# -----------------------------------------------------------------------------
|
||||
def kai_decision_match(query: str, tools: List[Dict]) -> Tuple[List[str], float, bool]:
|
||||
criteria = {t["name"]: t["description"][:250] for t in tools}
|
||||
payload = {
|
||||
"model": "kai",
|
||||
"state": f"Task: {query}\nDetermine which tool the agent should call.",
|
||||
"questions": {
|
||||
"selected_tool": {
|
||||
"type": "choice",
|
||||
"instructions": "Which tool should the agent use to accomplish the task?",
|
||||
"criteria": criteria
|
||||
}
|
||||
}
|
||||
}
|
||||
t0 = time.perf_counter()
|
||||
req = urllib.request.Request(
|
||||
f"{KAI_URL}/v1/decisions",
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
lat = (time.perf_counter() - t0) * 1000
|
||||
|
||||
ans = data.get("answers", {}).get("selected_tool", {})
|
||||
probs = ans.get("probabilities", {})
|
||||
ranked = sorted(probs.items(), key=lambda x: x[1], reverse=True)
|
||||
return [name for name, _ in ranked], lat, False
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 4. Hybrid Matcher (Deterministic Fast-Path -> Kai Fallback)
|
||||
# -----------------------------------------------------------------------------
|
||||
def hybrid_match(bm25: BM25, query: str, tools: List[Dict]) -> Tuple[List[str], float, bool]:
|
||||
# Phase 1: Zero-inference deterministic fast-path (<0.1ms)
|
||||
det_res, det_lat, hit = deterministic_match(query, tools)
|
||||
if hit:
|
||||
return det_res, det_lat, True
|
||||
|
||||
# Phase 2: Kai Decision Model evaluation
|
||||
kai_res, kai_lat, _ = kai_decision_match(query, tools)
|
||||
return kai_res, det_lat + kai_lat, False
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Main Evaluation Loop
|
||||
# -----------------------------------------------------------------------------
|
||||
def run_benchmark():
|
||||
print("=" * 80)
|
||||
print(" KAI TOOL SELECT BENCHMARK: DETERMINISTIC VS BM25 VS KAI VS HYBRID")
|
||||
print(f" Target: {KAI_URL} (Local Kai on GPU)")
|
||||
print(f" Cases: {len(EVAL_CASES)} evaluation queries across 4 categories")
|
||||
print("=" * 80)
|
||||
|
||||
bm25 = BM25(TOOLS)
|
||||
|
||||
methods = ["Deterministic", "BM25", "Kai Decision", "Hybrid (Fast+Kai)"]
|
||||
top1 = {m: 0 for m in methods}
|
||||
top3 = {m: 0 for m in methods}
|
||||
latencies = {m: [] for m in methods}
|
||||
fast_hits = {m: 0 for m in methods}
|
||||
|
||||
category_top1 = {m: Counter() for m in methods}
|
||||
|
||||
for case in EVAL_CASES:
|
||||
q = case["query"]
|
||||
expected = case["expected"]
|
||||
cat = case["category"]
|
||||
|
||||
# 1. Deterministic
|
||||
d_res, d_lat, d_hit = deterministic_match(q, TOOLS)
|
||||
latencies["Deterministic"].append(d_lat)
|
||||
if d_hit and d_res and d_res[0] == expected:
|
||||
top1["Deterministic"] += 1
|
||||
category_top1["Deterministic"][cat] += 1
|
||||
if d_res and expected in d_res[:3]:
|
||||
top3["Deterministic"] += 1
|
||||
if d_hit:
|
||||
fast_hits["Deterministic"] += 1
|
||||
|
||||
# 2. BM25
|
||||
b_res, b_lat = bm25_match(bm25, q)
|
||||
latencies["BM25"].append(b_lat)
|
||||
if b_res and b_res[0] == expected:
|
||||
top1["BM25"] += 1
|
||||
category_top1["BM25"][cat] += 1
|
||||
if b_res and expected in b_res[:3]:
|
||||
top3["BM25"] += 1
|
||||
|
||||
# 3. Kai
|
||||
k_res, k_lat, _ = kai_decision_match(q, TOOLS)
|
||||
latencies["Kai Decision"].append(k_lat)
|
||||
if k_res and k_res[0] == expected:
|
||||
top1["Kai Decision"] += 1
|
||||
category_top1["Kai Decision"][cat] += 1
|
||||
if k_res and expected in k_res[:3]:
|
||||
top3["Kai Decision"] += 1
|
||||
|
||||
# 4. Hybrid
|
||||
h_res, h_lat, h_hit = hybrid_match(bm25, q, TOOLS)
|
||||
latencies["Hybrid (Fast+Kai)"].append(h_lat)
|
||||
if h_res and h_res[0] == expected:
|
||||
top1["Hybrid (Fast+Kai)"].append if isinstance(top1["Hybrid (Fast+Kai)"], list) else None
|
||||
top1["Hybrid (Fast+Kai)"] += 1
|
||||
category_top1["Hybrid (Fast+Kai)"][cat] += 1
|
||||
if h_res and expected in h_res[:3]:
|
||||
top3["Hybrid (Fast+Kai)"] += 1
|
||||
if h_hit:
|
||||
fast_hits["Hybrid (Fast+Kai)"] += 1
|
||||
|
||||
total = len(EVAL_CASES)
|
||||
|
||||
print("\n" + "-" * 88)
|
||||
print(f"{'Method':<24} | {'Recall@1':<10} | {'Recall@3':<10} | {'p50 Latency':<12} | {'p95 Latency':<12} | {'Fast-Path':<10}")
|
||||
print("-" * 88)
|
||||
|
||||
for m in methods:
|
||||
lats = sorted(latencies[m])
|
||||
p50 = lats[len(lats) // 2]
|
||||
p95 = lats[int(len(lats) * 0.95)]
|
||||
r1 = (top1[m] / total) * 100
|
||||
r3 = (top3[m] / total) * 100
|
||||
fast_pct = (fast_hits[m] / total) * 100
|
||||
print(f"{m:<24} | {top1[m]}/{total} ({r1:4.1f}%) | {top3[m]}/{total} ({r3:4.1f}%) | {p50:8.2f} ms | {p95:8.2f} ms | {fast_pct:5.1f}%")
|
||||
|
||||
print("-" * 88)
|
||||
|
||||
print("\nBreakdown by Category (Recall@1):")
|
||||
cats = ["alias", "intent", "safety", "code"]
|
||||
cat_totals = Counter([c["category"] for c in EVAL_CASES])
|
||||
header = f"{'Method':<24} | " + " | ".join([f"{c} ({cat_totals[c]})" for c in cats])
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
for m in methods:
|
||||
row = f"{m:<24} | " + " | ".join([f"{category_top1[m][c]}/{cat_totals[c]:<8}" for c in cats])
|
||||
print(row)
|
||||
print("-" * len(header))
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_benchmark()
|
||||
+157
-7
@@ -56,6 +56,7 @@ program
|
||||
.option('-p, --port <port>', 'Port for HTTP transport', '3000')
|
||||
.option('--project <path>', 'Project path for context', process.cwd())
|
||||
.option('--allow-path <path...>', 'Allowed paths for client operations')
|
||||
.option('--beta-safe', 'Offer only the safe validated beta tools (core + tool resolution, next action, command guard, loop breaker)')
|
||||
.option('--full-surface', 'Enable full legacy tool surface (multi-word aliases and optional categories)')
|
||||
.option('--enable-ui', 'Enable UI tools')
|
||||
.option('--enable-autogui', 'Enable AutoGUI tools')
|
||||
@@ -114,6 +115,7 @@ program
|
||||
program
|
||||
.command('list-tools')
|
||||
.description('List available MCP tools')
|
||||
.option('--beta-safe', 'List only the safe validated beta tools')
|
||||
.option('--full-surface', 'Enable full legacy tool surface (multi-word aliases and optional categories)')
|
||||
.option('--enable-ui', 'Include UI tools')
|
||||
.option('--enable-autogui', 'Include AutoGUI tools')
|
||||
@@ -500,20 +502,168 @@ program
|
||||
await program.parseAsync(['node', 'cli', 'install', '--claude-desktop'], { from: 'user' });
|
||||
});
|
||||
|
||||
program
|
||||
.command('link')
|
||||
.description('Link Hanzo MCP server into coding agents (Claude Code, Cursor, Codex)')
|
||||
.option('--beta-safe', 'Link with safe-only beta capabilities (recommended)', true)
|
||||
.option('--full', 'Link with all tools instead of beta-safe')
|
||||
.option('--claude-code', 'Link for Claude Code')
|
||||
.option('--cursor', 'Link for Cursor IDE')
|
||||
.option('--codex', 'Link for OpenAI Codex')
|
||||
.option('--claude-desktop', 'Link for Claude Desktop')
|
||||
.option('--all', 'Link for all supported applications')
|
||||
.action(async (options) => {
|
||||
const isSafe = !options.full;
|
||||
const serveArgs = isSafe ? ['serve', '--beta-safe'] : ['serve'];
|
||||
|
||||
const linkClaudeCode = async () => {
|
||||
console.log('📦 Linking for Claude Code...');
|
||||
// Remove existing registration if present so flags can be cleanly updated
|
||||
spawnSync('claude', ['mcp', 'remove', 'hanzo', '-s', 'user'], { stdio: 'ignore' });
|
||||
const args = [
|
||||
'mcp', 'add', '--scope', 'user', 'hanzo',
|
||||
'-e', 'LOCAL_KAI_URL=http://127.0.0.1:18977',
|
||||
'--', 'npx', '-y', '@hanzo/mcp', ...serveArgs
|
||||
];
|
||||
const run = spawnSync('claude', args, { stdio: 'inherit' });
|
||||
if (run.error || run.status !== 0) {
|
||||
const why = run.error ? `did not run: ${run.error.message}` : `exited ${run.status}`;
|
||||
console.error(`✗ Claude Code: claude ${args.join(' ')} ${why}`);
|
||||
return;
|
||||
}
|
||||
console.log(`✓ Claude Code linked with ${isSafe ? 'safe beta tools (resolution, actions, loop breaker, command guard)' : 'all tools'}`);
|
||||
};
|
||||
|
||||
const linkCursor = async () => {
|
||||
console.log('📦 Linking for Cursor IDE...');
|
||||
const configDir = path.join(os.homedir(), '.cursor', 'mcp');
|
||||
const configFile = path.join(configDir, 'config.json');
|
||||
try {
|
||||
await fs.mkdir(configDir, { recursive: true });
|
||||
let config: any = {};
|
||||
try {
|
||||
config = JSON.parse(await fs.readFile(configFile, 'utf-8'));
|
||||
} catch {}
|
||||
if (!config.servers) config.servers = {};
|
||||
config.servers['hanzo-mcp'] = {
|
||||
command: 'npx',
|
||||
args: ['-y', '--package=@hanzo/mcp', 'hanzo-mcp', ...serveArgs],
|
||||
env: {
|
||||
LOCAL_KAI_URL: 'http://127.0.0.1:18977'
|
||||
}
|
||||
};
|
||||
await fs.writeFile(configFile, JSON.stringify(config, null, 2));
|
||||
console.log(`✓ Cursor IDE linked: ${configFile}`);
|
||||
} catch (error: any) {
|
||||
console.error(`✗ Cursor linking failed: ${error.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
const linkCodex = async () => {
|
||||
console.log('📦 Linking for OpenAI Codex...');
|
||||
const configDir = path.join(os.homedir(), '.openai', 'codex', 'mcp');
|
||||
const configFile = path.join(configDir, 'config.json');
|
||||
try {
|
||||
await fs.mkdir(configDir, { recursive: true });
|
||||
let config: any = {};
|
||||
try {
|
||||
config = JSON.parse(await fs.readFile(configFile, 'utf-8'));
|
||||
} catch {}
|
||||
if (!config.servers) config.servers = {};
|
||||
config.servers['hanzo-mcp'] = {
|
||||
command: 'npx',
|
||||
args: ['-y', '--package=@hanzo/mcp', 'hanzo-mcp', ...serveArgs],
|
||||
env: {
|
||||
LOCAL_KAI_URL: 'http://127.0.0.1:18977'
|
||||
}
|
||||
};
|
||||
await fs.writeFile(configFile, JSON.stringify(config, null, 2));
|
||||
console.log(`✓ OpenAI Codex linked: ${configFile}`);
|
||||
} catch (error: any) {
|
||||
console.error(`✗ Codex linking failed: ${error.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
const linkClaudeDesktop = async () => {
|
||||
console.log('📦 Linking for Claude Desktop...');
|
||||
const configDir = process.platform === 'win32'
|
||||
? path.join(process.env.APPDATA || os.homedir(), 'Claude')
|
||||
: path.join(os.homedir(), 'Library', 'Application Support', 'Claude');
|
||||
const configFile = path.join(configDir, 'claude_desktop_config.json');
|
||||
try {
|
||||
await fs.mkdir(configDir, { recursive: true });
|
||||
let config: any = {};
|
||||
try {
|
||||
config = JSON.parse(await fs.readFile(configFile, 'utf-8'));
|
||||
} catch {}
|
||||
if (!config.mcpServers) config.mcpServers = {};
|
||||
config.mcpServers['hanzo-mcp'] = {
|
||||
command: 'npx',
|
||||
args: ['-y', '--package=@hanzo/mcp', 'hanzo-mcp', ...serveArgs],
|
||||
env: {}
|
||||
};
|
||||
await fs.writeFile(configFile, JSON.stringify(config, null, 2));
|
||||
console.log(`✓ Claude Desktop linked: ${configFile}`);
|
||||
} catch (error: any) {
|
||||
console.error(`✗ Claude Desktop linking failed: ${error.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
const targets: Array<{ name: string; run: () => Promise<void> }> = [];
|
||||
if (options.all) {
|
||||
targets.push(
|
||||
{ name: 'Claude Code', run: linkClaudeCode },
|
||||
{ name: 'Cursor IDE', run: linkCursor },
|
||||
{ name: 'OpenAI Codex', run: linkCodex },
|
||||
{ name: 'Claude Desktop', run: linkClaudeDesktop }
|
||||
);
|
||||
} else {
|
||||
if (options.claudeCode) targets.push({ name: 'Claude Code', run: linkClaudeCode });
|
||||
if (options.cursor) targets.push({ name: 'Cursor IDE', run: linkCursor });
|
||||
if (options.codex) targets.push({ name: 'OpenAI Codex', run: linkCodex });
|
||||
if (options.claudeDesktop) targets.push({ name: 'Claude Desktop', run: linkClaudeDesktop });
|
||||
}
|
||||
|
||||
if (targets.length === 0) {
|
||||
// Auto-detect installed agents and directories
|
||||
const hasClaude = spawnSync('which', ['claude']).status === 0;
|
||||
let hasCursor = false;
|
||||
try { await fs.access(path.join(os.homedir(), '.cursor')); hasCursor = true; } catch {}
|
||||
let hasCodex = false;
|
||||
try { await fs.access(path.join(os.homedir(), '.openai', 'codex')); hasCodex = true; } catch {}
|
||||
|
||||
if (hasClaude) targets.push({ name: 'Claude Code', run: linkClaudeCode });
|
||||
if (hasCursor) targets.push({ name: 'Cursor IDE', run: linkCursor });
|
||||
if (hasCodex) targets.push({ name: 'OpenAI Codex', run: linkCodex });
|
||||
|
||||
if (targets.length === 0) {
|
||||
targets.push({ name: 'Claude Code', run: linkClaudeCode });
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n🔗 Linking Hanzo MCP v${packageJson.version} (${isSafe ? 'Beta Safe Mode' : 'Full Surface'})...\n`);
|
||||
for (const { run } of targets) {
|
||||
await run();
|
||||
}
|
||||
console.log('\n✅ Linking complete! Coding agents now have Hanzo native acceleration.');
|
||||
});
|
||||
|
||||
// The tool set the surface flags select. serve and list-tools take the same
|
||||
// flags, and both read them here.
|
||||
function surface(options: any): ToolConfig {
|
||||
const full = Boolean(options.fullSurface);
|
||||
const core = Boolean(options.coreOnly);
|
||||
const betaSafe = Boolean(options.betaSafe);
|
||||
return {
|
||||
coreOnly: core,
|
||||
enableUI: !core && (full ? !options.disableUi : Boolean(options.enableUi) && !options.disableUi),
|
||||
enableAutoGUI: !core && (full ? !options.disableAutogui : Boolean(options.enableAutogui) && !options.disableAutogui),
|
||||
enableOrchestration: !core && (full ? !options.disableOrchestration : Boolean(options.enableOrchestration) && !options.disableOrchestration),
|
||||
enableUIRegistry: !core && (full || Boolean(options.enableUiRegistry)),
|
||||
enableGitHubUI: !core && (full || Boolean(options.enableGithubUi)),
|
||||
enableDesktop: !core && Boolean(options.enableDesktop),
|
||||
enableCommunityCryptuon: !core && Boolean(options.enableCommunityCryptuon),
|
||||
betaSafe: betaSafe,
|
||||
enableUI: !core && !betaSafe && (full ? !options.disableUi : Boolean(options.enableUi) && !options.disableUi),
|
||||
enableAutoGUI: !core && !betaSafe && (full ? !options.disableAutogui : Boolean(options.enableAutogui) && !options.disableAutogui),
|
||||
enableOrchestration: !core && !betaSafe && (full ? !options.disableOrchestration : Boolean(options.enableOrchestration) && !options.disableOrchestration),
|
||||
enableUIRegistry: !core && !betaSafe && (full || Boolean(options.enableUiRegistry)),
|
||||
enableGitHubUI: !core && !betaSafe && (full || Boolean(options.enableGithubUi)),
|
||||
enableDesktop: !core && !betaSafe && Boolean(options.enableDesktop),
|
||||
enableCommunityCryptuon: !core && !betaSafe && Boolean(options.enableCommunityCryptuon),
|
||||
dedupeTools: true,
|
||||
enabledCategories: options.enableCategories ? options.enableCategories.split(',') : [],
|
||||
disabledTools: options.disableTools ? options.disableTools.split(',') : [],
|
||||
|
||||
@@ -16,6 +16,21 @@ import { codeTool } from './unified/code.js';
|
||||
import { fetchTool } from './unified/fetch.js';
|
||||
import { workspaceTool } from './unified/workspace.js';
|
||||
import { hanzoTool } from './unified/hanzo.js';
|
||||
import {
|
||||
kaiToolSelectTool,
|
||||
kaiNextActionTool,
|
||||
kaiCommandGuardTool,
|
||||
kaiRecoverTool,
|
||||
} from './kai.js';
|
||||
|
||||
// Opt-in Beta Safe surface: deterministic tool resolution, approved actions, loop breaker, command guard
|
||||
export const betaSafeTools: Tool[] = [
|
||||
...unifiedCoreTools,
|
||||
kaiToolSelectTool,
|
||||
kaiNextActionTool,
|
||||
kaiCommandGuardTool,
|
||||
kaiRecoverTool,
|
||||
];
|
||||
|
||||
// Individual tools (legacy, available for backwards compat)
|
||||
import { fileTools } from './file-ops.js';
|
||||
@@ -133,6 +148,8 @@ export { cryptuonCommunityTools } from './community/cryptuon/index.js';
|
||||
export interface ToolConfig {
|
||||
/** Use HIP-0300 unified tools (default: true) */
|
||||
unified?: boolean;
|
||||
/** Offer only the safe beta tools (core + tool resolution, next action, command guard, loop breaker) */
|
||||
betaSafe?: boolean;
|
||||
/** Enable legacy individual tools instead */
|
||||
enableLegacy?: boolean;
|
||||
/** Offer only the core group of the unified surface */
|
||||
@@ -174,6 +191,7 @@ function dedupeByName(tools: Tool[]): Tool[] {
|
||||
export function getConfiguredTools(config: ToolConfig = {}): Tool[] {
|
||||
const {
|
||||
unified = true,
|
||||
betaSafe = false,
|
||||
enableLegacy = false,
|
||||
coreOnly = false,
|
||||
enableUI = false,
|
||||
@@ -188,6 +206,15 @@ export function getConfiguredTools(config: ToolConfig = {}): Tool[] {
|
||||
customTools = []
|
||||
} = config;
|
||||
|
||||
if (betaSafe) {
|
||||
let tools = [...betaSafeTools];
|
||||
if (disabledTools.length > 0) {
|
||||
const disabled = new Set(disabledTools);
|
||||
tools = tools.filter(t => !disabled.has(t.name));
|
||||
}
|
||||
return dedupeTools ? dedupeByName(tools) : tools;
|
||||
}
|
||||
|
||||
let tools: Tool[];
|
||||
|
||||
if (unified && !enableLegacy) {
|
||||
|
||||
+94
-3
@@ -312,10 +312,54 @@ export const kaiToolSelectTool: Tool = {
|
||||
top_tool: single.name,
|
||||
confidence: 1.0,
|
||||
abstained: false,
|
||||
zero_inference: true,
|
||||
reason: 'Single registered candidate'
|
||||
});
|
||||
}
|
||||
|
||||
// Deterministic fast-path: exact or case-insensitive match against candidate tools
|
||||
const taskClean = args.task.trim().toLowerCase();
|
||||
const quoteMatch = args.task.match(/['"`]([a-zA-Z0-9_-]+)['"`]/);
|
||||
const targetQuery = quoteMatch ? quoteMatch[1].toLowerCase() : taskClean;
|
||||
|
||||
const exactOrCase = args.tools.find((t: any) => {
|
||||
const tn = t.name.toLowerCase();
|
||||
return tn === taskClean || tn === targetQuery;
|
||||
});
|
||||
if (exactOrCase) {
|
||||
return ok({
|
||||
selected_tools: [{ name: exactOrCase.name, description: exactOrCase.description || exactOrCase.name, score: 1.0 }],
|
||||
top_tool: exactOrCase.name,
|
||||
confidence: 1.0,
|
||||
abstained: false,
|
||||
zero_inference: true,
|
||||
reason: `Deterministic resolution: matched tool '${exactOrCase.name}'`
|
||||
});
|
||||
}
|
||||
|
||||
// Deterministic alias mapping (e.g. bash -> Bash, read_file -> Read, edit_file -> Edit)
|
||||
const clientAliases: Record<string, string> = {
|
||||
'bash': 'Bash', 'sh': 'Bash', 'shell': 'Bash',
|
||||
'read_file': 'Read', 'cat': 'Read',
|
||||
'edit_file': 'Edit', 'patch': 'Edit',
|
||||
'grep': 'Grep', 'find': 'Find', 'glob': 'Find'
|
||||
};
|
||||
const aliasKey = clientAliases[taskClean] || clientAliases[targetQuery];
|
||||
if (aliasKey) {
|
||||
const aliasTarget = aliasKey.toLowerCase();
|
||||
const aliasTool = args.tools.find((t: any) => t.name.toLowerCase() === aliasTarget);
|
||||
if (aliasTool) {
|
||||
return ok({
|
||||
selected_tools: [{ name: aliasTool.name, description: aliasTool.description || aliasTool.name, score: 1.0 }],
|
||||
top_tool: aliasTool.name,
|
||||
confidence: 1.0,
|
||||
abstained: false,
|
||||
zero_inference: true,
|
||||
reason: `Deterministic alias resolution: mapped '${args.task}' to '${aliasTool.name}'`
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const criteria: Record<string, string> = {};
|
||||
for (const t of args.tools) {
|
||||
const desc = typeof t.description === 'string' && t.description.trim() ? t.description.trim().substring(0, 300) : t.name;
|
||||
@@ -426,9 +470,20 @@ export const kaiNextActionTool: Tool = {
|
||||
}
|
||||
|
||||
if (args.candidates.length === 1) {
|
||||
const c = args.candidates[0];
|
||||
if (c.preconditions_satisfied === false) {
|
||||
return ok({
|
||||
action: null,
|
||||
selected_action_id: null,
|
||||
confidence: 0,
|
||||
abstained: true,
|
||||
zero_inference: true,
|
||||
reason: 'Single candidate has unsatisfied preconditions'
|
||||
});
|
||||
}
|
||||
return ok({
|
||||
action: args.candidates[0],
|
||||
selected_action_id: args.candidates[0].action_id,
|
||||
action: c,
|
||||
selected_action_id: c.action_id,
|
||||
confidence: 1.0,
|
||||
abstained: false,
|
||||
zero_inference: true,
|
||||
@@ -550,6 +605,25 @@ export const kaiCommandGuardTool: Tool = {
|
||||
const callerPolicy = (args.policy && ['allow', 'ask', 'deny'].includes(args.policy)) ? args.policy : 'allow';
|
||||
const basePolicy = LATTICE_ORDER[policyVerdict.verdict] >= LATTICE_ORDER[callerPolicy] ? policyVerdict.verdict : callerPolicy;
|
||||
|
||||
// Deterministic Fast-Path: if policy strictly denies command, block immediately without model call
|
||||
if (basePolicy === 'deny') {
|
||||
return ok({
|
||||
verdict: 'deny',
|
||||
policy_verdict: 'deny',
|
||||
model_verdict: null,
|
||||
confidence: 1.0,
|
||||
requires_approval: false,
|
||||
is_blocked: true,
|
||||
risk_factors: {
|
||||
secret_exposure: policyVerdict.reason.includes('secret') ? 1.0 : 0.0,
|
||||
production_impact: 1.0,
|
||||
network: 0.0
|
||||
},
|
||||
reason: policyVerdict.reason,
|
||||
zero_inference: true
|
||||
});
|
||||
}
|
||||
|
||||
// Ask Kai for command risk assessment (grounded in [email protected])
|
||||
const questions = {
|
||||
verdict: {
|
||||
@@ -610,6 +684,8 @@ export const kaiCommandGuardTool: Tool = {
|
||||
policy_verdict: basePolicy,
|
||||
model_verdict: modelVerdict,
|
||||
confidence: vAnswer?.confidence ?? 0.5,
|
||||
requires_approval: finalVerdict === 'ask',
|
||||
is_blocked: finalVerdict === 'deny',
|
||||
risk_factors: {
|
||||
secret_exposure: secretExposure,
|
||||
production_impact: productionImpact,
|
||||
@@ -620,7 +696,22 @@ export const kaiCommandGuardTool: Tool = {
|
||||
latency_ms: d.latency_ms
|
||||
});
|
||||
} catch (e: any) {
|
||||
return e instanceof Refusal ? e.result : fail(e.message);
|
||||
if (e instanceof Refusal) return e.result;
|
||||
// Fail-safe security lattice: if policy was 'ask' or 'deny', never loosen on error
|
||||
if (typeof basePolicy !== 'undefined' && (basePolicy === 'ask' || basePolicy === 'deny')) {
|
||||
return ok({
|
||||
verdict: basePolicy,
|
||||
policy_verdict: basePolicy,
|
||||
model_verdict: null,
|
||||
confidence: 1.0,
|
||||
requires_approval: basePolicy === 'ask',
|
||||
is_blocked: basePolicy === 'deny',
|
||||
risk_factors: { secret_exposure: 0.5, production_impact: 0.5, network: 0.0 },
|
||||
reason: `${policyVerdict.reason} (fail-safe policy fallback)`,
|
||||
zero_inference: true
|
||||
});
|
||||
}
|
||||
return fail(e.message);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -217,6 +217,22 @@ describe('Kai Purpose-Built Native MCP Tools', () => {
|
||||
expect(sent.length).toBe(0); // Zero inference!
|
||||
});
|
||||
|
||||
test('abstains when single candidate has unsatisfied preconditions', async () => {
|
||||
const candidates = [
|
||||
{ action_id: 'act_deploy', tool: 'Deploy', description: 'Deploy to prod', preconditions_satisfied: false }
|
||||
];
|
||||
|
||||
const res = await kaiNextActionTool.handler({
|
||||
goal: 'Deploy service',
|
||||
candidates
|
||||
});
|
||||
const data = parseResult(res);
|
||||
expect(data.action).toBeNull();
|
||||
expect(data.abstained).toBe(true);
|
||||
expect(data.confidence).toBe(0);
|
||||
expect(data.reason).toContain('unsatisfied preconditions');
|
||||
});
|
||||
|
||||
test('calls Kai when multiple candidates are active', async () => {
|
||||
mockReply = () => ({
|
||||
id: 'dec_act_1',
|
||||
@@ -283,8 +299,11 @@ describe('Kai Purpose-Built Native MCP Tools', () => {
|
||||
const data = parseResult(res);
|
||||
expect(data.verdict).toBe('deny');
|
||||
expect(data.policy_verdict).toBe('deny');
|
||||
expect(data.model_verdict).toBeNull();
|
||||
expect(data.zero_inference).toBe(true);
|
||||
expect(data.is_blocked).toBe(true);
|
||||
expect(data.reason).toContain('Deterministic security policy');
|
||||
expect(sent.length).toBe(1); // Kai is still asked for full audit, but verdict is locked to deny
|
||||
expect(sent.length).toBe(0); // Zero-inference fast-path blocks immediately without network call
|
||||
});
|
||||
|
||||
test('enforces deterministic ask for force push', async () => {
|
||||
|
||||
Reference in New Issue
Block a user