mirror of
https://github.com/hanzoai/python-sdk.git
synced 2026-10-23 20:23:00 +00:00
feat(tools): fix BaseTool dispatch + achieve 100% HIP-0300 conformance
- Fix BaseTool.register() to bypass FastMCP's broken **kwargs introspection by creating Tool objects manually with Pydantic extra="allow" models - Add PARAM_ALIASES support for cross-implementation parity (path→uri) - Fix ExecTool to use ShellExecutor.run_shell() (correct 5-tuple API) - Fix CodeTool tree-sitter import for tree-sitter-language-pack 0.25+ - Fix MemoryTool fallback to markdown backend when knowledge service unavailable - Add ExecTool to shell TOOLS list - Remove redundant register() overrides from code/fetch/plan/vcs/fs tools - Add code, network, mode test categories to conformance suite (23 cases) - Add parallel benchmark (benchmark_parallel.py) Results: Python 23/23 (100%), 350/350 parallel calls async-safe
This commit is contained in:
@@ -154,15 +154,19 @@ All operations are PURE - safe to cache and parallelize.
|
||||
def _get_tree_sitter(self):
|
||||
"""Lazy-load tree-sitter."""
|
||||
if self._tree_sitter is None:
|
||||
# Try tree-sitter-language-pack first (works with tree-sitter 0.24+)
|
||||
try:
|
||||
import tree_sitter_languages
|
||||
|
||||
self._tree_sitter = tree_sitter_languages
|
||||
import tree_sitter_language_pack
|
||||
self._tree_sitter = tree_sitter_language_pack
|
||||
except ImportError:
|
||||
raise ToolError(
|
||||
code="INTERNAL_ERROR",
|
||||
message="tree-sitter-languages not installed. Run: pip install tree-sitter-languages",
|
||||
)
|
||||
try:
|
||||
import tree_sitter_languages
|
||||
self._tree_sitter = tree_sitter_languages
|
||||
except ImportError:
|
||||
raise ToolError(
|
||||
code="INTERNAL_ERROR",
|
||||
message="tree-sitter-languages not installed. Run: pip install tree-sitter-language-pack",
|
||||
)
|
||||
return self._tree_sitter
|
||||
|
||||
def _parse_with_tree_sitter(self, text: str, lang: str) -> dict:
|
||||
@@ -636,20 +640,5 @@ All operations are PURE - safe to cache and parallelize.
|
||||
"next_actions": next_actions,
|
||||
}
|
||||
|
||||
def register(self, mcp_server: FastMCP) -> None:
|
||||
"""Register as 'code' tool with MCP server."""
|
||||
tool_name = self.name
|
||||
tool_description = self.description
|
||||
|
||||
@mcp_server.tool(name=tool_name, description=tool_description)
|
||||
async def handler(
|
||||
ctx: MCPContext,
|
||||
action: str = "help",
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
result = await self.call(ctx, action=action, **kwargs)
|
||||
return json.dumps(result, indent=2, default=str)
|
||||
|
||||
|
||||
# Singleton instance
|
||||
code_tool = CodeTool
|
||||
|
||||
@@ -173,6 +173,8 @@ class FsTool(BaseTool):
|
||||
|
||||
name: ClassVar[str] = "fs"
|
||||
VERSION: ClassVar[str] = "0.12.0"
|
||||
# Accept "path" as alias for "uri" (TS parity)
|
||||
PARAM_ALIASES: ClassVar[dict[str, str]] = {"path": "uri"}
|
||||
|
||||
def __init__(self, permission_manager: Optional[PermissionManager] = None):
|
||||
super().__init__()
|
||||
@@ -809,20 +811,5 @@ patch supports Rust grammar format: *** Begin Patch / *** Update File: / @@ / -o
|
||||
|
||||
return {"uri": file_uri(str(path)), "removed": True}
|
||||
|
||||
def register(self, mcp_server: FastMCP) -> None:
|
||||
"""Register as 'fs' tool with MCP server."""
|
||||
tool_name = self.name
|
||||
tool_description = self.description
|
||||
|
||||
@mcp_server.tool(name=tool_name, description=tool_description)
|
||||
async def handler(
|
||||
ctx: MCPContext,
|
||||
action: str = "help",
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
result = await self.call(ctx, action=action, **kwargs)
|
||||
return json.dumps(result, indent=2, default=str)
|
||||
|
||||
|
||||
# Backward compatibility
|
||||
fs_tool = FsTool
|
||||
|
||||
@@ -374,7 +374,7 @@ Examples:
|
||||
async def _facts_lookup(
|
||||
self, queries: List[str], kb: Optional[str], limit: int
|
||||
) -> str:
|
||||
"""Search knowledge bases."""
|
||||
"""Search knowledge bases (falls back to local markdown)."""
|
||||
if not queries:
|
||||
return "Error: query required for facts"
|
||||
|
||||
@@ -397,7 +397,20 @@ Examples:
|
||||
return "\n".join(lines)
|
||||
|
||||
except ImportError:
|
||||
return "Knowledge service not available"
|
||||
# Fall back to local markdown backend
|
||||
from hanzo_tools.memory.markdown_memory import get_markdown_backend
|
||||
|
||||
backend = get_markdown_backend()
|
||||
results = backend.search_memories(
|
||||
queries=queries, limit=limit, scope="project"
|
||||
)
|
||||
if not results:
|
||||
return "No facts found."
|
||||
|
||||
lines = [f"Found {len(results)} facts:"]
|
||||
for mem in results[:limit]:
|
||||
lines.append(f" • {mem.content[:100]}...")
|
||||
return "\n".join(lines)
|
||||
|
||||
async def _facts_store(
|
||||
self,
|
||||
@@ -405,7 +418,7 @@ Examples:
|
||||
kb: Optional[str],
|
||||
metadata: Optional[Dict],
|
||||
) -> str:
|
||||
"""Store facts in knowledge base."""
|
||||
"""Store facts in knowledge base (falls back to local markdown)."""
|
||||
if not facts:
|
||||
return "Error: fact or facts required"
|
||||
|
||||
@@ -426,7 +439,15 @@ Examples:
|
||||
return f"Stored {len(stored)} facts in '{kb or 'default'}'"
|
||||
|
||||
except ImportError:
|
||||
return "Knowledge service not available"
|
||||
# Fall back to local markdown backend
|
||||
from hanzo_tools.memory.markdown_memory import get_markdown_backend
|
||||
|
||||
backend = get_markdown_backend()
|
||||
stored = []
|
||||
for fact in facts:
|
||||
mem = backend.add_memory(content=fact, scope="session")
|
||||
stored.append(mem.memory_id)
|
||||
return f"Stored {len(stored)} fact(s) locally"
|
||||
|
||||
async def _summarize(
|
||||
self,
|
||||
|
||||
@@ -462,20 +462,5 @@ Effect: NONDETERMINISTIC_EFFECT (network I/O)
|
||||
|
||||
return result
|
||||
|
||||
def register(self, mcp_server: FastMCP) -> None:
|
||||
"""Register as 'net' tool with MCP server."""
|
||||
tool_name = self.name
|
||||
tool_description = self.description
|
||||
|
||||
@mcp_server.tool(name=tool_name, description=tool_description)
|
||||
async def handler(
|
||||
ctx: MCPContext,
|
||||
action: str = "help",
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
result = await self.call(ctx, action=action, **kwargs)
|
||||
return json.dumps(result, indent=2, default=str)
|
||||
|
||||
|
||||
# Singleton
|
||||
fetch_tool = FetchTool
|
||||
|
||||
@@ -642,20 +642,5 @@ Turns permissive natural language input into strict canonical operator chains.
|
||||
"total": len(result),
|
||||
}
|
||||
|
||||
def register(self, mcp_server: FastMCP) -> None:
|
||||
"""Register as 'plan' tool with MCP server."""
|
||||
tool_name = self.name
|
||||
tool_description = self.description
|
||||
|
||||
@mcp_server.tool(name=tool_name, description=tool_description)
|
||||
async def handler(
|
||||
ctx: MCPContext,
|
||||
action: str = "help",
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
result = await self.call(ctx, action=action, **kwargs)
|
||||
return json.dumps(result, indent=2, default=str)
|
||||
|
||||
|
||||
# Singleton
|
||||
plan_tool = PlanTool
|
||||
|
||||
@@ -176,7 +176,8 @@ def _get_detected_shell_tools() -> list:
|
||||
shell_tool_class = get_shell_tool_class(shell_name)
|
||||
|
||||
# Base tools (always included) - no cmd, just the detected shell
|
||||
tools = [PsTool, NpxTool, UvxTool, OpenTool, CurlTool, JqTool, WgetTool]
|
||||
# ExecTool = HIP-0300 unified exec tool (action-routed: exec, ps, kill, logs)
|
||||
tools = [ExecTool, PsTool, NpxTool, UvxTool, OpenTool, CurlTool, JqTool, WgetTool]
|
||||
|
||||
# Add detected shell tool, or fall back to ShellTool (smart auto-detect)
|
||||
if shell_tool_class:
|
||||
|
||||
@@ -155,38 +155,35 @@ Auto-backgrounds commands after {AUTO_BACKGROUND_TIMEOUT}s.
|
||||
executor = get_shell_executor()
|
||||
|
||||
try:
|
||||
# Execute with auto-background support
|
||||
result = await executor.execute(
|
||||
command,
|
||||
shell=shell_path,
|
||||
cwd=cwd,
|
||||
env=process_env,
|
||||
timeout=timeout or AUTO_BACKGROUND_TIMEOUT,
|
||||
# Execute with auto-background support via run_shell
|
||||
stdout_str, stderr_str, exit_code, was_bg, proc_id = (
|
||||
await executor.run_shell(
|
||||
command,
|
||||
shell=shell_path,
|
||||
cwd=cwd,
|
||||
env=process_env,
|
||||
timeout=float(timeout or AUTO_BACKGROUND_TIMEOUT),
|
||||
tool_name="exec",
|
||||
)
|
||||
)
|
||||
|
||||
# Check if backgrounded
|
||||
if result.get("backgrounded"):
|
||||
proc_id = result.get("proc_id", "unknown")
|
||||
if was_bg:
|
||||
return {
|
||||
"proc_id": proc_id,
|
||||
"proc_id": proc_id or "unknown",
|
||||
"exit_code": None,
|
||||
"stdout_ref": f"proc:{proc_id}:stdout",
|
||||
"stderr_ref": f"proc:{proc_id}:stderr",
|
||||
"status": "running",
|
||||
"message": f"Command backgrounded after {timeout or AUTO_BACKGROUND_TIMEOUT}s. Use proc(action='logs', proc_id='{proc_id}') to view output.",
|
||||
"message": f"Command backgrounded after {timeout or AUTO_BACKGROUND_TIMEOUT}s. Use exec(action='logs', proc_id='{proc_id}') to view output.",
|
||||
}
|
||||
|
||||
# Command completed
|
||||
exit_code = result.get("return_code", result.get("exit_code", 0))
|
||||
stdout = result.get("stdout", "")
|
||||
stderr = result.get("stderr", "")
|
||||
|
||||
return {
|
||||
"proc_id": result.get("proc_id"),
|
||||
"proc_id": proc_id,
|
||||
"exit_code": exit_code,
|
||||
"stdout": stdout,
|
||||
"stderr": stderr,
|
||||
"duration_ms": result.get("duration_ms"),
|
||||
"stdout": stdout_str,
|
||||
"stderr": stderr_str,
|
||||
"status": "success" if exit_code == 0 else "failed",
|
||||
}
|
||||
|
||||
@@ -485,20 +482,5 @@ Auto-backgrounds commands after {AUTO_BACKGROUND_TIMEOUT}s.
|
||||
message=f"Failed to read logs: {e}",
|
||||
)
|
||||
|
||||
def register(self, mcp_server: FastMCP) -> None:
|
||||
"""Register as 'proc' tool with MCP server."""
|
||||
tool_name = self.name
|
||||
tool_description = self.description
|
||||
|
||||
@mcp_server.tool(name=tool_name, description=tool_description)
|
||||
async def handler(
|
||||
ctx: MCPContext,
|
||||
action: str = "help",
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
result = await self.call(ctx, action=action, **kwargs)
|
||||
return json.dumps(result, indent=2, default=str)
|
||||
|
||||
|
||||
# Backward compatibility
|
||||
exec_tool = ExecTool
|
||||
|
||||
@@ -447,20 +447,5 @@ Outputs diffs in unified patch format for use with fs.apply_patch.
|
||||
"total": len(commits),
|
||||
}
|
||||
|
||||
def register(self, mcp_server: FastMCP) -> None:
|
||||
"""Register as 'vcs' tool with MCP server."""
|
||||
tool_name = self.name
|
||||
tool_description = self.description
|
||||
|
||||
@mcp_server.tool(name=tool_name, description=tool_description)
|
||||
async def handler(
|
||||
ctx: MCPContext,
|
||||
action: str = "help",
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
result = await self.call(ctx, action=action, **kwargs)
|
||||
return json.dumps(result, indent=2, default=str)
|
||||
|
||||
|
||||
# Backward compatibility
|
||||
git_tool = GitTool
|
||||
|
||||
@@ -145,6 +145,9 @@ class BaseTool(_BaseToolABC):
|
||||
# Version for meta envelope
|
||||
VERSION: ClassVar[str] = "0.12.0"
|
||||
|
||||
# Param aliases for cross-implementation parity (e.g., {"path": "uri"})
|
||||
PARAM_ALIASES: ClassVar[dict[str, str]] = {}
|
||||
|
||||
def __init__(self):
|
||||
self._handlers: dict[str, ActionHandler] = {}
|
||||
self._register_builtin_actions()
|
||||
@@ -350,11 +353,20 @@ class BaseTool(_BaseToolABC):
|
||||
Args:
|
||||
ctx: MCP context
|
||||
action: Action to execute (default: "help")
|
||||
**kwargs: Action parameters
|
||||
**kwargs: Action parameters (flat, matching TS wire format)
|
||||
|
||||
Returns:
|
||||
Unified response envelope
|
||||
"""
|
||||
# Unwrap kwargs wrapping from legacy clients: {"kwargs": {...}} → flat
|
||||
if "kwargs" in kwargs and isinstance(kwargs["kwargs"], dict) and len(kwargs) == 1:
|
||||
kwargs = kwargs["kwargs"]
|
||||
|
||||
# Apply param aliases for cross-implementation parity
|
||||
for alias, canonical in self.PARAM_ALIASES.items():
|
||||
if alias in kwargs and canonical not in kwargs:
|
||||
kwargs[canonical] = kwargs.pop(alias)
|
||||
|
||||
if action not in self._handlers:
|
||||
return self._error(
|
||||
"UNKNOWN_ACTION",
|
||||
@@ -375,22 +387,78 @@ class BaseTool(_BaseToolABC):
|
||||
def register(self, mcp_server: FastMCP) -> None:
|
||||
"""Register this tool with the MCP server.
|
||||
|
||||
Creates a single MCP tool with action parameter for routing.
|
||||
Creates a single MCP tool with flat params matching TS wire format.
|
||||
Bypasses FastMCP's function introspection to support arbitrary extra
|
||||
parameters (action-specific params) as top-level fields.
|
||||
|
||||
Wire format: {"action": "read", "path": "/tmp/foo"} (flat, same as TS)
|
||||
"""
|
||||
from pydantic import ConfigDict
|
||||
from mcp.server.fastmcp.tools.base import Tool as FastMCPTool
|
||||
from mcp.server.fastmcp.utilities.func_metadata import (
|
||||
FuncMetadata,
|
||||
ArgModelBase,
|
||||
)
|
||||
|
||||
tool_name = self.name
|
||||
tool_description = (
|
||||
f"{self.description}\n\nActions: {', '.join(self._handlers.keys())}"
|
||||
)
|
||||
|
||||
@mcp_server.tool(name=tool_name, description=tool_description)
|
||||
async def handler(
|
||||
ctx: MCPContext,
|
||||
action: str = "help",
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
result = await self.call(ctx, action=action, **kwargs)
|
||||
# Build combined JSON schema from all action handlers
|
||||
all_properties: dict[str, Any] = {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"description": "Action to perform",
|
||||
"default": "help",
|
||||
}
|
||||
}
|
||||
for ah in self._handlers.values():
|
||||
if ah.schema and "properties" in ah.schema:
|
||||
for k, v in ah.schema["properties"].items():
|
||||
if k not in all_properties and k != "ctx":
|
||||
all_properties[k] = v
|
||||
|
||||
parameters_schema = {
|
||||
"type": "object",
|
||||
"properties": all_properties,
|
||||
"required": ["action"],
|
||||
"additionalProperties": True,
|
||||
}
|
||||
|
||||
# Permissive Pydantic model that accepts any extra fields
|
||||
class _FlexArgs(ArgModelBase):
|
||||
model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)
|
||||
action: str = "help"
|
||||
|
||||
def model_dump_one_level(self) -> dict[str, Any]:
|
||||
result = super().model_dump_one_level()
|
||||
if self.model_extra:
|
||||
result.update(self.model_extra)
|
||||
return result
|
||||
|
||||
# Handler function — receives flat validated args
|
||||
tool_ref = self
|
||||
|
||||
async def _handler(ctx: MCPContext, action: str = "help", **kwargs: Any) -> str:
|
||||
result = await tool_ref.call(ctx, action=action, **kwargs)
|
||||
return json.dumps(result, indent=2, default=str)
|
||||
|
||||
metadata = FuncMetadata(arg_model=_FlexArgs)
|
||||
|
||||
tool = FastMCPTool(
|
||||
fn=_handler,
|
||||
name=tool_name,
|
||||
description=tool_description,
|
||||
parameters=parameters_schema,
|
||||
fn_metadata=metadata,
|
||||
is_async=True,
|
||||
context_kwarg="ctx",
|
||||
)
|
||||
|
||||
# Register directly, bypassing Tool.from_function() introspection
|
||||
mcp_server._tool_manager._tools[tool_name] = tool
|
||||
|
||||
|
||||
# Utility functions for composability
|
||||
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
HIP-0300 Parallel Performance Benchmark
|
||||
|
||||
Tests that all MCP tools work correctly when invoked concurrently.
|
||||
Measures throughput and latency under parallel load.
|
||||
|
||||
Usage:
|
||||
uv run python tests/conformance/benchmark_parallel.py
|
||||
uv run python tests/conformance/benchmark_parallel.py --concurrency 20
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
HANZO_ROOT = Path(os.environ.get("HANZO_ROOT", Path.home() / "work" / "hanzo"))
|
||||
TIMEOUT = 30
|
||||
STARTUP_TIMEOUT = 20
|
||||
|
||||
|
||||
class MCPClient:
|
||||
"""JSON-RPC 2.0 over stdio client."""
|
||||
|
||||
def __init__(self, command: list[str], cwd: Path, env: dict[str, str]):
|
||||
self.command = command
|
||||
self.cwd = cwd
|
||||
self.env = env
|
||||
self.proc: asyncio.subprocess.Process | None = None
|
||||
self._id = 0
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def start(self) -> None:
|
||||
full_env = {**os.environ, **self.env}
|
||||
self.proc = await asyncio.create_subprocess_exec(
|
||||
*self.command,
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=str(self.cwd),
|
||||
env=full_env,
|
||||
)
|
||||
resp = await self._request("initialize", {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {"tools": {}},
|
||||
"clientInfo": {"name": "benchmark", "version": "1.0.0"},
|
||||
})
|
||||
if "error" in resp:
|
||||
raise RuntimeError(f"Initialize failed: {resp['error']}")
|
||||
await self._notify("notifications/initialized")
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self.proc and self.proc.returncode is None:
|
||||
self.proc.terminate()
|
||||
try:
|
||||
await asyncio.wait_for(self.proc.wait(), timeout=5)
|
||||
except asyncio.TimeoutError:
|
||||
self.proc.kill()
|
||||
|
||||
async def call_tool(self, name: str, arguments: dict[str, Any]) -> dict:
|
||||
return await self._request("tools/call", {"name": name, "arguments": arguments})
|
||||
|
||||
async def list_tools(self) -> list[dict]:
|
||||
resp = await self._request("tools/list", {})
|
||||
return resp.get("result", {}).get("tools", [])
|
||||
|
||||
async def _request(self, method: str, params: dict) -> dict:
|
||||
async with self._lock:
|
||||
self._id += 1
|
||||
msg = {"jsonrpc": "2.0", "id": self._id, "method": method, "params": params}
|
||||
return await self._send_and_recv(msg)
|
||||
|
||||
async def _notify(self, method: str, params: dict | None = None) -> None:
|
||||
msg: dict[str, Any] = {"jsonrpc": "2.0", "method": method}
|
||||
if params:
|
||||
msg["params"] = params
|
||||
assert self.proc and self.proc.stdin
|
||||
self.proc.stdin.write(json.dumps(msg).encode() + b"\n")
|
||||
await self.proc.stdin.drain()
|
||||
|
||||
async def _send_and_recv(self, msg: dict) -> dict:
|
||||
assert self.proc and self.proc.stdin and self.proc.stdout
|
||||
data = json.dumps(msg).encode() + b"\n"
|
||||
self.proc.stdin.write(data)
|
||||
await self.proc.stdin.drain()
|
||||
deadline = time.monotonic() + TIMEOUT
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
line = await asyncio.wait_for(
|
||||
self.proc.stdout.readline(),
|
||||
timeout=max(0.1, deadline - time.monotonic()),
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
break
|
||||
if not line:
|
||||
break
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
resp = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if resp.get("id") == msg["id"]:
|
||||
return resp
|
||||
return {"error": {"code": -1, "message": "Timeout"}}
|
||||
|
||||
|
||||
@dataclass
|
||||
class BenchResult:
|
||||
tool: str
|
||||
action: str
|
||||
concurrency: int
|
||||
total_calls: int
|
||||
passed: int
|
||||
failed: int
|
||||
latencies_ms: list[float] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def p50(self) -> float:
|
||||
return statistics.median(self.latencies_ms) if self.latencies_ms else 0
|
||||
|
||||
@property
|
||||
def p95(self) -> float:
|
||||
if not self.latencies_ms:
|
||||
return 0
|
||||
sorted_l = sorted(self.latencies_ms)
|
||||
idx = int(len(sorted_l) * 0.95)
|
||||
return sorted_l[min(idx, len(sorted_l) - 1)]
|
||||
|
||||
@property
|
||||
def throughput(self) -> float:
|
||||
total_s = sum(self.latencies_ms) / 1000 if self.latencies_ms else 1
|
||||
return self.total_calls / (total_s / self.concurrency) if total_s > 0 else 0
|
||||
|
||||
|
||||
async def bench_tool(
|
||||
client: MCPClient,
|
||||
tool: str,
|
||||
action: str,
|
||||
params: dict[str, Any],
|
||||
concurrency: int = 10,
|
||||
iterations: int = 50,
|
||||
) -> BenchResult:
|
||||
"""Benchmark a tool with parallel invocations."""
|
||||
result = BenchResult(
|
||||
tool=tool, action=action, concurrency=concurrency,
|
||||
total_calls=iterations, passed=0, failed=0,
|
||||
)
|
||||
|
||||
sem = asyncio.Semaphore(concurrency)
|
||||
|
||||
async def single_call():
|
||||
async with sem:
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
resp = await asyncio.wait_for(
|
||||
client.call_tool(tool, {"action": action, **params}),
|
||||
timeout=TIMEOUT,
|
||||
)
|
||||
duration = (time.monotonic() - t0) * 1000
|
||||
result.latencies_ms.append(duration)
|
||||
if "error" in resp:
|
||||
result.failed += 1
|
||||
else:
|
||||
result.passed += 1
|
||||
except Exception:
|
||||
result.failed += 1
|
||||
result.latencies_ms.append((time.monotonic() - t0) * 1000)
|
||||
|
||||
# Run all calls concurrently
|
||||
await asyncio.gather(*[single_call() for _ in range(iterations)])
|
||||
return result
|
||||
|
||||
|
||||
async def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="HIP-0300 Parallel Benchmark")
|
||||
parser.add_argument("--concurrency", type=int, default=10, help="Max concurrent calls")
|
||||
parser.add_argument("--iterations", type=int, default=50, help="Total calls per test")
|
||||
parser.add_argument("--impl", default="python", choices=["python", "typescript"])
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.impl == "python":
|
||||
client = MCPClient(
|
||||
command=["uv", "run", "hanzo-mcp"],
|
||||
cwd=HANZO_ROOT / "python-sdk",
|
||||
env={"LOGLEVEL": "ERROR", "HANZO_MCP_LOG_LEVEL": "ERROR"},
|
||||
)
|
||||
else:
|
||||
client = MCPClient(
|
||||
command=["node", "dist/cli.js", "serve"],
|
||||
cwd=HANZO_ROOT / "mcp",
|
||||
env={"NODE_ENV": "test", "MCP_LOG_LEVEL": "error"},
|
||||
)
|
||||
|
||||
print(f"\nHIP-0300 Parallel Benchmark ({args.impl})")
|
||||
print(f"Concurrency: {args.concurrency}, Iterations: {args.iterations}")
|
||||
print(f"{'='*70}")
|
||||
|
||||
print("Starting server...", end=" ", flush=True)
|
||||
await asyncio.wait_for(client.start(), timeout=STARTUP_TIMEOUT)
|
||||
tools = await client.list_tools()
|
||||
tool_names = set(t["name"] for t in tools)
|
||||
print(f"OK ({len(tools)} tools)")
|
||||
|
||||
# Create temp dir for fs tests
|
||||
tmp = Path(tempfile.mkdtemp(prefix="mcp-bench-"))
|
||||
|
||||
benchmarks = []
|
||||
|
||||
# Define test workloads
|
||||
workloads = [
|
||||
("exec", "exec", {"command": "echo bench"}),
|
||||
("exec", "ps", {}),
|
||||
("git", "status", {}),
|
||||
("git", "log", {"limit": 3}),
|
||||
("think", "think", {"thought": "Benchmark test"}),
|
||||
]
|
||||
|
||||
# Only test tools that are registered
|
||||
workloads = [(t, a, p) for t, a, p in workloads if t in tool_names]
|
||||
|
||||
# Add fs tests with unique filenames
|
||||
if "fs" in tool_names:
|
||||
# Write files first
|
||||
for i in range(args.iterations):
|
||||
await client.call_tool("fs", {
|
||||
"action": "write",
|
||||
"path": str(tmp / f"bench_{i}.txt"),
|
||||
"content": f"benchmark content {i}",
|
||||
})
|
||||
workloads.append(("fs", "read", {"path": str(tmp / "bench_0.txt")}))
|
||||
workloads.append(("fs", "list", {"path": str(tmp)}))
|
||||
|
||||
print(f"\nRunning {len(workloads)} workloads...\n")
|
||||
print(f"{'Tool':<12} {'Action':<10} {'Pass':<6} {'Fail':<6} {'P50':<10} {'P95':<10} {'Throughput':<12}")
|
||||
print("-" * 70)
|
||||
|
||||
for tool, action, params in workloads:
|
||||
result = await bench_tool(
|
||||
client, tool, action, params,
|
||||
concurrency=args.concurrency,
|
||||
iterations=args.iterations,
|
||||
)
|
||||
benchmarks.append(result)
|
||||
print(
|
||||
f"{tool:<12} {action:<10} {result.passed:<6} {result.failed:<6} "
|
||||
f"{result.p50:>7.1f}ms {result.p95:>7.1f}ms "
|
||||
f"{result.throughput:>8.1f} ops/s"
|
||||
)
|
||||
|
||||
# Cleanup
|
||||
import shutil
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
await client.stop()
|
||||
|
||||
# Summary
|
||||
total_passed = sum(b.passed for b in benchmarks)
|
||||
total_failed = sum(b.failed for b in benchmarks)
|
||||
total = total_passed + total_failed
|
||||
all_latencies = [l for b in benchmarks for l in b.latencies_ms]
|
||||
|
||||
print(f"\n{'='*70}")
|
||||
print(f"Total: {total_passed}/{total} passed ({total_failed} failed)")
|
||||
if all_latencies:
|
||||
print(f"Overall P50: {statistics.median(all_latencies):.1f}ms P95: {sorted(all_latencies)[int(len(all_latencies)*0.95)]:.1f}ms")
|
||||
print(f"All tools async-safe: {'YES' if total_failed == 0 else 'NO'}")
|
||||
|
||||
sys.exit(0 if total_failed == 0 else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -94,12 +94,16 @@ def resolve_tool_call(
|
||||
category: str,
|
||||
action: str,
|
||||
params: dict[str, Any],
|
||||
tool_schemas: dict[str, dict] | None = None,
|
||||
) -> tuple[str, dict[str, Any]] | None:
|
||||
"""Resolve a logical test action to the correct tool name + args for an implementation.
|
||||
|
||||
Returns (tool_name, arguments) or None if the tool isn't available.
|
||||
"""
|
||||
# Unified names (TS default, Python future)
|
||||
tool_schemas = tool_schemas or {}
|
||||
# Unified names (TS default, Python HIP-0300)
|
||||
# Note: Python's BaseTool-derived tools (fs, git, code, fetch) require a `kwargs` wrapper
|
||||
# while think/memory/tasks/browser use flat params. We detect which format via tool_names.
|
||||
unified_map: dict[tuple[str, str], tuple[str, dict]] = {
|
||||
("filesystem", "read"): ("fs", {"action": "read", **params}),
|
||||
("filesystem", "write"): ("fs", {"action": "write", **params}),
|
||||
@@ -119,15 +123,37 @@ def resolve_tool_call(
|
||||
("network", "fetch"): ("fetch", {"action": "fetch", **params}),
|
||||
("network", "head"): ("fetch", {"action": "head", **params}),
|
||||
("reasoning", "think"): ("think", {"action": "think", **params}),
|
||||
("browser", "status"): ("browser", {"action": "status"}),
|
||||
("memory", "store"): ("memory", {"action": "store", **params}),
|
||||
("memory", "recall"): ("memory", {"action": "recall", **params}),
|
||||
("memory", "list"): ("memory", {"action": "list", **params}),
|
||||
("memory", "forget"): ("memory", {"action": "forget", **params}),
|
||||
("memory", "delete"): ("memory", {"action": "delete", **params}),
|
||||
("tasks", "add"): ("tasks", {"action": "add", **params}),
|
||||
("tasks", "list"): ("tasks", {"action": "list", **params}),
|
||||
("mode", "list"): ("mode", {"action": "list"}),
|
||||
}
|
||||
|
||||
# Python's HIP-0300 BaseTool uses kwargs wrapper for fs/git/code/fetch/exec
|
||||
# These tools require: {"action": "...", "kwargs": {...params}}
|
||||
py_kwargs_map: dict[tuple[str, str], tuple[str, dict]] = {
|
||||
("filesystem", "read"): ("fs", {"action": "read", "kwargs": params}),
|
||||
("filesystem", "write"): ("fs", {"action": "write", "kwargs": params}),
|
||||
("filesystem", "stat"): ("fs", {"action": "stat", "kwargs": params}),
|
||||
("filesystem", "list"): ("fs", {"action": "list", "kwargs": params}),
|
||||
("filesystem", "search"): ("fs", {"action": "search_text", "kwargs": params}),
|
||||
("filesystem", "mkdir"): ("fs", {"action": "mkdir", "kwargs": params}),
|
||||
("filesystem", "rm"): ("fs", {"action": "rm", "kwargs": params}),
|
||||
("shell", "exec"): ("exec", {"action": "exec", "kwargs": params}),
|
||||
("shell", "ps"): ("exec", {"action": "ps", "kwargs": {}}),
|
||||
("vcs", "status"): ("git", {"action": "status", "kwargs": params}),
|
||||
("vcs", "branch"): ("git", {"action": "branch", "kwargs": params}),
|
||||
("vcs", "log"): ("git", {"action": "log", "kwargs": params}),
|
||||
("vcs", "diff"): ("git", {"action": "diff", "kwargs": params}),
|
||||
("code", "symbols"): ("code", {"action": "symbols", "kwargs": params}),
|
||||
("code", "summarize"): ("code", {"action": "summarize", "kwargs": params}),
|
||||
("network", "fetch"): ("fetch", {"action": "fetch", "kwargs": params}),
|
||||
("network", "head"): ("fetch", {"action": "head", "kwargs": params}),
|
||||
}
|
||||
|
||||
# Legacy names (Python current)
|
||||
legacy_map: dict[tuple[str, str], tuple[str, dict]] = {
|
||||
("filesystem", "read"): ("read_file", params),
|
||||
@@ -143,12 +169,24 @@ def resolve_tool_call(
|
||||
("vcs", "branch"): ("run_command", {"command": "git branch"}),
|
||||
("vcs", "log"): ("run_command", {"command": f"git log --oneline -n {params.get('limit', 5)}"}),
|
||||
("vcs", "diff"): ("run_command", {"command": "git diff"}),
|
||||
("reasoning", "think"): ("critic_agent", params),
|
||||
("reasoning", "think"): ("critic", {"analysis": params.get("thought", params.get("analysis", ""))}),
|
||||
("browser", "status"): ("browser", {"action": "status"}),
|
||||
}
|
||||
|
||||
key = (category, action)
|
||||
|
||||
# Try unified name first
|
||||
# Check if tool uses kwargs wrapper (Python BaseTool pattern)
|
||||
def _needs_kwargs(tool_name: str) -> bool:
|
||||
schema = tool_schemas.get(tool_name, {})
|
||||
return "kwargs" in schema.get("required", []) or "kwargs" in schema.get("properties", {})
|
||||
|
||||
# Try kwargs-wrapped unified names (Python BaseTool pattern)
|
||||
if key in py_kwargs_map:
|
||||
tool, args = py_kwargs_map[key]
|
||||
if tool in tool_names and _needs_kwargs(tool):
|
||||
return tool, args
|
||||
|
||||
# Try flat unified names (TS pattern)
|
||||
if key in unified_map:
|
||||
tool, args = unified_map[key]
|
||||
if tool in tool_names:
|
||||
@@ -342,6 +380,39 @@ def build_test_cases() -> list[TestCase]:
|
||||
),
|
||||
])
|
||||
|
||||
# ── Code ──────────────────────────────────────────────────────────────
|
||||
code_file = tmp / "sample.py"
|
||||
def setup_code():
|
||||
tmp.mkdir(parents=True, exist_ok=True)
|
||||
code_file.write_text('def greet(name):\n return f"hello {name}"\n\nclass Greeter:\n pass\n')
|
||||
|
||||
cases.extend([
|
||||
TestCase(
|
||||
category="code", action="symbols", name="code.symbols lists symbols",
|
||||
params={"path": str(code_file)},
|
||||
expect_pattern="(?i)greet|hello|Greeter|function|class|def",
|
||||
setup=setup_code,
|
||||
),
|
||||
TestCase(
|
||||
category="code", action="summarize", name="code.summarize diff text",
|
||||
params={"text": "--- a/foo.py\n+++ b/foo.py\n@@ -1 +1 @@\n-old\n+new\n"},
|
||||
),
|
||||
])
|
||||
|
||||
# ── Network / Fetch ───────────────────────────────────────────────────
|
||||
cases.extend([
|
||||
TestCase(
|
||||
category="network", action="fetch", name="fetch.fetch retrieves URL",
|
||||
params={"url": "https://httpbin.org/get"},
|
||||
expect_pattern="(?i)origin|headers|url",
|
||||
),
|
||||
TestCase(
|
||||
category="network", action="head", name="fetch.head gets headers",
|
||||
params={"url": "https://httpbin.org/get"},
|
||||
expect_pattern="(?i)content-type|200|headers|status",
|
||||
),
|
||||
])
|
||||
|
||||
# ── Reasoning ─────────────────────────────────────────────────────────
|
||||
cases.append(
|
||||
TestCase(
|
||||
@@ -350,6 +421,54 @@ def build_test_cases() -> list[TestCase]:
|
||||
),
|
||||
)
|
||||
|
||||
# ── Browser (non-browser-requiring tests) ─────────────────────────────
|
||||
cases.extend([
|
||||
TestCase(
|
||||
category="browser", action="status", name="browser status check",
|
||||
params={"action": "status"},
|
||||
),
|
||||
])
|
||||
|
||||
# ── Memory ────────────────────────────────────────────────────────────
|
||||
# TS memory uses: key/value. Python memory uses: fact/query.
|
||||
cases.extend([
|
||||
TestCase(
|
||||
category="memory", action="store", name="memory store",
|
||||
params={"key": "conformance-test", "value": "HIP-0300 value", "fact": "conformance-test: HIP-0300 value"},
|
||||
expect_pattern="(?i)stored|saved|ok|success|written|remembered|added",
|
||||
),
|
||||
TestCase(
|
||||
category="memory", action="recall", name="memory recall",
|
||||
params={"key": "conformance-test", "query": "conformance-test"},
|
||||
expect_pattern="(?i)HIP-0300|conformance",
|
||||
),
|
||||
TestCase(
|
||||
category="memory", action="delete", name="memory delete",
|
||||
params={"key": "conformance-test", "query": "conformance-test"},
|
||||
),
|
||||
])
|
||||
|
||||
# ── Tasks ─────────────────────────────────────────────────────────────
|
||||
cases.extend([
|
||||
TestCase(
|
||||
category="tasks", action="add", name="tasks add",
|
||||
params={"content": "Conformance test task"},
|
||||
expect_pattern="(?i)added|created|task|ok",
|
||||
),
|
||||
TestCase(
|
||||
category="tasks", action="list", name="tasks list",
|
||||
params={},
|
||||
),
|
||||
])
|
||||
|
||||
# ── Mode ──────────────────────────────────────────────────────────────
|
||||
cases.append(
|
||||
TestCase(
|
||||
category="mode", action="list", name="mode.list shows modes",
|
||||
params={},
|
||||
),
|
||||
)
|
||||
|
||||
return cases
|
||||
|
||||
|
||||
@@ -416,6 +535,7 @@ async def run_tests(
|
||||
|
||||
tools = await client.list_tools()
|
||||
tool_names = set(t["name"] for t in tools)
|
||||
tool_schemas = {t["name"]: t.get("inputSchema", {}) for t in tools}
|
||||
print(f" Tools registered: {len(tools)}")
|
||||
|
||||
if verbose:
|
||||
@@ -426,7 +546,7 @@ async def run_tests(
|
||||
if categories_filter and tc.category not in categories_filter:
|
||||
continue
|
||||
|
||||
resolved = resolve_tool_call(impl.id, tool_names, tc.category, tc.action, tc.params)
|
||||
resolved = resolve_tool_call(impl.id, tool_names, tc.category, tc.action, tc.params, tool_schemas)
|
||||
if resolved is None:
|
||||
print(f" SKIP {tc.name} (no matching tool for {tc.category}.{tc.action})")
|
||||
results.append(TestResult(
|
||||
|
||||
Reference in New Issue
Block a user