fix: run ruff check --fix and format across entire codebase

This commit is contained in:
2025-12-26 04:31:06 -08:00
parent dacdc3b797
commit b21ca3a5ce
201 changed files with 1665 additions and 4195 deletions
+2 -6
View File
@@ -108,9 +108,7 @@ def build_package(package_dir: Path) -> bool:
subprocess.run(["rm", "-rf", str(egg_info)], check=True)
# Build the package
result = subprocess.run(
["python", "-m", "build"], cwd=package_dir, capture_output=True, text=True
)
result = subprocess.run(["python", "-m", "build"], cwd=package_dir, capture_output=True, text=True)
if result.returncode != 0:
print_error(f"Build failed: {result.stderr}")
@@ -226,9 +224,7 @@ def main():
capture_output=True,
)
except Exception:
print_warn(
"Could not install build tools. Make sure pip, build, and twine are available."
)
print_warn("Could not install build tools. Make sure pip, build, and twine are available.")
print_info("You can install them with: pip install build twine")
# Track results
+80 -99
View File
@@ -12,12 +12,13 @@ from typing import Any, Dict, List
# Example of how to use hanzo-mcp batch tool for parallel edits
# In practice, this would be called through the MCP interface
def create_parallel_edit_batch():
"""
Create batch invocations for parallel document editing.
Each agent works on a different file simultaneously.
"""
batch_invocations = [
{
"tool_name": "claude",
@@ -33,7 +34,7 @@ def create_parallel_edit_batch():
Return the complete enhanced content maintaining all existing sections.
Focus on Claude-specific strengths and unique features.
"""
}
},
},
{
"tool_name": "codex",
@@ -49,7 +50,7 @@ def create_parallel_edit_batch():
Return the complete enhanced content maintaining all existing sections.
Focus on OpenAI-specific optimizations and features.
"""
}
},
},
{
"tool_name": "gemini",
@@ -65,7 +66,7 @@ def create_parallel_edit_batch():
Return the complete enhanced content maintaining all existing sections.
Focus on Google-specific integrations and multimodal strengths.
"""
}
},
},
{
"tool_name": "grok",
@@ -83,18 +84,19 @@ def create_parallel_edit_batch():
Follow the same structure as other AI doc files.
Include code examples and practical patterns.
"""
}
}
},
},
]
return batch_invocations
def create_agent_editing_batch():
"""
Create batch for editing AGENTS.md using multiple agents.
Each agent enhances a different section.
"""
batch_invocations = [
{
"tool_name": "claude",
@@ -107,7 +109,7 @@ def create_agent_editing_batch():
- Include agent negotiation strategies
Write only the enhanced section content.
"""
}
},
},
{
"tool_name": "codex",
@@ -120,7 +122,7 @@ def create_agent_editing_batch():
- Include automated PR generation
Write only the enhanced section content.
"""
}
},
},
{
"tool_name": "gemini",
@@ -133,25 +135,26 @@ def create_agent_editing_batch():
- Include performance optimization
Write only the enhanced section content.
"""
}
}
},
},
]
return batch_invocations
def create_review_and_merge_batch(edited_content: Dict[str, str]):
"""
Create batch for reviewing and merging edits using consensus.
Multiple agents review each other's work.
"""
batch_invocations = [
{
"tool_name": "claude",
"input": {
"prompt": f"""
Review the Codex edits to LLM.md:
{edited_content.get('llm_edits', '')}
{edited_content.get("llm_edits", "")}
Check for:
1. Technical accuracy
@@ -161,14 +164,14 @@ def create_review_and_merge_batch(edited_content: Dict[str, str]):
Provide feedback and improved version if needed.
"""
}
},
},
{
"tool_name": "codex",
"input": {
"prompt": f"""
Review the Claude edits to CLAUDE.md:
{edited_content.get('claude_edits', '')}
{edited_content.get("claude_edits", "")}
Check for:
1. Implementation feasibility
@@ -178,7 +181,7 @@ def create_review_and_merge_batch(edited_content: Dict[str, str]):
Provide feedback and improved version if needed.
"""
}
},
},
{
"tool_name": "gemini",
@@ -198,104 +201,96 @@ def create_review_and_merge_batch(edited_content: Dict[str, str]):
3. Integration test cases
4. Documentation validation script
"""
}
}
},
},
]
return batch_invocations
class ParallelDocumentEditor:
"""
Orchestrates parallel editing of documentation using multiple AI agents.
"""
def __init__(self):
self.agents = ["claude", "codex", "gemini", "grok"]
self.files = ["LLM.md", "AGENTS.md", "GEMINI.md", "CLAUDE.md"]
async def execute_parallel_edits(self):
"""
Execute parallel edits using batch tool.
"""
# Phase 1: Initial parallel edits
print("🚀 Phase 1: Initiating parallel edits...")
initial_batch = create_parallel_edit_batch()
# This would be executed as:
# results = await batch(
# description="Edit AI documentation files",
# invocations=initial_batch
# )
# Simulated results for demonstration
results_phase1 = {
"claude": "Enhanced CLAUDE.md content...",
"codex": "Enhanced LLM.md content...",
"gemini": "Enhanced GEMINI.md content...",
"grok": "New GROK.md content..."
"grok": "New GROK.md content...",
}
print("✅ Phase 1 complete: All files edited in parallel")
# Phase 2: Parallel section enhancements
print("\n🚀 Phase 2: Enhancing specific sections...")
section_batch = create_agent_editing_batch()
# results = await batch(
# description="Enhance AGENTS.md sections",
# invocations=section_batch
# )
results_phase2 = {
"communication": "Enhanced communication section...",
"worktree": "Enhanced worktree section...",
"swarm": "Enhanced swarm section..."
"swarm": "Enhanced swarm section...",
}
print("✅ Phase 2 complete: Sections enhanced")
# Phase 3: Cross-review and validation
print("\n🚀 Phase 3: Cross-reviewing edits...")
review_batch = create_review_and_merge_batch(results_phase1)
# results = await batch(
# description="Review and validate edits",
# invocations=review_batch
# )
print("✅ Phase 3 complete: All edits reviewed and validated")
return {
"phase1": results_phase1,
"phase2": results_phase2,
"status": "completed"
}
return {"phase1": results_phase1, "phase2": results_phase2, "status": "completed"}
async def apply_edits_in_parallel(self, edits: Dict[str, str]):
"""
Apply validated edits to files in parallel.
"""
write_batch = [
{
"tool_name": "write",
"input": {
"file_path": f"/Users/z/work/hanzo/python-sdk/{file}",
"content": content
}
}
{"tool_name": "write", "input": {"file_path": f"/Users/z/work/hanzo/python-sdk/{file}", "content": content}}
for file, content in edits.items()
]
# Execute all writes in parallel
# results = await batch(
# description="Write updated documentation",
# invocations=write_batch
# )
print("✅ All files updated in parallel")
# Practical example of batch execution in MCP context
BATCH_EXAMPLE = """
# In Claude Desktop or hanzo-mcp CLI, you would execute:
@@ -328,116 +323,102 @@ batch --description "Edit AI docs in parallel" --invocations '[
]'
"""
def demonstrate_batch_patterns():
"""
Demonstrate various batch execution patterns.
"""
print("=" * 60)
print("PARALLEL AI DOCUMENTATION EDITING PATTERNS")
print("=" * 60)
# Pattern 1: Parallel file editing
print("\n📝 Pattern 1: Parallel File Editing")
print("-" * 40)
parallel_edit = {
"description": "Edit 4 files simultaneously",
"invocations": [
{"tool": "claude", "target": "CLAUDE.md"},
{"tool": "codex", "target": "LLM.md"},
{"tool": "gemini", "target": "GEMINI.md"},
{"tool": "grok", "target": "AGENTS.md"}
]
{"tool": "grok", "target": "AGENTS.md"},
],
}
print(json.dumps(parallel_edit, indent=2))
# Pattern 2: Sequential with parallel stages
print("\n📝 Pattern 2: Sequential Stages with Parallel Tasks")
print("-" * 40)
staged_execution = {
"stage1": {
"description": "Analyze all files in parallel",
"parallel": True,
"tasks": ["analyze_claude.md", "analyze_llm.md", "analyze_agents.md"]
"tasks": ["analyze_claude.md", "analyze_llm.md", "analyze_agents.md"],
},
"stage2": {
"description": "Edit based on analysis",
"parallel": True,
"tasks": ["edit_claude.md", "edit_llm.md", "edit_agents.md"]
"tasks": ["edit_claude.md", "edit_llm.md", "edit_agents.md"],
},
"stage3": {
"description": "Review all edits",
"parallel": False,
"tasks": ["consensus_review"]
}
"stage3": {"description": "Review all edits", "parallel": False, "tasks": ["consensus_review"]},
}
print(json.dumps(staged_execution, indent=2))
# Pattern 3: Divide and conquer
print("\n📝 Pattern 3: Divide and Conquer")
print("-" * 40)
divide_conquer = {
"description": "Each agent handles specific sections",
"file": "AGENTS.md",
"parallel_sections": [
{
"agent": "claude",
"section": "Communication Protocols",
"expertise": "System design"
},
{
"agent": "codex",
"section": "Code Generation",
"expertise": "Implementation"
},
{
"agent": "gemini",
"section": "Testing Strategies",
"expertise": "Quality assurance"
}
]
{"agent": "claude", "section": "Communication Protocols", "expertise": "System design"},
{"agent": "codex", "section": "Code Generation", "expertise": "Implementation"},
{"agent": "gemini", "section": "Testing Strategies", "expertise": "Quality assurance"},
],
}
print(json.dumps(divide_conquer, indent=2))
# Pattern 4: Consensus editing
print("\n📝 Pattern 4: Consensus-Based Editing")
print("-" * 40)
consensus_edit = {
"description": "Multiple agents edit same content",
"target": "architecture.md",
"agents": ["claude", "codex", "gemini"],
"strategy": "Each agent provides version, then consensus",
"final_review": "grok"
"final_review": "grok",
}
print(json.dumps(consensus_edit, indent=2))
if __name__ == "__main__":
import asyncio
# Demonstrate patterns
demonstrate_batch_patterns()
# Example execution (would be async in practice)
print("\n" + "=" * 60)
print("EXAMPLE PARALLEL EXECUTION")
print("=" * 60)
editor = ParallelDocumentEditor()
# In practice, this would be:
# asyncio.run(editor.execute_parallel_edits())
print("\n✨ Parallel editing demonstration complete!")
print("\n" + "=" * 60)
print("BATCH TOOL USAGE IN MCP")
print("=" * 60)
print(BATCH_EXAMPLE)
print(BATCH_EXAMPLE)
+4 -12
View File
@@ -34,9 +34,7 @@ async def main():
)
# Cloud agent as fallback
cloud_agent = agents.create_agent(
name="cloud-helper", model="anthropic/claude-3-5-sonnet-20241022"
)
cloud_agent = agents.create_agent(name="cloud-helper", model="anthropic/claude-3-5-sonnet-20241022")
# 3. Create an agent network
print("\nCreating agent network...")
@@ -47,24 +45,18 @@ async def main():
# 4. Create MCP server with tools
print("\nStarting MCP server...")
mcp_server = mcp.create_mcp_server(
name="hanzo-unified", allowed_paths=[".", "/tmp"], enable_agent_tool=True
)
mcp_server = mcp.create_mcp_server(name="hanzo-unified", allowed_paths=[".", "/tmp"], enable_agent_tool=True)
# 5. Example: Use local AI for simple tasks
print("\n--- Local AI Example ---")
local_result = await local_cluster.inference(
prompt="Write a haiku about distributed AI", max_tokens=50
)
local_result = await local_cluster.inference(prompt="Write a haiku about distributed AI", max_tokens=50)
print(f"Local AI: {local_result}")
# 6. Example: Use cloud AI for complex tasks
print("\n--- Cloud AI Example ---")
cloud_result = completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=[
{"role": "user", "content": "Explain the benefits of local AI clusters"}
],
messages=[{"role": "user", "content": "Explain the benefits of local AI clusters"}],
)
print(f"Cloud AI: {cloud_result}")
+44 -53
View File
@@ -13,8 +13,8 @@ import sys
import asyncio
# Add paths for development (remove in production)
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'pkg', 'hanzoai'))
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'pkg', 'hanzo', 'src'))
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "pkg", "hanzoai"))
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "pkg", "hanzo", "src"))
try:
from hanzoai import Hanzo, AsyncHanzo
@@ -30,86 +30,77 @@ except ImportError:
def basic_grok_usage():
"""Basic usage of Grok through Hanzo SDK."""
client = Hanzo() # Uses HANZO_API_KEY env var
# Simple completion
response = client.chat.completions.create(
model="grok-4", # or "grok" or "xai-grok"
messages=[
{"role": "system", "content": "You have real-time knowledge."},
{"role": "user", "content": "What are the latest AI developments this week?"}
{"role": "user", "content": "What are the latest AI developments this week?"},
],
temperature=0.7,
max_tokens=500
max_tokens=500,
)
print("Grok Response:")
print(response.choices[0].message.content)
print("\n" + "="*50 + "\n")
print("\n" + "=" * 50 + "\n")
def streaming_grok():
"""Stream responses from Grok for real-time output."""
client = Hanzo()
print("Streaming from Grok:")
stream = client.chat.completions.create(
model="grok-4",
messages=[
{"role": "user", "content": "Explain quantum computing in simple terms"}
],
stream=True
model="grok-4", messages=[{"role": "user", "content": "Explain quantum computing in simple terms"}], stream=True
)
for chunk in stream:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="", flush=True)
print("\n\n" + "="*50 + "\n")
print("\n\n" + "=" * 50 + "\n")
async def async_grok_usage():
"""Async usage of Grok for concurrent operations."""
client = AsyncHanzo()
# Concurrent requests to different models
tasks = [
client.chat.completions.create(
model="grok-4",
messages=[{"role": "user", "content": "What's happening in tech today?"}]
model="grok-4", messages=[{"role": "user", "content": "What's happening in tech today?"}]
),
client.chat.completions.create(
model="claude-3-5-sonnet",
messages=[{"role": "user", "content": "Explain machine learning"}]
model="claude-3-5-sonnet", messages=[{"role": "user", "content": "Explain machine learning"}]
),
client.chat.completions.create(
model="gpt-4",
messages=[{"role": "user", "content": "Write a haiku about AI"}]
)
client.chat.completions.create(model="gpt-4", messages=[{"role": "user", "content": "Write a haiku about AI"}]),
]
print("Concurrent requests to Grok, Claude, and GPT-4:")
responses = await asyncio.gather(*tasks)
for i, (model, response) in enumerate(zip(["Grok", "Claude", "GPT-4"], responses)):
print(f"\n{model} Response:")
print(response.choices[0].message.content[:200] + "...")
print("\n" + "="*50 + "\n")
print("\n" + "=" * 50 + "\n")
def batch_operations_with_grok():
"""Demonstrate batch operations using Grok."""
from hanzo.batch_orchestrator import BatchConfig, BatchOrchestrator
print("Batch Operations with Grok:")
# Parse different batch configurations
configs = [
"batch:5 agent:grok analyze code quality",
"consensus:3 agent:grok,claude,gemini review architecture",
"critic:2 agent:grok,gpt-4 chain:true security audit"
"critic:2 agent:grok,gpt-4 chain:true security audit",
]
for cmd in configs:
config = BatchConfig.from_command(cmd)
print(f"\nCommand: {cmd}")
@@ -119,73 +110,73 @@ def batch_operations_with_grok():
print(f" - Consensus models: {config.consensus_models}")
if config.critic_mode:
print(f" - Critic chain: {config.critic_chain}")
print("\n" + "="*50 + "\n")
print("\n" + "=" * 50 + "\n")
def grok_with_tools():
"""Use Grok with MCP tools."""
try:
from hanzo_mcp.tools.agent.cli_tools import GrokCLITool
print("Grok CLI Tool Configuration:")
grok_tool = GrokCLITool()
print(f"Tool name: {grok_tool.name}")
print(f"Description: {grok_tool.description}")
print(f"Default model: {grok_tool.default_model}")
print(f"API key env: {grok_tool.api_key_env}")
# Get auth environment
env = grok_tool.get_auth_env()
if "XAI_API_KEY" in env:
print("✅ XAI_API_KEY is configured")
else:
print("⚠️ XAI_API_KEY not found in environment")
except ImportError:
print("hanzo-mcp not installed. Install with: pip install hanzo-mcp")
print("\n" + "="*50 + "\n")
print("\n" + "=" * 50 + "\n")
def main():
"""Run all Grok examples."""
print("="*50)
print("=" * 50)
print("xAI Grok Examples with Hanzo SDK")
print("="*50 + "\n")
print("=" * 50 + "\n")
# Check for API key
if not os.environ.get("XAI_API_KEY"):
print("⚠️ Warning: XAI_API_KEY not set")
print("Get your API key from https://x.ai/api")
print("Set it with: export XAI_API_KEY='your-key-here'")
print("\nRunning examples that don't require actual API calls...\n")
# Run non-API examples
batch_operations_with_grok()
grok_with_tools()
return
# Run all examples
try:
# Basic usage
basic_grok_usage()
# Streaming
streaming_grok()
# Async operations
asyncio.run(async_grok_usage())
# Batch operations
batch_operations_with_grok()
# MCP tools
grok_with_tools()
print("✅ All Grok examples completed successfully!")
except Exception as e:
print(f"Error: {e}")
print("\nMake sure you have:")
@@ -196,4 +187,4 @@ def main():
if __name__ == "__main__":
main()
main()
+43 -55
View File
@@ -13,13 +13,14 @@ from typing import Any, Dict, List
# This would normally use the hanzo-mcp Python client
# For demonstration, showing the conceptual flow
class WorktreeOrchestrator:
"""Orchestrates multiple agents working on git worktrees."""
def __init__(self, base_branch: str = "main"):
self.base_branch = base_branch
self.worktrees: Dict[str, str] = {}
def create_worktree_agent_prompt(self, task_id: str, task_content: str) -> str:
"""Generate agent prompt for worktree task."""
return f"""
@@ -115,23 +116,18 @@ Be strict but fair. This code will go to production.
def execute_workflow(self, tasks: List[Dict[str, Any]]) -> Dict[str, Any]:
"""Execute the complete worktree workflow."""
results = {
"total_tasks": len(tasks),
"completed": [],
"failed": [],
"worktrees": []
}
results = {"total_tasks": len(tasks), "completed": [], "failed": [], "worktrees": []}
for task in tasks:
task_id = task["id"]
task_content = task["content"]
print(f"\n🚀 Starting task {task_id}: {task_content}")
# Step 1: Agent implements the task
agent_prompt = self.create_worktree_agent_prompt(task_id, task_content)
# This would call: agent(prompt=agent_prompt)
# For demo, showing the structure
agent_result = {
@@ -139,77 +135,69 @@ Be strict but fair. This code will go to production.
"worktree": f"worktree-{task_id}",
"files_modified": ["src/auth.py", "tests/test_auth.py"],
"tests_passed": True,
"ready_for_review": True
"ready_for_review": True,
}
print(f"✅ Agent completed implementation")
# Step 2: Critic reviews the implementation
if agent_result["ready_for_review"]:
critic_prompt = self.create_critic_prompt(
task_id,
json.dumps(agent_result, indent=2)
)
critic_prompt = self.create_critic_prompt(task_id, json.dumps(agent_result, indent=2))
# This would call: critic(analysis=critic_prompt)
critic_result = {
"verdict": "APPROVED",
"issues": [],
"score": 95
}
critic_result = {"verdict": "APPROVED", "issues": [], "score": 95}
if critic_result["verdict"] == "APPROVED":
print(f"✅ Critic approved implementation (score: {critic_result['score']})")
results["completed"].append(task_id)
# Step 3: Prepare for merge
self.worktrees[task_id] = f"feature/{task_id}"
results["worktrees"].append({
"task_id": task_id,
"branch": f"feature/{task_id}",
"path": f"../worktree-{task_id}",
"ready_to_merge": True
})
results["worktrees"].append(
{
"task_id": task_id,
"branch": f"feature/{task_id}",
"path": f"../worktree-{task_id}",
"ready_to_merge": True,
}
)
else:
print(f"⚠️ Critic requested fixes: {critic_result['issues']}")
# Step 4: Agent fixes issues
fix_prompt = f"""
Fix these issues in worktree-{task_id}:
{json.dumps(critic_result['issues'], indent=2)}
{json.dumps(critic_result["issues"], indent=2)}
"""
# Second attempt would go here
results["failed"].append({
"task_id": task_id,
"reason": "Needs fixes",
"issues": critic_result["issues"]
})
results["failed"].append(
{"task_id": task_id, "reason": "Needs fixes", "issues": critic_result["issues"]}
)
return results
def merge_completed_worktrees(self, results: Dict[str, Any]) -> None:
"""Merge all completed worktrees back to base branch."""
print(f"\n🔀 Merging completed worktrees to {self.base_branch}")
for worktree in results["worktrees"]:
if worktree["ready_to_merge"]:
branch = worktree["branch"]
# This would execute:
# git checkout main
# git merge --no-ff feature/{task_id}
# git push origin main
print(f"✅ Merged {branch} to {self.base_branch}")
print(f"\n🎉 Workflow complete! {len(results['completed'])}/{results['total_tasks']} tasks merged")
# Example usage
if __name__ == "__main__":
# Sample tasks from todo list
tasks = [
{"id": "auth-1", "content": "Implement JWT token generation", "priority": "high"},
@@ -217,23 +205,23 @@ if __name__ == "__main__":
{"id": "auth-3", "content": "Add password reset flow", "priority": "medium"},
{"id": "auth-4", "content": "Implement rate limiting", "priority": "medium"},
]
# Initialize orchestrator
orchestrator = WorktreeOrchestrator(base_branch="main")
# Execute parallel development workflow
results = orchestrator.execute_workflow(tasks)
# Merge successful implementations
orchestrator.merge_completed_worktrees(results)
# Output summary
print("\n📊 Workflow Summary:")
print(f" Total tasks: {results['total_tasks']}")
print(f" Completed: {len(results['completed'])}")
print(f" Failed: {len(results['failed'])}")
print(f" Worktrees created: {len(results['worktrees'])}")
# This summary could be sent to Linear or saved to a report
with open("workflow_report.json", "w") as f:
json.dump(results, f, indent=2)
json.dump(results, f, indent=2)
+4 -3
View File
@@ -38,7 +38,7 @@ filterwarnings =
"""
# Write temporary config and use it
with tempfile.NamedTemporaryFile(mode='w', suffix='.ini', delete=False) as f:
with tempfile.NamedTemporaryFile(mode="w", suffix=".ini", delete=False) as f:
f.write(pytest_ini_content)
pytest_ini_path = f.name
@@ -46,7 +46,8 @@ filterwarnings =
session.run(
"pytest",
"tests/", # Explicitly specify tests/ to avoid collecting from pkg/* subpackages
"-c", pytest_ini_path,
"-c",
pytest_ini_path,
"--showlocals",
"--ignore=tests/functional",
# Ignore tests that require optional dependencies (hanzo, hanzo_network, click, etc.)
@@ -60,7 +61,7 @@ filterwarnings =
"--ignore=tests/test_streaming.py",
"--ignore=tests/test_todo.py",
"--ignore=tests/test_hanzo_dev.py", # Requires hanzo package
*session.posargs
*session.posargs,
)
finally:
os.unlink(pytest_ini_path)
+2 -6
View File
@@ -1,9 +1,5 @@
CONTENT_TRUNCATED_NOTICE = "<response clipped><NOTE>Due to the max output limit, only part of the full response has been shown to you.</NOTE>"
FILE_CONTENT_TRUNCATED_NOTICE: str = (
"<response clipped><NOTE>Due to the max output limit, only part of this file has been shown to you. You should retry this tool after you have searched inside the file with `grep -n` in order to find the line numbers of what you are looking for.</NOTE>"
)
FILE_CONTENT_TRUNCATED_NOTICE: str = "<response clipped><NOTE>Due to the max output limit, only part of this file has been shown to you. You should retry this tool after you have searched inside the file with `grep -n` in order to find the line numbers of what you are looking for.</NOTE>"
DIRECTORY_CONTENT_TRUNCATED_NOTICE: str = (
"<response clipped><NOTE>Due to the max output limit, only part of this directory has been shown to you. You should use `ls -la` instead to view large directories incrementally.</NOTE>"
)
DIRECTORY_CONTENT_TRUNCATED_NOTICE: str = "<response clipped><NOTE>Due to the max output limit, only part of this directory has been shown to you. You should use `ls -la` instead to view large directories incrementally.</NOTE>"
@@ -32,9 +32,9 @@ def test_file_editor_happy_path(temp_file):
result,
re.DOTALL,
)
assert (
match
), "Output does not contain the expected <oh_aci_output_> tags in the correct format."
assert match, (
"Output does not contain the expected <oh_aci_output_> tags in the correct format."
)
result_dict = json.loads(match.group(1))
# Validate the formatted output in the result dictionary
@@ -89,9 +89,9 @@ match = re.search(
re.DOTALL,
)
assert (
match
), "Output does not contain the expected <oh_aci_output_> tags in the correct format."
assert match, (
"Output does not contain the expected <oh_aci_output_> tags in the correct format."
)
result_dict = json.loads(match.group(1))
# Validate the formatted output in the result dictionary
@@ -256,9 +256,9 @@ def test_large_history_insert():
# Check if we can still retrieve the last entry
last_content = manager.pop_last_history(Path(f"test_file_{num_files - 1}.txt"))
assert (
last_content == large_content
), "Failed to retrieve the last inserted content"
assert last_content == large_content, (
"Failed to retrieve the last inserted content"
)
# Check if the number of cache entries is correct
cache_entries = list(manager.cache)
@@ -266,4 +266,6 @@ def test_large_history_insert():
len(cache_entries)
== num_files * 2
- 1 # The cache entry for file content was removed, only metadata remains
), f"Expected {num_files * 2 - 1} cache entries ({num_files - 1} content + {num_files} metadata), but found {len(cache_entries)}"
), (
f"Expected {num_files * 2 - 1} cache entries ({num_files - 1} content + {num_files} metadata), but found {len(cache_entries)}"
)
@@ -91,9 +91,7 @@ async def run_node(node_id: str, port: int, agents_subset: list):
while True:
await asyncio.sleep(5)
status = network.get_network_status()
print(
f"📊 Node {node_id} status - Peers: {status['peer_count']}, Agents: {status['local_agents']}"
)
print(f"📊 Node {node_id} status - Peers: {status['peer_count']}, Agents: {status['local_agents']}")
# Show discovered peers
if status["peers"]:
+3 -9
View File
@@ -90,27 +90,21 @@ async def main():
# Test weather agent
print("\n🌤️ Testing weather agent...")
weather_result = await network.run(
prompt="What's the weather in Tokyo?", initial_agent=weather_agent
)
weather_result = await network.run(prompt="What's the weather in Tokyo?", initial_agent=weather_agent)
if weather_result["success"]:
print(f"Response: {weather_result['final_output']}")
# Test math agent
print("\n🔢 Testing math agent...")
math_result = await network.run(
prompt="Calculate 42 * 17 + 3", initial_agent=math_agent
)
math_result = await network.run(prompt="Calculate 42 * 17 + 3", initial_agent=math_agent)
if math_result["success"]:
print(f"Response: {math_result['final_output']}")
# Test with router (let network decide which agent)
print("\n🤔 Testing with router...")
auto_result = await network.run(
prompt="I need to know the weather in Paris and also calculate 100 / 4"
)
auto_result = await network.run(prompt="I need to know the weather in Paris and also calculate 100 / 4")
if auto_result["success"]:
print(f"Response: {auto_result['final_output']}")
@@ -108,38 +108,28 @@ class Agent:
# Call lifecycle hook
if self.lifecycle and self.lifecycle.on_start:
result = await self.lifecycle.on_start(
agent=self, messages=messages, state=state, context=context
)
result = await self.lifecycle.on_start(agent=self, messages=messages, state=state, context=context)
if result and result.get("stop"):
return result
messages = result.get("messages", messages)
try:
# Execute with appropriate backend
if (
self.model
and isinstance(self.model, ModelConfig)
and self.model.provider == ModelProvider.CLI
):
if self.model and isinstance(self.model, ModelConfig) and self.model.provider == ModelProvider.CLI:
result = await self._execute_cli(messages, state, context)
else:
result = await self._execute_llm(messages, state, context)
# Call finish hook
if self.lifecycle and self.lifecycle.on_finish:
await self.lifecycle.on_finish(
agent=self, result=result, state=state, context=context
)
await self.lifecycle.on_finish(agent=self, result=result, state=state, context=context)
return result
except Exception as e:
# Call error hook
if self.lifecycle and self.lifecycle.on_error:
await self.lifecycle.on_error(
agent=self, error=e, state=state, context=context
)
await self.lifecycle.on_error(agent=self, error=e, state=state, context=context)
raise
async def _execute_llm(
@@ -158,9 +148,7 @@ class Agent:
tools = [t.to_anthropic_tool() for t in self.tools]
elif self.model.provider == ModelProvider.LOCAL:
# Simple tool format for local LLMs
tools = [
{"name": t.name, "description": t.description} for t in self.tools
]
tools = [{"name": t.name, "description": t.description} for t in self.tools]
# Handle local LLM providers using hanzo/net
if self.model and self.model.provider == ModelProvider.LOCAL:
@@ -174,9 +162,7 @@ class Agent:
engine_type = "tinygrad"
# Create hanzo/net provider
provider = HanzoNetProvider(
engine_type=engine_type, base_url=self.model.base_url
)
provider = HanzoNetProvider(engine_type=engine_type, base_url=self.model.base_url)
# Generate using distributed inference
return await provider.generate(
@@ -209,9 +209,7 @@ class Network(Generic[T]):
# No router - stop
return None
def _build_agent_prompt(
self, agent: Agent, last_result: Optional[Dict[str, Any]]
) -> List[Any]:
def _build_agent_prompt(self, agent: Agent, last_result: Optional[Dict[str, Any]]) -> List[Any]:
"""Build prompt for agent including conversation history.
Args:
@@ -129,9 +129,7 @@ class RoutingAgent(Agent):
return "\n".join(prompt_parts)
def _parse_routing_decision(
self, result: Dict[str, Any], args: RouterArgs
) -> Optional[Agent]:
def _parse_routing_decision(self, result: Dict[str, Any], args: RouterArgs) -> Optional[Agent]:
"""Parse routing decision from agent result."""
output = result.get("output", [])
@@ -226,9 +224,7 @@ def get_default_routing_agent(
from .agent import ModelConfig, ModelProvider
# Use local dummy model for default router
model = ModelConfig(
provider=ModelProvider.LOCAL, model="llama3.2", temperature=0.3
)
model = ModelConfig(provider=ModelProvider.LOCAL, model="llama3.2", temperature=0.3)
return create_routing_agent(
name="default_router",
@@ -50,9 +50,7 @@ class NetworkState(Generic[T]):
self.execution_count: int = 0
self.metadata: Dict[str, Any] = {}
def add_message(
self, role: str, content: str, agent_id: Optional[str] = None, **metadata
) -> None:
def add_message(self, role: str, content: str, agent_id: Optional[str] = None, **metadata) -> None:
"""Add a message to the conversation history.
Args:
@@ -61,9 +59,7 @@ class NetworkState(Generic[T]):
agent_id: ID of the agent that generated this message
**metadata: Additional metadata for the message
"""
self.messages.append(
Message(role=role, content=content, agent_id=agent_id, metadata=metadata)
)
self.messages.append(Message(role=role, content=content, agent_id=agent_id, metadata=metadata))
def get_messages(
self,
@@ -17,9 +17,7 @@ class ToolContext(TypedDict):
class ToolHandler(Protocol):
"""Protocol for tool handler functions."""
async def __call__(
self, parameters: Dict[str, Any], context: ToolContext
) -> Any: ...
async def __call__(self, parameters: Dict[str, Any], context: ToolContext) -> Any: ...
@dataclass
@@ -34,9 +32,7 @@ class Tool:
parameters: type[BaseModel]
handler: ToolHandler
async def call(
self, parameters: Dict[str, Any], context: Optional[ToolContext] = None
) -> Any:
async def call(self, parameters: Dict[str, Any], context: Optional[ToolContext] = None) -> Any:
"""Call the tool with given parameters.
Args:
@@ -130,14 +126,8 @@ def create_tool(
continue
# Get type annotation
param_type = (
param.annotation
if param.annotation != inspect.Parameter.empty
else Any
)
default = (
param.default if param.default != inspect.Parameter.empty else ...
)
param_type = param.annotation if param.annotation != inspect.Parameter.empty else Any
default = param.default if param.default != inspect.Parameter.empty else ...
param_fields[param_name] = (param_type, default)
@@ -147,9 +137,7 @@ def create_tool(
else:
parameters = create_model(f"{name}_params")
return Tool(
name=name, description=description, parameters=parameters, handler=func
)
return Tool(name=name, description=description, parameters=parameters, handler=func)
if handler is not None:
# Direct call with handler
@@ -21,9 +21,7 @@ else:
class GRPCPeerHandle(PeerHandle):
def __init__(
self, _id: str, address: str, desc: str, device_capabilities: DeviceCapabilities
):
def __init__(self, _id: str, address: str, desc: str, device_capabilities: DeviceCapabilities):
self._id = _id
self.address = address
self.desc = desc
@@ -67,10 +65,7 @@ class GRPCPeerHandle(PeerHandle):
await asyncio.wait_for(self.channel.channel_ready(), timeout=10.0)
async def is_connected(self) -> bool:
return (
self.channel is not None
and self.channel.get_state() == grpc.ChannelConnectivity.READY
)
return self.channel is not None and self.channel.get_state() == grpc.ChannelConnectivity.READY
async def disconnect(self):
if self.channel:
@@ -121,11 +116,7 @@ class GRPCPeerHandle(PeerHandle):
n_layers=shard.n_layers,
),
request_id=request_id,
inference_state=(
None
if inference_state is None
else self.serialize_inference_state(inference_state)
),
inference_state=(None if inference_state is None else self.serialize_inference_state(inference_state)),
)
await self.stub.SendPrompt(request)
@@ -150,20 +141,14 @@ class GRPCPeerHandle(PeerHandle):
dtype=str(tensor.dtype),
),
request_id=request_id,
inference_state=(
None
if inference_state is None
else self.serialize_inference_state(inference_state)
),
inference_state=(None if inference_state is None else self.serialize_inference_state(inference_state)),
)
response = await self.stub.SendTensor(request)
if not response.tensor_data or not response.shape or not response.dtype:
return None
return np.frombuffer(
response.tensor_data, dtype=np.dtype(response.dtype)
).reshape(response.shape)
return np.frombuffer(response.tensor_data, dtype=np.dtype(response.dtype)).reshape(response.shape)
async def send_example(
self,
@@ -203,16 +188,14 @@ class GRPCPeerHandle(PeerHandle):
response = await self.stub.SendExample(request)
loss = response.loss
if train and not shard.is_first_layer():
grads = np.frombuffer(
response.grads.tensor_data, dtype=np.dtype(response.grads.dtype)
).reshape(response.grads.shape)
grads = np.frombuffer(response.grads.tensor_data, dtype=np.dtype(response.grads.dtype)).reshape(
response.grads.shape
)
return loss, grads
else:
return loss
async def send_loss(
self, shard: Shard, tensor: np.ndarray, request_id: Optional[str] = None
) -> Optional[np.array]:
async def send_loss(self, shard: Shard, tensor: np.ndarray, request_id: Optional[str] = None) -> Optional[np.array]:
await self._ensure_connected()
request = node_service_pb2.TensorRequest(
shard=node_service_pb2.Shard(
@@ -233,15 +216,11 @@ class GRPCPeerHandle(PeerHandle):
if not response.tensor_data or not response.shape or not response.dtype:
return None
return np.frombuffer(
response.tensor_data, dtype=np.dtype(response.dtype)
).reshape(response.shape)
return np.frombuffer(response.tensor_data, dtype=np.dtype(response.dtype)).reshape(response.shape)
async def collect_topology(self, visited: set[str], max_depth: int) -> Topology:
await self._ensure_connected()
request = node_service_pb2.CollectTopologyRequest(
visited=visited, max_depth=max_depth
)
request = node_service_pb2.CollectTopologyRequest(visited=visited, max_depth=max_depth)
response = await self.stub.CollectTopology(request)
topology = Topology()
for node_id, capabilities in response.nodes.items():
@@ -261,9 +240,7 @@ class GRPCPeerHandle(PeerHandle):
topology.add_edge(node_id, conn.to_id, conn.description)
return topology
async def send_result(
self, request_id: str, result: List[int], is_finished: bool
) -> None:
async def send_result(self, request_id: str, result: List[int], is_finished: bool) -> None:
await self._ensure_connected()
tensor = None
if isinstance(result, np.ndarray):
@@ -280,14 +257,10 @@ class GRPCPeerHandle(PeerHandle):
async def send_opaque_status(self, request_id: str, status: str) -> None:
await self._ensure_connected()
request = node_service_pb2.SendOpaqueStatusRequest(
request_id=request_id, status=status
)
request = node_service_pb2.SendOpaqueStatusRequest(request_id=request_id, status=status)
await asyncio.wait_for(self.stub.SendOpaqueStatus(request), timeout=10.0)
def serialize_inference_state(
self, inference_state: dict
) -> node_service_pb2.InferenceState:
def serialize_inference_state(self, inference_state: dict) -> node_service_pb2.InferenceState:
proto_inference_state = node_service_pb2.InferenceState()
other_data = {}
for k, v in inference_state.items():
@@ -74,20 +74,14 @@ class GRPCServer(node_service_pb2_grpc.NodeServiceServicer):
prompt = request.prompt
request_id = request.request_id
inference_state = (
None
if request.inference_state is None
else self.deserialize_inference_state(request.inference_state)
)
result = await self.node.process_prompt(
shard, prompt, request_id, inference_state
None if request.inference_state is None else self.deserialize_inference_state(request.inference_state)
)
result = await self.node.process_prompt(shard, prompt, request_id, inference_state)
if DEBUG >= 5:
print(f"SendPrompt {shard=} {prompt=} {request_id=} result: {result}")
tensor_data = result.tobytes() if result is not None else None
return (
node_service_pb2.Tensor(
tensor_data=tensor_data, shape=result.shape, dtype=str(result.dtype)
)
node_service_pb2.Tensor(tensor_data=tensor_data, shape=result.shape, dtype=str(result.dtype))
if result is not None
else node_service_pb2.Tensor()
)
@@ -99,29 +93,21 @@ class GRPCServer(node_service_pb2_grpc.NodeServiceServicer):
end_layer=request.shard.end_layer,
n_layers=request.shard.n_layers,
)
tensor = np.frombuffer(
request.tensor.tensor_data, dtype=np.dtype(request.tensor.dtype)
).reshape(request.tensor.shape)
tensor = np.frombuffer(request.tensor.tensor_data, dtype=np.dtype(request.tensor.dtype)).reshape(
request.tensor.shape
)
request_id = request.request_id
inference_state = (
None
if request.inference_state is None
else self.deserialize_inference_state(request.inference_state)
None if request.inference_state is None else self.deserialize_inference_state(request.inference_state)
)
result = await self.node.process_tensor(
shard, tensor, request_id, inference_state
)
result = await self.node.process_tensor(shard, tensor, request_id, inference_state)
if DEBUG >= 5:
print(
f"SendTensor tensor {shard=} {tensor=} {request_id=} result: {result}"
)
print(f"SendTensor tensor {shard=} {tensor=} {request_id=} result: {result}")
tensor_data = result.tobytes() if result is not None else None
return (
node_service_pb2.Tensor(
tensor_data=tensor_data, shape=result.shape, dtype=str(result.dtype)
)
node_service_pb2.Tensor(tensor_data=tensor_data, shape=result.shape, dtype=str(result.dtype))
if result is not None
else node_service_pb2.Tensor()
)
@@ -133,31 +119,25 @@ class GRPCServer(node_service_pb2_grpc.NodeServiceServicer):
end_layer=request.shard.end_layer,
n_layers=request.shard.n_layers,
)
example = np.frombuffer(
request.example.tensor_data, dtype=np.dtype(request.example.dtype)
).reshape(request.example.shape)
target = np.frombuffer(
request.target.tensor_data, dtype=np.dtype(request.target.dtype)
).reshape(request.target.shape)
length = np.frombuffer(
request.length.tensor_data, dtype=np.dtype(request.length.dtype)
).reshape(request.length.shape)
example = np.frombuffer(request.example.tensor_data, dtype=np.dtype(request.example.dtype)).reshape(
request.example.shape
)
target = np.frombuffer(request.target.tensor_data, dtype=np.dtype(request.target.dtype)).reshape(
request.target.shape
)
length = np.frombuffer(request.length.tensor_data, dtype=np.dtype(request.length.dtype)).reshape(
request.length.shape
)
train = request.train
request_id = request.request_id
if train and not shard.is_first_layer():
loss, grad = await self.node.process_example(
shard, example, target, length, train, request_id
)
loss, grad = await self.node.process_example(shard, example, target, length, train, request_id)
tensor_data = grad.tobytes()
grad_tensor = node_service_pb2.Tensor(
tensor_data=tensor_data, shape=grad.shape, dtype=str(grad.dtype)
)
grad_tensor = node_service_pb2.Tensor(tensor_data=tensor_data, shape=grad.shape, dtype=str(grad.dtype))
return node_service_pb2.Loss(loss=loss, grads=grad_tensor)
else:
loss = await self.node.process_example(
shard, example, target, length, train, request_id
)
loss = await self.node.process_example(shard, example, target, length, train, request_id)
return node_service_pb2.Loss(loss=loss, grads=None)
async def CollectTopology(self, request, context):
@@ -169,18 +149,14 @@ class GRPCServer(node_service_pb2_grpc.NodeServiceServicer):
model=cap.model,
chip=cap.chip,
memory=cap.memory,
flops=node_service_pb2.DeviceFlops(
fp32=cap.flops.fp32, fp16=cap.flops.fp16, int8=cap.flops.int8
),
flops=node_service_pb2.DeviceFlops(fp32=cap.flops.fp32, fp16=cap.flops.fp16, int8=cap.flops.int8),
)
for node_id, cap in topology.nodes.items()
}
peer_graph = {
node_id: node_service_pb2.PeerConnections(
connections=[
node_service_pb2.PeerConnection(
to_id=conn.to_id, description=conn.description
)
node_service_pb2.PeerConnection(to_id=conn.to_id, description=conn.description)
for conn in connections
]
)
@@ -196,14 +172,10 @@ class GRPCServer(node_service_pb2_grpc.NodeServiceServicer):
is_finished = request.is_finished
img = request.tensor
if DEBUG >= 5:
print(
f"Received SendResult request: {request_id=} {result=} {is_finished=}"
)
print(f"Received SendResult request: {request_id=} {result=} {is_finished=}")
result = list(result)
if len(img.tensor_data) > 0:
result = np.frombuffer(img.tensor_data, dtype=np.dtype(img.dtype)).reshape(
img.shape
)
result = np.frombuffer(img.tensor_data, dtype=np.dtype(img.dtype)).reshape(img.shape)
self.node.on_token.trigger_all(request_id, result, is_finished)
return node_service_pb2.Empty()
@@ -218,24 +190,16 @@ class GRPCServer(node_service_pb2_grpc.NodeServiceServicer):
async def HealthCheck(self, request, context):
return node_service_pb2.HealthCheckResponse(is_healthy=True)
def deserialize_inference_state(
self, inference_state_proto: node_service_pb2.InferenceState
) -> dict:
def deserialize_inference_state(self, inference_state_proto: node_service_pb2.InferenceState) -> dict:
inference_state = {}
for k, tensor_data in inference_state_proto.tensor_data.items():
np_array = np.frombuffer(
tensor_data.tensor_data, dtype=tensor_data.dtype
).reshape(tensor_data.shape)
np_array = np.frombuffer(tensor_data.tensor_data, dtype=tensor_data.dtype).reshape(tensor_data.shape)
inference_state[k] = mx.array(np_array)
for k, tensor_list in inference_state_proto.tensor_list_data.items():
inference_state[k] = [
mx.array(
np.frombuffer(tensor.tensor_data, dtype=tensor.dtype).reshape(
tensor.shape
)
)
mx.array(np.frombuffer(tensor.tensor_data, dtype=tensor.dtype).reshape(tensor.shape))
for tensor in tensor_list.tensors
]
@@ -11,9 +11,7 @@ from google.protobuf import runtime_version as _runtime_version
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC, 5, 27, 2, "", "node_service.proto"
)
_runtime_version.ValidateProtobufRuntimeVersion(_runtime_version.Domain.PUBLIC, 5, 27, 2, "", "node_service.proto")
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
@@ -12,9 +12,7 @@ _version_not_supported = False
try:
from grpc._utilities import first_version_is_lower
_version_not_supported = first_version_is_lower(
GRPC_VERSION, GRPC_GENERATED_VERSION
)
_version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION)
except ImportError:
_version_not_supported = True
@@ -165,13 +163,9 @@ def add_NodeServiceServicer_to_server(servicer, server):
response_serializer=node__service__pb2.HealthCheckResponse.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
"node_service.NodeService", rpc_method_handlers
)
generic_handler = grpc.method_handlers_generic_handler("node_service.NodeService", rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
server.add_registered_method_handlers(
"node_service.NodeService", rpc_method_handlers
)
server.add_registered_method_handlers("node_service.NodeService", rpc_method_handlers)
# This class is part of an EXPERIMENTAL API.
@@ -40,14 +40,10 @@ class ManualDiscovery(Discovery):
if wait_for_peers > 0:
while len(self.known_peers) < wait_for_peers:
if DEBUG_DISCOVERY >= 2:
print(
f"Current peers: {len(self.known_peers)}/{wait_for_peers}. Waiting for more peers..."
)
print(f"Current peers: {len(self.known_peers)}/{wait_for_peers}. Waiting for more peers...")
await asyncio.sleep(0.1)
if DEBUG_DISCOVERY >= 2:
print(
f"Discovered peers: {[peer.id() for peer in self.known_peers.values()]}"
)
print(f"Discovered peers: {[peer.id() for peer in self.known_peers.values()]}")
return list(self.known_peers.values())
async def task_find_peers_from_config(self):
@@ -59,9 +55,7 @@ class ManualDiscovery(Discovery):
for peer_id, peer_config in peers_from_config.items():
try:
if DEBUG_DISCOVERY >= 2:
print(
f"Checking peer {peer_id=} at {peer_config.address}:{peer_config.port}"
)
print(f"Checking peer {peer_id=} at {peer_config.address}:{peer_config.port}")
peer = self.known_peers.get(peer_id)
if not peer:
if DEBUG_DISCOVERY >= 2:
@@ -75,33 +69,23 @@ class ManualDiscovery(Discovery):
is_healthy = await peer.health_check()
if is_healthy:
if DEBUG_DISCOVERY >= 2:
print(
f"{peer_id=} at {peer_config.address}:{peer_config.port} is healthy."
)
print(f"{peer_id=} at {peer_config.address}:{peer_config.port} is healthy.")
new_known_peers[peer_id] = peer
elif DEBUG_DISCOVERY >= 2:
print(
f"{peer_id=} at {peer_config.address}:{peer_config.port} is not healthy. Removing."
)
print(f"{peer_id=} at {peer_config.address}:{peer_config.port} is not healthy. Removing.")
except Exception as e:
if DEBUG_DISCOVERY >= 2:
print(
f"Exception occurred when attempting to add {peer_id=}: {e}"
)
print(f"Exception occurred when attempting to add {peer_id=}: {e}")
self.known_peers = new_known_peers
await asyncio.sleep(5.0)
if DEBUG_DISCOVERY >= 2:
print(
f"Current known peers: {[peer.id() for peer in self.known_peers.values()]}"
)
print(f"Current known peers: {[peer.id() for peer in self.known_peers.values()]}")
async def _get_peers(self):
try:
loop = asyncio.get_running_loop()
current_mtime = await loop.run_in_executor(
self._file_executor, os.path.getmtime, self.network_config_path
)
current_mtime = await loop.run_in_executor(self._file_executor, os.path.getmtime, self.network_config_path)
if (
self._cached_peers is not None
@@ -29,6 +29,4 @@ class NetworkTopology(BaseModel):
try:
return cls.model_validate_json(config_data)
except ValidationError as e:
raise ValueError(
f"Error validating network topology config from {path}: {e}"
) from e
raise ValueError(f"Error validating network topology config from {path}: {e}") from e
@@ -72,12 +72,8 @@ class TestManualDiscoveryWithGRPCPeerHandle(unittest.IsolatedAsyncioTestCase):
self.node1 = mock.AsyncMock(spec=Node)
self.node2 = mock.AsyncMock(spec=Node)
self.server1 = GRPCServer(
self.node1, config.peers["node1"].address, config.peers["node1"].port
)
self.server2 = GRPCServer(
self.node2, config.peers["node2"].address, config.peers["node2"].port
)
self.server1 = GRPCServer(self.node1, config.peers["node1"].address, config.peers["node1"].port)
self.server2 = GRPCServer(self.node2, config.peers["node2"].address, config.peers["node2"].port)
await self.server1.start()
await self.server2.start()
self.discovery1 = ManualDiscovery(
@@ -30,9 +30,7 @@ class TestNetworkTopologyConfig(unittest.TestCase):
config = NetworkTopology.from_path(root_path + "test_config.json")
self.assertEqual(config.peers["node1"].port, 50051)
self.assertEqual(
config.peers["node1"].device_capabilities.model, "Unknown Model"
)
self.assertEqual(config.peers["node1"].device_capabilities.model, "Unknown Model")
self.assertEqual(config.peers["node1"].address, "localhost")
self.assertEqual(config.peers["node1"].device_capabilities.chip, "Unknown Chip")
self.assertEqual(config.peers["node1"].device_capabilities.memory, 0)
@@ -41,9 +39,7 @@ class TestNetworkTopologyConfig(unittest.TestCase):
self.assertEqual(config.peers["node1"].device_capabilities.flops.int8, 0)
self.assertEqual(config.peers["node2"].port, 50052)
self.assertEqual(
config.peers["node2"].device_capabilities.model, "Unknown Model"
)
self.assertEqual(config.peers["node2"].device_capabilities.model, "Unknown Model")
self.assertEqual(config.peers["node2"].address, "localhost")
self.assertEqual(config.peers["node2"].device_capabilities.chip, "Unknown Chip")
self.assertEqual(config.peers["node2"].device_capabilities.memory, 0)
@@ -65,9 +65,7 @@ class MinimalDiscovery(Discovery):
self.is_running = False
print(f"MinimalDiscovery stopped for node {self.node_id}")
async def discover_peers(
self, wait_for_peers: int = 0
) -> List[SimplifiedPeerHandle]:
async def discover_peers(self, wait_for_peers: int = 0) -> List[SimplifiedPeerHandle]:
"""Discover peers (returns empty list for now)."""
# For testing, we don't actually discover peers
# Real implementation would use UDP broadcast, mDNS, etc.
@@ -40,21 +40,15 @@ class PeerHandle(ABC):
pass
@abstractmethod
async def send_prompt(
self, shard: Shard, prompt: str, request_id: Optional[str] = None
) -> Optional[np.array]:
async def send_prompt(self, shard: Shard, prompt: str, request_id: Optional[str] = None) -> Optional[np.array]:
pass
@abstractmethod
async def send_tensor(
self, shard: Shard, tensor: np.array, request_id: Optional[str] = None
) -> Optional[np.array]:
async def send_tensor(self, shard: Shard, tensor: np.array, request_id: Optional[str] = None) -> Optional[np.array]:
pass
@abstractmethod
async def send_result(
self, request_id: str, result: List[int], is_finished: bool
) -> None:
async def send_result(self, request_id: str, result: List[int], is_finished: bool) -> None:
pass
@abstractmethod
@@ -53,9 +53,7 @@ class TailscaleDiscovery(Discovery):
self.device_capabilities = await device_capabilities()
self.discovery_task = asyncio.create_task(self.task_discover_peers())
self.cleanup_task = asyncio.create_task(self.task_cleanup_peers())
self.update_task = asyncio.create_task(
self.task_update_device_posture_attributes()
)
self.update_task = asyncio.create_task(self.task_update_device_posture_attributes())
async def task_update_device_posture_attributes(self):
while True:
@@ -87,60 +85,45 @@ class TailscaleDiscovery(Discovery):
async def task_discover_peers(self):
while True:
try:
devices: dict[str, Device] = await get_tailscale_devices(
self.tailscale_api_key, self.tailnet
)
devices: dict[str, Device] = await get_tailscale_devices(self.tailscale_api_key, self.tailnet)
current_time = time.time()
active_devices = {
name: device
for name, device in devices.items()
if device.last_seen is not None
and (current_time - device.last_seen.timestamp()) < 30
if device.last_seen is not None and (current_time - device.last_seen.timestamp()) < 30
}
if DEBUG_DISCOVERY >= 4:
print(f"Found tailscale devices: {devices}")
if DEBUG_DISCOVERY >= 2:
print(
f"Active tailscale devices: {len(active_devices)}/{len(devices)}"
)
print(f"Active tailscale devices: {len(active_devices)}/{len(devices)}")
if DEBUG_DISCOVERY >= 2:
print(
"Time since last seen tailscale devices",
[
(current_time - device.last_seen.timestamp())
for device in devices.values()
],
[(current_time - device.last_seen.timestamp()) for device in devices.values()],
)
for device in active_devices.values():
if device.name == self.node_id:
continue
peer_host = device.addresses[0]
peer_id, peer_port, device_capabilities = (
await get_device_attributes(
device.device_id, self.tailscale_api_key
)
peer_id, peer_port, device_capabilities = await get_device_attributes(
device.device_id, self.tailscale_api_key
)
if not peer_id:
if DEBUG_DISCOVERY >= 4:
print(
f"{device.device_id} does not have exo node attributes. skipping."
)
print(f"{device.device_id} does not have exo node attributes. skipping.")
continue
if self.allowed_node_ids and peer_id not in self.allowed_node_ids:
if DEBUG_DISCOVERY >= 2:
print(
f"Ignoring peer {peer_id} as it's not in the allowed node IDs list"
)
print(f"Ignoring peer {peer_id} as it's not in the allowed node IDs list")
continue
if (
peer_id not in self.known_peers
or self.known_peers[peer_id][0].addr()
!= f"{peer_host}:{peer_port}"
or self.known_peers[peer_id][0].addr() != f"{peer_host}:{peer_port}"
):
new_peer_handle = self.create_peer_handle(
peer_id,
@@ -150,9 +133,7 @@ class TailscaleDiscovery(Discovery):
)
if not await new_peer_handle.health_check():
if DEBUG >= 1:
print(
f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Skipping."
)
print(f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Skipping.")
continue
if DEBUG >= 1:
@@ -167,9 +148,7 @@ class TailscaleDiscovery(Discovery):
else:
if not await self.known_peers[peer_id][0].health_check():
if DEBUG >= 1:
print(
f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Removing."
)
print(f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Removing.")
if peer_id in self.known_peers:
del self.known_peers[peer_id]
continue
@@ -204,9 +183,7 @@ class TailscaleDiscovery(Discovery):
if wait_for_peers > 0:
while len(self.known_peers) < wait_for_peers:
if DEBUG_DISCOVERY >= 2:
print(
f"Current peers: {len(self.known_peers)}/{wait_for_peers}. Waiting for more peers..."
)
print(f"Current peers: {len(self.known_peers)}/{wait_for_peers}. Waiting for more peers...")
await asyncio.sleep(0.1)
return [peer_handle for peer_handle, _, _ in self.known_peers.values()]
@@ -239,9 +216,7 @@ class TailscaleDiscovery(Discovery):
if peer_id in self.known_peers:
del self.known_peers[peer_id]
if DEBUG_DISCOVERY >= 2:
print(
f"Removed peer {peer_id} due to inactivity or failed health check."
)
print(f"Removed peer {peer_id} due to inactivity or failed health check.")
except Exception as e:
print(f"Error in cleanup peers: {e}")
print(traceback.format_exc())
@@ -249,9 +224,7 @@ class TailscaleDiscovery(Discovery):
await asyncio.sleep(self.discovery_interval)
async def check_peer(self, peer_id: str, current_time: float) -> bool:
peer_handle, connected_at, last_seen = self.known_peers.get(
peer_id, (None, None, None)
)
peer_handle, connected_at, last_seen = self.known_peers.get(peer_id, (None, None, None))
if peer_handle is None:
return False
@@ -34,9 +34,7 @@ class Device:
def parse_datetime(date_string: Optional[str]) -> Optional[datetime]:
if not date_string:
return None
return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%SZ").replace(
tzinfo=timezone.utc
)
return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
async def get_device_id() -> str:
@@ -50,17 +48,13 @@ async def get_device_id() -> str:
)
stdout, stderr = await process.communicate()
if process.returncode != 0:
raise Exception(
f"Command failed with exit code {process.returncode}: {stderr.decode().strip()}."
)
raise Exception(f"Command failed with exit code {process.returncode}: {stderr.decode().strip()}.")
if DEBUG_DISCOVERY >= 4:
print(f"tailscale status: {stdout.decode()}")
data = json.loads(stdout.decode())
return data["Self"]["ID"]
except Exception as e:
raise Exception(
f"{str(e)} Do you have the tailscale cli installed? See: https://tailscale.com/kb/1080/cli"
)
raise Exception(f"{str(e)} Do you have the tailscale cli installed? See: https://tailscale.com/kb/1080/cli")
async def update_device_attributes(
@@ -80,44 +74,28 @@ async def update_device_attributes(
attributes = {
"custom:exo_node_id": node_id.replace("-", "_"),
"custom:exo_node_port": node_port,
"custom:exo_device_capability_chip": sanitize_attribute(
device_capabilities.chip
),
"custom:exo_device_capability_model": sanitize_attribute(
device_capabilities.model
),
"custom:exo_device_capability_chip": sanitize_attribute(device_capabilities.chip),
"custom:exo_device_capability_model": sanitize_attribute(device_capabilities.model),
"custom:exo_device_capability_memory": str(device_capabilities.memory),
"custom:exo_device_capability_flops_fp16": str(
device_capabilities.flops.fp16
),
"custom:exo_device_capability_flops_fp32": str(
device_capabilities.flops.fp32
),
"custom:exo_device_capability_flops_int8": str(
device_capabilities.flops.int8
),
"custom:exo_device_capability_flops_fp16": str(device_capabilities.flops.fp16),
"custom:exo_device_capability_flops_fp32": str(device_capabilities.flops.fp32),
"custom:exo_device_capability_flops_int8": str(device_capabilities.flops.int8),
}
for attr_name, attr_value in attributes.items():
url = f"{base_url}/{attr_name}"
data = {
"value": str(attr_value).replace(" ", "_")
} # Ensure all values are strings for JSON
data = {"value": str(attr_value).replace(" ", "_")} # Ensure all values are strings for JSON
async with session.post(url, headers=headers, json=data) as response:
if response.status == 200:
if DEBUG_DISCOVERY >= 1:
print(
f"Updated device posture attribute {attr_name} for device {device_id}"
)
print(f"Updated device posture attribute {attr_name} for device {device_id}")
else:
print(
f"Failed to update device posture attribute {attr_name}: {response.status} {await response.text()}"
)
async def get_device_attributes(
device_id: str, api_key: str
) -> Tuple[str, int, DeviceCapabilities]:
async def get_device_attributes(device_id: str, api_key: str) -> Tuple[str, int, DeviceCapabilities]:
async with aiohttp.ClientSession() as session:
url = f"https://api.tailscale.com/api/v2/device/{device_id}/attributes"
headers = {"Authorization": f"Bearer {api_key}"}
@@ -128,32 +106,18 @@ async def get_device_attributes(
node_id = attributes.get("custom:exo_node_id", "").replace("_", "-")
node_port = int(attributes.get("custom:exo_node_port", 0))
device_capabilities = DeviceCapabilities(
model=attributes.get(
"custom:exo_device_capability_model", ""
).replace("_", " "),
chip=attributes.get(
"custom:exo_device_capability_chip", ""
).replace("_", " "),
memory=int(
attributes.get("custom:exo_device_capability_memory", 0)
),
model=attributes.get("custom:exo_device_capability_model", "").replace("_", " "),
chip=attributes.get("custom:exo_device_capability_chip", "").replace("_", " "),
memory=int(attributes.get("custom:exo_device_capability_memory", 0)),
flops=DeviceFlops(
fp16=float(
attributes.get("custom:exo_device_capability_flops_fp16", 0)
),
fp32=float(
attributes.get("custom:exo_device_capability_flops_fp32", 0)
),
int8=float(
attributes.get("custom:exo_device_capability_flops_int8", 0)
),
fp16=float(attributes.get("custom:exo_device_capability_flops_fp16", 0)),
fp32=float(attributes.get("custom:exo_device_capability_flops_fp32", 0)),
int8=float(attributes.get("custom:exo_device_capability_flops_int8", 0)),
),
)
return node_id, node_port, device_capabilities
else:
print(
f"Failed to fetch posture attributes for {device_id}: {response.status}"
)
print(f"Failed to fetch posture attributes for {device_id}: {response.status}")
return (
"",
0,
@@ -52,13 +52,9 @@ class BroadcastProtocol(asyncio.DatagramProtocol):
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
# Try both subnet-specific and global broadcast
broadcast_addr = get_broadcast_address(self.source_ip)
transport.sendto(
self.message.encode("utf-8"), (broadcast_addr, self.broadcast_port)
)
transport.sendto(self.message.encode("utf-8"), (broadcast_addr, self.broadcast_port))
if broadcast_addr != "255.255.255.255":
transport.sendto(
self.message.encode("utf-8"), ("255.255.255.255", self.broadcast_port)
)
transport.sendto(self.message.encode("utf-8"), ("255.255.255.255", self.broadcast_port))
class UDPDiscovery(Discovery):
@@ -115,18 +111,14 @@ class UDPDiscovery(Discovery):
if wait_for_peers > 0:
while len(self.known_peers) < wait_for_peers:
if DEBUG_DISCOVERY >= 2:
print(
f"Current peers: {len(self.known_peers)}/{wait_for_peers}. Waiting for more peers..."
)
print(f"Current peers: {len(self.known_peers)}/{wait_for_peers}. Waiting for more peers...")
await asyncio.sleep(0.1)
return [peer_handle for peer_handle, _, _, _ in self.known_peers.values()]
async def task_broadcast_presence(self):
while True:
for addr, interface_name in get_all_ip_addresses_and_interfaces():
interface_priority, interface_type = (
await get_interface_priority_and_type(interface_name)
)
interface_priority, interface_type = await get_interface_priority_and_type(interface_name)
message = json.dumps(
{
"type": "discovery",
@@ -158,9 +150,7 @@ class UDPDiscovery(Discovery):
sock=sock,
)
except Exception as e:
print(
f"Error in broadcast presence ({addr} - {interface_name} - {interface_priority}): {e}"
)
print(f"Error in broadcast presence ({addr} - {interface_name} - {interface_priority}): {e}")
finally:
if transport:
try:
@@ -200,9 +190,7 @@ class UDPDiscovery(Discovery):
# Skip if peer_id is not in allowed list
if self.allowed_node_ids and peer_id not in self.allowed_node_ids:
if DEBUG_DISCOVERY >= 2:
print(
f"Ignoring peer {peer_id} as it's not in the allowed node IDs list"
)
print(f"Ignoring peer {peer_id} as it's not in the allowed node IDs list")
return
peer_host = addr[0]
@@ -212,10 +200,7 @@ class UDPDiscovery(Discovery):
peer_interface_type = message["interface_type"]
# Skip if interface type is not in allowed list
if (
self.allowed_interface_types
and peer_interface_type not in self.allowed_interface_types
):
if self.allowed_interface_types and peer_interface_type not in self.allowed_interface_types:
if DEBUG_DISCOVERY >= 2:
print(
f"Ignoring peer {peer_id} as its interface type {peer_interface_type} is not in the allowed interface types list"
@@ -224,10 +209,7 @@ class UDPDiscovery(Discovery):
device_capabilities = DeviceCapabilities(**message["device_capabilities"])
if (
peer_id not in self.known_peers
or self.known_peers[peer_id][0].addr() != f"{peer_host}:{peer_port}"
):
if peer_id not in self.known_peers or self.known_peers[peer_id][0].addr() != f"{peer_host}:{peer_port}":
if peer_id in self.known_peers:
existing_peer_prio = self.known_peers[peer_id][3]
if existing_peer_prio >= peer_prio:
@@ -244,9 +226,7 @@ class UDPDiscovery(Discovery):
)
if not await new_peer_handle.health_check():
if DEBUG >= 1:
print(
f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Skipping."
)
print(f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Skipping.")
return
if DEBUG >= 1:
print(
@@ -261,9 +241,7 @@ class UDPDiscovery(Discovery):
else:
if not await self.known_peers[peer_id][0].health_check():
if DEBUG >= 1:
print(
f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Removing."
)
print(f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Removing.")
if peer_id in self.known_peers:
del self.known_peers[peer_id]
return
@@ -312,9 +290,7 @@ class UDPDiscovery(Discovery):
if peer_id in self.known_peers:
del self.known_peers[peer_id]
if DEBUG_DISCOVERY >= 2:
print(
f"Removed peer {peer_id} due to inactivity or failed health check."
)
print(f"Removed peer {peer_id} due to inactivity or failed health check.")
except Exception as e:
print(f"Error in cleanup peers: {e}")
print(traceback.format_exc())
@@ -322,9 +298,7 @@ class UDPDiscovery(Discovery):
await asyncio.sleep(self.broadcast_interval)
async def check_peer(self, peer_id: str, current_time: float) -> bool:
peer_handle, connected_at, last_seen, prio = self.known_peers.get(
peer_id, (None, None, None, None)
)
peer_handle, connected_at, last_seen, prio = self.known_peers.get(peer_id, (None, None, None, None))
if peer_handle is None:
return False
@@ -62,13 +62,9 @@ class BroadcastProtocol(asyncio.DatagramProtocol):
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
# Try both subnet-specific and global broadcast
broadcast_addr = get_broadcast_address(self.source_ip)
transport.sendto(
self.message.encode("utf-8"), (broadcast_addr, self.broadcast_port)
)
transport.sendto(self.message.encode("utf-8"), (broadcast_addr, self.broadcast_port))
if broadcast_addr != "255.255.255.255":
transport.sendto(
self.message.encode("utf-8"), ("255.255.255.255", self.broadcast_port)
)
transport.sendto(self.message.encode("utf-8"), ("255.255.255.255", self.broadcast_port))
transport.close()
@@ -116,9 +112,7 @@ class UDPDiscovery(Discovery):
self.listen_task = asyncio.create_task(self._listen_loop())
self.cleanup_task = asyncio.create_task(self._cleanup_loop())
print(
f"UDP Discovery started for node {self.node_id} on port {self.listen_port}"
)
print(f"UDP Discovery started for node {self.node_id} on port {self.listen_port}")
async def stop(self) -> None:
"""Stop UDP discovery."""
@@ -150,9 +144,7 @@ class UDPDiscovery(Discovery):
start_time = time.time()
while len(self.known_peers) < wait_for_peers:
if time.time() - start_time > 30: # 30 second timeout
print(
f"Timeout waiting for {wait_for_peers} peers, found {len(self.known_peers)}"
)
print(f"Timeout waiting for {wait_for_peers} peers, found {len(self.known_peers)}")
break
await asyncio.sleep(0.1)
@@ -168,11 +160,7 @@ class UDPDiscovery(Discovery):
"type": "discovery",
"node_id": self.node_id,
"listen_port": self.listen_port,
"device_capabilities": (
self.device_capabilities.to_dict()
if self.device_capabilities
else {}
),
"device_capabilities": (self.device_capabilities.to_dict() if self.device_capabilities else {}),
"timestamp": time.time(),
}
)
@@ -188,9 +176,7 @@ class UDPDiscovery(Discovery):
sock.bind((self.local_ip, 0))
transport, _ = await asyncio.get_event_loop().create_datagram_endpoint(
lambda: BroadcastProtocol(
message, self.broadcast_port, self.local_ip
),
lambda: BroadcastProtocol(message, self.broadcast_port, self.local_ip),
sock=sock,
)
@@ -217,9 +203,7 @@ class UDPDiscovery(Discovery):
except Exception as e:
print(f"Error in listen loop: {e}")
async def _handle_discovery_message(
self, data: bytes, addr: Tuple[str, int]
) -> None:
async def _handle_discovery_message(self, data: bytes, addr: Tuple[str, int]) -> None:
"""Handle incoming discovery message."""
try:
# Decode message
@@ -252,9 +236,7 @@ class UDPDiscovery(Discovery):
if peer_id not in self.known_peers:
# New peer discovered
peer_handle = MinimalPeerHandle(
peer_id=peer_id, address=peer_address, capabilities=peer_caps
)
peer_handle = MinimalPeerHandle(peer_id=peer_id, address=peer_address, capabilities=peer_caps)
self.known_peers[peer_id] = (peer_handle, current_time, current_time)
print(f"Discovered new peer: {peer_id} at {peer_address}")
else:
@@ -97,9 +97,7 @@ class DistributedNetwork(Network[T]):
self.sync_task = asyncio.create_task(self._sync_loop())
self.is_running = True
print(
f"Distributed network node {self.node_id} started with {len(self.peers)} peers"
)
print(f"Distributed network node {self.node_id} started with {len(self.peers)} peers")
async def stop(self) -> None:
"""Stop the distributed network node."""
@@ -154,9 +152,7 @@ class DistributedNetwork(Network[T]):
peer = self._get_peer(peer_id)
if peer:
# Execute remotely
return await self._execute_remote(
peer, prompt, initial_agent, dist_context
)
return await self._execute_remote(peer, prompt, initial_agent, dist_context)
# Execute locally (using base implementation)
return await super().run(prompt, initial_agent, dist_context)
@@ -256,9 +252,7 @@ class DistributedNetwork(Network[T]):
{
"id": p.id,
"address": p.address,
"capabilities": (
p.device_capabilities.__dict__ if p.device_capabilities else {}
),
"capabilities": (p.device_capabilities.__dict__ if p.device_capabilities else {}),
}
for p in self.peers
],
@@ -275,9 +269,7 @@ class DistributedNetwork(Network[T]):
{
"name": name,
"type": agent.__class__.__name__,
"has_tools": (
len(agent.tools) > 0 if hasattr(agent, "tools") else False
),
"has_tools": (len(agent.tools) > 0 if hasattr(agent, "tools") else False),
}
)
+6 -21
View File
@@ -15,9 +15,7 @@ DEBUG_DISCOVERY = int(os.getenv("DEBUG_DISCOVERY", default="0"))
VERSION = "0.1.0"
# Single shared thread pool for subprocess operations
subprocess_pool = ThreadPoolExecutor(
max_workers=4, thread_name_prefix="subprocess_worker"
)
subprocess_pool = ThreadPoolExecutor(max_workers=4, thread_name_prefix="subprocess_worker")
def get_system_info():
@@ -33,9 +31,7 @@ def get_system_info():
return "Non-Mac, non-Linux system"
def find_available_port(
host: str = "", min_port: int = 49152, max_port: int = 65535
) -> int:
def find_available_port(host: str = "", min_port: int = 49152, max_port: int = 65535) -> int:
"""Find an available port in the specified range."""
for _ in range(100): # Try 100 times
port = random.randint(min_port, max_port)
@@ -82,12 +78,7 @@ async def get_interface_priority_and_type(ifname: str) -> Tuple[int, str]:
return (6, "Loopback")
# Container/virtual interfaces
if (
ifname.startswith(
("docker", "br-", "veth", "cni", "flannel", "calico", "weave")
)
or "bridge" in ifname
):
if ifname.startswith(("docker", "br-", "veth", "cni", "flannel", "calico", "weave")) or "bridge" in ifname:
return (7, "Container Virtual")
# Thunderbolt
@@ -115,22 +106,16 @@ async def get_mac_system_info() -> Tuple[str, str, int]:
try:
output = await asyncio.get_running_loop().run_in_executor(
subprocess_pool,
lambda: subprocess.check_output(
["system_profiler", "SPHardwareDataType"]
).decode("utf-8"),
lambda: subprocess.check_output(["system_profiler", "SPHardwareDataType"]).decode("utf-8"),
)
model_line = next(
(line for line in output.split("\n") if "Model Name" in line), None
)
model_line = next((line for line in output.split("\n") if "Model Name" in line), None)
model_id = model_line.split(": ")[1] if model_line else "Unknown Model"
chip_line = next((line for line in output.split("\n") if "Chip" in line), None)
chip_id = chip_line.split(": ")[1] if chip_line else "Unknown Chip"
memory_line = next(
(line for line in output.split("\n") if "Memory" in line), None
)
memory_line = next((line for line in output.split("\n") if "Memory" in line), None)
memory_str = memory_line.split(": ")[1] if memory_line else "Unknown Memory"
memory_units = memory_str.split()
memory_value = int(memory_units[0])
@@ -19,9 +19,7 @@ class DummyInferenceEngine(InferenceEngine):
async def encode(self, shard: Shard, prompt: str) -> np.ndarray:
return np.array(self.tokenizer.encode(prompt))
async def sample(
self, x: np.ndarray, temp: float = 0.0, top_p: float = 1.0
) -> np.ndarray:
async def sample(self, x: np.ndarray, temp: float = 0.0, top_p: float = 1.0) -> np.ndarray:
if x[0] > self.num_generate_dummy_tokens:
return np.array([self.tokenizer.eos_token_id])
return x
@@ -58,9 +58,7 @@ class InferenceEngine(ABC):
x = tokens.reshape(1, -1)
else:
x = tokens
output_data, inference_state = await self.infer_tensor(
request_id, shard, x, inference_state
)
output_data, inference_state = await self.infer_tensor(request_id, shard, x, inference_state)
return output_data, inference_state
@@ -22,9 +22,7 @@ def back_gradient_loss(model, inputs, gradients, lengths):
grad = gradients.astype(mx.float32)
# Mask padding tokens
length_mask = mx.repeat(
mx.arange(inputs.shape[1])[None, :] < lengths[:, None], out.shape[-1]
).reshape(out.shape)
length_mask = mx.repeat(mx.arange(inputs.shape[1])[None, :] < lengths[:, None], out.shape[-1]).reshape(out.shape)
masked_sum = (out * length_mask).sum(axis=1)
gradient_lens = mx.abs(grad * masked_sum)
@@ -29,13 +29,7 @@ class DiffusionConfig:
@classmethod
def from_dict(cls, params):
return cls(
**{
k: v
for k, v in params.items()
if k in inspect.signature(cls).parameters
}
)
return cls(**{k: v for k, v in params.items() if k in inspect.signature(cls).parameters})
# Sampler
@@ -66,22 +60,16 @@ class SimpleEulerSampler:
def __init__(self, config: DiffusionConfig):
# Compute the noise schedule
if config.beta_schedule == "linear":
betas = _linspace(
config.beta_start, config.beta_end, config.num_train_steps
)
betas = _linspace(config.beta_start, config.beta_end, config.num_train_steps)
elif config.beta_schedule == "scaled_linear":
betas = _linspace(
config.beta_start**0.5, config.beta_end**0.5, config.num_train_steps
).square()
betas = _linspace(config.beta_start**0.5, config.beta_end**0.5, config.num_train_steps).square()
else:
raise NotImplementedError(f"{config.beta_schedule} is not implemented.")
alphas = 1 - betas
alphas_cumprod = mx.cumprod(alphas)
self._sigmas = mx.concatenate(
[mx.zeros(1), ((1 - alphas_cumprod) / alphas_cumprod).sqrt()]
)
self._sigmas = mx.concatenate([mx.zeros(1), ((1 - alphas_cumprod) / alphas_cumprod).sqrt()])
@property
def max_time(self):
@@ -89,9 +77,7 @@ class SimpleEulerSampler:
def sample_prior(self, shape, dtype=mx.float32, key=None):
noise = mx.random.normal(shape, key=key)
return (
noise * self._sigmas[-1] * (self._sigmas[-1].square() + 1).rsqrt()
).astype(dtype)
return (noise * self._sigmas[-1] * (self._sigmas[-1].square() + 1).rsqrt()).astype(dtype)
def add_noise(self, x, t, key=None):
noise = mx.random.normal(x.shape, key=key)
@@ -145,13 +131,7 @@ class StableDiffusionConfig:
@classmethod
def from_dict(cls, params):
return cls(
**{
k: v
for k, v in params.items()
if k in inspect.signature(cls).parameters
}
)
return cls(**{k: v for k, v in params.items() if k in inspect.signature(cls).parameters})
@dataclass
@@ -163,9 +143,7 @@ class ModelArgs(StableDiffusionConfig):
self.shard = Shard(**self.shard)
if not isinstance(self.shard, Shard):
raise TypeError(
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
)
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
class Model(nn.Module):
@@ -175,17 +153,13 @@ class Model(nn.Module):
self.config = config
self.model_path = config.vae["path"].split("/vae")[0]
self.shard = config.shard
self.shard_clip, self.shard_encoder, self.shard_unet, self.shard_decoder = (
model_shards(config.shard)
)
self.shard_clip, self.shard_encoder, self.shard_unet, self.shard_decoder = model_shards(config.shard)
self.config_clip = CLIPArgs.from_dict(config.text_encoder["config"])
if self.shard_clip.start_layer != -1:
self.text_encoder = CLIPTextModel(self.config_clip, shard=self.shard_clip)
else:
self.text_encoder = nn.Identity()
self.tokenizer = load_tokenizer(
Path(self.model_path), "vocab.json", "merges.txt"
)
self.tokenizer = load_tokenizer(Path(self.model_path), "vocab.json", "merges.txt")
self.diffusion_config = DiffusionConfig.from_dict(config.scheduler["config"])
self.sampler = SimpleEulerSampler(self.diffusion_config)
if self.shard_unet.start_layer != -1:
@@ -195,15 +169,11 @@ class Model(nn.Module):
self.unet = nn.Identity()
self.config_vae = VAEArgs.from_dict(config.vae["config"])
if self.shard_encoder.start_layer != -1:
self.encoder = Autoencoder(
self.config_vae, self.shard_encoder, "vae_encoder"
)
self.encoder = Autoencoder(self.config_vae, self.shard_encoder, "vae_encoder")
else:
self.encoder = nn.Identity()
if self.shard_decoder.start_layer != -1:
self.decoder = Autoencoder(
self.config_vae, self.shard_decoder, "vae_decoder"
)
self.decoder = Autoencoder(self.config_vae, self.shard_decoder, "vae_decoder")
else:
self.decoder = nn.Identity()
@@ -223,9 +193,7 @@ class Model(nn.Module):
strength=0.65,
start_step=None,
):
t, t_prev = self.sampler.current_timestep(
step=step, total_steps=total_steps, start_time=start_step
)
t, t_prev = self.sampler.current_timestep(step=step, total_steps=total_steps, start_time=start_step)
is_finished = False
is_step_finished = False
if t.item() == 1000:
@@ -261,15 +229,11 @@ class Model(nn.Module):
if step < total_steps:
x = x_t_prev
if self.shard_unet.is_first_layer():
x_t_unet = (
mx.concatenate([x] * 2, axis=0) if cfg_weight > 1 else x
)
x_t_unet = mx.concatenate([x] * 2, axis=0) if cfg_weight > 1 else x
else:
x_t_unet = x
t_unet = mx.broadcast_to(t, [len(x_t_unet)])
x, residual = self.unet(
x_t_unet, t_unet, encoder_x=conditioning, residuals=residual
)
x, residual = self.unet(x_t_unet, t_unet, encoder_x=conditioning, residuals=residual)
if self.shard_unet.is_last_layer():
if cfg_weight > 1:
eps_text, eps_neg = x.split(2)
@@ -340,9 +304,7 @@ def model_shards(shard: ShardConfig):
# Adjust the layers relative to the model's range
relative_start = overlap_start - range_start
relative_end = overlap_end - range_start
shards[model_name] = Shard(
model_name, relative_start, relative_end, range_end - range_start
)
shards[model_name] = Shard(model_name, relative_start, relative_end, range_end - range_start)
else:
# If no overlap, create a zero-layer shard
shards[model_name] = Shard(model_name, -1, -1, range_end - range_start)
@@ -18,9 +18,7 @@ class ModelArgs(ModelArgs):
if isinstance(self.shard, Shard):
return
if not isinstance(self.shard, dict):
raise TypeError(
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
)
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
self.shard = Shard(**self.shard)
@@ -96,11 +94,7 @@ class Model(nn.Module):
for key, value in weights.items():
if key.startswith("model.layers."):
layer_num = int(key.split(".")[2])
if (
self.args.shard.start_layer
<= layer_num
<= self.args.shard.end_layer
):
if self.args.shard.start_layer <= layer_num <= self.args.shard.end_layer:
shard_state_dict[key] = value
elif (
self.args.shard.is_first_layer()
@@ -119,9 +113,7 @@ class Model(nn.Module):
shard_state_dict.pop(f"{prefix}.mlp.experts.{e}.{m}.{k}")
for e in range(self.args.n_routed_experts)
]
shard_state_dict[f"{prefix}.mlp.switch_mlp.{m}.{k}"] = mx.stack(
to_join
)
shard_state_dict[f"{prefix}.mlp.switch_mlp.{m}.{k}"] = mx.stack(to_join)
return shard_state_dict
@@ -21,9 +21,7 @@ class ModelArgs(V3ModelArgs):
if isinstance(self.shard, Shard):
return
if not isinstance(self.shard, dict):
raise TypeError(
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
)
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
self.shard = Shard(**self.shard)
@@ -99,11 +97,7 @@ class Model(nn.Module):
for key, value in weights.items():
if key.startswith("model.layers."):
layer_num = int(key.split(".")[2])
if (
self.args.shard.start_layer
<= layer_num
<= self.args.shard.end_layer
):
if self.args.shard.start_layer <= layer_num <= self.args.shard.end_layer:
shard_state_dict[key] = value
elif (
self.args.shard.is_first_layer()
@@ -123,9 +117,7 @@ class Model(nn.Module):
shard_state_dict.pop(f"{prefix}.mlp.experts.{e}.{m}.{k}")
for e in range(self.args.n_routed_experts)
]
shard_state_dict[f"{prefix}.mlp.switch_mlp.{m}.{k}"] = mx.stack(
to_join
)
shard_state_dict[f"{prefix}.mlp.switch_mlp.{m}.{k}"] = mx.stack(to_join)
return shard_state_dict
@@ -18,9 +18,7 @@ class ModelArgs(ModelArgs):
if isinstance(self.shard, Shard):
return
if not isinstance(self.shard, dict):
raise TypeError(
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
)
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
self.shard = Shard(**self.shard)
@@ -98,11 +96,7 @@ class Model(nn.Module):
continue
if key.startswith("model.layers."):
layer_num = int(key.split(".")[2])
if (
self.args.shard.start_layer
<= layer_num
<= self.args.shard.end_layer
):
if self.args.shard.start_layer <= layer_num <= self.args.shard.end_layer:
shard_state_dict[key] = value
elif (
(self.args.shard.is_first_layer() or self.args.shard.is_last_layer())
@@ -20,9 +20,7 @@ class ModelArgs(ModelArgs):
if isinstance(self.shard, Shard):
return
if not isinstance(self.shard, dict):
raise TypeError(
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
)
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
self.shard = Shard(**self.shard)
@@ -34,9 +32,7 @@ class LlamaModel(nn.Module):
self.vocab_size = args.vocab_size
self.num_hidden_layers = args.num_hidden_layers
assert self.vocab_size > 0
if args.shard.is_first_layer() or (
args.shard.is_last_layer() and args.tie_word_embeddings
):
if args.shard.is_first_layer() or (args.shard.is_last_layer() and args.tie_word_embeddings):
self.embed_tokens = nn.Embedding(args.vocab_size, args.hidden_size)
self.layers = []
for i in range(self.num_hidden_layers):
@@ -103,21 +99,14 @@ class Model(nn.Module):
continue
if key.startswith("model.layers."):
layer_num = int(key.split(".")[2])
if (
self.args.shard.start_layer
<= layer_num
<= self.args.shard.end_layer
):
if self.args.shard.start_layer <= layer_num <= self.args.shard.end_layer:
shard_state_dict[key] = value
elif (
self.args.shard.is_first_layer()
and key.startswith("model.embed_tokens")
or (self.args.shard.is_last_layer() and self.args.tie_word_embeddings)
and key.startswith("model.embed_tokens")
or (
self.args.shard.is_last_layer()
and not self.args.tie_word_embeddings
)
or (self.args.shard.is_last_layer() and not self.args.tie_word_embeddings)
and key.startswith("lm_head")
or self.args.shard.is_last_layer()
and (key.startswith("model.norm"))
@@ -132,9 +121,7 @@ class Model(nn.Module):
@property
def head_dim(self):
return (
self.args.head_dim or self.args.hidden_size // self.args.num_attention_heads
)
return self.args.head_dim or self.args.hidden_size // self.args.num_attention_heads
@property
def n_kv_heads(self):
@@ -29,13 +29,7 @@ class VisionConfig:
@classmethod
def from_dict(cls, params):
return cls(
**{
k: v
for k, v in params.items()
if k in inspect.signature(cls).parameters
}
)
return cls(**{k: v for k, v in params.items() if k in inspect.signature(cls).parameters})
class VisionAttention(nn.Module):
@@ -108,9 +102,7 @@ class VisionEncoderLayer(nn.Module):
def __init__(self, config: VisionConfig):
super().__init__()
self.embed_dim = config.hidden_size
self.self_attn = VisionAttention(
config.hidden_size, config.num_attention_heads, bias=True
)
self.self_attn = VisionAttention(config.hidden_size, config.num_attention_heads, bias=True)
self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
self.mlp = VisionMLP(config)
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
@@ -127,9 +119,7 @@ class VisionEncoderLayer(nn.Module):
class VisionEncoder(nn.Module):
def __init__(self, config: VisionConfig):
super().__init__()
self.layers = [
VisionEncoderLayer(config) for _ in range(config.num_hidden_layers)
]
self.layers = [VisionEncoderLayer(config) for _ in range(config.num_hidden_layers)]
class VisionEmbeddings(nn.Module):
@@ -159,9 +149,7 @@ class VisionEmbeddings(nn.Module):
patch_embeddings = self.patch_embedding(x)
patch_embeddings = mx.flatten(patch_embeddings, start_axis=1, end_axis=2)
embed_dim = patch_embeddings.shape[-1]
cls_embeddings = mx.broadcast_to(
self.class_embedding, (batch_size, 1, embed_dim)
)
cls_embeddings = mx.broadcast_to(self.class_embedding, (batch_size, 1, embed_dim))
embeddings = mx.concatenate((cls_embeddings, patch_embeddings), axis=1)
embeddings += self.position_embedding.weight
return embeddings
@@ -204,9 +192,7 @@ class VisionModel(nn.Module):
self.vision_model = ClipVisionModel(config)
def __call__(
self, x: mx.array, output_hidden_states: Optional[bool] = None
) -> mx.array:
def __call__(self, x: mx.array, output_hidden_states: Optional[bool] = None) -> mx.array:
return self.vision_model(x, output_hidden_states)
def sanitize(self, weights):
@@ -244,13 +230,7 @@ class TextConfig:
@classmethod
def from_dict(cls, params):
return cls(
**{
k: v
for k, v in params.items()
if k in inspect.signature(cls).parameters
}
)
return cls(**{k: v for k, v in params.items() if k in inspect.signature(cls).parameters})
def __post_init__(self):
if self.num_key_value_heads is None:
@@ -291,8 +271,7 @@ class TextAttention(nn.Module):
rope_scale = (
1 / config.rope_scaling["factor"]
if config.rope_scaling is not None
and config.rope_scaling["type"] == "linear"
if config.rope_scaling is not None and config.rope_scaling["type"] == "linear"
else 1
)
self.rope = nn.RoPE(
@@ -325,9 +304,7 @@ class TextAttention(nn.Module):
queries = self.rope(queries)
keys = self.rope(keys)
output = mx.fast.scaled_dot_product_attention(
queries, keys, values, scale=self.scale, mask=mask
)
output = mx.fast.scaled_dot_product_attention(queries, keys, values, scale=self.scale, mask=mask)
output = output.transpose(0, 2, 1, 3).reshape(B, L, -1)
return self.o_proj(output)
@@ -351,9 +328,7 @@ class TransformerBlock(nn.Module):
self.self_attn = TextAttention(config)
self.mlp = TextMLP(config.hidden_size, config.intermediate_size)
self.input_layernorm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = nn.RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.post_attention_layernorm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.config = config
def __call__(
@@ -427,9 +402,7 @@ class LanguageModel(nn.Module):
super().__init__()
self.model_type = config.model_type
if self.model_type != "llama":
raise ValueError(
f"Model type {self.model_type} not supported. Currently only 'llama' is supported"
)
raise ValueError(f"Model type {self.model_type} not supported. Currently only 'llama' is supported")
self.shard = shard
self.model = Llama(config, shard)
if self.shard.is_last_layer():
@@ -454,19 +427,13 @@ class LanguageModel(nn.Module):
if key.startswith("language_model.model.layers."):
layer_num = int(key.split(".")[3])
if (
layer_num < self.shard.start_layer
or layer_num > self.shard.end_layer
):
if layer_num < self.shard.start_layer or layer_num > self.shard.end_layer:
continue
if (
not self.shard.is_first_layer()
and key.startswith("language_model.model.embed_tokens")
or not self.shard.is_last_layer()
and (
key.startswith("language_model.model.norm")
or key.startswith("language_model.lm_head")
)
and (key.startswith("language_model.model.norm") or key.startswith("language_model.lm_head"))
):
continue
@@ -508,9 +475,7 @@ class ModelArgs(LlaVAConfig):
self.shard = Shard(**self.shard)
if not isinstance(self.shard, Shard):
raise TypeError(
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
)
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
if not self.shard.is_first_layer():
self.vision_config = None
@@ -519,13 +484,9 @@ class ModelArgs(LlaVAConfig):
class LlavaMultiModalProjector(nn.Module):
def __init__(self, config: LlaVAConfig):
super().__init__()
self.linear_1 = nn.Linear(
config.vision_config.hidden_size, config.text_config.hidden_size, bias=True
)
self.linear_1 = nn.Linear(config.vision_config.hidden_size, config.text_config.hidden_size, bias=True)
self.gelu = nn.GELU()
self.linear_2 = nn.Linear(
config.text_config.hidden_size, config.text_config.hidden_size, bias=True
)
self.linear_2 = nn.Linear(config.text_config.hidden_size, config.text_config.hidden_size, bias=True)
def __call__(self, x: mx.array) -> mx.array:
x = self.linear_1(x)
@@ -558,9 +519,7 @@ class Model(nn.Module):
inputs_embeds = self.language_model.model.embed_tokens(input_ids)
# Get the ouptut hidden states from the vision model
*_, hidden_states = self.vision_tower(
pixel_values.transpose(0, 2, 3, 1), output_hidden_states=True
)
*_, hidden_states = self.vision_tower(pixel_values.transpose(0, 2, 3, 1), output_hidden_states=True)
# Select the hidden states from the desired layer
selected_image_feature = hidden_states[self.vision_feature_layer]
@@ -570,22 +529,16 @@ class Model(nn.Module):
elif self.vision_feature_select_strategy == "full":
selected_image_feature = selected_image_feature
else:
raise ValueError(
f"Unexpected feature selection strategy: {self.vision_feature_select_strategy}"
)
raise ValueError(f"Unexpected feature selection strategy: {self.vision_feature_select_strategy}")
# Pass image features through the multi-modal projector
image_features = self.multi_modal_projector(selected_image_feature)
# Insert special image tokens in the input_ids
final_inputs_embeds = self._merge_input_ids_with_image_features(
image_features, inputs_embeds, input_ids
)
final_inputs_embeds = self._merge_input_ids_with_image_features(image_features, inputs_embeds, input_ids)
return final_inputs_embeds
def _merge_input_ids_with_image_features(
self, image_features, inputs_embeds, input_ids
):
def _merge_input_ids_with_image_features(self, image_features, inputs_embeds, input_ids):
image_token_index = self.config.image_token_index
num_images, num_image_patches, embed_dim = image_features.shape
@@ -617,9 +570,7 @@ class Model(nn.Module):
input_embddings = None
if pixel_values is not None:
input_embddings = self.get_input_embeddings(input_ids, pixel_values)
logits = self.language_model(
input_ids, cache=cache, inputs_embeds=input_embddings
)
logits = self.language_model(input_ids, cache=cache, inputs_embeds=input_embddings)
return logits
def sanitize(self, weights):
@@ -649,8 +600,7 @@ class Model(nn.Module):
def head_dim(self):
return (
self.language_model.model.head_dim
or self.language_model.model.hidden_size
// self.language_model.model.num_attention_heads
or self.language_model.model.hidden_size // self.language_model.model.num_attention_heads
)
@property
@@ -20,9 +20,7 @@ class ModelArgs(ModelArgs):
if isinstance(self.shard, Shard):
return
if not isinstance(self.shard, dict):
raise TypeError(
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
)
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
self.shard = Shard(**self.shard)
@@ -100,11 +98,7 @@ class Model(nn.Module):
continue
if key.startswith("model.layers."):
layer_num = int(key.split(".")[2])
if (
self.args.shard.start_layer
<= layer_num
<= self.args.shard.end_layer
):
if self.args.shard.start_layer <= layer_num <= self.args.shard.end_layer:
shard_state_dict[key] = value
elif (
self.args.shard.is_first_layer()
@@ -20,9 +20,7 @@ class ModelArgs(ModelArgs):
if isinstance(self.shard, Shard):
return
if not isinstance(self.shard, dict):
raise TypeError(
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
)
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
self.shard = Shard(**self.shard)
@@ -35,9 +33,7 @@ class Qwen2Model(nn.Module):
self.num_hidden_layers = args.num_hidden_layers
assert self.vocab_size > 0
if self.args.shard.is_first_layer() or (
self.args.shard.is_last_layer() and args.tie_word_embeddings
):
if self.args.shard.is_first_layer() or (self.args.shard.is_last_layer() and args.tie_word_embeddings):
self.embed_tokens = nn.Embedding(args.vocab_size, args.hidden_size)
self.layers = []
@@ -106,21 +102,14 @@ class Model(nn.Module):
continue
if key.startswith("model.layers."):
layer_num = int(key.split(".")[2])
if (
self.args.shard.start_layer
<= layer_num
<= self.args.shard.end_layer
):
if self.args.shard.start_layer <= layer_num <= self.args.shard.end_layer:
shard_state_dict[key] = value
elif (
self.args.shard.is_first_layer()
and key.startswith("model.embed_tokens")
or (self.args.shard.is_last_layer() and self.args.tie_word_embeddings)
and key.startswith("model.embed_tokens")
or (
self.args.shard.is_last_layer()
and not self.args.tie_word_embeddings
)
or (self.args.shard.is_last_layer() and not self.args.tie_word_embeddings)
and key.startswith("lm_head")
or self.args.shard.is_last_layer()
and (key.startswith("model.norm"))
@@ -31,11 +31,7 @@ class CLIPTextModelConfig:
num_heads=config["num_attention_heads"],
max_length=config["max_position_embeddings"],
vocab_size=config["vocab_size"],
projection_dim=(
config["projection_dim"]
if "WithProjection" in config["architectures"][0]
else None
),
projection_dim=(config["projection_dim"] if "WithProjection" in config["architectures"][0] else None),
hidden_act=config.get("hidden_act", "quick_gelu"),
weight_files=config.get("weight_files", []),
)
@@ -51,9 +47,7 @@ class ModelArgs(CLIPTextModelConfig):
self.shard = Shard(**self.shard)
if not isinstance(self.shard, Shard):
raise TypeError(
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
)
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
if not self.shard.is_first_layer():
self.vision_config = None
@@ -106,35 +100,23 @@ class CLIPTextModel(nn.Module):
super().__init__()
self.shard = shard
self.layers_range = range(
self.shard.start_layer * 2, self.shard.end_layer * 2 + 2
)
self.layers_range = range(self.shard.start_layer * 2, self.shard.end_layer * 2 + 2)
if self.shard.is_first_layer():
self.token_embedding = nn.Embedding(config.vocab_size, config.model_dims)
self.position_embedding = nn.Embedding(config.max_length, config.model_dims)
self.layers = []
for i in range(math.ceil(config.num_layers / 2)):
if 2 * i in self.layers_range:
self.layers.append(
CLIPEncoderLayer(
config.model_dims, config.num_heads, config.hidden_act
)
)
self.layers.append(CLIPEncoderLayer(config.model_dims, config.num_heads, config.hidden_act))
if 2 * i + 1 in self.layers_range and 2 * i + 1 < config.num_layers:
self.layers.append(
CLIPEncoderLayer(
config.model_dims, config.num_heads, config.hidden_act
)
)
self.layers.append(CLIPEncoderLayer(config.model_dims, config.num_heads, config.hidden_act))
else:
self.layers.append(IdentityBlock())
if self.shard.is_last_layer():
self.final_layer_norm = nn.LayerNorm(config.model_dims)
if config.projection_dim is not None:
self.text_projection = nn.Linear(
config.model_dims, config.projection_dim, bias=False
)
self.text_projection = nn.Linear(config.model_dims, config.projection_dim, bias=False)
def _get_mask(self, N, dtype):
indices = mx.arange(N)
@@ -49,9 +49,7 @@ class Tokenizer:
#
# Ported from https://github.com/huggingface/transformers/blob/main/src/transformers/models/clip/tokenization_clip.py
while unique_bigrams:
bigram = min(
unique_bigrams, key=lambda pair: self.bpe_ranks.get(pair, float("inf"))
)
bigram = min(unique_bigrams, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
if bigram not in self.bpe_ranks:
break
@@ -49,9 +49,7 @@ class UNetConfig:
out_channels=config["out_channels"],
block_out_channels=config["block_out_channels"],
layers_per_block=[config["layers_per_block"]] * n_blocks,
transformer_layers_per_block=config.get(
"transformer_layers_per_block", (1,) * 4
),
transformer_layers_per_block=config.get("transformer_layers_per_block", (1,) * 4),
num_attention_heads=(
[config["attention_head_dim"]] * n_blocks
if isinstance(config["attention_head_dim"], int)
@@ -63,9 +61,7 @@ class UNetConfig:
up_block_types=config["up_block_types"][::-1],
addition_embed_type=config.get("addition_embed_type", None),
addition_time_embed_dim=config.get("addition_time_embed_dim", None),
projection_class_embeddings_input_dim=config.get(
"projection_class_embeddings_input_dim", None
),
projection_class_embeddings_input_dim=config.get("projection_class_embeddings_input_dim", None),
weight_files=config.get("weight_files", []),
)
@@ -109,9 +105,7 @@ class TransformerBlock(nn.Module):
memory_dims = memory_dims or model_dims
self.norm2 = nn.LayerNorm(model_dims)
self.attn2 = nn.MultiHeadAttention(
model_dims, num_heads, key_input_dims=memory_dims
)
self.attn2 = nn.MultiHeadAttention(model_dims, num_heads, key_input_dims=memory_dims)
self.attn2.out_proj.bias = mx.zeros(model_dims)
hidden_dims = hidden_dims or 4 * model_dims
@@ -159,8 +153,7 @@ class Transformer2D(nn.Module):
self.norm = nn.GroupNorm(norm_num_groups, in_channels, pytorch_compatible=True)
self.proj_in = nn.Linear(in_channels, model_dims)
self.transformer_blocks = [
TransformerBlock(model_dims, num_heads, memory_dims=encoder_dims)
for i in range(num_layers)
TransformerBlock(model_dims, num_heads, memory_dims=encoder_dims) for i in range(num_layers)
]
self.proj_out = nn.Linear(model_dims, in_channels)
@@ -198,15 +191,11 @@ class ResnetBlock2D(nn.Module):
out_channels = out_channels or in_channels
self.norm1 = nn.GroupNorm(groups, in_channels, pytorch_compatible=True)
self.conv1 = nn.Conv2d(
in_channels, out_channels, kernel_size=3, stride=1, padding=1
)
self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
if temb_channels is not None:
self.time_emb_proj = nn.Linear(temb_channels, out_channels)
self.norm2 = nn.GroupNorm(groups, out_channels, pytorch_compatible=True)
self.conv2 = nn.Conv2d(
out_channels, out_channels, kernel_size=3, stride=1, padding=1
)
self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
if in_channels != out_channels:
self.conv_shortcut = nn.Linear(in_channels, out_channels)
@@ -256,9 +245,7 @@ class UNetBlock2D(nn.Module):
else:
in_channels_list = [prev_out_channels] + [out_channels] * (num_layers - 1)
res_channels_list = [out_channels] * (num_layers - 1) + [in_channels]
in_channels_list = [
a + b for a, b in zip(in_channels_list, res_channels_list)
]
in_channels_list = [a + b for a, b in zip(in_channels_list, res_channels_list)]
# Add resnet blocks that also process the time embedding
self.resnets = [
@@ -286,15 +273,11 @@ class UNetBlock2D(nn.Module):
# Add an optional downsampling layer
if add_downsample:
self.downsample = nn.Conv2d(
out_channels, out_channels, kernel_size=3, stride=2, padding=1
)
self.downsample = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=2, padding=1)
# or upsampling layer
if add_upsample:
self.upsample = nn.Conv2d(
out_channels, out_channels, kernel_size=3, stride=1, padding=1
)
self.upsample = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
def __call__(
self,
@@ -349,9 +332,7 @@ class UNetModel(nn.Module):
self.timesteps = nn.SinusoidalPositionalEncoding(
config.block_out_channels[0],
max_freq=1,
min_freq=math.exp(
-math.log(10000) + 2 * math.log(10000) / config.block_out_channels[0]
),
min_freq=math.exp(-math.log(10000) + 2 * math.log(10000) / config.block_out_channels[0]),
scale=1.0,
cos_first=True,
full_turns=False,
@@ -365,10 +346,7 @@ class UNetModel(nn.Module):
self.add_time_proj = nn.SinusoidalPositionalEncoding(
config.addition_time_embed_dim,
max_freq=1,
min_freq=math.exp(
-math.log(10000)
+ 2 * math.log(10000) / config.addition_time_embed_dim
),
min_freq=math.exp(-math.log(10000) + 2 * math.log(10000) / config.addition_time_embed_dim),
scale=1.0,
cos_first=True,
full_turns=False,
@@ -379,14 +357,10 @@ class UNetModel(nn.Module):
)
# Make the downsampling blocks
block_channels = [config.block_out_channels[0]] + list(
config.block_out_channels
)
block_channels = [config.block_out_channels[0]] + list(config.block_out_channels)
self.down_blocks = []
for i, (in_channels, out_channels) in enumerate(
zip(block_channels, block_channels[1:])
):
for i, (in_channels, out_channels) in enumerate(zip(block_channels, block_channels[1:])):
if i in self.layers_range:
self.down_blocks.append(
UNetBlock2D(
@@ -394,9 +368,7 @@ class UNetModel(nn.Module):
out_channels=out_channels,
temb_channels=config.block_out_channels[0] * 4,
num_layers=config.layers_per_block[i],
transformer_layers_per_block=config.transformer_layers_per_block[
i
],
transformer_layers_per_block=config.transformer_layers_per_block[i],
num_attention_heads=config.num_attention_heads[i],
cross_attention_dim=config.cross_attention_dim[i],
resnet_groups=config.norm_num_groups,
@@ -434,20 +406,14 @@ class UNetModel(nn.Module):
# Make the upsampling blocks
block_channels = (
[config.block_out_channels[0]]
+ list(config.block_out_channels)
+ [config.block_out_channels[-1]]
[config.block_out_channels[0]] + list(config.block_out_channels) + [config.block_out_channels[-1]]
)
total_items = len(block_channels) - 3
reversed_channels = list(
reversed(list(zip(block_channels, block_channels[1:], block_channels[2:])))
)
reversed_channels = list(reversed(list(zip(block_channels, block_channels[1:], block_channels[2:]))))
self.up_blocks = []
for rev_i, (in_channels, out_channels, prev_out_channels) in enumerate(
reversed_channels
):
for rev_i, (in_channels, out_channels, prev_out_channels) in enumerate(reversed_channels):
i = total_items - rev_i
if rev_i + 5 in self.layers_range:
self.up_blocks.append(
@@ -457,9 +423,7 @@ class UNetModel(nn.Module):
temb_channels=config.block_out_channels[0] * 4,
prev_out_channels=prev_out_channels,
num_layers=config.layers_per_block[i] + 1,
transformer_layers_per_block=config.transformer_layers_per_block[
i
],
transformer_layers_per_block=config.transformer_layers_per_block[i],
num_attention_heads=config.num_attention_heads[i],
cross_attention_dim=config.cross_attention_dim[i],
resnet_groups=config.norm_num_groups,
@@ -28,13 +28,7 @@ class AutoencoderConfig:
@classmethod
def from_dict(cls, params):
return cls(
**{
k: v
for k, v in params.items()
if k in inspect.signature(cls).parameters
}
)
return cls(**{k: v for k, v in params.items() if k in inspect.signature(cls).parameters})
@dataclass
@@ -46,9 +40,7 @@ class ModelArgs(AutoencoderConfig):
self.shard = Shard(**self.shard)
if not isinstance(self.shard, Shard):
raise TypeError(
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
)
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
if not self.shard.is_first_layer():
self.vision_config = None
@@ -110,15 +102,11 @@ class EncoderDecoderBlock2D(nn.Module):
# Add an optional downsampling layer
if add_downsample:
self.downsample = nn.Conv2d(
out_channels, out_channels, kernel_size=3, stride=2, padding=0
)
self.downsample = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=2, padding=0)
# or upsampling layer
if add_upsample:
self.upsample = nn.Conv2d(
out_channels, out_channels, kernel_size=3, stride=1, padding=1
)
self.upsample = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
def __call__(self, x):
for resnet in self.resnets:
@@ -149,9 +137,7 @@ class Encoder(nn.Module):
self.layers_range = layers_range
self.shard = shard
if self.shard.is_first_layer():
self.conv_in = nn.Conv2d(
in_channels, block_out_channels[0], kernel_size=3, stride=1, padding=1
)
self.conv_in = nn.Conv2d(in_channels, block_out_channels[0], kernel_size=3, stride=1, padding=1)
channels = [block_out_channels[0]] + list(block_out_channels)
self.down_blocks = []
@@ -187,12 +173,8 @@ class Encoder(nn.Module):
),
]
self.conv_norm_out = nn.GroupNorm(
resnet_groups, block_out_channels[-1], pytorch_compatible=True
)
self.conv_out = nn.Conv2d(
block_out_channels[-1], latent_channels_out, 3, padding=1
)
self.conv_norm_out = nn.GroupNorm(resnet_groups, block_out_channels[-1], pytorch_compatible=True)
self.conv_out = nn.Conv2d(block_out_channels[-1], latent_channels_out, 3, padding=1)
def __call__(self, x):
if self.shard.is_first_layer():
@@ -230,9 +212,7 @@ class Decoder(nn.Module):
self.out_channels = out_channels
self.layers_range = layer_range
if 0 in layer_range:
self.conv_in = nn.Conv2d(
in_channels, block_out_channels[-1], kernel_size=3, stride=1, padding=1
)
self.conv_in = nn.Conv2d(in_channels, block_out_channels[-1], kernel_size=3, stride=1, padding=1)
if 0 in layer_range:
self.mid_blocks = [
@@ -271,12 +251,8 @@ class Decoder(nn.Module):
self.up_blocks.append(IdentityBlock())
current_layer += 1
if 4 in layer_range:
self.conv_norm_out = nn.GroupNorm(
resnet_groups, block_out_channels[0], pytorch_compatible=True
)
self.conv_out = nn.Conv2d(
block_out_channels[0], self.out_channels, 3, padding=1
)
self.conv_norm_out = nn.GroupNorm(resnet_groups, block_out_channels[0], pytorch_compatible=True)
self.conv_out = nn.Conv2d(block_out_channels[0], self.out_channels, 3, padding=1)
def __call__(self, x):
if 0 in self.layers_range:
@@ -317,9 +293,7 @@ class Autoencoder(nn.Module):
shard=shard,
)
if self.shard.is_last_layer():
self.quant_proj = nn.Linear(
config.latent_channels_out, config.latent_channels_out
)
self.quant_proj = nn.Linear(config.latent_channels_out, config.latent_channels_out)
if self.model_shard == "vae_decoder":
self.decoder = Decoder(
config.latent_channels_in,
@@ -331,9 +305,7 @@ class Autoencoder(nn.Module):
resnet_groups=config.norm_num_groups,
)
if self.shard.is_first_layer():
self.post_quant_proj = nn.Linear(
config.latent_channels_in, config.latent_channels_in
)
self.post_quant_proj = nn.Linear(config.latent_channels_in, config.latent_channels_in)
def decode(self, z):
if self.shard.is_first_layer():
@@ -23,16 +23,12 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
self.sampler_params: tuple[float, float] = (0.0, 0.0, 0.0, 1)
self.sampler = make_sampler(*self.sampler_params)
self._mlx_thread = ThreadPoolExecutor(max_workers=1, thread_name_prefix="mlx")
self._tokenizer_thread = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="tokenizer"
)
self._tokenizer_thread = ThreadPoolExecutor(max_workers=1, thread_name_prefix="tokenizer")
self.session = {}
self._shard_lock = asyncio.Lock()
async def _eval_mlx(self, *args):
await asyncio.get_running_loop().run_in_executor(
self._mlx_thread, mx.eval, *args
)
await asyncio.get_running_loop().run_in_executor(self._mlx_thread, mx.eval, *args)
async def poll_state(self, request_id: str, max_caches=2):
if request_id in self.caches:
@@ -44,9 +40,7 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
self.caches[request_id] = newcache
return {"cache": self.caches[request_id]}
async def sample(
self, x: np.ndarray, temp: float = 0.0, top_p: float = 1.0
) -> np.ndarray:
async def sample(self, x: np.ndarray, temp: float = 0.0, top_p: float = 1.0) -> np.ndarray:
if (temp, top_p, 0.0, 1) != self.sampler_params:
self.sampler_params = (temp, top_p, 0.0, 1)
self.sampler = make_sampler(*self.sampler_params)
@@ -60,28 +54,20 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
async def encode(self, shard: Shard, prompt: str) -> np.ndarray:
await self.ensure_shard(shard)
return np.asarray(
await asyncio.get_running_loop().run_in_executor(
self._tokenizer_thread, self.tokenizer.encode, prompt
)
await asyncio.get_running_loop().run_in_executor(self._tokenizer_thread, self.tokenizer.encode, prompt)
)
async def decode(self, shard: Shard, tokens) -> str:
await self.ensure_shard(shard)
return await asyncio.get_running_loop().run_in_executor(
self._tokenizer_thread, self.tokenizer.decode, tokens
)
return await asyncio.get_running_loop().run_in_executor(self._tokenizer_thread, self.tokenizer.decode, tokens)
async def save_checkpoint(self, shard: Shard, path: str):
await self.ensure_shard(shard)
await asyncio.get_running_loop().run_in_executor(
self._mlx_thread, lambda: self.model.save_weights(path)
)
await asyncio.get_running_loop().run_in_executor(self._mlx_thread, lambda: self.model.save_weights(path))
async def load_checkpoint(self, shard: Shard, path: str):
await self.ensure_shard(shard)
await asyncio.get_running_loop().run_in_executor(
self._mlx_thread, lambda: self.model.load_weights(path)
)
await asyncio.get_running_loop().run_in_executor(self._mlx_thread, lambda: self.model.load_weights(path))
async def infer_tensor(
self,
@@ -91,11 +77,7 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
inference_state: Optional[dict] = None,
) -> tuple[np.ndarray, Optional[dict]]:
await self.ensure_shard(shard)
state = (
await self.poll_state(request_id)
if self.model.model_type != "StableDiffusionPipeline"
else {}
)
state = await self.poll_state(request_id) if self.model.model_type != "StableDiffusionPipeline" else {}
x = mx.array(input_data)
if self.model.model_type != "StableDiffusionPipeline":
@@ -147,37 +129,22 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
):
await self.ensure_shard(shard)
if (
"train_layers" not in self.session
or self.session["train_layers"] != trainable_layers
):
if "train_layers" not in self.session or self.session["train_layers"] != trainable_layers:
await self.save_session("train_layers", trainable_layers)
def freeze_unfreeze():
self.model.freeze()
self.model.apply_to_modules(
lambda k, v: (
v.unfreeze()
if any(
k.endswith(layer_name) for layer_name in trainable_layers
)
else None
v.unfreeze() if any(k.endswith(layer_name) for layer_name in trainable_layers) else None
)
)
await asyncio.get_running_loop().run_in_executor(
self._mlx_thread, freeze_unfreeze
)
await asyncio.get_running_loop().run_in_executor(self._mlx_thread, freeze_unfreeze)
if (
"lossname" not in self.session
or "LVaG" not in self.session
or self.session["lossname"] != loss
):
if "lossname" not in self.session or "LVaG" not in self.session or self.session["lossname"] != loss:
await self.save_session("lossname", loss)
await self.save_session(
"LVaG", nn.value_and_grad(self.model, loss_fns[loss])
)
await self.save_session("LVaG", nn.value_and_grad(self.model, loss_fns[loss]))
if "opt" not in self.session:
await self.save_session("opt", opt(lr))
@@ -214,11 +181,7 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
)
await self._eval_mlx(*eval_args)
layers = [
{k: v["weight"] for k, v in layer.items() if "weight" in v}
for layer in gradients
if layer
]
layers = [{k: v["weight"] for k, v in layer.items() if "weight" in v} for layer in gradients if layer]
first_layer = np.array(layers[0]["input_layernorm"], copy=False)
await self._eval_mlx(first_layer)
return score, first_layer
@@ -227,9 +190,7 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
async with self._shard_lock:
if self.shard == shard:
return
model_path = await self.shard_downloader.ensure_shard(
shard, self.__class__.__name__
)
model_path = await self.shard_downloader.ensure_shard(shard, self.__class__.__name__)
if self.shard != shard:
model_shard = await asyncio.get_running_loop().run_in_executor(
self._mlx_thread,
@@ -233,9 +233,7 @@ async def get_image_from_str(_image_str: str):
return img
else:
raise ValueError(
"Invalid image_str format. Must be a URL or a base64 encoded image."
)
raise ValueError("Invalid image_str format. Must be a URL or a base64 encoded image.")
# loading a combined config for all models in the index
@@ -256,13 +254,7 @@ def load_model_index(model_path: Path, model_index_path: Path):
model_config["path"] = model_path / model
if model_config["path"] / "*model.safetensors":
model_config["config"].update(
{
"weight_files": list(
glob.glob(
str(model_config["path"] / "*model.safetensors")
)
)
}
{"weight_files": list(glob.glob(str(model_config["path"] / "*model.safetensors")))}
)
model_config["path"] = str(model_path / model)
m = {}
@@ -12,9 +12,7 @@ async def test_non_blocking():
shard_downloader = NewShardDownloader()
engine = MLXDynamicShardInferenceEngine(shard_downloader)
_shard = build_base_shard("llama-3.1-8b", "MLXDynamicShardInferenceEngine")
shard = Shard(
_shard.model_id, _shard.start_layer, _shard.n_layers - 1, _shard.n_layers
)
shard = Shard(_shard.model_id, _shard.start_layer, _shard.n_layers - 1, _shard.n_layers)
await engine.ensure_shard(shard)
queue = asyncio.Queue()
@@ -55,9 +53,7 @@ async def test_non_blocking():
try:
while running:
timestamp = await queue.get()
latency = (
time.perf_counter_ns() - timestamp
) / 1_000_000 # Convert to ms
latency = (time.perf_counter_ns() - timestamp) / 1_000_000 # Convert to ms
measurements.append(latency)
queue.task_done()
except asyncio.CancelledError:
@@ -36,6 +36,6 @@ class Shard:
def shards_overlap(shard1: Shard, shard2: Shard) -> bool:
return shard1.model_id == shard2.model_id and max(
shard1.start_layer, shard2.start_layer
) <= min(shard1.end_layer, shard2.end_layer)
return shard1.model_id == shard2.model_id and max(shard1.start_layer, shard2.start_layer) <= min(
shard1.end_layer, shard2.end_layer
)
@@ -17,18 +17,14 @@ async def test_inference_engine(
prompt = "In a single word only, what is the last name of the current president of the USA?"
resp_full, _ = await inference_engine_1.infer_prompt(
"A",
shard=Shard(
model_id=model_id, start_layer=0, end_layer=n_layers - 1, n_layers=n_layers
),
shard=Shard(model_id=model_id, start_layer=0, end_layer=n_layers - 1, n_layers=n_layers),
prompt=prompt,
)
token_full = await inference_engine_1.sample(resp_full)
token_full = token_full.reshape(1, -1)
next_resp_full, _ = await inference_engine_1.infer_tensor(
"A",
shard=Shard(
model_id=model_id, start_layer=0, end_layer=n_layers - 1, n_layers=n_layers
),
shard=Shard(model_id=model_id, start_layer=0, end_layer=n_layers - 1, n_layers=n_layers),
input_data=token_full,
)
@@ -144,9 +144,7 @@ def build_transformer(model_path: Path, shard: Shard, model_size="8B", device=No
return model
_executor = ThreadPoolExecutor(
max_workers=1
) # singleton so tinygrad always runs on the same thread
_executor = ThreadPoolExecutor(max_workers=1) # singleton so tinygrad always runs on the same thread
class TinygradDynamicShardInferenceEngine(InferenceEngine):
@@ -166,50 +164,31 @@ class TinygradDynamicShardInferenceEngine(InferenceEngine):
state = self.states[request_id]
return {"start_pos": state.start, "cache": state.cache}
async def sample(
self, x: np.ndarray, temp=TEMPERATURE, top_p: float = 0.0
) -> np.ndarray:
async def sample(self, x: np.ndarray, temp=TEMPERATURE, top_p: float = 0.0) -> np.ndarray:
def sample_wrapper():
logits = x[:, -1, :]
return (
sample_logits(Tensor(logits).flatten(), temp, 0, 0.8, top_p, 0.0)
.realize()
.numpy()
.astype(int)
)
return sample_logits(Tensor(logits).flatten(), temp, 0, 0.8, top_p, 0.0).realize().numpy().astype(int)
return await asyncio.get_running_loop().run_in_executor(
self.executor, sample_wrapper
)
return await asyncio.get_running_loop().run_in_executor(self.executor, sample_wrapper)
async def encode(self, shard: Shard, prompt: str) -> np.ndarray:
await self.ensure_shard(shard)
tokens = await asyncio.get_running_loop().run_in_executor(
self.executor, self.tokenizer.encode, prompt
)
return await asyncio.get_running_loop().run_in_executor(
self.executor, np.array, tokens
)
tokens = await asyncio.get_running_loop().run_in_executor(self.executor, self.tokenizer.encode, prompt)
return await asyncio.get_running_loop().run_in_executor(self.executor, np.array, tokens)
async def decode(self, shard: Shard, tokens) -> str:
await self.ensure_shard(shard)
tokens = await asyncio.get_running_loop().run_in_executor(
self.executor, self.tokenizer.decode, tokens
)
tokens = await asyncio.get_running_loop().run_in_executor(self.executor, self.tokenizer.decode, tokens)
return tokens
async def load_checkpoint(self, shard: Shard, path: str):
await self.ensure_shard(shard)
state_dict = safe_load(path)
await asyncio.get_running_loop().run_in_executor(
self.executor, load_state_dict, self.model, state_dict
)
await asyncio.get_running_loop().run_in_executor(self.executor, load_state_dict, self.model, state_dict)
async def save_checkpoint(self, shard: Shard, path: str):
await self.ensure_shard(shard)
state_dict = await asyncio.get_running_loop().run_in_executor(
self.executor, get_state_dict, self.model
)
state_dict = await asyncio.get_running_loop().run_in_executor(self.executor, get_state_dict, self.model)
safe_save(state_dict, path)
async def infer_tensor(
@@ -229,9 +208,7 @@ class TinygradDynamicShardInferenceEngine(InferenceEngine):
self.states[request_id].start += x.shape[1]
return out.numpy()
output_data = await asyncio.get_running_loop().run_in_executor(
self.executor, wrap_infer
)
output_data = await asyncio.get_running_loop().run_in_executor(self.executor, wrap_infer)
return output_data, inference_state
async def evaluate(
@@ -286,28 +263,18 @@ class TinygradDynamicShardInferenceEngine(InferenceEngine):
if self.shard == shard:
return
model_path = await self.shard_downloader.ensure_shard(
shard, self.__class__.__name__
)
model_path = await self.shard_downloader.ensure_shard(shard, self.__class__.__name__)
if self.shard != shard:
loop = asyncio.get_running_loop()
parameters = (
"1B"
if "1b" in shard.model_id.lower()
else (
"3B"
if "3b" in shard.model_id.lower()
else "8B" if "8b" in shard.model_id.lower() else "70B"
)
)
model_shard = await loop.run_in_executor(
self.executor, build_transformer, model_path, shard, parameters
else ("3B" if "3b" in shard.model_id.lower() else "8B" if "8b" in shard.model_id.lower() else "70B")
)
model_shard = await loop.run_in_executor(self.executor, build_transformer, model_path, shard, parameters)
tokenizer_path = str(
(model_path if model_path.is_dir() else model_path.parent)
)
tokenizer_path = str((model_path if model_path.is_dir() else model_path.parent))
self.tokenizer = await resolve_tokenizer(tokenizer_path)
self.shard = shard
self.model = model_shard
@@ -7,13 +7,9 @@ def length_masked_ce_loss(model, inputs, targets, lengths):
logits = model(inputs).cast(dtypes.float32).contiguous()
# Mask padding tokens
length_mask = Tensor(
np.arange(inputs.shape[1])[None, :] < lengths[:, None], requires_grad=False
)
length_mask = Tensor(np.arange(inputs.shape[1])[None, :] < lengths[:, None], requires_grad=False)
# Calculate the loss
ce = logits.sparse_categorical_crossentropy(
Tensor(targets, requires_grad=False)
).mul(length_mask)
ce = logits.sparse_categorical_crossentropy(Tensor(targets, requires_grad=False)).mul(length_mask)
loss = ce.sum() / length_mask.sum()
return loss
@@ -27,9 +27,7 @@ def precompute_freqs_cis(
freqs = Tensor.arange(end).unsqueeze(dim=1) * freqs.unsqueeze(dim=0)
# TODO: move dtype outside this
return Tensor.stack(
freqs.cos().cast(dtype), freqs.sin().cast(dtype), dim=-1
).reshape(1, end, 1, dim // 2, 2)
return Tensor.stack(freqs.cos().cast(dtype), freqs.sin().cast(dtype), dim=-1).reshape(1, end, 1, dim // 2, 2)
# (a+i*b) * (c+i*d) = (ac-bd) + i*(ad+bc)
@@ -40,12 +38,10 @@ def complex_mult(A, c, d):
return ro.cat(co, dim=-1)
def apply_rotary_emb(
xq: Tensor, xk: Tensor, freqs_cis: Tensor
) -> Tuple[Tensor, Tensor]:
assert (
freqs_cis.shape[1] == xq.shape[1] == xk.shape[1]
), f"freqs_cis shape mismatch {freqs_cis.shape} xq:{xq.shape} xk:{xk.shape}"
def apply_rotary_emb(xq: Tensor, xk: Tensor, freqs_cis: Tensor) -> Tuple[Tensor, Tensor]:
assert freqs_cis.shape[1] == xq.shape[1] == xk.shape[1], (
f"freqs_cis shape mismatch {freqs_cis.shape} xq:{xq.shape} xk:{xk.shape}"
)
xq = xq.reshape(*xq.shape[0:-1], -1, 2)
xk = xk.reshape(*xk.shape[0:-1], -1, 2)
assert len(xq.shape) == len(xk.shape) == len(freqs_cis.shape) == 5
@@ -110,23 +106,13 @@ class Attention:
if cache is not None:
# update the cache
assert (
xk.dtype == xv.dtype == cache.dtype
), f"{xk.dtype=}, {xv.dtype=}, {cache.dtype=}"
cache.shrink(
(None, None, (start_pos, start_pos + seqlen), None, None)
).assign(Tensor.stack(xk, xv)).realize()
assert xk.dtype == xv.dtype == cache.dtype, f"{xk.dtype=}, {xv.dtype=}, {cache.dtype=}"
cache.shrink((None, None, (start_pos, start_pos + seqlen), None, None)).assign(
Tensor.stack(xk, xv)
).realize()
keys = (
cache[0].shrink((None, (0, start_pos + seqlen), None, None))
if start_pos > 0
else xk
)
values = (
cache[1].shrink((None, (0, start_pos + seqlen), None, None))
if start_pos > 0
else xv
)
keys = cache[0].shrink((None, (0, start_pos + seqlen), None, None)) if start_pos > 0 else xk
values = cache[1].shrink((None, (0, start_pos + seqlen), None, None)) if start_pos > 0 else xv
else:
keys = xk
values = xv
@@ -149,9 +135,7 @@ class FeedForward:
self.w3 = linear(dim, hidden_dim, bias=False) # the gate in Gated Linear Unit
def __call__(self, x: Tensor) -> Tensor:
return self.w2(
self.w1(x).silu() * self.w3(x)
) # SwiGLU [arxiv/2002.05202, eq (5)]
return self.w2(self.w1(x).silu() * self.w3(x)) # SwiGLU [arxiv/2002.05202, eq (5)]
class TransformerBlock:
@@ -179,16 +163,12 @@ class TransformerBlock:
mask: Optional[Tensor],
cache: Optional[Tensor] = None,
):
h = x + self.attention(
self.attention_norm(x), start_pos, freqs_cis, mask, cache=cache
)
h = x + self.attention(self.attention_norm(x), start_pos, freqs_cis, mask, cache=cache)
return (h + self.feed_forward(self.ffn_norm(h))).contiguous()
# standard openai sampling
def sample_logits(
logits: Tensor, temp: float, k: int, p: float, af: float, ap: float, sample=None
):
def sample_logits(logits: Tensor, temp: float, k: int, p: float, af: float, ap: float, sample=None):
assert logits.ndim == 1, "only works on 1d tensors"
assert 0 <= p <= 1, "p must be between 0 and 1"
assert 0 <= k <= logits.numel(), "k must be between 0 and numel"
@@ -224,13 +204,9 @@ def sample_logits(
Tensor.zeros(k, device=logits.device, dtype=dtypes.int32).contiguous(),
)
for i in range(k):
t_argmax = (
t.numel() - ((t == (t_max := t.max())) * counter2).max() - 1
).cast(dtypes.default_int)
t_argmax = (t.numel() - ((t == (t_max := t.max())) * counter2).max() - 1).cast(dtypes.default_int)
output = output + t_max.unsqueeze(0).pad(((i, k - i - 1),))
output_indices = output_indices + t_argmax.unsqueeze(0).pad(
((i, k - i - 1),)
)
output_indices = output_indices + t_argmax.unsqueeze(0).pad(((i, k - i - 1),))
t = (counter == t_argmax).where(0, t)
# approximate top p
@@ -247,9 +223,7 @@ def sample_logits(
# increase alpha counter
if af or ap:
sample.alpha_counter = (counter == output_token).where(
sample.alpha_counter + 1, sample.alpha_counter
)
sample.alpha_counter = (counter == output_token).where(sample.alpha_counter + 1, sample.alpha_counter)
return output_token
@@ -305,9 +279,7 @@ class Transformer:
cache: Optional[List[Tensor]] = None,
):
seqlen = x.shape[1]
freqs_cis = self.freqs_cis.shrink(
(None, (start_pos, start_pos + seqlen), None, None, None)
)
freqs_cis = self.freqs_cis.shrink((None, (start_pos, start_pos + seqlen), None, None, None))
mask = (
Tensor.full(
(1, 1, seqlen, start_pos + seqlen),
@@ -324,9 +296,7 @@ class Transformer:
h = x
if cache is None:
cache = [
None for _ in range(self.shard.start_layer, self.shard.end_layer + 1)
]
cache = [None for _ in range(self.shard.start_layer, self.shard.end_layer + 1)]
for i, c in zip(range(self.shard.start_layer, self.shard.end_layer + 1), cache):
layer = self.layers[i]
h = layer(h, start_pos, freqs_cis, mask, cache=c)
@@ -353,9 +323,7 @@ class Transformer:
)
return self.forward_base(x, start_pos, cache=cache)
def __call__(
self, x: Tensor, start_pos: Variable, cache: Optional[List[Tensor]] = None
):
def __call__(self, x: Tensor, start_pos: Variable, cache: Optional[List[Tensor]] = None):
# TODO: better way to handle the first call v.s. the rest?
h = self.embed(x)
return self.forward(h, start_pos, cache=cache)
@@ -369,22 +337,12 @@ class TransformerShard:
jit: bool = True,
):
shardrange = range(shard.start_layer, shard.end_layer + 1)
self.layers = [
layer
for layer, n in zip(base.layers, range(shard.n_layers))
if n in shardrange
]
self.layers = [layer for layer, n in zip(base.layers, range(shard.n_layers)) if n in shardrange]
self.norm = base.norm
self.tok_embeddings = base.tok_embeddings
self.embed = (
(lambda x: self.tok_embeddings(x))
if shard.is_first_layer()
else (lambda x: x)
)
self.embed = (lambda x: self.tok_embeddings(x)) if shard.is_first_layer() else (lambda x: x)
self.output = base.output
self.post = (
(lambda x: self.output(x)) if shard.is_last_layer() else (lambda x: x)
)
self.post = (lambda x: self.output(x)) if shard.is_last_layer() else (lambda x: x)
self.max_context = base.max_context
self.null_cache = [None for _ in shardrange]
self.freqs_cis = base.freqs_cis
@@ -392,9 +350,7 @@ class TransformerShard:
def forward_base(self, x: Tensor, start_pos: Union[Variable, int], cache):
seqlen = x.shape[1]
freqs_cis = self.freqs_cis.shrink(
(None, (start_pos, start_pos + seqlen), None, None, None)
)
freqs_cis = self.freqs_cis.shrink((None, (start_pos, start_pos + seqlen), None, None, None))
mask = (
Tensor.full(
(1, 1, seqlen, start_pos + seqlen),
@@ -423,28 +379,18 @@ class TransformerShard:
)
return self.forward_base(x, start_pos, cache=cache)
def __call__(
self, x: Tensor, start_pos: Variable, cache: Optional[List[Tensor]] = None
):
def __call__(self, x: Tensor, start_pos: Variable, cache: Optional[List[Tensor]] = None):
# TODO: better way to handle the first call v.s. the rest?
h = self.embed(x)
return self.forward(
h, start_pos, cache=self.null_cache if cache is None else cache
)
return self.forward(h, start_pos, cache=self.null_cache if cache is None else cache)
# *** helpers ***
def convert_from_huggingface(
weights: Dict[str, Tensor], model: Transformer, n_heads: int, n_kv_heads: int
):
def convert_from_huggingface(weights: Dict[str, Tensor], model: Transformer, n_heads: int, n_kv_heads: int):
def permute(v: Tensor, n_heads: int):
return (
v.reshape(n_heads, 2, v.shape[0] // n_heads // 2, v.shape[1])
.transpose(1, 2)
.reshape(*v.shape[:2])
)
return v.reshape(n_heads, 2, v.shape[0] // n_heads // 2, v.shape[1]).transpose(1, 2).reshape(*v.shape[:2])
keymap = {
"model.embed_tokens.weight": "tok_embeddings.weight",
@@ -490,29 +436,16 @@ def fix_bf16(weights: Dict[Any, Tensor]):
if Device.DEFAULT == "CLANG":
# TODO: without casting to float16, 70B llama OOM on tinybox.
return {
k: (
v.llvm_bf16_cast(dtypes.float32).to(v.device)
if v.dtype == dtypes.bfloat16
else v
)
k: (v.llvm_bf16_cast(dtypes.float32).to(v.device) if v.dtype == dtypes.bfloat16 else v)
for k, v in weights.items()
}
if getenv("SUPPORT_BF16", 1):
# TODO: without casting to float16, 70B llama OOM on tinybox.
return {
k: (
v.cast(dtypes.float32).cast(dtypes.float16)
if v.dtype == dtypes.bfloat16
else v
)
k: (v.cast(dtypes.float32).cast(dtypes.float16) if v.dtype == dtypes.bfloat16 else v)
for k, v in weights.items()
}
# TODO: check if device supports bf16
return {
k: (
v.llvm_bf16_cast(dtypes.half).to(v.device)
if v.dtype == dtypes.bfloat16
else v
)
for k, v in weights.items()
k: (v.llvm_bf16_cast(dtypes.half).to(v.device) if v.dtype == dtypes.bfloat16 else v) for k, v in weights.items()
}
@@ -18,9 +18,7 @@ def create_kv_cache(x: Tensor, layer):
)
if isinstance(x.device, tuple):
# TODO: instead of specifying how to shard, it can follow how xk and xv are being sharded
cache_kv.shard_(
(x.device), axis=3 if getenv("SHARD_KVCACHE") else None
).realize()
cache_kv.shard_((x.device), axis=3 if getenv("SHARD_KVCACHE") else None).realize()
return cache_kv.realize()
@@ -16,19 +16,11 @@ def concat_weights(models, device=None):
disk_tensors: List[Tensor] = [model[name] for model in models]
if len(disk_tensors) == 1 or len(disk_tensors[0].shape) == 1:
return disk_tensors[0].to(device=device)
axis = (
1
if name.endswith(".attention.wo.weight")
or name.endswith(".feed_forward.w2.weight")
else 0
)
axis = 1 if name.endswith(".attention.wo.weight") or name.endswith(".feed_forward.w2.weight") else 0
lazy_tensors = [data.to(device=device) for data in disk_tensors]
return lazy_tensors[0].cat(*lazy_tensors[1:], dim=axis)
return {
name: convert(name)
for name in {name: None for model in models for name in model}
}
return {name: convert(name) for name in {name: None for model in models for name in model}}
def load(fn: str, shard: Shard):
@@ -39,9 +31,7 @@ def load(fn: str, shard: Shard):
filtered_weight_map = {}
allow_patterns = get_allow_patterns(weight_map, shard)
for k, n in weight_map.items():
if allow_patterns is not None and not any(
fnmatch(n, r) for r in allow_patterns
):
if allow_patterns is not None and not any(fnmatch(n, r) for r in allow_patterns):
continue
if k.startswith("model.layers."):
layer_num = int(k.split(".")[2])
@@ -58,9 +48,7 @@ def load(fn: str, shard: Shard):
elif fn.endswith(".safetensors"):
weight_map = safe_load(fn)
for k in list(weight_map):
if (n := re.search(r"\.(\d+)\.", k)) and not (
shard.start_layer <= int(n.group(1)) <= shard.end_layer
):
if (n := re.search(r"\.(\d+)\.", k)) and not (shard.start_layer <= int(n.group(1)) <= shard.end_layer):
del weight_map[k]
return weight_map
else:
@@ -49,9 +49,7 @@ async def resolve_tokenizer(repo_id: Union[str, PathLike]):
return DummyTokenizer()
local_path = await ensure_downloads_dir() / str(repo_id).replace("/", "--")
if DEBUG >= 2:
print(
f"Checking if local path exists to load tokenizer from local {local_path=}"
)
print(f"Checking if local path exists to load tokenizer from local {local_path=}")
try:
if local_path and await aios.path.exists(local_path):
if DEBUG >= 2:
@@ -59,9 +57,7 @@ async def resolve_tokenizer(repo_id: Union[str, PathLike]):
return await _resolve_tokenizer(local_path)
except Exception:
if DEBUG >= 5:
print(
f"Local check for {local_path=} failed. Resolving tokenizer for {repo_id=} normally..."
)
print(f"Local check for {local_path=} failed. Resolving tokenizer for {repo_id=} normally...")
if DEBUG >= 5:
traceback.print_exc()
return await _resolve_tokenizer(repo_id)
@@ -81,13 +77,9 @@ async def _resolve_tokenizer(repo_id_or_local_path: Union[str, PathLike]):
processor, "tokenizer", getattr(processor, "_tokenizer", processor)
).eos_token_id
if not hasattr(processor, "encode"):
processor.encode = getattr(
processor, "tokenizer", getattr(processor, "_tokenizer", processor)
).encode
processor.encode = getattr(processor, "tokenizer", getattr(processor, "_tokenizer", processor)).encode
if not hasattr(processor, "decode"):
processor.decode = getattr(
processor, "tokenizer", getattr(processor, "_tokenizer", processor)
).decode
processor.decode = getattr(processor, "tokenizer", getattr(processor, "_tokenizer", processor)).decode
return processor
except Exception as e:
if DEBUG >= 4:
@@ -98,9 +90,7 @@ async def _resolve_tokenizer(repo_id_or_local_path: Union[str, PathLike]):
try:
if DEBUG >= 4:
print(f"Trying AutoTokenizer for {repo_id_or_local_path}")
return AutoTokenizer.from_pretrained(
repo_id_or_local_path, trust_remote_code=True
)
return AutoTokenizer.from_pretrained(repo_id_or_local_path, trust_remote_code=True)
except Exception as e:
if DEBUG >= 4:
print(
@@ -181,9 +181,7 @@ class HanzoNetProvider(LocalLLMProvider):
"stable-diffusion-2-1-base",
]
def _messages_to_prompt(
self, messages: List[Message], tools: Optional[List[Dict[str, Any]]]
) -> str:
def _messages_to_prompt(self, messages: List[Message], tools: Optional[List[Dict[str, Any]]]) -> str:
"""Convert messages to a prompt string."""
prompt = ""
@@ -214,9 +212,7 @@ class HanzoNetProvider(LocalLLMProvider):
prompt += "Assistant: "
return prompt
def _generate_dummy_response(
self, prompt: str, tools: Optional[List[Dict[str, Any]]]
) -> str:
def _generate_dummy_response(self, prompt: str, tools: Optional[List[Dict[str, Any]]]) -> str:
"""Generate a contextual dummy response based on prompt."""
prompt_lower = prompt.lower()
@@ -224,10 +220,7 @@ class HanzoNetProvider(LocalLLMProvider):
if tools:
for tool in tools:
tool_name = tool["name"]
if (
tool_name in prompt_lower
or tool["description"].lower() in prompt_lower
):
if tool_name in prompt_lower or tool["description"].lower() in prompt_lower:
# Generate a response that calls the tool
if "search" in tool_name:
return f"I'll search for that information using the {tool_name} tool.\n\n<tool_call>{tool_name}('authentication')</tool_call>"
@@ -244,7 +237,9 @@ class HanzoNetProvider(LocalLLMProvider):
elif "analyze" in prompt_lower:
return "After analyzing the code, I can see it follows good practices with clear structure and efficient implementation."
elif "generate" in prompt_lower or "test" in prompt_lower:
return "I've generated the requested code/tests following best practices and ensuring comprehensive coverage."
return (
"I've generated the requested code/tests following best practices and ensuring comprehensive coverage."
)
elif "explain" in prompt_lower or "what is" in prompt_lower:
return "Let me explain that concept: It's a fundamental programming technique that involves a function calling itself to solve smaller instances of the same problem."
else:
@@ -115,9 +115,7 @@ class LocalComputeNode:
models: List of available models
"""
self.node_id = node_id
self.wallet_address = (
wallet_address or f"0x{hashlib.sha256(node_id.encode()).hexdigest()[:40]}"
)
self.wallet_address = wallet_address or f"0x{hashlib.sha256(node_id.encode()).hexdigest()[:40]}"
self.models: Dict[str, ModelConfig] = {}
self.loaded_models: Dict[str, Any] = {}
@@ -254,9 +252,7 @@ class LocalComputeNode:
tokenizer = AutoTokenizer.from_pretrained(config.model_path)
model = AutoModelForCausalLM.from_pretrained(
config.model_path,
torch_dtype=(
torch.float16 if config.device != "cpu" else torch.float32
),
torch_dtype=(torch.float16 if config.device != "cpu" else torch.float32),
device_map="auto" if config.device == "cuda" else None,
)
@@ -332,9 +328,7 @@ class LocalComputeNode:
# Calculate cost
tokens_generated = len(result_text.split()) # Approximate
cost = (tokens_generated / 1000) * self.models[
selected_model
].price_per_1k_tokens
cost = (tokens_generated / 1000) * self.models[selected_model].price_per_1k_tokens
# Update statistics
self.total_requests += 1
@@ -411,9 +405,7 @@ class LocalComputeNode:
return response
def _create_attestation(
self, request: InferenceRequest, result: InferenceResult
) -> Dict[str, Any]:
def _create_attestation(self, request: InferenceRequest, result: InferenceResult) -> Dict[str, Any]:
"""Create attestation for inference result."""
# In production, this would use TEE attestation
attestation_data = {
@@ -484,13 +476,8 @@ class LocalComputeOrchestrator:
for node in self.nodes.values():
for model in node.list_models():
if model["available"]:
estimated_cost = (request.max_tokens / 1000) * model[
"price_per_1k_tokens"
]
if (
estimated_cost <= request.max_price_eth
and estimated_cost < best_price
):
estimated_cost = (request.max_tokens / 1000) * model["price_per_1k_tokens"]
if estimated_cost <= request.max_price_eth and estimated_cost < best_price:
best_node = node
best_price = estimated_cost
@@ -522,9 +509,7 @@ class LocalComputeOrchestrator:
"completed_requests": len(self.completed_requests),
"total_requests_processed": total_requests,
"total_earnings_eth": total_earnings,
"nodes_detail": {
node_id: node.get_stats() for node_id, node in self.nodes.items()
},
"nodes_detail": {node_id: node.get_stats() for node_id, node in self.nodes.items()},
}
@@ -536,9 +521,7 @@ orchestrator = LocalComputeOrchestrator()
async def demo_local_compute():
"""Demonstrate local compute capabilities."""
# Create a compute node
node1 = LocalComputeNode(
node_id="node_001", wallet_address="0x1234567890123456789012345678901234567890"
)
node1 = LocalComputeNode(node_id="node_001", wallet_address="0x1234567890123456789012345678901234567890")
# Register with orchestrator
orchestrator.register_node(node1)
@@ -546,9 +529,7 @@ async def demo_local_compute():
# List available models
print("\nAvailable models:")
for model in node1.list_models():
print(
f" - {model['name']}: {model['device']}, ${model['price_per_1k_tokens']}/1k tokens"
)
print(f" - {model['name']}: {model['device']}, ${model['price_per_1k_tokens']}/1k tokens")
# Create inference request
request = InferenceRequest(
@@ -30,9 +30,7 @@ def create_local_agent(
Agent configured for local LLM
"""
# Create model config for local provider
model_config = ModelConfig(
provider=ModelProvider.LOCAL, model=local_model, base_url=base_url
)
model_config = ModelConfig(provider=ModelProvider.LOCAL, model=local_model, base_url=base_url)
return create_agent(
name=name,
@@ -76,9 +74,7 @@ def create_local_distributed_network(
# Create a local router if one isn't provided
if "router" not in kwargs:
router_config = ModelConfig(
provider=ModelProvider.LOCAL, model=local_model, base_url=base_url
)
router_config = ModelConfig(provider=ModelProvider.LOCAL, model=local_model, base_url=base_url)
kwargs["router"] = create_routing_agent(
name="local_router", description="Local routing agent", model=router_config
@@ -132,18 +128,12 @@ async def check_local_llm_status(provider: str = "hanzo") -> dict:
if engine_type == "mlx":
import platform
status["platform"] = (
"Apple Silicon"
if platform.machine() in ["arm64", "aarch64"]
else platform.machine()
)
status["platform"] = "Apple Silicon" if platform.machine() in ["arm64", "aarch64"] else platform.machine()
if not is_available:
status["instructions"] = "MLX requires Apple Silicon (M1/M2/M3)"
elif engine_type == "tinygrad":
status["instructions"] = (
"Tinygrad engine ready for distributed inference"
if is_available
else "Install tinygrad"
"Tinygrad engine ready for distributed inference" if is_available else "Install tinygrad"
)
else: # dummy
status["instructions"] = "Using mock responses for testing"
@@ -73,9 +73,7 @@ class MemoryManager:
ids = []
for statement in statements:
memory_id = await self.store.add(
{"content": statement, "type": "statement"}
)
memory_id = await self.store.add({"content": statement, "type": "statement"})
ids.append(memory_id)
return ids
@@ -92,9 +90,7 @@ class MemoryManager:
results = []
for update in updates:
success = await self.store.update(
update["id"], {"content": update["statement"]}
)
success = await self.store.update(update["id"], {"content": update["statement"]})
results.append(success)
return results
@@ -139,9 +135,7 @@ def create_memory_tools(memory_manager: Optional[MemoryManager] = None) -> List[
name="recall_memories",
description="Recall memories relevant to one or more queries. Can run multiple queries in parallel.",
)
async def recall_memories_tool(
queries: List[str], limit: int = 10, context: ToolContext = None
) -> str:
async def recall_memories_tool(queries: List[str], limit: int = 10, context: ToolContext = None) -> str:
"""Recall relevant memories."""
memories = await memory_manager.recall(queries, limit)
@@ -162,9 +156,7 @@ def create_memory_tools(memory_manager: Optional[MemoryManager] = None) -> List[
name="create_memories",
description="Save one or more new pieces of information to memory.",
)
async def create_memories_tool(
statements: List[str], context: ToolContext = None
) -> str:
async def create_memories_tool(statements: List[str], context: ToolContext = None) -> str:
"""Create new memories."""
ids = await memory_manager.create(statements)
return f"Created {len(ids)} new memories."
@@ -176,9 +168,7 @@ def create_memory_tools(memory_manager: Optional[MemoryManager] = None) -> List[
name="update_memories",
description="Update existing memories with corrected information.",
)
async def update_memories_tool(
updates: List[Dict[str, str]], context: ToolContext = None
) -> str:
async def update_memories_tool(updates: List[Dict[str, str]], context: ToolContext = None) -> str:
"""Update memories."""
results = await memory_manager.update(updates)
success_count = sum(results)
@@ -59,49 +59,21 @@ UNKNOWN_DEVICE_CAPABILITIES = DeviceCapabilities(
CHIP_FLOPS = {
# Apple M series
"Apple M1": DeviceFlops(fp32=2.29 * TFLOPS, fp16=4.58 * TFLOPS, int8=9.16 * TFLOPS),
"Apple M1 Pro": DeviceFlops(
fp32=5.30 * TFLOPS, fp16=10.60 * TFLOPS, int8=21.20 * TFLOPS
),
"Apple M1 Max": DeviceFlops(
fp32=10.60 * TFLOPS, fp16=21.20 * TFLOPS, int8=42.40 * TFLOPS
),
"Apple M2": DeviceFlops(
fp32=3.55 * TFLOPS, fp16=7.10 * TFLOPS, int8=14.20 * TFLOPS
),
"Apple M2 Pro": DeviceFlops(
fp32=5.68 * TFLOPS, fp16=11.36 * TFLOPS, int8=22.72 * TFLOPS
),
"Apple M2 Max": DeviceFlops(
fp32=13.49 * TFLOPS, fp16=26.98 * TFLOPS, int8=53.96 * TFLOPS
),
"Apple M3": DeviceFlops(
fp32=3.55 * TFLOPS, fp16=7.10 * TFLOPS, int8=14.20 * TFLOPS
),
"Apple M3 Pro": DeviceFlops(
fp32=4.97 * TFLOPS, fp16=9.94 * TFLOPS, int8=19.88 * TFLOPS
),
"Apple M3 Max": DeviceFlops(
fp32=14.20 * TFLOPS, fp16=28.40 * TFLOPS, int8=56.80 * TFLOPS
),
"Apple M4": DeviceFlops(
fp32=4.26 * TFLOPS, fp16=8.52 * TFLOPS, int8=17.04 * TFLOPS
),
"Apple M1 Pro": DeviceFlops(fp32=5.30 * TFLOPS, fp16=10.60 * TFLOPS, int8=21.20 * TFLOPS),
"Apple M1 Max": DeviceFlops(fp32=10.60 * TFLOPS, fp16=21.20 * TFLOPS, int8=42.40 * TFLOPS),
"Apple M2": DeviceFlops(fp32=3.55 * TFLOPS, fp16=7.10 * TFLOPS, int8=14.20 * TFLOPS),
"Apple M2 Pro": DeviceFlops(fp32=5.68 * TFLOPS, fp16=11.36 * TFLOPS, int8=22.72 * TFLOPS),
"Apple M2 Max": DeviceFlops(fp32=13.49 * TFLOPS, fp16=26.98 * TFLOPS, int8=53.96 * TFLOPS),
"Apple M3": DeviceFlops(fp32=3.55 * TFLOPS, fp16=7.10 * TFLOPS, int8=14.20 * TFLOPS),
"Apple M3 Pro": DeviceFlops(fp32=4.97 * TFLOPS, fp16=9.94 * TFLOPS, int8=19.88 * TFLOPS),
"Apple M3 Max": DeviceFlops(fp32=14.20 * TFLOPS, fp16=28.40 * TFLOPS, int8=56.80 * TFLOPS),
"Apple M4": DeviceFlops(fp32=4.26 * TFLOPS, fp16=8.52 * TFLOPS, int8=17.04 * TFLOPS),
# NVIDIA GPUs
"NVIDIA GEFORCE RTX 4090": DeviceFlops(
fp32=82.58 * TFLOPS, fp16=165.16 * TFLOPS, int8=330.32 * TFLOPS
),
"NVIDIA GEFORCE RTX 4080": DeviceFlops(
fp32=48.74 * TFLOPS, fp16=97.48 * TFLOPS, int8=194.96 * TFLOPS
),
"NVIDIA GEFORCE RTX 4070": DeviceFlops(
fp32=29.0 * TFLOPS, fp16=58.0 * TFLOPS, int8=116.0 * TFLOPS
),
"NVIDIA GEFORCE RTX 3090": DeviceFlops(
fp32=35.6 * TFLOPS, fp16=71.2 * TFLOPS, int8=142.4 * TFLOPS
),
"NVIDIA GEFORCE RTX 3080": DeviceFlops(
fp32=29.8 * TFLOPS, fp16=59.6 * TFLOPS, int8=119.2 * TFLOPS
),
"NVIDIA GEFORCE RTX 4090": DeviceFlops(fp32=82.58 * TFLOPS, fp16=165.16 * TFLOPS, int8=330.32 * TFLOPS),
"NVIDIA GEFORCE RTX 4080": DeviceFlops(fp32=48.74 * TFLOPS, fp16=97.48 * TFLOPS, int8=194.96 * TFLOPS),
"NVIDIA GEFORCE RTX 4070": DeviceFlops(fp32=29.0 * TFLOPS, fp16=58.0 * TFLOPS, int8=116.0 * TFLOPS),
"NVIDIA GEFORCE RTX 3090": DeviceFlops(fp32=35.6 * TFLOPS, fp16=71.2 * TFLOPS, int8=142.4 * TFLOPS),
"NVIDIA GEFORCE RTX 3080": DeviceFlops(fp32=29.8 * TFLOPS, fp16=59.6 * TFLOPS, int8=119.2 * TFLOPS),
# Add more as needed
}
@@ -131,9 +103,7 @@ def mac_device_capabilities() -> DeviceCapabilities:
import subprocess
# Get model info
model_result = subprocess.run(
["system_profiler", "SPHardwareDataType"], capture_output=True, text=True
)
model_result = subprocess.run(["system_profiler", "SPHardwareDataType"], capture_output=True, text=True)
model = "Mac"
chip = "Unknown"
@@ -172,24 +142,24 @@ def linux_device_capabilities() -> DeviceCapabilities:
"""Get Linux device capabilities."""
try:
import subprocess
# Try to detect NVIDIA GPU using nvidia-smi
try:
result = subprocess.run(
["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader"],
capture_output=True,
text=True,
timeout=5
timeout=5,
)
if result.returncode == 0 and result.stdout.strip():
gpu_info = result.stdout.strip().split(",")
gpu_name = gpu_info[0].strip()
gpu_memory = int(gpu_info[1].strip().split()[0]) # Memory in MiB
# Lookup FLOPS from known GPUs
flops = CHIP_FLOPS.get(gpu_name.upper(), DeviceFlops(fp32=0, fp16=0, int8=0))
return DeviceCapabilities(
model="Linux Box with GPU",
chip=gpu_name,
@@ -198,23 +168,18 @@ def linux_device_capabilities() -> DeviceCapabilities:
)
except (subprocess.SubprocessError, FileNotFoundError, IndexError):
pass
# Try to detect AMD GPU using rocm-smi
try:
result = subprocess.run(
["rocm-smi", "--showproductname"],
capture_output=True,
text=True,
timeout=5
)
result = subprocess.run(["rocm-smi", "--showproductname"], capture_output=True, text=True, timeout=5)
if result.returncode == 0 and result.stdout.strip():
# Parse AMD GPU info
for line in result.stdout.split("\n"):
if "GPU" in line:
gpu_name = line.split(":")[-1].strip() if ":" in line else "AMD GPU"
memory = psutil.virtual_memory().total // 2**20
return DeviceCapabilities(
model="Linux Box with AMD GPU",
chip=gpu_name,
@@ -223,7 +188,7 @@ def linux_device_capabilities() -> DeviceCapabilities:
)
except (subprocess.SubprocessError, FileNotFoundError):
pass
# Fallback to CPU
return DeviceCapabilities(
model="Linux Box",
@@ -231,7 +196,7 @@ def linux_device_capabilities() -> DeviceCapabilities:
memory=psutil.virtual_memory().total // 2**20,
flops=DeviceFlops(fp32=0, fp16=0, int8=0),
)
except Exception as e:
if DEBUG >= 1:
print(f"Error getting Linux device capabilities: {e}")
@@ -247,7 +212,7 @@ def windows_device_capabilities() -> DeviceCapabilities:
"""Get Windows device capabilities."""
try:
import subprocess
# Try to detect NVIDIA GPU using nvidia-smi
try:
result = subprocess.run(
@@ -255,17 +220,17 @@ def windows_device_capabilities() -> DeviceCapabilities:
capture_output=True,
text=True,
timeout=5,
shell=True # Windows may need shell
shell=True, # Windows may need shell
)
if result.returncode == 0 and result.stdout.strip():
gpu_info = result.stdout.strip().split(",")
gpu_name = gpu_info[0].strip()
gpu_memory = int(gpu_info[1].strip().split()[0]) # Memory in MiB
# Lookup FLOPS from known GPUs
flops = CHIP_FLOPS.get(gpu_name.upper(), DeviceFlops(fp32=0, fp16=0, int8=0))
return DeviceCapabilities(
model="Windows Box with GPU",
chip=gpu_name,
@@ -274,7 +239,7 @@ def windows_device_capabilities() -> DeviceCapabilities:
)
except (subprocess.SubprocessError, FileNotFoundError, IndexError):
pass
# Fallback to CPU
return DeviceCapabilities(
model="Windows Box",
@@ -282,7 +247,7 @@ def windows_device_capabilities() -> DeviceCapabilities:
memory=psutil.virtual_memory().total // 2**20,
flops=DeviceFlops(fp32=0, fp16=0, int8=0),
)
except Exception as e:
if DEBUG >= 1:
print(f"Error getting Windows device capabilities: {e}")
@@ -19,9 +19,7 @@ class PartitioningStrategy(ABC):
pass
def map_partitions_to_shards(
partitions: List[Partition], num_layers: int, model_id: str
) -> List[Shard]:
def map_partitions_to_shards(partitions: List[Partition], num_layers: int, model_id: str) -> List[Shard]:
shards = []
for i, partition in enumerate(partitions):
start_layer = int(partition.start * num_layers)
@@ -88,9 +88,7 @@ Activation Lock Status: Disabled
assert result.memory == 8192 # 8 GB in MB
@pytest.mark.skip(
reason="Unskip this test when running on a MacBook Pro, Apple M3 Max, 128GB"
)
@pytest.mark.skip(reason="Unskip this test when running on a MacBook Pro, Apple M3 Max, 128GB")
def test_mac_device_capabilities_real():
# Call the function without mocking
result = mac_device_capabilities()
@@ -100,9 +98,7 @@ def test_mac_device_capabilities_real():
assert result.model == "MacBook Pro"
assert result.chip == "Apple M3 Max"
assert result.memory == 131072 # 128 GB in MB
assert result.flops == DeviceFlops(
fp32=14.20 * TFLOPS, fp16=28.40 * TFLOPS, int8=56.80 * TFLOPS
)
assert result.flops == DeviceFlops(fp32=14.20 * TFLOPS, fp16=28.40 * TFLOPS, int8=56.80 * TFLOPS)
assert (
str(result)
== "Model: MacBook Pro. Chip: Apple M3 Max. Memory: 131072MB. Flops: fp32: 14.20 TFLOPS, fp16: 28.40 TFLOPS, int8: 56.80 TFLOPS"
@@ -53,9 +53,7 @@ class TestRingMemoryWeightedPartitioningStrategy(unittest.TestCase):
def test_broken_map_partitions_to_shards(self):
# this was an old broken implementation that sometimes had rounding errors!
def _broken_map_partitions_to_shards(
partitions: List[Partition], num_layers, model_id: str
):
def _broken_map_partitions_to_shards(partitions: List[Partition], num_layers, model_id: str):
shards = []
for i, partition in enumerate(partitions):
start_layer = int(partition.start * num_layers)
@@ -53,21 +53,15 @@ class Topology:
self.add_edge(conn.from_id, conn.to_id, conn.description)
def __str__(self):
nodes_str = ", ".join(
f"{node_id}: {cap}" for node_id, cap in self.nodes.items()
)
nodes_str = ", ".join(f"{node_id}: {cap}" for node_id, cap in self.nodes.items())
edges_str = ", ".join(
f"{node}: {[f'{c.to_id}({c.description})' for c in conns]}"
for node, conns in self.peer_graph.items()
f"{node}: {[f'{c.to_id}({c.description})' for c in conns]}" for node, conns in self.peer_graph.items()
)
return f"Topology(Nodes: {{{nodes_str}}}, Edges: {{{edges_str}}})"
def to_json(self):
return {
"nodes": {
node_id: capabilities.to_dict()
for node_id, capabilities in self.nodes.items()
},
"nodes": {node_id: capabilities.to_dict() for node_id, capabilities in self.nodes.items()},
"peer_graph": {
node_id: [
{
@@ -66,9 +66,7 @@ async def test_distributed_network_start_stop():
"""Test starting and stopping a distributed network."""
agents = create_test_agents()
network = create_distributed_network(
agents=agents, name="test-network", listen_port=15679, broadcast_port=15679
)
network = create_distributed_network(agents=agents, name="test-network", listen_port=15679, broadcast_port=15679)
# Start network
await network.start(wait_for_peers=0)
@@ -91,9 +89,7 @@ async def test_distributed_network_local_execution():
"""Test executing agents locally in distributed network."""
agents = create_test_agents()
network = create_distributed_network(
agents=agents, name="test-network", listen_port=15680, broadcast_port=15680
)
network = create_distributed_network(agents=agents, name="test-network", listen_port=15680, broadcast_port=15680)
# Start network
await network.start(wait_for_peers=0)
@@ -110,9 +106,7 @@ async def test_distributed_network_local_execution():
assert "echo_agent" in echo_result["final_output"]
# Test math agent
math_result = await network.run(
prompt="Calculate 2 + 2", initial_agent=network.get_agent("math_agent")
)
math_result = await network.run(prompt="Calculate 2 + 2", initial_agent=network.get_agent("math_agent"))
assert math_result["success"]
# Mock agents return mock responses
+7 -23
View File
@@ -123,9 +123,7 @@ class ClaudeCodeBackend(Backend):
# Check authentication
if not self.authenticated and not self._check_auth():
console = Console()
console.print(
"[yellow]Not authenticated with Claude. Attempting login...[/yellow]"
)
console.print("[yellow]Not authenticated with Claude. Attempting login...[/yellow]")
if not await self.authenticate():
raise RuntimeError("Failed to authenticate with Claude")
@@ -150,14 +148,9 @@ class ClaudeCodeBackend(Backend):
if proc.returncode != 0:
# Check if it's an auth error
if (
"unauthorized" in stderr.decode().lower()
or "auth" in stderr.decode().lower()
):
if "unauthorized" in stderr.decode().lower() or "auth" in stderr.decode().lower():
self.authenticated = False
raise RuntimeError(
"Claude authentication expired. Please re-authenticate."
)
raise RuntimeError("Claude authentication expired. Please re-authenticate.")
raise RuntimeError(f"Claude Code error: {stderr.decode()}")
return stdout.decode()
@@ -257,9 +250,7 @@ class OpenAICodexBackend(Backend):
tool_choice="auto",
)
else:
response = await client.chat.completions.create(
model="gpt-4-turbo-preview", messages=messages
)
response = await client.chat.completions.create(model="gpt-4-turbo-preview", messages=messages)
return response.choices[0].message.content
@@ -277,9 +268,7 @@ class HanzoDevBackend(Backend):
"/usr/local/bin/hanzo-dev",
os.path.expanduser("~/.local/bin/hanzo-dev"),
# Check in parent directory
os.path.join(
os.path.dirname(os.path.dirname(__file__)), "..", "dev", "hanzo-dev"
),
os.path.join(os.path.dirname(os.path.dirname(__file__)), "..", "dev", "hanzo-dev"),
]
for path in paths:
@@ -342,9 +331,7 @@ class EmbeddedBackend(Backend):
{"role": "user", "content": message},
]
response = await self.llm_client.chat(
messages=messages, tools=tools, tool_choice="auto" if tools else None
)
response = await self.llm_client.chat(messages=messages, tools=tools, tool_choice="auto" if tools else None)
return response.choices[0].message.content
@@ -394,10 +381,7 @@ class BackendManager:
def list_backends(self) -> Dict[str, bool]:
"""List all backends and their availability."""
return {
name: backend.is_available() if backend else False
for name, backend in self.backends.items()
}
return {name: backend.is_available() if backend else False for name, backend in self.backends.items()}
async def load_config(self) -> Optional[str]:
"""Load configuration from the appropriate file."""
@@ -42,16 +42,12 @@ class CommandItem(ListItem):
"""Create child widgets."""
with Horizontal(classes="command-item"):
# Icon and category
yield Static(
f"{self.command.icon} [{self.command.category}]", classes="command-icon"
)
yield Static(f"{self.command.icon} [{self.command.category}]", classes="command-icon")
# Name and description
with Vertical(classes="command-info"):
yield Static(self.command.name, classes="command-name")
yield Static(
self.command.description[:60] + "...", classes="command-desc"
)
yield Static(self.command.description[:60] + "...", classes="command-desc")
class CommandPalette(Vertical):
@@ -163,9 +159,7 @@ class CommandPalette(Vertical):
description=tool.description,
category=category,
icon=icon,
parameters=(
tool.get_schema() if hasattr(tool, "get_schema") else None
),
parameters=(tool.get_schema() if hasattr(tool, "get_schema") else None),
)
)
@@ -188,9 +182,7 @@ class CommandPalette(Vertical):
id="palette-input",
)
yield ListView(id="palette-list")
yield Label(
"↑↓ Navigate · ⏎ Select · esc Close · ⇥ Complete", id="palette-hint"
)
yield Label("↑↓ Navigate · ⏎ Select · esc Close · ⇥ Complete", id="palette-hint")
def on_mount(self) -> None:
"""Focus input when mounted."""
@@ -76,13 +76,9 @@ class CommandSuggestions(Vertical):
SlashCommand("/backend", "Switch AI backend (claude/openai/embedded)"),
SlashCommand("/bug", "Submit feedback about Hanzo REPL"),
SlashCommand("/clear", "Clear conversation history and free up context"),
SlashCommand(
"/compact", "Clear conversation history but keep a summary in context"
),
SlashCommand("/compact", "Clear conversation history but keep a summary in context"),
SlashCommand("/config", "Open config panel", ["theme"]),
SlashCommand(
"/cost", "Show the total cost and duration of the current session"
),
SlashCommand("/cost", "Show the total cost and duration of the current session"),
SlashCommand("/doctor", "Checks the health of your Hanzo installation"),
SlashCommand("/exit", "Exit the REPL", ["quit"]),
SlashCommand("/help", "Show help and available commands"),
@@ -115,9 +111,7 @@ class CommandSuggestions(Vertical):
self.filtered_commands = self.COMMANDS
else:
query = self.query[1:] if self.query.startswith("/") else self.query
self.filtered_commands = [
cmd for cmd in self.COMMANDS if cmd.matches(query)
]
self.filtered_commands = [cmd for cmd in self.COMMANDS if cmd.matches(query)]
self.selected_index = 0
def compose(self) -> ComposeResult:
+7 -23
View File
@@ -36,9 +36,7 @@ class HanzoMagics(Magics):
# Run async chat in sync context
loop = asyncio.get_event_loop()
response = loop.run_until_complete(
self.repl.tool_executor.execute_with_tools(line)
)
response = loop.run_until_complete(self.repl.tool_executor.execute_with_tools(line))
self.console.print(Markdown(response))
@cell_magic
@@ -49,9 +47,7 @@ class HanzoMagics(Magics):
return
loop = asyncio.get_event_loop()
response = loop.run_until_complete(
self.repl.tool_executor.execute_with_tools(message)
)
response = loop.run_until_complete(self.repl.tool_executor.execute_with_tools(message))
self.console.print(Markdown(response))
@line_magic
@@ -130,9 +126,7 @@ class HanzoIPythonREPL:
self.llm_client = LLMClient()
if not self.llm_client.get_available_providers():
self.console.print("[red]No LLM providers available![/red]")
self.console.print(
"Set OPENAI_API_KEY, ANTHROPIC_API_KEY, or other provider keys"
)
self.console.print("Set OPENAI_API_KEY, ANTHROPIC_API_KEY, or other provider keys")
sys.exit(1)
# Initialize tool executor
@@ -142,11 +136,7 @@ class HanzoIPythonREPL:
"""List available tools with their methods."""
for _name, tool in sorted(self.tools.items()):
# Show available methods
methods = [
m
for m in dir(tool)
if not m.startswith("_") and callable(getattr(tool, m))
]
methods = [m for m in dir(tool) if not m.startswith("_") and callable(getattr(tool, m))]
for method in methods:
if method not in ["execute", "get_schema"]:
pass
@@ -200,9 +190,7 @@ class HanzoIPythonREPL:
def _chat_sync(self, message: str) -> str:
"""Synchronous chat wrapper."""
loop = asyncio.get_event_loop()
response = loop.run_until_complete(
self.tool_executor.execute_with_tools(message)
)
response = loop.run_until_complete(self.tool_executor.execute_with_tools(message))
self.console.print(Markdown(response))
return response
@@ -233,9 +221,7 @@ class HanzoIPythonREPL:
}
}
shell = InteractiveShellEmbed(
config=config, user_ns=namespace, exit_msg="Goodbye!"
)
shell = InteractiveShellEmbed(config=config, user_ns=namespace, exit_msg="Goodbye!")
# Register magic commands
shell.register_magics(HanzoMagics(shell, self))
@@ -247,9 +233,7 @@ class HanzoIPythonREPL:
# Print welcome info
self.console.print(f"[green]Model: {self.llm_client.current_model}[/green]")
self.console.print("[dim]Type ? for help, ?? for more details[/dim]")
self.console.print(
"[dim]Use %chat for AI chat, tools.<tab> for completion[/dim]"
)
self.console.print("[dim]Use %chat for AI chat, tools.<tab> for completion[/dim]")
# Start the shell
shell()
+4 -13
View File
@@ -62,17 +62,13 @@ class LLMClient:
for provider in preferred_order:
if provider in self.available_providers:
self.current_provider = provider
self.current_model = self.DEFAULT_MODELS.get(
provider, "gpt-3.5-turbo"
)
self.current_model = self.DEFAULT_MODELS.get(provider, "gpt-3.5-turbo")
break
# If no preferred provider, use the first available
if not self.current_provider:
self.current_provider = list(self.available_providers)[0]
self.current_model = self.DEFAULT_MODELS.get(
self.current_provider, "gpt-3.5-turbo"
)
self.current_model = self.DEFAULT_MODELS.get(self.current_provider, "gpt-3.5-turbo")
def _detect_providers(self) -> Set[str]:
"""Detect which LLM providers have API keys configured."""
@@ -81,9 +77,7 @@ class LLMClient:
for provider, env_vars in self.PROVIDER_ENV_VARS.items():
if not env_vars: # No API key needed (e.g., Ollama)
# Check if the provider is accessible
if provider == "ollama" and os.path.exists(
os.path.expanduser("~/.ollama")
):
if provider == "ollama" and os.path.exists(os.path.expanduser("~/.ollama")):
# TODO: Check if Ollama is running
available.add(provider)
continue
@@ -173,10 +167,7 @@ class LLMClient:
# Try to infer provider from model name
for prefix, provider in provider_prefixes.items():
if (
model.lower().startswith(prefix)
and provider in self.available_providers
):
if model.lower().startswith(prefix) and provider in self.available_providers:
self.current_provider = provider
self.current_model = model
return
+4 -11
View File
@@ -69,18 +69,14 @@ class HanzoREPL:
self.console.print("- etc.")
sys.exit(1)
self.console.print(
f"[green]Available providers: {', '.join(available_providers)}[/green]"
)
self.console.print(f"[green]Available providers: {', '.join(available_providers)}[/green]")
# Initialize tool executor
self.tool_executor = ToolExecutor(self.mcp_server, self.llm_client)
# Initialize prompt session
tool_names = [tool.name for tool in self.mcp_server.tools.values()]
completer = WordCompleter(
list(self.commands.keys()) + tool_names, ignore_case=True
)
completer = WordCompleter(list(self.commands.keys()) + tool_names, ignore_case=True)
self.session = PromptSession(
history=FileHistory(self.history_file),
auto_suggest=AutoSuggestFromHistory(),
@@ -203,15 +199,12 @@ Simply type your message and the REPL will:
status = "Active" if provider == self.llm_client.current_provider else ""
table.add_row(
provider,
", ".join(provider_models[:3])
+ ("..." if len(provider_models) > 3 else ""),
", ".join(provider_models[:3]) + ("..." if len(provider_models) > 3 else ""),
status,
)
self.console.print(table)
self.console.print(
f"\nCurrent model: [bold cyan]{self.llm_client.current_model}[/bold cyan]"
)
self.console.print(f"\nCurrent model: [bold cyan]{self.llm_client.current_model}[/bold cyan]")
async def set_model(self, command: str):
"""Set the LLM model to use."""
+2 -6
View File
@@ -9,9 +9,7 @@ from rich.console import Console
from rich.table import Table
async def run_tool_tests(
console: Console, mcp_server: HanzoMCPServer, tool_executor: Any
):
async def run_tool_tests(console: Console, mcp_server: HanzoMCPServer, tool_executor: Any):
"""Run comprehensive tests for MCP tools."""
console.print("\n[bold cyan]Running MCP Tool Tests[/bold cyan]\n")
@@ -209,9 +207,7 @@ async def test_agent_delegation(
try:
tool = mcp_server.tools.get("dispatch_agent")
if tool:
result = await tool.execute(
instruction="List the current directory contents"
)
result = await tool.execute(instruction="List the current directory contents")
success = result is not None
results.append(("Agent dispatch", success, "Agent completed task"))
else:
+14 -43
View File
@@ -239,9 +239,7 @@ class HanzoTextualREPL(App):
# Show specific info based on backend
if backend_name == "claude":
if hasattr(backend, "authenticated") and backend.authenticated:
messages.write(
Text("● Using Claude personal account", style="cyan")
)
messages.write(Text("● Using Claude personal account", style="cyan"))
else:
messages.write(
Text(
@@ -250,9 +248,7 @@ class HanzoTextualREPL(App):
)
)
elif backend_name == "embedded":
messages.write(
Text(f"● Model: {self.llm_client.current_model}", style="green")
)
messages.write(Text(f"● Model: {self.llm_client.current_model}", style="green"))
# Initialize tool executor with backend
self.tool_executor = ToolExecutor(self.mcp_server, self.backend_manager)
@@ -260,9 +256,7 @@ class HanzoTextualREPL(App):
# List available backends
backends = self.backend_manager.list_backends()
available = [name for name, avail in backends.items() if avail]
messages.write(
Text(f"● Available backends: {', '.join(available)}", style="dim")
)
messages.write(Text(f"● Available backends: {', '.join(available)}", style="dim"))
except Exception as e:
self.show_error(f"Initialization error: {e}")
@@ -505,9 +499,7 @@ class HanzoTextualREPL(App):
# Execute MCP tool
await self.execute_tool(command.name, command.parameters)
async def execute_tool(
self, tool_name: str, parameters: Optional[Dict] = None
) -> None:
async def execute_tool(self, tool_name: str, parameters: Optional[Dict] = None) -> None:
"""Execute an MCP tool."""
self.show_message(f"Executing: {tool_name}", "cyan")
@@ -522,10 +514,7 @@ class HanzoTextualREPL(App):
return
# Get parameters if needed
if parameters and any(
p.get("required", False)
for p in parameters.get("properties", {}).values()
):
if parameters and any(p.get("required", False) for p in parameters.get("properties", {}).values()):
# TODO: Show parameter input dialog
self.show_message("Tool requires parameters. Using defaults.", "yellow")
params = {}
@@ -637,9 +626,7 @@ class HanzoTextualREPL(App):
self.llm_client.set_model(arg)
self.show_message(f"Model changed to: {arg}", "green")
else:
self.show_message(
"Model selection only available for embedded backend", "yellow"
)
self.show_message("Model selection only available for embedded backend", "yellow")
return True
if command == "auth":
await self.handle_auth_command()
@@ -683,9 +670,7 @@ class HanzoTextualREPL(App):
from .voice_mode import VOICE_AVAILABLE, VoiceCommands, VoiceMode
if not VOICE_AVAILABLE:
self.show_error(
"Voice mode not available. Install: pip install speechrecognition pyttsx3 pyaudio"
)
self.show_error("Voice mode not available. Install: pip install speechrecognition pyttsx3 pyaudio")
return
if not hasattr(self, "voice_mode"):
@@ -700,17 +685,13 @@ class HanzoTextualREPL(App):
# Start voice mode
def on_speech(text: str):
# Process voice input
processed, should_stop = self.voice_commands.process_voice_input(
text
)
processed, should_stop = self.voice_commands.process_voice_input(text)
if should_stop:
self.call_from_thread(self.toggle_voice_mode)
elif processed:
# Show what was heard
self.call_from_thread(
self.show_message, f"Heard: {text}", "dim"
)
self.call_from_thread(self.show_message, f"Heard: {text}", "dim")
# Process the command
self.call_from_thread(self.process_voice_command, processed)
@@ -775,9 +756,7 @@ class HanzoTextualREPL(App):
try:
success = await backend.authenticate()
if success:
self.show_message(
"Successfully authenticated with Claude!", "green"
)
self.show_message("Successfully authenticated with Claude!", "green")
self.show_message(
"You can now use your personal Claude account without API keys.",
"cyan",
@@ -787,9 +766,7 @@ class HanzoTextualREPL(App):
except Exception as e:
self.show_error(f"Authentication error: {e}")
else:
self.show_message(
"Claude backend doesn't support authentication", "yellow"
)
self.show_message("Claude backend doesn't support authentication", "yellow")
else:
self.show_message(
f"Authentication not available for {self.backend_manager.current_backend} backend",
@@ -807,9 +784,7 @@ class HanzoTextualREPL(App):
for name, available in backends.items():
status = "✓" if available else "✗"
current = (
" (current)" if name == self.backend_manager.current_backend else ""
)
current = " (current)" if name == self.backend_manager.current_backend else ""
style = "green" if available else "red"
messages.write(Text(f" {status} {name}{current}", style=style))
@@ -842,9 +817,7 @@ class HanzoTextualREPL(App):
"""Handle /logout command."""
backend = self.backend_manager.get_backend()
if self.backend_manager.current_backend == "claude" and hasattr(
backend, "logout"
):
if self.backend_manager.current_backend == "claude" and hasattr(backend, "logout"):
await backend.logout()
self.show_message("Logged out from Claude account", "yellow")
else:
@@ -863,9 +836,7 @@ class HanzoTextualREPL(App):
if backend_name == "claude":
auth_status = (
"Authenticated"
if hasattr(backend, "authenticated") and backend.authenticated
else "Not authenticated"
"Authenticated" if hasattr(backend, "authenticated") and backend.authenticated else "Not authenticated"
)
messages.write(f" Auth status: {auth_status}")
elif backend_name == "embedded":
+5 -21
View File
@@ -81,9 +81,7 @@ class ToolExecutor:
return result
except Exception as e:
self.console.print(
Panel(f"[bold red]Error:[/bold red] {str(e)}", border_style="red")
)
self.console.print(Panel(f"[bold red]Error:[/bold red] {str(e)}", border_style="red"))
raise
async def execute_with_tools(self, user_message: str) -> str:
@@ -135,19 +133,11 @@ Available tool categories:
from types import SimpleNamespace
response = SimpleNamespace(
choices=[
SimpleNamespace(
message=SimpleNamespace(
content=response_text, tool_calls=None
)
)
]
choices=[SimpleNamespace(message=SimpleNamespace(content=response_text, tool_calls=None))]
)
else:
# Legacy LLMClient
response = await self.backend.chat(
messages=messages, tools=tools, tool_choice="auto"
)
response = await self.backend.chat(messages=messages, tools=tools, tool_choice="auto")
# Extract response
message = response.choices[0].message
@@ -180,11 +170,7 @@ Available tool categories:
{
"role": "tool",
"tool_call_id": tool_call.id,
"content": (
json.dumps(result)
if not isinstance(result, str)
else result
),
"content": (json.dumps(result) if not isinstance(result, str) else result),
}
)
@@ -205,9 +191,7 @@ Available tool categories:
final_response = message.content
# Add to history
self.conversation_history.append(
{"role": "assistant", "content": final_response}
)
self.conversation_history.append({"role": "assistant", "content": final_response})
break
+2 -6
View File
@@ -21,9 +21,7 @@ class VoiceMode:
def __init__(self):
if not VOICE_AVAILABLE:
raise ImportError(
"Voice dependencies not installed. Run: pip install speechrecognition pyttsx3 pyaudio"
)
raise ImportError("Voice dependencies not installed. Run: pip install speechrecognition pyttsx3 pyaudio")
# Speech recognition
self.recognizer = sr.Recognizer()
@@ -99,9 +97,7 @@ class VoiceMode:
self.on_listening_started()
# Listen with timeout
audio = self.recognizer.listen(
source, timeout=1, phrase_time_limit=10
)
audio = self.recognizer.listen(source, timeout=1, phrase_time_limit=10)
# Signal listening stopped
self.is_listening = False
+2 -2
View File
@@ -1,8 +1,8 @@
"""Test all hanzo-tools-* packages import and register correctly."""
import asyncio
import sys
import time
import asyncio
import pytest
@@ -143,7 +143,7 @@ class TestToolPackages:
)
def test_llm_tools(self):
"""Test hanzo-tools-llm imports (tools depend on litellm)."""
from hanzo_tools.llm import LLM_AVAILABLE, TOOLS
from hanzo_tools.llm import TOOLS, LLM_AVAILABLE
# LLM tools are optional, depend on litellm
if LLM_AVAILABLE:
+15 -15
View File
@@ -25,7 +25,7 @@ Individual packages:
Usage:
from hanzo_tools.filesystem import register_tools as register_fs
from hanzo_tools.shell import register_tools as register_shell
# Register with MCP server
register_fs(mcp_server, permission_manager)
register_shell(mcp_server)
@@ -39,16 +39,16 @@ __path__ = __import__("pkgutil").extend_path(__path__, __name__)
def discover_tools():
"""Discover all installed tool packages.
Uses entry points to find hanzo.tools plugins.
Returns:
Dict of package_name -> tools list
"""
import importlib.metadata
tools = {}
try:
eps = importlib.metadata.entry_points(group="hanzo.tools")
for ep in eps:
@@ -60,53 +60,53 @@ def discover_tools():
pass
except Exception:
pass
return tools
def register_all(mcp_server, permission_manager=None, enabled_tools=None):
"""Register all discovered tools with the MCP server.
Args:
mcp_server: FastMCP server instance
permission_manager: Optional permission manager
enabled_tools: Dict of tool_name -> enabled state
Returns:
List of registered tool instances
"""
import importlib.metadata
registered = []
enabled = enabled_tools or {}
try:
eps = importlib.metadata.entry_points(group="hanzo.tools")
for ep in eps:
# Check if package is enabled
if not enabled.get(ep.name, True):
continue
try:
module_parent = ep.value.rsplit(":", 1)[0]
module = __import__(module_parent, fromlist=["register_tools"])
if hasattr(module, "register_tools"):
try:
tools = module.register_tools(
mcp_server,
mcp_server,
permission_manager=permission_manager,
enabled_tools=enabled,
)
except TypeError:
# Some tools don't need permission_manager
tools = module.register_tools(mcp_server, enabled_tools=enabled)
if tools:
registered.extend(tools)
except Exception as e:
print(f"Failed to register {ep.name}: {e}")
except Exception:
pass
return registered
+82 -79
View File
@@ -14,16 +14,16 @@ from hanzo.tools.detector import AITool, ToolDetector
class TestToolDetector(unittest.TestCase):
"""Test AI tool detection functionality."""
def setUp(self):
"""Set up test fixtures."""
self.detector = ToolDetector()
def test_tool_initialization(self):
"""Test that tools are properly initialized."""
# Check that we have tools defined
self.assertGreater(len(self.detector.TOOLS), 0)
# Check that each tool has required attributes
for tool in self.detector.TOOLS:
self.assertIsInstance(tool.name, str)
@@ -31,29 +31,29 @@ class TestToolDetector(unittest.TestCase):
self.assertIsInstance(tool.provider, str)
self.assertIsInstance(tool.priority, int)
self.assertIsInstance(tool.detected, bool)
def test_priority_ordering(self):
"""Test that tools have correct priority ordering."""
tools = self.detector.TOOLS
# Find specific tools
hanzod = next((t for t in tools if t.name == "hanzod"), None)
hanzo_router = next((t for t in tools if t.name == "hanzo-router"), None)
claude_code = next((t for t in tools if t.name == "claude-code"), None)
# Hanzo Node should have highest priority (0)
if hanzod:
self.assertEqual(hanzod.priority, 0)
# Router should have high priority
if hanzo_router:
self.assertLessEqual(hanzo_router.priority, 2)
# Claude Code should have reasonable priority
if claude_code:
self.assertLessEqual(claude_code.priority, 5)
@patch('hanzo.tools.detector.httpx.post')
@patch("hanzo.tools.detector.httpx.post")
def test_hanzod_detection_success(self, mock_post):
"""Test successful Hanzo Node detection."""
# Mock successful response
@@ -61,94 +61,95 @@ class TestToolDetector(unittest.TestCase):
mock_response.status_code = 200
mock_response.json.return_value = {"status": "ok"}
mock_post.return_value = mock_response
hanzod = AITool(
name="hanzod",
command="hanzo node",
display_name="Hanzo Node",
provider="hanzo-local",
priority=0,
api_endpoint="http://localhost:3690/health"
api_endpoint="http://localhost:3690/health",
)
result = self.detector.detect_tool(hanzod)
self.assertTrue(result)
self.assertTrue(hanzod.detected)
@patch('hanzo.tools.detector.httpx.post')
@patch("hanzo.tools.detector.httpx.post")
def test_hanzod_detection_failure_404(self, mock_post):
"""Test Hanzo Node detection fails on 404."""
# Mock 404 response
mock_response = Mock()
mock_response.status_code = 404
mock_post.return_value = mock_response
hanzod = AITool(
name="hanzod",
command="hanzo node",
display_name="Hanzo Node",
provider="hanzo-local",
priority=0,
api_endpoint="http://localhost:3690/health"
api_endpoint="http://localhost:3690/health",
)
result = self.detector.detect_tool(hanzod)
self.assertFalse(result)
self.assertFalse(hanzod.detected)
@patch('hanzo.tools.detector.httpx.post')
@patch("hanzo.tools.detector.httpx.post")
def test_hanzod_detection_connection_refused(self, mock_post):
"""Test Hanzo Node detection handles connection refused."""
import httpx
# Mock connection error
mock_post.side_effect = httpx.ConnectError("Connection refused")
hanzod = AITool(
name="hanzod",
command="hanzo node",
display_name="Hanzo Node",
provider="hanzo-local",
priority=0,
api_endpoint="http://localhost:3690/health"
api_endpoint="http://localhost:3690/health",
)
result = self.detector.detect_tool(hanzod)
self.assertFalse(result)
self.assertFalse(hanzod.detected)
@patch('hanzo.tools.detector.shutil.which')
@patch("hanzo.tools.detector.shutil.which")
def test_command_detection(self, mock_which):
"""Test command-based tool detection."""
# Mock command exists
mock_which.return_value = "/usr/local/bin/claude"
claude_tool = AITool(
name="claude-code",
command="claude",
display_name="Claude Code",
provider="anthropic",
priority=3,
check_command="claude"
check_command="claude",
)
result = self.detector.detect_tool(claude_tool)
self.assertTrue(result)
self.assertTrue(claude_tool.detected)
# Mock command doesn't exist
mock_which.return_value = None
claude_tool.detected = False
result = self.detector.detect_tool(claude_tool)
self.assertFalse(result)
self.assertFalse(claude_tool.detected)
@patch.dict(os.environ, {'OPENAI_API_KEY': 'test-key'})
@patch('hanzo.tools.detector.shutil.which')
@patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"})
@patch("hanzo.tools.detector.shutil.which")
def test_env_var_detection(self, mock_which):
"""Test environment variable based detection."""
mock_which.return_value = "/usr/local/bin/openai"
openai_tool = AITool(
name="openai",
command="openai",
@@ -156,24 +157,24 @@ class TestToolDetector(unittest.TestCase):
provider="openai",
priority=4,
check_command="openai",
env_var="OPENAI_API_KEY"
env_var="OPENAI_API_KEY",
)
result = self.detector.detect_tool(openai_tool)
self.assertTrue(result)
self.assertTrue(openai_tool.detected)
@patch.dict(os.environ, {}, clear=True) # Clear all env vars
@patch('hanzo.tools.detector.shutil.which')
@patch("hanzo.tools.detector.shutil.which")
def test_env_var_missing(self, mock_which):
"""Test that tool is still detected when command exists but env var is missing.
The tool detection logic checks for command existence first,
and env_var is only a fallback. A tool can be "detected" but
may not be fully functional without the API key.
"""
mock_which.return_value = "/usr/local/bin/openai"
openai_tool = AITool(
name="openai",
command="openai",
@@ -181,20 +182,20 @@ class TestToolDetector(unittest.TestCase):
provider="openai",
priority=4,
check_command="openai",
env_var="OPENAI_API_KEY"
env_var="OPENAI_API_KEY",
)
result = self.detector.detect_tool(openai_tool)
# Tool is detected because command exists, even without API key
self.assertTrue(result)
self.assertTrue(openai_tool.detected)
@patch.dict(os.environ, {}, clear=True) # Clear all env vars
@patch('hanzo.tools.detector.shutil.which')
@patch("hanzo.tools.detector.shutil.which")
def test_no_command_no_env_var(self, mock_which):
"""Test detection fails when neither command nor env var exists."""
mock_which.return_value = None # Command doesn't exist
openai_tool = AITool(
name="openai",
command="openai",
@@ -202,68 +203,70 @@ class TestToolDetector(unittest.TestCase):
provider="openai",
priority=4,
check_command="openai",
env_var="OPENAI_API_KEY"
env_var="OPENAI_API_KEY",
)
result = self.detector.detect_tool(openai_tool)
self.assertFalse(result)
self.assertFalse(openai_tool.detected)
@patch('hanzo.tools.detector.subprocess.run')
@patch('hanzo.tools.detector.shutil.which')
@patch("hanzo.tools.detector.subprocess.run")
@patch("hanzo.tools.detector.shutil.which")
def test_version_detection(self, mock_which, mock_run):
"""Test version detection for tools."""
mock_which.return_value = "/usr/local/bin/hanzo"
# Mock version command output
mock_result = Mock()
mock_result.stdout = "hanzo version 0.3.23\n"
mock_result.returncode = 0
mock_run.return_value = mock_result
hanzo_tool = AITool(
name="hanzo-dev",
command="hanzo dev",
display_name="Hanzo Dev",
provider="hanzo",
priority=2,
check_command="hanzo"
check_command="hanzo",
)
result = self.detector.detect_tool(hanzo_tool)
self.assertTrue(result)
self.assertIn("0.3.23", hanzo_tool.version or "")
@patch('hanzo.tools.detector.httpx.post')
@patch('hanzo.tools.detector.shutil.which')
@patch("hanzo.tools.detector.httpx.post")
@patch("hanzo.tools.detector.shutil.which")
def test_detect_all(self, mock_which, mock_post):
"""Test detecting all available tools."""
# Mock some tools as available
def which_side_effect(cmd):
if cmd in ["claude", "hanzo", "openai"]:
return f"/usr/local/bin/{cmd}"
return None
mock_which.side_effect = which_side_effect
# Mock Hanzo Node as not running
import httpx
mock_post.side_effect = httpx.ConnectError("Connection refused")
# Set OpenAI API key for testing
with patch.dict(os.environ, {'OPENAI_API_KEY': 'test-key'}):
with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}):
detected_tools = self.detector.detect_all()
# Should have detected some tools
self.assertGreater(len(detected_tools), 0)
# Check that detected tools are marked as such
for tool in detected_tools:
if tool.name in ["claude-code", "hanzo-dev", "openai"]:
self.assertTrue(tool.detected, f"{tool.name} should be detected")
elif tool.name == "hanzod":
self.assertFalse(tool.detected, "hanzod should not be detected")
def test_get_default_tool(self):
"""Test getting the default tool."""
# Create mock tools with different priorities (already sorted)
@@ -272,51 +275,51 @@ class TestToolDetector(unittest.TestCase):
AITool("tool4", "cmd4", "Tool 4", "provider4", priority=3, detected=True),
AITool("tool1", "cmd1", "Tool 1", "provider1", priority=5, detected=True),
]
self.detector.detected_tools = tools
# Default should be the first tool (tool2 with priority 2)
default = self.detector.get_default_tool()
self.assertIsNotNone(default)
self.assertEqual(default.name, "tool2")
self.assertEqual(default.priority, 2)
def test_get_default_tool_none_detected(self):
"""Test get_default_tool returns None when no tools detected."""
# Empty detected_tools list
self.detector.detected_tools = []
# Mock detect_all to return empty list
with patch.object(self.detector, 'detect_all', return_value=[]):
with patch.object(self.detector, "detect_all", return_value=[]):
default = self.detector.get_default_tool()
self.assertIsNone(default)
@patch('hanzo.tools.detector.httpx.post')
@patch("hanzo.tools.detector.httpx.post")
def test_port_fallback_for_hanzod(self, mock_post):
"""Test that Hanzo Node tries both port 3690 and 8000."""
import httpx
# First call to port 3690 fails, second to 8000 succeeds
responses = [
httpx.ConnectError("Connection refused"), # Port 3690 fails
Mock(status_code=200) # Port 8000 succeeds
Mock(status_code=200), # Port 8000 succeeds
]
mock_post.side_effect = responses
hanzod = AITool(
name="hanzod",
command="hanzo node",
display_name="Hanzo Node",
provider="hanzo-local",
priority=0,
api_endpoint="http://localhost:3690/health"
api_endpoint="http://localhost:3690/health",
)
result = self.detector.detect_tool(hanzod)
# Should have tried both ports
self.assertEqual(mock_post.call_count, 2)
# Check that both ports were tried
calls = mock_post.call_args_list
urls = [call[0][0] for call in calls]
@@ -325,4 +328,4 @@ class TestToolDetector(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
unittest.main()
+4 -14
View File
@@ -125,9 +125,7 @@ class BaseAgent(ABC, Generic[TContext, TResult]):
# Log start
if context and hasattr(context, "log"):
await context.log(
f"Starting {self.name} with model {self.config.model}"
)
await context.log(f"Starting {self.name} with model {self.config.model}")
# Execute with retries
result = await self._execute_with_retries(prompt, context, env, **kwargs)
@@ -145,11 +143,7 @@ class BaseAgent(ABC, Generic[TContext, TResult]):
except Exception as e:
self._end_time = datetime.now()
duration = (
(self._end_time - self._start_time).total_seconds()
if self._start_time
else None
)
duration = (self._end_time - self._start_time).total_seconds() if self._start_time else None
logger.error(f"Agent {self.name} failed: {e}")
@@ -231,9 +225,7 @@ class BaseAgent(ABC, Generic[TContext, TResult]):
if attempt < self.config.max_retries - 1:
await asyncio.sleep(2**attempt)
raise Exception(
f"All {self.config.max_retries} attempts failed. Last error: {last_error}"
)
raise Exception(f"All {self.config.max_retries} attempts failed. Last error: {last_error}")
@abstractmethod
async def _execute_impl(
@@ -326,9 +318,7 @@ class CLIAgent(BaseAgent[TContext, str]):
)
except asyncio.TimeoutError:
process.kill()
raise asyncio.TimeoutError(
f"Command timed out after {self.config.timeout} seconds"
)
raise asyncio.TimeoutError(f"Command timed out after {self.config.timeout} seconds")
# Check for errors
if process.returncode != 0:
+7 -25
View File
@@ -55,11 +55,7 @@ class BatchTask:
@property
def success(self) -> bool:
"""Check if task succeeded."""
return (
self.status == "completed"
and self.result
and getattr(self.result, "success", True)
)
return self.status == "completed" and self.result and getattr(self.result, "success", True)
def get_error(self) -> Optional[str]:
"""Get error message from direct error or result."""
@@ -161,9 +157,7 @@ class BatchConfig:
if not config.consensus_mode and not config.critic_mode:
agent_match = re.search(r"agent:(\w+)", command)
if agent_match:
config.agent_model = agent_match.group(
1
) # Will be resolved in __post_init__
config.agent_model = agent_match.group(1) # Will be resolved in __post_init__
# Parse worktree option
worktree_match = re.search(r"worktree:(true|false)", command)
@@ -220,9 +214,7 @@ class BatchOrchestrator:
self._task_counter += 1
return f"task_{self._task_counter:04d}"
async def _setup_worktree(
self, task_id: str, config: BatchConfig
) -> Optional[Path]:
async def _setup_worktree(self, task_id: str, config: BatchConfig) -> Optional[Path]:
"""Setup git worktree for parallel editing.
Args:
@@ -536,9 +528,7 @@ class BatchOrchestrator:
# If consensus reached, combine insights
if consensus_result["consensus_reached"] and successful_responses:
combined = "\n\n".join(
[f"[{r['model']}]: {r['response']}" for r in successful_responses]
)
combined = "\n\n".join([f"[{r['model']}]: {r['response']}" for r in successful_responses])
consensus_result["combined_response"] = combined
return consensus_result
@@ -627,9 +617,7 @@ Provide your critical analysis:"""
return {
"critic_chain": reviews,
"final_review": (
reviews[-1]["review"] if reviews and "review" in reviews[-1] else None
),
"final_review": (reviews[-1]["review"] if reviews and "review" in reviews[-1] else None),
"models_used": models,
"chain_length": len(reviews),
}
@@ -706,11 +694,7 @@ Provide your critical analysis:"""
if "review" in review:
console.print(
Panel(
(
review["review"][:500] + "..."
if len(review["review"]) > 500
else review["review"]
),
(review["review"][:500] + "..." if len(review["review"]) > 500 else review["review"]),
title=f"[cyan]Critic {review['iteration']}: {review['model']}[/cyan]",
)
)
@@ -785,9 +769,7 @@ Provide your critical analysis:"""
del self.active_tasks[task.id]
# Generate summary
total_duration = sum(
t.duration() or 0 for t in self.completed_tasks + self.failed_tasks
)
total_duration = sum(t.duration() or 0 for t in self.completed_tasks + self.failed_tasks)
summary = {
"total_tasks": len(tasks),
+14 -42
View File
@@ -115,15 +115,9 @@ def serve(ctx, name: str, port: int):
@cli.command()
@click.option("--name", "-n", help="Node name (auto-generated if not provided)")
@click.option(
"--port", "-p", default=52415, help="Node port (default: 52415 for hanzo/net)"
)
@click.option(
"--network", default="local", help="Network to join (mainnet/testnet/local)"
)
@click.option(
"--models", "-m", multiple=True, help="Models to serve (e.g., llama-3.2-3b)"
)
@click.option("--port", "-p", default=52415, help="Node port (default: 52415 for hanzo/net)")
@click.option("--network", default="local", help="Network to join (mainnet/testnet/local)")
@click.option("--models", "-m", multiple=True, help="Models to serve (e.g., llama-3.2-3b)")
@click.option("--max-jobs", type=int, default=10, help="Max concurrent jobs")
@click.pass_context
def net(ctx, name: str, port: int, network: str, models: tuple, max_jobs: int):
@@ -137,15 +131,9 @@ def net(ctx, name: str, port: int, network: str, models: tuple, max_jobs: int):
@cli.command()
@click.option("--name", "-n", help="Node name (auto-generated if not provided)")
@click.option(
"--port", "-p", default=52415, help="Node port (default: 52415 for hanzo/net)"
)
@click.option(
"--network", default="local", help="Network to join (mainnet/testnet/local)"
)
@click.option(
"--models", "-m", multiple=True, help="Models to serve (e.g., llama-3.2-3b)"
)
@click.option("--port", "-p", default=52415, help="Node port (default: 52415 for hanzo/net)")
@click.option("--network", default="local", help="Network to join (mainnet/testnet/local)")
@click.option("--models", "-m", multiple=True, help="Models to serve (e.g., llama-3.2-3b)")
@click.option("--max-jobs", type=int, default=10, help="Max concurrent jobs")
@click.pass_context
def node(ctx, name: str, port: int, network: str, models: tuple, max_jobs: int):
@@ -180,15 +168,9 @@ def node(ctx, name: str, port: int, network: str, models: tuple, max_jobs: int):
@click.option("--repl", is_flag=True, help="Start REPL interface (default)")
@click.option("--instances", type=int, default=2, help="Number of worker agents")
@click.option("--mcp-tools", is_flag=True, default=True, help="Enable all MCP tools")
@click.option(
"--network-mode", is_flag=True, default=True, help="Network agents together"
)
@click.option(
"--guardrails", is_flag=True, default=True, help="Enable code quality guardrails"
)
@click.option(
"--use-network/--no-network", default=True, help="Use hanzo-network if available"
)
@click.option("--network-mode", is_flag=True, default=True, help="Network agents together")
@click.option("--guardrails", is_flag=True, default=True, help="Enable code quality guardrails")
@click.option("--use-network/--no-network", default=True, help="Use hanzo-network if available")
@click.option(
"--use-hanzo-net",
is_flag=True,
@@ -266,9 +248,7 @@ def dev(
console.print(f" Router Endpoint: {orch_config.router.endpoint}")
if orch_config.codex:
console.print(f" Codex Model: {orch_config.codex.model}")
console.print(
f" Cost Optimization: {'Enabled' if orch_config.enable_cost_optimization else 'Disabled'}"
)
console.print(f" Cost Optimization: {'Enabled' if orch_config.enable_cost_optimization else 'Disabled'}")
console.print()
asyncio.run(
@@ -351,9 +331,7 @@ async def start_compute_node(
console.print(f"\n[green]✓[/green] Node initialized")
console.print(f" Port: {port}")
console.print(
f" Models: {', '.join(models) if models else 'auto-detect'}"
)
console.print(f" Models: {', '.join(models) if models else 'auto-detect'}")
console.print("\n[bold green]Hanzo Net is running![/bold green]")
console.print("WebUI: http://localhost:52415")
console.print("API: http://localhost:52415/v1/chat/completions")
@@ -375,9 +353,7 @@ async def start_compute_node(
stop_task = asyncio.create_task(stop_event.wait())
# Wait for either net to complete or stop signal
done, pending = await asyncio.wait(
[net_task, stop_task], return_when=asyncio.FIRST_COMPLETED
)
done, pending = await asyncio.wait([net_task, stop_task], return_when=asyncio.FIRST_COMPLETED)
# Cancel pending tasks
for task in pending:
@@ -413,15 +389,11 @@ async def start_compute_node(
env["NET_MODELS"] = ",".join(models)
if name:
env["NET_NODE_NAME"] = name
env["PYTHONPATH"] = (
os.path.join(net_path, "src") + ":" + env.get("PYTHONPATH", "")
)
env["PYTHONPATH"] = os.path.join(net_path, "src") + ":" + env.get("PYTHONPATH", "")
console.print(f"\n[green]✓[/green] Starting net node")
console.print(f" Port: {port}")
console.print(
f" Models: {', '.join(models) if models else 'auto-detect'}"
)
console.print(f" Models: {', '.join(models) if models else 'auto-detect'}")
console.print("\n[bold green]Hanzo Net is running![/bold green]")
console.print("WebUI: http://localhost:52415")
console.print("API: http://localhost:52415/v1/chat/completions")
+1 -5
View File
@@ -88,11 +88,7 @@ async def run(ctx, agents: tuple, task: str, parallel: bool, timeout: Optional[i
network = create_network(agents=agent_list)
# Run task
result = (
await asyncio.wait_for(network.run(task), timeout=timeout)
if timeout
else await network.run(task)
)
result = await asyncio.wait_for(network.run(task), timeout=timeout) if timeout else await network.run(task)
console.print("[green]Task completed![/green]")
console.print(result)
+2 -6
View File
@@ -127,9 +127,7 @@ def login(ctx, email: str, password: str, api_key: str, sso: bool):
hanzo_auth = HanzoAuth()
# Email auth not implemented yet
console.print(
"[yellow]Email authentication not yet implemented[/yellow]"
)
console.print("[yellow]Email authentication not yet implemented[/yellow]")
console.print("[dim]Saving credentials locally for development[/dim]")
auth = {
@@ -249,9 +247,7 @@ def whoami():
content = "\n".join(lines) if lines else "[dim]No user information available[/dim]"
console.print(
Panel(content, title="[bold cyan]User Information[/bold cyan]", box=box.ROUNDED)
)
console.print(Panel(content, title="[bold cyan]User Information[/bold cyan]", box=box.ROUNDED))
@auth_group.command(name="set-key")
+3 -9
View File
@@ -18,9 +18,7 @@ from ..utils.output import console
@click.option("--local/--cloud", default=True, help="Use local or cloud model")
@click.option("--once", is_flag=True, help="Single question mode")
@click.option("--system", "-s", help="System prompt")
@click.option(
"--repl", is_flag=True, help="Start full REPL interface (like Claude Code)"
)
@click.option("--repl", is_flag=True, help="Start full REPL interface (like Claude Code)")
@click.option("--ipython", is_flag=True, help="Use IPython REPL interface")
@click.option("--tui", is_flag=True, help="Use beautiful TUI interface")
@click.argument("prompt", nargs=-1)
@@ -80,9 +78,7 @@ def chat_command(
asyncio.run(interactive_chat(ctx, model, local, system))
async def ask_once(
ctx, prompt: str, model: str, local: bool, system: Optional[str] = None
):
async def ask_once(ctx, prompt: str, model: str, local: bool, system: Optional[str] = None):
"""Ask a single question."""
messages = []
if system:
@@ -172,9 +168,7 @@ async def interactive_chat(ctx, model: str, local: bool, system: Optional[str]):
from prompt_toolkit import PromptSession
from prompt_toolkit.history import FileHistory
console.print(
f"[cyan]Chat session started[/cyan] (model: {model}, mode: {'local' if local else 'cloud'})"
)
console.print(f"[cyan]Chat session started[/cyan] (model: {model}, mode: {'local' if local else 'cloud'})")
console.print("Type 'exit' or Ctrl+D to quit\n")
session = PromptSession(history=FileHistory(".hanzo_chat_history"))
+1 -3
View File
@@ -115,9 +115,7 @@ def set(ctx, key: str, value: str, is_global: bool, is_local: bool):
# Save config
save_config(config_path, config)
console.print(
f"[green]✓[/green] Set {key} = {parsed_value} in {config_name} config"
)
console.print(f"[green]✓[/green] Set {key} = {parsed_value} in {config_name} config")
@config_group.command()
+2 -6
View File
@@ -96,9 +96,7 @@ async def tools(ctx, category: str):
# Display tools
for cat, tools in sorted(categories.items()):
table = Table(
title=f"{cat.title()} Tools" if len(categories) > 1 else "MCP Tools"
)
table = Table(title=f"{cat.title()} Tools" if len(categories) > 1 else "MCP Tools")
table.add_column("Name", style="cyan", no_wrap=True)
table.add_column("Description")
@@ -241,9 +239,7 @@ def install(ctx, path: str):
existing["mcpServers"]["hanzo-mcp"] = config["mcpServers"]["hanzo-mcp"]
config = existing
except Exception as e:
console.print(
f"[yellow]Warning: Could not read existing config: {e}[/yellow]"
)
console.print(f"[yellow]Warning: Could not read existing config: {e}[/yellow]")
# Write config
config_path.parent.mkdir(parents=True, exist_ok=True)
+7 -21
View File
@@ -50,9 +50,7 @@ async def start(
if not click.confirm("Continue without wallet?"):
return
miner = HanzoMiner(
name=name, wallet=wallet, device=device, network=network, min_stake=min_stake
)
miner = HanzoMiner(name=name, wallet=wallet, device=device, network=network, min_stake=min_stake)
with Progress(
SpinnerColumn(),
@@ -94,9 +92,7 @@ async def start(
if event["type"] == "log":
console.print(event["message"], end="")
elif event["type"] == "job":
console.print(
f"[green]Job completed:[/green] {event['job_id']} (+{event['tokens']} tokens)"
)
console.print(f"[green]Job completed:[/green] {event['job_id']} (+{event['tokens']} tokens)")
elif event["type"] == "error":
console.print(f"[red]Error:[/red] {event['message']}")
except KeyboardInterrupt:
@@ -109,9 +105,7 @@ async def start(
console.print("\n[cyan]Session Summary:[/cyan]")
console.print(f" Jobs completed: {final_stats.get('jobs_completed', 0)}")
console.print(f" Tokens earned: {final_stats.get('tokens_earned', 0)}")
console.print(
f" Average job time: {final_stats.get('avg_job_time', 'N/A')}"
)
console.print(f" Average job time: {final_stats.get('avg_job_time', 'N/A')}")
console.print("[green]✓[/green] Miner stopped")
@@ -207,15 +201,11 @@ async def status(ctx, name: str, detailed: bool):
console.print(f" Network: {stats.get('network', 'unknown')}")
console.print(f" Wallet: {stats.get('wallet', 'Not set')}")
console.print(f" Models: {', '.join(stats.get('models', []))}")
console.print(
f" Memory: {stats.get('memory_used', 0)} / {stats.get('memory_total', 0)} MB"
)
console.print(f" Memory: {stats.get('memory_used', 0)} / {stats.get('memory_total', 0)} MB")
console.print(f" CPU: {stats.get('cpu_percent', 0)}%")
if gpu := stats.get("gpu"):
console.print(
f" GPU: {gpu['name']} ({gpu['memory_used']} / {gpu['memory_total']} MB)"
)
console.print(f" GPU: {gpu['name']} ({gpu['memory_used']} / {gpu['memory_total']} MB)")
console.print(f"\n[cyan]Performance:[/cyan]")
console.print(f" Average job time: {stats.get('avg_job_time', 'N/A')}")
@@ -292,9 +282,7 @@ async def earnings(ctx, wallet: str, network: str):
if history := data.get("recent_jobs"):
console.print("\n[cyan]Recent Jobs:[/cyan]")
for job in history[:5]:
console.print(
f" • {job['timestamp']}: +{job['tokens']} tokens ({job['model']})"
)
console.print(f" • {job['timestamp']}: +{job['tokens']} tokens ({job['model']})")
@miner_group.command()
@@ -323,9 +311,7 @@ async def withdraw(ctx, amount: float, wallet: str, to: str, network: str):
with console.status("Processing withdrawal..."):
try:
result = await withdraw_earnings(
wallet=wallet, amount=amount, destination=to, network=network
)
result = await withdraw_earnings(wallet=wallet, amount=amount, destination=to, network=network)
console.print(f"[green]✓[/green] Withdrawal successful")
console.print(f" Transaction: {result['tx_hash']}")
+8 -25
View File
@@ -26,9 +26,7 @@ def network_group():
@click.option("--consensus", is_flag=True, help="Require consensus")
@click.option("--timeout", "-t", type=int, default=300, help="Timeout in seconds")
@click.pass_context
async def dispatch(
ctx, prompt: str, agents: int, model: str, mode: str, consensus: bool, timeout: int
):
async def dispatch(ctx, prompt: str, agents: int, model: str, mode: str, consensus: bool, timeout: int):
"""Dispatch work to agent network."""
try:
from hanzo_network import NetworkDispatcher
@@ -391,13 +389,9 @@ def topology(ctx, json: bool):
)
if gpu_available:
console.print(
"[green]✓[/green] GPU/Accelerator detected and available for AI workloads"
)
console.print("[green]✓[/green] GPU/Accelerator detected and available for AI workloads")
if "APPLE M" in caps.chip.upper():
console.print(
" [dim]Apple Silicon Neural Engine available for on-device AI[/dim]"
)
console.print(" [dim]Apple Silicon Neural Engine available for on-device AI[/dim]")
else:
console.print("[yellow]⚠[/yellow] No GPU detected - using CPU only")
console.print(" To enable GPU support:")
@@ -547,9 +541,7 @@ def topology_add(
try:
import qrcode
except ImportError:
console.print(
"\n[yellow]QR code generation requires qrcode library[/yellow]"
)
console.print("\n[yellow]QR code generation requires qrcode library[/yellow]")
console.print("Install with: pip install qrcode[pil]")
return
@@ -700,10 +692,7 @@ def models(ctx, endpoint: str):
models_list = [
model
for model in all_models
if not any(
keyword in model.get("id", "").lower()
for keyword in embedding_keywords
)
if not any(keyword in model.get("id", "").lower() for keyword in embedding_keywords)
]
if models_list:
@@ -741,16 +730,12 @@ def models(ctx, endpoint: str):
table.add_row(str(i), model_id, tier, provider)
console.print(table)
console.print(
f"\n[dim]Total LLMs: {len(models_list)} (embedding models filtered out)[/dim]"
)
console.print(f"\n[dim]Total LLMs: {len(models_list)} (embedding models filtered out)[/dim]")
else:
console.print("[yellow]No models available on gateway[/yellow]")
else:
# Show static list of known models
console.print(
"[yellow]Could not fetch live models, showing known models:[/yellow]\n"
)
console.print("[yellow]Could not fetch live models, showing known models:[/yellow]\n")
_show_static_models()
except Exception as e:
console.print(f"[yellow]Error fetching models: {e}[/yellow]\n")
@@ -758,9 +743,7 @@ def models(ctx, endpoint: str):
console.print("\n[dim]To use a model:[/dim]")
console.print(" [cyan]hanzo dev --model llama3-8b-instruct[/cyan]")
console.print(
"\n[dim cyan]Free tier:[/dim cyan] [green]Most models available without login![/green]"
)
console.print("\n[dim cyan]Free tier:[/dim cyan] [green]Most models available without login![/green]")
console.print(
"[dim cyan]Premium models:[/dim cyan] [yellow]gpt-4, claude-3-opus, o1-preview[/yellow] - [cyan]hanzo auth login[/cyan]"
)
+6 -18
View File
@@ -31,9 +31,7 @@ async def start(ctx, name: str, port: int, models: tuple, device: str):
await start_node(ctx, name, port, list(models) if models else None, device)
async def start_node(
ctx, name: str, port: int, models: Optional[List[str]] = None, device: str = "auto"
):
async def start_node(ctx, name: str, port: int, models: Optional[List[str]] = None, device: str = "auto"):
"""Start a local node via hanzo-cluster."""
try:
from hanzo_cluster import HanzoCluster
@@ -140,9 +138,7 @@ async def status(ctx, name: str):
if workers := status.get("node_details", []):
console.print("\n[cyan]Workers:[/cyan]")
for worker in workers:
console.print(
f" • {worker.get('name', 'unknown')} ({worker.get('state', 'unknown')})"
)
console.print(f" • {worker.get('name', 'unknown')} ({worker.get('state', 'unknown')})")
if device := worker.get("device"):
console.print(f" Device: {device}")
else:
@@ -256,9 +252,7 @@ def worker_group():
default="auto",
help="Device to use",
)
@click.option(
"--port", "-p", type=int, help="Worker port (auto-assigned if not specified)"
)
@click.option("--port", "-p", type=int, help="Worker port (auto-assigned if not specified)")
@click.option("--blockchain", is_flag=True, help="Enable blockchain features")
@click.option("--network", is_flag=True, help="Enable network discovery")
@click.pass_context
@@ -431,9 +425,7 @@ async def worker_info(ctx, name: str):
if resources := info.get("resources"):
console.print("\n[cyan]Resources:[/cyan]")
console.print(f" CPU: {resources.get('cpu_percent', 'N/A')}%")
console.print(
f" Memory: {resources.get('memory_used', 'N/A')} / {resources.get('memory_total', 'N/A')}"
)
console.print(f" Memory: {resources.get('memory_used', 'N/A')} / {resources.get('memory_total', 'N/A')}")
if gpu := resources.get("gpu"):
console.print(
f" GPU: {gpu.get('name', 'N/A')} ({gpu.get('memory_used', 'N/A')} / {gpu.get('memory_total', 'N/A')})"
@@ -446,12 +438,8 @@ async def worker_info(ctx, name: str):
if network := info.get("network"):
console.print("\n[cyan]Network:[/cyan]")
console.print(
f" Blockchain: {'enabled' if network.get('blockchain') else 'disabled'}"
)
console.print(
f" Discovery: {'enabled' if network.get('discovery') else 'disabled'}"
)
console.print(f" Blockchain: {'enabled' if network.get('blockchain') else 'disabled'}")
console.print(f" Discovery: {'enabled' if network.get('discovery') else 'disabled'}")
if peers := network.get("peers"):
console.print(f" Peers: {len(peers)}")
+3 -9
View File
@@ -105,18 +105,14 @@ def info(ctx):
if not providers:
console.print("\n[yellow]No LLM providers configured[/yellow]")
console.print(
"Set one of: OPENAI_API_KEY, ANTHROPIC_API_KEY, HANZO_API_KEY"
)
console.print("Set one of: OPENAI_API_KEY, ANTHROPIC_API_KEY, HANZO_API_KEY")
except ImportError:
console.print("[red]Error:[/red] hanzo-repl not installed")
@repl_group.command()
@click.option(
"--interface", type=click.Choice(["all", "ipython", "tui", "voice"]), default="all"
)
@click.option("--interface", type=click.Choice(["all", "ipython", "tui", "voice"]), default="all")
@click.pass_context
def install_extras(ctx, interface: str):
"""Install optional REPL components."""
@@ -169,9 +165,7 @@ def exec(ctx, command: tuple, model: str):
except ImportError as e:
console.print(f"[red]Import Error:[/red] {e}")
console.print(
"[yellow]Note:[/yellow] hanzo-repl may not be installed correctly"
)
console.print("[yellow]Note:[/yellow] hanzo-repl may not be installed correctly")
except Exception as e:
console.print(f"[red]Error: {e}[/red]")
+6 -20
View File
@@ -40,9 +40,7 @@ def start_router(ctx, port: int, config: Optional[str], detach: bool):
if not router_dir:
console.print("[red]Error:[/red] Hanzo router not found")
console.print("\nPlease clone the router:")
console.print(
" git clone https://github.com/hanzoai/router.git ~/work/hanzo/router"
)
console.print(" git clone https://github.com/hanzoai/router.git ~/work/hanzo/router")
return
console.print(f"[green]✓[/green] Found router at {router_dir}")
@@ -91,9 +89,7 @@ def start_router(ctx, port: int, config: Optional[str], detach: bool):
stderr=subprocess.DEVNULL,
start_new_session=True,
)
console.print(
f"[green]✓[/green] Router started in background (PID: {process.pid})"
)
console.print(f"[green]✓[/green] Router started in background (PID: {process.pid})")
console.print(f"Check status: curl http://localhost:{port}/health")
else:
# Run in foreground
@@ -116,14 +112,8 @@ def stop_router(port: int):
for proc in psutil.process_iter(["pid", "cmdline"]):
try:
cmdline = proc.info["cmdline"]
if (
cmdline
and "proxy_server" in " ".join(cmdline)
and str(port) in " ".join(cmdline)
):
console.print(
f"[yellow]Stopping router (PID: {proc.info['pid']})[/yellow]"
)
if cmdline and "proxy_server" in " ".join(cmdline) and str(port) in " ".join(cmdline):
console.print(f"[yellow]Stopping router (PID: {proc.info['pid']})[/yellow]")
proc.send_signal(signal.SIGTERM)
proc.wait(timeout=5)
found = True
@@ -148,9 +138,7 @@ def router_status(port: int):
# Try to get models
try:
models_response = httpx.get(
f"http://localhost:{port}/models", timeout=2.0
)
models_response = httpx.get(f"http://localhost:{port}/models", timeout=2.0)
if models_response.status_code == 200:
data = models_response.json()
if "data" in data:
@@ -158,9 +146,7 @@ def router_status(port: int):
except Exception:
pass
else:
console.print(
f"[yellow]Router responding but unhealthy (status: {response.status_code})[/yellow]"
)
console.print(f"[yellow]Router responding but unhealthy (status: {response.status_code})[/yellow]")
except httpx.ConnectError:
console.print(f"[red]Router not running on port {port}[/red]")
console.print("\nStart with: hanzo router start")
+2 -6
View File
@@ -30,9 +30,7 @@ async def list_tools(ctx, category: str, installed: bool):
with console.status("Loading tools..."):
try:
tools = await registry.list_tools(
category=category, installed_only=installed
)
tools = await registry.list_tools(category=category, installed_only=installed)
except Exception as e:
console.print(f"[red]Failed to load tools: {e}[/red]")
return
@@ -88,9 +86,7 @@ async def install(ctx, tool_name: str, version: str):
try:
result = await registry.install_tool(name=tool_name, version=version)
console.print(
f"[green]✓[/green] Installed {tool_name} v{result['version']}"
)
console.print(f"[green]✓[/green] Installed {tool_name} v{result['version']}")
if deps := result.get("dependencies_installed"):
console.print(f" Dependencies: {', '.join(deps)}")
+69 -215
View File
@@ -221,9 +221,7 @@ class HanzoDevOrchestrator:
grpo_dir = self.workspace_dir / "grpo"
grpo_dir.mkdir(exist_ok=True)
self.experience_manager = ExperienceManager(
str(grpo_dir / "experience_library.json")
)
self.experience_manager = ExperienceManager(str(grpo_dir / "experience_library.json"))
# Initialize with DeepSeek if API key available
deepseek_key = os.getenv("DEEPSEEK_API_KEY")
@@ -254,9 +252,7 @@ class HanzoDevOrchestrator:
for path in possible_paths:
expanded = Path(path).expanduser()
if expanded.exists() or (
path == "claude" and os.system(f"which {path} >/dev/null 2>&1") == 0
):
if expanded.exists() or (path == "claude" and os.system(f"which {path} >/dev/null 2>&1") == 0):
return str(expanded) if expanded.exists() else path
raise RuntimeError("Claude Code not found. Please specify path.")
@@ -343,9 +339,7 @@ class HanzoDevOrchestrator:
else:
return "Execute standard workflow with monitoring"
def _assess_risks(
self, problem: str, alternatives: List[str], context: Dict[str, Any]
) -> List[str]:
def _assess_risks(self, problem: str, alternatives: List[str], context: Dict[str, Any]) -> List[str]:
"""Assess risks of different approaches."""
risks = []
@@ -369,9 +363,7 @@ class HanzoDevOrchestrator:
) -> str:
"""Synthesize a decision from analysis."""
if len(risks) > 2:
return (
"Proceed cautiously with incremental approach and rollback capability"
)
return "Proceed cautiously with incremental approach and rollback capability"
elif alternatives:
return f"Execute primary approach: {alternatives[0]}"
else:
@@ -416,9 +408,7 @@ class HanzoDevOrchestrator:
if resume:
checkpoint_file = self._get_latest_checkpoint()
if checkpoint_file:
console.print(
f"[green]Resuming from checkpoint: {checkpoint_file.name}[/green]"
)
console.print(f"[green]Resuming from checkpoint: {checkpoint_file.name}[/green]")
# Prepare command
cmd = [self.claude_code_path]
@@ -536,12 +526,8 @@ class HanzoDevOrchestrator:
"timestamp": datetime.now().isoformat(),
"agent_state": self.agent_state.value,
"runtime_health": asdict(self.runtime_health),
"current_context": (
asdict(self.current_context) if self.current_context else None
),
"thinking_history": [
asdict(t) for t in self.thinking_history[-10:]
], # Last 10
"current_context": (asdict(self.current_context) if self.current_context else None),
"thinking_history": [asdict(t) for t in self.thinking_history[-10:]], # Last 10
}
with open(checkpoint_file, "w") as f:
@@ -559,9 +545,7 @@ class HanzoDevOrchestrator:
self.agent_state = AgentState(data["agent_state"])
# Restore other state as needed
console.print(
f"[green]✓ Restored from checkpoint: {checkpoint_file.name}[/green]"
)
console.print(f"[green]✓ Restored from checkpoint: {checkpoint_file.name}[/green]")
return True
except Exception as e:
console.print(f"[red]Failed to restore checkpoint: {e}[/red]")
@@ -577,9 +561,7 @@ class HanzoDevOrchestrator:
healthy = await self.health_check()
if not healthy:
console.print(
f"[yellow]Health check failed. State: {self.runtime_health.state.value}[/yellow]"
)
console.print(f"[yellow]Health check failed. State: {self.runtime_health.state.value}[/yellow]")
# Use System 2 thinking to decide what to do
thinking_result = await self.think(
@@ -588,9 +570,7 @@ class HanzoDevOrchestrator:
)
console.print(f"[cyan]Decision: {thinking_result.decision}[/cyan]")
console.print(
f"[cyan]Confidence: {thinking_result.confidence:.2f}[/cyan]"
)
console.print(f"[cyan]Confidence: {thinking_result.confidence:.2f}[/cyan]")
# Execute decision
if thinking_result.confidence > 0.6:
@@ -744,14 +724,10 @@ class HanzoDevOrchestrator:
experiences_after = len(self.experience_manager.experiences)
# Save updated library
self.experience_manager.save(
str(self.workspace_dir / "grpo" / "experience_library.json")
)
self.experience_manager.save(str(self.workspace_dir / "grpo" / "experience_library.json"))
learned_count = experiences_after - experiences_before
console.print(
f"[green]✓ Learned {learned_count} new experiences (total: {experiences_after})[/green]"
)
console.print(f"[green]✓ Learned {learned_count} new experiences (total: {experiences_after})[/green]")
return learned_count
@@ -854,9 +830,7 @@ class HanzoDevREPL:
handler = FallbackHandler()
if not handler.fallback_order:
console.print("[yellow]⚠️ No API keys detected[/yellow]")
console.print(
"[dim]Set OPENAI_API_KEY or ANTHROPIC_API_KEY to enable AI[/dim]"
)
console.print("[dim]Set OPENAI_API_KEY or ANTHROPIC_API_KEY to enable AI[/dim]")
console.print()
else:
primary = handler.fallback_order[0][1]
@@ -902,13 +876,9 @@ class HanzoDevREPL:
# Handle memory/context commands
from .memory_manager import handle_memory_command
handled = handle_memory_command(
user_input, self.memory_manager, console
)
handled = handle_memory_command(user_input, self.memory_manager, console)
if not handled:
console.print(
"[yellow]Unknown memory command. Use #memory help[/yellow]"
)
console.print("[yellow]Unknown memory command. Use #memory help[/yellow]")
else:
# Natural chat - send directly to AI agents
@@ -951,17 +921,13 @@ class HanzoDevREPL:
table.add_row("Agent State", self.orchestrator.agent_state.value)
table.add_row("Runtime State", self.orchestrator.runtime_health.state.value)
table.add_row(
"Last Response", str(self.orchestrator.runtime_health.last_response)
)
table.add_row("Last Response", str(self.orchestrator.runtime_health.last_response))
table.add_row(
"Response Time",
f"{self.orchestrator.runtime_health.response_time_ms:.2f}ms",
)
table.add_row("Error Count", str(self.orchestrator.runtime_health.error_count))
table.add_row(
"Restart Count", str(self.orchestrator.runtime_health.restart_count)
)
table.add_row("Restart Count", str(self.orchestrator.runtime_health.restart_count))
console.print(table)
@@ -1008,9 +974,7 @@ class HanzoDevREPL:
"""Restore from checkpoint."""
if not args:
# Show available checkpoints
checkpoints = list(
self.orchestrator.checkpoint_dir.glob("checkpoint_*.json")
)
checkpoints = list(self.orchestrator.checkpoint_dir.glob("checkpoint_*.json"))
if checkpoints:
console.print("[cyan]Available checkpoints:[/cyan]")
for cp in checkpoints:
@@ -1087,10 +1051,7 @@ Examples:
enhanced_message = message
# Try smart fallback if no specific model configured
if (
not hasattr(self.orchestrator, "orchestrator_model")
or self.orchestrator.orchestrator_model == "auto"
):
if not hasattr(self.orchestrator, "orchestrator_model") or self.orchestrator.orchestrator_model == "auto":
# Use streaming if available
from .streaming import stream_with_fallback
@@ -1102,9 +1063,7 @@ Examples:
# Response already displayed by streaming handler
return
else:
console.print(
"[red]No AI options available. Please configure API keys or install tools.[/red]"
)
console.print("[red]No AI options available. Please configure API keys or install tools.[/red]")
return
# For codex and other CLI tools, go straight to direct API chat
@@ -1345,26 +1304,14 @@ Examples:
# No API keys available
console.print("[red]No AI API keys configured![/red]")
console.print(
"[yellow]Try these options that don't need your API key:[/yellow]"
)
console.print("[yellow]Try these options that don't need your API key:[/yellow]")
console.print("\n[bold]CLI Tools (use existing tools):[/bold]")
console.print(
" • hanzo dev --orchestrator codex # OpenAI CLI (if installed)"
)
console.print(
" • hanzo dev --orchestrator claude # Claude Desktop (if installed)"
)
console.print(
" • hanzo dev --orchestrator gemini # Gemini CLI (if installed)"
)
console.print(
" • hanzo dev --orchestrator hanzo-ide # Hanzo IDE from ~/work/hanzo/ide"
)
console.print(" • hanzo dev --orchestrator codex # OpenAI CLI (if installed)")
console.print(" • hanzo dev --orchestrator claude # Claude Desktop (if installed)")
console.print(" • hanzo dev --orchestrator gemini # Gemini CLI (if installed)")
console.print(" • hanzo dev --orchestrator hanzo-ide # Hanzo IDE from ~/work/hanzo/ide")
console.print("\n[bold]Free APIs (rate limited):[/bold]")
console.print(
" • hanzo dev --orchestrator codestral # Free Mistral Codestral"
)
console.print(" • hanzo dev --orchestrator codestral # Free Mistral Codestral")
console.print(" • hanzo dev --orchestrator starcoder # Free StarCoder")
console.print("\n[bold]Local Models (unlimited):[/bold]")
console.print(" • hanzo dev --orchestrator local:llama3.2 # Via Ollama")
@@ -1407,16 +1354,10 @@ Examples:
if response.status_code == 200:
data = response.json()
if data.get("choices"):
console.print(
f"[cyan]Codestral:[/cyan] {data['choices'][0]['message']['content']}"
)
console.print(f"[cyan]Codestral:[/cyan] {data['choices'][0]['message']['content']}")
else:
console.print(
"[yellow]Free tier limit reached. Try local models instead:[/yellow]"
)
console.print(
" • Install Ollama: curl -fsSL https://ollama.com/install.sh | sh"
)
console.print("[yellow]Free tier limit reached. Try local models instead:[/yellow]")
console.print(" • Install Ollama: curl -fsSL https://ollama.com/install.sh | sh")
console.print(" • Run: ollama pull codellama")
console.print(" • Use: hanzo dev --orchestrator local:codellama")
@@ -1453,13 +1394,9 @@ Examples:
if response.status_code == 200:
data = response.json()
if isinstance(data, list) and data:
console.print(
f"[cyan]StarCoder:[/cyan] {data[0].get('generated_text', '')}"
)
console.print(f"[cyan]StarCoder:[/cyan] {data[0].get('generated_text', '')}")
else:
console.print(
"[yellow]API limit reached. Install local models:[/yellow]"
)
console.print("[yellow]API limit reached. Install local models:[/yellow]")
console.print(" • brew install ollama")
console.print(" • ollama pull starcoder2")
console.print(" • hanzo dev --orchestrator local:starcoder2")
@@ -1496,9 +1433,7 @@ Examples:
message,
]
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
stdout, stderr = process.communicate(timeout=30)
@@ -1530,11 +1465,7 @@ Examples:
claude_path = None
for path in claude_paths:
if (
os.path.exists(path)
or subprocess.run(["which", path], capture_output=True).returncode
== 0
):
if os.path.exists(path) or subprocess.run(["which", path], capture_output=True).returncode == 0:
claude_path = path
break
@@ -1563,9 +1494,7 @@ Examples:
"""
subprocess.run(["osascript", "-e", script])
console.print(
"[cyan]Sent to Claude Desktop. Check the app for response.[/cyan]"
)
console.print("[cyan]Sent to Claude Desktop. Check the app for response.[/cyan]")
else:
# Try direct CLI invocation
process = subprocess.Popen(
@@ -1604,9 +1533,7 @@ Examples:
# Use gemini CLI
cmd = ["gemini", "chat", message]
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
stdout, stderr = process.communicate(timeout=30)
@@ -1634,9 +1561,7 @@ Examples:
console.print("[red]Hanzo Dev IDE not found![/red]")
console.print("[yellow]Expected location: ~/work/hanzo/ide[/yellow]")
console.print("To set up:")
console.print(
" • git clone https://github.com/hanzoai/ide ~/work/hanzo/ide"
)
console.print(" • git clone https://github.com/hanzoai/ide ~/work/hanzo/ide")
console.print(" • cd ~/work/hanzo/ide && npm install")
return
@@ -1672,9 +1597,7 @@ Examples:
cmd = [cli_path, "chat", message]
cwd = None
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=cwd
)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=cwd)
stdout, stderr = process.communicate(timeout=30)
@@ -1721,9 +1644,7 @@ Examples:
if response.status_code == 200:
data = response.json()
if data.get("message"):
console.print(
f"[cyan]{model_name}:[/cyan] {data['message']['content']}"
)
console.print(f"[cyan]{model_name}:[/cyan] {data['message']['content']}")
return
except Exception:
@@ -1754,9 +1675,7 @@ Examples:
if response.status_code == 200:
data = response.json()
if data.get("choices"):
console.print(
f"[cyan]{model_name}:[/cyan] {data['choices'][0]['message']['content']}"
)
console.print(f"[cyan]{model_name}:[/cyan] {data['choices'][0]['message']['content']}")
return
except Exception:
@@ -1834,9 +1753,7 @@ async def run_dev_orchestrator(**kwargs):
# Check if we should use network mode
# For now, disable network mode since hanzo-network isn't available
if False and use_network and NETWORK_AVAILABLE:
console_obj.print(
f"[cyan]Mode: Network Orchestration with hanzo-network[/cyan]"
)
console_obj.print(f"[cyan]Mode: Network Orchestration with hanzo-network[/cyan]")
console_obj.print(f"Orchestrator: {orchestrator_model}")
console_obj.print(f"Workers: {instances} agents")
console_obj.print(f"Critics: {max(1, instances // 2)} agents")
@@ -1942,25 +1859,19 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
# Check if we can use hanzo-network
if not NETWORK_AVAILABLE:
self.console.print(
"[yellow]Warning: hanzo-network not available, falling back to basic mode[/yellow]"
)
self.console.print("[yellow]Warning: hanzo-network not available, falling back to basic mode[/yellow]")
async def initialize(self):
"""Initialize the agent network with orchestrator and workers."""
if not NETWORK_AVAILABLE:
self.console.print(
"[red]Cannot initialize network mode without hanzo-network[/red]"
)
self.console.print("[red]Cannot initialize network mode without hanzo-network[/red]")
return False
# Start hanzo net if requested for local orchestration
if self.use_hanzo_net or self.orchestrator_model.startswith("local:"):
await self._start_hanzo_net()
self.console.print(
f"[cyan]Initializing agent network with {self.orchestrator_model} orchestrator...[/cyan]"
)
self.console.print(f"[cyan]Initializing agent network with {self.orchestrator_model} orchestrator...[/cyan]")
# Create orchestrator agent (GPT-5, local, or other model)
self.orchestrator_agent = await self._create_orchestrator_agent()
@@ -1977,9 +1888,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
for i in range(num_local_workers):
local_worker = await self._create_local_worker_agent(i)
self.worker_agents.append(local_worker)
self.console.print(
f"[green]Added {num_local_workers} local workers for cost optimization[/green]"
)
self.console.print(f"[green]Added {num_local_workers} local workers for cost optimization[/green]")
# Create critic agents for System 2 thinking
if self.enable_guardrails:
@@ -2002,21 +1911,15 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
self.agent_network = create_network(
agents=all_agents,
router=router,
default_agent=(
self.orchestrator_agent.name if self.orchestrator_agent else None
),
default_agent=(self.orchestrator_agent.name if self.orchestrator_agent else None),
)
self.console.print(
f"[green]✓ Agent network initialized with {len(all_agents)} agents[/green]"
)
self.console.print(f"[green]✓ Agent network initialized with {len(all_agents)} agents[/green]")
return True
async def _start_hanzo_net(self):
"""Start hanzo net for local AI orchestration."""
self.console.print(
"[cyan]Starting hanzo/net for local AI orchestration...[/cyan]"
)
self.console.print("[cyan]Starting hanzo/net for local AI orchestration...[/cyan]")
# Check if hanzo net is already running
import socket
@@ -2026,9 +1929,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
sock.close()
if result == 0:
self.console.print(
f"[yellow]hanzo/net already running on port {self.hanzo_net_port}[/yellow]"
)
self.console.print(f"[yellow]hanzo/net already running on port {self.hanzo_net_port}[/yellow]")
return
# Start hanzo net
@@ -2089,9 +1990,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
tools=[],
)
self.console.print(
f"[green]✓ Created local {model_name} orchestrator via hanzo/net[/green]"
)
self.console.print(f"[green]✓ Created local {model_name} orchestrator via hanzo/net[/green]")
return orchestrator
# Parse model string to get provider and model
@@ -2147,9 +2046,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
tools=[], # Orchestrator tools will be added
)
self.console.print(
f"[green]✓ Created {self.orchestrator_model} orchestrator[/green]"
)
self.console.print(f"[green]✓ Created {self.orchestrator_model} orchestrator[/green]")
return orchestrator
def _get_orchestrator_system_prompt(self) -> str:
@@ -2282,10 +2179,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
"count",
"find",
]
if (
any(keyword in prompt_lower for keyword in simple_keywords)
and self.local_workers
):
if any(keyword in prompt_lower for keyword in simple_keywords) and self.local_workers:
return self.local_workers[0].name
# Complex implementation → API workers (Claude)
@@ -2297,10 +2191,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
"design",
"architect",
]
if (
any(keyword in prompt_lower for keyword in complex_keywords)
and self.api_workers
):
if any(keyword in prompt_lower for keyword in complex_keywords) and self.api_workers:
return self.api_workers[0].name
# Review tasks → Critics
@@ -2311,10 +2202,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
"improve",
"validate code",
]
if (
any(keyword in prompt_lower for keyword in review_keywords)
and self.critics
):
if any(keyword in prompt_lower for keyword in review_keywords) and self.critics:
return self.critics[0].name
# Strategic decisions → Orchestrator
@@ -2335,20 +2223,12 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
return self.local_workers[0].name
# Fall back to API workers for complex tasks
return (
self.api_workers[0].name
if self.api_workers
else self.orchestrator.name
)
return self.api_workers[0].name if self.api_workers else self.orchestrator.name
# Create the cost-optimized router
router = CostOptimizedRouter(
self.orchestrator_agent, self.worker_agents, self.critic_agents
)
router = CostOptimizedRouter(self.orchestrator_agent, self.worker_agents, self.critic_agents)
self.console.print(
"[green]✓ Created cost-optimized router (local models preferred)[/green]"
)
self.console.print("[green]✓ Created cost-optimized router (local models preferred)[/green]")
return router
async def _create_intelligent_router(self) -> Router:
@@ -2383,9 +2263,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
return router
async def execute_with_network(
self, task: str, context: Optional[Dict] = None
) -> Dict:
async def execute_with_network(self, task: str, context: Optional[Dict] = None) -> Dict:
"""Execute a task using the agent network.
Args:
@@ -2523,9 +2401,7 @@ class MultiClaudeOrchestrator(HanzoDevOrchestrator):
config = await self._create_instance_config(i, role)
self.instance_configs.append(config)
self.console.print(
f" [{i + 1}/{self.num_instances}] {role} instance configured"
)
self.console.print(f" [{i + 1}/{self.num_instances}] {role} instance configured")
# If networking enabled, configure MCP connections between instances
if self.enable_networking:
@@ -2587,9 +2463,7 @@ class MultiClaudeOrchestrator(HanzoDevOrchestrator):
async def _setup_mcp_networking(self):
"""Set up MCP networking between Claude instances."""
self.console.print(
"[cyan]Setting up MCP networking between instances...[/cyan]"
)
self.console.print("[cyan]Setting up MCP networking between instances...[/cyan]")
# Each instance gets MCP servers for all other instances
for i, config in enumerate(self.instance_configs):
@@ -2732,14 +2606,10 @@ class MultiClaudeOrchestrator(HanzoDevOrchestrator):
# Validate improvement didn't degrade quality
if await self._validate_improvement(result, improved):
self.console.print(
"[green]✓ Solution improved with System 2 feedback[/green]"
)
self.console.print("[green]✓ Solution improved with System 2 feedback[/green]")
return improved
else:
self.console.print(
"[yellow]⚠ Keeping original solution (improvement validation failed)[/yellow]"
)
self.console.print("[yellow]⚠ Keeping original solution (improvement validation failed)[/yellow]")
return result
@@ -2816,9 +2686,7 @@ class MultiClaudeOrchestrator(HanzoDevOrchestrator):
try:
import subprocess
result = subprocess.run(
["gemini", "chat", prompt], capture_output=True, text=True, timeout=30
)
result = subprocess.run(["gemini", "chat", prompt], capture_output=True, text=True, timeout=30)
if result.returncode == 0 and result.stdout:
return {"output": result.stdout.strip(), "success": True}
except Exception as e:
@@ -2882,9 +2750,7 @@ class MultiClaudeOrchestrator(HanzoDevOrchestrator):
logger.error(f"OpenAI API error: {e}")
# Try Anthropic
anthropic_key = os.environ.get("ANTHROPIC_API_KEY") or os.getenv(
"ANTHROPIC_API_KEY"
)
anthropic_key = os.environ.get("ANTHROPIC_API_KEY") or os.getenv("ANTHROPIC_API_KEY")
if anthropic_key:
try:
from anthropic import AsyncAnthropic
@@ -2903,9 +2769,7 @@ class MultiClaudeOrchestrator(HanzoDevOrchestrator):
# Try fallback handler as last resort
from .fallback_handler import smart_chat
response = await smart_chat(
prompt, console=None
) # No console to avoid duplicate messages
response = await smart_chat(prompt, console=None) # No console to avoid duplicate messages
if response:
return {"output": response, "success": True}
@@ -2948,25 +2812,15 @@ async def main():
"""Main entry point for hanzo-dev."""
import argparse
parser = argparse.ArgumentParser(
description="Hanzo Dev - System 2 Meta-AI Orchestrator"
)
parser.add_argument(
"--workspace", default="~/.hanzo/dev", help="Workspace directory"
)
parser = argparse.ArgumentParser(description="Hanzo Dev - System 2 Meta-AI Orchestrator")
parser.add_argument("--workspace", default="~/.hanzo/dev", help="Workspace directory")
parser.add_argument("--claude-path", help="Path to Claude Code executable")
parser.add_argument("--monitor", action="store_true", help="Start in monitor mode")
parser.add_argument("--repl", action="store_true", help="Start REPL interface")
parser.add_argument(
"--instances", type=int, default=2, help="Number of Claude instances"
)
parser.add_argument("--instances", type=int, default=2, help="Number of Claude instances")
parser.add_argument("--no-mcp", action="store_true", help="Disable MCP tools")
parser.add_argument(
"--no-network", action="store_true", help="Disable instance networking"
)
parser.add_argument(
"--no-guardrails", action="store_true", help="Disable guardrails"
)
parser.add_argument("--no-network", action="store_true", help="Disable instance networking")
parser.add_argument("--no-guardrails", action="store_true", help="Disable guardrails")
args = parser.parse_args()
+7 -21
View File
@@ -23,9 +23,7 @@ class FallbackHandler:
"deepseek_api": bool(os.getenv("DEEPSEEK_API_KEY")), # Added DeepSeek
"openai_api": bool(os.getenv("OPENAI_API_KEY")),
"anthropic_api": bool(os.getenv("ANTHROPIC_API_KEY")),
"google_api": bool(
os.getenv("GOOGLE_API_KEY") or os.getenv("GEMINI_API_KEY")
),
"google_api": bool(os.getenv("GOOGLE_API_KEY") or os.getenv("GEMINI_API_KEY")),
"openai_cli": shutil.which("openai") is not None,
"claude_cli": shutil.which("claude") is not None,
"gemini_cli": shutil.which("gemini") is not None,
@@ -103,9 +101,7 @@ class FallbackHandler:
suggestions = []
if not self.available_options["deepseek_api"]:
suggestions.append(
"• Set DEEPSEEK_API_KEY for cost-effective DeepSeek access ($0.14/M tokens)"
)
suggestions.append("• Set DEEPSEEK_API_KEY for cost-effective DeepSeek access ($0.14/M tokens)")
if not self.available_options["openai_api"]:
suggestions.append("• Set OPENAI_API_KEY for GPT-4/GPT-5 access")
@@ -114,30 +110,22 @@ class FallbackHandler:
suggestions.append("• Set ANTHROPIC_API_KEY for Claude access")
if not self.available_options["ollama"]:
suggestions.append(
"• Install Ollama: curl -fsSL https://ollama.com/install.sh | sh"
)
suggestions.append("• Install Ollama: curl -fsSL https://ollama.com/install.sh | sh")
suggestions.append(" Then run: ollama pull llama3.2")
if not self.available_options["openai_cli"]:
suggestions.append("• Install OpenAI CLI: pip install openai-cli")
if not self.available_options["claude_cli"]:
suggestions.append(
"• Install Claude Desktop from https://claude.ai/download"
)
suggestions.append("• Install Claude Desktop from https://claude.ai/download")
return (
"\n".join(suggestions) if suggestions else "All AI options are available!"
)
return "\n".join(suggestions) if suggestions else "All AI options are available!"
def print_status(self, console):
"""Print the current status of available AI options."""
from rich.table import Table
table = Table(
title="Available AI Options", show_header=True, header_style="bold magenta"
)
table = Table(title="Available AI Options", show_header=True, header_style="bold magenta")
table.add_column("Option", style="cyan", width=20)
table.add_column("Status", width=10)
table.add_column("Model", width=20)
@@ -164,9 +152,7 @@ class FallbackHandler:
console.print(table)
if self.fallback_order:
console.print(
f"\n[green]Primary option: {self.fallback_order[0][1]}[/green]"
)
console.print(f"\n[green]Primary option: {self.fallback_order[0][1]}[/green]")
if len(self.fallback_order) > 1:
fallbacks = ", ".join([opt[1] for opt in self.fallback_order[1:]])
console.print(f"[yellow]Fallback options: {fallbacks}[/yellow]")
+34 -112
View File
@@ -98,9 +98,7 @@ class EnhancedHanzoREPL:
self.failed_tools = set() # Track tools that have failed this session
# Initialize background task manager
self.task_manager = (
BackgroundTaskManager(console) if BackgroundTaskManager else None
)
self.task_manager = BackgroundTaskManager(console) if BackgroundTaskManager else None
# Initialize todo manager
self.todo_manager = TodoManager(console) if TodoManager else None
@@ -114,9 +112,7 @@ class EnhancedHanzoREPL:
if default_tool:
self.current_model = f"tool:{default_tool.name}"
self.current_tool = default_tool
self.console.print(
f"[green]✓ Detected {default_tool.display_name} as default AI assistant[/green]"
)
self.console.print(f"[green]✓ Detected {default_tool.display_name} as default AI assistant[/green]")
else:
# Fallback to regular models
self.current_model = self.config.get("default_model", "gpt-3.5-turbo")
@@ -355,9 +351,7 @@ class EnhancedHanzoREPL:
# Router status
try:
response = httpx.get("http://localhost:4000/health", timeout=1)
router_status = (
"✅ Running" if response.status_code == 200 else "⚠️ Unhealthy"
)
router_status = "✅ Running" if response.status_code == 200 else "⚠️ Unhealthy"
router_details = "Port 4000"
except Exception:
router_status = "❌ Offline"
@@ -410,11 +404,7 @@ class EnhancedHanzoREPL:
args = f"tool:{tool.name}"
else:
# It's a model selection
model_idx = (
num - len(self.detected_tools) - 1
if self.detected_tools
else num - 1
)
model_idx = num - len(self.detected_tools) - 1 if self.detected_tools else num - 1
models_list = list(self.MODELS.keys())
if 0 <= model_idx < len(models_list):
args = models_list[model_idx]
@@ -427,11 +417,7 @@ class EnhancedHanzoREPL:
return
# Check if it's a tool
if (
args.startswith("tool:") or args in [t.name for t in self.detected_tools]
if self.detected_tools
else False
):
if args.startswith("tool:") or args in [t.name for t in self.detected_tools] if self.detected_tools else False:
# Handle tool selection
tool_name = args.replace("tool:", "") if args.startswith("tool:") else args
@@ -465,9 +451,7 @@ class EnhancedHanzoREPL:
# Show hint if it's a gateway model (not in our predefined list)
if args not in self.MODELS and not args.startswith("local:"):
self.console.print(
"[dim]Using gateway model - all models free! Use /models to see full list.[/dim]"
)
self.console.print("[dim]Using gateway model - all models free! Use /models to see full list.[/dim]")
async def list_tools(self, args: str = ""):
"""List available AI tools."""
@@ -480,14 +464,10 @@ class EnhancedHanzoREPL:
"""List available models from gateway.hanzo.ai."""
# Show tools first if available
if self.detected_tools:
self.console.print(
"[bold cyan]AI Coding Assistants (Detected):[/bold cyan]"
)
self.console.print("[bold cyan]AI Coding Assistants (Detected):[/bold cyan]")
for i, tool in enumerate(self.detected_tools, 1):
marker = "→" if self.current_model == f"tool:{tool.name}" else " "
self.console.print(
f" {marker} {i}. {tool.display_name} ({tool.provider})"
)
self.console.print(f" {marker} {i}. {tool.display_name} ({tool.provider})")
self.console.print()
# Try to fetch live models from gateway
@@ -507,20 +487,14 @@ class EnhancedHanzoREPL:
models_list = [
model
for model in all_models
if not any(
keyword in model.get("id", "").lower()
for keyword in embedding_keywords
)
if not any(keyword in model.get("id", "").lower() for keyword in embedding_keywords)
]
except Exception:
pass
# Fallback to static models if gateway fetch fails
if not models_list:
models_list = [
{"id": model_id, "name": model_name}
for model_id, model_name in self.MODELS.items()
]
models_list = [{"id": model_id, "name": model_name} for model_id, model_name in self.MODELS.items()]
# Create table
table = Table(title="Gateway LLMs (Chat Models)", box=box.ROUNDED)
@@ -559,22 +533,16 @@ class EnhancedHanzoREPL:
# Highlight current model
if model_id == self.current_model:
table.add_row(
str(i), f"[bold green]→ {model_id}[/bold green]", tier, provider
)
table.add_row(str(i), f"[bold green]→ {model_id}[/bold green]", tier, provider)
else:
table.add_row(str(i), model_id, tier, provider)
self.console.print(table)
self.console.print(
"\n[dim cyan]Free tier:[/dim cyan] [green]Most models available without login![/green]"
)
self.console.print("\n[dim cyan]Free tier:[/dim cyan] [green]Most models available without login![/green]")
self.console.print(
"[dim cyan]Premium models:[/dim cyan] [yellow]gpt-4, claude-3-opus, o1-preview[/yellow] - [cyan]hanzo auth login[/cyan]"
)
self.console.print(
"\n[dim]Use /model <name> or /model <number> to switch[/dim]"
)
self.console.print("\n[dim]Use /model <name> or /model <number> to switch[/dim]")
async def login(self, args: str = ""):
"""Login to Hanzo."""
@@ -697,9 +665,7 @@ class EnhancedHanzoREPL:
self.console.print(f"[dim]Executing: {full_cmd}[/dim]")
try:
result = subprocess.run(
full_cmd, shell=True, capture_output=True, text=True
)
result = subprocess.run(full_cmd, shell=True, capture_output=True, text=True)
if result.stdout:
self.console.print(result.stdout)
@@ -720,19 +686,13 @@ class EnhancedHanzoREPL:
if tool.name not in self.failed_tools:
self.current_tool = tool
self.current_model = f"tool:{tool.name}"
self.console.print(
f"[yellow]Switched to {tool.display_name}[/yellow]"
)
self.console.print(f"[yellow]Switched to {tool.display_name}[/yellow]")
break
# Try the current tool
if self.current_tool.name not in self.failed_tools:
self.console.print(
f"[dim]Using {self.current_tool.display_name}...[/dim]"
)
success, output = self.tool_detector.execute_with_tool(
self.current_tool, message
)
self.console.print(f"[dim]Using {self.current_tool.display_name}...[/dim]")
success, output = self.tool_detector.execute_with_tool(self.current_tool, message)
if success:
self.console.print(output)
@@ -747,12 +707,8 @@ class EnhancedHanzoREPL:
if self.tool_detector and self.tool_detector.detected_tools:
for fallback_tool in self.tool_detector.detected_tools:
if fallback_tool.name not in self.failed_tools:
self.console.print(
f"[yellow]Trying {fallback_tool.display_name}...[/yellow]"
)
success, output = self.tool_detector.execute_with_tool(
fallback_tool, message
)
self.console.print(f"[yellow]Trying {fallback_tool.display_name}...[/yellow]")
success, output = self.tool_detector.execute_with_tool(fallback_tool, message)
if success:
self.console.print(output)
# Automatically switch to this working tool
@@ -768,21 +724,15 @@ class EnhancedHanzoREPL:
else:
# Mark as failed
self.failed_tools.add(fallback_tool.name)
self.console.print(
f"[red]{fallback_tool.display_name} also failed[/red]"
)
self.console.print(f"[red]{fallback_tool.display_name} also failed[/red]")
if not found_working:
# Final fallback to cloud model
self.console.print(f"[yellow]Falling back to cloud model...[/yellow]")
await self.execute_command(
"ask", f"--cloud --model gpt-3.5-turbo {message}"
)
await self.execute_command("ask", f"--cloud --model gpt-3.5-turbo {message}")
else:
# Use regular model through hanzo ask
await self.execute_command(
"ask", f"--cloud --model {self.current_model} {message}"
)
await self.execute_command("ask", f"--cloud --model {self.current_model} {message}")
async def quick_model_select(self, args: str = ""):
"""Quick model selector with arrow keys."""
@@ -791,11 +741,7 @@ class EnhancedHanzoREPL:
return
# Prepare tools and models
tools = (
[(f"tool:{t.name}", t.display_name) for t in self.detected_tools]
if self.detected_tools
else []
)
tools = [(f"tool:{t.name}", t.display_name) for t in self.detected_tools] if self.detected_tools else []
models = list(self.MODELS.items())
selector = QuickModelSelector(models, tools, self.current_model)
@@ -826,9 +772,7 @@ class EnhancedHanzoREPL:
else:
# Show tasks and prompt for selection
self.task_manager.list_tasks()
self.console.print(
"\n[cyan]Enter task ID to kill (or 'all' for all tasks):[/cyan]"
)
self.console.print("\n[cyan]Enter task ID to kill (or 'all' for all tasks):[/cyan]")
try:
task_id = await self.session.prompt_async("> ")
if task_id:
@@ -863,9 +807,7 @@ class EnhancedHanzoREPL:
# Quick add
try:
todo = self.todo_manager.quick_add(rest)
self.console.print(
f"[green]✅ Added todo: {todo.title} (ID: {todo.id})[/green]"
)
self.console.print(f"[green]✅ Added todo: {todo.title} (ID: {todo.id})[/green]")
except ValueError as e:
self.console.print(f"[red]Error: {e}[/red]")
else:
@@ -891,9 +833,7 @@ class EnhancedHanzoREPL:
elif key in ["tag", "t"]:
tag = value
todos = self.todo_manager.list_todos(
status=status, priority=priority, tag=tag
)
todos = self.todo_manager.list_todos(status=status, priority=priority, tag=tag)
title = "Filtered Todos" if (status or priority or tag) else "All Todos"
self.todo_manager.display_todos(todos, title)
@@ -902,9 +842,7 @@ class EnhancedHanzoREPL:
if rest:
todo = self.todo_manager.update_todo(rest, status="done")
if todo:
self.console.print(
f"[green]✅ Marked as done: {todo.title}[/green]"
)
self.console.print(f"[green]✅ Marked as done: {todo.title}[/green]")
else:
self.console.print(f"[red]Todo not found: {rest}[/red]")
else:
@@ -960,9 +898,7 @@ class EnhancedHanzoREPL:
elif subcommand in ["clear", "reset"]:
# Clear all todos (with confirmation)
try:
confirm = await self.session.prompt_async(
"Are you sure you want to delete ALL todos? (yes/no): "
)
confirm = await self.session.prompt_async("Are you sure you want to delete ALL todos? (yes/no): ")
if confirm.lower() in ["yes", "y"]:
self.todo_manager.todos = []
self.todo_manager.save_todos()
@@ -980,13 +916,9 @@ class EnhancedHanzoREPL:
# Unknown subcommand, treat as quick add
try:
todo = self.todo_manager.quick_add(args)
self.console.print(
f"[green]✅ Added todo: {todo.title} (ID: {todo.id})[/green]"
)
self.console.print(f"[green]✅ Added todo: {todo.title} (ID: {todo.id})[/green]")
except ValueError:
self.console.print(
f"[yellow]Unknown todo command: {subcommand}[/yellow]"
)
self.console.print(f"[yellow]Unknown todo command: {subcommand}[/yellow]")
self.console.print("[dim]Use /todo help for available commands[/dim]")
async def add_todo_interactive(self):
@@ -1002,21 +934,13 @@ class EnhancedHanzoREPL:
description = await self.session.prompt_async("Description (optional): ")
# Get priority
priority = await self.session.prompt_async(
"Priority (low/medium/high/urgent) [medium]: "
)
priority = await self.session.prompt_async("Priority (low/medium/high/urgent) [medium]: ")
if not priority:
priority = "medium"
# Get tags
tags_input = await self.session.prompt_async(
"Tags (comma-separated, optional): "
)
tags = (
[t.strip() for t in tags_input.split(",") if t.strip()]
if tags_input
else []
)
tags_input = await self.session.prompt_async("Tags (comma-separated, optional): ")
tags = [t.strip() for t in tags_input.split(",") if t.strip()] if tags_input else []
# Get due date
due_date = await self.session.prompt_async("Due date (optional): ")
@@ -1030,9 +954,7 @@ class EnhancedHanzoREPL:
due_date=due_date if due_date else None,
)
self.console.print(
f"[green]✅ Added todo: {todo.title} (ID: {todo.id})[/green]"
)
self.console.print(f"[green]✅ Added todo: {todo.title} (ID: {todo.id})[/green]")
except (KeyboardInterrupt, EOFError):
self.console.print("[yellow]Cancelled[/yellow]")
@@ -14,9 +14,7 @@ from prompt_toolkit.layout.containers import HSplit, Window
class QuickModelSelector:
"""Quick model selector with arrow navigation."""
def __init__(
self, models: List[Tuple[str, str]], tools: List[Tuple[str, str]], current: str
):
def __init__(self, models: List[Tuple[str, str]], tools: List[Tuple[str, str]], current: str):
self.models = models
self.tools = tools
self.current = current
@@ -99,9 +97,7 @@ class QuickModelSelector:
layout = Layout(Window(FormattedTextControl(get_text), wrap_lines=False))
app = Application(
layout=layout, key_bindings=kb, full_screen=False, mouse_support=True
)
app = Application(layout=layout, key_bindings=kb, full_screen=False, mouse_support=True)
return await app.run_async()
@@ -138,9 +134,7 @@ class BackgroundTaskManager:
task = self.tasks[task_id]
if task["process"].poll() is None:
task["process"].terminate()
self.console.print(
f"[yellow]Terminated {task_id}: {task['name']}[/yellow]"
)
self.console.print(f"[yellow]Terminated {task_id}: {task['name']}[/yellow]")
else:
self.console.print(f"[dim]Task {task_id} already stopped[/dim]")
del self.tasks[task_id]
+1 -3
View File
@@ -196,9 +196,7 @@ hanzo> mcp run read_file --arg path=README.md
except subprocess.TimeoutExpired:
self.console.print("[red]Command timed out[/red]")
except FileNotFoundError:
self.console.print(
"[red]Command not found. Make sure 'hanzo' is installed.[/red]"
)
self.console.print("[red]Command not found. Make sure 'hanzo' is installed.[/red]")
except Exception as e:
self.console.print(f"[red]Command error: {e}[/red]")

Some files were not shown because too many files have changed in this diff Show More