mirror of
https://github.com/hanzoai/python-sdk.git
synced 2026-10-23 20:23:00 +00:00
fix: run ruff check --fix and format across entire codebase
This commit is contained in:
@@ -108,9 +108,7 @@ def build_package(package_dir: Path) -> bool:
|
||||
subprocess.run(["rm", "-rf", str(egg_info)], check=True)
|
||||
|
||||
# Build the package
|
||||
result = subprocess.run(
|
||||
["python", "-m", "build"], cwd=package_dir, capture_output=True, text=True
|
||||
)
|
||||
result = subprocess.run(["python", "-m", "build"], cwd=package_dir, capture_output=True, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
print_error(f"Build failed: {result.stderr}")
|
||||
@@ -226,9 +224,7 @@ def main():
|
||||
capture_output=True,
|
||||
)
|
||||
except Exception:
|
||||
print_warn(
|
||||
"Could not install build tools. Make sure pip, build, and twine are available."
|
||||
)
|
||||
print_warn("Could not install build tools. Make sure pip, build, and twine are available.")
|
||||
print_info("You can install them with: pip install build twine")
|
||||
|
||||
# Track results
|
||||
|
||||
@@ -12,12 +12,13 @@ from typing import Any, Dict, List
|
||||
# Example of how to use hanzo-mcp batch tool for parallel edits
|
||||
# In practice, this would be called through the MCP interface
|
||||
|
||||
|
||||
def create_parallel_edit_batch():
|
||||
"""
|
||||
Create batch invocations for parallel document editing.
|
||||
Each agent works on a different file simultaneously.
|
||||
"""
|
||||
|
||||
|
||||
batch_invocations = [
|
||||
{
|
||||
"tool_name": "claude",
|
||||
@@ -33,7 +34,7 @@ def create_parallel_edit_batch():
|
||||
Return the complete enhanced content maintaining all existing sections.
|
||||
Focus on Claude-specific strengths and unique features.
|
||||
"""
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"tool_name": "codex",
|
||||
@@ -49,7 +50,7 @@ def create_parallel_edit_batch():
|
||||
Return the complete enhanced content maintaining all existing sections.
|
||||
Focus on OpenAI-specific optimizations and features.
|
||||
"""
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"tool_name": "gemini",
|
||||
@@ -65,7 +66,7 @@ def create_parallel_edit_batch():
|
||||
Return the complete enhanced content maintaining all existing sections.
|
||||
Focus on Google-specific integrations and multimodal strengths.
|
||||
"""
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"tool_name": "grok",
|
||||
@@ -83,18 +84,19 @@ def create_parallel_edit_batch():
|
||||
Follow the same structure as other AI doc files.
|
||||
Include code examples and practical patterns.
|
||||
"""
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
return batch_invocations
|
||||
|
||||
|
||||
def create_agent_editing_batch():
|
||||
"""
|
||||
Create batch for editing AGENTS.md using multiple agents.
|
||||
Each agent enhances a different section.
|
||||
"""
|
||||
|
||||
|
||||
batch_invocations = [
|
||||
{
|
||||
"tool_name": "claude",
|
||||
@@ -107,7 +109,7 @@ def create_agent_editing_batch():
|
||||
- Include agent negotiation strategies
|
||||
Write only the enhanced section content.
|
||||
"""
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"tool_name": "codex",
|
||||
@@ -120,7 +122,7 @@ def create_agent_editing_batch():
|
||||
- Include automated PR generation
|
||||
Write only the enhanced section content.
|
||||
"""
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"tool_name": "gemini",
|
||||
@@ -133,25 +135,26 @@ def create_agent_editing_batch():
|
||||
- Include performance optimization
|
||||
Write only the enhanced section content.
|
||||
"""
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
return batch_invocations
|
||||
|
||||
|
||||
def create_review_and_merge_batch(edited_content: Dict[str, str]):
|
||||
"""
|
||||
Create batch for reviewing and merging edits using consensus.
|
||||
Multiple agents review each other's work.
|
||||
"""
|
||||
|
||||
|
||||
batch_invocations = [
|
||||
{
|
||||
"tool_name": "claude",
|
||||
"input": {
|
||||
"prompt": f"""
|
||||
Review the Codex edits to LLM.md:
|
||||
{edited_content.get('llm_edits', '')}
|
||||
{edited_content.get("llm_edits", "")}
|
||||
|
||||
Check for:
|
||||
1. Technical accuracy
|
||||
@@ -161,14 +164,14 @@ def create_review_and_merge_batch(edited_content: Dict[str, str]):
|
||||
|
||||
Provide feedback and improved version if needed.
|
||||
"""
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"tool_name": "codex",
|
||||
"input": {
|
||||
"prompt": f"""
|
||||
Review the Claude edits to CLAUDE.md:
|
||||
{edited_content.get('claude_edits', '')}
|
||||
{edited_content.get("claude_edits", "")}
|
||||
|
||||
Check for:
|
||||
1. Implementation feasibility
|
||||
@@ -178,7 +181,7 @@ def create_review_and_merge_batch(edited_content: Dict[str, str]):
|
||||
|
||||
Provide feedback and improved version if needed.
|
||||
"""
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"tool_name": "gemini",
|
||||
@@ -198,104 +201,96 @@ def create_review_and_merge_batch(edited_content: Dict[str, str]):
|
||||
3. Integration test cases
|
||||
4. Documentation validation script
|
||||
"""
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
return batch_invocations
|
||||
|
||||
|
||||
class ParallelDocumentEditor:
|
||||
"""
|
||||
Orchestrates parallel editing of documentation using multiple AI agents.
|
||||
"""
|
||||
|
||||
|
||||
def __init__(self):
|
||||
self.agents = ["claude", "codex", "gemini", "grok"]
|
||||
self.files = ["LLM.md", "AGENTS.md", "GEMINI.md", "CLAUDE.md"]
|
||||
|
||||
|
||||
async def execute_parallel_edits(self):
|
||||
"""
|
||||
Execute parallel edits using batch tool.
|
||||
"""
|
||||
|
||||
|
||||
# Phase 1: Initial parallel edits
|
||||
print("🚀 Phase 1: Initiating parallel edits...")
|
||||
initial_batch = create_parallel_edit_batch()
|
||||
|
||||
|
||||
# This would be executed as:
|
||||
# results = await batch(
|
||||
# description="Edit AI documentation files",
|
||||
# invocations=initial_batch
|
||||
# )
|
||||
|
||||
|
||||
# Simulated results for demonstration
|
||||
results_phase1 = {
|
||||
"claude": "Enhanced CLAUDE.md content...",
|
||||
"codex": "Enhanced LLM.md content...",
|
||||
"gemini": "Enhanced GEMINI.md content...",
|
||||
"grok": "New GROK.md content..."
|
||||
"grok": "New GROK.md content...",
|
||||
}
|
||||
|
||||
|
||||
print("✅ Phase 1 complete: All files edited in parallel")
|
||||
|
||||
|
||||
# Phase 2: Parallel section enhancements
|
||||
print("\n🚀 Phase 2: Enhancing specific sections...")
|
||||
section_batch = create_agent_editing_batch()
|
||||
|
||||
|
||||
# results = await batch(
|
||||
# description="Enhance AGENTS.md sections",
|
||||
# invocations=section_batch
|
||||
# )
|
||||
|
||||
|
||||
results_phase2 = {
|
||||
"communication": "Enhanced communication section...",
|
||||
"worktree": "Enhanced worktree section...",
|
||||
"swarm": "Enhanced swarm section..."
|
||||
"swarm": "Enhanced swarm section...",
|
||||
}
|
||||
|
||||
|
||||
print("✅ Phase 2 complete: Sections enhanced")
|
||||
|
||||
|
||||
# Phase 3: Cross-review and validation
|
||||
print("\n🚀 Phase 3: Cross-reviewing edits...")
|
||||
review_batch = create_review_and_merge_batch(results_phase1)
|
||||
|
||||
|
||||
# results = await batch(
|
||||
# description="Review and validate edits",
|
||||
# invocations=review_batch
|
||||
# )
|
||||
|
||||
|
||||
print("✅ Phase 3 complete: All edits reviewed and validated")
|
||||
|
||||
return {
|
||||
"phase1": results_phase1,
|
||||
"phase2": results_phase2,
|
||||
"status": "completed"
|
||||
}
|
||||
|
||||
|
||||
return {"phase1": results_phase1, "phase2": results_phase2, "status": "completed"}
|
||||
|
||||
async def apply_edits_in_parallel(self, edits: Dict[str, str]):
|
||||
"""
|
||||
Apply validated edits to files in parallel.
|
||||
"""
|
||||
|
||||
|
||||
write_batch = [
|
||||
{
|
||||
"tool_name": "write",
|
||||
"input": {
|
||||
"file_path": f"/Users/z/work/hanzo/python-sdk/{file}",
|
||||
"content": content
|
||||
}
|
||||
}
|
||||
{"tool_name": "write", "input": {"file_path": f"/Users/z/work/hanzo/python-sdk/{file}", "content": content}}
|
||||
for file, content in edits.items()
|
||||
]
|
||||
|
||||
|
||||
# Execute all writes in parallel
|
||||
# results = await batch(
|
||||
# description="Write updated documentation",
|
||||
# invocations=write_batch
|
||||
# )
|
||||
|
||||
|
||||
print("✅ All files updated in parallel")
|
||||
|
||||
|
||||
# Practical example of batch execution in MCP context
|
||||
BATCH_EXAMPLE = """
|
||||
# In Claude Desktop or hanzo-mcp CLI, you would execute:
|
||||
@@ -328,116 +323,102 @@ batch --description "Edit AI docs in parallel" --invocations '[
|
||||
]'
|
||||
"""
|
||||
|
||||
|
||||
def demonstrate_batch_patterns():
|
||||
"""
|
||||
Demonstrate various batch execution patterns.
|
||||
"""
|
||||
|
||||
|
||||
print("=" * 60)
|
||||
print("PARALLEL AI DOCUMENTATION EDITING PATTERNS")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
# Pattern 1: Parallel file editing
|
||||
print("\n📝 Pattern 1: Parallel File Editing")
|
||||
print("-" * 40)
|
||||
|
||||
|
||||
parallel_edit = {
|
||||
"description": "Edit 4 files simultaneously",
|
||||
"invocations": [
|
||||
{"tool": "claude", "target": "CLAUDE.md"},
|
||||
{"tool": "codex", "target": "LLM.md"},
|
||||
{"tool": "gemini", "target": "GEMINI.md"},
|
||||
{"tool": "grok", "target": "AGENTS.md"}
|
||||
]
|
||||
{"tool": "grok", "target": "AGENTS.md"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
print(json.dumps(parallel_edit, indent=2))
|
||||
|
||||
|
||||
# Pattern 2: Sequential with parallel stages
|
||||
print("\n📝 Pattern 2: Sequential Stages with Parallel Tasks")
|
||||
print("-" * 40)
|
||||
|
||||
|
||||
staged_execution = {
|
||||
"stage1": {
|
||||
"description": "Analyze all files in parallel",
|
||||
"parallel": True,
|
||||
"tasks": ["analyze_claude.md", "analyze_llm.md", "analyze_agents.md"]
|
||||
"tasks": ["analyze_claude.md", "analyze_llm.md", "analyze_agents.md"],
|
||||
},
|
||||
"stage2": {
|
||||
"description": "Edit based on analysis",
|
||||
"parallel": True,
|
||||
"tasks": ["edit_claude.md", "edit_llm.md", "edit_agents.md"]
|
||||
"tasks": ["edit_claude.md", "edit_llm.md", "edit_agents.md"],
|
||||
},
|
||||
"stage3": {
|
||||
"description": "Review all edits",
|
||||
"parallel": False,
|
||||
"tasks": ["consensus_review"]
|
||||
}
|
||||
"stage3": {"description": "Review all edits", "parallel": False, "tasks": ["consensus_review"]},
|
||||
}
|
||||
|
||||
|
||||
print(json.dumps(staged_execution, indent=2))
|
||||
|
||||
|
||||
# Pattern 3: Divide and conquer
|
||||
print("\n📝 Pattern 3: Divide and Conquer")
|
||||
print("-" * 40)
|
||||
|
||||
|
||||
divide_conquer = {
|
||||
"description": "Each agent handles specific sections",
|
||||
"file": "AGENTS.md",
|
||||
"parallel_sections": [
|
||||
{
|
||||
"agent": "claude",
|
||||
"section": "Communication Protocols",
|
||||
"expertise": "System design"
|
||||
},
|
||||
{
|
||||
"agent": "codex",
|
||||
"section": "Code Generation",
|
||||
"expertise": "Implementation"
|
||||
},
|
||||
{
|
||||
"agent": "gemini",
|
||||
"section": "Testing Strategies",
|
||||
"expertise": "Quality assurance"
|
||||
}
|
||||
]
|
||||
{"agent": "claude", "section": "Communication Protocols", "expertise": "System design"},
|
||||
{"agent": "codex", "section": "Code Generation", "expertise": "Implementation"},
|
||||
{"agent": "gemini", "section": "Testing Strategies", "expertise": "Quality assurance"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
print(json.dumps(divide_conquer, indent=2))
|
||||
|
||||
|
||||
# Pattern 4: Consensus editing
|
||||
print("\n📝 Pattern 4: Consensus-Based Editing")
|
||||
print("-" * 40)
|
||||
|
||||
|
||||
consensus_edit = {
|
||||
"description": "Multiple agents edit same content",
|
||||
"target": "architecture.md",
|
||||
"agents": ["claude", "codex", "gemini"],
|
||||
"strategy": "Each agent provides version, then consensus",
|
||||
"final_review": "grok"
|
||||
"final_review": "grok",
|
||||
}
|
||||
|
||||
|
||||
print(json.dumps(consensus_edit, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import asyncio
|
||||
|
||||
|
||||
# Demonstrate patterns
|
||||
demonstrate_batch_patterns()
|
||||
|
||||
|
||||
# Example execution (would be async in practice)
|
||||
print("\n" + "=" * 60)
|
||||
print("EXAMPLE PARALLEL EXECUTION")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
editor = ParallelDocumentEditor()
|
||||
|
||||
|
||||
# In practice, this would be:
|
||||
# asyncio.run(editor.execute_parallel_edits())
|
||||
|
||||
|
||||
print("\n✨ Parallel editing demonstration complete!")
|
||||
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("BATCH TOOL USAGE IN MCP")
|
||||
print("=" * 60)
|
||||
print(BATCH_EXAMPLE)
|
||||
print(BATCH_EXAMPLE)
|
||||
|
||||
@@ -34,9 +34,7 @@ async def main():
|
||||
)
|
||||
|
||||
# Cloud agent as fallback
|
||||
cloud_agent = agents.create_agent(
|
||||
name="cloud-helper", model="anthropic/claude-3-5-sonnet-20241022"
|
||||
)
|
||||
cloud_agent = agents.create_agent(name="cloud-helper", model="anthropic/claude-3-5-sonnet-20241022")
|
||||
|
||||
# 3. Create an agent network
|
||||
print("\nCreating agent network...")
|
||||
@@ -47,24 +45,18 @@ async def main():
|
||||
|
||||
# 4. Create MCP server with tools
|
||||
print("\nStarting MCP server...")
|
||||
mcp_server = mcp.create_mcp_server(
|
||||
name="hanzo-unified", allowed_paths=[".", "/tmp"], enable_agent_tool=True
|
||||
)
|
||||
mcp_server = mcp.create_mcp_server(name="hanzo-unified", allowed_paths=[".", "/tmp"], enable_agent_tool=True)
|
||||
|
||||
# 5. Example: Use local AI for simple tasks
|
||||
print("\n--- Local AI Example ---")
|
||||
local_result = await local_cluster.inference(
|
||||
prompt="Write a haiku about distributed AI", max_tokens=50
|
||||
)
|
||||
local_result = await local_cluster.inference(prompt="Write a haiku about distributed AI", max_tokens=50)
|
||||
print(f"Local AI: {local_result}")
|
||||
|
||||
# 6. Example: Use cloud AI for complex tasks
|
||||
print("\n--- Cloud AI Example ---")
|
||||
cloud_result = completion(
|
||||
model="anthropic/claude-3-5-sonnet-20241022",
|
||||
messages=[
|
||||
{"role": "user", "content": "Explain the benefits of local AI clusters"}
|
||||
],
|
||||
messages=[{"role": "user", "content": "Explain the benefits of local AI clusters"}],
|
||||
)
|
||||
print(f"Cloud AI: {cloud_result}")
|
||||
|
||||
|
||||
+44
-53
@@ -13,8 +13,8 @@ import sys
|
||||
import asyncio
|
||||
|
||||
# Add paths for development (remove in production)
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'pkg', 'hanzoai'))
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'pkg', 'hanzo', 'src'))
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "pkg", "hanzoai"))
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "pkg", "hanzo", "src"))
|
||||
|
||||
try:
|
||||
from hanzoai import Hanzo, AsyncHanzo
|
||||
@@ -30,86 +30,77 @@ except ImportError:
|
||||
def basic_grok_usage():
|
||||
"""Basic usage of Grok through Hanzo SDK."""
|
||||
client = Hanzo() # Uses HANZO_API_KEY env var
|
||||
|
||||
|
||||
# Simple completion
|
||||
response = client.chat.completions.create(
|
||||
model="grok-4", # or "grok" or "xai-grok"
|
||||
messages=[
|
||||
{"role": "system", "content": "You have real-time knowledge."},
|
||||
{"role": "user", "content": "What are the latest AI developments this week?"}
|
||||
{"role": "user", "content": "What are the latest AI developments this week?"},
|
||||
],
|
||||
temperature=0.7,
|
||||
max_tokens=500
|
||||
max_tokens=500,
|
||||
)
|
||||
|
||||
|
||||
print("Grok Response:")
|
||||
print(response.choices[0].message.content)
|
||||
print("\n" + "="*50 + "\n")
|
||||
print("\n" + "=" * 50 + "\n")
|
||||
|
||||
|
||||
def streaming_grok():
|
||||
"""Stream responses from Grok for real-time output."""
|
||||
client = Hanzo()
|
||||
|
||||
|
||||
print("Streaming from Grok:")
|
||||
stream = client.chat.completions.create(
|
||||
model="grok-4",
|
||||
messages=[
|
||||
{"role": "user", "content": "Explain quantum computing in simple terms"}
|
||||
],
|
||||
stream=True
|
||||
model="grok-4", messages=[{"role": "user", "content": "Explain quantum computing in simple terms"}], stream=True
|
||||
)
|
||||
|
||||
|
||||
for chunk in stream:
|
||||
if chunk.choices[0].delta.content:
|
||||
print(chunk.choices[0].delta.content, end="", flush=True)
|
||||
|
||||
print("\n\n" + "="*50 + "\n")
|
||||
|
||||
print("\n\n" + "=" * 50 + "\n")
|
||||
|
||||
|
||||
async def async_grok_usage():
|
||||
"""Async usage of Grok for concurrent operations."""
|
||||
client = AsyncHanzo()
|
||||
|
||||
|
||||
# Concurrent requests to different models
|
||||
tasks = [
|
||||
client.chat.completions.create(
|
||||
model="grok-4",
|
||||
messages=[{"role": "user", "content": "What's happening in tech today?"}]
|
||||
model="grok-4", messages=[{"role": "user", "content": "What's happening in tech today?"}]
|
||||
),
|
||||
client.chat.completions.create(
|
||||
model="claude-3-5-sonnet",
|
||||
messages=[{"role": "user", "content": "Explain machine learning"}]
|
||||
model="claude-3-5-sonnet", messages=[{"role": "user", "content": "Explain machine learning"}]
|
||||
),
|
||||
client.chat.completions.create(
|
||||
model="gpt-4",
|
||||
messages=[{"role": "user", "content": "Write a haiku about AI"}]
|
||||
)
|
||||
client.chat.completions.create(model="gpt-4", messages=[{"role": "user", "content": "Write a haiku about AI"}]),
|
||||
]
|
||||
|
||||
|
||||
print("Concurrent requests to Grok, Claude, and GPT-4:")
|
||||
responses = await asyncio.gather(*tasks)
|
||||
|
||||
|
||||
for i, (model, response) in enumerate(zip(["Grok", "Claude", "GPT-4"], responses)):
|
||||
print(f"\n{model} Response:")
|
||||
print(response.choices[0].message.content[:200] + "...")
|
||||
|
||||
print("\n" + "="*50 + "\n")
|
||||
|
||||
print("\n" + "=" * 50 + "\n")
|
||||
|
||||
|
||||
def batch_operations_with_grok():
|
||||
"""Demonstrate batch operations using Grok."""
|
||||
from hanzo.batch_orchestrator import BatchConfig, BatchOrchestrator
|
||||
|
||||
|
||||
print("Batch Operations with Grok:")
|
||||
|
||||
|
||||
# Parse different batch configurations
|
||||
configs = [
|
||||
"batch:5 agent:grok analyze code quality",
|
||||
"consensus:3 agent:grok,claude,gemini review architecture",
|
||||
"critic:2 agent:grok,gpt-4 chain:true security audit"
|
||||
"critic:2 agent:grok,gpt-4 chain:true security audit",
|
||||
]
|
||||
|
||||
|
||||
for cmd in configs:
|
||||
config = BatchConfig.from_command(cmd)
|
||||
print(f"\nCommand: {cmd}")
|
||||
@@ -119,73 +110,73 @@ def batch_operations_with_grok():
|
||||
print(f" - Consensus models: {config.consensus_models}")
|
||||
if config.critic_mode:
|
||||
print(f" - Critic chain: {config.critic_chain}")
|
||||
|
||||
print("\n" + "="*50 + "\n")
|
||||
|
||||
print("\n" + "=" * 50 + "\n")
|
||||
|
||||
|
||||
def grok_with_tools():
|
||||
"""Use Grok with MCP tools."""
|
||||
try:
|
||||
from hanzo_mcp.tools.agent.cli_tools import GrokCLITool
|
||||
|
||||
|
||||
print("Grok CLI Tool Configuration:")
|
||||
grok_tool = GrokCLITool()
|
||||
|
||||
|
||||
print(f"Tool name: {grok_tool.name}")
|
||||
print(f"Description: {grok_tool.description}")
|
||||
print(f"Default model: {grok_tool.default_model}")
|
||||
print(f"API key env: {grok_tool.api_key_env}")
|
||||
|
||||
|
||||
# Get auth environment
|
||||
env = grok_tool.get_auth_env()
|
||||
if "XAI_API_KEY" in env:
|
||||
print("✅ XAI_API_KEY is configured")
|
||||
else:
|
||||
print("⚠️ XAI_API_KEY not found in environment")
|
||||
|
||||
|
||||
except ImportError:
|
||||
print("hanzo-mcp not installed. Install with: pip install hanzo-mcp")
|
||||
|
||||
print("\n" + "="*50 + "\n")
|
||||
|
||||
print("\n" + "=" * 50 + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
"""Run all Grok examples."""
|
||||
print("="*50)
|
||||
print("=" * 50)
|
||||
print("xAI Grok Examples with Hanzo SDK")
|
||||
print("="*50 + "\n")
|
||||
|
||||
print("=" * 50 + "\n")
|
||||
|
||||
# Check for API key
|
||||
if not os.environ.get("XAI_API_KEY"):
|
||||
print("⚠️ Warning: XAI_API_KEY not set")
|
||||
print("Get your API key from https://x.ai/api")
|
||||
print("Set it with: export XAI_API_KEY='your-key-here'")
|
||||
print("\nRunning examples that don't require actual API calls...\n")
|
||||
|
||||
|
||||
# Run non-API examples
|
||||
batch_operations_with_grok()
|
||||
grok_with_tools()
|
||||
return
|
||||
|
||||
|
||||
# Run all examples
|
||||
try:
|
||||
# Basic usage
|
||||
basic_grok_usage()
|
||||
|
||||
|
||||
# Streaming
|
||||
streaming_grok()
|
||||
|
||||
|
||||
# Async operations
|
||||
asyncio.run(async_grok_usage())
|
||||
|
||||
|
||||
# Batch operations
|
||||
batch_operations_with_grok()
|
||||
|
||||
|
||||
# MCP tools
|
||||
grok_with_tools()
|
||||
|
||||
|
||||
print("✅ All Grok examples completed successfully!")
|
||||
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
print("\nMake sure you have:")
|
||||
@@ -196,4 +187,4 @@ def main():
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
main()
|
||||
|
||||
@@ -13,13 +13,14 @@ from typing import Any, Dict, List
|
||||
# This would normally use the hanzo-mcp Python client
|
||||
# For demonstration, showing the conceptual flow
|
||||
|
||||
|
||||
class WorktreeOrchestrator:
|
||||
"""Orchestrates multiple agents working on git worktrees."""
|
||||
|
||||
|
||||
def __init__(self, base_branch: str = "main"):
|
||||
self.base_branch = base_branch
|
||||
self.worktrees: Dict[str, str] = {}
|
||||
|
||||
|
||||
def create_worktree_agent_prompt(self, task_id: str, task_content: str) -> str:
|
||||
"""Generate agent prompt for worktree task."""
|
||||
return f"""
|
||||
@@ -115,23 +116,18 @@ Be strict but fair. This code will go to production.
|
||||
|
||||
def execute_workflow(self, tasks: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""Execute the complete worktree workflow."""
|
||||
|
||||
results = {
|
||||
"total_tasks": len(tasks),
|
||||
"completed": [],
|
||||
"failed": [],
|
||||
"worktrees": []
|
||||
}
|
||||
|
||||
|
||||
results = {"total_tasks": len(tasks), "completed": [], "failed": [], "worktrees": []}
|
||||
|
||||
for task in tasks:
|
||||
task_id = task["id"]
|
||||
task_content = task["content"]
|
||||
|
||||
|
||||
print(f"\n🚀 Starting task {task_id}: {task_content}")
|
||||
|
||||
|
||||
# Step 1: Agent implements the task
|
||||
agent_prompt = self.create_worktree_agent_prompt(task_id, task_content)
|
||||
|
||||
|
||||
# This would call: agent(prompt=agent_prompt)
|
||||
# For demo, showing the structure
|
||||
agent_result = {
|
||||
@@ -139,77 +135,69 @@ Be strict but fair. This code will go to production.
|
||||
"worktree": f"worktree-{task_id}",
|
||||
"files_modified": ["src/auth.py", "tests/test_auth.py"],
|
||||
"tests_passed": True,
|
||||
"ready_for_review": True
|
||||
"ready_for_review": True,
|
||||
}
|
||||
|
||||
|
||||
print(f"✅ Agent completed implementation")
|
||||
|
||||
|
||||
# Step 2: Critic reviews the implementation
|
||||
if agent_result["ready_for_review"]:
|
||||
critic_prompt = self.create_critic_prompt(
|
||||
task_id,
|
||||
json.dumps(agent_result, indent=2)
|
||||
)
|
||||
|
||||
critic_prompt = self.create_critic_prompt(task_id, json.dumps(agent_result, indent=2))
|
||||
|
||||
# This would call: critic(analysis=critic_prompt)
|
||||
critic_result = {
|
||||
"verdict": "APPROVED",
|
||||
"issues": [],
|
||||
"score": 95
|
||||
}
|
||||
|
||||
critic_result = {"verdict": "APPROVED", "issues": [], "score": 95}
|
||||
|
||||
if critic_result["verdict"] == "APPROVED":
|
||||
print(f"✅ Critic approved implementation (score: {critic_result['score']})")
|
||||
results["completed"].append(task_id)
|
||||
|
||||
|
||||
# Step 3: Prepare for merge
|
||||
self.worktrees[task_id] = f"feature/{task_id}"
|
||||
results["worktrees"].append({
|
||||
"task_id": task_id,
|
||||
"branch": f"feature/{task_id}",
|
||||
"path": f"../worktree-{task_id}",
|
||||
"ready_to_merge": True
|
||||
})
|
||||
results["worktrees"].append(
|
||||
{
|
||||
"task_id": task_id,
|
||||
"branch": f"feature/{task_id}",
|
||||
"path": f"../worktree-{task_id}",
|
||||
"ready_to_merge": True,
|
||||
}
|
||||
)
|
||||
else:
|
||||
print(f"⚠️ Critic requested fixes: {critic_result['issues']}")
|
||||
|
||||
|
||||
# Step 4: Agent fixes issues
|
||||
fix_prompt = f"""
|
||||
Fix these issues in worktree-{task_id}:
|
||||
{json.dumps(critic_result['issues'], indent=2)}
|
||||
{json.dumps(critic_result["issues"], indent=2)}
|
||||
"""
|
||||
|
||||
|
||||
# Second attempt would go here
|
||||
results["failed"].append({
|
||||
"task_id": task_id,
|
||||
"reason": "Needs fixes",
|
||||
"issues": critic_result["issues"]
|
||||
})
|
||||
|
||||
results["failed"].append(
|
||||
{"task_id": task_id, "reason": "Needs fixes", "issues": critic_result["issues"]}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def merge_completed_worktrees(self, results: Dict[str, Any]) -> None:
|
||||
"""Merge all completed worktrees back to base branch."""
|
||||
|
||||
|
||||
print(f"\n🔀 Merging completed worktrees to {self.base_branch}")
|
||||
|
||||
|
||||
for worktree in results["worktrees"]:
|
||||
if worktree["ready_to_merge"]:
|
||||
branch = worktree["branch"]
|
||||
|
||||
|
||||
# This would execute:
|
||||
# git checkout main
|
||||
# git merge --no-ff feature/{task_id}
|
||||
# git push origin main
|
||||
|
||||
|
||||
print(f"✅ Merged {branch} to {self.base_branch}")
|
||||
|
||||
|
||||
print(f"\n🎉 Workflow complete! {len(results['completed'])}/{results['total_tasks']} tasks merged")
|
||||
|
||||
|
||||
# Example usage
|
||||
if __name__ == "__main__":
|
||||
|
||||
# Sample tasks from todo list
|
||||
tasks = [
|
||||
{"id": "auth-1", "content": "Implement JWT token generation", "priority": "high"},
|
||||
@@ -217,23 +205,23 @@ if __name__ == "__main__":
|
||||
{"id": "auth-3", "content": "Add password reset flow", "priority": "medium"},
|
||||
{"id": "auth-4", "content": "Implement rate limiting", "priority": "medium"},
|
||||
]
|
||||
|
||||
|
||||
# Initialize orchestrator
|
||||
orchestrator = WorktreeOrchestrator(base_branch="main")
|
||||
|
||||
|
||||
# Execute parallel development workflow
|
||||
results = orchestrator.execute_workflow(tasks)
|
||||
|
||||
|
||||
# Merge successful implementations
|
||||
orchestrator.merge_completed_worktrees(results)
|
||||
|
||||
|
||||
# Output summary
|
||||
print("\n📊 Workflow Summary:")
|
||||
print(f" Total tasks: {results['total_tasks']}")
|
||||
print(f" Completed: {len(results['completed'])}")
|
||||
print(f" Failed: {len(results['failed'])}")
|
||||
print(f" Worktrees created: {len(results['worktrees'])}")
|
||||
|
||||
|
||||
# This summary could be sent to Linear or saved to a report
|
||||
with open("workflow_report.json", "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
json.dump(results, f, indent=2)
|
||||
|
||||
+4
-3
@@ -38,7 +38,7 @@ filterwarnings =
|
||||
"""
|
||||
|
||||
# Write temporary config and use it
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.ini', delete=False) as f:
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".ini", delete=False) as f:
|
||||
f.write(pytest_ini_content)
|
||||
pytest_ini_path = f.name
|
||||
|
||||
@@ -46,7 +46,8 @@ filterwarnings =
|
||||
session.run(
|
||||
"pytest",
|
||||
"tests/", # Explicitly specify tests/ to avoid collecting from pkg/* subpackages
|
||||
"-c", pytest_ini_path,
|
||||
"-c",
|
||||
pytest_ini_path,
|
||||
"--showlocals",
|
||||
"--ignore=tests/functional",
|
||||
# Ignore tests that require optional dependencies (hanzo, hanzo_network, click, etc.)
|
||||
@@ -60,7 +61,7 @@ filterwarnings =
|
||||
"--ignore=tests/test_streaming.py",
|
||||
"--ignore=tests/test_todo.py",
|
||||
"--ignore=tests/test_hanzo_dev.py", # Requires hanzo package
|
||||
*session.posargs
|
||||
*session.posargs,
|
||||
)
|
||||
finally:
|
||||
os.unlink(pytest_ini_path)
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
CONTENT_TRUNCATED_NOTICE = "<response clipped><NOTE>Due to the max output limit, only part of the full response has been shown to you.</NOTE>"
|
||||
|
||||
FILE_CONTENT_TRUNCATED_NOTICE: str = (
|
||||
"<response clipped><NOTE>Due to the max output limit, only part of this file has been shown to you. You should retry this tool after you have searched inside the file with `grep -n` in order to find the line numbers of what you are looking for.</NOTE>"
|
||||
)
|
||||
FILE_CONTENT_TRUNCATED_NOTICE: str = "<response clipped><NOTE>Due to the max output limit, only part of this file has been shown to you. You should retry this tool after you have searched inside the file with `grep -n` in order to find the line numbers of what you are looking for.</NOTE>"
|
||||
|
||||
DIRECTORY_CONTENT_TRUNCATED_NOTICE: str = (
|
||||
"<response clipped><NOTE>Due to the max output limit, only part of this directory has been shown to you. You should use `ls -la` instead to view large directories incrementally.</NOTE>"
|
||||
)
|
||||
DIRECTORY_CONTENT_TRUNCATED_NOTICE: str = "<response clipped><NOTE>Due to the max output limit, only part of this directory has been shown to you. You should use `ls -la` instead to view large directories incrementally.</NOTE>"
|
||||
|
||||
@@ -32,9 +32,9 @@ def test_file_editor_happy_path(temp_file):
|
||||
result,
|
||||
re.DOTALL,
|
||||
)
|
||||
assert (
|
||||
match
|
||||
), "Output does not contain the expected <oh_aci_output_> tags in the correct format."
|
||||
assert match, (
|
||||
"Output does not contain the expected <oh_aci_output_> tags in the correct format."
|
||||
)
|
||||
result_dict = json.loads(match.group(1))
|
||||
|
||||
# Validate the formatted output in the result dictionary
|
||||
@@ -89,9 +89,9 @@ match = re.search(
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
assert (
|
||||
match
|
||||
), "Output does not contain the expected <oh_aci_output_> tags in the correct format."
|
||||
assert match, (
|
||||
"Output does not contain the expected <oh_aci_output_> tags in the correct format."
|
||||
)
|
||||
result_dict = json.loads(match.group(1))
|
||||
|
||||
# Validate the formatted output in the result dictionary
|
||||
|
||||
@@ -256,9 +256,9 @@ def test_large_history_insert():
|
||||
|
||||
# Check if we can still retrieve the last entry
|
||||
last_content = manager.pop_last_history(Path(f"test_file_{num_files - 1}.txt"))
|
||||
assert (
|
||||
last_content == large_content
|
||||
), "Failed to retrieve the last inserted content"
|
||||
assert last_content == large_content, (
|
||||
"Failed to retrieve the last inserted content"
|
||||
)
|
||||
|
||||
# Check if the number of cache entries is correct
|
||||
cache_entries = list(manager.cache)
|
||||
@@ -266,4 +266,6 @@ def test_large_history_insert():
|
||||
len(cache_entries)
|
||||
== num_files * 2
|
||||
- 1 # The cache entry for file content was removed, only metadata remains
|
||||
), f"Expected {num_files * 2 - 1} cache entries ({num_files - 1} content + {num_files} metadata), but found {len(cache_entries)}"
|
||||
), (
|
||||
f"Expected {num_files * 2 - 1} cache entries ({num_files - 1} content + {num_files} metadata), but found {len(cache_entries)}"
|
||||
)
|
||||
|
||||
@@ -91,9 +91,7 @@ async def run_node(node_id: str, port: int, agents_subset: list):
|
||||
while True:
|
||||
await asyncio.sleep(5)
|
||||
status = network.get_network_status()
|
||||
print(
|
||||
f"📊 Node {node_id} status - Peers: {status['peer_count']}, Agents: {status['local_agents']}"
|
||||
)
|
||||
print(f"📊 Node {node_id} status - Peers: {status['peer_count']}, Agents: {status['local_agents']}")
|
||||
|
||||
# Show discovered peers
|
||||
if status["peers"]:
|
||||
|
||||
@@ -90,27 +90,21 @@ async def main():
|
||||
|
||||
# Test weather agent
|
||||
print("\n🌤️ Testing weather agent...")
|
||||
weather_result = await network.run(
|
||||
prompt="What's the weather in Tokyo?", initial_agent=weather_agent
|
||||
)
|
||||
weather_result = await network.run(prompt="What's the weather in Tokyo?", initial_agent=weather_agent)
|
||||
|
||||
if weather_result["success"]:
|
||||
print(f"Response: {weather_result['final_output']}")
|
||||
|
||||
# Test math agent
|
||||
print("\n🔢 Testing math agent...")
|
||||
math_result = await network.run(
|
||||
prompt="Calculate 42 * 17 + 3", initial_agent=math_agent
|
||||
)
|
||||
math_result = await network.run(prompt="Calculate 42 * 17 + 3", initial_agent=math_agent)
|
||||
|
||||
if math_result["success"]:
|
||||
print(f"Response: {math_result['final_output']}")
|
||||
|
||||
# Test with router (let network decide which agent)
|
||||
print("\n🤔 Testing with router...")
|
||||
auto_result = await network.run(
|
||||
prompt="I need to know the weather in Paris and also calculate 100 / 4"
|
||||
)
|
||||
auto_result = await network.run(prompt="I need to know the weather in Paris and also calculate 100 / 4")
|
||||
|
||||
if auto_result["success"]:
|
||||
print(f"Response: {auto_result['final_output']}")
|
||||
|
||||
@@ -108,38 +108,28 @@ class Agent:
|
||||
|
||||
# Call lifecycle hook
|
||||
if self.lifecycle and self.lifecycle.on_start:
|
||||
result = await self.lifecycle.on_start(
|
||||
agent=self, messages=messages, state=state, context=context
|
||||
)
|
||||
result = await self.lifecycle.on_start(agent=self, messages=messages, state=state, context=context)
|
||||
if result and result.get("stop"):
|
||||
return result
|
||||
messages = result.get("messages", messages)
|
||||
|
||||
try:
|
||||
# Execute with appropriate backend
|
||||
if (
|
||||
self.model
|
||||
and isinstance(self.model, ModelConfig)
|
||||
and self.model.provider == ModelProvider.CLI
|
||||
):
|
||||
if self.model and isinstance(self.model, ModelConfig) and self.model.provider == ModelProvider.CLI:
|
||||
result = await self._execute_cli(messages, state, context)
|
||||
else:
|
||||
result = await self._execute_llm(messages, state, context)
|
||||
|
||||
# Call finish hook
|
||||
if self.lifecycle and self.lifecycle.on_finish:
|
||||
await self.lifecycle.on_finish(
|
||||
agent=self, result=result, state=state, context=context
|
||||
)
|
||||
await self.lifecycle.on_finish(agent=self, result=result, state=state, context=context)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
# Call error hook
|
||||
if self.lifecycle and self.lifecycle.on_error:
|
||||
await self.lifecycle.on_error(
|
||||
agent=self, error=e, state=state, context=context
|
||||
)
|
||||
await self.lifecycle.on_error(agent=self, error=e, state=state, context=context)
|
||||
raise
|
||||
|
||||
async def _execute_llm(
|
||||
@@ -158,9 +148,7 @@ class Agent:
|
||||
tools = [t.to_anthropic_tool() for t in self.tools]
|
||||
elif self.model.provider == ModelProvider.LOCAL:
|
||||
# Simple tool format for local LLMs
|
||||
tools = [
|
||||
{"name": t.name, "description": t.description} for t in self.tools
|
||||
]
|
||||
tools = [{"name": t.name, "description": t.description} for t in self.tools]
|
||||
|
||||
# Handle local LLM providers using hanzo/net
|
||||
if self.model and self.model.provider == ModelProvider.LOCAL:
|
||||
@@ -174,9 +162,7 @@ class Agent:
|
||||
engine_type = "tinygrad"
|
||||
|
||||
# Create hanzo/net provider
|
||||
provider = HanzoNetProvider(
|
||||
engine_type=engine_type, base_url=self.model.base_url
|
||||
)
|
||||
provider = HanzoNetProvider(engine_type=engine_type, base_url=self.model.base_url)
|
||||
|
||||
# Generate using distributed inference
|
||||
return await provider.generate(
|
||||
|
||||
@@ -209,9 +209,7 @@ class Network(Generic[T]):
|
||||
# No router - stop
|
||||
return None
|
||||
|
||||
def _build_agent_prompt(
|
||||
self, agent: Agent, last_result: Optional[Dict[str, Any]]
|
||||
) -> List[Any]:
|
||||
def _build_agent_prompt(self, agent: Agent, last_result: Optional[Dict[str, Any]]) -> List[Any]:
|
||||
"""Build prompt for agent including conversation history.
|
||||
|
||||
Args:
|
||||
|
||||
@@ -129,9 +129,7 @@ class RoutingAgent(Agent):
|
||||
|
||||
return "\n".join(prompt_parts)
|
||||
|
||||
def _parse_routing_decision(
|
||||
self, result: Dict[str, Any], args: RouterArgs
|
||||
) -> Optional[Agent]:
|
||||
def _parse_routing_decision(self, result: Dict[str, Any], args: RouterArgs) -> Optional[Agent]:
|
||||
"""Parse routing decision from agent result."""
|
||||
output = result.get("output", [])
|
||||
|
||||
@@ -226,9 +224,7 @@ def get_default_routing_agent(
|
||||
from .agent import ModelConfig, ModelProvider
|
||||
|
||||
# Use local dummy model for default router
|
||||
model = ModelConfig(
|
||||
provider=ModelProvider.LOCAL, model="llama3.2", temperature=0.3
|
||||
)
|
||||
model = ModelConfig(provider=ModelProvider.LOCAL, model="llama3.2", temperature=0.3)
|
||||
|
||||
return create_routing_agent(
|
||||
name="default_router",
|
||||
|
||||
@@ -50,9 +50,7 @@ class NetworkState(Generic[T]):
|
||||
self.execution_count: int = 0
|
||||
self.metadata: Dict[str, Any] = {}
|
||||
|
||||
def add_message(
|
||||
self, role: str, content: str, agent_id: Optional[str] = None, **metadata
|
||||
) -> None:
|
||||
def add_message(self, role: str, content: str, agent_id: Optional[str] = None, **metadata) -> None:
|
||||
"""Add a message to the conversation history.
|
||||
|
||||
Args:
|
||||
@@ -61,9 +59,7 @@ class NetworkState(Generic[T]):
|
||||
agent_id: ID of the agent that generated this message
|
||||
**metadata: Additional metadata for the message
|
||||
"""
|
||||
self.messages.append(
|
||||
Message(role=role, content=content, agent_id=agent_id, metadata=metadata)
|
||||
)
|
||||
self.messages.append(Message(role=role, content=content, agent_id=agent_id, metadata=metadata))
|
||||
|
||||
def get_messages(
|
||||
self,
|
||||
|
||||
@@ -17,9 +17,7 @@ class ToolContext(TypedDict):
|
||||
class ToolHandler(Protocol):
|
||||
"""Protocol for tool handler functions."""
|
||||
|
||||
async def __call__(
|
||||
self, parameters: Dict[str, Any], context: ToolContext
|
||||
) -> Any: ...
|
||||
async def __call__(self, parameters: Dict[str, Any], context: ToolContext) -> Any: ...
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -34,9 +32,7 @@ class Tool:
|
||||
parameters: type[BaseModel]
|
||||
handler: ToolHandler
|
||||
|
||||
async def call(
|
||||
self, parameters: Dict[str, Any], context: Optional[ToolContext] = None
|
||||
) -> Any:
|
||||
async def call(self, parameters: Dict[str, Any], context: Optional[ToolContext] = None) -> Any:
|
||||
"""Call the tool with given parameters.
|
||||
|
||||
Args:
|
||||
@@ -130,14 +126,8 @@ def create_tool(
|
||||
continue
|
||||
|
||||
# Get type annotation
|
||||
param_type = (
|
||||
param.annotation
|
||||
if param.annotation != inspect.Parameter.empty
|
||||
else Any
|
||||
)
|
||||
default = (
|
||||
param.default if param.default != inspect.Parameter.empty else ...
|
||||
)
|
||||
param_type = param.annotation if param.annotation != inspect.Parameter.empty else Any
|
||||
default = param.default if param.default != inspect.Parameter.empty else ...
|
||||
|
||||
param_fields[param_name] = (param_type, default)
|
||||
|
||||
@@ -147,9 +137,7 @@ def create_tool(
|
||||
else:
|
||||
parameters = create_model(f"{name}_params")
|
||||
|
||||
return Tool(
|
||||
name=name, description=description, parameters=parameters, handler=func
|
||||
)
|
||||
return Tool(name=name, description=description, parameters=parameters, handler=func)
|
||||
|
||||
if handler is not None:
|
||||
# Direct call with handler
|
||||
|
||||
@@ -21,9 +21,7 @@ else:
|
||||
|
||||
|
||||
class GRPCPeerHandle(PeerHandle):
|
||||
def __init__(
|
||||
self, _id: str, address: str, desc: str, device_capabilities: DeviceCapabilities
|
||||
):
|
||||
def __init__(self, _id: str, address: str, desc: str, device_capabilities: DeviceCapabilities):
|
||||
self._id = _id
|
||||
self.address = address
|
||||
self.desc = desc
|
||||
@@ -67,10 +65,7 @@ class GRPCPeerHandle(PeerHandle):
|
||||
await asyncio.wait_for(self.channel.channel_ready(), timeout=10.0)
|
||||
|
||||
async def is_connected(self) -> bool:
|
||||
return (
|
||||
self.channel is not None
|
||||
and self.channel.get_state() == grpc.ChannelConnectivity.READY
|
||||
)
|
||||
return self.channel is not None and self.channel.get_state() == grpc.ChannelConnectivity.READY
|
||||
|
||||
async def disconnect(self):
|
||||
if self.channel:
|
||||
@@ -121,11 +116,7 @@ class GRPCPeerHandle(PeerHandle):
|
||||
n_layers=shard.n_layers,
|
||||
),
|
||||
request_id=request_id,
|
||||
inference_state=(
|
||||
None
|
||||
if inference_state is None
|
||||
else self.serialize_inference_state(inference_state)
|
||||
),
|
||||
inference_state=(None if inference_state is None else self.serialize_inference_state(inference_state)),
|
||||
)
|
||||
await self.stub.SendPrompt(request)
|
||||
|
||||
@@ -150,20 +141,14 @@ class GRPCPeerHandle(PeerHandle):
|
||||
dtype=str(tensor.dtype),
|
||||
),
|
||||
request_id=request_id,
|
||||
inference_state=(
|
||||
None
|
||||
if inference_state is None
|
||||
else self.serialize_inference_state(inference_state)
|
||||
),
|
||||
inference_state=(None if inference_state is None else self.serialize_inference_state(inference_state)),
|
||||
)
|
||||
response = await self.stub.SendTensor(request)
|
||||
|
||||
if not response.tensor_data or not response.shape or not response.dtype:
|
||||
return None
|
||||
|
||||
return np.frombuffer(
|
||||
response.tensor_data, dtype=np.dtype(response.dtype)
|
||||
).reshape(response.shape)
|
||||
return np.frombuffer(response.tensor_data, dtype=np.dtype(response.dtype)).reshape(response.shape)
|
||||
|
||||
async def send_example(
|
||||
self,
|
||||
@@ -203,16 +188,14 @@ class GRPCPeerHandle(PeerHandle):
|
||||
response = await self.stub.SendExample(request)
|
||||
loss = response.loss
|
||||
if train and not shard.is_first_layer():
|
||||
grads = np.frombuffer(
|
||||
response.grads.tensor_data, dtype=np.dtype(response.grads.dtype)
|
||||
).reshape(response.grads.shape)
|
||||
grads = np.frombuffer(response.grads.tensor_data, dtype=np.dtype(response.grads.dtype)).reshape(
|
||||
response.grads.shape
|
||||
)
|
||||
return loss, grads
|
||||
else:
|
||||
return loss
|
||||
|
||||
async def send_loss(
|
||||
self, shard: Shard, tensor: np.ndarray, request_id: Optional[str] = None
|
||||
) -> Optional[np.array]:
|
||||
async def send_loss(self, shard: Shard, tensor: np.ndarray, request_id: Optional[str] = None) -> Optional[np.array]:
|
||||
await self._ensure_connected()
|
||||
request = node_service_pb2.TensorRequest(
|
||||
shard=node_service_pb2.Shard(
|
||||
@@ -233,15 +216,11 @@ class GRPCPeerHandle(PeerHandle):
|
||||
if not response.tensor_data or not response.shape or not response.dtype:
|
||||
return None
|
||||
|
||||
return np.frombuffer(
|
||||
response.tensor_data, dtype=np.dtype(response.dtype)
|
||||
).reshape(response.shape)
|
||||
return np.frombuffer(response.tensor_data, dtype=np.dtype(response.dtype)).reshape(response.shape)
|
||||
|
||||
async def collect_topology(self, visited: set[str], max_depth: int) -> Topology:
|
||||
await self._ensure_connected()
|
||||
request = node_service_pb2.CollectTopologyRequest(
|
||||
visited=visited, max_depth=max_depth
|
||||
)
|
||||
request = node_service_pb2.CollectTopologyRequest(visited=visited, max_depth=max_depth)
|
||||
response = await self.stub.CollectTopology(request)
|
||||
topology = Topology()
|
||||
for node_id, capabilities in response.nodes.items():
|
||||
@@ -261,9 +240,7 @@ class GRPCPeerHandle(PeerHandle):
|
||||
topology.add_edge(node_id, conn.to_id, conn.description)
|
||||
return topology
|
||||
|
||||
async def send_result(
|
||||
self, request_id: str, result: List[int], is_finished: bool
|
||||
) -> None:
|
||||
async def send_result(self, request_id: str, result: List[int], is_finished: bool) -> None:
|
||||
await self._ensure_connected()
|
||||
tensor = None
|
||||
if isinstance(result, np.ndarray):
|
||||
@@ -280,14 +257,10 @@ class GRPCPeerHandle(PeerHandle):
|
||||
|
||||
async def send_opaque_status(self, request_id: str, status: str) -> None:
|
||||
await self._ensure_connected()
|
||||
request = node_service_pb2.SendOpaqueStatusRequest(
|
||||
request_id=request_id, status=status
|
||||
)
|
||||
request = node_service_pb2.SendOpaqueStatusRequest(request_id=request_id, status=status)
|
||||
await asyncio.wait_for(self.stub.SendOpaqueStatus(request), timeout=10.0)
|
||||
|
||||
def serialize_inference_state(
|
||||
self, inference_state: dict
|
||||
) -> node_service_pb2.InferenceState:
|
||||
def serialize_inference_state(self, inference_state: dict) -> node_service_pb2.InferenceState:
|
||||
proto_inference_state = node_service_pb2.InferenceState()
|
||||
other_data = {}
|
||||
for k, v in inference_state.items():
|
||||
|
||||
@@ -74,20 +74,14 @@ class GRPCServer(node_service_pb2_grpc.NodeServiceServicer):
|
||||
prompt = request.prompt
|
||||
request_id = request.request_id
|
||||
inference_state = (
|
||||
None
|
||||
if request.inference_state is None
|
||||
else self.deserialize_inference_state(request.inference_state)
|
||||
)
|
||||
result = await self.node.process_prompt(
|
||||
shard, prompt, request_id, inference_state
|
||||
None if request.inference_state is None else self.deserialize_inference_state(request.inference_state)
|
||||
)
|
||||
result = await self.node.process_prompt(shard, prompt, request_id, inference_state)
|
||||
if DEBUG >= 5:
|
||||
print(f"SendPrompt {shard=} {prompt=} {request_id=} result: {result}")
|
||||
tensor_data = result.tobytes() if result is not None else None
|
||||
return (
|
||||
node_service_pb2.Tensor(
|
||||
tensor_data=tensor_data, shape=result.shape, dtype=str(result.dtype)
|
||||
)
|
||||
node_service_pb2.Tensor(tensor_data=tensor_data, shape=result.shape, dtype=str(result.dtype))
|
||||
if result is not None
|
||||
else node_service_pb2.Tensor()
|
||||
)
|
||||
@@ -99,29 +93,21 @@ class GRPCServer(node_service_pb2_grpc.NodeServiceServicer):
|
||||
end_layer=request.shard.end_layer,
|
||||
n_layers=request.shard.n_layers,
|
||||
)
|
||||
tensor = np.frombuffer(
|
||||
request.tensor.tensor_data, dtype=np.dtype(request.tensor.dtype)
|
||||
).reshape(request.tensor.shape)
|
||||
tensor = np.frombuffer(request.tensor.tensor_data, dtype=np.dtype(request.tensor.dtype)).reshape(
|
||||
request.tensor.shape
|
||||
)
|
||||
request_id = request.request_id
|
||||
|
||||
inference_state = (
|
||||
None
|
||||
if request.inference_state is None
|
||||
else self.deserialize_inference_state(request.inference_state)
|
||||
None if request.inference_state is None else self.deserialize_inference_state(request.inference_state)
|
||||
)
|
||||
|
||||
result = await self.node.process_tensor(
|
||||
shard, tensor, request_id, inference_state
|
||||
)
|
||||
result = await self.node.process_tensor(shard, tensor, request_id, inference_state)
|
||||
if DEBUG >= 5:
|
||||
print(
|
||||
f"SendTensor tensor {shard=} {tensor=} {request_id=} result: {result}"
|
||||
)
|
||||
print(f"SendTensor tensor {shard=} {tensor=} {request_id=} result: {result}")
|
||||
tensor_data = result.tobytes() if result is not None else None
|
||||
return (
|
||||
node_service_pb2.Tensor(
|
||||
tensor_data=tensor_data, shape=result.shape, dtype=str(result.dtype)
|
||||
)
|
||||
node_service_pb2.Tensor(tensor_data=tensor_data, shape=result.shape, dtype=str(result.dtype))
|
||||
if result is not None
|
||||
else node_service_pb2.Tensor()
|
||||
)
|
||||
@@ -133,31 +119,25 @@ class GRPCServer(node_service_pb2_grpc.NodeServiceServicer):
|
||||
end_layer=request.shard.end_layer,
|
||||
n_layers=request.shard.n_layers,
|
||||
)
|
||||
example = np.frombuffer(
|
||||
request.example.tensor_data, dtype=np.dtype(request.example.dtype)
|
||||
).reshape(request.example.shape)
|
||||
target = np.frombuffer(
|
||||
request.target.tensor_data, dtype=np.dtype(request.target.dtype)
|
||||
).reshape(request.target.shape)
|
||||
length = np.frombuffer(
|
||||
request.length.tensor_data, dtype=np.dtype(request.length.dtype)
|
||||
).reshape(request.length.shape)
|
||||
example = np.frombuffer(request.example.tensor_data, dtype=np.dtype(request.example.dtype)).reshape(
|
||||
request.example.shape
|
||||
)
|
||||
target = np.frombuffer(request.target.tensor_data, dtype=np.dtype(request.target.dtype)).reshape(
|
||||
request.target.shape
|
||||
)
|
||||
length = np.frombuffer(request.length.tensor_data, dtype=np.dtype(request.length.dtype)).reshape(
|
||||
request.length.shape
|
||||
)
|
||||
train = request.train
|
||||
request_id = request.request_id
|
||||
|
||||
if train and not shard.is_first_layer():
|
||||
loss, grad = await self.node.process_example(
|
||||
shard, example, target, length, train, request_id
|
||||
)
|
||||
loss, grad = await self.node.process_example(shard, example, target, length, train, request_id)
|
||||
tensor_data = grad.tobytes()
|
||||
grad_tensor = node_service_pb2.Tensor(
|
||||
tensor_data=tensor_data, shape=grad.shape, dtype=str(grad.dtype)
|
||||
)
|
||||
grad_tensor = node_service_pb2.Tensor(tensor_data=tensor_data, shape=grad.shape, dtype=str(grad.dtype))
|
||||
return node_service_pb2.Loss(loss=loss, grads=grad_tensor)
|
||||
else:
|
||||
loss = await self.node.process_example(
|
||||
shard, example, target, length, train, request_id
|
||||
)
|
||||
loss = await self.node.process_example(shard, example, target, length, train, request_id)
|
||||
return node_service_pb2.Loss(loss=loss, grads=None)
|
||||
|
||||
async def CollectTopology(self, request, context):
|
||||
@@ -169,18 +149,14 @@ class GRPCServer(node_service_pb2_grpc.NodeServiceServicer):
|
||||
model=cap.model,
|
||||
chip=cap.chip,
|
||||
memory=cap.memory,
|
||||
flops=node_service_pb2.DeviceFlops(
|
||||
fp32=cap.flops.fp32, fp16=cap.flops.fp16, int8=cap.flops.int8
|
||||
),
|
||||
flops=node_service_pb2.DeviceFlops(fp32=cap.flops.fp32, fp16=cap.flops.fp16, int8=cap.flops.int8),
|
||||
)
|
||||
for node_id, cap in topology.nodes.items()
|
||||
}
|
||||
peer_graph = {
|
||||
node_id: node_service_pb2.PeerConnections(
|
||||
connections=[
|
||||
node_service_pb2.PeerConnection(
|
||||
to_id=conn.to_id, description=conn.description
|
||||
)
|
||||
node_service_pb2.PeerConnection(to_id=conn.to_id, description=conn.description)
|
||||
for conn in connections
|
||||
]
|
||||
)
|
||||
@@ -196,14 +172,10 @@ class GRPCServer(node_service_pb2_grpc.NodeServiceServicer):
|
||||
is_finished = request.is_finished
|
||||
img = request.tensor
|
||||
if DEBUG >= 5:
|
||||
print(
|
||||
f"Received SendResult request: {request_id=} {result=} {is_finished=}"
|
||||
)
|
||||
print(f"Received SendResult request: {request_id=} {result=} {is_finished=}")
|
||||
result = list(result)
|
||||
if len(img.tensor_data) > 0:
|
||||
result = np.frombuffer(img.tensor_data, dtype=np.dtype(img.dtype)).reshape(
|
||||
img.shape
|
||||
)
|
||||
result = np.frombuffer(img.tensor_data, dtype=np.dtype(img.dtype)).reshape(img.shape)
|
||||
self.node.on_token.trigger_all(request_id, result, is_finished)
|
||||
return node_service_pb2.Empty()
|
||||
|
||||
@@ -218,24 +190,16 @@ class GRPCServer(node_service_pb2_grpc.NodeServiceServicer):
|
||||
async def HealthCheck(self, request, context):
|
||||
return node_service_pb2.HealthCheckResponse(is_healthy=True)
|
||||
|
||||
def deserialize_inference_state(
|
||||
self, inference_state_proto: node_service_pb2.InferenceState
|
||||
) -> dict:
|
||||
def deserialize_inference_state(self, inference_state_proto: node_service_pb2.InferenceState) -> dict:
|
||||
inference_state = {}
|
||||
|
||||
for k, tensor_data in inference_state_proto.tensor_data.items():
|
||||
np_array = np.frombuffer(
|
||||
tensor_data.tensor_data, dtype=tensor_data.dtype
|
||||
).reshape(tensor_data.shape)
|
||||
np_array = np.frombuffer(tensor_data.tensor_data, dtype=tensor_data.dtype).reshape(tensor_data.shape)
|
||||
inference_state[k] = mx.array(np_array)
|
||||
|
||||
for k, tensor_list in inference_state_proto.tensor_list_data.items():
|
||||
inference_state[k] = [
|
||||
mx.array(
|
||||
np.frombuffer(tensor.tensor_data, dtype=tensor.dtype).reshape(
|
||||
tensor.shape
|
||||
)
|
||||
)
|
||||
mx.array(np.frombuffer(tensor.tensor_data, dtype=tensor.dtype).reshape(tensor.shape))
|
||||
for tensor in tensor_list.tensors
|
||||
]
|
||||
|
||||
|
||||
@@ -11,9 +11,7 @@ from google.protobuf import runtime_version as _runtime_version
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
from google.protobuf.internal import builder as _builder
|
||||
|
||||
_runtime_version.ValidateProtobufRuntimeVersion(
|
||||
_runtime_version.Domain.PUBLIC, 5, 27, 2, "", "node_service.proto"
|
||||
)
|
||||
_runtime_version.ValidateProtobufRuntimeVersion(_runtime_version.Domain.PUBLIC, 5, 27, 2, "", "node_service.proto")
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
@@ -12,9 +12,7 @@ _version_not_supported = False
|
||||
try:
|
||||
from grpc._utilities import first_version_is_lower
|
||||
|
||||
_version_not_supported = first_version_is_lower(
|
||||
GRPC_VERSION, GRPC_GENERATED_VERSION
|
||||
)
|
||||
_version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION)
|
||||
except ImportError:
|
||||
_version_not_supported = True
|
||||
|
||||
@@ -165,13 +163,9 @@ def add_NodeServiceServicer_to_server(servicer, server):
|
||||
response_serializer=node__service__pb2.HealthCheckResponse.SerializeToString,
|
||||
),
|
||||
}
|
||||
generic_handler = grpc.method_handlers_generic_handler(
|
||||
"node_service.NodeService", rpc_method_handlers
|
||||
)
|
||||
generic_handler = grpc.method_handlers_generic_handler("node_service.NodeService", rpc_method_handlers)
|
||||
server.add_generic_rpc_handlers((generic_handler,))
|
||||
server.add_registered_method_handlers(
|
||||
"node_service.NodeService", rpc_method_handlers
|
||||
)
|
||||
server.add_registered_method_handlers("node_service.NodeService", rpc_method_handlers)
|
||||
|
||||
|
||||
# This class is part of an EXPERIMENTAL API.
|
||||
|
||||
@@ -40,14 +40,10 @@ class ManualDiscovery(Discovery):
|
||||
if wait_for_peers > 0:
|
||||
while len(self.known_peers) < wait_for_peers:
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"Current peers: {len(self.known_peers)}/{wait_for_peers}. Waiting for more peers..."
|
||||
)
|
||||
print(f"Current peers: {len(self.known_peers)}/{wait_for_peers}. Waiting for more peers...")
|
||||
await asyncio.sleep(0.1)
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"Discovered peers: {[peer.id() for peer in self.known_peers.values()]}"
|
||||
)
|
||||
print(f"Discovered peers: {[peer.id() for peer in self.known_peers.values()]}")
|
||||
return list(self.known_peers.values())
|
||||
|
||||
async def task_find_peers_from_config(self):
|
||||
@@ -59,9 +55,7 @@ class ManualDiscovery(Discovery):
|
||||
for peer_id, peer_config in peers_from_config.items():
|
||||
try:
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"Checking peer {peer_id=} at {peer_config.address}:{peer_config.port}"
|
||||
)
|
||||
print(f"Checking peer {peer_id=} at {peer_config.address}:{peer_config.port}")
|
||||
peer = self.known_peers.get(peer_id)
|
||||
if not peer:
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
@@ -75,33 +69,23 @@ class ManualDiscovery(Discovery):
|
||||
is_healthy = await peer.health_check()
|
||||
if is_healthy:
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"{peer_id=} at {peer_config.address}:{peer_config.port} is healthy."
|
||||
)
|
||||
print(f"{peer_id=} at {peer_config.address}:{peer_config.port} is healthy.")
|
||||
new_known_peers[peer_id] = peer
|
||||
elif DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"{peer_id=} at {peer_config.address}:{peer_config.port} is not healthy. Removing."
|
||||
)
|
||||
print(f"{peer_id=} at {peer_config.address}:{peer_config.port} is not healthy. Removing.")
|
||||
except Exception as e:
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"Exception occurred when attempting to add {peer_id=}: {e}"
|
||||
)
|
||||
print(f"Exception occurred when attempting to add {peer_id=}: {e}")
|
||||
self.known_peers = new_known_peers
|
||||
await asyncio.sleep(5.0)
|
||||
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"Current known peers: {[peer.id() for peer in self.known_peers.values()]}"
|
||||
)
|
||||
print(f"Current known peers: {[peer.id() for peer in self.known_peers.values()]}")
|
||||
|
||||
async def _get_peers(self):
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
current_mtime = await loop.run_in_executor(
|
||||
self._file_executor, os.path.getmtime, self.network_config_path
|
||||
)
|
||||
current_mtime = await loop.run_in_executor(self._file_executor, os.path.getmtime, self.network_config_path)
|
||||
|
||||
if (
|
||||
self._cached_peers is not None
|
||||
|
||||
@@ -29,6 +29,4 @@ class NetworkTopology(BaseModel):
|
||||
try:
|
||||
return cls.model_validate_json(config_data)
|
||||
except ValidationError as e:
|
||||
raise ValueError(
|
||||
f"Error validating network topology config from {path}: {e}"
|
||||
) from e
|
||||
raise ValueError(f"Error validating network topology config from {path}: {e}") from e
|
||||
|
||||
@@ -72,12 +72,8 @@ class TestManualDiscoveryWithGRPCPeerHandle(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
self.node1 = mock.AsyncMock(spec=Node)
|
||||
self.node2 = mock.AsyncMock(spec=Node)
|
||||
self.server1 = GRPCServer(
|
||||
self.node1, config.peers["node1"].address, config.peers["node1"].port
|
||||
)
|
||||
self.server2 = GRPCServer(
|
||||
self.node2, config.peers["node2"].address, config.peers["node2"].port
|
||||
)
|
||||
self.server1 = GRPCServer(self.node1, config.peers["node1"].address, config.peers["node1"].port)
|
||||
self.server2 = GRPCServer(self.node2, config.peers["node2"].address, config.peers["node2"].port)
|
||||
await self.server1.start()
|
||||
await self.server2.start()
|
||||
self.discovery1 = ManualDiscovery(
|
||||
|
||||
+2
-6
@@ -30,9 +30,7 @@ class TestNetworkTopologyConfig(unittest.TestCase):
|
||||
config = NetworkTopology.from_path(root_path + "test_config.json")
|
||||
|
||||
self.assertEqual(config.peers["node1"].port, 50051)
|
||||
self.assertEqual(
|
||||
config.peers["node1"].device_capabilities.model, "Unknown Model"
|
||||
)
|
||||
self.assertEqual(config.peers["node1"].device_capabilities.model, "Unknown Model")
|
||||
self.assertEqual(config.peers["node1"].address, "localhost")
|
||||
self.assertEqual(config.peers["node1"].device_capabilities.chip, "Unknown Chip")
|
||||
self.assertEqual(config.peers["node1"].device_capabilities.memory, 0)
|
||||
@@ -41,9 +39,7 @@ class TestNetworkTopologyConfig(unittest.TestCase):
|
||||
self.assertEqual(config.peers["node1"].device_capabilities.flops.int8, 0)
|
||||
|
||||
self.assertEqual(config.peers["node2"].port, 50052)
|
||||
self.assertEqual(
|
||||
config.peers["node2"].device_capabilities.model, "Unknown Model"
|
||||
)
|
||||
self.assertEqual(config.peers["node2"].device_capabilities.model, "Unknown Model")
|
||||
self.assertEqual(config.peers["node2"].address, "localhost")
|
||||
self.assertEqual(config.peers["node2"].device_capabilities.chip, "Unknown Chip")
|
||||
self.assertEqual(config.peers["node2"].device_capabilities.memory, 0)
|
||||
|
||||
@@ -65,9 +65,7 @@ class MinimalDiscovery(Discovery):
|
||||
self.is_running = False
|
||||
print(f"MinimalDiscovery stopped for node {self.node_id}")
|
||||
|
||||
async def discover_peers(
|
||||
self, wait_for_peers: int = 0
|
||||
) -> List[SimplifiedPeerHandle]:
|
||||
async def discover_peers(self, wait_for_peers: int = 0) -> List[SimplifiedPeerHandle]:
|
||||
"""Discover peers (returns empty list for now)."""
|
||||
# For testing, we don't actually discover peers
|
||||
# Real implementation would use UDP broadcast, mDNS, etc.
|
||||
|
||||
@@ -40,21 +40,15 @@ class PeerHandle(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def send_prompt(
|
||||
self, shard: Shard, prompt: str, request_id: Optional[str] = None
|
||||
) -> Optional[np.array]:
|
||||
async def send_prompt(self, shard: Shard, prompt: str, request_id: Optional[str] = None) -> Optional[np.array]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def send_tensor(
|
||||
self, shard: Shard, tensor: np.array, request_id: Optional[str] = None
|
||||
) -> Optional[np.array]:
|
||||
async def send_tensor(self, shard: Shard, tensor: np.array, request_id: Optional[str] = None) -> Optional[np.array]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def send_result(
|
||||
self, request_id: str, result: List[int], is_finished: bool
|
||||
) -> None:
|
||||
async def send_result(self, request_id: str, result: List[int], is_finished: bool) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -53,9 +53,7 @@ class TailscaleDiscovery(Discovery):
|
||||
self.device_capabilities = await device_capabilities()
|
||||
self.discovery_task = asyncio.create_task(self.task_discover_peers())
|
||||
self.cleanup_task = asyncio.create_task(self.task_cleanup_peers())
|
||||
self.update_task = asyncio.create_task(
|
||||
self.task_update_device_posture_attributes()
|
||||
)
|
||||
self.update_task = asyncio.create_task(self.task_update_device_posture_attributes())
|
||||
|
||||
async def task_update_device_posture_attributes(self):
|
||||
while True:
|
||||
@@ -87,60 +85,45 @@ class TailscaleDiscovery(Discovery):
|
||||
async def task_discover_peers(self):
|
||||
while True:
|
||||
try:
|
||||
devices: dict[str, Device] = await get_tailscale_devices(
|
||||
self.tailscale_api_key, self.tailnet
|
||||
)
|
||||
devices: dict[str, Device] = await get_tailscale_devices(self.tailscale_api_key, self.tailnet)
|
||||
current_time = time.time()
|
||||
|
||||
active_devices = {
|
||||
name: device
|
||||
for name, device in devices.items()
|
||||
if device.last_seen is not None
|
||||
and (current_time - device.last_seen.timestamp()) < 30
|
||||
if device.last_seen is not None and (current_time - device.last_seen.timestamp()) < 30
|
||||
}
|
||||
|
||||
if DEBUG_DISCOVERY >= 4:
|
||||
print(f"Found tailscale devices: {devices}")
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"Active tailscale devices: {len(active_devices)}/{len(devices)}"
|
||||
)
|
||||
print(f"Active tailscale devices: {len(active_devices)}/{len(devices)}")
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
"Time since last seen tailscale devices",
|
||||
[
|
||||
(current_time - device.last_seen.timestamp())
|
||||
for device in devices.values()
|
||||
],
|
||||
[(current_time - device.last_seen.timestamp()) for device in devices.values()],
|
||||
)
|
||||
|
||||
for device in active_devices.values():
|
||||
if device.name == self.node_id:
|
||||
continue
|
||||
peer_host = device.addresses[0]
|
||||
peer_id, peer_port, device_capabilities = (
|
||||
await get_device_attributes(
|
||||
device.device_id, self.tailscale_api_key
|
||||
)
|
||||
peer_id, peer_port, device_capabilities = await get_device_attributes(
|
||||
device.device_id, self.tailscale_api_key
|
||||
)
|
||||
if not peer_id:
|
||||
if DEBUG_DISCOVERY >= 4:
|
||||
print(
|
||||
f"{device.device_id} does not have exo node attributes. skipping."
|
||||
)
|
||||
print(f"{device.device_id} does not have exo node attributes. skipping.")
|
||||
continue
|
||||
|
||||
if self.allowed_node_ids and peer_id not in self.allowed_node_ids:
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"Ignoring peer {peer_id} as it's not in the allowed node IDs list"
|
||||
)
|
||||
print(f"Ignoring peer {peer_id} as it's not in the allowed node IDs list")
|
||||
continue
|
||||
|
||||
if (
|
||||
peer_id not in self.known_peers
|
||||
or self.known_peers[peer_id][0].addr()
|
||||
!= f"{peer_host}:{peer_port}"
|
||||
or self.known_peers[peer_id][0].addr() != f"{peer_host}:{peer_port}"
|
||||
):
|
||||
new_peer_handle = self.create_peer_handle(
|
||||
peer_id,
|
||||
@@ -150,9 +133,7 @@ class TailscaleDiscovery(Discovery):
|
||||
)
|
||||
if not await new_peer_handle.health_check():
|
||||
if DEBUG >= 1:
|
||||
print(
|
||||
f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Skipping."
|
||||
)
|
||||
print(f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Skipping.")
|
||||
continue
|
||||
|
||||
if DEBUG >= 1:
|
||||
@@ -167,9 +148,7 @@ class TailscaleDiscovery(Discovery):
|
||||
else:
|
||||
if not await self.known_peers[peer_id][0].health_check():
|
||||
if DEBUG >= 1:
|
||||
print(
|
||||
f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Removing."
|
||||
)
|
||||
print(f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Removing.")
|
||||
if peer_id in self.known_peers:
|
||||
del self.known_peers[peer_id]
|
||||
continue
|
||||
@@ -204,9 +183,7 @@ class TailscaleDiscovery(Discovery):
|
||||
if wait_for_peers > 0:
|
||||
while len(self.known_peers) < wait_for_peers:
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"Current peers: {len(self.known_peers)}/{wait_for_peers}. Waiting for more peers..."
|
||||
)
|
||||
print(f"Current peers: {len(self.known_peers)}/{wait_for_peers}. Waiting for more peers...")
|
||||
await asyncio.sleep(0.1)
|
||||
return [peer_handle for peer_handle, _, _ in self.known_peers.values()]
|
||||
|
||||
@@ -239,9 +216,7 @@ class TailscaleDiscovery(Discovery):
|
||||
if peer_id in self.known_peers:
|
||||
del self.known_peers[peer_id]
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"Removed peer {peer_id} due to inactivity or failed health check."
|
||||
)
|
||||
print(f"Removed peer {peer_id} due to inactivity or failed health check.")
|
||||
except Exception as e:
|
||||
print(f"Error in cleanup peers: {e}")
|
||||
print(traceback.format_exc())
|
||||
@@ -249,9 +224,7 @@ class TailscaleDiscovery(Discovery):
|
||||
await asyncio.sleep(self.discovery_interval)
|
||||
|
||||
async def check_peer(self, peer_id: str, current_time: float) -> bool:
|
||||
peer_handle, connected_at, last_seen = self.known_peers.get(
|
||||
peer_id, (None, None, None)
|
||||
)
|
||||
peer_handle, connected_at, last_seen = self.known_peers.get(peer_id, (None, None, None))
|
||||
if peer_handle is None:
|
||||
return False
|
||||
|
||||
|
||||
@@ -34,9 +34,7 @@ class Device:
|
||||
def parse_datetime(date_string: Optional[str]) -> Optional[datetime]:
|
||||
if not date_string:
|
||||
return None
|
||||
return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%SZ").replace(
|
||||
tzinfo=timezone.utc
|
||||
)
|
||||
return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
async def get_device_id() -> str:
|
||||
@@ -50,17 +48,13 @@ async def get_device_id() -> str:
|
||||
)
|
||||
stdout, stderr = await process.communicate()
|
||||
if process.returncode != 0:
|
||||
raise Exception(
|
||||
f"Command failed with exit code {process.returncode}: {stderr.decode().strip()}."
|
||||
)
|
||||
raise Exception(f"Command failed with exit code {process.returncode}: {stderr.decode().strip()}.")
|
||||
if DEBUG_DISCOVERY >= 4:
|
||||
print(f"tailscale status: {stdout.decode()}")
|
||||
data = json.loads(stdout.decode())
|
||||
return data["Self"]["ID"]
|
||||
except Exception as e:
|
||||
raise Exception(
|
||||
f"{str(e)} Do you have the tailscale cli installed? See: https://tailscale.com/kb/1080/cli"
|
||||
)
|
||||
raise Exception(f"{str(e)} Do you have the tailscale cli installed? See: https://tailscale.com/kb/1080/cli")
|
||||
|
||||
|
||||
async def update_device_attributes(
|
||||
@@ -80,44 +74,28 @@ async def update_device_attributes(
|
||||
attributes = {
|
||||
"custom:exo_node_id": node_id.replace("-", "_"),
|
||||
"custom:exo_node_port": node_port,
|
||||
"custom:exo_device_capability_chip": sanitize_attribute(
|
||||
device_capabilities.chip
|
||||
),
|
||||
"custom:exo_device_capability_model": sanitize_attribute(
|
||||
device_capabilities.model
|
||||
),
|
||||
"custom:exo_device_capability_chip": sanitize_attribute(device_capabilities.chip),
|
||||
"custom:exo_device_capability_model": sanitize_attribute(device_capabilities.model),
|
||||
"custom:exo_device_capability_memory": str(device_capabilities.memory),
|
||||
"custom:exo_device_capability_flops_fp16": str(
|
||||
device_capabilities.flops.fp16
|
||||
),
|
||||
"custom:exo_device_capability_flops_fp32": str(
|
||||
device_capabilities.flops.fp32
|
||||
),
|
||||
"custom:exo_device_capability_flops_int8": str(
|
||||
device_capabilities.flops.int8
|
||||
),
|
||||
"custom:exo_device_capability_flops_fp16": str(device_capabilities.flops.fp16),
|
||||
"custom:exo_device_capability_flops_fp32": str(device_capabilities.flops.fp32),
|
||||
"custom:exo_device_capability_flops_int8": str(device_capabilities.flops.int8),
|
||||
}
|
||||
|
||||
for attr_name, attr_value in attributes.items():
|
||||
url = f"{base_url}/{attr_name}"
|
||||
data = {
|
||||
"value": str(attr_value).replace(" ", "_")
|
||||
} # Ensure all values are strings for JSON
|
||||
data = {"value": str(attr_value).replace(" ", "_")} # Ensure all values are strings for JSON
|
||||
async with session.post(url, headers=headers, json=data) as response:
|
||||
if response.status == 200:
|
||||
if DEBUG_DISCOVERY >= 1:
|
||||
print(
|
||||
f"Updated device posture attribute {attr_name} for device {device_id}"
|
||||
)
|
||||
print(f"Updated device posture attribute {attr_name} for device {device_id}")
|
||||
else:
|
||||
print(
|
||||
f"Failed to update device posture attribute {attr_name}: {response.status} {await response.text()}"
|
||||
)
|
||||
|
||||
|
||||
async def get_device_attributes(
|
||||
device_id: str, api_key: str
|
||||
) -> Tuple[str, int, DeviceCapabilities]:
|
||||
async def get_device_attributes(device_id: str, api_key: str) -> Tuple[str, int, DeviceCapabilities]:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
url = f"https://api.tailscale.com/api/v2/device/{device_id}/attributes"
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
@@ -128,32 +106,18 @@ async def get_device_attributes(
|
||||
node_id = attributes.get("custom:exo_node_id", "").replace("_", "-")
|
||||
node_port = int(attributes.get("custom:exo_node_port", 0))
|
||||
device_capabilities = DeviceCapabilities(
|
||||
model=attributes.get(
|
||||
"custom:exo_device_capability_model", ""
|
||||
).replace("_", " "),
|
||||
chip=attributes.get(
|
||||
"custom:exo_device_capability_chip", ""
|
||||
).replace("_", " "),
|
||||
memory=int(
|
||||
attributes.get("custom:exo_device_capability_memory", 0)
|
||||
),
|
||||
model=attributes.get("custom:exo_device_capability_model", "").replace("_", " "),
|
||||
chip=attributes.get("custom:exo_device_capability_chip", "").replace("_", " "),
|
||||
memory=int(attributes.get("custom:exo_device_capability_memory", 0)),
|
||||
flops=DeviceFlops(
|
||||
fp16=float(
|
||||
attributes.get("custom:exo_device_capability_flops_fp16", 0)
|
||||
),
|
||||
fp32=float(
|
||||
attributes.get("custom:exo_device_capability_flops_fp32", 0)
|
||||
),
|
||||
int8=float(
|
||||
attributes.get("custom:exo_device_capability_flops_int8", 0)
|
||||
),
|
||||
fp16=float(attributes.get("custom:exo_device_capability_flops_fp16", 0)),
|
||||
fp32=float(attributes.get("custom:exo_device_capability_flops_fp32", 0)),
|
||||
int8=float(attributes.get("custom:exo_device_capability_flops_int8", 0)),
|
||||
),
|
||||
)
|
||||
return node_id, node_port, device_capabilities
|
||||
else:
|
||||
print(
|
||||
f"Failed to fetch posture attributes for {device_id}: {response.status}"
|
||||
)
|
||||
print(f"Failed to fetch posture attributes for {device_id}: {response.status}")
|
||||
return (
|
||||
"",
|
||||
0,
|
||||
|
||||
@@ -52,13 +52,9 @@ class BroadcastProtocol(asyncio.DatagramProtocol):
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
|
||||
# Try both subnet-specific and global broadcast
|
||||
broadcast_addr = get_broadcast_address(self.source_ip)
|
||||
transport.sendto(
|
||||
self.message.encode("utf-8"), (broadcast_addr, self.broadcast_port)
|
||||
)
|
||||
transport.sendto(self.message.encode("utf-8"), (broadcast_addr, self.broadcast_port))
|
||||
if broadcast_addr != "255.255.255.255":
|
||||
transport.sendto(
|
||||
self.message.encode("utf-8"), ("255.255.255.255", self.broadcast_port)
|
||||
)
|
||||
transport.sendto(self.message.encode("utf-8"), ("255.255.255.255", self.broadcast_port))
|
||||
|
||||
|
||||
class UDPDiscovery(Discovery):
|
||||
@@ -115,18 +111,14 @@ class UDPDiscovery(Discovery):
|
||||
if wait_for_peers > 0:
|
||||
while len(self.known_peers) < wait_for_peers:
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"Current peers: {len(self.known_peers)}/{wait_for_peers}. Waiting for more peers..."
|
||||
)
|
||||
print(f"Current peers: {len(self.known_peers)}/{wait_for_peers}. Waiting for more peers...")
|
||||
await asyncio.sleep(0.1)
|
||||
return [peer_handle for peer_handle, _, _, _ in self.known_peers.values()]
|
||||
|
||||
async def task_broadcast_presence(self):
|
||||
while True:
|
||||
for addr, interface_name in get_all_ip_addresses_and_interfaces():
|
||||
interface_priority, interface_type = (
|
||||
await get_interface_priority_and_type(interface_name)
|
||||
)
|
||||
interface_priority, interface_type = await get_interface_priority_and_type(interface_name)
|
||||
message = json.dumps(
|
||||
{
|
||||
"type": "discovery",
|
||||
@@ -158,9 +150,7 @@ class UDPDiscovery(Discovery):
|
||||
sock=sock,
|
||||
)
|
||||
except Exception as e:
|
||||
print(
|
||||
f"Error in broadcast presence ({addr} - {interface_name} - {interface_priority}): {e}"
|
||||
)
|
||||
print(f"Error in broadcast presence ({addr} - {interface_name} - {interface_priority}): {e}")
|
||||
finally:
|
||||
if transport:
|
||||
try:
|
||||
@@ -200,9 +190,7 @@ class UDPDiscovery(Discovery):
|
||||
# Skip if peer_id is not in allowed list
|
||||
if self.allowed_node_ids and peer_id not in self.allowed_node_ids:
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"Ignoring peer {peer_id} as it's not in the allowed node IDs list"
|
||||
)
|
||||
print(f"Ignoring peer {peer_id} as it's not in the allowed node IDs list")
|
||||
return
|
||||
|
||||
peer_host = addr[0]
|
||||
@@ -212,10 +200,7 @@ class UDPDiscovery(Discovery):
|
||||
peer_interface_type = message["interface_type"]
|
||||
|
||||
# Skip if interface type is not in allowed list
|
||||
if (
|
||||
self.allowed_interface_types
|
||||
and peer_interface_type not in self.allowed_interface_types
|
||||
):
|
||||
if self.allowed_interface_types and peer_interface_type not in self.allowed_interface_types:
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"Ignoring peer {peer_id} as its interface type {peer_interface_type} is not in the allowed interface types list"
|
||||
@@ -224,10 +209,7 @@ class UDPDiscovery(Discovery):
|
||||
|
||||
device_capabilities = DeviceCapabilities(**message["device_capabilities"])
|
||||
|
||||
if (
|
||||
peer_id not in self.known_peers
|
||||
or self.known_peers[peer_id][0].addr() != f"{peer_host}:{peer_port}"
|
||||
):
|
||||
if peer_id not in self.known_peers or self.known_peers[peer_id][0].addr() != f"{peer_host}:{peer_port}":
|
||||
if peer_id in self.known_peers:
|
||||
existing_peer_prio = self.known_peers[peer_id][3]
|
||||
if existing_peer_prio >= peer_prio:
|
||||
@@ -244,9 +226,7 @@ class UDPDiscovery(Discovery):
|
||||
)
|
||||
if not await new_peer_handle.health_check():
|
||||
if DEBUG >= 1:
|
||||
print(
|
||||
f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Skipping."
|
||||
)
|
||||
print(f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Skipping.")
|
||||
return
|
||||
if DEBUG >= 1:
|
||||
print(
|
||||
@@ -261,9 +241,7 @@ class UDPDiscovery(Discovery):
|
||||
else:
|
||||
if not await self.known_peers[peer_id][0].health_check():
|
||||
if DEBUG >= 1:
|
||||
print(
|
||||
f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Removing."
|
||||
)
|
||||
print(f"Peer {peer_id} at {peer_host}:{peer_port} is not healthy. Removing.")
|
||||
if peer_id in self.known_peers:
|
||||
del self.known_peers[peer_id]
|
||||
return
|
||||
@@ -312,9 +290,7 @@ class UDPDiscovery(Discovery):
|
||||
if peer_id in self.known_peers:
|
||||
del self.known_peers[peer_id]
|
||||
if DEBUG_DISCOVERY >= 2:
|
||||
print(
|
||||
f"Removed peer {peer_id} due to inactivity or failed health check."
|
||||
)
|
||||
print(f"Removed peer {peer_id} due to inactivity or failed health check.")
|
||||
except Exception as e:
|
||||
print(f"Error in cleanup peers: {e}")
|
||||
print(traceback.format_exc())
|
||||
@@ -322,9 +298,7 @@ class UDPDiscovery(Discovery):
|
||||
await asyncio.sleep(self.broadcast_interval)
|
||||
|
||||
async def check_peer(self, peer_id: str, current_time: float) -> bool:
|
||||
peer_handle, connected_at, last_seen, prio = self.known_peers.get(
|
||||
peer_id, (None, None, None, None)
|
||||
)
|
||||
peer_handle, connected_at, last_seen, prio = self.known_peers.get(peer_id, (None, None, None, None))
|
||||
if peer_handle is None:
|
||||
return False
|
||||
|
||||
|
||||
@@ -62,13 +62,9 @@ class BroadcastProtocol(asyncio.DatagramProtocol):
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
|
||||
# Try both subnet-specific and global broadcast
|
||||
broadcast_addr = get_broadcast_address(self.source_ip)
|
||||
transport.sendto(
|
||||
self.message.encode("utf-8"), (broadcast_addr, self.broadcast_port)
|
||||
)
|
||||
transport.sendto(self.message.encode("utf-8"), (broadcast_addr, self.broadcast_port))
|
||||
if broadcast_addr != "255.255.255.255":
|
||||
transport.sendto(
|
||||
self.message.encode("utf-8"), ("255.255.255.255", self.broadcast_port)
|
||||
)
|
||||
transport.sendto(self.message.encode("utf-8"), ("255.255.255.255", self.broadcast_port))
|
||||
transport.close()
|
||||
|
||||
|
||||
@@ -116,9 +112,7 @@ class UDPDiscovery(Discovery):
|
||||
self.listen_task = asyncio.create_task(self._listen_loop())
|
||||
self.cleanup_task = asyncio.create_task(self._cleanup_loop())
|
||||
|
||||
print(
|
||||
f"UDP Discovery started for node {self.node_id} on port {self.listen_port}"
|
||||
)
|
||||
print(f"UDP Discovery started for node {self.node_id} on port {self.listen_port}")
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Stop UDP discovery."""
|
||||
@@ -150,9 +144,7 @@ class UDPDiscovery(Discovery):
|
||||
start_time = time.time()
|
||||
while len(self.known_peers) < wait_for_peers:
|
||||
if time.time() - start_time > 30: # 30 second timeout
|
||||
print(
|
||||
f"Timeout waiting for {wait_for_peers} peers, found {len(self.known_peers)}"
|
||||
)
|
||||
print(f"Timeout waiting for {wait_for_peers} peers, found {len(self.known_peers)}")
|
||||
break
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
@@ -168,11 +160,7 @@ class UDPDiscovery(Discovery):
|
||||
"type": "discovery",
|
||||
"node_id": self.node_id,
|
||||
"listen_port": self.listen_port,
|
||||
"device_capabilities": (
|
||||
self.device_capabilities.to_dict()
|
||||
if self.device_capabilities
|
||||
else {}
|
||||
),
|
||||
"device_capabilities": (self.device_capabilities.to_dict() if self.device_capabilities else {}),
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
)
|
||||
@@ -188,9 +176,7 @@ class UDPDiscovery(Discovery):
|
||||
sock.bind((self.local_ip, 0))
|
||||
|
||||
transport, _ = await asyncio.get_event_loop().create_datagram_endpoint(
|
||||
lambda: BroadcastProtocol(
|
||||
message, self.broadcast_port, self.local_ip
|
||||
),
|
||||
lambda: BroadcastProtocol(message, self.broadcast_port, self.local_ip),
|
||||
sock=sock,
|
||||
)
|
||||
|
||||
@@ -217,9 +203,7 @@ class UDPDiscovery(Discovery):
|
||||
except Exception as e:
|
||||
print(f"Error in listen loop: {e}")
|
||||
|
||||
async def _handle_discovery_message(
|
||||
self, data: bytes, addr: Tuple[str, int]
|
||||
) -> None:
|
||||
async def _handle_discovery_message(self, data: bytes, addr: Tuple[str, int]) -> None:
|
||||
"""Handle incoming discovery message."""
|
||||
try:
|
||||
# Decode message
|
||||
@@ -252,9 +236,7 @@ class UDPDiscovery(Discovery):
|
||||
|
||||
if peer_id not in self.known_peers:
|
||||
# New peer discovered
|
||||
peer_handle = MinimalPeerHandle(
|
||||
peer_id=peer_id, address=peer_address, capabilities=peer_caps
|
||||
)
|
||||
peer_handle = MinimalPeerHandle(peer_id=peer_id, address=peer_address, capabilities=peer_caps)
|
||||
self.known_peers[peer_id] = (peer_handle, current_time, current_time)
|
||||
print(f"Discovered new peer: {peer_id} at {peer_address}")
|
||||
else:
|
||||
|
||||
@@ -97,9 +97,7 @@ class DistributedNetwork(Network[T]):
|
||||
self.sync_task = asyncio.create_task(self._sync_loop())
|
||||
|
||||
self.is_running = True
|
||||
print(
|
||||
f"Distributed network node {self.node_id} started with {len(self.peers)} peers"
|
||||
)
|
||||
print(f"Distributed network node {self.node_id} started with {len(self.peers)} peers")
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Stop the distributed network node."""
|
||||
@@ -154,9 +152,7 @@ class DistributedNetwork(Network[T]):
|
||||
peer = self._get_peer(peer_id)
|
||||
if peer:
|
||||
# Execute remotely
|
||||
return await self._execute_remote(
|
||||
peer, prompt, initial_agent, dist_context
|
||||
)
|
||||
return await self._execute_remote(peer, prompt, initial_agent, dist_context)
|
||||
|
||||
# Execute locally (using base implementation)
|
||||
return await super().run(prompt, initial_agent, dist_context)
|
||||
@@ -256,9 +252,7 @@ class DistributedNetwork(Network[T]):
|
||||
{
|
||||
"id": p.id,
|
||||
"address": p.address,
|
||||
"capabilities": (
|
||||
p.device_capabilities.__dict__ if p.device_capabilities else {}
|
||||
),
|
||||
"capabilities": (p.device_capabilities.__dict__ if p.device_capabilities else {}),
|
||||
}
|
||||
for p in self.peers
|
||||
],
|
||||
@@ -275,9 +269,7 @@ class DistributedNetwork(Network[T]):
|
||||
{
|
||||
"name": name,
|
||||
"type": agent.__class__.__name__,
|
||||
"has_tools": (
|
||||
len(agent.tools) > 0 if hasattr(agent, "tools") else False
|
||||
),
|
||||
"has_tools": (len(agent.tools) > 0 if hasattr(agent, "tools") else False),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -15,9 +15,7 @@ DEBUG_DISCOVERY = int(os.getenv("DEBUG_DISCOVERY", default="0"))
|
||||
VERSION = "0.1.0"
|
||||
|
||||
# Single shared thread pool for subprocess operations
|
||||
subprocess_pool = ThreadPoolExecutor(
|
||||
max_workers=4, thread_name_prefix="subprocess_worker"
|
||||
)
|
||||
subprocess_pool = ThreadPoolExecutor(max_workers=4, thread_name_prefix="subprocess_worker")
|
||||
|
||||
|
||||
def get_system_info():
|
||||
@@ -33,9 +31,7 @@ def get_system_info():
|
||||
return "Non-Mac, non-Linux system"
|
||||
|
||||
|
||||
def find_available_port(
|
||||
host: str = "", min_port: int = 49152, max_port: int = 65535
|
||||
) -> int:
|
||||
def find_available_port(host: str = "", min_port: int = 49152, max_port: int = 65535) -> int:
|
||||
"""Find an available port in the specified range."""
|
||||
for _ in range(100): # Try 100 times
|
||||
port = random.randint(min_port, max_port)
|
||||
@@ -82,12 +78,7 @@ async def get_interface_priority_and_type(ifname: str) -> Tuple[int, str]:
|
||||
return (6, "Loopback")
|
||||
|
||||
# Container/virtual interfaces
|
||||
if (
|
||||
ifname.startswith(
|
||||
("docker", "br-", "veth", "cni", "flannel", "calico", "weave")
|
||||
)
|
||||
or "bridge" in ifname
|
||||
):
|
||||
if ifname.startswith(("docker", "br-", "veth", "cni", "flannel", "calico", "weave")) or "bridge" in ifname:
|
||||
return (7, "Container Virtual")
|
||||
|
||||
# Thunderbolt
|
||||
@@ -115,22 +106,16 @@ async def get_mac_system_info() -> Tuple[str, str, int]:
|
||||
try:
|
||||
output = await asyncio.get_running_loop().run_in_executor(
|
||||
subprocess_pool,
|
||||
lambda: subprocess.check_output(
|
||||
["system_profiler", "SPHardwareDataType"]
|
||||
).decode("utf-8"),
|
||||
lambda: subprocess.check_output(["system_profiler", "SPHardwareDataType"]).decode("utf-8"),
|
||||
)
|
||||
|
||||
model_line = next(
|
||||
(line for line in output.split("\n") if "Model Name" in line), None
|
||||
)
|
||||
model_line = next((line for line in output.split("\n") if "Model Name" in line), None)
|
||||
model_id = model_line.split(": ")[1] if model_line else "Unknown Model"
|
||||
|
||||
chip_line = next((line for line in output.split("\n") if "Chip" in line), None)
|
||||
chip_id = chip_line.split(": ")[1] if chip_line else "Unknown Chip"
|
||||
|
||||
memory_line = next(
|
||||
(line for line in output.split("\n") if "Memory" in line), None
|
||||
)
|
||||
memory_line = next((line for line in output.split("\n") if "Memory" in line), None)
|
||||
memory_str = memory_line.split(": ")[1] if memory_line else "Unknown Memory"
|
||||
memory_units = memory_str.split()
|
||||
memory_value = int(memory_units[0])
|
||||
|
||||
@@ -19,9 +19,7 @@ class DummyInferenceEngine(InferenceEngine):
|
||||
async def encode(self, shard: Shard, prompt: str) -> np.ndarray:
|
||||
return np.array(self.tokenizer.encode(prompt))
|
||||
|
||||
async def sample(
|
||||
self, x: np.ndarray, temp: float = 0.0, top_p: float = 1.0
|
||||
) -> np.ndarray:
|
||||
async def sample(self, x: np.ndarray, temp: float = 0.0, top_p: float = 1.0) -> np.ndarray:
|
||||
if x[0] > self.num_generate_dummy_tokens:
|
||||
return np.array([self.tokenizer.eos_token_id])
|
||||
return x
|
||||
|
||||
@@ -58,9 +58,7 @@ class InferenceEngine(ABC):
|
||||
x = tokens.reshape(1, -1)
|
||||
else:
|
||||
x = tokens
|
||||
output_data, inference_state = await self.infer_tensor(
|
||||
request_id, shard, x, inference_state
|
||||
)
|
||||
output_data, inference_state = await self.infer_tensor(request_id, shard, x, inference_state)
|
||||
|
||||
return output_data, inference_state
|
||||
|
||||
|
||||
@@ -22,9 +22,7 @@ def back_gradient_loss(model, inputs, gradients, lengths):
|
||||
grad = gradients.astype(mx.float32)
|
||||
|
||||
# Mask padding tokens
|
||||
length_mask = mx.repeat(
|
||||
mx.arange(inputs.shape[1])[None, :] < lengths[:, None], out.shape[-1]
|
||||
).reshape(out.shape)
|
||||
length_mask = mx.repeat(mx.arange(inputs.shape[1])[None, :] < lengths[:, None], out.shape[-1]).reshape(out.shape)
|
||||
|
||||
masked_sum = (out * length_mask).sum(axis=1)
|
||||
gradient_lens = mx.abs(grad * masked_sum)
|
||||
|
||||
+15
-53
@@ -29,13 +29,7 @@ class DiffusionConfig:
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, params):
|
||||
return cls(
|
||||
**{
|
||||
k: v
|
||||
for k, v in params.items()
|
||||
if k in inspect.signature(cls).parameters
|
||||
}
|
||||
)
|
||||
return cls(**{k: v for k, v in params.items() if k in inspect.signature(cls).parameters})
|
||||
|
||||
|
||||
# Sampler
|
||||
@@ -66,22 +60,16 @@ class SimpleEulerSampler:
|
||||
def __init__(self, config: DiffusionConfig):
|
||||
# Compute the noise schedule
|
||||
if config.beta_schedule == "linear":
|
||||
betas = _linspace(
|
||||
config.beta_start, config.beta_end, config.num_train_steps
|
||||
)
|
||||
betas = _linspace(config.beta_start, config.beta_end, config.num_train_steps)
|
||||
elif config.beta_schedule == "scaled_linear":
|
||||
betas = _linspace(
|
||||
config.beta_start**0.5, config.beta_end**0.5, config.num_train_steps
|
||||
).square()
|
||||
betas = _linspace(config.beta_start**0.5, config.beta_end**0.5, config.num_train_steps).square()
|
||||
else:
|
||||
raise NotImplementedError(f"{config.beta_schedule} is not implemented.")
|
||||
|
||||
alphas = 1 - betas
|
||||
alphas_cumprod = mx.cumprod(alphas)
|
||||
|
||||
self._sigmas = mx.concatenate(
|
||||
[mx.zeros(1), ((1 - alphas_cumprod) / alphas_cumprod).sqrt()]
|
||||
)
|
||||
self._sigmas = mx.concatenate([mx.zeros(1), ((1 - alphas_cumprod) / alphas_cumprod).sqrt()])
|
||||
|
||||
@property
|
||||
def max_time(self):
|
||||
@@ -89,9 +77,7 @@ class SimpleEulerSampler:
|
||||
|
||||
def sample_prior(self, shape, dtype=mx.float32, key=None):
|
||||
noise = mx.random.normal(shape, key=key)
|
||||
return (
|
||||
noise * self._sigmas[-1] * (self._sigmas[-1].square() + 1).rsqrt()
|
||||
).astype(dtype)
|
||||
return (noise * self._sigmas[-1] * (self._sigmas[-1].square() + 1).rsqrt()).astype(dtype)
|
||||
|
||||
def add_noise(self, x, t, key=None):
|
||||
noise = mx.random.normal(x.shape, key=key)
|
||||
@@ -145,13 +131,7 @@ class StableDiffusionConfig:
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, params):
|
||||
return cls(
|
||||
**{
|
||||
k: v
|
||||
for k, v in params.items()
|
||||
if k in inspect.signature(cls).parameters
|
||||
}
|
||||
)
|
||||
return cls(**{k: v for k, v in params.items() if k in inspect.signature(cls).parameters})
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -163,9 +143,7 @@ class ModelArgs(StableDiffusionConfig):
|
||||
self.shard = Shard(**self.shard)
|
||||
|
||||
if not isinstance(self.shard, Shard):
|
||||
raise TypeError(
|
||||
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
|
||||
)
|
||||
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
|
||||
|
||||
|
||||
class Model(nn.Module):
|
||||
@@ -175,17 +153,13 @@ class Model(nn.Module):
|
||||
self.config = config
|
||||
self.model_path = config.vae["path"].split("/vae")[0]
|
||||
self.shard = config.shard
|
||||
self.shard_clip, self.shard_encoder, self.shard_unet, self.shard_decoder = (
|
||||
model_shards(config.shard)
|
||||
)
|
||||
self.shard_clip, self.shard_encoder, self.shard_unet, self.shard_decoder = model_shards(config.shard)
|
||||
self.config_clip = CLIPArgs.from_dict(config.text_encoder["config"])
|
||||
if self.shard_clip.start_layer != -1:
|
||||
self.text_encoder = CLIPTextModel(self.config_clip, shard=self.shard_clip)
|
||||
else:
|
||||
self.text_encoder = nn.Identity()
|
||||
self.tokenizer = load_tokenizer(
|
||||
Path(self.model_path), "vocab.json", "merges.txt"
|
||||
)
|
||||
self.tokenizer = load_tokenizer(Path(self.model_path), "vocab.json", "merges.txt")
|
||||
self.diffusion_config = DiffusionConfig.from_dict(config.scheduler["config"])
|
||||
self.sampler = SimpleEulerSampler(self.diffusion_config)
|
||||
if self.shard_unet.start_layer != -1:
|
||||
@@ -195,15 +169,11 @@ class Model(nn.Module):
|
||||
self.unet = nn.Identity()
|
||||
self.config_vae = VAEArgs.from_dict(config.vae["config"])
|
||||
if self.shard_encoder.start_layer != -1:
|
||||
self.encoder = Autoencoder(
|
||||
self.config_vae, self.shard_encoder, "vae_encoder"
|
||||
)
|
||||
self.encoder = Autoencoder(self.config_vae, self.shard_encoder, "vae_encoder")
|
||||
else:
|
||||
self.encoder = nn.Identity()
|
||||
if self.shard_decoder.start_layer != -1:
|
||||
self.decoder = Autoencoder(
|
||||
self.config_vae, self.shard_decoder, "vae_decoder"
|
||||
)
|
||||
self.decoder = Autoencoder(self.config_vae, self.shard_decoder, "vae_decoder")
|
||||
else:
|
||||
self.decoder = nn.Identity()
|
||||
|
||||
@@ -223,9 +193,7 @@ class Model(nn.Module):
|
||||
strength=0.65,
|
||||
start_step=None,
|
||||
):
|
||||
t, t_prev = self.sampler.current_timestep(
|
||||
step=step, total_steps=total_steps, start_time=start_step
|
||||
)
|
||||
t, t_prev = self.sampler.current_timestep(step=step, total_steps=total_steps, start_time=start_step)
|
||||
is_finished = False
|
||||
is_step_finished = False
|
||||
if t.item() == 1000:
|
||||
@@ -261,15 +229,11 @@ class Model(nn.Module):
|
||||
if step < total_steps:
|
||||
x = x_t_prev
|
||||
if self.shard_unet.is_first_layer():
|
||||
x_t_unet = (
|
||||
mx.concatenate([x] * 2, axis=0) if cfg_weight > 1 else x
|
||||
)
|
||||
x_t_unet = mx.concatenate([x] * 2, axis=0) if cfg_weight > 1 else x
|
||||
else:
|
||||
x_t_unet = x
|
||||
t_unet = mx.broadcast_to(t, [len(x_t_unet)])
|
||||
x, residual = self.unet(
|
||||
x_t_unet, t_unet, encoder_x=conditioning, residuals=residual
|
||||
)
|
||||
x, residual = self.unet(x_t_unet, t_unet, encoder_x=conditioning, residuals=residual)
|
||||
if self.shard_unet.is_last_layer():
|
||||
if cfg_weight > 1:
|
||||
eps_text, eps_neg = x.split(2)
|
||||
@@ -340,9 +304,7 @@ def model_shards(shard: ShardConfig):
|
||||
# Adjust the layers relative to the model's range
|
||||
relative_start = overlap_start - range_start
|
||||
relative_end = overlap_end - range_start
|
||||
shards[model_name] = Shard(
|
||||
model_name, relative_start, relative_end, range_end - range_start
|
||||
)
|
||||
shards[model_name] = Shard(model_name, relative_start, relative_end, range_end - range_start)
|
||||
else:
|
||||
# If no overlap, create a zero-layer shard
|
||||
shards[model_name] = Shard(model_name, -1, -1, range_end - range_start)
|
||||
|
||||
@@ -18,9 +18,7 @@ class ModelArgs(ModelArgs):
|
||||
if isinstance(self.shard, Shard):
|
||||
return
|
||||
if not isinstance(self.shard, dict):
|
||||
raise TypeError(
|
||||
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
|
||||
)
|
||||
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
|
||||
|
||||
self.shard = Shard(**self.shard)
|
||||
|
||||
@@ -96,11 +94,7 @@ class Model(nn.Module):
|
||||
for key, value in weights.items():
|
||||
if key.startswith("model.layers."):
|
||||
layer_num = int(key.split(".")[2])
|
||||
if (
|
||||
self.args.shard.start_layer
|
||||
<= layer_num
|
||||
<= self.args.shard.end_layer
|
||||
):
|
||||
if self.args.shard.start_layer <= layer_num <= self.args.shard.end_layer:
|
||||
shard_state_dict[key] = value
|
||||
elif (
|
||||
self.args.shard.is_first_layer()
|
||||
@@ -119,9 +113,7 @@ class Model(nn.Module):
|
||||
shard_state_dict.pop(f"{prefix}.mlp.experts.{e}.{m}.{k}")
|
||||
for e in range(self.args.n_routed_experts)
|
||||
]
|
||||
shard_state_dict[f"{prefix}.mlp.switch_mlp.{m}.{k}"] = mx.stack(
|
||||
to_join
|
||||
)
|
||||
shard_state_dict[f"{prefix}.mlp.switch_mlp.{m}.{k}"] = mx.stack(to_join)
|
||||
|
||||
return shard_state_dict
|
||||
|
||||
|
||||
@@ -21,9 +21,7 @@ class ModelArgs(V3ModelArgs):
|
||||
if isinstance(self.shard, Shard):
|
||||
return
|
||||
if not isinstance(self.shard, dict):
|
||||
raise TypeError(
|
||||
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
|
||||
)
|
||||
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
|
||||
|
||||
self.shard = Shard(**self.shard)
|
||||
|
||||
@@ -99,11 +97,7 @@ class Model(nn.Module):
|
||||
for key, value in weights.items():
|
||||
if key.startswith("model.layers."):
|
||||
layer_num = int(key.split(".")[2])
|
||||
if (
|
||||
self.args.shard.start_layer
|
||||
<= layer_num
|
||||
<= self.args.shard.end_layer
|
||||
):
|
||||
if self.args.shard.start_layer <= layer_num <= self.args.shard.end_layer:
|
||||
shard_state_dict[key] = value
|
||||
elif (
|
||||
self.args.shard.is_first_layer()
|
||||
@@ -123,9 +117,7 @@ class Model(nn.Module):
|
||||
shard_state_dict.pop(f"{prefix}.mlp.experts.{e}.{m}.{k}")
|
||||
for e in range(self.args.n_routed_experts)
|
||||
]
|
||||
shard_state_dict[f"{prefix}.mlp.switch_mlp.{m}.{k}"] = mx.stack(
|
||||
to_join
|
||||
)
|
||||
shard_state_dict[f"{prefix}.mlp.switch_mlp.{m}.{k}"] = mx.stack(to_join)
|
||||
|
||||
return shard_state_dict
|
||||
|
||||
|
||||
@@ -18,9 +18,7 @@ class ModelArgs(ModelArgs):
|
||||
if isinstance(self.shard, Shard):
|
||||
return
|
||||
if not isinstance(self.shard, dict):
|
||||
raise TypeError(
|
||||
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
|
||||
)
|
||||
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
|
||||
|
||||
self.shard = Shard(**self.shard)
|
||||
|
||||
@@ -98,11 +96,7 @@ class Model(nn.Module):
|
||||
continue
|
||||
if key.startswith("model.layers."):
|
||||
layer_num = int(key.split(".")[2])
|
||||
if (
|
||||
self.args.shard.start_layer
|
||||
<= layer_num
|
||||
<= self.args.shard.end_layer
|
||||
):
|
||||
if self.args.shard.start_layer <= layer_num <= self.args.shard.end_layer:
|
||||
shard_state_dict[key] = value
|
||||
elif (
|
||||
(self.args.shard.is_first_layer() or self.args.shard.is_last_layer())
|
||||
|
||||
@@ -20,9 +20,7 @@ class ModelArgs(ModelArgs):
|
||||
if isinstance(self.shard, Shard):
|
||||
return
|
||||
if not isinstance(self.shard, dict):
|
||||
raise TypeError(
|
||||
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
|
||||
)
|
||||
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
|
||||
|
||||
self.shard = Shard(**self.shard)
|
||||
|
||||
@@ -34,9 +32,7 @@ class LlamaModel(nn.Module):
|
||||
self.vocab_size = args.vocab_size
|
||||
self.num_hidden_layers = args.num_hidden_layers
|
||||
assert self.vocab_size > 0
|
||||
if args.shard.is_first_layer() or (
|
||||
args.shard.is_last_layer() and args.tie_word_embeddings
|
||||
):
|
||||
if args.shard.is_first_layer() or (args.shard.is_last_layer() and args.tie_word_embeddings):
|
||||
self.embed_tokens = nn.Embedding(args.vocab_size, args.hidden_size)
|
||||
self.layers = []
|
||||
for i in range(self.num_hidden_layers):
|
||||
@@ -103,21 +99,14 @@ class Model(nn.Module):
|
||||
continue
|
||||
if key.startswith("model.layers."):
|
||||
layer_num = int(key.split(".")[2])
|
||||
if (
|
||||
self.args.shard.start_layer
|
||||
<= layer_num
|
||||
<= self.args.shard.end_layer
|
||||
):
|
||||
if self.args.shard.start_layer <= layer_num <= self.args.shard.end_layer:
|
||||
shard_state_dict[key] = value
|
||||
elif (
|
||||
self.args.shard.is_first_layer()
|
||||
and key.startswith("model.embed_tokens")
|
||||
or (self.args.shard.is_last_layer() and self.args.tie_word_embeddings)
|
||||
and key.startswith("model.embed_tokens")
|
||||
or (
|
||||
self.args.shard.is_last_layer()
|
||||
and not self.args.tie_word_embeddings
|
||||
)
|
||||
or (self.args.shard.is_last_layer() and not self.args.tie_word_embeddings)
|
||||
and key.startswith("lm_head")
|
||||
or self.args.shard.is_last_layer()
|
||||
and (key.startswith("model.norm"))
|
||||
@@ -132,9 +121,7 @@ class Model(nn.Module):
|
||||
|
||||
@property
|
||||
def head_dim(self):
|
||||
return (
|
||||
self.args.head_dim or self.args.hidden_size // self.args.num_attention_heads
|
||||
)
|
||||
return self.args.head_dim or self.args.hidden_size // self.args.num_attention_heads
|
||||
|
||||
@property
|
||||
def n_kv_heads(self):
|
||||
|
||||
@@ -29,13 +29,7 @@ class VisionConfig:
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, params):
|
||||
return cls(
|
||||
**{
|
||||
k: v
|
||||
for k, v in params.items()
|
||||
if k in inspect.signature(cls).parameters
|
||||
}
|
||||
)
|
||||
return cls(**{k: v for k, v in params.items() if k in inspect.signature(cls).parameters})
|
||||
|
||||
|
||||
class VisionAttention(nn.Module):
|
||||
@@ -108,9 +102,7 @@ class VisionEncoderLayer(nn.Module):
|
||||
def __init__(self, config: VisionConfig):
|
||||
super().__init__()
|
||||
self.embed_dim = config.hidden_size
|
||||
self.self_attn = VisionAttention(
|
||||
config.hidden_size, config.num_attention_heads, bias=True
|
||||
)
|
||||
self.self_attn = VisionAttention(config.hidden_size, config.num_attention_heads, bias=True)
|
||||
self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
|
||||
self.mlp = VisionMLP(config)
|
||||
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
|
||||
@@ -127,9 +119,7 @@ class VisionEncoderLayer(nn.Module):
|
||||
class VisionEncoder(nn.Module):
|
||||
def __init__(self, config: VisionConfig):
|
||||
super().__init__()
|
||||
self.layers = [
|
||||
VisionEncoderLayer(config) for _ in range(config.num_hidden_layers)
|
||||
]
|
||||
self.layers = [VisionEncoderLayer(config) for _ in range(config.num_hidden_layers)]
|
||||
|
||||
|
||||
class VisionEmbeddings(nn.Module):
|
||||
@@ -159,9 +149,7 @@ class VisionEmbeddings(nn.Module):
|
||||
patch_embeddings = self.patch_embedding(x)
|
||||
patch_embeddings = mx.flatten(patch_embeddings, start_axis=1, end_axis=2)
|
||||
embed_dim = patch_embeddings.shape[-1]
|
||||
cls_embeddings = mx.broadcast_to(
|
||||
self.class_embedding, (batch_size, 1, embed_dim)
|
||||
)
|
||||
cls_embeddings = mx.broadcast_to(self.class_embedding, (batch_size, 1, embed_dim))
|
||||
embeddings = mx.concatenate((cls_embeddings, patch_embeddings), axis=1)
|
||||
embeddings += self.position_embedding.weight
|
||||
return embeddings
|
||||
@@ -204,9 +192,7 @@ class VisionModel(nn.Module):
|
||||
|
||||
self.vision_model = ClipVisionModel(config)
|
||||
|
||||
def __call__(
|
||||
self, x: mx.array, output_hidden_states: Optional[bool] = None
|
||||
) -> mx.array:
|
||||
def __call__(self, x: mx.array, output_hidden_states: Optional[bool] = None) -> mx.array:
|
||||
return self.vision_model(x, output_hidden_states)
|
||||
|
||||
def sanitize(self, weights):
|
||||
@@ -244,13 +230,7 @@ class TextConfig:
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, params):
|
||||
return cls(
|
||||
**{
|
||||
k: v
|
||||
for k, v in params.items()
|
||||
if k in inspect.signature(cls).parameters
|
||||
}
|
||||
)
|
||||
return cls(**{k: v for k, v in params.items() if k in inspect.signature(cls).parameters})
|
||||
|
||||
def __post_init__(self):
|
||||
if self.num_key_value_heads is None:
|
||||
@@ -291,8 +271,7 @@ class TextAttention(nn.Module):
|
||||
|
||||
rope_scale = (
|
||||
1 / config.rope_scaling["factor"]
|
||||
if config.rope_scaling is not None
|
||||
and config.rope_scaling["type"] == "linear"
|
||||
if config.rope_scaling is not None and config.rope_scaling["type"] == "linear"
|
||||
else 1
|
||||
)
|
||||
self.rope = nn.RoPE(
|
||||
@@ -325,9 +304,7 @@ class TextAttention(nn.Module):
|
||||
queries = self.rope(queries)
|
||||
keys = self.rope(keys)
|
||||
|
||||
output = mx.fast.scaled_dot_product_attention(
|
||||
queries, keys, values, scale=self.scale, mask=mask
|
||||
)
|
||||
output = mx.fast.scaled_dot_product_attention(queries, keys, values, scale=self.scale, mask=mask)
|
||||
output = output.transpose(0, 2, 1, 3).reshape(B, L, -1)
|
||||
return self.o_proj(output)
|
||||
|
||||
@@ -351,9 +328,7 @@ class TransformerBlock(nn.Module):
|
||||
self.self_attn = TextAttention(config)
|
||||
self.mlp = TextMLP(config.hidden_size, config.intermediate_size)
|
||||
self.input_layernorm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = nn.RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.post_attention_layernorm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.config = config
|
||||
|
||||
def __call__(
|
||||
@@ -427,9 +402,7 @@ class LanguageModel(nn.Module):
|
||||
super().__init__()
|
||||
self.model_type = config.model_type
|
||||
if self.model_type != "llama":
|
||||
raise ValueError(
|
||||
f"Model type {self.model_type} not supported. Currently only 'llama' is supported"
|
||||
)
|
||||
raise ValueError(f"Model type {self.model_type} not supported. Currently only 'llama' is supported")
|
||||
self.shard = shard
|
||||
self.model = Llama(config, shard)
|
||||
if self.shard.is_last_layer():
|
||||
@@ -454,19 +427,13 @@ class LanguageModel(nn.Module):
|
||||
|
||||
if key.startswith("language_model.model.layers."):
|
||||
layer_num = int(key.split(".")[3])
|
||||
if (
|
||||
layer_num < self.shard.start_layer
|
||||
or layer_num > self.shard.end_layer
|
||||
):
|
||||
if layer_num < self.shard.start_layer or layer_num > self.shard.end_layer:
|
||||
continue
|
||||
if (
|
||||
not self.shard.is_first_layer()
|
||||
and key.startswith("language_model.model.embed_tokens")
|
||||
or not self.shard.is_last_layer()
|
||||
and (
|
||||
key.startswith("language_model.model.norm")
|
||||
or key.startswith("language_model.lm_head")
|
||||
)
|
||||
and (key.startswith("language_model.model.norm") or key.startswith("language_model.lm_head"))
|
||||
):
|
||||
continue
|
||||
|
||||
@@ -508,9 +475,7 @@ class ModelArgs(LlaVAConfig):
|
||||
self.shard = Shard(**self.shard)
|
||||
|
||||
if not isinstance(self.shard, Shard):
|
||||
raise TypeError(
|
||||
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
|
||||
)
|
||||
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
|
||||
|
||||
if not self.shard.is_first_layer():
|
||||
self.vision_config = None
|
||||
@@ -519,13 +484,9 @@ class ModelArgs(LlaVAConfig):
|
||||
class LlavaMultiModalProjector(nn.Module):
|
||||
def __init__(self, config: LlaVAConfig):
|
||||
super().__init__()
|
||||
self.linear_1 = nn.Linear(
|
||||
config.vision_config.hidden_size, config.text_config.hidden_size, bias=True
|
||||
)
|
||||
self.linear_1 = nn.Linear(config.vision_config.hidden_size, config.text_config.hidden_size, bias=True)
|
||||
self.gelu = nn.GELU()
|
||||
self.linear_2 = nn.Linear(
|
||||
config.text_config.hidden_size, config.text_config.hidden_size, bias=True
|
||||
)
|
||||
self.linear_2 = nn.Linear(config.text_config.hidden_size, config.text_config.hidden_size, bias=True)
|
||||
|
||||
def __call__(self, x: mx.array) -> mx.array:
|
||||
x = self.linear_1(x)
|
||||
@@ -558,9 +519,7 @@ class Model(nn.Module):
|
||||
inputs_embeds = self.language_model.model.embed_tokens(input_ids)
|
||||
|
||||
# Get the ouptut hidden states from the vision model
|
||||
*_, hidden_states = self.vision_tower(
|
||||
pixel_values.transpose(0, 2, 3, 1), output_hidden_states=True
|
||||
)
|
||||
*_, hidden_states = self.vision_tower(pixel_values.transpose(0, 2, 3, 1), output_hidden_states=True)
|
||||
|
||||
# Select the hidden states from the desired layer
|
||||
selected_image_feature = hidden_states[self.vision_feature_layer]
|
||||
@@ -570,22 +529,16 @@ class Model(nn.Module):
|
||||
elif self.vision_feature_select_strategy == "full":
|
||||
selected_image_feature = selected_image_feature
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unexpected feature selection strategy: {self.vision_feature_select_strategy}"
|
||||
)
|
||||
raise ValueError(f"Unexpected feature selection strategy: {self.vision_feature_select_strategy}")
|
||||
|
||||
# Pass image features through the multi-modal projector
|
||||
image_features = self.multi_modal_projector(selected_image_feature)
|
||||
|
||||
# Insert special image tokens in the input_ids
|
||||
final_inputs_embeds = self._merge_input_ids_with_image_features(
|
||||
image_features, inputs_embeds, input_ids
|
||||
)
|
||||
final_inputs_embeds = self._merge_input_ids_with_image_features(image_features, inputs_embeds, input_ids)
|
||||
return final_inputs_embeds
|
||||
|
||||
def _merge_input_ids_with_image_features(
|
||||
self, image_features, inputs_embeds, input_ids
|
||||
):
|
||||
def _merge_input_ids_with_image_features(self, image_features, inputs_embeds, input_ids):
|
||||
image_token_index = self.config.image_token_index
|
||||
num_images, num_image_patches, embed_dim = image_features.shape
|
||||
|
||||
@@ -617,9 +570,7 @@ class Model(nn.Module):
|
||||
input_embddings = None
|
||||
if pixel_values is not None:
|
||||
input_embddings = self.get_input_embeddings(input_ids, pixel_values)
|
||||
logits = self.language_model(
|
||||
input_ids, cache=cache, inputs_embeds=input_embddings
|
||||
)
|
||||
logits = self.language_model(input_ids, cache=cache, inputs_embeds=input_embddings)
|
||||
return logits
|
||||
|
||||
def sanitize(self, weights):
|
||||
@@ -649,8 +600,7 @@ class Model(nn.Module):
|
||||
def head_dim(self):
|
||||
return (
|
||||
self.language_model.model.head_dim
|
||||
or self.language_model.model.hidden_size
|
||||
// self.language_model.model.num_attention_heads
|
||||
or self.language_model.model.hidden_size // self.language_model.model.num_attention_heads
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
@@ -20,9 +20,7 @@ class ModelArgs(ModelArgs):
|
||||
if isinstance(self.shard, Shard):
|
||||
return
|
||||
if not isinstance(self.shard, dict):
|
||||
raise TypeError(
|
||||
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
|
||||
)
|
||||
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
|
||||
|
||||
self.shard = Shard(**self.shard)
|
||||
|
||||
@@ -100,11 +98,7 @@ class Model(nn.Module):
|
||||
continue
|
||||
if key.startswith("model.layers."):
|
||||
layer_num = int(key.split(".")[2])
|
||||
if (
|
||||
self.args.shard.start_layer
|
||||
<= layer_num
|
||||
<= self.args.shard.end_layer
|
||||
):
|
||||
if self.args.shard.start_layer <= layer_num <= self.args.shard.end_layer:
|
||||
shard_state_dict[key] = value
|
||||
elif (
|
||||
self.args.shard.is_first_layer()
|
||||
|
||||
@@ -20,9 +20,7 @@ class ModelArgs(ModelArgs):
|
||||
if isinstance(self.shard, Shard):
|
||||
return
|
||||
if not isinstance(self.shard, dict):
|
||||
raise TypeError(
|
||||
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
|
||||
)
|
||||
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
|
||||
|
||||
self.shard = Shard(**self.shard)
|
||||
|
||||
@@ -35,9 +33,7 @@ class Qwen2Model(nn.Module):
|
||||
self.num_hidden_layers = args.num_hidden_layers
|
||||
assert self.vocab_size > 0
|
||||
|
||||
if self.args.shard.is_first_layer() or (
|
||||
self.args.shard.is_last_layer() and args.tie_word_embeddings
|
||||
):
|
||||
if self.args.shard.is_first_layer() or (self.args.shard.is_last_layer() and args.tie_word_embeddings):
|
||||
self.embed_tokens = nn.Embedding(args.vocab_size, args.hidden_size)
|
||||
|
||||
self.layers = []
|
||||
@@ -106,21 +102,14 @@ class Model(nn.Module):
|
||||
continue
|
||||
if key.startswith("model.layers."):
|
||||
layer_num = int(key.split(".")[2])
|
||||
if (
|
||||
self.args.shard.start_layer
|
||||
<= layer_num
|
||||
<= self.args.shard.end_layer
|
||||
):
|
||||
if self.args.shard.start_layer <= layer_num <= self.args.shard.end_layer:
|
||||
shard_state_dict[key] = value
|
||||
elif (
|
||||
self.args.shard.is_first_layer()
|
||||
and key.startswith("model.embed_tokens")
|
||||
or (self.args.shard.is_last_layer() and self.args.tie_word_embeddings)
|
||||
and key.startswith("model.embed_tokens")
|
||||
or (
|
||||
self.args.shard.is_last_layer()
|
||||
and not self.args.tie_word_embeddings
|
||||
)
|
||||
or (self.args.shard.is_last_layer() and not self.args.tie_word_embeddings)
|
||||
and key.startswith("lm_head")
|
||||
or self.args.shard.is_last_layer()
|
||||
and (key.startswith("model.norm"))
|
||||
|
||||
@@ -31,11 +31,7 @@ class CLIPTextModelConfig:
|
||||
num_heads=config["num_attention_heads"],
|
||||
max_length=config["max_position_embeddings"],
|
||||
vocab_size=config["vocab_size"],
|
||||
projection_dim=(
|
||||
config["projection_dim"]
|
||||
if "WithProjection" in config["architectures"][0]
|
||||
else None
|
||||
),
|
||||
projection_dim=(config["projection_dim"] if "WithProjection" in config["architectures"][0] else None),
|
||||
hidden_act=config.get("hidden_act", "quick_gelu"),
|
||||
weight_files=config.get("weight_files", []),
|
||||
)
|
||||
@@ -51,9 +47,7 @@ class ModelArgs(CLIPTextModelConfig):
|
||||
self.shard = Shard(**self.shard)
|
||||
|
||||
if not isinstance(self.shard, Shard):
|
||||
raise TypeError(
|
||||
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
|
||||
)
|
||||
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
|
||||
|
||||
if not self.shard.is_first_layer():
|
||||
self.vision_config = None
|
||||
@@ -106,35 +100,23 @@ class CLIPTextModel(nn.Module):
|
||||
super().__init__()
|
||||
|
||||
self.shard = shard
|
||||
self.layers_range = range(
|
||||
self.shard.start_layer * 2, self.shard.end_layer * 2 + 2
|
||||
)
|
||||
self.layers_range = range(self.shard.start_layer * 2, self.shard.end_layer * 2 + 2)
|
||||
if self.shard.is_first_layer():
|
||||
self.token_embedding = nn.Embedding(config.vocab_size, config.model_dims)
|
||||
self.position_embedding = nn.Embedding(config.max_length, config.model_dims)
|
||||
self.layers = []
|
||||
for i in range(math.ceil(config.num_layers / 2)):
|
||||
if 2 * i in self.layers_range:
|
||||
self.layers.append(
|
||||
CLIPEncoderLayer(
|
||||
config.model_dims, config.num_heads, config.hidden_act
|
||||
)
|
||||
)
|
||||
self.layers.append(CLIPEncoderLayer(config.model_dims, config.num_heads, config.hidden_act))
|
||||
if 2 * i + 1 in self.layers_range and 2 * i + 1 < config.num_layers:
|
||||
self.layers.append(
|
||||
CLIPEncoderLayer(
|
||||
config.model_dims, config.num_heads, config.hidden_act
|
||||
)
|
||||
)
|
||||
self.layers.append(CLIPEncoderLayer(config.model_dims, config.num_heads, config.hidden_act))
|
||||
else:
|
||||
self.layers.append(IdentityBlock())
|
||||
if self.shard.is_last_layer():
|
||||
self.final_layer_norm = nn.LayerNorm(config.model_dims)
|
||||
|
||||
if config.projection_dim is not None:
|
||||
self.text_projection = nn.Linear(
|
||||
config.model_dims, config.projection_dim, bias=False
|
||||
)
|
||||
self.text_projection = nn.Linear(config.model_dims, config.projection_dim, bias=False)
|
||||
|
||||
def _get_mask(self, N, dtype):
|
||||
indices = mx.arange(N)
|
||||
|
||||
@@ -49,9 +49,7 @@ class Tokenizer:
|
||||
#
|
||||
# Ported from https://github.com/huggingface/transformers/blob/main/src/transformers/models/clip/tokenization_clip.py
|
||||
while unique_bigrams:
|
||||
bigram = min(
|
||||
unique_bigrams, key=lambda pair: self.bpe_ranks.get(pair, float("inf"))
|
||||
)
|
||||
bigram = min(unique_bigrams, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
|
||||
if bigram not in self.bpe_ranks:
|
||||
break
|
||||
|
||||
|
||||
@@ -49,9 +49,7 @@ class UNetConfig:
|
||||
out_channels=config["out_channels"],
|
||||
block_out_channels=config["block_out_channels"],
|
||||
layers_per_block=[config["layers_per_block"]] * n_blocks,
|
||||
transformer_layers_per_block=config.get(
|
||||
"transformer_layers_per_block", (1,) * 4
|
||||
),
|
||||
transformer_layers_per_block=config.get("transformer_layers_per_block", (1,) * 4),
|
||||
num_attention_heads=(
|
||||
[config["attention_head_dim"]] * n_blocks
|
||||
if isinstance(config["attention_head_dim"], int)
|
||||
@@ -63,9 +61,7 @@ class UNetConfig:
|
||||
up_block_types=config["up_block_types"][::-1],
|
||||
addition_embed_type=config.get("addition_embed_type", None),
|
||||
addition_time_embed_dim=config.get("addition_time_embed_dim", None),
|
||||
projection_class_embeddings_input_dim=config.get(
|
||||
"projection_class_embeddings_input_dim", None
|
||||
),
|
||||
projection_class_embeddings_input_dim=config.get("projection_class_embeddings_input_dim", None),
|
||||
weight_files=config.get("weight_files", []),
|
||||
)
|
||||
|
||||
@@ -109,9 +105,7 @@ class TransformerBlock(nn.Module):
|
||||
|
||||
memory_dims = memory_dims or model_dims
|
||||
self.norm2 = nn.LayerNorm(model_dims)
|
||||
self.attn2 = nn.MultiHeadAttention(
|
||||
model_dims, num_heads, key_input_dims=memory_dims
|
||||
)
|
||||
self.attn2 = nn.MultiHeadAttention(model_dims, num_heads, key_input_dims=memory_dims)
|
||||
self.attn2.out_proj.bias = mx.zeros(model_dims)
|
||||
|
||||
hidden_dims = hidden_dims or 4 * model_dims
|
||||
@@ -159,8 +153,7 @@ class Transformer2D(nn.Module):
|
||||
self.norm = nn.GroupNorm(norm_num_groups, in_channels, pytorch_compatible=True)
|
||||
self.proj_in = nn.Linear(in_channels, model_dims)
|
||||
self.transformer_blocks = [
|
||||
TransformerBlock(model_dims, num_heads, memory_dims=encoder_dims)
|
||||
for i in range(num_layers)
|
||||
TransformerBlock(model_dims, num_heads, memory_dims=encoder_dims) for i in range(num_layers)
|
||||
]
|
||||
self.proj_out = nn.Linear(model_dims, in_channels)
|
||||
|
||||
@@ -198,15 +191,11 @@ class ResnetBlock2D(nn.Module):
|
||||
out_channels = out_channels or in_channels
|
||||
|
||||
self.norm1 = nn.GroupNorm(groups, in_channels, pytorch_compatible=True)
|
||||
self.conv1 = nn.Conv2d(
|
||||
in_channels, out_channels, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
|
||||
if temb_channels is not None:
|
||||
self.time_emb_proj = nn.Linear(temb_channels, out_channels)
|
||||
self.norm2 = nn.GroupNorm(groups, out_channels, pytorch_compatible=True)
|
||||
self.conv2 = nn.Conv2d(
|
||||
out_channels, out_channels, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
|
||||
|
||||
if in_channels != out_channels:
|
||||
self.conv_shortcut = nn.Linear(in_channels, out_channels)
|
||||
@@ -256,9 +245,7 @@ class UNetBlock2D(nn.Module):
|
||||
else:
|
||||
in_channels_list = [prev_out_channels] + [out_channels] * (num_layers - 1)
|
||||
res_channels_list = [out_channels] * (num_layers - 1) + [in_channels]
|
||||
in_channels_list = [
|
||||
a + b for a, b in zip(in_channels_list, res_channels_list)
|
||||
]
|
||||
in_channels_list = [a + b for a, b in zip(in_channels_list, res_channels_list)]
|
||||
|
||||
# Add resnet blocks that also process the time embedding
|
||||
self.resnets = [
|
||||
@@ -286,15 +273,11 @@ class UNetBlock2D(nn.Module):
|
||||
|
||||
# Add an optional downsampling layer
|
||||
if add_downsample:
|
||||
self.downsample = nn.Conv2d(
|
||||
out_channels, out_channels, kernel_size=3, stride=2, padding=1
|
||||
)
|
||||
self.downsample = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=2, padding=1)
|
||||
|
||||
# or upsampling layer
|
||||
if add_upsample:
|
||||
self.upsample = nn.Conv2d(
|
||||
out_channels, out_channels, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
self.upsample = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
@@ -349,9 +332,7 @@ class UNetModel(nn.Module):
|
||||
self.timesteps = nn.SinusoidalPositionalEncoding(
|
||||
config.block_out_channels[0],
|
||||
max_freq=1,
|
||||
min_freq=math.exp(
|
||||
-math.log(10000) + 2 * math.log(10000) / config.block_out_channels[0]
|
||||
),
|
||||
min_freq=math.exp(-math.log(10000) + 2 * math.log(10000) / config.block_out_channels[0]),
|
||||
scale=1.0,
|
||||
cos_first=True,
|
||||
full_turns=False,
|
||||
@@ -365,10 +346,7 @@ class UNetModel(nn.Module):
|
||||
self.add_time_proj = nn.SinusoidalPositionalEncoding(
|
||||
config.addition_time_embed_dim,
|
||||
max_freq=1,
|
||||
min_freq=math.exp(
|
||||
-math.log(10000)
|
||||
+ 2 * math.log(10000) / config.addition_time_embed_dim
|
||||
),
|
||||
min_freq=math.exp(-math.log(10000) + 2 * math.log(10000) / config.addition_time_embed_dim),
|
||||
scale=1.0,
|
||||
cos_first=True,
|
||||
full_turns=False,
|
||||
@@ -379,14 +357,10 @@ class UNetModel(nn.Module):
|
||||
)
|
||||
|
||||
# Make the downsampling blocks
|
||||
block_channels = [config.block_out_channels[0]] + list(
|
||||
config.block_out_channels
|
||||
)
|
||||
block_channels = [config.block_out_channels[0]] + list(config.block_out_channels)
|
||||
self.down_blocks = []
|
||||
|
||||
for i, (in_channels, out_channels) in enumerate(
|
||||
zip(block_channels, block_channels[1:])
|
||||
):
|
||||
for i, (in_channels, out_channels) in enumerate(zip(block_channels, block_channels[1:])):
|
||||
if i in self.layers_range:
|
||||
self.down_blocks.append(
|
||||
UNetBlock2D(
|
||||
@@ -394,9 +368,7 @@ class UNetModel(nn.Module):
|
||||
out_channels=out_channels,
|
||||
temb_channels=config.block_out_channels[0] * 4,
|
||||
num_layers=config.layers_per_block[i],
|
||||
transformer_layers_per_block=config.transformer_layers_per_block[
|
||||
i
|
||||
],
|
||||
transformer_layers_per_block=config.transformer_layers_per_block[i],
|
||||
num_attention_heads=config.num_attention_heads[i],
|
||||
cross_attention_dim=config.cross_attention_dim[i],
|
||||
resnet_groups=config.norm_num_groups,
|
||||
@@ -434,20 +406,14 @@ class UNetModel(nn.Module):
|
||||
|
||||
# Make the upsampling blocks
|
||||
block_channels = (
|
||||
[config.block_out_channels[0]]
|
||||
+ list(config.block_out_channels)
|
||||
+ [config.block_out_channels[-1]]
|
||||
[config.block_out_channels[0]] + list(config.block_out_channels) + [config.block_out_channels[-1]]
|
||||
)
|
||||
|
||||
total_items = len(block_channels) - 3
|
||||
reversed_channels = list(
|
||||
reversed(list(zip(block_channels, block_channels[1:], block_channels[2:])))
|
||||
)
|
||||
reversed_channels = list(reversed(list(zip(block_channels, block_channels[1:], block_channels[2:]))))
|
||||
|
||||
self.up_blocks = []
|
||||
for rev_i, (in_channels, out_channels, prev_out_channels) in enumerate(
|
||||
reversed_channels
|
||||
):
|
||||
for rev_i, (in_channels, out_channels, prev_out_channels) in enumerate(reversed_channels):
|
||||
i = total_items - rev_i
|
||||
if rev_i + 5 in self.layers_range:
|
||||
self.up_blocks.append(
|
||||
@@ -457,9 +423,7 @@ class UNetModel(nn.Module):
|
||||
temb_channels=config.block_out_channels[0] * 4,
|
||||
prev_out_channels=prev_out_channels,
|
||||
num_layers=config.layers_per_block[i] + 1,
|
||||
transformer_layers_per_block=config.transformer_layers_per_block[
|
||||
i
|
||||
],
|
||||
transformer_layers_per_block=config.transformer_layers_per_block[i],
|
||||
num_attention_heads=config.num_attention_heads[i],
|
||||
cross_attention_dim=config.cross_attention_dim[i],
|
||||
resnet_groups=config.norm_num_groups,
|
||||
|
||||
@@ -28,13 +28,7 @@ class AutoencoderConfig:
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, params):
|
||||
return cls(
|
||||
**{
|
||||
k: v
|
||||
for k, v in params.items()
|
||||
if k in inspect.signature(cls).parameters
|
||||
}
|
||||
)
|
||||
return cls(**{k: v for k, v in params.items() if k in inspect.signature(cls).parameters})
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -46,9 +40,7 @@ class ModelArgs(AutoencoderConfig):
|
||||
self.shard = Shard(**self.shard)
|
||||
|
||||
if not isinstance(self.shard, Shard):
|
||||
raise TypeError(
|
||||
f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead"
|
||||
)
|
||||
raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
|
||||
|
||||
if not self.shard.is_first_layer():
|
||||
self.vision_config = None
|
||||
@@ -110,15 +102,11 @@ class EncoderDecoderBlock2D(nn.Module):
|
||||
|
||||
# Add an optional downsampling layer
|
||||
if add_downsample:
|
||||
self.downsample = nn.Conv2d(
|
||||
out_channels, out_channels, kernel_size=3, stride=2, padding=0
|
||||
)
|
||||
self.downsample = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=2, padding=0)
|
||||
|
||||
# or upsampling layer
|
||||
if add_upsample:
|
||||
self.upsample = nn.Conv2d(
|
||||
out_channels, out_channels, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
self.upsample = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
|
||||
|
||||
def __call__(self, x):
|
||||
for resnet in self.resnets:
|
||||
@@ -149,9 +137,7 @@ class Encoder(nn.Module):
|
||||
self.layers_range = layers_range
|
||||
self.shard = shard
|
||||
if self.shard.is_first_layer():
|
||||
self.conv_in = nn.Conv2d(
|
||||
in_channels, block_out_channels[0], kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
self.conv_in = nn.Conv2d(in_channels, block_out_channels[0], kernel_size=3, stride=1, padding=1)
|
||||
|
||||
channels = [block_out_channels[0]] + list(block_out_channels)
|
||||
self.down_blocks = []
|
||||
@@ -187,12 +173,8 @@ class Encoder(nn.Module):
|
||||
),
|
||||
]
|
||||
|
||||
self.conv_norm_out = nn.GroupNorm(
|
||||
resnet_groups, block_out_channels[-1], pytorch_compatible=True
|
||||
)
|
||||
self.conv_out = nn.Conv2d(
|
||||
block_out_channels[-1], latent_channels_out, 3, padding=1
|
||||
)
|
||||
self.conv_norm_out = nn.GroupNorm(resnet_groups, block_out_channels[-1], pytorch_compatible=True)
|
||||
self.conv_out = nn.Conv2d(block_out_channels[-1], latent_channels_out, 3, padding=1)
|
||||
|
||||
def __call__(self, x):
|
||||
if self.shard.is_first_layer():
|
||||
@@ -230,9 +212,7 @@ class Decoder(nn.Module):
|
||||
self.out_channels = out_channels
|
||||
self.layers_range = layer_range
|
||||
if 0 in layer_range:
|
||||
self.conv_in = nn.Conv2d(
|
||||
in_channels, block_out_channels[-1], kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
self.conv_in = nn.Conv2d(in_channels, block_out_channels[-1], kernel_size=3, stride=1, padding=1)
|
||||
|
||||
if 0 in layer_range:
|
||||
self.mid_blocks = [
|
||||
@@ -271,12 +251,8 @@ class Decoder(nn.Module):
|
||||
self.up_blocks.append(IdentityBlock())
|
||||
current_layer += 1
|
||||
if 4 in layer_range:
|
||||
self.conv_norm_out = nn.GroupNorm(
|
||||
resnet_groups, block_out_channels[0], pytorch_compatible=True
|
||||
)
|
||||
self.conv_out = nn.Conv2d(
|
||||
block_out_channels[0], self.out_channels, 3, padding=1
|
||||
)
|
||||
self.conv_norm_out = nn.GroupNorm(resnet_groups, block_out_channels[0], pytorch_compatible=True)
|
||||
self.conv_out = nn.Conv2d(block_out_channels[0], self.out_channels, 3, padding=1)
|
||||
|
||||
def __call__(self, x):
|
||||
if 0 in self.layers_range:
|
||||
@@ -317,9 +293,7 @@ class Autoencoder(nn.Module):
|
||||
shard=shard,
|
||||
)
|
||||
if self.shard.is_last_layer():
|
||||
self.quant_proj = nn.Linear(
|
||||
config.latent_channels_out, config.latent_channels_out
|
||||
)
|
||||
self.quant_proj = nn.Linear(config.latent_channels_out, config.latent_channels_out)
|
||||
if self.model_shard == "vae_decoder":
|
||||
self.decoder = Decoder(
|
||||
config.latent_channels_in,
|
||||
@@ -331,9 +305,7 @@ class Autoencoder(nn.Module):
|
||||
resnet_groups=config.norm_num_groups,
|
||||
)
|
||||
if self.shard.is_first_layer():
|
||||
self.post_quant_proj = nn.Linear(
|
||||
config.latent_channels_in, config.latent_channels_in
|
||||
)
|
||||
self.post_quant_proj = nn.Linear(config.latent_channels_in, config.latent_channels_in)
|
||||
|
||||
def decode(self, z):
|
||||
if self.shard.is_first_layer():
|
||||
|
||||
@@ -23,16 +23,12 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
|
||||
self.sampler_params: tuple[float, float] = (0.0, 0.0, 0.0, 1)
|
||||
self.sampler = make_sampler(*self.sampler_params)
|
||||
self._mlx_thread = ThreadPoolExecutor(max_workers=1, thread_name_prefix="mlx")
|
||||
self._tokenizer_thread = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="tokenizer"
|
||||
)
|
||||
self._tokenizer_thread = ThreadPoolExecutor(max_workers=1, thread_name_prefix="tokenizer")
|
||||
self.session = {}
|
||||
self._shard_lock = asyncio.Lock()
|
||||
|
||||
async def _eval_mlx(self, *args):
|
||||
await asyncio.get_running_loop().run_in_executor(
|
||||
self._mlx_thread, mx.eval, *args
|
||||
)
|
||||
await asyncio.get_running_loop().run_in_executor(self._mlx_thread, mx.eval, *args)
|
||||
|
||||
async def poll_state(self, request_id: str, max_caches=2):
|
||||
if request_id in self.caches:
|
||||
@@ -44,9 +40,7 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
|
||||
self.caches[request_id] = newcache
|
||||
return {"cache": self.caches[request_id]}
|
||||
|
||||
async def sample(
|
||||
self, x: np.ndarray, temp: float = 0.0, top_p: float = 1.0
|
||||
) -> np.ndarray:
|
||||
async def sample(self, x: np.ndarray, temp: float = 0.0, top_p: float = 1.0) -> np.ndarray:
|
||||
if (temp, top_p, 0.0, 1) != self.sampler_params:
|
||||
self.sampler_params = (temp, top_p, 0.0, 1)
|
||||
self.sampler = make_sampler(*self.sampler_params)
|
||||
@@ -60,28 +54,20 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
|
||||
async def encode(self, shard: Shard, prompt: str) -> np.ndarray:
|
||||
await self.ensure_shard(shard)
|
||||
return np.asarray(
|
||||
await asyncio.get_running_loop().run_in_executor(
|
||||
self._tokenizer_thread, self.tokenizer.encode, prompt
|
||||
)
|
||||
await asyncio.get_running_loop().run_in_executor(self._tokenizer_thread, self.tokenizer.encode, prompt)
|
||||
)
|
||||
|
||||
async def decode(self, shard: Shard, tokens) -> str:
|
||||
await self.ensure_shard(shard)
|
||||
return await asyncio.get_running_loop().run_in_executor(
|
||||
self._tokenizer_thread, self.tokenizer.decode, tokens
|
||||
)
|
||||
return await asyncio.get_running_loop().run_in_executor(self._tokenizer_thread, self.tokenizer.decode, tokens)
|
||||
|
||||
async def save_checkpoint(self, shard: Shard, path: str):
|
||||
await self.ensure_shard(shard)
|
||||
await asyncio.get_running_loop().run_in_executor(
|
||||
self._mlx_thread, lambda: self.model.save_weights(path)
|
||||
)
|
||||
await asyncio.get_running_loop().run_in_executor(self._mlx_thread, lambda: self.model.save_weights(path))
|
||||
|
||||
async def load_checkpoint(self, shard: Shard, path: str):
|
||||
await self.ensure_shard(shard)
|
||||
await asyncio.get_running_loop().run_in_executor(
|
||||
self._mlx_thread, lambda: self.model.load_weights(path)
|
||||
)
|
||||
await asyncio.get_running_loop().run_in_executor(self._mlx_thread, lambda: self.model.load_weights(path))
|
||||
|
||||
async def infer_tensor(
|
||||
self,
|
||||
@@ -91,11 +77,7 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
|
||||
inference_state: Optional[dict] = None,
|
||||
) -> tuple[np.ndarray, Optional[dict]]:
|
||||
await self.ensure_shard(shard)
|
||||
state = (
|
||||
await self.poll_state(request_id)
|
||||
if self.model.model_type != "StableDiffusionPipeline"
|
||||
else {}
|
||||
)
|
||||
state = await self.poll_state(request_id) if self.model.model_type != "StableDiffusionPipeline" else {}
|
||||
x = mx.array(input_data)
|
||||
|
||||
if self.model.model_type != "StableDiffusionPipeline":
|
||||
@@ -147,37 +129,22 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
|
||||
):
|
||||
await self.ensure_shard(shard)
|
||||
|
||||
if (
|
||||
"train_layers" not in self.session
|
||||
or self.session["train_layers"] != trainable_layers
|
||||
):
|
||||
if "train_layers" not in self.session or self.session["train_layers"] != trainable_layers:
|
||||
await self.save_session("train_layers", trainable_layers)
|
||||
|
||||
def freeze_unfreeze():
|
||||
self.model.freeze()
|
||||
self.model.apply_to_modules(
|
||||
lambda k, v: (
|
||||
v.unfreeze()
|
||||
if any(
|
||||
k.endswith(layer_name) for layer_name in trainable_layers
|
||||
)
|
||||
else None
|
||||
v.unfreeze() if any(k.endswith(layer_name) for layer_name in trainable_layers) else None
|
||||
)
|
||||
)
|
||||
|
||||
await asyncio.get_running_loop().run_in_executor(
|
||||
self._mlx_thread, freeze_unfreeze
|
||||
)
|
||||
await asyncio.get_running_loop().run_in_executor(self._mlx_thread, freeze_unfreeze)
|
||||
|
||||
if (
|
||||
"lossname" not in self.session
|
||||
or "LVaG" not in self.session
|
||||
or self.session["lossname"] != loss
|
||||
):
|
||||
if "lossname" not in self.session or "LVaG" not in self.session or self.session["lossname"] != loss:
|
||||
await self.save_session("lossname", loss)
|
||||
await self.save_session(
|
||||
"LVaG", nn.value_and_grad(self.model, loss_fns[loss])
|
||||
)
|
||||
await self.save_session("LVaG", nn.value_and_grad(self.model, loss_fns[loss]))
|
||||
|
||||
if "opt" not in self.session:
|
||||
await self.save_session("opt", opt(lr))
|
||||
@@ -214,11 +181,7 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
|
||||
)
|
||||
await self._eval_mlx(*eval_args)
|
||||
|
||||
layers = [
|
||||
{k: v["weight"] for k, v in layer.items() if "weight" in v}
|
||||
for layer in gradients
|
||||
if layer
|
||||
]
|
||||
layers = [{k: v["weight"] for k, v in layer.items() if "weight" in v} for layer in gradients if layer]
|
||||
first_layer = np.array(layers[0]["input_layernorm"], copy=False)
|
||||
await self._eval_mlx(first_layer)
|
||||
return score, first_layer
|
||||
@@ -227,9 +190,7 @@ class MLXDynamicShardInferenceEngine(InferenceEngine):
|
||||
async with self._shard_lock:
|
||||
if self.shard == shard:
|
||||
return
|
||||
model_path = await self.shard_downloader.ensure_shard(
|
||||
shard, self.__class__.__name__
|
||||
)
|
||||
model_path = await self.shard_downloader.ensure_shard(shard, self.__class__.__name__)
|
||||
if self.shard != shard:
|
||||
model_shard = await asyncio.get_running_loop().run_in_executor(
|
||||
self._mlx_thread,
|
||||
|
||||
@@ -233,9 +233,7 @@ async def get_image_from_str(_image_str: str):
|
||||
|
||||
return img
|
||||
else:
|
||||
raise ValueError(
|
||||
"Invalid image_str format. Must be a URL or a base64 encoded image."
|
||||
)
|
||||
raise ValueError("Invalid image_str format. Must be a URL or a base64 encoded image.")
|
||||
|
||||
|
||||
# loading a combined config for all models in the index
|
||||
@@ -256,13 +254,7 @@ def load_model_index(model_path: Path, model_index_path: Path):
|
||||
model_config["path"] = model_path / model
|
||||
if model_config["path"] / "*model.safetensors":
|
||||
model_config["config"].update(
|
||||
{
|
||||
"weight_files": list(
|
||||
glob.glob(
|
||||
str(model_config["path"] / "*model.safetensors")
|
||||
)
|
||||
)
|
||||
}
|
||||
{"weight_files": list(glob.glob(str(model_config["path"] / "*model.safetensors")))}
|
||||
)
|
||||
model_config["path"] = str(model_path / model)
|
||||
m = {}
|
||||
|
||||
@@ -12,9 +12,7 @@ async def test_non_blocking():
|
||||
shard_downloader = NewShardDownloader()
|
||||
engine = MLXDynamicShardInferenceEngine(shard_downloader)
|
||||
_shard = build_base_shard("llama-3.1-8b", "MLXDynamicShardInferenceEngine")
|
||||
shard = Shard(
|
||||
_shard.model_id, _shard.start_layer, _shard.n_layers - 1, _shard.n_layers
|
||||
)
|
||||
shard = Shard(_shard.model_id, _shard.start_layer, _shard.n_layers - 1, _shard.n_layers)
|
||||
await engine.ensure_shard(shard)
|
||||
|
||||
queue = asyncio.Queue()
|
||||
@@ -55,9 +53,7 @@ async def test_non_blocking():
|
||||
try:
|
||||
while running:
|
||||
timestamp = await queue.get()
|
||||
latency = (
|
||||
time.perf_counter_ns() - timestamp
|
||||
) / 1_000_000 # Convert to ms
|
||||
latency = (time.perf_counter_ns() - timestamp) / 1_000_000 # Convert to ms
|
||||
measurements.append(latency)
|
||||
queue.task_done()
|
||||
except asyncio.CancelledError:
|
||||
|
||||
@@ -36,6 +36,6 @@ class Shard:
|
||||
|
||||
|
||||
def shards_overlap(shard1: Shard, shard2: Shard) -> bool:
|
||||
return shard1.model_id == shard2.model_id and max(
|
||||
shard1.start_layer, shard2.start_layer
|
||||
) <= min(shard1.end_layer, shard2.end_layer)
|
||||
return shard1.model_id == shard2.model_id and max(shard1.start_layer, shard2.start_layer) <= min(
|
||||
shard1.end_layer, shard2.end_layer
|
||||
)
|
||||
|
||||
@@ -17,18 +17,14 @@ async def test_inference_engine(
|
||||
prompt = "In a single word only, what is the last name of the current president of the USA?"
|
||||
resp_full, _ = await inference_engine_1.infer_prompt(
|
||||
"A",
|
||||
shard=Shard(
|
||||
model_id=model_id, start_layer=0, end_layer=n_layers - 1, n_layers=n_layers
|
||||
),
|
||||
shard=Shard(model_id=model_id, start_layer=0, end_layer=n_layers - 1, n_layers=n_layers),
|
||||
prompt=prompt,
|
||||
)
|
||||
token_full = await inference_engine_1.sample(resp_full)
|
||||
token_full = token_full.reshape(1, -1)
|
||||
next_resp_full, _ = await inference_engine_1.infer_tensor(
|
||||
"A",
|
||||
shard=Shard(
|
||||
model_id=model_id, start_layer=0, end_layer=n_layers - 1, n_layers=n_layers
|
||||
),
|
||||
shard=Shard(model_id=model_id, start_layer=0, end_layer=n_layers - 1, n_layers=n_layers),
|
||||
input_data=token_full,
|
||||
)
|
||||
|
||||
|
||||
@@ -144,9 +144,7 @@ def build_transformer(model_path: Path, shard: Shard, model_size="8B", device=No
|
||||
return model
|
||||
|
||||
|
||||
_executor = ThreadPoolExecutor(
|
||||
max_workers=1
|
||||
) # singleton so tinygrad always runs on the same thread
|
||||
_executor = ThreadPoolExecutor(max_workers=1) # singleton so tinygrad always runs on the same thread
|
||||
|
||||
|
||||
class TinygradDynamicShardInferenceEngine(InferenceEngine):
|
||||
@@ -166,50 +164,31 @@ class TinygradDynamicShardInferenceEngine(InferenceEngine):
|
||||
state = self.states[request_id]
|
||||
return {"start_pos": state.start, "cache": state.cache}
|
||||
|
||||
async def sample(
|
||||
self, x: np.ndarray, temp=TEMPERATURE, top_p: float = 0.0
|
||||
) -> np.ndarray:
|
||||
async def sample(self, x: np.ndarray, temp=TEMPERATURE, top_p: float = 0.0) -> np.ndarray:
|
||||
def sample_wrapper():
|
||||
logits = x[:, -1, :]
|
||||
return (
|
||||
sample_logits(Tensor(logits).flatten(), temp, 0, 0.8, top_p, 0.0)
|
||||
.realize()
|
||||
.numpy()
|
||||
.astype(int)
|
||||
)
|
||||
return sample_logits(Tensor(logits).flatten(), temp, 0, 0.8, top_p, 0.0).realize().numpy().astype(int)
|
||||
|
||||
return await asyncio.get_running_loop().run_in_executor(
|
||||
self.executor, sample_wrapper
|
||||
)
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, sample_wrapper)
|
||||
|
||||
async def encode(self, shard: Shard, prompt: str) -> np.ndarray:
|
||||
await self.ensure_shard(shard)
|
||||
tokens = await asyncio.get_running_loop().run_in_executor(
|
||||
self.executor, self.tokenizer.encode, prompt
|
||||
)
|
||||
return await asyncio.get_running_loop().run_in_executor(
|
||||
self.executor, np.array, tokens
|
||||
)
|
||||
tokens = await asyncio.get_running_loop().run_in_executor(self.executor, self.tokenizer.encode, prompt)
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, np.array, tokens)
|
||||
|
||||
async def decode(self, shard: Shard, tokens) -> str:
|
||||
await self.ensure_shard(shard)
|
||||
tokens = await asyncio.get_running_loop().run_in_executor(
|
||||
self.executor, self.tokenizer.decode, tokens
|
||||
)
|
||||
tokens = await asyncio.get_running_loop().run_in_executor(self.executor, self.tokenizer.decode, tokens)
|
||||
return tokens
|
||||
|
||||
async def load_checkpoint(self, shard: Shard, path: str):
|
||||
await self.ensure_shard(shard)
|
||||
state_dict = safe_load(path)
|
||||
await asyncio.get_running_loop().run_in_executor(
|
||||
self.executor, load_state_dict, self.model, state_dict
|
||||
)
|
||||
await asyncio.get_running_loop().run_in_executor(self.executor, load_state_dict, self.model, state_dict)
|
||||
|
||||
async def save_checkpoint(self, shard: Shard, path: str):
|
||||
await self.ensure_shard(shard)
|
||||
state_dict = await asyncio.get_running_loop().run_in_executor(
|
||||
self.executor, get_state_dict, self.model
|
||||
)
|
||||
state_dict = await asyncio.get_running_loop().run_in_executor(self.executor, get_state_dict, self.model)
|
||||
safe_save(state_dict, path)
|
||||
|
||||
async def infer_tensor(
|
||||
@@ -229,9 +208,7 @@ class TinygradDynamicShardInferenceEngine(InferenceEngine):
|
||||
self.states[request_id].start += x.shape[1]
|
||||
return out.numpy()
|
||||
|
||||
output_data = await asyncio.get_running_loop().run_in_executor(
|
||||
self.executor, wrap_infer
|
||||
)
|
||||
output_data = await asyncio.get_running_loop().run_in_executor(self.executor, wrap_infer)
|
||||
return output_data, inference_state
|
||||
|
||||
async def evaluate(
|
||||
@@ -286,28 +263,18 @@ class TinygradDynamicShardInferenceEngine(InferenceEngine):
|
||||
if self.shard == shard:
|
||||
return
|
||||
|
||||
model_path = await self.shard_downloader.ensure_shard(
|
||||
shard, self.__class__.__name__
|
||||
)
|
||||
model_path = await self.shard_downloader.ensure_shard(shard, self.__class__.__name__)
|
||||
|
||||
if self.shard != shard:
|
||||
loop = asyncio.get_running_loop()
|
||||
parameters = (
|
||||
"1B"
|
||||
if "1b" in shard.model_id.lower()
|
||||
else (
|
||||
"3B"
|
||||
if "3b" in shard.model_id.lower()
|
||||
else "8B" if "8b" in shard.model_id.lower() else "70B"
|
||||
)
|
||||
)
|
||||
model_shard = await loop.run_in_executor(
|
||||
self.executor, build_transformer, model_path, shard, parameters
|
||||
else ("3B" if "3b" in shard.model_id.lower() else "8B" if "8b" in shard.model_id.lower() else "70B")
|
||||
)
|
||||
model_shard = await loop.run_in_executor(self.executor, build_transformer, model_path, shard, parameters)
|
||||
|
||||
tokenizer_path = str(
|
||||
(model_path if model_path.is_dir() else model_path.parent)
|
||||
)
|
||||
tokenizer_path = str((model_path if model_path.is_dir() else model_path.parent))
|
||||
self.tokenizer = await resolve_tokenizer(tokenizer_path)
|
||||
self.shard = shard
|
||||
self.model = model_shard
|
||||
|
||||
@@ -7,13 +7,9 @@ def length_masked_ce_loss(model, inputs, targets, lengths):
|
||||
logits = model(inputs).cast(dtypes.float32).contiguous()
|
||||
|
||||
# Mask padding tokens
|
||||
length_mask = Tensor(
|
||||
np.arange(inputs.shape[1])[None, :] < lengths[:, None], requires_grad=False
|
||||
)
|
||||
length_mask = Tensor(np.arange(inputs.shape[1])[None, :] < lengths[:, None], requires_grad=False)
|
||||
|
||||
# Calculate the loss
|
||||
ce = logits.sparse_categorical_crossentropy(
|
||||
Tensor(targets, requires_grad=False)
|
||||
).mul(length_mask)
|
||||
ce = logits.sparse_categorical_crossentropy(Tensor(targets, requires_grad=False)).mul(length_mask)
|
||||
loss = ce.sum() / length_mask.sum()
|
||||
return loss
|
||||
|
||||
@@ -27,9 +27,7 @@ def precompute_freqs_cis(
|
||||
|
||||
freqs = Tensor.arange(end).unsqueeze(dim=1) * freqs.unsqueeze(dim=0)
|
||||
# TODO: move dtype outside this
|
||||
return Tensor.stack(
|
||||
freqs.cos().cast(dtype), freqs.sin().cast(dtype), dim=-1
|
||||
).reshape(1, end, 1, dim // 2, 2)
|
||||
return Tensor.stack(freqs.cos().cast(dtype), freqs.sin().cast(dtype), dim=-1).reshape(1, end, 1, dim // 2, 2)
|
||||
|
||||
|
||||
# (a+i*b) * (c+i*d) = (ac-bd) + i*(ad+bc)
|
||||
@@ -40,12 +38,10 @@ def complex_mult(A, c, d):
|
||||
return ro.cat(co, dim=-1)
|
||||
|
||||
|
||||
def apply_rotary_emb(
|
||||
xq: Tensor, xk: Tensor, freqs_cis: Tensor
|
||||
) -> Tuple[Tensor, Tensor]:
|
||||
assert (
|
||||
freqs_cis.shape[1] == xq.shape[1] == xk.shape[1]
|
||||
), f"freqs_cis shape mismatch {freqs_cis.shape} xq:{xq.shape} xk:{xk.shape}"
|
||||
def apply_rotary_emb(xq: Tensor, xk: Tensor, freqs_cis: Tensor) -> Tuple[Tensor, Tensor]:
|
||||
assert freqs_cis.shape[1] == xq.shape[1] == xk.shape[1], (
|
||||
f"freqs_cis shape mismatch {freqs_cis.shape} xq:{xq.shape} xk:{xk.shape}"
|
||||
)
|
||||
xq = xq.reshape(*xq.shape[0:-1], -1, 2)
|
||||
xk = xk.reshape(*xk.shape[0:-1], -1, 2)
|
||||
assert len(xq.shape) == len(xk.shape) == len(freqs_cis.shape) == 5
|
||||
@@ -110,23 +106,13 @@ class Attention:
|
||||
|
||||
if cache is not None:
|
||||
# update the cache
|
||||
assert (
|
||||
xk.dtype == xv.dtype == cache.dtype
|
||||
), f"{xk.dtype=}, {xv.dtype=}, {cache.dtype=}"
|
||||
cache.shrink(
|
||||
(None, None, (start_pos, start_pos + seqlen), None, None)
|
||||
).assign(Tensor.stack(xk, xv)).realize()
|
||||
assert xk.dtype == xv.dtype == cache.dtype, f"{xk.dtype=}, {xv.dtype=}, {cache.dtype=}"
|
||||
cache.shrink((None, None, (start_pos, start_pos + seqlen), None, None)).assign(
|
||||
Tensor.stack(xk, xv)
|
||||
).realize()
|
||||
|
||||
keys = (
|
||||
cache[0].shrink((None, (0, start_pos + seqlen), None, None))
|
||||
if start_pos > 0
|
||||
else xk
|
||||
)
|
||||
values = (
|
||||
cache[1].shrink((None, (0, start_pos + seqlen), None, None))
|
||||
if start_pos > 0
|
||||
else xv
|
||||
)
|
||||
keys = cache[0].shrink((None, (0, start_pos + seqlen), None, None)) if start_pos > 0 else xk
|
||||
values = cache[1].shrink((None, (0, start_pos + seqlen), None, None)) if start_pos > 0 else xv
|
||||
else:
|
||||
keys = xk
|
||||
values = xv
|
||||
@@ -149,9 +135,7 @@ class FeedForward:
|
||||
self.w3 = linear(dim, hidden_dim, bias=False) # the gate in Gated Linear Unit
|
||||
|
||||
def __call__(self, x: Tensor) -> Tensor:
|
||||
return self.w2(
|
||||
self.w1(x).silu() * self.w3(x)
|
||||
) # SwiGLU [arxiv/2002.05202, eq (5)]
|
||||
return self.w2(self.w1(x).silu() * self.w3(x)) # SwiGLU [arxiv/2002.05202, eq (5)]
|
||||
|
||||
|
||||
class TransformerBlock:
|
||||
@@ -179,16 +163,12 @@ class TransformerBlock:
|
||||
mask: Optional[Tensor],
|
||||
cache: Optional[Tensor] = None,
|
||||
):
|
||||
h = x + self.attention(
|
||||
self.attention_norm(x), start_pos, freqs_cis, mask, cache=cache
|
||||
)
|
||||
h = x + self.attention(self.attention_norm(x), start_pos, freqs_cis, mask, cache=cache)
|
||||
return (h + self.feed_forward(self.ffn_norm(h))).contiguous()
|
||||
|
||||
|
||||
# standard openai sampling
|
||||
def sample_logits(
|
||||
logits: Tensor, temp: float, k: int, p: float, af: float, ap: float, sample=None
|
||||
):
|
||||
def sample_logits(logits: Tensor, temp: float, k: int, p: float, af: float, ap: float, sample=None):
|
||||
assert logits.ndim == 1, "only works on 1d tensors"
|
||||
assert 0 <= p <= 1, "p must be between 0 and 1"
|
||||
assert 0 <= k <= logits.numel(), "k must be between 0 and numel"
|
||||
@@ -224,13 +204,9 @@ def sample_logits(
|
||||
Tensor.zeros(k, device=logits.device, dtype=dtypes.int32).contiguous(),
|
||||
)
|
||||
for i in range(k):
|
||||
t_argmax = (
|
||||
t.numel() - ((t == (t_max := t.max())) * counter2).max() - 1
|
||||
).cast(dtypes.default_int)
|
||||
t_argmax = (t.numel() - ((t == (t_max := t.max())) * counter2).max() - 1).cast(dtypes.default_int)
|
||||
output = output + t_max.unsqueeze(0).pad(((i, k - i - 1),))
|
||||
output_indices = output_indices + t_argmax.unsqueeze(0).pad(
|
||||
((i, k - i - 1),)
|
||||
)
|
||||
output_indices = output_indices + t_argmax.unsqueeze(0).pad(((i, k - i - 1),))
|
||||
t = (counter == t_argmax).where(0, t)
|
||||
|
||||
# approximate top p
|
||||
@@ -247,9 +223,7 @@ def sample_logits(
|
||||
|
||||
# increase alpha counter
|
||||
if af or ap:
|
||||
sample.alpha_counter = (counter == output_token).where(
|
||||
sample.alpha_counter + 1, sample.alpha_counter
|
||||
)
|
||||
sample.alpha_counter = (counter == output_token).where(sample.alpha_counter + 1, sample.alpha_counter)
|
||||
|
||||
return output_token
|
||||
|
||||
@@ -305,9 +279,7 @@ class Transformer:
|
||||
cache: Optional[List[Tensor]] = None,
|
||||
):
|
||||
seqlen = x.shape[1]
|
||||
freqs_cis = self.freqs_cis.shrink(
|
||||
(None, (start_pos, start_pos + seqlen), None, None, None)
|
||||
)
|
||||
freqs_cis = self.freqs_cis.shrink((None, (start_pos, start_pos + seqlen), None, None, None))
|
||||
mask = (
|
||||
Tensor.full(
|
||||
(1, 1, seqlen, start_pos + seqlen),
|
||||
@@ -324,9 +296,7 @@ class Transformer:
|
||||
h = x
|
||||
|
||||
if cache is None:
|
||||
cache = [
|
||||
None for _ in range(self.shard.start_layer, self.shard.end_layer + 1)
|
||||
]
|
||||
cache = [None for _ in range(self.shard.start_layer, self.shard.end_layer + 1)]
|
||||
for i, c in zip(range(self.shard.start_layer, self.shard.end_layer + 1), cache):
|
||||
layer = self.layers[i]
|
||||
h = layer(h, start_pos, freqs_cis, mask, cache=c)
|
||||
@@ -353,9 +323,7 @@ class Transformer:
|
||||
)
|
||||
return self.forward_base(x, start_pos, cache=cache)
|
||||
|
||||
def __call__(
|
||||
self, x: Tensor, start_pos: Variable, cache: Optional[List[Tensor]] = None
|
||||
):
|
||||
def __call__(self, x: Tensor, start_pos: Variable, cache: Optional[List[Tensor]] = None):
|
||||
# TODO: better way to handle the first call v.s. the rest?
|
||||
h = self.embed(x)
|
||||
return self.forward(h, start_pos, cache=cache)
|
||||
@@ -369,22 +337,12 @@ class TransformerShard:
|
||||
jit: bool = True,
|
||||
):
|
||||
shardrange = range(shard.start_layer, shard.end_layer + 1)
|
||||
self.layers = [
|
||||
layer
|
||||
for layer, n in zip(base.layers, range(shard.n_layers))
|
||||
if n in shardrange
|
||||
]
|
||||
self.layers = [layer for layer, n in zip(base.layers, range(shard.n_layers)) if n in shardrange]
|
||||
self.norm = base.norm
|
||||
self.tok_embeddings = base.tok_embeddings
|
||||
self.embed = (
|
||||
(lambda x: self.tok_embeddings(x))
|
||||
if shard.is_first_layer()
|
||||
else (lambda x: x)
|
||||
)
|
||||
self.embed = (lambda x: self.tok_embeddings(x)) if shard.is_first_layer() else (lambda x: x)
|
||||
self.output = base.output
|
||||
self.post = (
|
||||
(lambda x: self.output(x)) if shard.is_last_layer() else (lambda x: x)
|
||||
)
|
||||
self.post = (lambda x: self.output(x)) if shard.is_last_layer() else (lambda x: x)
|
||||
self.max_context = base.max_context
|
||||
self.null_cache = [None for _ in shardrange]
|
||||
self.freqs_cis = base.freqs_cis
|
||||
@@ -392,9 +350,7 @@ class TransformerShard:
|
||||
|
||||
def forward_base(self, x: Tensor, start_pos: Union[Variable, int], cache):
|
||||
seqlen = x.shape[1]
|
||||
freqs_cis = self.freqs_cis.shrink(
|
||||
(None, (start_pos, start_pos + seqlen), None, None, None)
|
||||
)
|
||||
freqs_cis = self.freqs_cis.shrink((None, (start_pos, start_pos + seqlen), None, None, None))
|
||||
mask = (
|
||||
Tensor.full(
|
||||
(1, 1, seqlen, start_pos + seqlen),
|
||||
@@ -423,28 +379,18 @@ class TransformerShard:
|
||||
)
|
||||
return self.forward_base(x, start_pos, cache=cache)
|
||||
|
||||
def __call__(
|
||||
self, x: Tensor, start_pos: Variable, cache: Optional[List[Tensor]] = None
|
||||
):
|
||||
def __call__(self, x: Tensor, start_pos: Variable, cache: Optional[List[Tensor]] = None):
|
||||
# TODO: better way to handle the first call v.s. the rest?
|
||||
h = self.embed(x)
|
||||
return self.forward(
|
||||
h, start_pos, cache=self.null_cache if cache is None else cache
|
||||
)
|
||||
return self.forward(h, start_pos, cache=self.null_cache if cache is None else cache)
|
||||
|
||||
|
||||
# *** helpers ***
|
||||
|
||||
|
||||
def convert_from_huggingface(
|
||||
weights: Dict[str, Tensor], model: Transformer, n_heads: int, n_kv_heads: int
|
||||
):
|
||||
def convert_from_huggingface(weights: Dict[str, Tensor], model: Transformer, n_heads: int, n_kv_heads: int):
|
||||
def permute(v: Tensor, n_heads: int):
|
||||
return (
|
||||
v.reshape(n_heads, 2, v.shape[0] // n_heads // 2, v.shape[1])
|
||||
.transpose(1, 2)
|
||||
.reshape(*v.shape[:2])
|
||||
)
|
||||
return v.reshape(n_heads, 2, v.shape[0] // n_heads // 2, v.shape[1]).transpose(1, 2).reshape(*v.shape[:2])
|
||||
|
||||
keymap = {
|
||||
"model.embed_tokens.weight": "tok_embeddings.weight",
|
||||
@@ -490,29 +436,16 @@ def fix_bf16(weights: Dict[Any, Tensor]):
|
||||
if Device.DEFAULT == "CLANG":
|
||||
# TODO: without casting to float16, 70B llama OOM on tinybox.
|
||||
return {
|
||||
k: (
|
||||
v.llvm_bf16_cast(dtypes.float32).to(v.device)
|
||||
if v.dtype == dtypes.bfloat16
|
||||
else v
|
||||
)
|
||||
k: (v.llvm_bf16_cast(dtypes.float32).to(v.device) if v.dtype == dtypes.bfloat16 else v)
|
||||
for k, v in weights.items()
|
||||
}
|
||||
if getenv("SUPPORT_BF16", 1):
|
||||
# TODO: without casting to float16, 70B llama OOM on tinybox.
|
||||
return {
|
||||
k: (
|
||||
v.cast(dtypes.float32).cast(dtypes.float16)
|
||||
if v.dtype == dtypes.bfloat16
|
||||
else v
|
||||
)
|
||||
k: (v.cast(dtypes.float32).cast(dtypes.float16) if v.dtype == dtypes.bfloat16 else v)
|
||||
for k, v in weights.items()
|
||||
}
|
||||
# TODO: check if device supports bf16
|
||||
return {
|
||||
k: (
|
||||
v.llvm_bf16_cast(dtypes.half).to(v.device)
|
||||
if v.dtype == dtypes.bfloat16
|
||||
else v
|
||||
)
|
||||
for k, v in weights.items()
|
||||
k: (v.llvm_bf16_cast(dtypes.half).to(v.device) if v.dtype == dtypes.bfloat16 else v) for k, v in weights.items()
|
||||
}
|
||||
|
||||
@@ -18,9 +18,7 @@ def create_kv_cache(x: Tensor, layer):
|
||||
)
|
||||
if isinstance(x.device, tuple):
|
||||
# TODO: instead of specifying how to shard, it can follow how xk and xv are being sharded
|
||||
cache_kv.shard_(
|
||||
(x.device), axis=3 if getenv("SHARD_KVCACHE") else None
|
||||
).realize()
|
||||
cache_kv.shard_((x.device), axis=3 if getenv("SHARD_KVCACHE") else None).realize()
|
||||
return cache_kv.realize()
|
||||
|
||||
|
||||
|
||||
@@ -16,19 +16,11 @@ def concat_weights(models, device=None):
|
||||
disk_tensors: List[Tensor] = [model[name] for model in models]
|
||||
if len(disk_tensors) == 1 or len(disk_tensors[0].shape) == 1:
|
||||
return disk_tensors[0].to(device=device)
|
||||
axis = (
|
||||
1
|
||||
if name.endswith(".attention.wo.weight")
|
||||
or name.endswith(".feed_forward.w2.weight")
|
||||
else 0
|
||||
)
|
||||
axis = 1 if name.endswith(".attention.wo.weight") or name.endswith(".feed_forward.w2.weight") else 0
|
||||
lazy_tensors = [data.to(device=device) for data in disk_tensors]
|
||||
return lazy_tensors[0].cat(*lazy_tensors[1:], dim=axis)
|
||||
|
||||
return {
|
||||
name: convert(name)
|
||||
for name in {name: None for model in models for name in model}
|
||||
}
|
||||
return {name: convert(name) for name in {name: None for model in models for name in model}}
|
||||
|
||||
|
||||
def load(fn: str, shard: Shard):
|
||||
@@ -39,9 +31,7 @@ def load(fn: str, shard: Shard):
|
||||
filtered_weight_map = {}
|
||||
allow_patterns = get_allow_patterns(weight_map, shard)
|
||||
for k, n in weight_map.items():
|
||||
if allow_patterns is not None and not any(
|
||||
fnmatch(n, r) for r in allow_patterns
|
||||
):
|
||||
if allow_patterns is not None and not any(fnmatch(n, r) for r in allow_patterns):
|
||||
continue
|
||||
if k.startswith("model.layers."):
|
||||
layer_num = int(k.split(".")[2])
|
||||
@@ -58,9 +48,7 @@ def load(fn: str, shard: Shard):
|
||||
elif fn.endswith(".safetensors"):
|
||||
weight_map = safe_load(fn)
|
||||
for k in list(weight_map):
|
||||
if (n := re.search(r"\.(\d+)\.", k)) and not (
|
||||
shard.start_layer <= int(n.group(1)) <= shard.end_layer
|
||||
):
|
||||
if (n := re.search(r"\.(\d+)\.", k)) and not (shard.start_layer <= int(n.group(1)) <= shard.end_layer):
|
||||
del weight_map[k]
|
||||
return weight_map
|
||||
else:
|
||||
|
||||
@@ -49,9 +49,7 @@ async def resolve_tokenizer(repo_id: Union[str, PathLike]):
|
||||
return DummyTokenizer()
|
||||
local_path = await ensure_downloads_dir() / str(repo_id).replace("/", "--")
|
||||
if DEBUG >= 2:
|
||||
print(
|
||||
f"Checking if local path exists to load tokenizer from local {local_path=}"
|
||||
)
|
||||
print(f"Checking if local path exists to load tokenizer from local {local_path=}")
|
||||
try:
|
||||
if local_path and await aios.path.exists(local_path):
|
||||
if DEBUG >= 2:
|
||||
@@ -59,9 +57,7 @@ async def resolve_tokenizer(repo_id: Union[str, PathLike]):
|
||||
return await _resolve_tokenizer(local_path)
|
||||
except Exception:
|
||||
if DEBUG >= 5:
|
||||
print(
|
||||
f"Local check for {local_path=} failed. Resolving tokenizer for {repo_id=} normally..."
|
||||
)
|
||||
print(f"Local check for {local_path=} failed. Resolving tokenizer for {repo_id=} normally...")
|
||||
if DEBUG >= 5:
|
||||
traceback.print_exc()
|
||||
return await _resolve_tokenizer(repo_id)
|
||||
@@ -81,13 +77,9 @@ async def _resolve_tokenizer(repo_id_or_local_path: Union[str, PathLike]):
|
||||
processor, "tokenizer", getattr(processor, "_tokenizer", processor)
|
||||
).eos_token_id
|
||||
if not hasattr(processor, "encode"):
|
||||
processor.encode = getattr(
|
||||
processor, "tokenizer", getattr(processor, "_tokenizer", processor)
|
||||
).encode
|
||||
processor.encode = getattr(processor, "tokenizer", getattr(processor, "_tokenizer", processor)).encode
|
||||
if not hasattr(processor, "decode"):
|
||||
processor.decode = getattr(
|
||||
processor, "tokenizer", getattr(processor, "_tokenizer", processor)
|
||||
).decode
|
||||
processor.decode = getattr(processor, "tokenizer", getattr(processor, "_tokenizer", processor)).decode
|
||||
return processor
|
||||
except Exception as e:
|
||||
if DEBUG >= 4:
|
||||
@@ -98,9 +90,7 @@ async def _resolve_tokenizer(repo_id_or_local_path: Union[str, PathLike]):
|
||||
try:
|
||||
if DEBUG >= 4:
|
||||
print(f"Trying AutoTokenizer for {repo_id_or_local_path}")
|
||||
return AutoTokenizer.from_pretrained(
|
||||
repo_id_or_local_path, trust_remote_code=True
|
||||
)
|
||||
return AutoTokenizer.from_pretrained(repo_id_or_local_path, trust_remote_code=True)
|
||||
except Exception as e:
|
||||
if DEBUG >= 4:
|
||||
print(
|
||||
|
||||
@@ -181,9 +181,7 @@ class HanzoNetProvider(LocalLLMProvider):
|
||||
"stable-diffusion-2-1-base",
|
||||
]
|
||||
|
||||
def _messages_to_prompt(
|
||||
self, messages: List[Message], tools: Optional[List[Dict[str, Any]]]
|
||||
) -> str:
|
||||
def _messages_to_prompt(self, messages: List[Message], tools: Optional[List[Dict[str, Any]]]) -> str:
|
||||
"""Convert messages to a prompt string."""
|
||||
prompt = ""
|
||||
|
||||
@@ -214,9 +212,7 @@ class HanzoNetProvider(LocalLLMProvider):
|
||||
prompt += "Assistant: "
|
||||
return prompt
|
||||
|
||||
def _generate_dummy_response(
|
||||
self, prompt: str, tools: Optional[List[Dict[str, Any]]]
|
||||
) -> str:
|
||||
def _generate_dummy_response(self, prompt: str, tools: Optional[List[Dict[str, Any]]]) -> str:
|
||||
"""Generate a contextual dummy response based on prompt."""
|
||||
prompt_lower = prompt.lower()
|
||||
|
||||
@@ -224,10 +220,7 @@ class HanzoNetProvider(LocalLLMProvider):
|
||||
if tools:
|
||||
for tool in tools:
|
||||
tool_name = tool["name"]
|
||||
if (
|
||||
tool_name in prompt_lower
|
||||
or tool["description"].lower() in prompt_lower
|
||||
):
|
||||
if tool_name in prompt_lower or tool["description"].lower() in prompt_lower:
|
||||
# Generate a response that calls the tool
|
||||
if "search" in tool_name:
|
||||
return f"I'll search for that information using the {tool_name} tool.\n\n<tool_call>{tool_name}('authentication')</tool_call>"
|
||||
@@ -244,7 +237,9 @@ class HanzoNetProvider(LocalLLMProvider):
|
||||
elif "analyze" in prompt_lower:
|
||||
return "After analyzing the code, I can see it follows good practices with clear structure and efficient implementation."
|
||||
elif "generate" in prompt_lower or "test" in prompt_lower:
|
||||
return "I've generated the requested code/tests following best practices and ensuring comprehensive coverage."
|
||||
return (
|
||||
"I've generated the requested code/tests following best practices and ensuring comprehensive coverage."
|
||||
)
|
||||
elif "explain" in prompt_lower or "what is" in prompt_lower:
|
||||
return "Let me explain that concept: It's a fundamental programming technique that involves a function calling itself to solve smaller instances of the same problem."
|
||||
else:
|
||||
|
||||
@@ -115,9 +115,7 @@ class LocalComputeNode:
|
||||
models: List of available models
|
||||
"""
|
||||
self.node_id = node_id
|
||||
self.wallet_address = (
|
||||
wallet_address or f"0x{hashlib.sha256(node_id.encode()).hexdigest()[:40]}"
|
||||
)
|
||||
self.wallet_address = wallet_address or f"0x{hashlib.sha256(node_id.encode()).hexdigest()[:40]}"
|
||||
self.models: Dict[str, ModelConfig] = {}
|
||||
self.loaded_models: Dict[str, Any] = {}
|
||||
|
||||
@@ -254,9 +252,7 @@ class LocalComputeNode:
|
||||
tokenizer = AutoTokenizer.from_pretrained(config.model_path)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
config.model_path,
|
||||
torch_dtype=(
|
||||
torch.float16 if config.device != "cpu" else torch.float32
|
||||
),
|
||||
torch_dtype=(torch.float16 if config.device != "cpu" else torch.float32),
|
||||
device_map="auto" if config.device == "cuda" else None,
|
||||
)
|
||||
|
||||
@@ -332,9 +328,7 @@ class LocalComputeNode:
|
||||
|
||||
# Calculate cost
|
||||
tokens_generated = len(result_text.split()) # Approximate
|
||||
cost = (tokens_generated / 1000) * self.models[
|
||||
selected_model
|
||||
].price_per_1k_tokens
|
||||
cost = (tokens_generated / 1000) * self.models[selected_model].price_per_1k_tokens
|
||||
|
||||
# Update statistics
|
||||
self.total_requests += 1
|
||||
@@ -411,9 +405,7 @@ class LocalComputeNode:
|
||||
|
||||
return response
|
||||
|
||||
def _create_attestation(
|
||||
self, request: InferenceRequest, result: InferenceResult
|
||||
) -> Dict[str, Any]:
|
||||
def _create_attestation(self, request: InferenceRequest, result: InferenceResult) -> Dict[str, Any]:
|
||||
"""Create attestation for inference result."""
|
||||
# In production, this would use TEE attestation
|
||||
attestation_data = {
|
||||
@@ -484,13 +476,8 @@ class LocalComputeOrchestrator:
|
||||
for node in self.nodes.values():
|
||||
for model in node.list_models():
|
||||
if model["available"]:
|
||||
estimated_cost = (request.max_tokens / 1000) * model[
|
||||
"price_per_1k_tokens"
|
||||
]
|
||||
if (
|
||||
estimated_cost <= request.max_price_eth
|
||||
and estimated_cost < best_price
|
||||
):
|
||||
estimated_cost = (request.max_tokens / 1000) * model["price_per_1k_tokens"]
|
||||
if estimated_cost <= request.max_price_eth and estimated_cost < best_price:
|
||||
best_node = node
|
||||
best_price = estimated_cost
|
||||
|
||||
@@ -522,9 +509,7 @@ class LocalComputeOrchestrator:
|
||||
"completed_requests": len(self.completed_requests),
|
||||
"total_requests_processed": total_requests,
|
||||
"total_earnings_eth": total_earnings,
|
||||
"nodes_detail": {
|
||||
node_id: node.get_stats() for node_id, node in self.nodes.items()
|
||||
},
|
||||
"nodes_detail": {node_id: node.get_stats() for node_id, node in self.nodes.items()},
|
||||
}
|
||||
|
||||
|
||||
@@ -536,9 +521,7 @@ orchestrator = LocalComputeOrchestrator()
|
||||
async def demo_local_compute():
|
||||
"""Demonstrate local compute capabilities."""
|
||||
# Create a compute node
|
||||
node1 = LocalComputeNode(
|
||||
node_id="node_001", wallet_address="0x1234567890123456789012345678901234567890"
|
||||
)
|
||||
node1 = LocalComputeNode(node_id="node_001", wallet_address="0x1234567890123456789012345678901234567890")
|
||||
|
||||
# Register with orchestrator
|
||||
orchestrator.register_node(node1)
|
||||
@@ -546,9 +529,7 @@ async def demo_local_compute():
|
||||
# List available models
|
||||
print("\nAvailable models:")
|
||||
for model in node1.list_models():
|
||||
print(
|
||||
f" - {model['name']}: {model['device']}, ${model['price_per_1k_tokens']}/1k tokens"
|
||||
)
|
||||
print(f" - {model['name']}: {model['device']}, ${model['price_per_1k_tokens']}/1k tokens")
|
||||
|
||||
# Create inference request
|
||||
request = InferenceRequest(
|
||||
|
||||
@@ -30,9 +30,7 @@ def create_local_agent(
|
||||
Agent configured for local LLM
|
||||
"""
|
||||
# Create model config for local provider
|
||||
model_config = ModelConfig(
|
||||
provider=ModelProvider.LOCAL, model=local_model, base_url=base_url
|
||||
)
|
||||
model_config = ModelConfig(provider=ModelProvider.LOCAL, model=local_model, base_url=base_url)
|
||||
|
||||
return create_agent(
|
||||
name=name,
|
||||
@@ -76,9 +74,7 @@ def create_local_distributed_network(
|
||||
|
||||
# Create a local router if one isn't provided
|
||||
if "router" not in kwargs:
|
||||
router_config = ModelConfig(
|
||||
provider=ModelProvider.LOCAL, model=local_model, base_url=base_url
|
||||
)
|
||||
router_config = ModelConfig(provider=ModelProvider.LOCAL, model=local_model, base_url=base_url)
|
||||
|
||||
kwargs["router"] = create_routing_agent(
|
||||
name="local_router", description="Local routing agent", model=router_config
|
||||
@@ -132,18 +128,12 @@ async def check_local_llm_status(provider: str = "hanzo") -> dict:
|
||||
if engine_type == "mlx":
|
||||
import platform
|
||||
|
||||
status["platform"] = (
|
||||
"Apple Silicon"
|
||||
if platform.machine() in ["arm64", "aarch64"]
|
||||
else platform.machine()
|
||||
)
|
||||
status["platform"] = "Apple Silicon" if platform.machine() in ["arm64", "aarch64"] else platform.machine()
|
||||
if not is_available:
|
||||
status["instructions"] = "MLX requires Apple Silicon (M1/M2/M3)"
|
||||
elif engine_type == "tinygrad":
|
||||
status["instructions"] = (
|
||||
"Tinygrad engine ready for distributed inference"
|
||||
if is_available
|
||||
else "Install tinygrad"
|
||||
"Tinygrad engine ready for distributed inference" if is_available else "Install tinygrad"
|
||||
)
|
||||
else: # dummy
|
||||
status["instructions"] = "Using mock responses for testing"
|
||||
|
||||
@@ -73,9 +73,7 @@ class MemoryManager:
|
||||
ids = []
|
||||
|
||||
for statement in statements:
|
||||
memory_id = await self.store.add(
|
||||
{"content": statement, "type": "statement"}
|
||||
)
|
||||
memory_id = await self.store.add({"content": statement, "type": "statement"})
|
||||
ids.append(memory_id)
|
||||
|
||||
return ids
|
||||
@@ -92,9 +90,7 @@ class MemoryManager:
|
||||
results = []
|
||||
|
||||
for update in updates:
|
||||
success = await self.store.update(
|
||||
update["id"], {"content": update["statement"]}
|
||||
)
|
||||
success = await self.store.update(update["id"], {"content": update["statement"]})
|
||||
results.append(success)
|
||||
|
||||
return results
|
||||
@@ -139,9 +135,7 @@ def create_memory_tools(memory_manager: Optional[MemoryManager] = None) -> List[
|
||||
name="recall_memories",
|
||||
description="Recall memories relevant to one or more queries. Can run multiple queries in parallel.",
|
||||
)
|
||||
async def recall_memories_tool(
|
||||
queries: List[str], limit: int = 10, context: ToolContext = None
|
||||
) -> str:
|
||||
async def recall_memories_tool(queries: List[str], limit: int = 10, context: ToolContext = None) -> str:
|
||||
"""Recall relevant memories."""
|
||||
memories = await memory_manager.recall(queries, limit)
|
||||
|
||||
@@ -162,9 +156,7 @@ def create_memory_tools(memory_manager: Optional[MemoryManager] = None) -> List[
|
||||
name="create_memories",
|
||||
description="Save one or more new pieces of information to memory.",
|
||||
)
|
||||
async def create_memories_tool(
|
||||
statements: List[str], context: ToolContext = None
|
||||
) -> str:
|
||||
async def create_memories_tool(statements: List[str], context: ToolContext = None) -> str:
|
||||
"""Create new memories."""
|
||||
ids = await memory_manager.create(statements)
|
||||
return f"Created {len(ids)} new memories."
|
||||
@@ -176,9 +168,7 @@ def create_memory_tools(memory_manager: Optional[MemoryManager] = None) -> List[
|
||||
name="update_memories",
|
||||
description="Update existing memories with corrected information.",
|
||||
)
|
||||
async def update_memories_tool(
|
||||
updates: List[Dict[str, str]], context: ToolContext = None
|
||||
) -> str:
|
||||
async def update_memories_tool(updates: List[Dict[str, str]], context: ToolContext = None) -> str:
|
||||
"""Update memories."""
|
||||
results = await memory_manager.update(updates)
|
||||
success_count = sum(results)
|
||||
|
||||
@@ -59,49 +59,21 @@ UNKNOWN_DEVICE_CAPABILITIES = DeviceCapabilities(
|
||||
CHIP_FLOPS = {
|
||||
# Apple M series
|
||||
"Apple M1": DeviceFlops(fp32=2.29 * TFLOPS, fp16=4.58 * TFLOPS, int8=9.16 * TFLOPS),
|
||||
"Apple M1 Pro": DeviceFlops(
|
||||
fp32=5.30 * TFLOPS, fp16=10.60 * TFLOPS, int8=21.20 * TFLOPS
|
||||
),
|
||||
"Apple M1 Max": DeviceFlops(
|
||||
fp32=10.60 * TFLOPS, fp16=21.20 * TFLOPS, int8=42.40 * TFLOPS
|
||||
),
|
||||
"Apple M2": DeviceFlops(
|
||||
fp32=3.55 * TFLOPS, fp16=7.10 * TFLOPS, int8=14.20 * TFLOPS
|
||||
),
|
||||
"Apple M2 Pro": DeviceFlops(
|
||||
fp32=5.68 * TFLOPS, fp16=11.36 * TFLOPS, int8=22.72 * TFLOPS
|
||||
),
|
||||
"Apple M2 Max": DeviceFlops(
|
||||
fp32=13.49 * TFLOPS, fp16=26.98 * TFLOPS, int8=53.96 * TFLOPS
|
||||
),
|
||||
"Apple M3": DeviceFlops(
|
||||
fp32=3.55 * TFLOPS, fp16=7.10 * TFLOPS, int8=14.20 * TFLOPS
|
||||
),
|
||||
"Apple M3 Pro": DeviceFlops(
|
||||
fp32=4.97 * TFLOPS, fp16=9.94 * TFLOPS, int8=19.88 * TFLOPS
|
||||
),
|
||||
"Apple M3 Max": DeviceFlops(
|
||||
fp32=14.20 * TFLOPS, fp16=28.40 * TFLOPS, int8=56.80 * TFLOPS
|
||||
),
|
||||
"Apple M4": DeviceFlops(
|
||||
fp32=4.26 * TFLOPS, fp16=8.52 * TFLOPS, int8=17.04 * TFLOPS
|
||||
),
|
||||
"Apple M1 Pro": DeviceFlops(fp32=5.30 * TFLOPS, fp16=10.60 * TFLOPS, int8=21.20 * TFLOPS),
|
||||
"Apple M1 Max": DeviceFlops(fp32=10.60 * TFLOPS, fp16=21.20 * TFLOPS, int8=42.40 * TFLOPS),
|
||||
"Apple M2": DeviceFlops(fp32=3.55 * TFLOPS, fp16=7.10 * TFLOPS, int8=14.20 * TFLOPS),
|
||||
"Apple M2 Pro": DeviceFlops(fp32=5.68 * TFLOPS, fp16=11.36 * TFLOPS, int8=22.72 * TFLOPS),
|
||||
"Apple M2 Max": DeviceFlops(fp32=13.49 * TFLOPS, fp16=26.98 * TFLOPS, int8=53.96 * TFLOPS),
|
||||
"Apple M3": DeviceFlops(fp32=3.55 * TFLOPS, fp16=7.10 * TFLOPS, int8=14.20 * TFLOPS),
|
||||
"Apple M3 Pro": DeviceFlops(fp32=4.97 * TFLOPS, fp16=9.94 * TFLOPS, int8=19.88 * TFLOPS),
|
||||
"Apple M3 Max": DeviceFlops(fp32=14.20 * TFLOPS, fp16=28.40 * TFLOPS, int8=56.80 * TFLOPS),
|
||||
"Apple M4": DeviceFlops(fp32=4.26 * TFLOPS, fp16=8.52 * TFLOPS, int8=17.04 * TFLOPS),
|
||||
# NVIDIA GPUs
|
||||
"NVIDIA GEFORCE RTX 4090": DeviceFlops(
|
||||
fp32=82.58 * TFLOPS, fp16=165.16 * TFLOPS, int8=330.32 * TFLOPS
|
||||
),
|
||||
"NVIDIA GEFORCE RTX 4080": DeviceFlops(
|
||||
fp32=48.74 * TFLOPS, fp16=97.48 * TFLOPS, int8=194.96 * TFLOPS
|
||||
),
|
||||
"NVIDIA GEFORCE RTX 4070": DeviceFlops(
|
||||
fp32=29.0 * TFLOPS, fp16=58.0 * TFLOPS, int8=116.0 * TFLOPS
|
||||
),
|
||||
"NVIDIA GEFORCE RTX 3090": DeviceFlops(
|
||||
fp32=35.6 * TFLOPS, fp16=71.2 * TFLOPS, int8=142.4 * TFLOPS
|
||||
),
|
||||
"NVIDIA GEFORCE RTX 3080": DeviceFlops(
|
||||
fp32=29.8 * TFLOPS, fp16=59.6 * TFLOPS, int8=119.2 * TFLOPS
|
||||
),
|
||||
"NVIDIA GEFORCE RTX 4090": DeviceFlops(fp32=82.58 * TFLOPS, fp16=165.16 * TFLOPS, int8=330.32 * TFLOPS),
|
||||
"NVIDIA GEFORCE RTX 4080": DeviceFlops(fp32=48.74 * TFLOPS, fp16=97.48 * TFLOPS, int8=194.96 * TFLOPS),
|
||||
"NVIDIA GEFORCE RTX 4070": DeviceFlops(fp32=29.0 * TFLOPS, fp16=58.0 * TFLOPS, int8=116.0 * TFLOPS),
|
||||
"NVIDIA GEFORCE RTX 3090": DeviceFlops(fp32=35.6 * TFLOPS, fp16=71.2 * TFLOPS, int8=142.4 * TFLOPS),
|
||||
"NVIDIA GEFORCE RTX 3080": DeviceFlops(fp32=29.8 * TFLOPS, fp16=59.6 * TFLOPS, int8=119.2 * TFLOPS),
|
||||
# Add more as needed
|
||||
}
|
||||
|
||||
@@ -131,9 +103,7 @@ def mac_device_capabilities() -> DeviceCapabilities:
|
||||
import subprocess
|
||||
|
||||
# Get model info
|
||||
model_result = subprocess.run(
|
||||
["system_profiler", "SPHardwareDataType"], capture_output=True, text=True
|
||||
)
|
||||
model_result = subprocess.run(["system_profiler", "SPHardwareDataType"], capture_output=True, text=True)
|
||||
|
||||
model = "Mac"
|
||||
chip = "Unknown"
|
||||
@@ -172,24 +142,24 @@ def linux_device_capabilities() -> DeviceCapabilities:
|
||||
"""Get Linux device capabilities."""
|
||||
try:
|
||||
import subprocess
|
||||
|
||||
|
||||
# Try to detect NVIDIA GPU using nvidia-smi
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5
|
||||
timeout=5,
|
||||
)
|
||||
|
||||
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
gpu_info = result.stdout.strip().split(",")
|
||||
gpu_name = gpu_info[0].strip()
|
||||
gpu_memory = int(gpu_info[1].strip().split()[0]) # Memory in MiB
|
||||
|
||||
|
||||
# Lookup FLOPS from known GPUs
|
||||
flops = CHIP_FLOPS.get(gpu_name.upper(), DeviceFlops(fp32=0, fp16=0, int8=0))
|
||||
|
||||
|
||||
return DeviceCapabilities(
|
||||
model="Linux Box with GPU",
|
||||
chip=gpu_name,
|
||||
@@ -198,23 +168,18 @@ def linux_device_capabilities() -> DeviceCapabilities:
|
||||
)
|
||||
except (subprocess.SubprocessError, FileNotFoundError, IndexError):
|
||||
pass
|
||||
|
||||
|
||||
# Try to detect AMD GPU using rocm-smi
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["rocm-smi", "--showproductname"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5
|
||||
)
|
||||
|
||||
result = subprocess.run(["rocm-smi", "--showproductname"], capture_output=True, text=True, timeout=5)
|
||||
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
# Parse AMD GPU info
|
||||
for line in result.stdout.split("\n"):
|
||||
if "GPU" in line:
|
||||
gpu_name = line.split(":")[-1].strip() if ":" in line else "AMD GPU"
|
||||
memory = psutil.virtual_memory().total // 2**20
|
||||
|
||||
|
||||
return DeviceCapabilities(
|
||||
model="Linux Box with AMD GPU",
|
||||
chip=gpu_name,
|
||||
@@ -223,7 +188,7 @@ def linux_device_capabilities() -> DeviceCapabilities:
|
||||
)
|
||||
except (subprocess.SubprocessError, FileNotFoundError):
|
||||
pass
|
||||
|
||||
|
||||
# Fallback to CPU
|
||||
return DeviceCapabilities(
|
||||
model="Linux Box",
|
||||
@@ -231,7 +196,7 @@ def linux_device_capabilities() -> DeviceCapabilities:
|
||||
memory=psutil.virtual_memory().total // 2**20,
|
||||
flops=DeviceFlops(fp32=0, fp16=0, int8=0),
|
||||
)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
if DEBUG >= 1:
|
||||
print(f"Error getting Linux device capabilities: {e}")
|
||||
@@ -247,7 +212,7 @@ def windows_device_capabilities() -> DeviceCapabilities:
|
||||
"""Get Windows device capabilities."""
|
||||
try:
|
||||
import subprocess
|
||||
|
||||
|
||||
# Try to detect NVIDIA GPU using nvidia-smi
|
||||
try:
|
||||
result = subprocess.run(
|
||||
@@ -255,17 +220,17 @@ def windows_device_capabilities() -> DeviceCapabilities:
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
shell=True # Windows may need shell
|
||||
shell=True, # Windows may need shell
|
||||
)
|
||||
|
||||
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
gpu_info = result.stdout.strip().split(",")
|
||||
gpu_name = gpu_info[0].strip()
|
||||
gpu_memory = int(gpu_info[1].strip().split()[0]) # Memory in MiB
|
||||
|
||||
|
||||
# Lookup FLOPS from known GPUs
|
||||
flops = CHIP_FLOPS.get(gpu_name.upper(), DeviceFlops(fp32=0, fp16=0, int8=0))
|
||||
|
||||
|
||||
return DeviceCapabilities(
|
||||
model="Windows Box with GPU",
|
||||
chip=gpu_name,
|
||||
@@ -274,7 +239,7 @@ def windows_device_capabilities() -> DeviceCapabilities:
|
||||
)
|
||||
except (subprocess.SubprocessError, FileNotFoundError, IndexError):
|
||||
pass
|
||||
|
||||
|
||||
# Fallback to CPU
|
||||
return DeviceCapabilities(
|
||||
model="Windows Box",
|
||||
@@ -282,7 +247,7 @@ def windows_device_capabilities() -> DeviceCapabilities:
|
||||
memory=psutil.virtual_memory().total // 2**20,
|
||||
flops=DeviceFlops(fp32=0, fp16=0, int8=0),
|
||||
)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
if DEBUG >= 1:
|
||||
print(f"Error getting Windows device capabilities: {e}")
|
||||
|
||||
@@ -19,9 +19,7 @@ class PartitioningStrategy(ABC):
|
||||
pass
|
||||
|
||||
|
||||
def map_partitions_to_shards(
|
||||
partitions: List[Partition], num_layers: int, model_id: str
|
||||
) -> List[Shard]:
|
||||
def map_partitions_to_shards(partitions: List[Partition], num_layers: int, model_id: str) -> List[Shard]:
|
||||
shards = []
|
||||
for i, partition in enumerate(partitions):
|
||||
start_layer = int(partition.start * num_layers)
|
||||
|
||||
@@ -88,9 +88,7 @@ Activation Lock Status: Disabled
|
||||
assert result.memory == 8192 # 8 GB in MB
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Unskip this test when running on a MacBook Pro, Apple M3 Max, 128GB"
|
||||
)
|
||||
@pytest.mark.skip(reason="Unskip this test when running on a MacBook Pro, Apple M3 Max, 128GB")
|
||||
def test_mac_device_capabilities_real():
|
||||
# Call the function without mocking
|
||||
result = mac_device_capabilities()
|
||||
@@ -100,9 +98,7 @@ def test_mac_device_capabilities_real():
|
||||
assert result.model == "MacBook Pro"
|
||||
assert result.chip == "Apple M3 Max"
|
||||
assert result.memory == 131072 # 128 GB in MB
|
||||
assert result.flops == DeviceFlops(
|
||||
fp32=14.20 * TFLOPS, fp16=28.40 * TFLOPS, int8=56.80 * TFLOPS
|
||||
)
|
||||
assert result.flops == DeviceFlops(fp32=14.20 * TFLOPS, fp16=28.40 * TFLOPS, int8=56.80 * TFLOPS)
|
||||
assert (
|
||||
str(result)
|
||||
== "Model: MacBook Pro. Chip: Apple M3 Max. Memory: 131072MB. Flops: fp32: 14.20 TFLOPS, fp16: 28.40 TFLOPS, int8: 56.80 TFLOPS"
|
||||
|
||||
@@ -53,9 +53,7 @@ class TestRingMemoryWeightedPartitioningStrategy(unittest.TestCase):
|
||||
|
||||
def test_broken_map_partitions_to_shards(self):
|
||||
# this was an old broken implementation that sometimes had rounding errors!
|
||||
def _broken_map_partitions_to_shards(
|
||||
partitions: List[Partition], num_layers, model_id: str
|
||||
):
|
||||
def _broken_map_partitions_to_shards(partitions: List[Partition], num_layers, model_id: str):
|
||||
shards = []
|
||||
for i, partition in enumerate(partitions):
|
||||
start_layer = int(partition.start * num_layers)
|
||||
|
||||
@@ -53,21 +53,15 @@ class Topology:
|
||||
self.add_edge(conn.from_id, conn.to_id, conn.description)
|
||||
|
||||
def __str__(self):
|
||||
nodes_str = ", ".join(
|
||||
f"{node_id}: {cap}" for node_id, cap in self.nodes.items()
|
||||
)
|
||||
nodes_str = ", ".join(f"{node_id}: {cap}" for node_id, cap in self.nodes.items())
|
||||
edges_str = ", ".join(
|
||||
f"{node}: {[f'{c.to_id}({c.description})' for c in conns]}"
|
||||
for node, conns in self.peer_graph.items()
|
||||
f"{node}: {[f'{c.to_id}({c.description})' for c in conns]}" for node, conns in self.peer_graph.items()
|
||||
)
|
||||
return f"Topology(Nodes: {{{nodes_str}}}, Edges: {{{edges_str}}})"
|
||||
|
||||
def to_json(self):
|
||||
return {
|
||||
"nodes": {
|
||||
node_id: capabilities.to_dict()
|
||||
for node_id, capabilities in self.nodes.items()
|
||||
},
|
||||
"nodes": {node_id: capabilities.to_dict() for node_id, capabilities in self.nodes.items()},
|
||||
"peer_graph": {
|
||||
node_id: [
|
||||
{
|
||||
|
||||
@@ -66,9 +66,7 @@ async def test_distributed_network_start_stop():
|
||||
"""Test starting and stopping a distributed network."""
|
||||
agents = create_test_agents()
|
||||
|
||||
network = create_distributed_network(
|
||||
agents=agents, name="test-network", listen_port=15679, broadcast_port=15679
|
||||
)
|
||||
network = create_distributed_network(agents=agents, name="test-network", listen_port=15679, broadcast_port=15679)
|
||||
|
||||
# Start network
|
||||
await network.start(wait_for_peers=0)
|
||||
@@ -91,9 +89,7 @@ async def test_distributed_network_local_execution():
|
||||
"""Test executing agents locally in distributed network."""
|
||||
agents = create_test_agents()
|
||||
|
||||
network = create_distributed_network(
|
||||
agents=agents, name="test-network", listen_port=15680, broadcast_port=15680
|
||||
)
|
||||
network = create_distributed_network(agents=agents, name="test-network", listen_port=15680, broadcast_port=15680)
|
||||
|
||||
# Start network
|
||||
await network.start(wait_for_peers=0)
|
||||
@@ -110,9 +106,7 @@ async def test_distributed_network_local_execution():
|
||||
assert "echo_agent" in echo_result["final_output"]
|
||||
|
||||
# Test math agent
|
||||
math_result = await network.run(
|
||||
prompt="Calculate 2 + 2", initial_agent=network.get_agent("math_agent")
|
||||
)
|
||||
math_result = await network.run(prompt="Calculate 2 + 2", initial_agent=network.get_agent("math_agent"))
|
||||
|
||||
assert math_result["success"]
|
||||
# Mock agents return mock responses
|
||||
|
||||
@@ -123,9 +123,7 @@ class ClaudeCodeBackend(Backend):
|
||||
# Check authentication
|
||||
if not self.authenticated and not self._check_auth():
|
||||
console = Console()
|
||||
console.print(
|
||||
"[yellow]Not authenticated with Claude. Attempting login...[/yellow]"
|
||||
)
|
||||
console.print("[yellow]Not authenticated with Claude. Attempting login...[/yellow]")
|
||||
if not await self.authenticate():
|
||||
raise RuntimeError("Failed to authenticate with Claude")
|
||||
|
||||
@@ -150,14 +148,9 @@ class ClaudeCodeBackend(Backend):
|
||||
|
||||
if proc.returncode != 0:
|
||||
# Check if it's an auth error
|
||||
if (
|
||||
"unauthorized" in stderr.decode().lower()
|
||||
or "auth" in stderr.decode().lower()
|
||||
):
|
||||
if "unauthorized" in stderr.decode().lower() or "auth" in stderr.decode().lower():
|
||||
self.authenticated = False
|
||||
raise RuntimeError(
|
||||
"Claude authentication expired. Please re-authenticate."
|
||||
)
|
||||
raise RuntimeError("Claude authentication expired. Please re-authenticate.")
|
||||
raise RuntimeError(f"Claude Code error: {stderr.decode()}")
|
||||
|
||||
return stdout.decode()
|
||||
@@ -257,9 +250,7 @@ class OpenAICodexBackend(Backend):
|
||||
tool_choice="auto",
|
||||
)
|
||||
else:
|
||||
response = await client.chat.completions.create(
|
||||
model="gpt-4-turbo-preview", messages=messages
|
||||
)
|
||||
response = await client.chat.completions.create(model="gpt-4-turbo-preview", messages=messages)
|
||||
|
||||
return response.choices[0].message.content
|
||||
|
||||
@@ -277,9 +268,7 @@ class HanzoDevBackend(Backend):
|
||||
"/usr/local/bin/hanzo-dev",
|
||||
os.path.expanduser("~/.local/bin/hanzo-dev"),
|
||||
# Check in parent directory
|
||||
os.path.join(
|
||||
os.path.dirname(os.path.dirname(__file__)), "..", "dev", "hanzo-dev"
|
||||
),
|
||||
os.path.join(os.path.dirname(os.path.dirname(__file__)), "..", "dev", "hanzo-dev"),
|
||||
]
|
||||
|
||||
for path in paths:
|
||||
@@ -342,9 +331,7 @@ class EmbeddedBackend(Backend):
|
||||
{"role": "user", "content": message},
|
||||
]
|
||||
|
||||
response = await self.llm_client.chat(
|
||||
messages=messages, tools=tools, tool_choice="auto" if tools else None
|
||||
)
|
||||
response = await self.llm_client.chat(messages=messages, tools=tools, tool_choice="auto" if tools else None)
|
||||
|
||||
return response.choices[0].message.content
|
||||
|
||||
@@ -394,10 +381,7 @@ class BackendManager:
|
||||
|
||||
def list_backends(self) -> Dict[str, bool]:
|
||||
"""List all backends and their availability."""
|
||||
return {
|
||||
name: backend.is_available() if backend else False
|
||||
for name, backend in self.backends.items()
|
||||
}
|
||||
return {name: backend.is_available() if backend else False for name, backend in self.backends.items()}
|
||||
|
||||
async def load_config(self) -> Optional[str]:
|
||||
"""Load configuration from the appropriate file."""
|
||||
|
||||
@@ -42,16 +42,12 @@ class CommandItem(ListItem):
|
||||
"""Create child widgets."""
|
||||
with Horizontal(classes="command-item"):
|
||||
# Icon and category
|
||||
yield Static(
|
||||
f"{self.command.icon} [{self.command.category}]", classes="command-icon"
|
||||
)
|
||||
yield Static(f"{self.command.icon} [{self.command.category}]", classes="command-icon")
|
||||
|
||||
# Name and description
|
||||
with Vertical(classes="command-info"):
|
||||
yield Static(self.command.name, classes="command-name")
|
||||
yield Static(
|
||||
self.command.description[:60] + "...", classes="command-desc"
|
||||
)
|
||||
yield Static(self.command.description[:60] + "...", classes="command-desc")
|
||||
|
||||
|
||||
class CommandPalette(Vertical):
|
||||
@@ -163,9 +159,7 @@ class CommandPalette(Vertical):
|
||||
description=tool.description,
|
||||
category=category,
|
||||
icon=icon,
|
||||
parameters=(
|
||||
tool.get_schema() if hasattr(tool, "get_schema") else None
|
||||
),
|
||||
parameters=(tool.get_schema() if hasattr(tool, "get_schema") else None),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -188,9 +182,7 @@ class CommandPalette(Vertical):
|
||||
id="palette-input",
|
||||
)
|
||||
yield ListView(id="palette-list")
|
||||
yield Label(
|
||||
"↑↓ Navigate · ⏎ Select · esc Close · ⇥ Complete", id="palette-hint"
|
||||
)
|
||||
yield Label("↑↓ Navigate · ⏎ Select · esc Close · ⇥ Complete", id="palette-hint")
|
||||
|
||||
def on_mount(self) -> None:
|
||||
"""Focus input when mounted."""
|
||||
|
||||
@@ -76,13 +76,9 @@ class CommandSuggestions(Vertical):
|
||||
SlashCommand("/backend", "Switch AI backend (claude/openai/embedded)"),
|
||||
SlashCommand("/bug", "Submit feedback about Hanzo REPL"),
|
||||
SlashCommand("/clear", "Clear conversation history and free up context"),
|
||||
SlashCommand(
|
||||
"/compact", "Clear conversation history but keep a summary in context"
|
||||
),
|
||||
SlashCommand("/compact", "Clear conversation history but keep a summary in context"),
|
||||
SlashCommand("/config", "Open config panel", ["theme"]),
|
||||
SlashCommand(
|
||||
"/cost", "Show the total cost and duration of the current session"
|
||||
),
|
||||
SlashCommand("/cost", "Show the total cost and duration of the current session"),
|
||||
SlashCommand("/doctor", "Checks the health of your Hanzo installation"),
|
||||
SlashCommand("/exit", "Exit the REPL", ["quit"]),
|
||||
SlashCommand("/help", "Show help and available commands"),
|
||||
@@ -115,9 +111,7 @@ class CommandSuggestions(Vertical):
|
||||
self.filtered_commands = self.COMMANDS
|
||||
else:
|
||||
query = self.query[1:] if self.query.startswith("/") else self.query
|
||||
self.filtered_commands = [
|
||||
cmd for cmd in self.COMMANDS if cmd.matches(query)
|
||||
]
|
||||
self.filtered_commands = [cmd for cmd in self.COMMANDS if cmd.matches(query)]
|
||||
self.selected_index = 0
|
||||
|
||||
def compose(self) -> ComposeResult:
|
||||
|
||||
@@ -36,9 +36,7 @@ class HanzoMagics(Magics):
|
||||
|
||||
# Run async chat in sync context
|
||||
loop = asyncio.get_event_loop()
|
||||
response = loop.run_until_complete(
|
||||
self.repl.tool_executor.execute_with_tools(line)
|
||||
)
|
||||
response = loop.run_until_complete(self.repl.tool_executor.execute_with_tools(line))
|
||||
self.console.print(Markdown(response))
|
||||
|
||||
@cell_magic
|
||||
@@ -49,9 +47,7 @@ class HanzoMagics(Magics):
|
||||
return
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
response = loop.run_until_complete(
|
||||
self.repl.tool_executor.execute_with_tools(message)
|
||||
)
|
||||
response = loop.run_until_complete(self.repl.tool_executor.execute_with_tools(message))
|
||||
self.console.print(Markdown(response))
|
||||
|
||||
@line_magic
|
||||
@@ -130,9 +126,7 @@ class HanzoIPythonREPL:
|
||||
self.llm_client = LLMClient()
|
||||
if not self.llm_client.get_available_providers():
|
||||
self.console.print("[red]No LLM providers available![/red]")
|
||||
self.console.print(
|
||||
"Set OPENAI_API_KEY, ANTHROPIC_API_KEY, or other provider keys"
|
||||
)
|
||||
self.console.print("Set OPENAI_API_KEY, ANTHROPIC_API_KEY, or other provider keys")
|
||||
sys.exit(1)
|
||||
|
||||
# Initialize tool executor
|
||||
@@ -142,11 +136,7 @@ class HanzoIPythonREPL:
|
||||
"""List available tools with their methods."""
|
||||
for _name, tool in sorted(self.tools.items()):
|
||||
# Show available methods
|
||||
methods = [
|
||||
m
|
||||
for m in dir(tool)
|
||||
if not m.startswith("_") and callable(getattr(tool, m))
|
||||
]
|
||||
methods = [m for m in dir(tool) if not m.startswith("_") and callable(getattr(tool, m))]
|
||||
for method in methods:
|
||||
if method not in ["execute", "get_schema"]:
|
||||
pass
|
||||
@@ -200,9 +190,7 @@ class HanzoIPythonREPL:
|
||||
def _chat_sync(self, message: str) -> str:
|
||||
"""Synchronous chat wrapper."""
|
||||
loop = asyncio.get_event_loop()
|
||||
response = loop.run_until_complete(
|
||||
self.tool_executor.execute_with_tools(message)
|
||||
)
|
||||
response = loop.run_until_complete(self.tool_executor.execute_with_tools(message))
|
||||
self.console.print(Markdown(response))
|
||||
return response
|
||||
|
||||
@@ -233,9 +221,7 @@ class HanzoIPythonREPL:
|
||||
}
|
||||
}
|
||||
|
||||
shell = InteractiveShellEmbed(
|
||||
config=config, user_ns=namespace, exit_msg="Goodbye!"
|
||||
)
|
||||
shell = InteractiveShellEmbed(config=config, user_ns=namespace, exit_msg="Goodbye!")
|
||||
|
||||
# Register magic commands
|
||||
shell.register_magics(HanzoMagics(shell, self))
|
||||
@@ -247,9 +233,7 @@ class HanzoIPythonREPL:
|
||||
# Print welcome info
|
||||
self.console.print(f"[green]Model: {self.llm_client.current_model}[/green]")
|
||||
self.console.print("[dim]Type ? for help, ?? for more details[/dim]")
|
||||
self.console.print(
|
||||
"[dim]Use %chat for AI chat, tools.<tab> for completion[/dim]"
|
||||
)
|
||||
self.console.print("[dim]Use %chat for AI chat, tools.<tab> for completion[/dim]")
|
||||
|
||||
# Start the shell
|
||||
shell()
|
||||
|
||||
@@ -62,17 +62,13 @@ class LLMClient:
|
||||
for provider in preferred_order:
|
||||
if provider in self.available_providers:
|
||||
self.current_provider = provider
|
||||
self.current_model = self.DEFAULT_MODELS.get(
|
||||
provider, "gpt-3.5-turbo"
|
||||
)
|
||||
self.current_model = self.DEFAULT_MODELS.get(provider, "gpt-3.5-turbo")
|
||||
break
|
||||
|
||||
# If no preferred provider, use the first available
|
||||
if not self.current_provider:
|
||||
self.current_provider = list(self.available_providers)[0]
|
||||
self.current_model = self.DEFAULT_MODELS.get(
|
||||
self.current_provider, "gpt-3.5-turbo"
|
||||
)
|
||||
self.current_model = self.DEFAULT_MODELS.get(self.current_provider, "gpt-3.5-turbo")
|
||||
|
||||
def _detect_providers(self) -> Set[str]:
|
||||
"""Detect which LLM providers have API keys configured."""
|
||||
@@ -81,9 +77,7 @@ class LLMClient:
|
||||
for provider, env_vars in self.PROVIDER_ENV_VARS.items():
|
||||
if not env_vars: # No API key needed (e.g., Ollama)
|
||||
# Check if the provider is accessible
|
||||
if provider == "ollama" and os.path.exists(
|
||||
os.path.expanduser("~/.ollama")
|
||||
):
|
||||
if provider == "ollama" and os.path.exists(os.path.expanduser("~/.ollama")):
|
||||
# TODO: Check if Ollama is running
|
||||
available.add(provider)
|
||||
continue
|
||||
@@ -173,10 +167,7 @@ class LLMClient:
|
||||
|
||||
# Try to infer provider from model name
|
||||
for prefix, provider in provider_prefixes.items():
|
||||
if (
|
||||
model.lower().startswith(prefix)
|
||||
and provider in self.available_providers
|
||||
):
|
||||
if model.lower().startswith(prefix) and provider in self.available_providers:
|
||||
self.current_provider = provider
|
||||
self.current_model = model
|
||||
return
|
||||
|
||||
@@ -69,18 +69,14 @@ class HanzoREPL:
|
||||
self.console.print("- etc.")
|
||||
sys.exit(1)
|
||||
|
||||
self.console.print(
|
||||
f"[green]Available providers: {', '.join(available_providers)}[/green]"
|
||||
)
|
||||
self.console.print(f"[green]Available providers: {', '.join(available_providers)}[/green]")
|
||||
|
||||
# Initialize tool executor
|
||||
self.tool_executor = ToolExecutor(self.mcp_server, self.llm_client)
|
||||
|
||||
# Initialize prompt session
|
||||
tool_names = [tool.name for tool in self.mcp_server.tools.values()]
|
||||
completer = WordCompleter(
|
||||
list(self.commands.keys()) + tool_names, ignore_case=True
|
||||
)
|
||||
completer = WordCompleter(list(self.commands.keys()) + tool_names, ignore_case=True)
|
||||
self.session = PromptSession(
|
||||
history=FileHistory(self.history_file),
|
||||
auto_suggest=AutoSuggestFromHistory(),
|
||||
@@ -203,15 +199,12 @@ Simply type your message and the REPL will:
|
||||
status = "Active" if provider == self.llm_client.current_provider else ""
|
||||
table.add_row(
|
||||
provider,
|
||||
", ".join(provider_models[:3])
|
||||
+ ("..." if len(provider_models) > 3 else ""),
|
||||
", ".join(provider_models[:3]) + ("..." if len(provider_models) > 3 else ""),
|
||||
status,
|
||||
)
|
||||
|
||||
self.console.print(table)
|
||||
self.console.print(
|
||||
f"\nCurrent model: [bold cyan]{self.llm_client.current_model}[/bold cyan]"
|
||||
)
|
||||
self.console.print(f"\nCurrent model: [bold cyan]{self.llm_client.current_model}[/bold cyan]")
|
||||
|
||||
async def set_model(self, command: str):
|
||||
"""Set the LLM model to use."""
|
||||
|
||||
@@ -9,9 +9,7 @@ from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
|
||||
async def run_tool_tests(
|
||||
console: Console, mcp_server: HanzoMCPServer, tool_executor: Any
|
||||
):
|
||||
async def run_tool_tests(console: Console, mcp_server: HanzoMCPServer, tool_executor: Any):
|
||||
"""Run comprehensive tests for MCP tools."""
|
||||
|
||||
console.print("\n[bold cyan]Running MCP Tool Tests[/bold cyan]\n")
|
||||
@@ -209,9 +207,7 @@ async def test_agent_delegation(
|
||||
try:
|
||||
tool = mcp_server.tools.get("dispatch_agent")
|
||||
if tool:
|
||||
result = await tool.execute(
|
||||
instruction="List the current directory contents"
|
||||
)
|
||||
result = await tool.execute(instruction="List the current directory contents")
|
||||
success = result is not None
|
||||
results.append(("Agent dispatch", success, "Agent completed task"))
|
||||
else:
|
||||
|
||||
@@ -239,9 +239,7 @@ class HanzoTextualREPL(App):
|
||||
# Show specific info based on backend
|
||||
if backend_name == "claude":
|
||||
if hasattr(backend, "authenticated") and backend.authenticated:
|
||||
messages.write(
|
||||
Text("● Using Claude personal account", style="cyan")
|
||||
)
|
||||
messages.write(Text("● Using Claude personal account", style="cyan"))
|
||||
else:
|
||||
messages.write(
|
||||
Text(
|
||||
@@ -250,9 +248,7 @@ class HanzoTextualREPL(App):
|
||||
)
|
||||
)
|
||||
elif backend_name == "embedded":
|
||||
messages.write(
|
||||
Text(f"● Model: {self.llm_client.current_model}", style="green")
|
||||
)
|
||||
messages.write(Text(f"● Model: {self.llm_client.current_model}", style="green"))
|
||||
|
||||
# Initialize tool executor with backend
|
||||
self.tool_executor = ToolExecutor(self.mcp_server, self.backend_manager)
|
||||
@@ -260,9 +256,7 @@ class HanzoTextualREPL(App):
|
||||
# List available backends
|
||||
backends = self.backend_manager.list_backends()
|
||||
available = [name for name, avail in backends.items() if avail]
|
||||
messages.write(
|
||||
Text(f"● Available backends: {', '.join(available)}", style="dim")
|
||||
)
|
||||
messages.write(Text(f"● Available backends: {', '.join(available)}", style="dim"))
|
||||
|
||||
except Exception as e:
|
||||
self.show_error(f"Initialization error: {e}")
|
||||
@@ -505,9 +499,7 @@ class HanzoTextualREPL(App):
|
||||
# Execute MCP tool
|
||||
await self.execute_tool(command.name, command.parameters)
|
||||
|
||||
async def execute_tool(
|
||||
self, tool_name: str, parameters: Optional[Dict] = None
|
||||
) -> None:
|
||||
async def execute_tool(self, tool_name: str, parameters: Optional[Dict] = None) -> None:
|
||||
"""Execute an MCP tool."""
|
||||
self.show_message(f"Executing: {tool_name}", "cyan")
|
||||
|
||||
@@ -522,10 +514,7 @@ class HanzoTextualREPL(App):
|
||||
return
|
||||
|
||||
# Get parameters if needed
|
||||
if parameters and any(
|
||||
p.get("required", False)
|
||||
for p in parameters.get("properties", {}).values()
|
||||
):
|
||||
if parameters and any(p.get("required", False) for p in parameters.get("properties", {}).values()):
|
||||
# TODO: Show parameter input dialog
|
||||
self.show_message("Tool requires parameters. Using defaults.", "yellow")
|
||||
params = {}
|
||||
@@ -637,9 +626,7 @@ class HanzoTextualREPL(App):
|
||||
self.llm_client.set_model(arg)
|
||||
self.show_message(f"Model changed to: {arg}", "green")
|
||||
else:
|
||||
self.show_message(
|
||||
"Model selection only available for embedded backend", "yellow"
|
||||
)
|
||||
self.show_message("Model selection only available for embedded backend", "yellow")
|
||||
return True
|
||||
if command == "auth":
|
||||
await self.handle_auth_command()
|
||||
@@ -683,9 +670,7 @@ class HanzoTextualREPL(App):
|
||||
from .voice_mode import VOICE_AVAILABLE, VoiceCommands, VoiceMode
|
||||
|
||||
if not VOICE_AVAILABLE:
|
||||
self.show_error(
|
||||
"Voice mode not available. Install: pip install speechrecognition pyttsx3 pyaudio"
|
||||
)
|
||||
self.show_error("Voice mode not available. Install: pip install speechrecognition pyttsx3 pyaudio")
|
||||
return
|
||||
|
||||
if not hasattr(self, "voice_mode"):
|
||||
@@ -700,17 +685,13 @@ class HanzoTextualREPL(App):
|
||||
# Start voice mode
|
||||
def on_speech(text: str):
|
||||
# Process voice input
|
||||
processed, should_stop = self.voice_commands.process_voice_input(
|
||||
text
|
||||
)
|
||||
processed, should_stop = self.voice_commands.process_voice_input(text)
|
||||
|
||||
if should_stop:
|
||||
self.call_from_thread(self.toggle_voice_mode)
|
||||
elif processed:
|
||||
# Show what was heard
|
||||
self.call_from_thread(
|
||||
self.show_message, f"Heard: {text}", "dim"
|
||||
)
|
||||
self.call_from_thread(self.show_message, f"Heard: {text}", "dim")
|
||||
# Process the command
|
||||
self.call_from_thread(self.process_voice_command, processed)
|
||||
|
||||
@@ -775,9 +756,7 @@ class HanzoTextualREPL(App):
|
||||
try:
|
||||
success = await backend.authenticate()
|
||||
if success:
|
||||
self.show_message(
|
||||
"Successfully authenticated with Claude!", "green"
|
||||
)
|
||||
self.show_message("Successfully authenticated with Claude!", "green")
|
||||
self.show_message(
|
||||
"You can now use your personal Claude account without API keys.",
|
||||
"cyan",
|
||||
@@ -787,9 +766,7 @@ class HanzoTextualREPL(App):
|
||||
except Exception as e:
|
||||
self.show_error(f"Authentication error: {e}")
|
||||
else:
|
||||
self.show_message(
|
||||
"Claude backend doesn't support authentication", "yellow"
|
||||
)
|
||||
self.show_message("Claude backend doesn't support authentication", "yellow")
|
||||
else:
|
||||
self.show_message(
|
||||
f"Authentication not available for {self.backend_manager.current_backend} backend",
|
||||
@@ -807,9 +784,7 @@ class HanzoTextualREPL(App):
|
||||
|
||||
for name, available in backends.items():
|
||||
status = "✓" if available else "✗"
|
||||
current = (
|
||||
" (current)" if name == self.backend_manager.current_backend else ""
|
||||
)
|
||||
current = " (current)" if name == self.backend_manager.current_backend else ""
|
||||
style = "green" if available else "red"
|
||||
messages.write(Text(f" {status} {name}{current}", style=style))
|
||||
|
||||
@@ -842,9 +817,7 @@ class HanzoTextualREPL(App):
|
||||
"""Handle /logout command."""
|
||||
backend = self.backend_manager.get_backend()
|
||||
|
||||
if self.backend_manager.current_backend == "claude" and hasattr(
|
||||
backend, "logout"
|
||||
):
|
||||
if self.backend_manager.current_backend == "claude" and hasattr(backend, "logout"):
|
||||
await backend.logout()
|
||||
self.show_message("Logged out from Claude account", "yellow")
|
||||
else:
|
||||
@@ -863,9 +836,7 @@ class HanzoTextualREPL(App):
|
||||
|
||||
if backend_name == "claude":
|
||||
auth_status = (
|
||||
"Authenticated"
|
||||
if hasattr(backend, "authenticated") and backend.authenticated
|
||||
else "Not authenticated"
|
||||
"Authenticated" if hasattr(backend, "authenticated") and backend.authenticated else "Not authenticated"
|
||||
)
|
||||
messages.write(f" Auth status: {auth_status}")
|
||||
elif backend_name == "embedded":
|
||||
|
||||
@@ -81,9 +81,7 @@ class ToolExecutor:
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
self.console.print(
|
||||
Panel(f"[bold red]Error:[/bold red] {str(e)}", border_style="red")
|
||||
)
|
||||
self.console.print(Panel(f"[bold red]Error:[/bold red] {str(e)}", border_style="red"))
|
||||
raise
|
||||
|
||||
async def execute_with_tools(self, user_message: str) -> str:
|
||||
@@ -135,19 +133,11 @@ Available tool categories:
|
||||
from types import SimpleNamespace
|
||||
|
||||
response = SimpleNamespace(
|
||||
choices=[
|
||||
SimpleNamespace(
|
||||
message=SimpleNamespace(
|
||||
content=response_text, tool_calls=None
|
||||
)
|
||||
)
|
||||
]
|
||||
choices=[SimpleNamespace(message=SimpleNamespace(content=response_text, tool_calls=None))]
|
||||
)
|
||||
else:
|
||||
# Legacy LLMClient
|
||||
response = await self.backend.chat(
|
||||
messages=messages, tools=tools, tool_choice="auto"
|
||||
)
|
||||
response = await self.backend.chat(messages=messages, tools=tools, tool_choice="auto")
|
||||
|
||||
# Extract response
|
||||
message = response.choices[0].message
|
||||
@@ -180,11 +170,7 @@ Available tool categories:
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": tool_call.id,
|
||||
"content": (
|
||||
json.dumps(result)
|
||||
if not isinstance(result, str)
|
||||
else result
|
||||
),
|
||||
"content": (json.dumps(result) if not isinstance(result, str) else result),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -205,9 +191,7 @@ Available tool categories:
|
||||
final_response = message.content
|
||||
|
||||
# Add to history
|
||||
self.conversation_history.append(
|
||||
{"role": "assistant", "content": final_response}
|
||||
)
|
||||
self.conversation_history.append({"role": "assistant", "content": final_response})
|
||||
|
||||
break
|
||||
|
||||
|
||||
@@ -21,9 +21,7 @@ class VoiceMode:
|
||||
|
||||
def __init__(self):
|
||||
if not VOICE_AVAILABLE:
|
||||
raise ImportError(
|
||||
"Voice dependencies not installed. Run: pip install speechrecognition pyttsx3 pyaudio"
|
||||
)
|
||||
raise ImportError("Voice dependencies not installed. Run: pip install speechrecognition pyttsx3 pyaudio")
|
||||
|
||||
# Speech recognition
|
||||
self.recognizer = sr.Recognizer()
|
||||
@@ -99,9 +97,7 @@ class VoiceMode:
|
||||
self.on_listening_started()
|
||||
|
||||
# Listen with timeout
|
||||
audio = self.recognizer.listen(
|
||||
source, timeout=1, phrase_time_limit=10
|
||||
)
|
||||
audio = self.recognizer.listen(source, timeout=1, phrase_time_limit=10)
|
||||
|
||||
# Signal listening stopped
|
||||
self.is_listening = False
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
"""Test all hanzo-tools-* packages import and register correctly."""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import time
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -143,7 +143,7 @@ class TestToolPackages:
|
||||
)
|
||||
def test_llm_tools(self):
|
||||
"""Test hanzo-tools-llm imports (tools depend on litellm)."""
|
||||
from hanzo_tools.llm import LLM_AVAILABLE, TOOLS
|
||||
from hanzo_tools.llm import TOOLS, LLM_AVAILABLE
|
||||
|
||||
# LLM tools are optional, depend on litellm
|
||||
if LLM_AVAILABLE:
|
||||
|
||||
@@ -25,7 +25,7 @@ Individual packages:
|
||||
Usage:
|
||||
from hanzo_tools.filesystem import register_tools as register_fs
|
||||
from hanzo_tools.shell import register_tools as register_shell
|
||||
|
||||
|
||||
# Register with MCP server
|
||||
register_fs(mcp_server, permission_manager)
|
||||
register_shell(mcp_server)
|
||||
@@ -39,16 +39,16 @@ __path__ = __import__("pkgutil").extend_path(__path__, __name__)
|
||||
|
||||
def discover_tools():
|
||||
"""Discover all installed tool packages.
|
||||
|
||||
|
||||
Uses entry points to find hanzo.tools plugins.
|
||||
|
||||
|
||||
Returns:
|
||||
Dict of package_name -> tools list
|
||||
"""
|
||||
import importlib.metadata
|
||||
|
||||
|
||||
tools = {}
|
||||
|
||||
|
||||
try:
|
||||
eps = importlib.metadata.entry_points(group="hanzo.tools")
|
||||
for ep in eps:
|
||||
@@ -60,53 +60,53 @@ def discover_tools():
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
return tools
|
||||
|
||||
|
||||
def register_all(mcp_server, permission_manager=None, enabled_tools=None):
|
||||
"""Register all discovered tools with the MCP server.
|
||||
|
||||
|
||||
Args:
|
||||
mcp_server: FastMCP server instance
|
||||
permission_manager: Optional permission manager
|
||||
enabled_tools: Dict of tool_name -> enabled state
|
||||
|
||||
|
||||
Returns:
|
||||
List of registered tool instances
|
||||
"""
|
||||
import importlib.metadata
|
||||
|
||||
|
||||
registered = []
|
||||
enabled = enabled_tools or {}
|
||||
|
||||
|
||||
try:
|
||||
eps = importlib.metadata.entry_points(group="hanzo.tools")
|
||||
for ep in eps:
|
||||
# Check if package is enabled
|
||||
if not enabled.get(ep.name, True):
|
||||
continue
|
||||
|
||||
|
||||
try:
|
||||
module_parent = ep.value.rsplit(":", 1)[0]
|
||||
module = __import__(module_parent, fromlist=["register_tools"])
|
||||
|
||||
|
||||
if hasattr(module, "register_tools"):
|
||||
try:
|
||||
tools = module.register_tools(
|
||||
mcp_server,
|
||||
mcp_server,
|
||||
permission_manager=permission_manager,
|
||||
enabled_tools=enabled,
|
||||
)
|
||||
except TypeError:
|
||||
# Some tools don't need permission_manager
|
||||
tools = module.register_tools(mcp_server, enabled_tools=enabled)
|
||||
|
||||
|
||||
if tools:
|
||||
registered.extend(tools)
|
||||
except Exception as e:
|
||||
print(f"Failed to register {ep.name}: {e}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
return registered
|
||||
|
||||
@@ -14,16 +14,16 @@ from hanzo.tools.detector import AITool, ToolDetector
|
||||
|
||||
class TestToolDetector(unittest.TestCase):
|
||||
"""Test AI tool detection functionality."""
|
||||
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test fixtures."""
|
||||
self.detector = ToolDetector()
|
||||
|
||||
|
||||
def test_tool_initialization(self):
|
||||
"""Test that tools are properly initialized."""
|
||||
# Check that we have tools defined
|
||||
self.assertGreater(len(self.detector.TOOLS), 0)
|
||||
|
||||
|
||||
# Check that each tool has required attributes
|
||||
for tool in self.detector.TOOLS:
|
||||
self.assertIsInstance(tool.name, str)
|
||||
@@ -31,29 +31,29 @@ class TestToolDetector(unittest.TestCase):
|
||||
self.assertIsInstance(tool.provider, str)
|
||||
self.assertIsInstance(tool.priority, int)
|
||||
self.assertIsInstance(tool.detected, bool)
|
||||
|
||||
|
||||
def test_priority_ordering(self):
|
||||
"""Test that tools have correct priority ordering."""
|
||||
tools = self.detector.TOOLS
|
||||
|
||||
|
||||
# Find specific tools
|
||||
hanzod = next((t for t in tools if t.name == "hanzod"), None)
|
||||
hanzo_router = next((t for t in tools if t.name == "hanzo-router"), None)
|
||||
claude_code = next((t for t in tools if t.name == "claude-code"), None)
|
||||
|
||||
|
||||
# Hanzo Node should have highest priority (0)
|
||||
if hanzod:
|
||||
self.assertEqual(hanzod.priority, 0)
|
||||
|
||||
|
||||
# Router should have high priority
|
||||
if hanzo_router:
|
||||
self.assertLessEqual(hanzo_router.priority, 2)
|
||||
|
||||
|
||||
# Claude Code should have reasonable priority
|
||||
if claude_code:
|
||||
self.assertLessEqual(claude_code.priority, 5)
|
||||
|
||||
@patch('hanzo.tools.detector.httpx.post')
|
||||
|
||||
@patch("hanzo.tools.detector.httpx.post")
|
||||
def test_hanzod_detection_success(self, mock_post):
|
||||
"""Test successful Hanzo Node detection."""
|
||||
# Mock successful response
|
||||
@@ -61,94 +61,95 @@ class TestToolDetector(unittest.TestCase):
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {"status": "ok"}
|
||||
mock_post.return_value = mock_response
|
||||
|
||||
|
||||
hanzod = AITool(
|
||||
name="hanzod",
|
||||
command="hanzo node",
|
||||
display_name="Hanzo Node",
|
||||
provider="hanzo-local",
|
||||
priority=0,
|
||||
api_endpoint="http://localhost:3690/health"
|
||||
api_endpoint="http://localhost:3690/health",
|
||||
)
|
||||
|
||||
|
||||
result = self.detector.detect_tool(hanzod)
|
||||
self.assertTrue(result)
|
||||
self.assertTrue(hanzod.detected)
|
||||
|
||||
@patch('hanzo.tools.detector.httpx.post')
|
||||
|
||||
@patch("hanzo.tools.detector.httpx.post")
|
||||
def test_hanzod_detection_failure_404(self, mock_post):
|
||||
"""Test Hanzo Node detection fails on 404."""
|
||||
# Mock 404 response
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 404
|
||||
mock_post.return_value = mock_response
|
||||
|
||||
|
||||
hanzod = AITool(
|
||||
name="hanzod",
|
||||
command="hanzo node",
|
||||
display_name="Hanzo Node",
|
||||
provider="hanzo-local",
|
||||
priority=0,
|
||||
api_endpoint="http://localhost:3690/health"
|
||||
api_endpoint="http://localhost:3690/health",
|
||||
)
|
||||
|
||||
|
||||
result = self.detector.detect_tool(hanzod)
|
||||
self.assertFalse(result)
|
||||
self.assertFalse(hanzod.detected)
|
||||
|
||||
@patch('hanzo.tools.detector.httpx.post')
|
||||
|
||||
@patch("hanzo.tools.detector.httpx.post")
|
||||
def test_hanzod_detection_connection_refused(self, mock_post):
|
||||
"""Test Hanzo Node detection handles connection refused."""
|
||||
import httpx
|
||||
|
||||
# Mock connection error
|
||||
mock_post.side_effect = httpx.ConnectError("Connection refused")
|
||||
|
||||
|
||||
hanzod = AITool(
|
||||
name="hanzod",
|
||||
command="hanzo node",
|
||||
display_name="Hanzo Node",
|
||||
provider="hanzo-local",
|
||||
priority=0,
|
||||
api_endpoint="http://localhost:3690/health"
|
||||
api_endpoint="http://localhost:3690/health",
|
||||
)
|
||||
|
||||
|
||||
result = self.detector.detect_tool(hanzod)
|
||||
self.assertFalse(result)
|
||||
self.assertFalse(hanzod.detected)
|
||||
|
||||
@patch('hanzo.tools.detector.shutil.which')
|
||||
|
||||
@patch("hanzo.tools.detector.shutil.which")
|
||||
def test_command_detection(self, mock_which):
|
||||
"""Test command-based tool detection."""
|
||||
# Mock command exists
|
||||
mock_which.return_value = "/usr/local/bin/claude"
|
||||
|
||||
|
||||
claude_tool = AITool(
|
||||
name="claude-code",
|
||||
command="claude",
|
||||
display_name="Claude Code",
|
||||
provider="anthropic",
|
||||
priority=3,
|
||||
check_command="claude"
|
||||
check_command="claude",
|
||||
)
|
||||
|
||||
|
||||
result = self.detector.detect_tool(claude_tool)
|
||||
self.assertTrue(result)
|
||||
self.assertTrue(claude_tool.detected)
|
||||
|
||||
|
||||
# Mock command doesn't exist
|
||||
mock_which.return_value = None
|
||||
claude_tool.detected = False
|
||||
|
||||
|
||||
result = self.detector.detect_tool(claude_tool)
|
||||
self.assertFalse(result)
|
||||
self.assertFalse(claude_tool.detected)
|
||||
|
||||
@patch.dict(os.environ, {'OPENAI_API_KEY': 'test-key'})
|
||||
@patch('hanzo.tools.detector.shutil.which')
|
||||
|
||||
@patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"})
|
||||
@patch("hanzo.tools.detector.shutil.which")
|
||||
def test_env_var_detection(self, mock_which):
|
||||
"""Test environment variable based detection."""
|
||||
mock_which.return_value = "/usr/local/bin/openai"
|
||||
|
||||
|
||||
openai_tool = AITool(
|
||||
name="openai",
|
||||
command="openai",
|
||||
@@ -156,24 +157,24 @@ class TestToolDetector(unittest.TestCase):
|
||||
provider="openai",
|
||||
priority=4,
|
||||
check_command="openai",
|
||||
env_var="OPENAI_API_KEY"
|
||||
env_var="OPENAI_API_KEY",
|
||||
)
|
||||
|
||||
|
||||
result = self.detector.detect_tool(openai_tool)
|
||||
self.assertTrue(result)
|
||||
self.assertTrue(openai_tool.detected)
|
||||
|
||||
|
||||
@patch.dict(os.environ, {}, clear=True) # Clear all env vars
|
||||
@patch('hanzo.tools.detector.shutil.which')
|
||||
@patch("hanzo.tools.detector.shutil.which")
|
||||
def test_env_var_missing(self, mock_which):
|
||||
"""Test that tool is still detected when command exists but env var is missing.
|
||||
|
||||
|
||||
The tool detection logic checks for command existence first,
|
||||
and env_var is only a fallback. A tool can be "detected" but
|
||||
may not be fully functional without the API key.
|
||||
"""
|
||||
mock_which.return_value = "/usr/local/bin/openai"
|
||||
|
||||
|
||||
openai_tool = AITool(
|
||||
name="openai",
|
||||
command="openai",
|
||||
@@ -181,20 +182,20 @@ class TestToolDetector(unittest.TestCase):
|
||||
provider="openai",
|
||||
priority=4,
|
||||
check_command="openai",
|
||||
env_var="OPENAI_API_KEY"
|
||||
env_var="OPENAI_API_KEY",
|
||||
)
|
||||
|
||||
|
||||
result = self.detector.detect_tool(openai_tool)
|
||||
# Tool is detected because command exists, even without API key
|
||||
self.assertTrue(result)
|
||||
self.assertTrue(openai_tool.detected)
|
||||
|
||||
|
||||
@patch.dict(os.environ, {}, clear=True) # Clear all env vars
|
||||
@patch('hanzo.tools.detector.shutil.which')
|
||||
@patch("hanzo.tools.detector.shutil.which")
|
||||
def test_no_command_no_env_var(self, mock_which):
|
||||
"""Test detection fails when neither command nor env var exists."""
|
||||
mock_which.return_value = None # Command doesn't exist
|
||||
|
||||
|
||||
openai_tool = AITool(
|
||||
name="openai",
|
||||
command="openai",
|
||||
@@ -202,68 +203,70 @@ class TestToolDetector(unittest.TestCase):
|
||||
provider="openai",
|
||||
priority=4,
|
||||
check_command="openai",
|
||||
env_var="OPENAI_API_KEY"
|
||||
env_var="OPENAI_API_KEY",
|
||||
)
|
||||
|
||||
|
||||
result = self.detector.detect_tool(openai_tool)
|
||||
self.assertFalse(result)
|
||||
self.assertFalse(openai_tool.detected)
|
||||
|
||||
@patch('hanzo.tools.detector.subprocess.run')
|
||||
@patch('hanzo.tools.detector.shutil.which')
|
||||
|
||||
@patch("hanzo.tools.detector.subprocess.run")
|
||||
@patch("hanzo.tools.detector.shutil.which")
|
||||
def test_version_detection(self, mock_which, mock_run):
|
||||
"""Test version detection for tools."""
|
||||
mock_which.return_value = "/usr/local/bin/hanzo"
|
||||
|
||||
|
||||
# Mock version command output
|
||||
mock_result = Mock()
|
||||
mock_result.stdout = "hanzo version 0.3.23\n"
|
||||
mock_result.returncode = 0
|
||||
mock_run.return_value = mock_result
|
||||
|
||||
|
||||
hanzo_tool = AITool(
|
||||
name="hanzo-dev",
|
||||
command="hanzo dev",
|
||||
display_name="Hanzo Dev",
|
||||
provider="hanzo",
|
||||
priority=2,
|
||||
check_command="hanzo"
|
||||
check_command="hanzo",
|
||||
)
|
||||
|
||||
|
||||
result = self.detector.detect_tool(hanzo_tool)
|
||||
self.assertTrue(result)
|
||||
self.assertIn("0.3.23", hanzo_tool.version or "")
|
||||
|
||||
@patch('hanzo.tools.detector.httpx.post')
|
||||
@patch('hanzo.tools.detector.shutil.which')
|
||||
|
||||
@patch("hanzo.tools.detector.httpx.post")
|
||||
@patch("hanzo.tools.detector.shutil.which")
|
||||
def test_detect_all(self, mock_which, mock_post):
|
||||
"""Test detecting all available tools."""
|
||||
|
||||
# Mock some tools as available
|
||||
def which_side_effect(cmd):
|
||||
if cmd in ["claude", "hanzo", "openai"]:
|
||||
return f"/usr/local/bin/{cmd}"
|
||||
return None
|
||||
|
||||
|
||||
mock_which.side_effect = which_side_effect
|
||||
|
||||
|
||||
# Mock Hanzo Node as not running
|
||||
import httpx
|
||||
|
||||
mock_post.side_effect = httpx.ConnectError("Connection refused")
|
||||
|
||||
|
||||
# Set OpenAI API key for testing
|
||||
with patch.dict(os.environ, {'OPENAI_API_KEY': 'test-key'}):
|
||||
with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}):
|
||||
detected_tools = self.detector.detect_all()
|
||||
|
||||
|
||||
# Should have detected some tools
|
||||
self.assertGreater(len(detected_tools), 0)
|
||||
|
||||
|
||||
# Check that detected tools are marked as such
|
||||
for tool in detected_tools:
|
||||
if tool.name in ["claude-code", "hanzo-dev", "openai"]:
|
||||
self.assertTrue(tool.detected, f"{tool.name} should be detected")
|
||||
elif tool.name == "hanzod":
|
||||
self.assertFalse(tool.detected, "hanzod should not be detected")
|
||||
|
||||
|
||||
def test_get_default_tool(self):
|
||||
"""Test getting the default tool."""
|
||||
# Create mock tools with different priorities (already sorted)
|
||||
@@ -272,51 +275,51 @@ class TestToolDetector(unittest.TestCase):
|
||||
AITool("tool4", "cmd4", "Tool 4", "provider4", priority=3, detected=True),
|
||||
AITool("tool1", "cmd1", "Tool 1", "provider1", priority=5, detected=True),
|
||||
]
|
||||
|
||||
|
||||
self.detector.detected_tools = tools
|
||||
|
||||
|
||||
# Default should be the first tool (tool2 with priority 2)
|
||||
default = self.detector.get_default_tool()
|
||||
self.assertIsNotNone(default)
|
||||
self.assertEqual(default.name, "tool2")
|
||||
self.assertEqual(default.priority, 2)
|
||||
|
||||
|
||||
def test_get_default_tool_none_detected(self):
|
||||
"""Test get_default_tool returns None when no tools detected."""
|
||||
# Empty detected_tools list
|
||||
self.detector.detected_tools = []
|
||||
|
||||
|
||||
# Mock detect_all to return empty list
|
||||
with patch.object(self.detector, 'detect_all', return_value=[]):
|
||||
with patch.object(self.detector, "detect_all", return_value=[]):
|
||||
default = self.detector.get_default_tool()
|
||||
self.assertIsNone(default)
|
||||
|
||||
@patch('hanzo.tools.detector.httpx.post')
|
||||
|
||||
@patch("hanzo.tools.detector.httpx.post")
|
||||
def test_port_fallback_for_hanzod(self, mock_post):
|
||||
"""Test that Hanzo Node tries both port 3690 and 8000."""
|
||||
import httpx
|
||||
|
||||
|
||||
# First call to port 3690 fails, second to 8000 succeeds
|
||||
responses = [
|
||||
httpx.ConnectError("Connection refused"), # Port 3690 fails
|
||||
Mock(status_code=200) # Port 8000 succeeds
|
||||
Mock(status_code=200), # Port 8000 succeeds
|
||||
]
|
||||
mock_post.side_effect = responses
|
||||
|
||||
|
||||
hanzod = AITool(
|
||||
name="hanzod",
|
||||
command="hanzo node",
|
||||
display_name="Hanzo Node",
|
||||
provider="hanzo-local",
|
||||
priority=0,
|
||||
api_endpoint="http://localhost:3690/health"
|
||||
api_endpoint="http://localhost:3690/health",
|
||||
)
|
||||
|
||||
|
||||
result = self.detector.detect_tool(hanzod)
|
||||
|
||||
|
||||
# Should have tried both ports
|
||||
self.assertEqual(mock_post.call_count, 2)
|
||||
|
||||
|
||||
# Check that both ports were tried
|
||||
calls = mock_post.call_args_list
|
||||
urls = [call[0][0] for call in calls]
|
||||
@@ -325,4 +328,4 @@ class TestToolDetector(unittest.TestCase):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
unittest.main()
|
||||
|
||||
@@ -125,9 +125,7 @@ class BaseAgent(ABC, Generic[TContext, TResult]):
|
||||
|
||||
# Log start
|
||||
if context and hasattr(context, "log"):
|
||||
await context.log(
|
||||
f"Starting {self.name} with model {self.config.model}"
|
||||
)
|
||||
await context.log(f"Starting {self.name} with model {self.config.model}")
|
||||
|
||||
# Execute with retries
|
||||
result = await self._execute_with_retries(prompt, context, env, **kwargs)
|
||||
@@ -145,11 +143,7 @@ class BaseAgent(ABC, Generic[TContext, TResult]):
|
||||
|
||||
except Exception as e:
|
||||
self._end_time = datetime.now()
|
||||
duration = (
|
||||
(self._end_time - self._start_time).total_seconds()
|
||||
if self._start_time
|
||||
else None
|
||||
)
|
||||
duration = (self._end_time - self._start_time).total_seconds() if self._start_time else None
|
||||
|
||||
logger.error(f"Agent {self.name} failed: {e}")
|
||||
|
||||
@@ -231,9 +225,7 @@ class BaseAgent(ABC, Generic[TContext, TResult]):
|
||||
if attempt < self.config.max_retries - 1:
|
||||
await asyncio.sleep(2**attempt)
|
||||
|
||||
raise Exception(
|
||||
f"All {self.config.max_retries} attempts failed. Last error: {last_error}"
|
||||
)
|
||||
raise Exception(f"All {self.config.max_retries} attempts failed. Last error: {last_error}")
|
||||
|
||||
@abstractmethod
|
||||
async def _execute_impl(
|
||||
@@ -326,9 +318,7 @@ class CLIAgent(BaseAgent[TContext, str]):
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
process.kill()
|
||||
raise asyncio.TimeoutError(
|
||||
f"Command timed out after {self.config.timeout} seconds"
|
||||
)
|
||||
raise asyncio.TimeoutError(f"Command timed out after {self.config.timeout} seconds")
|
||||
|
||||
# Check for errors
|
||||
if process.returncode != 0:
|
||||
|
||||
@@ -55,11 +55,7 @@ class BatchTask:
|
||||
@property
|
||||
def success(self) -> bool:
|
||||
"""Check if task succeeded."""
|
||||
return (
|
||||
self.status == "completed"
|
||||
and self.result
|
||||
and getattr(self.result, "success", True)
|
||||
)
|
||||
return self.status == "completed" and self.result and getattr(self.result, "success", True)
|
||||
|
||||
def get_error(self) -> Optional[str]:
|
||||
"""Get error message from direct error or result."""
|
||||
@@ -161,9 +157,7 @@ class BatchConfig:
|
||||
if not config.consensus_mode and not config.critic_mode:
|
||||
agent_match = re.search(r"agent:(\w+)", command)
|
||||
if agent_match:
|
||||
config.agent_model = agent_match.group(
|
||||
1
|
||||
) # Will be resolved in __post_init__
|
||||
config.agent_model = agent_match.group(1) # Will be resolved in __post_init__
|
||||
|
||||
# Parse worktree option
|
||||
worktree_match = re.search(r"worktree:(true|false)", command)
|
||||
@@ -220,9 +214,7 @@ class BatchOrchestrator:
|
||||
self._task_counter += 1
|
||||
return f"task_{self._task_counter:04d}"
|
||||
|
||||
async def _setup_worktree(
|
||||
self, task_id: str, config: BatchConfig
|
||||
) -> Optional[Path]:
|
||||
async def _setup_worktree(self, task_id: str, config: BatchConfig) -> Optional[Path]:
|
||||
"""Setup git worktree for parallel editing.
|
||||
|
||||
Args:
|
||||
@@ -536,9 +528,7 @@ class BatchOrchestrator:
|
||||
|
||||
# If consensus reached, combine insights
|
||||
if consensus_result["consensus_reached"] and successful_responses:
|
||||
combined = "\n\n".join(
|
||||
[f"[{r['model']}]: {r['response']}" for r in successful_responses]
|
||||
)
|
||||
combined = "\n\n".join([f"[{r['model']}]: {r['response']}" for r in successful_responses])
|
||||
consensus_result["combined_response"] = combined
|
||||
|
||||
return consensus_result
|
||||
@@ -627,9 +617,7 @@ Provide your critical analysis:"""
|
||||
|
||||
return {
|
||||
"critic_chain": reviews,
|
||||
"final_review": (
|
||||
reviews[-1]["review"] if reviews and "review" in reviews[-1] else None
|
||||
),
|
||||
"final_review": (reviews[-1]["review"] if reviews and "review" in reviews[-1] else None),
|
||||
"models_used": models,
|
||||
"chain_length": len(reviews),
|
||||
}
|
||||
@@ -706,11 +694,7 @@ Provide your critical analysis:"""
|
||||
if "review" in review:
|
||||
console.print(
|
||||
Panel(
|
||||
(
|
||||
review["review"][:500] + "..."
|
||||
if len(review["review"]) > 500
|
||||
else review["review"]
|
||||
),
|
||||
(review["review"][:500] + "..." if len(review["review"]) > 500 else review["review"]),
|
||||
title=f"[cyan]Critic {review['iteration']}: {review['model']}[/cyan]",
|
||||
)
|
||||
)
|
||||
@@ -785,9 +769,7 @@ Provide your critical analysis:"""
|
||||
del self.active_tasks[task.id]
|
||||
|
||||
# Generate summary
|
||||
total_duration = sum(
|
||||
t.duration() or 0 for t in self.completed_tasks + self.failed_tasks
|
||||
)
|
||||
total_duration = sum(t.duration() or 0 for t in self.completed_tasks + self.failed_tasks)
|
||||
|
||||
summary = {
|
||||
"total_tasks": len(tasks),
|
||||
|
||||
+14
-42
@@ -115,15 +115,9 @@ def serve(ctx, name: str, port: int):
|
||||
|
||||
@cli.command()
|
||||
@click.option("--name", "-n", help="Node name (auto-generated if not provided)")
|
||||
@click.option(
|
||||
"--port", "-p", default=52415, help="Node port (default: 52415 for hanzo/net)"
|
||||
)
|
||||
@click.option(
|
||||
"--network", default="local", help="Network to join (mainnet/testnet/local)"
|
||||
)
|
||||
@click.option(
|
||||
"--models", "-m", multiple=True, help="Models to serve (e.g., llama-3.2-3b)"
|
||||
)
|
||||
@click.option("--port", "-p", default=52415, help="Node port (default: 52415 for hanzo/net)")
|
||||
@click.option("--network", default="local", help="Network to join (mainnet/testnet/local)")
|
||||
@click.option("--models", "-m", multiple=True, help="Models to serve (e.g., llama-3.2-3b)")
|
||||
@click.option("--max-jobs", type=int, default=10, help="Max concurrent jobs")
|
||||
@click.pass_context
|
||||
def net(ctx, name: str, port: int, network: str, models: tuple, max_jobs: int):
|
||||
@@ -137,15 +131,9 @@ def net(ctx, name: str, port: int, network: str, models: tuple, max_jobs: int):
|
||||
|
||||
@cli.command()
|
||||
@click.option("--name", "-n", help="Node name (auto-generated if not provided)")
|
||||
@click.option(
|
||||
"--port", "-p", default=52415, help="Node port (default: 52415 for hanzo/net)"
|
||||
)
|
||||
@click.option(
|
||||
"--network", default="local", help="Network to join (mainnet/testnet/local)"
|
||||
)
|
||||
@click.option(
|
||||
"--models", "-m", multiple=True, help="Models to serve (e.g., llama-3.2-3b)"
|
||||
)
|
||||
@click.option("--port", "-p", default=52415, help="Node port (default: 52415 for hanzo/net)")
|
||||
@click.option("--network", default="local", help="Network to join (mainnet/testnet/local)")
|
||||
@click.option("--models", "-m", multiple=True, help="Models to serve (e.g., llama-3.2-3b)")
|
||||
@click.option("--max-jobs", type=int, default=10, help="Max concurrent jobs")
|
||||
@click.pass_context
|
||||
def node(ctx, name: str, port: int, network: str, models: tuple, max_jobs: int):
|
||||
@@ -180,15 +168,9 @@ def node(ctx, name: str, port: int, network: str, models: tuple, max_jobs: int):
|
||||
@click.option("--repl", is_flag=True, help="Start REPL interface (default)")
|
||||
@click.option("--instances", type=int, default=2, help="Number of worker agents")
|
||||
@click.option("--mcp-tools", is_flag=True, default=True, help="Enable all MCP tools")
|
||||
@click.option(
|
||||
"--network-mode", is_flag=True, default=True, help="Network agents together"
|
||||
)
|
||||
@click.option(
|
||||
"--guardrails", is_flag=True, default=True, help="Enable code quality guardrails"
|
||||
)
|
||||
@click.option(
|
||||
"--use-network/--no-network", default=True, help="Use hanzo-network if available"
|
||||
)
|
||||
@click.option("--network-mode", is_flag=True, default=True, help="Network agents together")
|
||||
@click.option("--guardrails", is_flag=True, default=True, help="Enable code quality guardrails")
|
||||
@click.option("--use-network/--no-network", default=True, help="Use hanzo-network if available")
|
||||
@click.option(
|
||||
"--use-hanzo-net",
|
||||
is_flag=True,
|
||||
@@ -266,9 +248,7 @@ def dev(
|
||||
console.print(f" Router Endpoint: {orch_config.router.endpoint}")
|
||||
if orch_config.codex:
|
||||
console.print(f" Codex Model: {orch_config.codex.model}")
|
||||
console.print(
|
||||
f" Cost Optimization: {'Enabled' if orch_config.enable_cost_optimization else 'Disabled'}"
|
||||
)
|
||||
console.print(f" Cost Optimization: {'Enabled' if orch_config.enable_cost_optimization else 'Disabled'}")
|
||||
console.print()
|
||||
|
||||
asyncio.run(
|
||||
@@ -351,9 +331,7 @@ async def start_compute_node(
|
||||
|
||||
console.print(f"\n[green]✓[/green] Node initialized")
|
||||
console.print(f" Port: {port}")
|
||||
console.print(
|
||||
f" Models: {', '.join(models) if models else 'auto-detect'}"
|
||||
)
|
||||
console.print(f" Models: {', '.join(models) if models else 'auto-detect'}")
|
||||
console.print("\n[bold green]Hanzo Net is running![/bold green]")
|
||||
console.print("WebUI: http://localhost:52415")
|
||||
console.print("API: http://localhost:52415/v1/chat/completions")
|
||||
@@ -375,9 +353,7 @@ async def start_compute_node(
|
||||
stop_task = asyncio.create_task(stop_event.wait())
|
||||
|
||||
# Wait for either net to complete or stop signal
|
||||
done, pending = await asyncio.wait(
|
||||
[net_task, stop_task], return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
done, pending = await asyncio.wait([net_task, stop_task], return_when=asyncio.FIRST_COMPLETED)
|
||||
|
||||
# Cancel pending tasks
|
||||
for task in pending:
|
||||
@@ -413,15 +389,11 @@ async def start_compute_node(
|
||||
env["NET_MODELS"] = ",".join(models)
|
||||
if name:
|
||||
env["NET_NODE_NAME"] = name
|
||||
env["PYTHONPATH"] = (
|
||||
os.path.join(net_path, "src") + ":" + env.get("PYTHONPATH", "")
|
||||
)
|
||||
env["PYTHONPATH"] = os.path.join(net_path, "src") + ":" + env.get("PYTHONPATH", "")
|
||||
|
||||
console.print(f"\n[green]✓[/green] Starting net node")
|
||||
console.print(f" Port: {port}")
|
||||
console.print(
|
||||
f" Models: {', '.join(models) if models else 'auto-detect'}"
|
||||
)
|
||||
console.print(f" Models: {', '.join(models) if models else 'auto-detect'}")
|
||||
console.print("\n[bold green]Hanzo Net is running![/bold green]")
|
||||
console.print("WebUI: http://localhost:52415")
|
||||
console.print("API: http://localhost:52415/v1/chat/completions")
|
||||
|
||||
@@ -88,11 +88,7 @@ async def run(ctx, agents: tuple, task: str, parallel: bool, timeout: Optional[i
|
||||
network = create_network(agents=agent_list)
|
||||
|
||||
# Run task
|
||||
result = (
|
||||
await asyncio.wait_for(network.run(task), timeout=timeout)
|
||||
if timeout
|
||||
else await network.run(task)
|
||||
)
|
||||
result = await asyncio.wait_for(network.run(task), timeout=timeout) if timeout else await network.run(task)
|
||||
|
||||
console.print("[green]Task completed![/green]")
|
||||
console.print(result)
|
||||
|
||||
@@ -127,9 +127,7 @@ def login(ctx, email: str, password: str, api_key: str, sso: bool):
|
||||
|
||||
hanzo_auth = HanzoAuth()
|
||||
# Email auth not implemented yet
|
||||
console.print(
|
||||
"[yellow]Email authentication not yet implemented[/yellow]"
|
||||
)
|
||||
console.print("[yellow]Email authentication not yet implemented[/yellow]")
|
||||
console.print("[dim]Saving credentials locally for development[/dim]")
|
||||
|
||||
auth = {
|
||||
@@ -249,9 +247,7 @@ def whoami():
|
||||
|
||||
content = "\n".join(lines) if lines else "[dim]No user information available[/dim]"
|
||||
|
||||
console.print(
|
||||
Panel(content, title="[bold cyan]User Information[/bold cyan]", box=box.ROUNDED)
|
||||
)
|
||||
console.print(Panel(content, title="[bold cyan]User Information[/bold cyan]", box=box.ROUNDED))
|
||||
|
||||
|
||||
@auth_group.command(name="set-key")
|
||||
|
||||
@@ -18,9 +18,7 @@ from ..utils.output import console
|
||||
@click.option("--local/--cloud", default=True, help="Use local or cloud model")
|
||||
@click.option("--once", is_flag=True, help="Single question mode")
|
||||
@click.option("--system", "-s", help="System prompt")
|
||||
@click.option(
|
||||
"--repl", is_flag=True, help="Start full REPL interface (like Claude Code)"
|
||||
)
|
||||
@click.option("--repl", is_flag=True, help="Start full REPL interface (like Claude Code)")
|
||||
@click.option("--ipython", is_flag=True, help="Use IPython REPL interface")
|
||||
@click.option("--tui", is_flag=True, help="Use beautiful TUI interface")
|
||||
@click.argument("prompt", nargs=-1)
|
||||
@@ -80,9 +78,7 @@ def chat_command(
|
||||
asyncio.run(interactive_chat(ctx, model, local, system))
|
||||
|
||||
|
||||
async def ask_once(
|
||||
ctx, prompt: str, model: str, local: bool, system: Optional[str] = None
|
||||
):
|
||||
async def ask_once(ctx, prompt: str, model: str, local: bool, system: Optional[str] = None):
|
||||
"""Ask a single question."""
|
||||
messages = []
|
||||
if system:
|
||||
@@ -172,9 +168,7 @@ async def interactive_chat(ctx, model: str, local: bool, system: Optional[str]):
|
||||
from prompt_toolkit import PromptSession
|
||||
from prompt_toolkit.history import FileHistory
|
||||
|
||||
console.print(
|
||||
f"[cyan]Chat session started[/cyan] (model: {model}, mode: {'local' if local else 'cloud'})"
|
||||
)
|
||||
console.print(f"[cyan]Chat session started[/cyan] (model: {model}, mode: {'local' if local else 'cloud'})")
|
||||
console.print("Type 'exit' or Ctrl+D to quit\n")
|
||||
|
||||
session = PromptSession(history=FileHistory(".hanzo_chat_history"))
|
||||
|
||||
@@ -115,9 +115,7 @@ def set(ctx, key: str, value: str, is_global: bool, is_local: bool):
|
||||
# Save config
|
||||
save_config(config_path, config)
|
||||
|
||||
console.print(
|
||||
f"[green]✓[/green] Set {key} = {parsed_value} in {config_name} config"
|
||||
)
|
||||
console.print(f"[green]✓[/green] Set {key} = {parsed_value} in {config_name} config")
|
||||
|
||||
|
||||
@config_group.command()
|
||||
|
||||
@@ -96,9 +96,7 @@ async def tools(ctx, category: str):
|
||||
|
||||
# Display tools
|
||||
for cat, tools in sorted(categories.items()):
|
||||
table = Table(
|
||||
title=f"{cat.title()} Tools" if len(categories) > 1 else "MCP Tools"
|
||||
)
|
||||
table = Table(title=f"{cat.title()} Tools" if len(categories) > 1 else "MCP Tools")
|
||||
table.add_column("Name", style="cyan", no_wrap=True)
|
||||
table.add_column("Description")
|
||||
|
||||
@@ -241,9 +239,7 @@ def install(ctx, path: str):
|
||||
existing["mcpServers"]["hanzo-mcp"] = config["mcpServers"]["hanzo-mcp"]
|
||||
config = existing
|
||||
except Exception as e:
|
||||
console.print(
|
||||
f"[yellow]Warning: Could not read existing config: {e}[/yellow]"
|
||||
)
|
||||
console.print(f"[yellow]Warning: Could not read existing config: {e}[/yellow]")
|
||||
|
||||
# Write config
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -50,9 +50,7 @@ async def start(
|
||||
if not click.confirm("Continue without wallet?"):
|
||||
return
|
||||
|
||||
miner = HanzoMiner(
|
||||
name=name, wallet=wallet, device=device, network=network, min_stake=min_stake
|
||||
)
|
||||
miner = HanzoMiner(name=name, wallet=wallet, device=device, network=network, min_stake=min_stake)
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
@@ -94,9 +92,7 @@ async def start(
|
||||
if event["type"] == "log":
|
||||
console.print(event["message"], end="")
|
||||
elif event["type"] == "job":
|
||||
console.print(
|
||||
f"[green]Job completed:[/green] {event['job_id']} (+{event['tokens']} tokens)"
|
||||
)
|
||||
console.print(f"[green]Job completed:[/green] {event['job_id']} (+{event['tokens']} tokens)")
|
||||
elif event["type"] == "error":
|
||||
console.print(f"[red]Error:[/red] {event['message']}")
|
||||
except KeyboardInterrupt:
|
||||
@@ -109,9 +105,7 @@ async def start(
|
||||
console.print("\n[cyan]Session Summary:[/cyan]")
|
||||
console.print(f" Jobs completed: {final_stats.get('jobs_completed', 0)}")
|
||||
console.print(f" Tokens earned: {final_stats.get('tokens_earned', 0)}")
|
||||
console.print(
|
||||
f" Average job time: {final_stats.get('avg_job_time', 'N/A')}"
|
||||
)
|
||||
console.print(f" Average job time: {final_stats.get('avg_job_time', 'N/A')}")
|
||||
|
||||
console.print("[green]✓[/green] Miner stopped")
|
||||
|
||||
@@ -207,15 +201,11 @@ async def status(ctx, name: str, detailed: bool):
|
||||
console.print(f" Network: {stats.get('network', 'unknown')}")
|
||||
console.print(f" Wallet: {stats.get('wallet', 'Not set')}")
|
||||
console.print(f" Models: {', '.join(stats.get('models', []))}")
|
||||
console.print(
|
||||
f" Memory: {stats.get('memory_used', 0)} / {stats.get('memory_total', 0)} MB"
|
||||
)
|
||||
console.print(f" Memory: {stats.get('memory_used', 0)} / {stats.get('memory_total', 0)} MB")
|
||||
console.print(f" CPU: {stats.get('cpu_percent', 0)}%")
|
||||
|
||||
if gpu := stats.get("gpu"):
|
||||
console.print(
|
||||
f" GPU: {gpu['name']} ({gpu['memory_used']} / {gpu['memory_total']} MB)"
|
||||
)
|
||||
console.print(f" GPU: {gpu['name']} ({gpu['memory_used']} / {gpu['memory_total']} MB)")
|
||||
|
||||
console.print(f"\n[cyan]Performance:[/cyan]")
|
||||
console.print(f" Average job time: {stats.get('avg_job_time', 'N/A')}")
|
||||
@@ -292,9 +282,7 @@ async def earnings(ctx, wallet: str, network: str):
|
||||
if history := data.get("recent_jobs"):
|
||||
console.print("\n[cyan]Recent Jobs:[/cyan]")
|
||||
for job in history[:5]:
|
||||
console.print(
|
||||
f" • {job['timestamp']}: +{job['tokens']} tokens ({job['model']})"
|
||||
)
|
||||
console.print(f" • {job['timestamp']}: +{job['tokens']} tokens ({job['model']})")
|
||||
|
||||
|
||||
@miner_group.command()
|
||||
@@ -323,9 +311,7 @@ async def withdraw(ctx, amount: float, wallet: str, to: str, network: str):
|
||||
|
||||
with console.status("Processing withdrawal..."):
|
||||
try:
|
||||
result = await withdraw_earnings(
|
||||
wallet=wallet, amount=amount, destination=to, network=network
|
||||
)
|
||||
result = await withdraw_earnings(wallet=wallet, amount=amount, destination=to, network=network)
|
||||
|
||||
console.print(f"[green]✓[/green] Withdrawal successful")
|
||||
console.print(f" Transaction: {result['tx_hash']}")
|
||||
|
||||
@@ -26,9 +26,7 @@ def network_group():
|
||||
@click.option("--consensus", is_flag=True, help="Require consensus")
|
||||
@click.option("--timeout", "-t", type=int, default=300, help="Timeout in seconds")
|
||||
@click.pass_context
|
||||
async def dispatch(
|
||||
ctx, prompt: str, agents: int, model: str, mode: str, consensus: bool, timeout: int
|
||||
):
|
||||
async def dispatch(ctx, prompt: str, agents: int, model: str, mode: str, consensus: bool, timeout: int):
|
||||
"""Dispatch work to agent network."""
|
||||
try:
|
||||
from hanzo_network import NetworkDispatcher
|
||||
@@ -391,13 +389,9 @@ def topology(ctx, json: bool):
|
||||
)
|
||||
|
||||
if gpu_available:
|
||||
console.print(
|
||||
"[green]✓[/green] GPU/Accelerator detected and available for AI workloads"
|
||||
)
|
||||
console.print("[green]✓[/green] GPU/Accelerator detected and available for AI workloads")
|
||||
if "APPLE M" in caps.chip.upper():
|
||||
console.print(
|
||||
" [dim]Apple Silicon Neural Engine available for on-device AI[/dim]"
|
||||
)
|
||||
console.print(" [dim]Apple Silicon Neural Engine available for on-device AI[/dim]")
|
||||
else:
|
||||
console.print("[yellow]⚠[/yellow] No GPU detected - using CPU only")
|
||||
console.print(" To enable GPU support:")
|
||||
@@ -547,9 +541,7 @@ def topology_add(
|
||||
try:
|
||||
import qrcode
|
||||
except ImportError:
|
||||
console.print(
|
||||
"\n[yellow]QR code generation requires qrcode library[/yellow]"
|
||||
)
|
||||
console.print("\n[yellow]QR code generation requires qrcode library[/yellow]")
|
||||
console.print("Install with: pip install qrcode[pil]")
|
||||
return
|
||||
|
||||
@@ -700,10 +692,7 @@ def models(ctx, endpoint: str):
|
||||
models_list = [
|
||||
model
|
||||
for model in all_models
|
||||
if not any(
|
||||
keyword in model.get("id", "").lower()
|
||||
for keyword in embedding_keywords
|
||||
)
|
||||
if not any(keyword in model.get("id", "").lower() for keyword in embedding_keywords)
|
||||
]
|
||||
|
||||
if models_list:
|
||||
@@ -741,16 +730,12 @@ def models(ctx, endpoint: str):
|
||||
table.add_row(str(i), model_id, tier, provider)
|
||||
|
||||
console.print(table)
|
||||
console.print(
|
||||
f"\n[dim]Total LLMs: {len(models_list)} (embedding models filtered out)[/dim]"
|
||||
)
|
||||
console.print(f"\n[dim]Total LLMs: {len(models_list)} (embedding models filtered out)[/dim]")
|
||||
else:
|
||||
console.print("[yellow]No models available on gateway[/yellow]")
|
||||
else:
|
||||
# Show static list of known models
|
||||
console.print(
|
||||
"[yellow]Could not fetch live models, showing known models:[/yellow]\n"
|
||||
)
|
||||
console.print("[yellow]Could not fetch live models, showing known models:[/yellow]\n")
|
||||
_show_static_models()
|
||||
except Exception as e:
|
||||
console.print(f"[yellow]Error fetching models: {e}[/yellow]\n")
|
||||
@@ -758,9 +743,7 @@ def models(ctx, endpoint: str):
|
||||
|
||||
console.print("\n[dim]To use a model:[/dim]")
|
||||
console.print(" [cyan]hanzo dev --model llama3-8b-instruct[/cyan]")
|
||||
console.print(
|
||||
"\n[dim cyan]Free tier:[/dim cyan] [green]Most models available without login![/green]"
|
||||
)
|
||||
console.print("\n[dim cyan]Free tier:[/dim cyan] [green]Most models available without login![/green]")
|
||||
console.print(
|
||||
"[dim cyan]Premium models:[/dim cyan] [yellow]gpt-4, claude-3-opus, o1-preview[/yellow] - [cyan]hanzo auth login[/cyan]"
|
||||
)
|
||||
|
||||
@@ -31,9 +31,7 @@ async def start(ctx, name: str, port: int, models: tuple, device: str):
|
||||
await start_node(ctx, name, port, list(models) if models else None, device)
|
||||
|
||||
|
||||
async def start_node(
|
||||
ctx, name: str, port: int, models: Optional[List[str]] = None, device: str = "auto"
|
||||
):
|
||||
async def start_node(ctx, name: str, port: int, models: Optional[List[str]] = None, device: str = "auto"):
|
||||
"""Start a local node via hanzo-cluster."""
|
||||
try:
|
||||
from hanzo_cluster import HanzoCluster
|
||||
@@ -140,9 +138,7 @@ async def status(ctx, name: str):
|
||||
if workers := status.get("node_details", []):
|
||||
console.print("\n[cyan]Workers:[/cyan]")
|
||||
for worker in workers:
|
||||
console.print(
|
||||
f" • {worker.get('name', 'unknown')} ({worker.get('state', 'unknown')})"
|
||||
)
|
||||
console.print(f" • {worker.get('name', 'unknown')} ({worker.get('state', 'unknown')})")
|
||||
if device := worker.get("device"):
|
||||
console.print(f" Device: {device}")
|
||||
else:
|
||||
@@ -256,9 +252,7 @@ def worker_group():
|
||||
default="auto",
|
||||
help="Device to use",
|
||||
)
|
||||
@click.option(
|
||||
"--port", "-p", type=int, help="Worker port (auto-assigned if not specified)"
|
||||
)
|
||||
@click.option("--port", "-p", type=int, help="Worker port (auto-assigned if not specified)")
|
||||
@click.option("--blockchain", is_flag=True, help="Enable blockchain features")
|
||||
@click.option("--network", is_flag=True, help="Enable network discovery")
|
||||
@click.pass_context
|
||||
@@ -431,9 +425,7 @@ async def worker_info(ctx, name: str):
|
||||
if resources := info.get("resources"):
|
||||
console.print("\n[cyan]Resources:[/cyan]")
|
||||
console.print(f" CPU: {resources.get('cpu_percent', 'N/A')}%")
|
||||
console.print(
|
||||
f" Memory: {resources.get('memory_used', 'N/A')} / {resources.get('memory_total', 'N/A')}"
|
||||
)
|
||||
console.print(f" Memory: {resources.get('memory_used', 'N/A')} / {resources.get('memory_total', 'N/A')}")
|
||||
if gpu := resources.get("gpu"):
|
||||
console.print(
|
||||
f" GPU: {gpu.get('name', 'N/A')} ({gpu.get('memory_used', 'N/A')} / {gpu.get('memory_total', 'N/A')})"
|
||||
@@ -446,12 +438,8 @@ async def worker_info(ctx, name: str):
|
||||
|
||||
if network := info.get("network"):
|
||||
console.print("\n[cyan]Network:[/cyan]")
|
||||
console.print(
|
||||
f" Blockchain: {'enabled' if network.get('blockchain') else 'disabled'}"
|
||||
)
|
||||
console.print(
|
||||
f" Discovery: {'enabled' if network.get('discovery') else 'disabled'}"
|
||||
)
|
||||
console.print(f" Blockchain: {'enabled' if network.get('blockchain') else 'disabled'}")
|
||||
console.print(f" Discovery: {'enabled' if network.get('discovery') else 'disabled'}")
|
||||
if peers := network.get("peers"):
|
||||
console.print(f" Peers: {len(peers)}")
|
||||
|
||||
|
||||
@@ -105,18 +105,14 @@ def info(ctx):
|
||||
|
||||
if not providers:
|
||||
console.print("\n[yellow]No LLM providers configured[/yellow]")
|
||||
console.print(
|
||||
"Set one of: OPENAI_API_KEY, ANTHROPIC_API_KEY, HANZO_API_KEY"
|
||||
)
|
||||
console.print("Set one of: OPENAI_API_KEY, ANTHROPIC_API_KEY, HANZO_API_KEY")
|
||||
|
||||
except ImportError:
|
||||
console.print("[red]Error:[/red] hanzo-repl not installed")
|
||||
|
||||
|
||||
@repl_group.command()
|
||||
@click.option(
|
||||
"--interface", type=click.Choice(["all", "ipython", "tui", "voice"]), default="all"
|
||||
)
|
||||
@click.option("--interface", type=click.Choice(["all", "ipython", "tui", "voice"]), default="all")
|
||||
@click.pass_context
|
||||
def install_extras(ctx, interface: str):
|
||||
"""Install optional REPL components."""
|
||||
@@ -169,9 +165,7 @@ def exec(ctx, command: tuple, model: str):
|
||||
|
||||
except ImportError as e:
|
||||
console.print(f"[red]Import Error:[/red] {e}")
|
||||
console.print(
|
||||
"[yellow]Note:[/yellow] hanzo-repl may not be installed correctly"
|
||||
)
|
||||
console.print("[yellow]Note:[/yellow] hanzo-repl may not be installed correctly")
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
|
||||
|
||||
@@ -40,9 +40,7 @@ def start_router(ctx, port: int, config: Optional[str], detach: bool):
|
||||
if not router_dir:
|
||||
console.print("[red]Error:[/red] Hanzo router not found")
|
||||
console.print("\nPlease clone the router:")
|
||||
console.print(
|
||||
" git clone https://github.com/hanzoai/router.git ~/work/hanzo/router"
|
||||
)
|
||||
console.print(" git clone https://github.com/hanzoai/router.git ~/work/hanzo/router")
|
||||
return
|
||||
|
||||
console.print(f"[green]✓[/green] Found router at {router_dir}")
|
||||
@@ -91,9 +89,7 @@ def start_router(ctx, port: int, config: Optional[str], detach: bool):
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
)
|
||||
console.print(
|
||||
f"[green]✓[/green] Router started in background (PID: {process.pid})"
|
||||
)
|
||||
console.print(f"[green]✓[/green] Router started in background (PID: {process.pid})")
|
||||
console.print(f"Check status: curl http://localhost:{port}/health")
|
||||
else:
|
||||
# Run in foreground
|
||||
@@ -116,14 +112,8 @@ def stop_router(port: int):
|
||||
for proc in psutil.process_iter(["pid", "cmdline"]):
|
||||
try:
|
||||
cmdline = proc.info["cmdline"]
|
||||
if (
|
||||
cmdline
|
||||
and "proxy_server" in " ".join(cmdline)
|
||||
and str(port) in " ".join(cmdline)
|
||||
):
|
||||
console.print(
|
||||
f"[yellow]Stopping router (PID: {proc.info['pid']})[/yellow]"
|
||||
)
|
||||
if cmdline and "proxy_server" in " ".join(cmdline) and str(port) in " ".join(cmdline):
|
||||
console.print(f"[yellow]Stopping router (PID: {proc.info['pid']})[/yellow]")
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
proc.wait(timeout=5)
|
||||
found = True
|
||||
@@ -148,9 +138,7 @@ def router_status(port: int):
|
||||
|
||||
# Try to get models
|
||||
try:
|
||||
models_response = httpx.get(
|
||||
f"http://localhost:{port}/models", timeout=2.0
|
||||
)
|
||||
models_response = httpx.get(f"http://localhost:{port}/models", timeout=2.0)
|
||||
if models_response.status_code == 200:
|
||||
data = models_response.json()
|
||||
if "data" in data:
|
||||
@@ -158,9 +146,7 @@ def router_status(port: int):
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
console.print(
|
||||
f"[yellow]Router responding but unhealthy (status: {response.status_code})[/yellow]"
|
||||
)
|
||||
console.print(f"[yellow]Router responding but unhealthy (status: {response.status_code})[/yellow]")
|
||||
except httpx.ConnectError:
|
||||
console.print(f"[red]Router not running on port {port}[/red]")
|
||||
console.print("\nStart with: hanzo router start")
|
||||
|
||||
@@ -30,9 +30,7 @@ async def list_tools(ctx, category: str, installed: bool):
|
||||
|
||||
with console.status("Loading tools..."):
|
||||
try:
|
||||
tools = await registry.list_tools(
|
||||
category=category, installed_only=installed
|
||||
)
|
||||
tools = await registry.list_tools(category=category, installed_only=installed)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Failed to load tools: {e}[/red]")
|
||||
return
|
||||
@@ -88,9 +86,7 @@ async def install(ctx, tool_name: str, version: str):
|
||||
try:
|
||||
result = await registry.install_tool(name=tool_name, version=version)
|
||||
|
||||
console.print(
|
||||
f"[green]✓[/green] Installed {tool_name} v{result['version']}"
|
||||
)
|
||||
console.print(f"[green]✓[/green] Installed {tool_name} v{result['version']}")
|
||||
|
||||
if deps := result.get("dependencies_installed"):
|
||||
console.print(f" Dependencies: {', '.join(deps)}")
|
||||
|
||||
+69
-215
@@ -221,9 +221,7 @@ class HanzoDevOrchestrator:
|
||||
grpo_dir = self.workspace_dir / "grpo"
|
||||
grpo_dir.mkdir(exist_ok=True)
|
||||
|
||||
self.experience_manager = ExperienceManager(
|
||||
str(grpo_dir / "experience_library.json")
|
||||
)
|
||||
self.experience_manager = ExperienceManager(str(grpo_dir / "experience_library.json"))
|
||||
|
||||
# Initialize with DeepSeek if API key available
|
||||
deepseek_key = os.getenv("DEEPSEEK_API_KEY")
|
||||
@@ -254,9 +252,7 @@ class HanzoDevOrchestrator:
|
||||
|
||||
for path in possible_paths:
|
||||
expanded = Path(path).expanduser()
|
||||
if expanded.exists() or (
|
||||
path == "claude" and os.system(f"which {path} >/dev/null 2>&1") == 0
|
||||
):
|
||||
if expanded.exists() or (path == "claude" and os.system(f"which {path} >/dev/null 2>&1") == 0):
|
||||
return str(expanded) if expanded.exists() else path
|
||||
|
||||
raise RuntimeError("Claude Code not found. Please specify path.")
|
||||
@@ -343,9 +339,7 @@ class HanzoDevOrchestrator:
|
||||
else:
|
||||
return "Execute standard workflow with monitoring"
|
||||
|
||||
def _assess_risks(
|
||||
self, problem: str, alternatives: List[str], context: Dict[str, Any]
|
||||
) -> List[str]:
|
||||
def _assess_risks(self, problem: str, alternatives: List[str], context: Dict[str, Any]) -> List[str]:
|
||||
"""Assess risks of different approaches."""
|
||||
risks = []
|
||||
|
||||
@@ -369,9 +363,7 @@ class HanzoDevOrchestrator:
|
||||
) -> str:
|
||||
"""Synthesize a decision from analysis."""
|
||||
if len(risks) > 2:
|
||||
return (
|
||||
"Proceed cautiously with incremental approach and rollback capability"
|
||||
)
|
||||
return "Proceed cautiously with incremental approach and rollback capability"
|
||||
elif alternatives:
|
||||
return f"Execute primary approach: {alternatives[0]}"
|
||||
else:
|
||||
@@ -416,9 +408,7 @@ class HanzoDevOrchestrator:
|
||||
if resume:
|
||||
checkpoint_file = self._get_latest_checkpoint()
|
||||
if checkpoint_file:
|
||||
console.print(
|
||||
f"[green]Resuming from checkpoint: {checkpoint_file.name}[/green]"
|
||||
)
|
||||
console.print(f"[green]Resuming from checkpoint: {checkpoint_file.name}[/green]")
|
||||
|
||||
# Prepare command
|
||||
cmd = [self.claude_code_path]
|
||||
@@ -536,12 +526,8 @@ class HanzoDevOrchestrator:
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"agent_state": self.agent_state.value,
|
||||
"runtime_health": asdict(self.runtime_health),
|
||||
"current_context": (
|
||||
asdict(self.current_context) if self.current_context else None
|
||||
),
|
||||
"thinking_history": [
|
||||
asdict(t) for t in self.thinking_history[-10:]
|
||||
], # Last 10
|
||||
"current_context": (asdict(self.current_context) if self.current_context else None),
|
||||
"thinking_history": [asdict(t) for t in self.thinking_history[-10:]], # Last 10
|
||||
}
|
||||
|
||||
with open(checkpoint_file, "w") as f:
|
||||
@@ -559,9 +545,7 @@ class HanzoDevOrchestrator:
|
||||
self.agent_state = AgentState(data["agent_state"])
|
||||
# Restore other state as needed
|
||||
|
||||
console.print(
|
||||
f"[green]✓ Restored from checkpoint: {checkpoint_file.name}[/green]"
|
||||
)
|
||||
console.print(f"[green]✓ Restored from checkpoint: {checkpoint_file.name}[/green]")
|
||||
return True
|
||||
except Exception as e:
|
||||
console.print(f"[red]Failed to restore checkpoint: {e}[/red]")
|
||||
@@ -577,9 +561,7 @@ class HanzoDevOrchestrator:
|
||||
healthy = await self.health_check()
|
||||
|
||||
if not healthy:
|
||||
console.print(
|
||||
f"[yellow]Health check failed. State: {self.runtime_health.state.value}[/yellow]"
|
||||
)
|
||||
console.print(f"[yellow]Health check failed. State: {self.runtime_health.state.value}[/yellow]")
|
||||
|
||||
# Use System 2 thinking to decide what to do
|
||||
thinking_result = await self.think(
|
||||
@@ -588,9 +570,7 @@ class HanzoDevOrchestrator:
|
||||
)
|
||||
|
||||
console.print(f"[cyan]Decision: {thinking_result.decision}[/cyan]")
|
||||
console.print(
|
||||
f"[cyan]Confidence: {thinking_result.confidence:.2f}[/cyan]"
|
||||
)
|
||||
console.print(f"[cyan]Confidence: {thinking_result.confidence:.2f}[/cyan]")
|
||||
|
||||
# Execute decision
|
||||
if thinking_result.confidence > 0.6:
|
||||
@@ -744,14 +724,10 @@ class HanzoDevOrchestrator:
|
||||
experiences_after = len(self.experience_manager.experiences)
|
||||
|
||||
# Save updated library
|
||||
self.experience_manager.save(
|
||||
str(self.workspace_dir / "grpo" / "experience_library.json")
|
||||
)
|
||||
self.experience_manager.save(str(self.workspace_dir / "grpo" / "experience_library.json"))
|
||||
|
||||
learned_count = experiences_after - experiences_before
|
||||
console.print(
|
||||
f"[green]✓ Learned {learned_count} new experiences (total: {experiences_after})[/green]"
|
||||
)
|
||||
console.print(f"[green]✓ Learned {learned_count} new experiences (total: {experiences_after})[/green]")
|
||||
|
||||
return learned_count
|
||||
|
||||
@@ -854,9 +830,7 @@ class HanzoDevREPL:
|
||||
handler = FallbackHandler()
|
||||
if not handler.fallback_order:
|
||||
console.print("[yellow]⚠️ No API keys detected[/yellow]")
|
||||
console.print(
|
||||
"[dim]Set OPENAI_API_KEY or ANTHROPIC_API_KEY to enable AI[/dim]"
|
||||
)
|
||||
console.print("[dim]Set OPENAI_API_KEY or ANTHROPIC_API_KEY to enable AI[/dim]")
|
||||
console.print()
|
||||
else:
|
||||
primary = handler.fallback_order[0][1]
|
||||
@@ -902,13 +876,9 @@ class HanzoDevREPL:
|
||||
# Handle memory/context commands
|
||||
from .memory_manager import handle_memory_command
|
||||
|
||||
handled = handle_memory_command(
|
||||
user_input, self.memory_manager, console
|
||||
)
|
||||
handled = handle_memory_command(user_input, self.memory_manager, console)
|
||||
if not handled:
|
||||
console.print(
|
||||
"[yellow]Unknown memory command. Use #memory help[/yellow]"
|
||||
)
|
||||
console.print("[yellow]Unknown memory command. Use #memory help[/yellow]")
|
||||
|
||||
else:
|
||||
# Natural chat - send directly to AI agents
|
||||
@@ -951,17 +921,13 @@ class HanzoDevREPL:
|
||||
|
||||
table.add_row("Agent State", self.orchestrator.agent_state.value)
|
||||
table.add_row("Runtime State", self.orchestrator.runtime_health.state.value)
|
||||
table.add_row(
|
||||
"Last Response", str(self.orchestrator.runtime_health.last_response)
|
||||
)
|
||||
table.add_row("Last Response", str(self.orchestrator.runtime_health.last_response))
|
||||
table.add_row(
|
||||
"Response Time",
|
||||
f"{self.orchestrator.runtime_health.response_time_ms:.2f}ms",
|
||||
)
|
||||
table.add_row("Error Count", str(self.orchestrator.runtime_health.error_count))
|
||||
table.add_row(
|
||||
"Restart Count", str(self.orchestrator.runtime_health.restart_count)
|
||||
)
|
||||
table.add_row("Restart Count", str(self.orchestrator.runtime_health.restart_count))
|
||||
|
||||
console.print(table)
|
||||
|
||||
@@ -1008,9 +974,7 @@ class HanzoDevREPL:
|
||||
"""Restore from checkpoint."""
|
||||
if not args:
|
||||
# Show available checkpoints
|
||||
checkpoints = list(
|
||||
self.orchestrator.checkpoint_dir.glob("checkpoint_*.json")
|
||||
)
|
||||
checkpoints = list(self.orchestrator.checkpoint_dir.glob("checkpoint_*.json"))
|
||||
if checkpoints:
|
||||
console.print("[cyan]Available checkpoints:[/cyan]")
|
||||
for cp in checkpoints:
|
||||
@@ -1087,10 +1051,7 @@ Examples:
|
||||
enhanced_message = message
|
||||
|
||||
# Try smart fallback if no specific model configured
|
||||
if (
|
||||
not hasattr(self.orchestrator, "orchestrator_model")
|
||||
or self.orchestrator.orchestrator_model == "auto"
|
||||
):
|
||||
if not hasattr(self.orchestrator, "orchestrator_model") or self.orchestrator.orchestrator_model == "auto":
|
||||
# Use streaming if available
|
||||
from .streaming import stream_with_fallback
|
||||
|
||||
@@ -1102,9 +1063,7 @@ Examples:
|
||||
# Response already displayed by streaming handler
|
||||
return
|
||||
else:
|
||||
console.print(
|
||||
"[red]No AI options available. Please configure API keys or install tools.[/red]"
|
||||
)
|
||||
console.print("[red]No AI options available. Please configure API keys or install tools.[/red]")
|
||||
return
|
||||
|
||||
# For codex and other CLI tools, go straight to direct API chat
|
||||
@@ -1345,26 +1304,14 @@ Examples:
|
||||
|
||||
# No API keys available
|
||||
console.print("[red]No AI API keys configured![/red]")
|
||||
console.print(
|
||||
"[yellow]Try these options that don't need your API key:[/yellow]"
|
||||
)
|
||||
console.print("[yellow]Try these options that don't need your API key:[/yellow]")
|
||||
console.print("\n[bold]CLI Tools (use existing tools):[/bold]")
|
||||
console.print(
|
||||
" • hanzo dev --orchestrator codex # OpenAI CLI (if installed)"
|
||||
)
|
||||
console.print(
|
||||
" • hanzo dev --orchestrator claude # Claude Desktop (if installed)"
|
||||
)
|
||||
console.print(
|
||||
" • hanzo dev --orchestrator gemini # Gemini CLI (if installed)"
|
||||
)
|
||||
console.print(
|
||||
" • hanzo dev --orchestrator hanzo-ide # Hanzo IDE from ~/work/hanzo/ide"
|
||||
)
|
||||
console.print(" • hanzo dev --orchestrator codex # OpenAI CLI (if installed)")
|
||||
console.print(" • hanzo dev --orchestrator claude # Claude Desktop (if installed)")
|
||||
console.print(" • hanzo dev --orchestrator gemini # Gemini CLI (if installed)")
|
||||
console.print(" • hanzo dev --orchestrator hanzo-ide # Hanzo IDE from ~/work/hanzo/ide")
|
||||
console.print("\n[bold]Free APIs (rate limited):[/bold]")
|
||||
console.print(
|
||||
" • hanzo dev --orchestrator codestral # Free Mistral Codestral"
|
||||
)
|
||||
console.print(" • hanzo dev --orchestrator codestral # Free Mistral Codestral")
|
||||
console.print(" • hanzo dev --orchestrator starcoder # Free StarCoder")
|
||||
console.print("\n[bold]Local Models (unlimited):[/bold]")
|
||||
console.print(" • hanzo dev --orchestrator local:llama3.2 # Via Ollama")
|
||||
@@ -1407,16 +1354,10 @@ Examples:
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get("choices"):
|
||||
console.print(
|
||||
f"[cyan]Codestral:[/cyan] {data['choices'][0]['message']['content']}"
|
||||
)
|
||||
console.print(f"[cyan]Codestral:[/cyan] {data['choices'][0]['message']['content']}")
|
||||
else:
|
||||
console.print(
|
||||
"[yellow]Free tier limit reached. Try local models instead:[/yellow]"
|
||||
)
|
||||
console.print(
|
||||
" • Install Ollama: curl -fsSL https://ollama.com/install.sh | sh"
|
||||
)
|
||||
console.print("[yellow]Free tier limit reached. Try local models instead:[/yellow]")
|
||||
console.print(" • Install Ollama: curl -fsSL https://ollama.com/install.sh | sh")
|
||||
console.print(" • Run: ollama pull codellama")
|
||||
console.print(" • Use: hanzo dev --orchestrator local:codellama")
|
||||
|
||||
@@ -1453,13 +1394,9 @@ Examples:
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if isinstance(data, list) and data:
|
||||
console.print(
|
||||
f"[cyan]StarCoder:[/cyan] {data[0].get('generated_text', '')}"
|
||||
)
|
||||
console.print(f"[cyan]StarCoder:[/cyan] {data[0].get('generated_text', '')}")
|
||||
else:
|
||||
console.print(
|
||||
"[yellow]API limit reached. Install local models:[/yellow]"
|
||||
)
|
||||
console.print("[yellow]API limit reached. Install local models:[/yellow]")
|
||||
console.print(" • brew install ollama")
|
||||
console.print(" • ollama pull starcoder2")
|
||||
console.print(" • hanzo dev --orchestrator local:starcoder2")
|
||||
@@ -1496,9 +1433,7 @@ Examples:
|
||||
message,
|
||||
]
|
||||
|
||||
process = subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
|
||||
)
|
||||
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
|
||||
stdout, stderr = process.communicate(timeout=30)
|
||||
|
||||
@@ -1530,11 +1465,7 @@ Examples:
|
||||
|
||||
claude_path = None
|
||||
for path in claude_paths:
|
||||
if (
|
||||
os.path.exists(path)
|
||||
or subprocess.run(["which", path], capture_output=True).returncode
|
||||
== 0
|
||||
):
|
||||
if os.path.exists(path) or subprocess.run(["which", path], capture_output=True).returncode == 0:
|
||||
claude_path = path
|
||||
break
|
||||
|
||||
@@ -1563,9 +1494,7 @@ Examples:
|
||||
"""
|
||||
|
||||
subprocess.run(["osascript", "-e", script])
|
||||
console.print(
|
||||
"[cyan]Sent to Claude Desktop. Check the app for response.[/cyan]"
|
||||
)
|
||||
console.print("[cyan]Sent to Claude Desktop. Check the app for response.[/cyan]")
|
||||
else:
|
||||
# Try direct CLI invocation
|
||||
process = subprocess.Popen(
|
||||
@@ -1604,9 +1533,7 @@ Examples:
|
||||
# Use gemini CLI
|
||||
cmd = ["gemini", "chat", message]
|
||||
|
||||
process = subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
|
||||
)
|
||||
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
|
||||
stdout, stderr = process.communicate(timeout=30)
|
||||
|
||||
@@ -1634,9 +1561,7 @@ Examples:
|
||||
console.print("[red]Hanzo Dev IDE not found![/red]")
|
||||
console.print("[yellow]Expected location: ~/work/hanzo/ide[/yellow]")
|
||||
console.print("To set up:")
|
||||
console.print(
|
||||
" • git clone https://github.com/hanzoai/ide ~/work/hanzo/ide"
|
||||
)
|
||||
console.print(" • git clone https://github.com/hanzoai/ide ~/work/hanzo/ide")
|
||||
console.print(" • cd ~/work/hanzo/ide && npm install")
|
||||
return
|
||||
|
||||
@@ -1672,9 +1597,7 @@ Examples:
|
||||
cmd = [cli_path, "chat", message]
|
||||
cwd = None
|
||||
|
||||
process = subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=cwd
|
||||
)
|
||||
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=cwd)
|
||||
|
||||
stdout, stderr = process.communicate(timeout=30)
|
||||
|
||||
@@ -1721,9 +1644,7 @@ Examples:
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get("message"):
|
||||
console.print(
|
||||
f"[cyan]{model_name}:[/cyan] {data['message']['content']}"
|
||||
)
|
||||
console.print(f"[cyan]{model_name}:[/cyan] {data['message']['content']}")
|
||||
return
|
||||
|
||||
except Exception:
|
||||
@@ -1754,9 +1675,7 @@ Examples:
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get("choices"):
|
||||
console.print(
|
||||
f"[cyan]{model_name}:[/cyan] {data['choices'][0]['message']['content']}"
|
||||
)
|
||||
console.print(f"[cyan]{model_name}:[/cyan] {data['choices'][0]['message']['content']}")
|
||||
return
|
||||
|
||||
except Exception:
|
||||
@@ -1834,9 +1753,7 @@ async def run_dev_orchestrator(**kwargs):
|
||||
# Check if we should use network mode
|
||||
# For now, disable network mode since hanzo-network isn't available
|
||||
if False and use_network and NETWORK_AVAILABLE:
|
||||
console_obj.print(
|
||||
f"[cyan]Mode: Network Orchestration with hanzo-network[/cyan]"
|
||||
)
|
||||
console_obj.print(f"[cyan]Mode: Network Orchestration with hanzo-network[/cyan]")
|
||||
console_obj.print(f"Orchestrator: {orchestrator_model}")
|
||||
console_obj.print(f"Workers: {instances} agents")
|
||||
console_obj.print(f"Critics: {max(1, instances // 2)} agents")
|
||||
@@ -1942,25 +1859,19 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
|
||||
|
||||
# Check if we can use hanzo-network
|
||||
if not NETWORK_AVAILABLE:
|
||||
self.console.print(
|
||||
"[yellow]Warning: hanzo-network not available, falling back to basic mode[/yellow]"
|
||||
)
|
||||
self.console.print("[yellow]Warning: hanzo-network not available, falling back to basic mode[/yellow]")
|
||||
|
||||
async def initialize(self):
|
||||
"""Initialize the agent network with orchestrator and workers."""
|
||||
if not NETWORK_AVAILABLE:
|
||||
self.console.print(
|
||||
"[red]Cannot initialize network mode without hanzo-network[/red]"
|
||||
)
|
||||
self.console.print("[red]Cannot initialize network mode without hanzo-network[/red]")
|
||||
return False
|
||||
|
||||
# Start hanzo net if requested for local orchestration
|
||||
if self.use_hanzo_net or self.orchestrator_model.startswith("local:"):
|
||||
await self._start_hanzo_net()
|
||||
|
||||
self.console.print(
|
||||
f"[cyan]Initializing agent network with {self.orchestrator_model} orchestrator...[/cyan]"
|
||||
)
|
||||
self.console.print(f"[cyan]Initializing agent network with {self.orchestrator_model} orchestrator...[/cyan]")
|
||||
|
||||
# Create orchestrator agent (GPT-5, local, or other model)
|
||||
self.orchestrator_agent = await self._create_orchestrator_agent()
|
||||
@@ -1977,9 +1888,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
|
||||
for i in range(num_local_workers):
|
||||
local_worker = await self._create_local_worker_agent(i)
|
||||
self.worker_agents.append(local_worker)
|
||||
self.console.print(
|
||||
f"[green]Added {num_local_workers} local workers for cost optimization[/green]"
|
||||
)
|
||||
self.console.print(f"[green]Added {num_local_workers} local workers for cost optimization[/green]")
|
||||
|
||||
# Create critic agents for System 2 thinking
|
||||
if self.enable_guardrails:
|
||||
@@ -2002,21 +1911,15 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
|
||||
self.agent_network = create_network(
|
||||
agents=all_agents,
|
||||
router=router,
|
||||
default_agent=(
|
||||
self.orchestrator_agent.name if self.orchestrator_agent else None
|
||||
),
|
||||
default_agent=(self.orchestrator_agent.name if self.orchestrator_agent else None),
|
||||
)
|
||||
|
||||
self.console.print(
|
||||
f"[green]✓ Agent network initialized with {len(all_agents)} agents[/green]"
|
||||
)
|
||||
self.console.print(f"[green]✓ Agent network initialized with {len(all_agents)} agents[/green]")
|
||||
return True
|
||||
|
||||
async def _start_hanzo_net(self):
|
||||
"""Start hanzo net for local AI orchestration."""
|
||||
self.console.print(
|
||||
"[cyan]Starting hanzo/net for local AI orchestration...[/cyan]"
|
||||
)
|
||||
self.console.print("[cyan]Starting hanzo/net for local AI orchestration...[/cyan]")
|
||||
|
||||
# Check if hanzo net is already running
|
||||
import socket
|
||||
@@ -2026,9 +1929,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
|
||||
sock.close()
|
||||
|
||||
if result == 0:
|
||||
self.console.print(
|
||||
f"[yellow]hanzo/net already running on port {self.hanzo_net_port}[/yellow]"
|
||||
)
|
||||
self.console.print(f"[yellow]hanzo/net already running on port {self.hanzo_net_port}[/yellow]")
|
||||
return
|
||||
|
||||
# Start hanzo net
|
||||
@@ -2089,9 +1990,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
|
||||
tools=[],
|
||||
)
|
||||
|
||||
self.console.print(
|
||||
f"[green]✓ Created local {model_name} orchestrator via hanzo/net[/green]"
|
||||
)
|
||||
self.console.print(f"[green]✓ Created local {model_name} orchestrator via hanzo/net[/green]")
|
||||
return orchestrator
|
||||
|
||||
# Parse model string to get provider and model
|
||||
@@ -2147,9 +2046,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
|
||||
tools=[], # Orchestrator tools will be added
|
||||
)
|
||||
|
||||
self.console.print(
|
||||
f"[green]✓ Created {self.orchestrator_model} orchestrator[/green]"
|
||||
)
|
||||
self.console.print(f"[green]✓ Created {self.orchestrator_model} orchestrator[/green]")
|
||||
return orchestrator
|
||||
|
||||
def _get_orchestrator_system_prompt(self) -> str:
|
||||
@@ -2282,10 +2179,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
|
||||
"count",
|
||||
"find",
|
||||
]
|
||||
if (
|
||||
any(keyword in prompt_lower for keyword in simple_keywords)
|
||||
and self.local_workers
|
||||
):
|
||||
if any(keyword in prompt_lower for keyword in simple_keywords) and self.local_workers:
|
||||
return self.local_workers[0].name
|
||||
|
||||
# Complex implementation → API workers (Claude)
|
||||
@@ -2297,10 +2191,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
|
||||
"design",
|
||||
"architect",
|
||||
]
|
||||
if (
|
||||
any(keyword in prompt_lower for keyword in complex_keywords)
|
||||
and self.api_workers
|
||||
):
|
||||
if any(keyword in prompt_lower for keyword in complex_keywords) and self.api_workers:
|
||||
return self.api_workers[0].name
|
||||
|
||||
# Review tasks → Critics
|
||||
@@ -2311,10 +2202,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
|
||||
"improve",
|
||||
"validate code",
|
||||
]
|
||||
if (
|
||||
any(keyword in prompt_lower for keyword in review_keywords)
|
||||
and self.critics
|
||||
):
|
||||
if any(keyword in prompt_lower for keyword in review_keywords) and self.critics:
|
||||
return self.critics[0].name
|
||||
|
||||
# Strategic decisions → Orchestrator
|
||||
@@ -2335,20 +2223,12 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
|
||||
return self.local_workers[0].name
|
||||
|
||||
# Fall back to API workers for complex tasks
|
||||
return (
|
||||
self.api_workers[0].name
|
||||
if self.api_workers
|
||||
else self.orchestrator.name
|
||||
)
|
||||
return self.api_workers[0].name if self.api_workers else self.orchestrator.name
|
||||
|
||||
# Create the cost-optimized router
|
||||
router = CostOptimizedRouter(
|
||||
self.orchestrator_agent, self.worker_agents, self.critic_agents
|
||||
)
|
||||
router = CostOptimizedRouter(self.orchestrator_agent, self.worker_agents, self.critic_agents)
|
||||
|
||||
self.console.print(
|
||||
"[green]✓ Created cost-optimized router (local models preferred)[/green]"
|
||||
)
|
||||
self.console.print("[green]✓ Created cost-optimized router (local models preferred)[/green]")
|
||||
return router
|
||||
|
||||
async def _create_intelligent_router(self) -> Router:
|
||||
@@ -2383,9 +2263,7 @@ class NetworkOrchestrator(HanzoDevOrchestrator):
|
||||
|
||||
return router
|
||||
|
||||
async def execute_with_network(
|
||||
self, task: str, context: Optional[Dict] = None
|
||||
) -> Dict:
|
||||
async def execute_with_network(self, task: str, context: Optional[Dict] = None) -> Dict:
|
||||
"""Execute a task using the agent network.
|
||||
|
||||
Args:
|
||||
@@ -2523,9 +2401,7 @@ class MultiClaudeOrchestrator(HanzoDevOrchestrator):
|
||||
config = await self._create_instance_config(i, role)
|
||||
self.instance_configs.append(config)
|
||||
|
||||
self.console.print(
|
||||
f" [{i + 1}/{self.num_instances}] {role} instance configured"
|
||||
)
|
||||
self.console.print(f" [{i + 1}/{self.num_instances}] {role} instance configured")
|
||||
|
||||
# If networking enabled, configure MCP connections between instances
|
||||
if self.enable_networking:
|
||||
@@ -2587,9 +2463,7 @@ class MultiClaudeOrchestrator(HanzoDevOrchestrator):
|
||||
|
||||
async def _setup_mcp_networking(self):
|
||||
"""Set up MCP networking between Claude instances."""
|
||||
self.console.print(
|
||||
"[cyan]Setting up MCP networking between instances...[/cyan]"
|
||||
)
|
||||
self.console.print("[cyan]Setting up MCP networking between instances...[/cyan]")
|
||||
|
||||
# Each instance gets MCP servers for all other instances
|
||||
for i, config in enumerate(self.instance_configs):
|
||||
@@ -2732,14 +2606,10 @@ class MultiClaudeOrchestrator(HanzoDevOrchestrator):
|
||||
|
||||
# Validate improvement didn't degrade quality
|
||||
if await self._validate_improvement(result, improved):
|
||||
self.console.print(
|
||||
"[green]✓ Solution improved with System 2 feedback[/green]"
|
||||
)
|
||||
self.console.print("[green]✓ Solution improved with System 2 feedback[/green]")
|
||||
return improved
|
||||
else:
|
||||
self.console.print(
|
||||
"[yellow]⚠ Keeping original solution (improvement validation failed)[/yellow]"
|
||||
)
|
||||
self.console.print("[yellow]⚠ Keeping original solution (improvement validation failed)[/yellow]")
|
||||
|
||||
return result
|
||||
|
||||
@@ -2816,9 +2686,7 @@ class MultiClaudeOrchestrator(HanzoDevOrchestrator):
|
||||
try:
|
||||
import subprocess
|
||||
|
||||
result = subprocess.run(
|
||||
["gemini", "chat", prompt], capture_output=True, text=True, timeout=30
|
||||
)
|
||||
result = subprocess.run(["gemini", "chat", prompt], capture_output=True, text=True, timeout=30)
|
||||
if result.returncode == 0 and result.stdout:
|
||||
return {"output": result.stdout.strip(), "success": True}
|
||||
except Exception as e:
|
||||
@@ -2882,9 +2750,7 @@ class MultiClaudeOrchestrator(HanzoDevOrchestrator):
|
||||
logger.error(f"OpenAI API error: {e}")
|
||||
|
||||
# Try Anthropic
|
||||
anthropic_key = os.environ.get("ANTHROPIC_API_KEY") or os.getenv(
|
||||
"ANTHROPIC_API_KEY"
|
||||
)
|
||||
anthropic_key = os.environ.get("ANTHROPIC_API_KEY") or os.getenv("ANTHROPIC_API_KEY")
|
||||
if anthropic_key:
|
||||
try:
|
||||
from anthropic import AsyncAnthropic
|
||||
@@ -2903,9 +2769,7 @@ class MultiClaudeOrchestrator(HanzoDevOrchestrator):
|
||||
# Try fallback handler as last resort
|
||||
from .fallback_handler import smart_chat
|
||||
|
||||
response = await smart_chat(
|
||||
prompt, console=None
|
||||
) # No console to avoid duplicate messages
|
||||
response = await smart_chat(prompt, console=None) # No console to avoid duplicate messages
|
||||
if response:
|
||||
return {"output": response, "success": True}
|
||||
|
||||
@@ -2948,25 +2812,15 @@ async def main():
|
||||
"""Main entry point for hanzo-dev."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Hanzo Dev - System 2 Meta-AI Orchestrator"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--workspace", default="~/.hanzo/dev", help="Workspace directory"
|
||||
)
|
||||
parser = argparse.ArgumentParser(description="Hanzo Dev - System 2 Meta-AI Orchestrator")
|
||||
parser.add_argument("--workspace", default="~/.hanzo/dev", help="Workspace directory")
|
||||
parser.add_argument("--claude-path", help="Path to Claude Code executable")
|
||||
parser.add_argument("--monitor", action="store_true", help="Start in monitor mode")
|
||||
parser.add_argument("--repl", action="store_true", help="Start REPL interface")
|
||||
parser.add_argument(
|
||||
"--instances", type=int, default=2, help="Number of Claude instances"
|
||||
)
|
||||
parser.add_argument("--instances", type=int, default=2, help="Number of Claude instances")
|
||||
parser.add_argument("--no-mcp", action="store_true", help="Disable MCP tools")
|
||||
parser.add_argument(
|
||||
"--no-network", action="store_true", help="Disable instance networking"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-guardrails", action="store_true", help="Disable guardrails"
|
||||
)
|
||||
parser.add_argument("--no-network", action="store_true", help="Disable instance networking")
|
||||
parser.add_argument("--no-guardrails", action="store_true", help="Disable guardrails")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
@@ -23,9 +23,7 @@ class FallbackHandler:
|
||||
"deepseek_api": bool(os.getenv("DEEPSEEK_API_KEY")), # Added DeepSeek
|
||||
"openai_api": bool(os.getenv("OPENAI_API_KEY")),
|
||||
"anthropic_api": bool(os.getenv("ANTHROPIC_API_KEY")),
|
||||
"google_api": bool(
|
||||
os.getenv("GOOGLE_API_KEY") or os.getenv("GEMINI_API_KEY")
|
||||
),
|
||||
"google_api": bool(os.getenv("GOOGLE_API_KEY") or os.getenv("GEMINI_API_KEY")),
|
||||
"openai_cli": shutil.which("openai") is not None,
|
||||
"claude_cli": shutil.which("claude") is not None,
|
||||
"gemini_cli": shutil.which("gemini") is not None,
|
||||
@@ -103,9 +101,7 @@ class FallbackHandler:
|
||||
suggestions = []
|
||||
|
||||
if not self.available_options["deepseek_api"]:
|
||||
suggestions.append(
|
||||
"• Set DEEPSEEK_API_KEY for cost-effective DeepSeek access ($0.14/M tokens)"
|
||||
)
|
||||
suggestions.append("• Set DEEPSEEK_API_KEY for cost-effective DeepSeek access ($0.14/M tokens)")
|
||||
|
||||
if not self.available_options["openai_api"]:
|
||||
suggestions.append("• Set OPENAI_API_KEY for GPT-4/GPT-5 access")
|
||||
@@ -114,30 +110,22 @@ class FallbackHandler:
|
||||
suggestions.append("• Set ANTHROPIC_API_KEY for Claude access")
|
||||
|
||||
if not self.available_options["ollama"]:
|
||||
suggestions.append(
|
||||
"• Install Ollama: curl -fsSL https://ollama.com/install.sh | sh"
|
||||
)
|
||||
suggestions.append("• Install Ollama: curl -fsSL https://ollama.com/install.sh | sh")
|
||||
suggestions.append(" Then run: ollama pull llama3.2")
|
||||
|
||||
if not self.available_options["openai_cli"]:
|
||||
suggestions.append("• Install OpenAI CLI: pip install openai-cli")
|
||||
|
||||
if not self.available_options["claude_cli"]:
|
||||
suggestions.append(
|
||||
"• Install Claude Desktop from https://claude.ai/download"
|
||||
)
|
||||
suggestions.append("• Install Claude Desktop from https://claude.ai/download")
|
||||
|
||||
return (
|
||||
"\n".join(suggestions) if suggestions else "All AI options are available!"
|
||||
)
|
||||
return "\n".join(suggestions) if suggestions else "All AI options are available!"
|
||||
|
||||
def print_status(self, console):
|
||||
"""Print the current status of available AI options."""
|
||||
from rich.table import Table
|
||||
|
||||
table = Table(
|
||||
title="Available AI Options", show_header=True, header_style="bold magenta"
|
||||
)
|
||||
table = Table(title="Available AI Options", show_header=True, header_style="bold magenta")
|
||||
table.add_column("Option", style="cyan", width=20)
|
||||
table.add_column("Status", width=10)
|
||||
table.add_column("Model", width=20)
|
||||
@@ -164,9 +152,7 @@ class FallbackHandler:
|
||||
console.print(table)
|
||||
|
||||
if self.fallback_order:
|
||||
console.print(
|
||||
f"\n[green]Primary option: {self.fallback_order[0][1]}[/green]"
|
||||
)
|
||||
console.print(f"\n[green]Primary option: {self.fallback_order[0][1]}[/green]")
|
||||
if len(self.fallback_order) > 1:
|
||||
fallbacks = ", ".join([opt[1] for opt in self.fallback_order[1:]])
|
||||
console.print(f"[yellow]Fallback options: {fallbacks}[/yellow]")
|
||||
|
||||
@@ -98,9 +98,7 @@ class EnhancedHanzoREPL:
|
||||
self.failed_tools = set() # Track tools that have failed this session
|
||||
|
||||
# Initialize background task manager
|
||||
self.task_manager = (
|
||||
BackgroundTaskManager(console) if BackgroundTaskManager else None
|
||||
)
|
||||
self.task_manager = BackgroundTaskManager(console) if BackgroundTaskManager else None
|
||||
|
||||
# Initialize todo manager
|
||||
self.todo_manager = TodoManager(console) if TodoManager else None
|
||||
@@ -114,9 +112,7 @@ class EnhancedHanzoREPL:
|
||||
if default_tool:
|
||||
self.current_model = f"tool:{default_tool.name}"
|
||||
self.current_tool = default_tool
|
||||
self.console.print(
|
||||
f"[green]✓ Detected {default_tool.display_name} as default AI assistant[/green]"
|
||||
)
|
||||
self.console.print(f"[green]✓ Detected {default_tool.display_name} as default AI assistant[/green]")
|
||||
else:
|
||||
# Fallback to regular models
|
||||
self.current_model = self.config.get("default_model", "gpt-3.5-turbo")
|
||||
@@ -355,9 +351,7 @@ class EnhancedHanzoREPL:
|
||||
# Router status
|
||||
try:
|
||||
response = httpx.get("http://localhost:4000/health", timeout=1)
|
||||
router_status = (
|
||||
"✅ Running" if response.status_code == 200 else "⚠️ Unhealthy"
|
||||
)
|
||||
router_status = "✅ Running" if response.status_code == 200 else "⚠️ Unhealthy"
|
||||
router_details = "Port 4000"
|
||||
except Exception:
|
||||
router_status = "❌ Offline"
|
||||
@@ -410,11 +404,7 @@ class EnhancedHanzoREPL:
|
||||
args = f"tool:{tool.name}"
|
||||
else:
|
||||
# It's a model selection
|
||||
model_idx = (
|
||||
num - len(self.detected_tools) - 1
|
||||
if self.detected_tools
|
||||
else num - 1
|
||||
)
|
||||
model_idx = num - len(self.detected_tools) - 1 if self.detected_tools else num - 1
|
||||
models_list = list(self.MODELS.keys())
|
||||
if 0 <= model_idx < len(models_list):
|
||||
args = models_list[model_idx]
|
||||
@@ -427,11 +417,7 @@ class EnhancedHanzoREPL:
|
||||
return
|
||||
|
||||
# Check if it's a tool
|
||||
if (
|
||||
args.startswith("tool:") or args in [t.name for t in self.detected_tools]
|
||||
if self.detected_tools
|
||||
else False
|
||||
):
|
||||
if args.startswith("tool:") or args in [t.name for t in self.detected_tools] if self.detected_tools else False:
|
||||
# Handle tool selection
|
||||
tool_name = args.replace("tool:", "") if args.startswith("tool:") else args
|
||||
|
||||
@@ -465,9 +451,7 @@ class EnhancedHanzoREPL:
|
||||
|
||||
# Show hint if it's a gateway model (not in our predefined list)
|
||||
if args not in self.MODELS and not args.startswith("local:"):
|
||||
self.console.print(
|
||||
"[dim]Using gateway model - all models free! Use /models to see full list.[/dim]"
|
||||
)
|
||||
self.console.print("[dim]Using gateway model - all models free! Use /models to see full list.[/dim]")
|
||||
|
||||
async def list_tools(self, args: str = ""):
|
||||
"""List available AI tools."""
|
||||
@@ -480,14 +464,10 @@ class EnhancedHanzoREPL:
|
||||
"""List available models from gateway.hanzo.ai."""
|
||||
# Show tools first if available
|
||||
if self.detected_tools:
|
||||
self.console.print(
|
||||
"[bold cyan]AI Coding Assistants (Detected):[/bold cyan]"
|
||||
)
|
||||
self.console.print("[bold cyan]AI Coding Assistants (Detected):[/bold cyan]")
|
||||
for i, tool in enumerate(self.detected_tools, 1):
|
||||
marker = "→" if self.current_model == f"tool:{tool.name}" else " "
|
||||
self.console.print(
|
||||
f" {marker} {i}. {tool.display_name} ({tool.provider})"
|
||||
)
|
||||
self.console.print(f" {marker} {i}. {tool.display_name} ({tool.provider})")
|
||||
self.console.print()
|
||||
|
||||
# Try to fetch live models from gateway
|
||||
@@ -507,20 +487,14 @@ class EnhancedHanzoREPL:
|
||||
models_list = [
|
||||
model
|
||||
for model in all_models
|
||||
if not any(
|
||||
keyword in model.get("id", "").lower()
|
||||
for keyword in embedding_keywords
|
||||
)
|
||||
if not any(keyword in model.get("id", "").lower() for keyword in embedding_keywords)
|
||||
]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback to static models if gateway fetch fails
|
||||
if not models_list:
|
||||
models_list = [
|
||||
{"id": model_id, "name": model_name}
|
||||
for model_id, model_name in self.MODELS.items()
|
||||
]
|
||||
models_list = [{"id": model_id, "name": model_name} for model_id, model_name in self.MODELS.items()]
|
||||
|
||||
# Create table
|
||||
table = Table(title="Gateway LLMs (Chat Models)", box=box.ROUNDED)
|
||||
@@ -559,22 +533,16 @@ class EnhancedHanzoREPL:
|
||||
|
||||
# Highlight current model
|
||||
if model_id == self.current_model:
|
||||
table.add_row(
|
||||
str(i), f"[bold green]→ {model_id}[/bold green]", tier, provider
|
||||
)
|
||||
table.add_row(str(i), f"[bold green]→ {model_id}[/bold green]", tier, provider)
|
||||
else:
|
||||
table.add_row(str(i), model_id, tier, provider)
|
||||
|
||||
self.console.print(table)
|
||||
self.console.print(
|
||||
"\n[dim cyan]Free tier:[/dim cyan] [green]Most models available without login![/green]"
|
||||
)
|
||||
self.console.print("\n[dim cyan]Free tier:[/dim cyan] [green]Most models available without login![/green]")
|
||||
self.console.print(
|
||||
"[dim cyan]Premium models:[/dim cyan] [yellow]gpt-4, claude-3-opus, o1-preview[/yellow] - [cyan]hanzo auth login[/cyan]"
|
||||
)
|
||||
self.console.print(
|
||||
"\n[dim]Use /model <name> or /model <number> to switch[/dim]"
|
||||
)
|
||||
self.console.print("\n[dim]Use /model <name> or /model <number> to switch[/dim]")
|
||||
|
||||
async def login(self, args: str = ""):
|
||||
"""Login to Hanzo."""
|
||||
@@ -697,9 +665,7 @@ class EnhancedHanzoREPL:
|
||||
self.console.print(f"[dim]Executing: {full_cmd}[/dim]")
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
full_cmd, shell=True, capture_output=True, text=True
|
||||
)
|
||||
result = subprocess.run(full_cmd, shell=True, capture_output=True, text=True)
|
||||
|
||||
if result.stdout:
|
||||
self.console.print(result.stdout)
|
||||
@@ -720,19 +686,13 @@ class EnhancedHanzoREPL:
|
||||
if tool.name not in self.failed_tools:
|
||||
self.current_tool = tool
|
||||
self.current_model = f"tool:{tool.name}"
|
||||
self.console.print(
|
||||
f"[yellow]Switched to {tool.display_name}[/yellow]"
|
||||
)
|
||||
self.console.print(f"[yellow]Switched to {tool.display_name}[/yellow]")
|
||||
break
|
||||
|
||||
# Try the current tool
|
||||
if self.current_tool.name not in self.failed_tools:
|
||||
self.console.print(
|
||||
f"[dim]Using {self.current_tool.display_name}...[/dim]"
|
||||
)
|
||||
success, output = self.tool_detector.execute_with_tool(
|
||||
self.current_tool, message
|
||||
)
|
||||
self.console.print(f"[dim]Using {self.current_tool.display_name}...[/dim]")
|
||||
success, output = self.tool_detector.execute_with_tool(self.current_tool, message)
|
||||
|
||||
if success:
|
||||
self.console.print(output)
|
||||
@@ -747,12 +707,8 @@ class EnhancedHanzoREPL:
|
||||
if self.tool_detector and self.tool_detector.detected_tools:
|
||||
for fallback_tool in self.tool_detector.detected_tools:
|
||||
if fallback_tool.name not in self.failed_tools:
|
||||
self.console.print(
|
||||
f"[yellow]Trying {fallback_tool.display_name}...[/yellow]"
|
||||
)
|
||||
success, output = self.tool_detector.execute_with_tool(
|
||||
fallback_tool, message
|
||||
)
|
||||
self.console.print(f"[yellow]Trying {fallback_tool.display_name}...[/yellow]")
|
||||
success, output = self.tool_detector.execute_with_tool(fallback_tool, message)
|
||||
if success:
|
||||
self.console.print(output)
|
||||
# Automatically switch to this working tool
|
||||
@@ -768,21 +724,15 @@ class EnhancedHanzoREPL:
|
||||
else:
|
||||
# Mark as failed
|
||||
self.failed_tools.add(fallback_tool.name)
|
||||
self.console.print(
|
||||
f"[red]{fallback_tool.display_name} also failed[/red]"
|
||||
)
|
||||
self.console.print(f"[red]{fallback_tool.display_name} also failed[/red]")
|
||||
|
||||
if not found_working:
|
||||
# Final fallback to cloud model
|
||||
self.console.print(f"[yellow]Falling back to cloud model...[/yellow]")
|
||||
await self.execute_command(
|
||||
"ask", f"--cloud --model gpt-3.5-turbo {message}"
|
||||
)
|
||||
await self.execute_command("ask", f"--cloud --model gpt-3.5-turbo {message}")
|
||||
else:
|
||||
# Use regular model through hanzo ask
|
||||
await self.execute_command(
|
||||
"ask", f"--cloud --model {self.current_model} {message}"
|
||||
)
|
||||
await self.execute_command("ask", f"--cloud --model {self.current_model} {message}")
|
||||
|
||||
async def quick_model_select(self, args: str = ""):
|
||||
"""Quick model selector with arrow keys."""
|
||||
@@ -791,11 +741,7 @@ class EnhancedHanzoREPL:
|
||||
return
|
||||
|
||||
# Prepare tools and models
|
||||
tools = (
|
||||
[(f"tool:{t.name}", t.display_name) for t in self.detected_tools]
|
||||
if self.detected_tools
|
||||
else []
|
||||
)
|
||||
tools = [(f"tool:{t.name}", t.display_name) for t in self.detected_tools] if self.detected_tools else []
|
||||
models = list(self.MODELS.items())
|
||||
|
||||
selector = QuickModelSelector(models, tools, self.current_model)
|
||||
@@ -826,9 +772,7 @@ class EnhancedHanzoREPL:
|
||||
else:
|
||||
# Show tasks and prompt for selection
|
||||
self.task_manager.list_tasks()
|
||||
self.console.print(
|
||||
"\n[cyan]Enter task ID to kill (or 'all' for all tasks):[/cyan]"
|
||||
)
|
||||
self.console.print("\n[cyan]Enter task ID to kill (or 'all' for all tasks):[/cyan]")
|
||||
try:
|
||||
task_id = await self.session.prompt_async("> ")
|
||||
if task_id:
|
||||
@@ -863,9 +807,7 @@ class EnhancedHanzoREPL:
|
||||
# Quick add
|
||||
try:
|
||||
todo = self.todo_manager.quick_add(rest)
|
||||
self.console.print(
|
||||
f"[green]✅ Added todo: {todo.title} (ID: {todo.id})[/green]"
|
||||
)
|
||||
self.console.print(f"[green]✅ Added todo: {todo.title} (ID: {todo.id})[/green]")
|
||||
except ValueError as e:
|
||||
self.console.print(f"[red]Error: {e}[/red]")
|
||||
else:
|
||||
@@ -891,9 +833,7 @@ class EnhancedHanzoREPL:
|
||||
elif key in ["tag", "t"]:
|
||||
tag = value
|
||||
|
||||
todos = self.todo_manager.list_todos(
|
||||
status=status, priority=priority, tag=tag
|
||||
)
|
||||
todos = self.todo_manager.list_todos(status=status, priority=priority, tag=tag)
|
||||
title = "Filtered Todos" if (status or priority or tag) else "All Todos"
|
||||
self.todo_manager.display_todos(todos, title)
|
||||
|
||||
@@ -902,9 +842,7 @@ class EnhancedHanzoREPL:
|
||||
if rest:
|
||||
todo = self.todo_manager.update_todo(rest, status="done")
|
||||
if todo:
|
||||
self.console.print(
|
||||
f"[green]✅ Marked as done: {todo.title}[/green]"
|
||||
)
|
||||
self.console.print(f"[green]✅ Marked as done: {todo.title}[/green]")
|
||||
else:
|
||||
self.console.print(f"[red]Todo not found: {rest}[/red]")
|
||||
else:
|
||||
@@ -960,9 +898,7 @@ class EnhancedHanzoREPL:
|
||||
elif subcommand in ["clear", "reset"]:
|
||||
# Clear all todos (with confirmation)
|
||||
try:
|
||||
confirm = await self.session.prompt_async(
|
||||
"Are you sure you want to delete ALL todos? (yes/no): "
|
||||
)
|
||||
confirm = await self.session.prompt_async("Are you sure you want to delete ALL todos? (yes/no): ")
|
||||
if confirm.lower() in ["yes", "y"]:
|
||||
self.todo_manager.todos = []
|
||||
self.todo_manager.save_todos()
|
||||
@@ -980,13 +916,9 @@ class EnhancedHanzoREPL:
|
||||
# Unknown subcommand, treat as quick add
|
||||
try:
|
||||
todo = self.todo_manager.quick_add(args)
|
||||
self.console.print(
|
||||
f"[green]✅ Added todo: {todo.title} (ID: {todo.id})[/green]"
|
||||
)
|
||||
self.console.print(f"[green]✅ Added todo: {todo.title} (ID: {todo.id})[/green]")
|
||||
except ValueError:
|
||||
self.console.print(
|
||||
f"[yellow]Unknown todo command: {subcommand}[/yellow]"
|
||||
)
|
||||
self.console.print(f"[yellow]Unknown todo command: {subcommand}[/yellow]")
|
||||
self.console.print("[dim]Use /todo help for available commands[/dim]")
|
||||
|
||||
async def add_todo_interactive(self):
|
||||
@@ -1002,21 +934,13 @@ class EnhancedHanzoREPL:
|
||||
description = await self.session.prompt_async("Description (optional): ")
|
||||
|
||||
# Get priority
|
||||
priority = await self.session.prompt_async(
|
||||
"Priority (low/medium/high/urgent) [medium]: "
|
||||
)
|
||||
priority = await self.session.prompt_async("Priority (low/medium/high/urgent) [medium]: ")
|
||||
if not priority:
|
||||
priority = "medium"
|
||||
|
||||
# Get tags
|
||||
tags_input = await self.session.prompt_async(
|
||||
"Tags (comma-separated, optional): "
|
||||
)
|
||||
tags = (
|
||||
[t.strip() for t in tags_input.split(",") if t.strip()]
|
||||
if tags_input
|
||||
else []
|
||||
)
|
||||
tags_input = await self.session.prompt_async("Tags (comma-separated, optional): ")
|
||||
tags = [t.strip() for t in tags_input.split(",") if t.strip()] if tags_input else []
|
||||
|
||||
# Get due date
|
||||
due_date = await self.session.prompt_async("Due date (optional): ")
|
||||
@@ -1030,9 +954,7 @@ class EnhancedHanzoREPL:
|
||||
due_date=due_date if due_date else None,
|
||||
)
|
||||
|
||||
self.console.print(
|
||||
f"[green]✅ Added todo: {todo.title} (ID: {todo.id})[/green]"
|
||||
)
|
||||
self.console.print(f"[green]✅ Added todo: {todo.title} (ID: {todo.id})[/green]")
|
||||
|
||||
except (KeyboardInterrupt, EOFError):
|
||||
self.console.print("[yellow]Cancelled[/yellow]")
|
||||
|
||||
@@ -14,9 +14,7 @@ from prompt_toolkit.layout.containers import HSplit, Window
|
||||
class QuickModelSelector:
|
||||
"""Quick model selector with arrow navigation."""
|
||||
|
||||
def __init__(
|
||||
self, models: List[Tuple[str, str]], tools: List[Tuple[str, str]], current: str
|
||||
):
|
||||
def __init__(self, models: List[Tuple[str, str]], tools: List[Tuple[str, str]], current: str):
|
||||
self.models = models
|
||||
self.tools = tools
|
||||
self.current = current
|
||||
@@ -99,9 +97,7 @@ class QuickModelSelector:
|
||||
|
||||
layout = Layout(Window(FormattedTextControl(get_text), wrap_lines=False))
|
||||
|
||||
app = Application(
|
||||
layout=layout, key_bindings=kb, full_screen=False, mouse_support=True
|
||||
)
|
||||
app = Application(layout=layout, key_bindings=kb, full_screen=False, mouse_support=True)
|
||||
|
||||
return await app.run_async()
|
||||
|
||||
@@ -138,9 +134,7 @@ class BackgroundTaskManager:
|
||||
task = self.tasks[task_id]
|
||||
if task["process"].poll() is None:
|
||||
task["process"].terminate()
|
||||
self.console.print(
|
||||
f"[yellow]Terminated {task_id}: {task['name']}[/yellow]"
|
||||
)
|
||||
self.console.print(f"[yellow]Terminated {task_id}: {task['name']}[/yellow]")
|
||||
else:
|
||||
self.console.print(f"[dim]Task {task_id} already stopped[/dim]")
|
||||
del self.tasks[task_id]
|
||||
|
||||
@@ -196,9 +196,7 @@ hanzo> mcp run read_file --arg path=README.md
|
||||
except subprocess.TimeoutExpired:
|
||||
self.console.print("[red]Command timed out[/red]")
|
||||
except FileNotFoundError:
|
||||
self.console.print(
|
||||
"[red]Command not found. Make sure 'hanzo' is installed.[/red]"
|
||||
)
|
||||
self.console.print("[red]Command not found. Make sure 'hanzo' is installed.[/red]")
|
||||
except Exception as e:
|
||||
self.console.print(f"[red]Command error: {e}[/red]")
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user