mirror of
https://github.com/hanzoai/python-sdk.git
synced 2026-10-27 04:20:09 +00:00
zeekay
4a2f0cdfb5
- Create full E2E test demonstrating local AI orchestration capabilities - Test Qwen3, Llama 3.2, Mistral, DeepSeek as local orchestrators - Configure hybrid network with Claude, GPT-4, Gemini as API workers - Implement task routing tests (simple → local, complex → API) - Add cost optimization validation (>50% local usage expected) - Create Docker Compose setup for local model serving (Ollama, LocalAI, vLLM) - Add setup scripts for automatic model downloading - Include Makefile for easy test execution - Document full architecture and usage patterns Test scenarios: 1. Task Routing: Validates correct agent selection based on complexity 2. Cost Optimization: Verifies 60-90% cost reduction through local models 3. Integration: Tests full hanzo dev command with local orchestrator This E2E test proves the system can: - Use local private AI (Qwen3, etc.) as orchestrator - Manage subnets of expensive API models - Route intelligently to minimize costs - Maintain full capability while reducing expenses by up to 90%
111 lines
2.6 KiB
YAML
111 lines
2.6 KiB
YAML
version: '3.8'
|
|
|
|
services:
|
|
# Ollama for running local models
|
|
ollama:
|
|
image: ollama/ollama:latest
|
|
container_name: hanzo-ollama
|
|
ports:
|
|
- "11434:11434"
|
|
volumes:
|
|
- ollama_data:/root/.ollama
|
|
environment:
|
|
- OLLAMA_HOST=0.0.0.0
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:11434/api/tags"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
|
|
# LocalAI for additional model support
|
|
localai:
|
|
image: quay.io/go-skynet/local-ai:latest
|
|
container_name: hanzo-localai
|
|
ports:
|
|
- "8080:8080"
|
|
volumes:
|
|
- ./models:/models
|
|
- ./images:/tmp/images
|
|
environment:
|
|
- DEBUG=true
|
|
- MODELS_PATH=/models
|
|
- GALLERIES=[{"name":"model-gallery","url":"github:go-skynet/model-gallery/index.yaml"}]
|
|
- PRELOAD_MODELS=[{"id":"qwen3","name":"qwen3","urls":["https://huggingface.co/Qwen/Qwen2.5-3B-Instruct-GGUF"]}]
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
memory: 8G
|
|
cpus: '4'
|
|
|
|
# Text Generation Web UI for testing
|
|
text-generation-webui:
|
|
image: atinoda/text-generation-webui:default
|
|
container_name: hanzo-text-gen
|
|
ports:
|
|
- "7860:7860"
|
|
- "5000:5000" # API port
|
|
volumes:
|
|
- ./models:/app/models
|
|
- ./characters:/app/characters
|
|
- ./presets:/app/presets
|
|
environment:
|
|
- CLI_ARGS=--api --listen --model-dir /app/models
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
memory: 16G
|
|
cpus: '8'
|
|
|
|
# vLLM for high-performance inference
|
|
vllm:
|
|
image: vllm/vllm-openai:latest
|
|
container_name: hanzo-vllm
|
|
ports:
|
|
- "8000:8000"
|
|
volumes:
|
|
- ./models:/models
|
|
environment:
|
|
- HF_HOME=/models
|
|
command: >
|
|
--model mistralai/Mistral-7B-Instruct-v0.1
|
|
--host 0.0.0.0
|
|
--port 8000
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|
|
limits:
|
|
memory: 24G
|
|
|
|
# MLX Server for Apple Silicon (Mac only)
|
|
# Uncomment if running on Mac with Apple Silicon
|
|
# mlx-server:
|
|
# build:
|
|
# context: .
|
|
# dockerfile: Dockerfile.mlx
|
|
# container_name: hanzo-mlx
|
|
# ports:
|
|
# - "5001:5001"
|
|
# volumes:
|
|
# - ./models:/models
|
|
# environment:
|
|
# - MLX_MODEL=mlx-community/Llama-3.2-3B-Instruct-4bit
|
|
# platform: linux/arm64
|
|
|
|
volumes:
|
|
ollama_data:
|
|
|
|
networks:
|
|
default:
|
|
name: hanzo-net
|
|
driver: bridge |