mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
997 lines
48 KiB
HTML
997 lines
48 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>ml-multi-model-orchestration</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/ml-multi-model-orchestration.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/ml-multi-model-orchestration">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/ml-multi-model-orchestration.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>ml-multi-model-orchestration</h1><p class="lede">Coordinating multiple LLMs in pipelines, ensembles, and cascading workflows</p><h1>Multi-Model Orchestration</h1>
|
|
<p><strong>Scope</strong>: Pipeline patterns, ensemble methods, specialist routing, cascading, context management, error handling <strong>Lines</strong>: ~400 <strong>Last Updated</strong>: 2025-10-26</p>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Building workflows that combine strengths of multiple models</li><li>Implementing consensus mechanisms for critical decisions</li><li>Cascading from fast models to accurate models based on confidence</li><li>Designing specialist systems where different models handle different subtasks</li><li>Chaining models in sequential pipelines (planning -> execution -> review)</li><li>Managing context and state across multi-model calls</li><li>Handling failures with fallback models</li><li>Optimizing for both quality and cost through model mixing</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Orchestration Patterns</h3>
|
|
<p><strong>1. Pipeline (Sequential)</strong>:</p>
|
|
<ul><li>Models execute in sequence, each using previous output</li><li>Example: Planner -> Coder -> Reviewer -> Refiner</li><li><strong>Benefit</strong>: Specialization per stage</li><li><strong>Cost</strong>: Cumulative latency</li></ul>
|
|
<p><strong>2. Ensemble (Parallel)</strong>:</p>
|
|
<ul><li>Multiple models process same input, outputs combined</li><li>Example: GPT-4o + Claude + Gemini -> Vote or merge</li><li><strong>Benefit</strong>: Higher accuracy, reduced bias</li><li><strong>Cost</strong>: 3x+ API costs</li></ul>
|
|
<p><strong>3. Specialist Routing</strong>:</p>
|
|
<ul><li>Route to model specialized for subtask</li><li>Example: Code->Claude, Math->DeepSeek R1, General->GPT-4o</li><li><strong>Benefit</strong>: Optimal model per task</li><li><strong>Cost</strong>: Routing overhead</li></ul>
|
|
<p><strong>4. Cascade (Conditional)</strong>:</p>
|
|
<ul><li>Start with cheap/fast model, escalate if needed</li><li>Example: GPT-4o mini -> (if unsure) -> GPT-4o</li><li><strong>Benefit</strong>: Cost savings with quality backstop</li><li><strong>Cost</strong>: Complexity in confidence detection</li></ul>
|
|
<p><strong>5. Hybrid (Combined)</strong>:</p>
|
|
<ul><li>Mix patterns: Pipeline + Ensemble, Cascade + Specialist</li><li>Example: Route by topic -> Ensemble -> Pipeline</li><li><strong>Benefit</strong>: Maximum flexibility</li><li><strong>Cost</strong>: Complex orchestration logic</li></ul>
|
|
<h3>Context Management</h3>
|
|
<p><strong>Challenges</strong>:</p>
|
|
<ul><li>Passing large context between models (token costs)</li><li>Maintaining conversation state across calls</li><li>Avoiding context window overflow</li><li>Preserving semantic coherence</li></ul>
|
|
<p><strong>Solutions</strong>:</p>
|
|
<ul><li>Prompt chaining (minimal context passing)</li><li>Shared memory/cache layer (Redis, database)</li><li>Summarization between stages (lossy but efficient)</li><li>Context compression (extract key information)</li></ul>
|
|
<h3>Error Handling Strategies</h3>
|
|
<p><strong>Fallback Chains</strong>:</p>
|
|
<pre><code>Primary model -> (on error) -> Fallback 1 -> Fallback 2 -> Default response</code></pre>
|
|
<p><strong>Retry with Backoff</strong>:</p>
|
|
<pre><code>Attempt 1 (immediate) -> Attempt 2 (+1s) -> Attempt 3 (+3s) -> Fail gracefully</code></pre>
|
|
<p><strong>Graceful Degradation</strong>:</p>
|
|
<pre><code>Try ensemble -> (on partial failure) -> Use successful responses -> (on total failure) -> Single model</code></pre>
|
|
<h3>Arize Phoenix Multi-Model Tracing</h3>
|
|
<p><strong>Capabilities</strong>:</p>
|
|
<ul><li>Span-level tracing across model calls</li><li>Routing decision visibility</li><li>Cost/latency breakdown per model</li><li>Error rate tracking per model</li><li>A/B test comparison</li></ul>
|
|
<p><strong>Span Structure</strong>:</p>
|
|
<pre><code>Root Span (orchestration)
|
|
├─ Span: routing_decision
|
|
├─ Span: model_call_1 (claude-3.5-sonnet)
|
|
│ ├─ Attribute: tokens_used
|
|
│ ├─ Attribute: latency_ms
|
|
│ └─ Attribute: cost_usd
|
|
├─ Span: model_call_2 (gpt-4o)
|
|
└─ Span: result_aggregation</code></pre>
|
|
<hr>
|
|
<h2>Implementation Patterns</h2>
|
|
<h3>Pipeline Pattern: Plan-Code-Review</h3>
|
|
<pre><code class="lang-python">import openai
|
|
from typing import Dict, Any
|
|
|
|
class CodeGenerationPipeline:
|
|
"""Multi-stage pipeline: planning -> coding -> review"""
|
|
|
|
def __init__(self):
|
|
self.planner_model = "gpt-4o"
|
|
self.coder_model = "claude-3-5-sonnet-20241022"
|
|
self.reviewer_model = "gpt-4o"
|
|
|
|
def plan(self, requirement: str) -> str:
|
|
"""Stage 1: Planning with GPT-4o"""
|
|
response = openai.ChatCompletion.create(
|
|
model=self.planner_model,
|
|
messages=[{
|
|
"role": "system",
|
|
"content": "You are a software architect. Create a detailed implementation plan."
|
|
}, {
|
|
"role": "user",
|
|
"content": f"Plan implementation for: {requirement}"
|
|
}],
|
|
temperature=0.7
|
|
)
|
|
return response.choices[0].message.content
|
|
|
|
def code(self, plan: str) -> str:
|
|
"""Stage 2: Coding with Claude (best at code)"""
|
|
response = openai.ChatCompletion.create(
|
|
model=self.coder_model,
|
|
messages=[{
|
|
"role": "system",
|
|
"content": "You are an expert programmer. Implement the plan precisely."
|
|
}, {
|
|
"role": "user",
|
|
"content": f"Implement this plan:\n\n{plan}"
|
|
}],
|
|
temperature=0.2
|
|
)
|
|
return response.choices[0].message.content
|
|
|
|
def review(self, code: str, plan: str) -> Dict[str, Any]:
|
|
"""Stage 3: Review with GPT-4o"""
|
|
response = openai.ChatCompletion.create(
|
|
model=self.reviewer_model,
|
|
messages=[{
|
|
"role": "system",
|
|
"content": "You are a code reviewer. Assess quality, correctness, and adherence to plan."
|
|
}, {
|
|
"role": "user",
|
|
"content": f"Review this code against the plan.\n\nPlan:\n{plan}\n\nCode:\n{code}"
|
|
}],
|
|
temperature=0.3
|
|
)
|
|
|
|
review_text = response.choices[0].message.content
|
|
|
|
# Extract approval (simple heuristic)
|
|
approved = "approved" in review_text.lower() or "looks good" in review_text.lower()
|
|
|
|
return {
|
|
"approved": approved,
|
|
"review": review_text,
|
|
"code": code
|
|
}
|
|
|
|
def execute(self, requirement: str) -> Dict[str, Any]:
|
|
"""Execute full pipeline"""
|
|
print("Stage 1: Planning...")
|
|
plan = self.plan(requirement)
|
|
|
|
print("Stage 2: Coding...")
|
|
code = self.code(plan)
|
|
|
|
print("Stage 3: Review...")
|
|
result = self.review(code, plan)
|
|
|
|
return {
|
|
"requirement": requirement,
|
|
"plan": plan,
|
|
"code": result["code"],
|
|
"review": result["review"],
|
|
"approved": result["approved"]
|
|
}
|
|
|
|
# Usage
|
|
pipeline = CodeGenerationPipeline()
|
|
result = pipeline.execute("Build a REST API for user authentication with JWT")
|
|
|
|
print(f"Approved: {result['approved']}")
|
|
print(f"Code:\n{result['code'][:500]}...")</code></pre>
|
|
<h3>Ensemble Pattern: Consensus Voting</h3>
|
|
<pre><code class="lang-python">import asyncio
|
|
from typing import List, Dict
|
|
from collections import Counter
|
|
|
|
class EnsembleOrchestrator:
|
|
"""Run multiple models in parallel and vote on results"""
|
|
|
|
def __init__(self, models: List[str] = None):
|
|
self.models = models or [
|
|
"gpt-4o",
|
|
"claude-3-5-sonnet-20241022",
|
|
"gemini-2.5-pro"
|
|
]
|
|
|
|
async def call_model(self, model: str, prompt: str) -> str:
|
|
"""Call single model asynchronously"""
|
|
response = await asyncio.to_thread(
|
|
openai.ChatCompletion.create,
|
|
model=model,
|
|
messages=[{"role": "user", "content": prompt}],
|
|
temperature=0.3
|
|
)
|
|
return response.choices[0].message.content
|
|
|
|
async def ensemble_vote(
|
|
self,
|
|
prompt: str,
|
|
extract_answer_fn = None
|
|
) -> Dict[str, Any]:
|
|
"""Run ensemble and vote on answers"""
|
|
|
|
# Call all models in parallel
|
|
tasks = [
|
|
self.call_model(model, prompt)
|
|
for model in self.models
|
|
]
|
|
responses = await asyncio.gather(*tasks)
|
|
|
|
# Extract answers (default: use full response)
|
|
if extract_answer_fn is None:
|
|
extract_answer_fn = lambda x: x.strip().lower()
|
|
|
|
answers = [extract_answer_fn(r) for r in responses]
|
|
|
|
# Vote
|
|
vote_counts = Counter(answers)
|
|
majority_answer = vote_counts.most_common(1)[0][0]
|
|
majority_count = vote_counts[majority_answer]
|
|
|
|
# Check consensus (>50%)
|
|
consensus = majority_count > len(self.models) / 2
|
|
|
|
return {
|
|
"models": self.models,
|
|
"responses": responses,
|
|
"answers": answers,
|
|
"majority_answer": majority_answer,
|
|
"vote_counts": dict(vote_counts),
|
|
"consensus": consensus,
|
|
"confidence": majority_count / len(self.models)
|
|
}
|
|
|
|
# Usage
|
|
async def main():
|
|
ensemble = EnsembleOrchestrator()
|
|
|
|
# Multiple choice question
|
|
prompt = """
|
|
What is the capital of Australia?
|
|
A) Sydney
|
|
B) Melbourne
|
|
C) Canberra
|
|
D) Brisbane
|
|
|
|
Answer with just the letter (A, B, C, or D).
|
|
"""
|
|
|
|
def extract_letter(response: str) -> str:
|
|
# Extract first letter A-D
|
|
import re
|
|
match = re.search(r'[ABCD]', response.upper())
|
|
return match.group(0) if match else ""
|
|
|
|
result = await ensemble.ensemble_vote(prompt, extract_answer_fn=extract_letter)
|
|
|
|
print(f"Majority answer: {result['majority_answer']}")
|
|
print(f"Consensus: {result['consensus']} ({result['confidence']:.0%})")
|
|
print(f"Vote counts: {result['vote_counts']}")
|
|
|
|
asyncio.run(main())</code></pre>
|
|
<h3>Cascade Pattern: Confidence-Based Escalation</h3>
|
|
<pre><code class="lang-python">import re
|
|
from typing import Optional, Dict, Any
|
|
|
|
class CascadeOrchestrator:
|
|
"""Start with cheap model, escalate to expensive if uncertain"""
|
|
|
|
def __init__(
|
|
self,
|
|
fast_model: str = "gpt-4o-mini",
|
|
strong_model: str = "gpt-4o",
|
|
confidence_threshold: float = 0.8
|
|
):
|
|
self.fast = fast_model
|
|
self.strong = strong_model
|
|
self.threshold = confidence_threshold
|
|
|
|
def extract_confidence(self, response: str) -> Optional[float]:
|
|
"""Extract confidence from response (if model provides it)"""
|
|
# Look for patterns like "Confidence: 85%" or "(confidence: 0.85)"
|
|
patterns = [
|
|
r"confidence[:\s]+(\d+(?:\.\d+)?)\%?",
|
|
r"\(confidence[:\s]+(\d+(?:\.\d+)?)\)",
|
|
]
|
|
|
|
for pattern in patterns:
|
|
match = re.search(pattern, response.lower())
|
|
if match:
|
|
conf = float(match.group(1))
|
|
return conf / 100 if conf > 1 else conf
|
|
|
|
return None
|
|
|
|
def call_with_confidence(
|
|
self,
|
|
model: str,
|
|
prompt: str,
|
|
request_confidence: bool = True
|
|
) -> Dict[str, Any]:
|
|
"""Call model and extract confidence"""
|
|
|
|
if request_confidence:
|
|
prompt += "\n\nProvide your confidence level (0-100%) at the end."
|
|
|
|
response = openai.ChatCompletion.create(
|
|
model=model,
|
|
messages=[{"role": "user", "content": prompt}],
|
|
temperature=0.3
|
|
)
|
|
|
|
text = response.choices[0].message.content
|
|
confidence = self.extract_confidence(text)
|
|
|
|
return {
|
|
"model": model,
|
|
"response": text,
|
|
"confidence": confidence,
|
|
"tokens": response.usage.total_tokens
|
|
}
|
|
|
|
def cascade(self, prompt: str) -> Dict[str, Any]:
|
|
"""Execute cascade: fast -> strong if needed"""
|
|
|
|
print(f"Trying fast model: {self.fast}")
|
|
fast_result = self.call_with_confidence(self.fast, prompt)
|
|
|
|
# Check if confident enough
|
|
if fast_result["confidence"] and fast_result["confidence"] >= self.threshold:
|
|
print(f"Fast model confident ({fast_result['confidence']:.0%}), using result")
|
|
return {
|
|
**fast_result,
|
|
"escalated": False,
|
|
"total_tokens": fast_result["tokens"]
|
|
}
|
|
|
|
# Escalate to strong model
|
|
print(f"Low confidence ({fast_result['confidence']:.0%}), escalating to {self.strong}")
|
|
strong_result = self.call_with_confidence(self.strong, prompt)
|
|
|
|
return {
|
|
**strong_result,
|
|
"escalated": True,
|
|
"fast_result": fast_result,
|
|
"total_tokens": fast_result["tokens"] + strong_result["tokens"]
|
|
}
|
|
|
|
# Usage
|
|
cascade = CascadeOrchestrator(confidence_threshold=0.8)
|
|
|
|
result = cascade.cascade("What are the ethical implications of AGI development?")
|
|
|
|
print(f"Final model: {result['model']}")
|
|
print(f"Escalated: {result['escalated']}")
|
|
print(f"Total tokens: {result['total_tokens']}")</code></pre>
|
|
<h3>Specialist Routing with Context Passing</h3>
|
|
<pre><code class="lang-python">from typing import List, Dict, Any
|
|
|
|
class SpecialistOrchestrator:
|
|
"""Route subtasks to specialist models"""
|
|
|
|
def __init__(self):
|
|
self.specialists = {
|
|
"code": "claude-3-5-sonnet-20241022",
|
|
"math": "deepseek-r1-distilled-70b",
|
|
"general": "gpt-4o",
|
|
"fast": "gemini-2.5-flash"
|
|
}
|
|
self.conversation_history: List[Dict] = []
|
|
|
|
def detect_task_type(self, query: str) -> str:
|
|
"""Simple task detection (use semantic routing for production)"""
|
|
query_lower = query.lower()
|
|
|
|
if any(kw in query_lower for kw in ["code", "function", "implement", "debug"]):
|
|
return "code"
|
|
if any(kw in query_lower for kw in ["calculate", "solve", "math", "equation"]):
|
|
return "math"
|
|
if len(query) < 50:
|
|
return "fast"
|
|
|
|
return "general"
|
|
|
|
def call_specialist(
|
|
self,
|
|
task_type: str,
|
|
query: str,
|
|
include_history: bool = False
|
|
) -> Dict[str, Any]:
|
|
"""Route to specialist and maintain context"""
|
|
|
|
model = self.specialists[task_type]
|
|
|
|
# Build messages with optional history
|
|
messages = []
|
|
if include_history:
|
|
messages.extend(self.conversation_history)
|
|
|
|
messages.append({"role": "user", "content": query})
|
|
|
|
response = openai.ChatCompletion.create(
|
|
model=model,
|
|
messages=messages,
|
|
temperature=0.3
|
|
)
|
|
|
|
answer = response.choices[0].message.content
|
|
|
|
# Update conversation history
|
|
self.conversation_history.append({"role": "user", "content": query})
|
|
self.conversation_history.append({"role": "assistant", "content": answer})
|
|
|
|
return {
|
|
"task_type": task_type,
|
|
"model": model,
|
|
"query": query,
|
|
"answer": answer,
|
|
"tokens": response.usage.total_tokens
|
|
}
|
|
|
|
def multi_turn_conversation(self, queries: List[str]) -> List[Dict[str, Any]]:
|
|
"""Handle multi-turn conversation with specialist routing"""
|
|
|
|
results = []
|
|
for query in queries:
|
|
task_type = self.detect_task_type(query)
|
|
print(f"Query: {query[:60]}... -> {task_type} ({self.specialists[task_type]})")
|
|
|
|
result = self.call_specialist(
|
|
task_type,
|
|
query,
|
|
include_history=True # Maintain context
|
|
)
|
|
results.append(result)
|
|
|
|
return results
|
|
|
|
# Usage
|
|
orchestrator = SpecialistOrchestrator()
|
|
|
|
conversation = [
|
|
"Write a function to calculate Fibonacci numbers",
|
|
"Now optimize it for large inputs",
|
|
"What's the time complexity?",
|
|
"Calculate the 50th Fibonacci number using your implementation"
|
|
]
|
|
|
|
results = orchestrator.multi_turn_conversation(conversation)
|
|
|
|
for r in results:
|
|
print(f"\n[{r['task_type']}] {r['model']}")
|
|
print(f"Answer: {r['answer'][:200]}...")</code></pre>
|
|
<h3>Phoenix Observability Integration</h3>
|
|
<pre><code class="lang-python">import phoenix as px
|
|
from phoenix.trace import using_project
|
|
from opentelemetry import trace
|
|
|
|
# Start Phoenix
|
|
px.launch_app()
|
|
|
|
tracer = trace.get_tracer(__name__)
|
|
|
|
@using_project("multi-model-orchestration")
|
|
def orchestrated_pipeline(requirement: str):
|
|
"""Pipeline with full observability"""
|
|
|
|
with tracer.start_as_current_span("pipeline") as root_span:
|
|
root_span.set_attribute("requirement", requirement)
|
|
|
|
# Stage 1: Planning
|
|
with tracer.start_as_current_span("stage_planning") as plan_span:
|
|
plan_span.set_attribute("model", "gpt-4o")
|
|
|
|
plan_response = openai.ChatCompletion.create(
|
|
model="gpt-4o",
|
|
messages=[{
|
|
"role": "user",
|
|
"content": f"Plan: {requirement}"
|
|
}]
|
|
)
|
|
|
|
plan = plan_response.choices[0].message.content
|
|
plan_span.set_attribute("tokens", plan_response.usage.total_tokens)
|
|
plan_span.set_attribute("cost_usd", calculate_cost(plan_response, "gpt-4o"))
|
|
|
|
# Stage 2: Coding
|
|
with tracer.start_as_current_span("stage_coding") as code_span:
|
|
code_span.set_attribute("model", "claude-3-5-sonnet")
|
|
|
|
code_response = openai.ChatCompletion.create(
|
|
model="claude-3-5-sonnet-20241022",
|
|
messages=[{
|
|
"role": "user",
|
|
"content": f"Implement:\n{plan}"
|
|
}]
|
|
)
|
|
|
|
code = code_response.choices[0].message.content
|
|
code_span.set_attribute("tokens", code_response.usage.total_tokens)
|
|
code_span.set_attribute("cost_usd", calculate_cost(code_response, "claude-3-5-sonnet"))
|
|
|
|
# Stage 3: Review
|
|
with tracer.start_as_current_span("stage_review") as review_span:
|
|
review_span.set_attribute("model", "gpt-4o")
|
|
|
|
review_response = openai.ChatCompletion.create(
|
|
model="gpt-4o",
|
|
messages=[{
|
|
"role": "user",
|
|
"content": f"Review:\nPlan: {plan}\nCode: {code}"
|
|
}]
|
|
)
|
|
|
|
review = review_response.choices[0].message.content
|
|
review_span.set_attribute("tokens", review_response.usage.total_tokens)
|
|
review_span.set_attribute("cost_usd", calculate_cost(review_response, "gpt-4o"))
|
|
|
|
# Calculate total metrics
|
|
total_cost = (
|
|
calculate_cost(plan_response, "gpt-4o") +
|
|
calculate_cost(code_response, "claude-3-5-sonnet") +
|
|
calculate_cost(review_response, "gpt-4o")
|
|
)
|
|
total_tokens = (
|
|
plan_response.usage.total_tokens +
|
|
code_response.usage.total_tokens +
|
|
review_response.usage.total_tokens
|
|
)
|
|
|
|
root_span.set_attribute("total_cost_usd", total_cost)
|
|
root_span.set_attribute("total_tokens", total_tokens)
|
|
|
|
return {
|
|
"plan": plan,
|
|
"code": code,
|
|
"review": review,
|
|
"cost": total_cost,
|
|
"tokens": total_tokens
|
|
}
|
|
|
|
def calculate_cost(response, model: str) -> float:
|
|
"""Calculate cost based on token usage"""
|
|
pricing = {
|
|
"gpt-4o": {"input": 5.0, "output": 20.0},
|
|
"claude-3-5-sonnet": {"input": 3.0, "output": 15.0},
|
|
"gpt-4o-mini": {"input": 0.15, "output": 0.60},
|
|
"gemini-2.5-flash": {"input": 0.10, "output": 0.40}
|
|
}
|
|
|
|
model_pricing = pricing.get(model, pricing["gpt-4o"])
|
|
usage = response.usage
|
|
|
|
cost = (
|
|
(usage.prompt_tokens / 1_000_000) * model_pricing["input"] +
|
|
(usage.completion_tokens / 1_000_000) * model_pricing["output"]
|
|
)
|
|
|
|
return cost</code></pre>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Orchestration Pattern Selection</h3>
|
|
<pre><code>Use Case | Recommended Pattern
|
|
----------------------------------|------------------------------------
|
|
Sequential specialization | Pipeline
|
|
Critical decision (safety) | Ensemble with voting
|
|
Cost optimization | Cascade (cheap -> expensive)
|
|
Domain-specific routing | Specialist routing
|
|
Complex multi-step workflows | Hybrid (Pipeline + Specialist)
|
|
High-reliability requirements | Ensemble + Fallback chains</code></pre>
|
|
<h3>Context Passing Strategies</h3>
|
|
<pre><code>Strategy | Cost | Latency | Use Case
|
|
-------------------|--------|---------|---------------------------
|
|
Full history | High | High | Short conversations (<10 turns)
|
|
Summarization | Medium | Medium | Long conversations
|
|
Key extraction | Low | Low | Factual pipelines
|
|
Shared state (DB) | Low | Medium | Multi-user systems
|
|
No context | Lowest | Lowest | Independent tasks</code></pre>
|
|
<h3>Error Handling Patterns</h3>
|
|
<pre><code class="lang-python"># Pattern 1: Fallback chain
|
|
try:
|
|
response = call_model("claude-3-5-sonnet", prompt)
|
|
except Exception:
|
|
try:
|
|
response = call_model("gpt-4o", prompt)
|
|
except Exception:
|
|
response = call_model("gpt-4o-mini", prompt)
|
|
|
|
# Pattern 2: Retry with exponential backoff
|
|
for attempt in range(3):
|
|
try:
|
|
response = call_model("gpt-4o", prompt)
|
|
break
|
|
except Exception as e:
|
|
if attempt == 2:
|
|
raise
|
|
time.sleep(2 ** attempt)
|
|
|
|
# Pattern 3: Graceful degradation
|
|
results = []
|
|
for model in ["gpt-4o", "claude-3-5-sonnet", "gemini-2.5-pro"]:
|
|
try:
|
|
results.append(call_model(model, prompt))
|
|
except Exception:
|
|
pass # Continue with other models
|
|
|
|
if results:
|
|
return aggregate(results)
|
|
else:
|
|
return default_response()</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<p>❌ <strong>Passing full context between all stages</strong>: Wastes tokens and money ✅ Summarize or extract key information between pipeline stages</p>
|
|
<p>❌ <strong>Ensemble without voting mechanism</strong>: Can't resolve disagreements ✅ Implement majority voting or weighted consensus</p>
|
|
<p>❌ <strong>Cascade without confidence scores</strong>: Can't decide when to escalate ✅ Prompt models for confidence or use logprobs for uncertainty estimation</p>
|
|
<p>❌ <strong>No fallback on model failures</strong>: Single point of failure ✅ Implement fallback chains with multiple model options</p>
|
|
<p>❌ <strong>Ignoring cost in orchestration</strong>: Ensemble costs 3x per query ✅ Track costs per pattern; use ensembles only for critical decisions</p>
|
|
<p>❌ <strong>Synchronous calls in ensemble</strong>: 3x latency penalty ✅ Use async/parallel calls for ensemble and specialist routing</p>
|
|
<p>❌ <strong>No observability in multi-model flows</strong>: Can't debug or optimize ✅ Use Arize Phoenix or similar for span-level tracing</p>
|
|
<p>❌ <strong>Hard-coded model names in orchestration</strong>: Brittle to model updates ✅ Use configuration or model registry for flexibility</p>
|
|
<hr>
|
|
<h2>Hanzo SDK: Simplified Multi-Model Orchestration</h2>
|
|
<p>Instead of manually implementing orchestration patterns, use <strong>Hanzo SDK</strong> for automatic multi-model coordination with built-in routing, caching, and failover.</p>
|
|
<h3>Pipeline Pattern with Hanzo</h3>
|
|
<pre><code class="lang-python">from hanzo import Hanzo
|
|
|
|
# Initialize with different strategies for different stages
|
|
hanzo_fast = Hanzo(routing_strategy='latency_optimized')
|
|
hanzo_smart = Hanzo(routing_strategy='quality_focused')
|
|
|
|
async def research_pipeline(topic: str) -> str:
|
|
# Stage 1: Fast outline generation
|
|
outline = await hanzo_fast.chat.completions.create_async(
|
|
messages=[{
|
|
'role': 'user',
|
|
'content': f'Create a research outline for: {topic}'
|
|
}]
|
|
)
|
|
|
|
# Stage 2: Quality content generation
|
|
content = await hanzo_smart.chat.completions.create_async(
|
|
messages=[{
|
|
'role': 'user',
|
|
'content': f'Write detailed content for: {outline.content}'
|
|
}]
|
|
)
|
|
|
|
return content.choices[0].message.content
|
|
|
|
# Hanzo automatically:
|
|
# - Routed stage 1 to fastest model (Gemini Flash)
|
|
# - Routed stage 2 to best model (GPT-4)
|
|
# - Handled retries and fallbacks
|
|
# - Tracked costs per stage</code></pre>
|
|
<h3>Ensemble Pattern with Hanzo</h3>
|
|
<pre><code class="lang-python">from hanzo import Hanzo
|
|
|
|
async def ensemble_decision(query: str) -> str:
|
|
# Hanzo can query multiple models in parallel
|
|
hanzo = Hanzo(inference_mode='hybrid')
|
|
|
|
# Parallel calls to different models
|
|
responses = await hanzo.chat.completions.batch_create_async([
|
|
{'messages': [{'role': 'user', 'content': query}], 'model': 'gpt-4'},
|
|
{'messages': [{'role': 'user', 'content': query}], 'model': 'claude-3-5-sonnet'},
|
|
{'messages': [{'role': 'user', 'content': query}], 'model': 'gemini-2.5-pro'}
|
|
])
|
|
|
|
# Extract answers
|
|
answers = [r.choices[0].message.content for r in responses]
|
|
|
|
# Aggregate with another model
|
|
final = await hanzo.chat.completions.create_async(
|
|
messages=[{
|
|
'role': 'user',
|
|
'content': f'Synthesize these expert opinions:\n\n' + '\n\n'.join(answers)
|
|
}],
|
|
model='gpt-4' # Use best model for synthesis
|
|
)
|
|
|
|
return final.choices[0].message.content</code></pre>
|
|
<h3>Specialist Routing with Hanzo MCP</h3>
|
|
<pre><code class="lang-python">from hanzo import Hanzo
|
|
from hanzo.mcp import MCPClient
|
|
|
|
# Hanzo can use MCP tools for specialized tasks
|
|
hanzo = Hanzo(
|
|
inference_mode='hybrid',
|
|
mcp_servers=['http://localhost:8081']
|
|
)
|
|
|
|
mcp = MCPClient('http://localhost:8081')
|
|
|
|
async def route_by_intent(query: str) -> str:
|
|
# First, classify intent
|
|
intent = await hanzo.chat.completions.create_async(
|
|
messages=[{
|
|
'role': 'user',
|
|
'content': f'Classify this query intent (code/search/analyze): {query}'
|
|
}],
|
|
model='gpt-4o-mini' # Fast classifier
|
|
)
|
|
|
|
if 'code' in intent.content.lower():
|
|
# Use code specialist model
|
|
return await hanzo.chat.completions.create_async(
|
|
messages=[{'role': 'user', 'content': query}],
|
|
model='claude-3-5-sonnet' # Best for code
|
|
)
|
|
elif 'search' in intent.content.lower():
|
|
# Use MCP search tool
|
|
tools = mcp.get_tools()
|
|
return await hanzo.chat.completions.create_async(
|
|
messages=[{'role': 'user', 'content': query}],
|
|
tools=tools,
|
|
tool_choice='auto'
|
|
)
|
|
else:
|
|
# General analysis
|
|
return await hanzo.chat.completions.create_async(
|
|
messages=[{'role': 'user', 'content': query}],
|
|
model='gpt-4' # Best for analysis
|
|
)</code></pre>
|
|
<h3>Cascade Pattern with Hanzo</h3>
|
|
<pre><code class="lang-python">from hanzo import Hanzo
|
|
|
|
hanzo_local = Hanzo(inference_mode='local')
|
|
hanzo_cloud = Hanzo(inference_mode='cloud')
|
|
|
|
async def cascade_query(query: str, complexity_threshold: float = 0.7) -> str:
|
|
# Try fast local model first
|
|
try:
|
|
response = await hanzo_local.chat.completions.create_async(
|
|
messages=[{'role': 'user', 'content': query}],
|
|
model='llama-3-8b',
|
|
timeout=5 # Fast timeout
|
|
)
|
|
|
|
# Check confidence (if model provides it)
|
|
if hasattr(response, 'confidence') and response.confidence > complexity_threshold:
|
|
return response.choices[0].message.content
|
|
except TimeoutError:
|
|
pass # Fall through to cloud
|
|
|
|
# Escalate to cloud model
|
|
response = await hanzo_cloud.chat.completions.create_async(
|
|
messages=[{'role': 'user', 'content': query}],
|
|
model='gpt-4' # Best model for complex queries
|
|
)
|
|
|
|
return response.choices[0].message.content</code></pre>
|
|
<h3>Comparison: Manual Orchestration vs Hanzo SDK</h3>
|
|
<p>| Pattern | Manual Implementation | Hanzo SDK | |---------|----------------------|-----------| | <strong>Pipeline</strong> | 50+ lines, manual routing | 10-15 lines, auto-routing | | <strong>Ensemble</strong> | Custom parallel execution | Built-in batch API | | <strong>Specialist</strong> | Manual intent classification | Automatic or MCP-based | | <strong>Cascade</strong> | Custom timeout/retry logic | Built-in with fallbacks | | <strong>Cost Tracking</strong> | Custom per-pattern | Automatic all patterns | | <strong>Caching</strong> | Manual between stages | Automatic semantic cache | | <strong>Observability</strong> | Manual OpenTelemetry | Built-in metrics | | <strong>Error Handling</strong> | Custom per model | Automatic retry/circuit breaker |</p>
|
|
<h3>Advanced: Multi-Model Workflow</h3>
|
|
<pre><code class="lang-python">from hanzo import Hanzo
|
|
from hanzo.workflows import Pipeline
|
|
|
|
# Define reusable workflow
|
|
class ResearchWorkflow(Pipeline):
|
|
def __init__(self):
|
|
super().__init__()
|
|
|
|
# Different Hanzo instances for different stages
|
|
self.fast_hanzo = Hanzo(routing_strategy='latency_optimized')
|
|
self.smart_hanzo = Hanzo(routing_strategy='quality_focused')
|
|
self.local_hanzo = Hanzo(inference_mode='local')
|
|
|
|
async def research(self, topic: str) -> dict:
|
|
# Stage 1: Generate outline (fast)
|
|
outline = await self.fast_hanzo.chat.completions.create_async(
|
|
messages=[{'role': 'user', 'content': f'Outline: {topic}'}]
|
|
)
|
|
|
|
# Stage 2: Parallel research on each section
|
|
sections = outline.content.split('\n')
|
|
research_tasks = [
|
|
self.local_hanzo.chat.completions.create_async(
|
|
messages=[{'role': 'user', 'content': f'Research: {section}'}]
|
|
)
|
|
for section in sections
|
|
]
|
|
research_results = await asyncio.gather(*research_tasks)
|
|
|
|
# Stage 3: Synthesize with best model
|
|
synthesis = await self.smart_hanzo.chat.completions.create_async(
|
|
messages=[{
|
|
'role': 'user',
|
|
'content': f'Synthesize:\n\n' + '\n\n'.join([r.content for r in research_results])
|
|
}]
|
|
)
|
|
|
|
return {
|
|
'outline': outline.content,
|
|
'research': [r.content for r in research_results],
|
|
'synthesis': synthesis.content,
|
|
'total_cost': sum([
|
|
outline.cost,
|
|
sum(r.cost for r in research_results),
|
|
synthesis.cost
|
|
])
|
|
}
|
|
|
|
# Use workflow
|
|
workflow = ResearchWorkflow()
|
|
result = await workflow.research('Quantum Computing Applications')
|
|
print(f"Total cost: ${result['total_cost']:.4f}")</code></pre>
|
|
<p><strong>Key Benefits</strong>:</p>
|
|
<ul><li><strong>10x less code</strong>: Hanzo SDK handles routing, retries, caching automatically</li><li><strong>Built-in observability</strong>: Automatic cost tracking and performance metrics</li><li><strong>Privacy-first</strong>: Local models via Hanzo Node before cloud escalation</li><li><strong>Production-ready</strong>: Circuit breakers, retries, fallbacks included</li><li><strong>Type-safe</strong>: Full TypeScript support with Python compatibility</li></ul>
|
|
<p><strong>See Also</strong>:</p>
|
|
<ul><li><strong>python-sdk.md</strong> - Complete Hanzo SDK documentation</li><li><strong>hanzo-mcp.md</strong> - MCP tool integration for workflows</li><li><strong>hanzo-node.md</strong> - Local model inference</li><li><strong>llm-model-routing.md</strong> - Automatic routing strategies</li></ul>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>llm-model-routing.md</code> - Routing strategies and RouteLLM framework</li><li><code>llm-model-selection.md</code> - Choosing models for capabilities and cost</li><li><code>api-error-handling.md</code> - Robust error handling across providers</li><li><code>observability-distributed-tracing.md</code> - OpenTelemetry and Phoenix</li><li><code>modal-web-endpoints.md</code> - Deploying orchestration as FastAPI services</li><li><code>redis-caching.md</code> - Shared state and caching for multi-model systems</li><li><strong><code>python-sdk.md</code></strong> - Hanzo SDK for multi-model orchestration</li><li><strong><code>hanzo-mcp.md</code></strong> - MCP tools for specialized tasks</li><li><strong><code>hanzo-node.md</code></strong> - Local inference infrastructure</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-28 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/ml-multi-model-orchestration.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |