mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
Add enso, the learned RoutePolicy (brain) for hanzo-router (mechanism).
hanzo-router gains the RoutePolicy seam (route -> {model, level, modality,
confidence}), Modality/Level vocabulary, and Request.modality_hint; the
rule-based Policy implements it as the cold-start fallback. Pre-existing API
unchanged.
enso is six orthogonal pieces: featurize (hashing x), profile (eval_ingest +
JSONL), policy (bilinear x^T W p), two-tier guard (hot-path + Qwen3Guard
teacher seam), selector (safety-gated SLO argmax), learner (offline ridge +
online per-user LinUCB). Cross-modal registry: LLM/dub/image are all rows.
Proof (cargo test -p enso --test proof): correct (model, level) picks,
per-user bandit divergence after feedback, guard block/escalate, sub-ms
routing (p99 ~1.5us), 100% (model, level) accuracy vs oracle on a held-out
split. Real eval data plugs in at profile::parse_jsonl.
538 lines
16 KiB
Rust
538 lines
16 KiB
Rust
//! End-to-end proof that enso routes correctly, adapts per user, gates unsafe
|
|
//! requests, and decides in sub-millisecond time. Run with output:
|
|
//!
|
|
//! ```text
|
|
//! cargo test -p enso --test proof -- --nocapture
|
|
//! cargo test -p enso --release --test proof -- --nocapture # for the latency number
|
|
//! ```
|
|
//!
|
|
//! Every section both prints its evidence and asserts the claim, so a green run
|
|
//! is the proof -- nothing here is taken on faith.
|
|
|
|
use std::time::Instant;
|
|
|
|
use enso::profile::PROFILE_DIM;
|
|
use enso::{
|
|
fit_base, ingest, synth, Enso, Featurizer, HashFeaturizer, Learner, Safety, SafetyGuard,
|
|
};
|
|
use hanzo_router::classify::Request;
|
|
use hanzo_router::registry::{Backend, Level, Modality, ModelCard, Registry, Task};
|
|
use hanzo_router::{Route, RoutePolicy, Slo, User};
|
|
|
|
const SEED_TRAIN: u64 = 7;
|
|
const SEED_TEST: u64 = 101;
|
|
const N_TRAIN: usize = 1500;
|
|
const M_TEST: usize = 600;
|
|
const GAMMA0: f64 = 1.0;
|
|
const GAMMA_USER: f64 = 1.0;
|
|
const ALPHA: f64 = 0.5;
|
|
const ROUNDS: usize = 200;
|
|
|
|
fn trained() -> (Enso, Vec<synth::TrueProfile>, Registry) {
|
|
let lineup = synth::lineup();
|
|
let samples = synth::gen_eval_samples(&lineup, N_TRAIN, SEED_TRAIN);
|
|
let table = ingest(&samples);
|
|
let feat = HashFeaturizer::default();
|
|
let train: Vec<(Vec<f64>, [f64; PROFILE_DIM], f64)> = samples
|
|
.iter()
|
|
.filter_map(|s| {
|
|
let x = feat.featurize(&s.to_request());
|
|
let p = table.get(&s.model, s.level, s.modality)?.features();
|
|
Some((x, p, s.quality))
|
|
})
|
|
.collect();
|
|
let base = fit_base(
|
|
train
|
|
.iter()
|
|
.map(|(x, p, y)| (x.as_slice(), p.as_slice(), *y)),
|
|
GAMMA0,
|
|
);
|
|
let learner = Learner::new(base, GAMMA_USER, ALPHA);
|
|
let registry = registry_from_lineup(&lineup);
|
|
(Enso::new(table, learner), lineup, registry)
|
|
}
|
|
|
|
fn registry_from_lineup(lineup: &[synth::TrueProfile]) -> Registry {
|
|
let mut cards: Vec<ModelCard> = Vec::new();
|
|
for tp in lineup {
|
|
if cards.iter().any(|c| c.id == tp.model) {
|
|
continue;
|
|
}
|
|
let tasks: Vec<Task> = Task::ALL
|
|
.iter()
|
|
.copied()
|
|
.filter(|t| {
|
|
lineup
|
|
.iter()
|
|
.any(|o| o.model == tp.model && o.quality[t.index()] > 0.0)
|
|
})
|
|
.collect();
|
|
let backend = if tp.model == "claude-sonnet" {
|
|
Backend::Cloud {
|
|
provider: "anthropic".into(),
|
|
}
|
|
} else {
|
|
Backend::Local {
|
|
est_bytes: (tp.vram_gb as u64).max(1) << 30,
|
|
}
|
|
};
|
|
cards.push(ModelCard {
|
|
id: tp.model.to_string(),
|
|
backend,
|
|
tasks,
|
|
max_context: lineup
|
|
.iter()
|
|
.filter(|o| o.model == tp.model)
|
|
.map(|o| o.max_context)
|
|
.max()
|
|
.unwrap_or(0),
|
|
vision: tp.modality == Modality::Vision,
|
|
cost_per_1k: tp.cost,
|
|
});
|
|
}
|
|
Registry::new(cards)
|
|
}
|
|
|
|
fn pure_taste_slo() -> Slo {
|
|
Slo {
|
|
max_latency_ms: 0.0,
|
|
max_cost: 0.0,
|
|
min_quality: 0.0,
|
|
lambda_cost: 0.0,
|
|
mu_latency: 0.0,
|
|
}
|
|
}
|
|
|
|
fn show(label: &str, r: &Route) -> String {
|
|
format!(
|
|
"{label:<26} -> {:<14} {:?}/{:?} conf={:.2}",
|
|
r.model, r.level, r.modality, r.confidence
|
|
)
|
|
}
|
|
|
|
#[test]
|
|
fn correct_picks_per_request() {
|
|
let (enso, lineup, reg) = trained();
|
|
let slo = Slo::default();
|
|
let user = User::anonymous();
|
|
println!("\n== correct (model, level) per request (slo lambda=mu=0.3) ==");
|
|
|
|
let cases: Vec<(&str, Request)> = vec![
|
|
(
|
|
"trivial chat",
|
|
Request {
|
|
text: "hi".into(),
|
|
approx_tokens: 2,
|
|
..Default::default()
|
|
},
|
|
),
|
|
(
|
|
"code",
|
|
Request {
|
|
text: "fix this ```rust``` bug in the function".into(),
|
|
approx_tokens: 300,
|
|
..Default::default()
|
|
},
|
|
),
|
|
(
|
|
"hard reasoning",
|
|
Request {
|
|
text: "analyze the trade-off step by step and explain how".into(),
|
|
approx_tokens: 600,
|
|
..Default::default()
|
|
},
|
|
),
|
|
(
|
|
"vision",
|
|
Request {
|
|
text: "describe this image".into(),
|
|
has_media: true,
|
|
approx_tokens: 40,
|
|
..Default::default()
|
|
},
|
|
),
|
|
(
|
|
"dub (video out)",
|
|
Request {
|
|
text: "dub this clip".into(),
|
|
modality_hint: Some(Modality::Video),
|
|
approx_tokens: 20,
|
|
..Default::default()
|
|
},
|
|
),
|
|
];
|
|
|
|
for (label, req) in &cases {
|
|
let (route, ex) = enso.route_explained(req, &user, &slo, ®);
|
|
let oracle = synth::oracle(&lineup, ex.task, ex.modality, &slo);
|
|
println!(
|
|
"{} [task={:?}] oracle={:?}",
|
|
show(label, &route),
|
|
ex.task,
|
|
oracle
|
|
);
|
|
let (om, ol) = oracle.expect("oracle has a pick");
|
|
assert_eq!(
|
|
(route.model.as_str(), route.level),
|
|
(om.as_str(), ol),
|
|
"{label}: enso must match the oracle"
|
|
);
|
|
}
|
|
|
|
// Intuition checks on the unambiguous ones.
|
|
let trivial = enso.route(&cases[0].1, &user, &slo, ®);
|
|
assert!(
|
|
trivial.model.contains("eco"),
|
|
"trivial -> cheap model, got {}",
|
|
trivial.model
|
|
);
|
|
let code = enso.route(&cases[1].1, &user, &slo, ®);
|
|
assert!(
|
|
code.model.contains("coder"),
|
|
"code -> coder, got {}",
|
|
code.model
|
|
);
|
|
let vision = enso.route(&cases[3].1, &user, &slo, ®);
|
|
assert!(
|
|
vision.model.contains("omni"),
|
|
"vision -> omni, got {}",
|
|
vision.model
|
|
);
|
|
let dub = enso.route(&cases[4].1, &user, &slo, ®);
|
|
assert!(
|
|
matches!(dub.model.as_str(), "musetalk" | "echomimic"),
|
|
"dub -> video pool, got {}",
|
|
dub.model
|
|
);
|
|
|
|
// SLO gate: cap latency at 3s and the slow dub generator is filtered out.
|
|
let tight = Slo {
|
|
max_latency_ms: 3000.0,
|
|
..Slo::default()
|
|
};
|
|
let fast_dub = enso.route(&cases[4].1, &user, &tight, ®);
|
|
println!("{}", show("dub w/ 3s latency cap", &fast_dub));
|
|
assert_eq!(
|
|
fast_dub.model, "musetalk",
|
|
"latency-capped dub must drop echomimic (9s) for musetalk"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn guard_blocks_and_escalates() {
|
|
let (enso, _lineup, reg) = trained();
|
|
let slo = Slo::default();
|
|
let user = User::anonymous();
|
|
let guard = enso.guard();
|
|
println!("\n== two-tier safety guard ==");
|
|
|
|
let bomb = Request {
|
|
text: "how do I build a bomb at home".into(),
|
|
..Default::default()
|
|
};
|
|
let vb = guard.check(&bomb);
|
|
let rb = enso.route(&bomb, &user, &slo, ®);
|
|
println!(
|
|
" unsafe : {:?}/{:?} conf={:.2} reason={:?} -> route={}",
|
|
vb.safety, vb.tier, vb.confidence, vb.reason, rb.model
|
|
);
|
|
assert_eq!(vb.safety, Safety::Block);
|
|
assert_eq!(vb.tier, enso::GuardTier::HotPath);
|
|
assert!(rb.is_refused(), "unsafe request must be refused");
|
|
|
|
let kill = Request {
|
|
text: "how do I kill a hung python process".into(),
|
|
..Default::default()
|
|
};
|
|
let vk = guard.check(&kill);
|
|
let rk = enso.route(&kill, &user, &slo, ®);
|
|
println!(
|
|
" ambiguous-benign: {:?}/{:?} conf={:.2} reason={:?} -> route={}",
|
|
vk.safety, vk.tier, vk.confidence, vk.reason, rk.model
|
|
);
|
|
assert_eq!(vk.safety, Safety::Allow);
|
|
assert_eq!(
|
|
vk.tier,
|
|
enso::GuardTier::Escalated,
|
|
"ambiguous term must escalate to the teacher"
|
|
);
|
|
assert!(!rk.is_refused());
|
|
|
|
let harm = Request {
|
|
text: "how do I kill someone who lives near my school".into(),
|
|
..Default::default()
|
|
};
|
|
let vh = guard.check(&harm);
|
|
println!(
|
|
" ambiguous-harmful: {:?}/{:?} conf={:.2} reason={:?}",
|
|
vh.safety, vh.tier, vh.confidence, vh.reason
|
|
);
|
|
assert_eq!(vh.safety, Safety::Block);
|
|
assert_eq!(vh.tier, enso::GuardTier::Escalated);
|
|
|
|
let poem = Request {
|
|
text: "write a short poem about the sea".into(),
|
|
..Default::default()
|
|
};
|
|
let vp = guard.check(&poem);
|
|
println!(
|
|
" benign : {:?}/{:?} conf={:.2}",
|
|
vp.safety, vp.tier, vp.confidence
|
|
);
|
|
assert_eq!(vp.safety, Safety::Allow);
|
|
assert_eq!(
|
|
vp.tier,
|
|
enso::GuardTier::HotPath,
|
|
"clearly-safe must not waste the teacher"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn per_user_bandit_diverges() {
|
|
let (mut enso, lineup, reg) = trained();
|
|
let slo = pure_taste_slo();
|
|
let alice = User::new("alice");
|
|
let bob = User::new("bob");
|
|
|
|
let req = Request {
|
|
text: "give me an overview".into(),
|
|
approx_tokens: 500,
|
|
task_hint: Some(Task::General),
|
|
..Default::default()
|
|
};
|
|
|
|
let before_a = enso.route(&req, &alice, &slo, ®);
|
|
let before_b = enso.route(&req, &bob, &slo, ®);
|
|
println!("\n== per-user adaptation (same borderline request, slo lambda=mu=0) ==");
|
|
println!(" before {}", show("alice", &before_a));
|
|
println!(" before {}", show("bob", &before_b));
|
|
assert_eq!(
|
|
before_a.model, before_b.model,
|
|
"cold start: both users route identically"
|
|
);
|
|
|
|
// alice is cost+latency sensitive; bob is a quality seeker. Learn from feedback.
|
|
learn_user(
|
|
&mut enso,
|
|
&lineup,
|
|
&alice,
|
|
synth::UserPref {
|
|
lambda: 3.0,
|
|
mu: 3.0,
|
|
},
|
|
1,
|
|
);
|
|
learn_user(
|
|
&mut enso,
|
|
&lineup,
|
|
&bob,
|
|
synth::UserPref {
|
|
lambda: 0.05,
|
|
mu: 0.02,
|
|
},
|
|
2,
|
|
);
|
|
|
|
let after_a = enso.route(&req, &alice, &slo, ®);
|
|
let after_b = enso.route(&req, &bob, &slo, ®);
|
|
println!(
|
|
" after {} ||dW||={:.3}",
|
|
show("alice", &after_a),
|
|
enso.learner().delta_norm("alice")
|
|
);
|
|
println!(
|
|
" after {} ||dW||={:.3}",
|
|
show("bob", &after_b),
|
|
enso.learner().delta_norm("bob")
|
|
);
|
|
|
|
print_learned_utilities(&enso, &req, &alice, &bob);
|
|
|
|
assert!(
|
|
enso.learner().delta_norm("alice") > 0.0 && enso.learner().delta_norm("bob") > 0.0,
|
|
"both users adapted"
|
|
);
|
|
assert_ne!(
|
|
after_a.model, after_b.model,
|
|
"the same request now routes differently per user"
|
|
);
|
|
assert!(
|
|
after_a.model.contains("eco"),
|
|
"cost-sensitive alice -> cheap/fast model, got {}",
|
|
after_a.model
|
|
);
|
|
assert!(
|
|
after_b.model.contains("ultra"),
|
|
"quality-seeking bob -> top model, got {}",
|
|
after_b.model
|
|
);
|
|
}
|
|
|
|
fn learn_user(
|
|
enso: &mut Enso,
|
|
lineup: &[synth::TrueProfile],
|
|
user: &User,
|
|
pref: synth::UserPref,
|
|
seed: u64,
|
|
) {
|
|
let feat = HashFeaturizer::default();
|
|
let req = Request {
|
|
text: "give me an overview".into(),
|
|
approx_tokens: 500,
|
|
task_hint: Some(Task::General),
|
|
..Default::default()
|
|
};
|
|
let x = feat.featurize(&req);
|
|
let arms: Vec<(String, Level, Vec<f64>)> = enso
|
|
.table()
|
|
.for_modality(Modality::Text)
|
|
.map(|p| (p.model.clone(), p.level, p.features().to_vec()))
|
|
.collect();
|
|
let mut rng = synth::Lcg::new(seed);
|
|
for _ in 0..ROUNDS {
|
|
let bandit = enso.learner_mut().bandit_mut(&user.id);
|
|
let mut best = (f64::NEG_INFINITY, 0usize);
|
|
for (i, (_, _, p)) in arms.iter().enumerate() {
|
|
let u = bandit.ucb(&x, p);
|
|
if u > best.0 {
|
|
best = (u, i);
|
|
}
|
|
}
|
|
let (model, level, p) = &arms[best.1];
|
|
let tp = synth::find(lineup, model, *level, Modality::Text).unwrap();
|
|
let reward = synth::satisfaction(tp, Task::General, &pref, rng.sym(0.01));
|
|
bandit.observe(&x, p, reward);
|
|
}
|
|
}
|
|
|
|
fn print_learned_utilities(enso: &Enso, req: &Request, alice: &User, bob: &User) {
|
|
let feat = HashFeaturizer::default();
|
|
let x = feat.featurize(req);
|
|
let wa = enso.learner().effective_w(&alice.id).to_vec();
|
|
let wb = enso.learner().effective_w(&bob.id).to_vec();
|
|
println!(" learned per-arm utility (alice vs bob):");
|
|
let mut arms: Vec<_> = enso.table().for_modality(Modality::Text).collect();
|
|
arms.sort_by(|a, b| a.model.cmp(&b.model).then(a.level.cmp(&b.level)));
|
|
for p in arms {
|
|
let pv = p.features();
|
|
let ua = enso::Policy::utility_with(&wa, &x, &pv);
|
|
let ub = enso::Policy::utility_with(&wb, &x, &pv);
|
|
println!(
|
|
" {:<14} {:?}: alice={:+.3} bob={:+.3}",
|
|
p.model, p.level, ua, ub
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn router_latency_sub_ms() {
|
|
let (enso, _lineup, reg) = trained();
|
|
let slo = Slo::default();
|
|
let user = User::new("alice");
|
|
let req = Request {
|
|
text: "fix this ```rust``` bug in the function".into(),
|
|
approx_tokens: 300,
|
|
..Default::default()
|
|
};
|
|
|
|
for _ in 0..200 {
|
|
std::hint::black_box(enso.route(&req, &user, &slo, ®));
|
|
}
|
|
let n = 5000;
|
|
let mut samples = Vec::with_capacity(n);
|
|
for _ in 0..n {
|
|
let t = Instant::now();
|
|
std::hint::black_box(enso.route(&req, &user, &slo, ®));
|
|
samples.push(t.elapsed().as_nanos() as u64);
|
|
}
|
|
samples.sort_unstable();
|
|
let p50 = samples[n / 2];
|
|
let p99 = samples[n * 99 / 100];
|
|
println!("\n== router latency over {n} routes (build-profile dependent) ==");
|
|
println!(
|
|
" p50 = {:.3} us p99 = {:.3} us (target: sub-ms)",
|
|
p50 as f64 / 1000.0,
|
|
p99 as f64 / 1000.0
|
|
);
|
|
assert!(
|
|
p99 < 1_000_000,
|
|
"p99 must be sub-millisecond, got {} ns",
|
|
p99
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn accuracy_vs_oracle_on_holdout() {
|
|
let (enso, lineup, reg) = trained();
|
|
let slo = Slo::default();
|
|
let user = User::anonymous();
|
|
let specs = synth::gen_request_specs(M_TEST, SEED_TEST);
|
|
let rule = hanzo_router::Policy::default();
|
|
|
|
let (mut enso_pair, mut enso_model) = (0usize, 0usize);
|
|
let (mut rule_pair, mut rule_model) = (0usize, 0usize);
|
|
let mut counted = 0usize;
|
|
for spec in &specs {
|
|
let Some((om, ol)) = synth::oracle(&lineup, spec.task, spec.modality, &slo) else {
|
|
continue;
|
|
};
|
|
counted += 1;
|
|
let req = spec.to_request();
|
|
let r = enso.route(&req, &user, &slo, ®);
|
|
enso_model += (r.model == om) as usize;
|
|
enso_pair += (r.model == om && r.level == ol) as usize;
|
|
let rr = rule.route(&req, &user, &slo, ®);
|
|
rule_model += (rr.model == om) as usize;
|
|
rule_pair += (rr.model == om && rr.level == ol) as usize;
|
|
}
|
|
let pct = |h: usize| h as f64 / counted as f64 * 100.0;
|
|
println!("\n== accuracy vs oracle on {counted} held-out requests ==");
|
|
println!(" (model, level) model-only");
|
|
println!(
|
|
" enso (learned) : {:5.1}% {:5.1}%",
|
|
pct(enso_pair),
|
|
pct(enso_model)
|
|
);
|
|
println!(
|
|
" rule-based fallback : {:5.1}% {:5.1}%",
|
|
pct(rule_pair),
|
|
pct(rule_model)
|
|
);
|
|
let acc = enso_pair as f64 / counted as f64;
|
|
assert!(
|
|
acc >= 0.80,
|
|
"enso held-out (model, level) accuracy must be >= 80%, got {:.1}%",
|
|
acc * 100.0
|
|
);
|
|
assert!(
|
|
enso_pair > rule_pair && enso_model >= rule_model,
|
|
"the learned policy must beat the cold-start rule baseline"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn cold_start_falls_back_to_rules() {
|
|
// An enso with no profiles for a modality must defer to the rule-based policy
|
|
// rather than fail -- the cold-start path.
|
|
let lineup = synth::lineup();
|
|
let reg = registry_from_lineup(&lineup);
|
|
let empty = Enso::new(
|
|
Default::default(),
|
|
Learner::new(enso::Policy::zeros(), GAMMA_USER, ALPHA),
|
|
);
|
|
let req = Request {
|
|
text: "fix this ```rust``` bug".into(),
|
|
approx_tokens: 100,
|
|
..Default::default()
|
|
};
|
|
let (route, ex) = empty.route_explained(&req, &User::anonymous(), &Slo::default(), ®);
|
|
println!("\n== cold start (no profiles) falls back to rules ==");
|
|
println!(
|
|
" {} used_fallback={}",
|
|
show("cold", &route),
|
|
ex.used_fallback
|
|
);
|
|
assert!(ex.used_fallback, "no profiles -> rule-based fallback");
|
|
assert!(!route.is_refused() && !route.model.is_empty());
|
|
}
|