mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
449 lines
54 KiB
HTML
449 lines
54 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>hip-0125</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/hip-0125.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/hip-0125">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/hip-0125.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>hip-0125</h1><p class="lede">HIP-125: Consensus-Backed Plugin-Placement Platform. Status Final. Hanzo architectural specification.</p><h1>HIP-0125: Consensus-Backed Plugin-Placement Platform</h1>
|
|
<h2>Abstract</h2>
|
|
<p>This proposal turns the unified <code>cloud</code> binary (HIP-0106) into a <strong>planet-scale, consensus-backed, plugin-placement PaaS runtime</strong>. A plugin author implements one small interface and calls <code>cloud.Register(name, order, fn)</code> (HIP-0105); in return the runtime gives that plugin <strong>placement, per-tenant scale, streaming replication, per-connection pinning, and high-availability for free</strong> — the plugin author never writes a lease, a failover, a proxy hop, or a replication loop.</p>
|
|
<p>The mechanism is **two planes, decomplected, where consensus orders <em>decisions</em>, not *bytes***:</p>
|
|
<ul><li>A <strong>control plane</strong> — one small <strong>Quasar</strong> post-quantum BFT replicated</li></ul>
|
|
<p> state machine (RSM) across a fixed seven-voter quorum (<code>luxfi/consensus/protocol/quasar</code> <code>NewEngine</code>; ordered finality, strict-PQ from day one) — whose finalized block sequence holds only tiny metadata: cluster membership, the placement map <code>(plugin, shard) → {writer, followers}</code>, and writer leases carrying fencing epochs. This RSM <strong>replaces</strong> the interim Kubernetes-Lease writer-fence in <code>hanzoai/kms/lease.go</code>.</p>
|
|
<ul><li>A <strong>data plane</strong> — the existing <code>hanzoai/kms/replication.go</code> async</li></ul>
|
|
<p> S3/ZapDB log-ship (HIP-0107), reused <strong>unchanged</strong>. The <em>only</em> edit is the write-fence seam: <code>canPush</code> (<code>replication.go:452</code>) flips its question from "do I hold the Kubernetes Lease?" to "does the control plane grant this shard's lease at epoch <em>E</em>?", backed by an S3 epoch compare-and-swap (CAS).</p>
|
|
<p>Consensus sits <strong>off the data path</strong>. Losing the control-plane quorum freezes placement and failover but <strong>does not stop serving</strong>: every current writer keeps its lease and keeps writing. This is the Chubby/etcd shape — a small, strongly-consistent coordination kernel steering a large, weakly- coupled fleet — applied to the Hanzo plugin runtime.</p>
|
|
<h2>Motivation</h2>
|
|
<p>HIP-0106 collapsed ~35 Hanzo Go subsystems into one binary and one artifact. That solved <em>packaging</em>. It did not solve <em>placement</em>: which replica owns the write side of a given subsystem for a given tenant, how a second replica is prevented from writing the same state, and how a new plugin inherits all of that without re-implementing it.</p>
|
|
<p>Today the answer is partial and per-service:</p>
|
|
<ul><li><strong>KMS</strong> already ships single-writer durability: <code>replication.go</code> log-ships</li></ul>
|
|
<p> ZapDB to an age-encrypted S3 mirror (HIP-0107), and <code>lease.go</code> fences the writer with a <strong>Kubernetes Lease object</strong>. This works, but the fence lives in <code>hanzoai/kms</code> alone, is coupled to the Kubernetes API server, and gives exactly one writer for the <em>whole</em> service — no per-tenant sharding, no runtime failover story beyond "kube-controller reassigns the Lease."</p>
|
|
<ul><li><strong>IAM</strong> carries an explicit <code>Replicas > 1</code> guard — it refuses to run more</li></ul>
|
|
<p> than one replica because sessions and JWKS are single-writer with no coordinated fence.</p>
|
|
<ul><li>Every <strong>other</strong> stateful subsystem (<code>tasks</code>, <code>pubsub</code>, the 24 SQLite-</li></ul>
|
|
<p> backed services) reinvents "who writes" ad hoc, or runs <code>Deployment(replicas: 1)</code> and calls it HA.</p>
|
|
<p>Three "almost-the-same" single-writer patterns, three failure stories, and a hard ceiling at one replica for anything with write state. A new plugin author inherits none of it — they get a route mount and a data directory, and must solve placement themselves.</p>
|
|
<p>The unifying observation: <strong>placement is a tiny, strongly-consistent decision problem sitting on top of a large, already-solved, weakly- consistent byte problem.</strong> The bytes (WAL frames, ZapDB deltas) are already handled by HIP-0107's one pipeline. What is missing is a single authority that says, linearizably, <em>who</em> may push those bytes for <em>which</em> shard at <em>which</em> epoch. That authority is a replicated state machine. This HIP adds exactly that RSM, wires the existing fence to consult it, and exposes the whole thing to plugin authors as an inherited property of <code>cloud.Register</code>.</p>
|
|
<h2>Specification</h2>
|
|
<h3>Two planes, decomplected</h3>
|
|
<p>The load-bearing decision is that <strong>consensus orders decisions, not bytes.</strong> The two planes never braid:</p>
|
|
<p>| | Control plane | Data plane | |---|---|---| | <strong>What it orders</strong> | Placement decisions (membership, <code>(plugin,shard)→{writer,followers}</code>, writer leases + epochs) | Durable bytes (SQLite WAL frames, ZapDB deltas, snapshots) | | <strong>Substrate</strong> | <strong>Quasar</strong> PQ-BFT RSM (<code>luxfi/consensus/protocol/quasar</code>), 7-voter quorum (5-of-7), ordered finality | <code>hanzoai/replicate</code> over <code>hanzo/vfs</code>, async age-encrypted S3 log-ship (HIP-0107) | | <strong>Size</strong> | Kilobytes. The whole placement map is small enough to fit in memory on every node. | Terabytes. Per-tenant WAL/delta streams. | | <strong>Consistency</strong> | Linearizable (the RSM is the single source of truth for who-owns-what) | Eventually consistent to the mirror; strongly consistent at the single writer | | <strong>On the request path?</strong> | <strong>No</strong> | <strong>Yes</strong> (the writer serves; followers proxy) | | <strong>Failure if it stops</strong> | Placement frozen; no new failovers or shard moves; <strong>serving continues</strong> | Serving stops for the affected shard until a writer is (re)placed |</p>
|
|
<p>Because the control plane is off the request path, a control-plane quorum loss is a <strong>degradation of agility, not of availability</strong>: existing writers hold their leases and keep serving reads and writes; only <em>rebalancing</em> and <em>failover</em> pause until quorum returns. This is deliberately the Chubby lock- service / etcd shape — a small consensus kernel that hands out leases and gets out of the way.</p>
|
|
<h3>Control plane: the placement RSM</h3>
|
|
<p>The control plane is one <strong>Quasar</strong> replicated state machine. Quasar is a <strong>post-quantum, Byzantine-fault-tolerant</strong> consensus engine with ordered finality — the correct substrate when the decision being ordered is "who is allowed to write," because a <em>wrong</em> or <em>equivocating</em> answer must be <strong>impossible</strong>, not merely improbable, and because the lease-granting quorum itself must be quantum-secure on-doctrine. The RSM drives the engine (<code>luxfi/consensus/protocol/quasar</code> <code>NewEngine(cfg)</code>) through its ordered-finality API — <code>Submit(block)</code> appends a placement command, <code>Finalized() <-chan *Block</code> yields the finalized block sequence, and <code>IsFinalized(id)</code> checks a specific command — and the <strong>placement map is the state machine built by deterministically reducing that finalized sequence</strong>. Each finalized block carries small, idempotent commands:</p>
|
|
<pre><code>PlacementCommand {
|
|
Kind // Join | Leave | Assign | Revoke | RenewLease | Rebalance
|
|
Plugin // plugin name from cloud.Register(name, ...)
|
|
Shard // shard key (see "Unit and shard key")
|
|
NodeID // stable StatefulSet NodeID
|
|
Epoch // monotonically increasing fencing epoch for this shard
|
|
}</code></pre>
|
|
<p>The RSM's deterministic reduction of that log yields the <strong>placement map</strong>:</p>
|
|
<pre><code>PlacementMap {
|
|
Members map[NodeID]MemberInfo // current voter + data-node set
|
|
Owners map[Shard]Assignment // Assignment{ Writer NodeID; Followers []NodeID; Epoch uint64 }
|
|
}</code></pre>
|
|
<p>Two invariants the RSM enforces by construction:</p>
|
|
<ol><li><strong>One writer per shard per epoch.</strong> An <code>Assign(shard, node, epoch)</code> is</li></ol>
|
|
<p> only committed if <code>epoch</code> is strictly greater than the shard's current epoch. Because the log is linearizable, two nodes can never both hold a valid lease for the same shard at the same epoch.</p>
|
|
<ol><li><strong>Epochs are monotonic per shard.</strong> A revoked or superseded writer's</li></ol>
|
|
<p> epoch is permanently in the past; it can never be revived. This is the fencing token the data plane checks (below).</p>
|
|
<p>The RSM log holds <em>only</em> this metadata. It never sees a WAL frame, a secret, or a tenant byte. It is a coordination kernel, not a database.</p>
|
|
<p>This RSM <strong>replaces the interim Kubernetes-Lease fence</strong> in <code>hanzoai/kms/lease.go</code>. That file's <code>writerLease</code> (a lease object polled against the Kubernetes API server) was always a placeholder for "a real distributed lease with fencing." The RSM <em>is</em> that lease, with a fencing epoch the Kubernetes Lease never carried, and with no dependency on the Kubernetes control plane being reachable on the write path.</p>
|
|
<p><strong>Post-quantum from day one.</strong> The engine runs under a strict-PQ cert profile from the start — <em>not</em> classical-now / PQ-later. The RSM calls <code>SetProfile(*config.ChainSecurityProfile)</code> to bind the strict profile, and the <code>Certifier</code>'s <code>demandsTriple</code> gate then requires the post-quantum triple-mode certificate before a block finalizes. The threshold signer is <strong>Pulsar RoundSigner</strong> (<code>protocol/quasar/pulsar/round_signer.go</code> — a <code>PulsarRoundSigner</code> that adapts <code>luxfi/pulsar</code>; <code>NewPQ</code> = "BLS + Corona threshold," Corona being the Ring-LWE production threshold), and it <strong>fails closed</strong> to the PQ path rather than degrading to a classical signature. The quorum that grants a writer lease is therefore quantum-secure by construction, consistent with the platform's strict-PQ doctrine.</p>
|
|
<p><strong>Full Byzantine agreement, N nodes.</strong> The quorum shape is decided: the control plane runs <strong>true N-node Byzantine agreement</strong>, not crash-fault replication. Each finalized block is certified by a threshold signature whose shares are <strong>split across the voter pods</strong> and aggregated through the Pulsar RoundSigner's <code>Round1</code> / <code>Round2</code> / <code>Finalize</code> protocol carried <strong>over ZAP</strong> (HIP-0114); every voter then independently <strong>verifies the aggregated post-quantum certificate before applying the finalized block</strong> to its placement-map state machine. A block lacking a valid triple-mode PQ cert never finalizes, so a Byzantine minority of voters can neither forge a placement decision nor equivocate a lease grant. Byzantine tolerance and the strict-PQ cert are <strong>orthogonal and both on from day one</strong>: the first is <em>how many</em> faulty voters are survived; the second is <em>what</em> signs the result.</p>
|
|
<h3>What Quasar provides — and what it does not</h3>
|
|
<p>Grounded in the actual <code>github.com/luxfi/consensus/protocol/quasar</code> package:</p>
|
|
<p><strong>Provides:</strong></p>
|
|
<ul><li><strong>Native membership.</strong> The <code>luxfi/validators</code> Manager tracks a validator</li></ul>
|
|
<p> set per network ID — the control-plane voter set maps directly onto it.</p>
|
|
<ul><li><strong>An ordered-finality log.</strong> <code>NewEngine(cfg)</code></li></ul>
|
|
<p> (<code>protocol/quasar/engine.go:59</code>) returns an <code>Engine</code> whose <code>Submit(block)</code> / <code>Finalized() <-chan *Block</code> / <code>IsFinalized(id)</code> API is a totally ordered, finalized command stream — exactly the substrate a replicated state machine reduces into the placement map.</p>
|
|
<ul><li><strong>Post-quantum BFT finality.</strong> <code>SetProfile(*config.ChainSecurityProfile)</code></li></ul>
|
|
<p> binds a strict-PQ profile; the <code>Certifier</code>/<code>demandsTriple</code> triple-mode gate requires the PQ certificate before a block finalizes; the <strong>Pulsar RoundSigner</strong> (<code>pulsar/round_signer.go</code>, adapting <code>luxfi/pulsar</code>) is the quantum-secure threshold signer. BFT ordered finality suitable for a small fixed quorum — quantum-secure from the start.</p>
|
|
<ul><li><strong>Pure-Go embed.</strong> Builds with <code>CGO_ENABLED=0</code>, so the control plane links</li></ul>
|
|
<p> into the same static <code>cloud</code> artifact (HIP-0106) with no native toolchain and no second process.</p>
|
|
<p><strong>Does not provide (by design):</strong></p>
|
|
<ul><li><strong>No pinned-writer primitive.</strong> Quasar is a <em>leaderless</em> engine — there is</li></ul>
|
|
<p> deliberately no built-in "this node is the leader for this key" notion. <strong>The host builds writer-pinning on top, as RSM leases.</strong> That is the work this HIP specifies: the placement map and its epoch-fenced <code>Assign</code>/<code>Revoke</code> commands are the pinned-writer layer the engine intentionally omits.</p>
|
|
<p>The takeaway: Quasar gives ordered, post-quantum agreement and membership; the <em>meaning</em> of the ordered commands (leases, placement) is the host's, and this HIP defines it.</p>
|
|
<p><strong>Dependency prerequisite (external, not yet satisfied).</strong> The strict-PQ path does <strong>not</strong> build from published module tags today, and this HIP does not pretend otherwise. The RoundSigner primitive the adapter consumes <strong>is</strong> published — <code>github.com/luxfi/pulsar/ref/go/pkg/pulsar</code> in <code>luxfi/pulsar</code> <strong>v1.1.5</strong> (build-proven since v1.1.3). The gap is <strong>consensus-side</strong>: the cert-profile sub-package <code>protocol/quasar/pulsar</code> (the <code>PulsarRoundSigner</code> adapter itself) is <strong>unpublished</strong> — it appears in <strong>0 of 268</strong> <code>luxfi/consensus</code> release tags and lives only on the dev branch <code>feat/quasar-pulsar-profile</code> (@<code>009ad013</code>), where it is wired via a <code>replace github.com/luxfi/pulsar => ../luxfi/pulsar</code> that <strong>Go ignores for downstream consumers</strong>. So a <code>cloud</code> importing any published <code>consensus</code> tag cannot import <code>protocol/quasar/pulsar</code> at all. The base <code>protocol/quasar</code> engine (<code>NewEngine</code>/<code>Submit</code>/<code>Finalized</code>/…) is already published; only the cert-profile sub-package needs releasing. The single prerequisite is therefore a <strong><code>luxfi/consensus</code> release tag that carries <code>protocol/quasar/pulsar</code>, <code>require</code>s published <code>luxfi/pulsar v1.1.5</code>, and drops the dev-branch <code>replace</code></strong> — a forward-port that is in progress now. Until that tag lands, the control plane builds strict-PQ only against local sibling checkouts. Stating this is the honest status — a HIP is a proposal, and this is a real, unmet external dependency the proposal is blocked on.</p>
|
|
<h3>Data plane: replication reused unchanged</h3>
|
|
<p>The data plane is HIP-0107's pipeline, reused <strong>verbatim</strong>: <code>replicate.Source</code> → frame-pump → <code>luxfi/age</code> encrypt → <code>hanzo/vfs</code> sink. KMS's <code>replication.go</code> already runs it against ZapDB with an age-encrypted S3 mirror. Nothing about the byte path changes — same frames, same encryption boundary, same bucket layout (<code>vfs://<backend>/<bucket>/replicate/<org>/<service>/<source>/</code>), same recovery objectives.</p>
|
|
<h3>The write-fence seam — the one code edit</h3>
|
|
<p>The entire coupling between the two planes is a single function. Today <code>canPush</code> (<code>hanzoai/kms/replication.go:452</code>) gates the primary's push loop on the Kubernetes Lease:</p>
|
|
<pre><code class="lang-go">// today (interim): push only if we hold the K8s Lease and the store is hydrated
|
|
func canPush(hydrated *atomic.Bool, fence *writerLease) bool {
|
|
return hydrated.Load() && fence.Held()
|
|
}</code></pre>
|
|
<p>Under this HIP the seam flips to consult the control plane, and the S3 write is additionally guarded by an epoch CAS:</p>
|
|
<pre><code class="lang-go">// proposed: push only if the control plane grants THIS shard's lease at epoch E,
|
|
// and the S3 mirror's fencing epoch is not already ahead of us.
|
|
func canPush(hydrated *atomic.Bool, grant LeaseGrant) bool {
|
|
return hydrated.Load() && grant.Valid() // grant = placement map's Assignment for our shard
|
|
}
|
|
// ... and every push CASes the mirror's `latest` fencing epoch:
|
|
// write succeeds iff mirrorEpoch <= grant.Epoch, then bumps mirrorEpoch := grant.Epoch</code></pre>
|
|
<p>Two fences, defense in depth:</p>
|
|
<ol><li><strong>RSM grant</strong> (linearizable, in-memory): a node pushes only for shards</li></ol>
|
|
<p> the placement map says it writes, at the current epoch.</p>
|
|
<ol><li><strong>S3 epoch-CAS</strong> (durable, at the sink): even a partitioned node that</li></ol>
|
|
<p> <em>believes</em> it still holds the lease is fenced at the mirror — a push carrying a stale epoch loses the CAS against a <code>latest</code> pointer already advanced by the successor writer. A deposed writer physically cannot corrupt the shared log.</p>
|
|
<p><code>lease.go</code>'s Kubernetes-Lease path stays in the tree <strong>behind a flag</strong> for one release (stage 2 below), so the cutover is reversible.</p>
|
|
<h3>Topology</h3>
|
|
<pre><code> ┌─────────────────── control plane (fast, fixed) ───────────────────┐
|
|
│ voter-0 voter-1 voter-2 voter-3 voter-4 │
|
|
│ └── Quasar PQ-BFT RSM · 7-voter quorum (5-of-7) · placement map ───┘
|
|
└───────────────────────────────┬───────────────────────────────────┘
|
|
│ watch: read-only placement map
|
|
┌────────────────────────────────┼──────────────────────────────────┐
|
|
│ data-0 data-1 data-2 … data-(N-1) (non-voting, unbounded)│
|
|
│ each: hosts shards · runs replication.go · serves or proxies │
|
|
└───────────────────────────────────────────────────────────────────┘
|
|
│
|
|
per-tenant ZapDB/SQLite + age-encrypted S3 mirror (HIP-0107)</code></pre>
|
|
<ul><li><strong>7 voter nodes</strong> run the BFT control plane. Seven is the smallest set</li></ul>
|
|
<p> that tolerates two simultaneous <strong>Byzantine</strong> failures (<code>N ≥ 3f+1</code>, so <code>f=2</code>) with a quorum of five (<code>2f+1</code>); the count is fixed so consensus stays fast (agreement among seven, not among the whole fleet).</p>
|
|
<ul><li><strong>N non-voting data nodes</strong> each <em>watch</em> a read-only replica of the</li></ul>
|
|
<p> placement map (the RSM streams committed updates to them), host the shards the map assigns them, run the unchanged <code>replication.go</code>, and serve or proxy requests. Data nodes carry no consensus weight, so <strong>N scales unbounded</strong> without slowing agreement.</p>
|
|
<p>A node may be both a voter and a data node in small deployments; at scale they separate so control-plane latency is independent of data-plane fan-out.</p>
|
|
<h3>Unit and shard key</h3>
|
|
<p>The <strong>unit of placement is the plugin</strong> — precisely the <code>(name, order, fn)</code> registered via <code>cloud.Register</code> (<code>hanzoai/cloud/build.go:351</code>) — crossed with a <strong>shard</strong>:</p>
|
|
<ul><li><strong>Coarse first (ship this):</strong> <code>shard = the whole plugin</code>. One global</li></ul>
|
|
<p> writer per plugin, every tenant's data behind it. <strong>Zero storage change</strong> — the plugin's existing single data directory is the single shard. This is the KMS shape today, generalized to every plugin.</p>
|
|
<ul><li><strong>Fine later (opt-in, stage 8):</strong> <code>shard = (plugin, org)</code>. Per-tenant</li></ul>
|
|
<p> writers, using the HIP-0302 per-org file layout <code>{DataDir}/orgs/{org}/{svc}.db</code> that already exists. A hot tenant gets its own writer on its own node; a cold tenant costs nothing until touched.</p>
|
|
<p>Node <strong>specialization</strong> is expressed two ways, both orthogonal to code:</p>
|
|
<ol><li><strong>Which shards/writers live where</strong> — the placement map, <em>dynamic</em>, moved</li></ol>
|
|
<p> by the RSM at runtime. This is failover and rebalancing.</p>
|
|
<ol><li><strong>Which plugins a node-pool may run at all</strong> — the <code>--enable</code> subsystem</li></ol>
|
|
<p> set from HIP-0106, <em>static</em> per node-pool, for resource isolation (e.g. a GPU pool enables only inference plugins).</p>
|
|
<p>Specialization is <strong>never</strong> a runtime mount/unmount of plugin code. Every <code>cloud</code> binary contains every compiled plugin (HIP-0106); placement decides which <em>shards</em> are <em>active</em> where, not which <em>code</em> is <em>present</em>. One artifact, many roles, by data — not by rebuild.</p>
|
|
<h3>Per-connection pinning</h3>
|
|
<p>Pinning is one middleware, mounted immediately after <code>IdentityMiddleware</code> (<code>hanzoai/cloud/serve.go:97</code>) so it runs with a validated org already in context:</p>
|
|
<pre><code>1. Identity middleware resolves org from the JWT owner claim (HIP-0026).
|
|
2. Placement middleware computes shard = (plugin, org) [coarse: shard = plugin].
|
|
3. It looks up Owners[shard] in the watched, read-only placement map (in-memory, no RPC).
|
|
4a. This node is the writer → serve locally.
|
|
4b. Read that requires read-your-writes → reverse-proxy to the writer.
|
|
4c. Stale-OK read → serve from a local follower copy, or proxy to any follower.
|
|
4d. This node is neither → reverse-proxy to the writer.</code></pre>
|
|
<p>Proxying is <strong>pod-to-pod over the StatefulSet headless service</strong> — a node dials <code>cloud-<N>.cloud-headless</code> directly by stable DNS, no load balancer in the middle, no extra hop through ingress. Writes and read-your-writes go to the shard's writer; explicitly stale-OK reads may be served by any follower (the followers are the HA standbys — see Decided forks). The client sees one endpoint; the fleet routes the connection to the byte-owner underneath.</p>
|
|
<h3>Plugin authoring — the payoff</h3>
|
|
<p>This is the point of the HIP. A plugin author writes one small interface implementation and one <code>cloud.Register</code> call, and <strong>inherits consensus placement, per-tenant sharding, shared ZapDB replication, per-connection pinning, and HA — for free.</strong> They write none of those mechanisms.</p>
|
|
<p>The default wire is <strong>ZAP-native</strong> (HIP-0114), high-performance by construction: <code>hanzoai/s3</code> is already ZAP-native (gRPC call-count = 0), and KMS's <code>embed.go</code> runs a ZAP server today. A new plugin gets the same transport with no extra work.</p>
|
|
<p>A complete "hello, sharded, replicated, HA plugin" — about twenty lines:</p>
|
|
<pre><code class="lang-go">package hello
|
|
|
|
import (
|
|
"github.com/hanzoai/cloud"
|
|
"github.com/hanzoai/zip"
|
|
)
|
|
|
|
// Store is the plugin's per-shard state; the runtime opens it at the
|
|
// HIP-0302 path for the shard the placement map assigns to this node,
|
|
// and log-ships it via HIP-0107 replication. The author does not manage it.
|
|
type Store struct{ db cloud.ShardDB }
|
|
|
|
func mount(app *zip.App, deps cloud.Deps) error {
|
|
s := &Store{db: deps.OpenShard("hello")} // coarse: one shard for the plugin
|
|
app.Get("/v1/hello/:key", func(c zip.Ctx) error {
|
|
return c.JSON(s.db.Get(c.Param("key"))) // pinned to a follower or writer automatically
|
|
})
|
|
app.Put("/v1/hello/:key", func(c zip.Ctx) error {
|
|
return s.db.Put(c.Param("key"), c.Body()) // pinned to the writer automatically
|
|
})
|
|
return nil
|
|
}
|
|
|
|
func init() { cloud.Register("hello", 500, mount) } // that's the whole integration</code></pre>
|
|
<p>Everything the author did <em>not</em> write, but got: the RSM chose which node writes <code>hello</code>; the placement middleware routed each connection to that node; <code>replication.go</code> mirrored the ZapDB to age-encrypted S3; a follower stands ready to be promoted on writer loss; and the epoch fence guarantees no split-brain write. The plugin is planet-scale and HA because it was registered, not because it was engineered to be.</p>
|
|
<h3>Deployment: Deployment → StatefulSet</h3>
|
|
<p>The runtime needs <strong>stable identity</strong> (a node's <code>NodeID</code> must survive a restart so the RSM membership and placement map stay valid) and <strong>stable storage</strong> (a node must reopen the same shard files). That is a StatefulSet, not a Deployment.</p>
|
|
<p>| Property | Today (HIP-0106) | This HIP | |---|---|---| | Workload kind | <code>Deployment(strategy: Recreate, replicas: 1)</code> | <code>StatefulSet(replicas: N)</code> | | Identity | Random pod name | Stable <code>NodeID</code> = ordinal (<code>cloud-0 … cloud-(N-1)</code>) | | Networking | Service only | <strong>Headless service</strong> for pod-to-pod (<code>cloud-<N>.cloud-headless</code>) + existing Services | | Storage | Shared/ephemeral | <code>volumeClaimTemplates</code> → per-pod PVC (each node's shards) | | Upgrade | Recreate (full stop) | <code>RollingUpdate{maxUnavailable: 1}</code>, <code>OrderedReady</code> |</p>
|
|
<p>Because <code>NodeID</code>s are <strong>stable across image upgrades</strong>, a rolling upgrade is <em>not</em> a membership change: the RSM sees the same seven voters and the same data-node identities before and after, so <strong>no epoch reconfiguration and no placement churn</strong> happens on a deploy. <code>maxUnavailable: 1</code> keeps the voter quorum (6 of 7, above the 5-of-7 threshold) live throughout the roll.</p>
|
|
<p>The existing Service aliases keep resolving: <code>kms</code> and <code>cloud-api</code> Services select <code>app.kubernetes.io/name: cloud</code>, which the StatefulSet pods still carry, so nothing downstream re-points. The operator's <code>hanzo.ai/v1 kind: Service</code> CR (HIP-0400) gains a <strong>StatefulSet-emitting mode</strong> — a boolean on the CR that makes the operator render a StatefulSet + headless service + volumeClaimTemplates instead of a Deployment. One CR field, one new render path; the CR contract is otherwise unchanged.</p>
|
|
<h2>Migration plan</h2>
|
|
<p>Nine stages. <strong>Each is independently correct, canary-able, and reversible.</strong> <code>[M]</code> = mechanical (config/plumbing, no behavior flip). <code>[BR]</code> = blue→red (a behavior change that must be adversarially reviewed before it ships).</p>
|
|
<h3>Stage 0 — dependencies + inert config <code>[M]</code></h3>
|
|
<p>Vendor <code>luxfi/consensus</code> (Quasar) and <code>luxfi/pulsar</code> into <code>cloud</code> — pure-Go, <code>CGO_ENABLED=0</code>. <strong>Gated on the external dependency prerequisite</strong> (see "What Quasar provides"): the strict-PQ cert-profile package <code>protocol/quasar/pulsar</code> is unpublished (dev branch only), so until a <code>luxfi/consensus</code> release tag carries it (requiring published <code>luxfi/pulsar v1.1.5</code>, <code>replace</code> removed), this stage builds strict-PQ only against local sibling checkouts — not yet reproducible from module tags. Add the placement-map types and config knobs <strong>inert</strong> — compiled in, read nothing, change nothing. Ship. Nothing behaves differently; the surface exists.</p>
|
|
<h3>Stage 1 — control-plane RSM in SHADOW <code>[BR]</code></h3>
|
|
<p>Mount the placement RSM as a new <code>cluster</code> subsystem (its own <code>cloud.Register</code>) running in <strong>shadow</strong>: the seven voters agree on membership and a placement map, data nodes watch it, but <strong>nothing consults it yet</strong> — the live fence is still the Kubernetes Lease. Verify in production that the shadow map matches reality (the RSM's chosen writer == the actual Lease holder) before trusting it.</p>
|
|
<h3>Stage 2 — flip the KMS <code>canPush</code> fence seam <code>[BR]</code></h3>
|
|
<p>Point <code>canPush</code> (<code>replication.go:452</code>) at the RSM grant instead of <code>fence.Held()</code>, and add the S3 epoch-CAS to the push. Keep <code>lease.go</code>'s Kubernetes-Lease path <strong>behind a flag</strong> for one release so the cutover is a config toggle, reversible in seconds. This is the single highest-value edit; it is <code>[BR]</code> because it changes who-may-write.</p>
|
|
<h3>Stage 3 — operator emits StatefulSet <code>[M]</code></h3>
|
|
<p>Teach the operator's <code>Service</code> CR the StatefulSet-emitting mode (stable <code>NodeID</code>, headless service, <code>volumeClaimTemplates</code>, <code>RollingUpdate maxUnavailable:1 OrderedReady</code>). Migrate KMS's own workload to it. Still <code>replicas: 1</code> — this stage only changes the <em>kind</em>, not the count.</p>
|
|
<h3>Stage 4 — KMS <code>N > 1</code>, coarse <code>[BR]</code></h3>
|
|
<p>Raise KMS to multiple replicas with the whole service as one coarse shard: one RSM-elected writer, the rest followers/standbys, per-connection pinning live. First real multi-writer-capable subsystem. Prove failover: kill the writer, watch the RSM <code>Assign</code> a follower at epoch+1, watch the old writer get fenced by the S3 CAS.</p>
|
|
<h3>Stage 5 — Infisical cutover <code>[BR]</code></h3>
|
|
<p>Retire the external Infisical secret store into KMS-on-this-runtime. Import and <strong>verify</strong> every secret (count + checksum). Stand up an <code>/api</code> compat shim for legacy Infisical clients — <strong>this shim is the cross-tenant-leak surface and the primary red-team target of the whole migration</strong> (a shim that resolves the wrong org's secret is a breach). Merge the <code>kms</code> Service alias into the <code>cloud</code> alias. Verify the three <code>DEPRECATED.md</code> gates and all 30 machine identities still authenticate. Only then kill Infisical.</p>
|
|
<h3>Stage 6 — roll coarse-writer to <code>tasks</code>, <code>pubsub</code>, the 24 SQLite services <code>[BR]</code></h3>
|
|
<p>Bring the remaining stateful subsystems under coarse placement. The <strong>audit subsystem is specifically <code>[BR]</code></strong>: it keeps an in-memory sequence counter / head pointer that must move to the replicated store <em>before</em> it can have a second replica — otherwise two writers mint colliding sequence numbers. Do that move first, review it, then raise its replica count.</p>
|
|
<h3>Stage 7 — IAM <code>N > 1</code> <code>[BR]</code></h3>
|
|
<p><strong>Last, because it is the highest blast radius.</strong> Move sessions to the replicated KV shard, hold a single JWKS in KMS (one signing key, RSM-placed writer), and <strong>lift the <code>Replicas > 1</code> gate</strong> that IAM carries today. Auth is on every request path, so this ships last and is reviewed hardest.</p>
|
|
<h3>Stage 8 — fine <code>(plugin, org)</code> sharding <code>[BR]</code> (future, opt-in)</h3>
|
|
<p>Split coarse plugin-shards into per-tenant <code>(plugin, org)</code> shards on the HIP-0302 per-org layout. Opt-in per plugin. A hot tenant gets a dedicated writer; cold tenants stay free. This is a placement-map refinement — no new mechanism, just a finer shard key — and it is explicitly future work.</p>
|
|
<h2>Decided forks</h2>
|
|
<p>Where the design chose one path, and why:</p>
|
|
<p>| Decision | Chosen | Rationale | |---|---|---| | Control-plane consensus | <strong>Quasar PQ-BFT</strong> (<code>protocol/quasar</code> <code>NewEngine</code>) | Placement decides <em>who may write</em>; a wrong/equivocating answer must be <strong>impossible</strong>, not improbable. <strong>Full N-node Byzantine agreement</strong> (decided): threshold shares split across voter pods, aggregated via Pulsar RoundSigner <code>Round1</code>/<code>Round2</code>/<code>Finalize</code> over ZAP, each voter verifying the aggregated PQ cert before applying a finalized block. <strong>Strict-PQ from day one</strong> (<code>ChainSecurityProfile</code> triple-mode gate) — orthogonal to and simultaneous with the Byzantine guarantee, <em>not</em> classical-now / PQ-later. | | Quorum shape | <strong>7 voters + N data nodes</strong> | <code>N ≥ 3f+1</code> for <code>f=2</code>: seven is the smallest voter set with a Byzantine quorum (<code>2f+1 = 5</code>) tolerating 2 faults; consensus stays fast among a fixed 7 while data nodes scale unbounded. | | Shard granularity (v1) | <strong>Coarse, per-plugin</strong> | One global writer per plugin = <strong>zero storage change</strong> (existing data dir is the shard). Fine <code>(plugin,org)</code> is a later, opt-in refinement. | | KMS read path | <strong>Writer serves reads</strong>; followers are HA standbys | Simplest correct default; reads are already fast at the writer. Follower-serves-stale-reads is available per-connection for read-heavy plugins. | | Replication mode (default) | <strong>Async S3 log-ship</strong> (reuse <code>replication.go</code>) | Reuses HIP-0107 unchanged; RPO is small and already characterized. Sync (RPO = 0) is a <strong>per-shard opt-in</strong> for plugins that need it, not the default tax. | | Default counts | <strong>7 voters / 3 data replicas</strong> | 3 data replicas = one writer + two standbys, tolerating one node loss with a warm successor. |</p>
|
|
<h2>Failure modes</h2>
|
|
<p>| Failure | Behavior | Why it is safe | |---|---|---| | <strong>Control-plane quorum loss</strong> (≥3 voters down) | Placement <strong>frozen</strong>: no new failovers, no rebalancing. <strong>Serving continues</strong> — every current writer keeps its lease and keeps writing. | Consensus is off the data path. The placement map is last-known-good and every data node already has it cached. Agility degrades; availability does not. | | <strong>Split-brain write attempt</strong> (a partitioned old writer thinks it still owns a shard) | The stale writer is fenced. | Two independent fences: (1) the RSM never grants two valid leases for one shard/epoch (linearizable); (2) the S3 epoch-CAS rejects any push whose epoch is behind the mirror's <code>latest</code> — the deposed writer physically cannot append. | | <strong>Writer node loss</strong> | RSM commits <code>Assign(shard, follower, epoch+1)</code>; a warm follower is promoted; pinning re-routes new connections to it. | Followers are HA standbys already replicating the shard; promotion is a placement-map update, not a data copy. | | <strong>Rolling image upgrade</strong> | No membership change, no epoch reconfig, no placement churn. | Stable <code>NodeID</code>s make the roll invisible to the RSM; <code>maxUnavailable:1</code> keeps 6/7 voters live — above the 5-of-7 quorum — throughout. | | <strong>Data migration correctness</strong> (coarse→fine, or Infisical import) | Verified by <strong>count + checksum</strong> before the old source is retired. | No cutover trusts the copy; every stage proves equivalence first, then flips. | | <strong><code>/api</code> compat-shim mis-scoping</strong> (stage 5) | Explicitly called out as <strong>the</strong> cross-tenant-leak surface. | The shim translates legacy paths to org-scoped KMS lookups; a scoping bug leaks another tenant's secret. It is the primary red-team target and ships behind the most review. |</p>
|
|
<h2>Non-goals</h2>
|
|
<ul><li><strong>Replacing HIP-0107's byte pipeline.</strong> The data plane is reused</li></ul>
|
|
<p> unchanged. This HIP adds the <em>decision</em> layer above it, nothing below.</p>
|
|
<ul><li><strong>Cross-deployment / cross-region consensus.</strong> The RSM coordinates one</li></ul>
|
|
<p> deployment's fleet. Geo-failover and multi-region topology are an operator concern over the <code>hanzo/vfs</code> backend (HIP-0107 non-goal), not this RSM.</p>
|
|
<ul><li><strong>Making consensus a dependency of serving.</strong> Explicitly rejected — the</li></ul>
|
|
<p> whole point is that serving survives quorum loss. Any design that puts the RSM on the request path violates this HIP.</p>
|
|
<ul><li><strong>A general scheduler.</strong> This is placement of <em>plugin write ownership</em>,</li></ul>
|
|
<p> not a Kubernetes replacement. Node-pool <code>--enable</code> sets (HIP-0106) and the operator (HIP-0400) still own coarse scheduling.</p>
|
|
<ul><li><strong>Folding <code>vault</code> / <code>payments</code>.</strong> Per HIP-0106's solo-vault CDE rule, the</li></ul>
|
|
<p> PCI systems stay their own deployments and are not placed by this runtime.</p>
|
|
<h2>Open questions</h2>
|
|
<ol><li><strong>Voter/data co-residence threshold.</strong> At what fleet size do voters and</li></ol>
|
|
<p> data nodes separate onto distinct pools? Small deployments co-locate; the crossover point wants measurement.</p>
|
|
<ol><li><strong>Placement-map watch fan-out.</strong> N data nodes each stream committed</li></ol>
|
|
<p> placement updates from the RSM. Confirm the fan-out is a broadcast from voters (cheap) and not N point-to-point subscriptions at scale.</p>
|
|
<ol><li><strong>Epoch-CAS backend contract.</strong> The S3 fence needs a CAS on the <code>latest</code></li></ol>
|
|
<p> pointer. Confirm the <code>hanzo/vfs</code> backend exposes a conditional-write / if-match primitive on every sink (S3 If-Match, GCS generation-match), or define the fallback for backends that do not.</p>
|
|
<ol><li><strong>Rebalancing policy.</strong> The RSM <em>can</em> move a shard's writer for load;</li></ol>
|
|
<p> <em>when</em> it should (hysteresis, cost of a writer move = a follower promotion + connection re-pin) is a policy this HIP leaves to a follow-up.</p>
|
|
<ol><li><strong>Sync replication opt-in shape.</strong> Per-shard RPO-0 (sync log-ship) is a</li></ol>
|
|
<p> decided future option; the exact per-plugin knob (a field on <code>cloud.Register</code>? a shard annotation?) is unspecified here.</p>
|
|
<h2>References</h2>
|
|
<ul><li>HIP-0026 — Identity and Access Management (org from JWT <code>owner</code> claim;</li></ul>
|
|
<p> the pinning key's tenant component)</p>
|
|
<ul><li>HIP-0027 — Secrets Management (KMS; the first subsystem placed, and the</li></ul>
|
|
<p> holder of the single JWKS in stage 7)</p>
|
|
<ul><li>HIP-0105 — In-Process Extension Runtime (<code>cloud.Register</code>, the plugin</li></ul>
|
|
<p> unit this HIP places)</p>
|
|
<ul><li>HIP-0106 — Cloud, Unified Hanzo Binary (the one artifact and <code>--enable</code></li></ul>
|
|
<p> subsystem model this HIP makes stateful and HA)</p>
|
|
<ul><li>HIP-0107 — Streaming Replication over VFS (the data plane, reused</li></ul>
|
|
<p> unchanged)</p>
|
|
<ul><li>HIP-0400 — Service CRD (the operator <code>Service</code> CR that gains a</li></ul>
|
|
<p> StatefulSet-emitting mode)</p>
|
|
<ul><li>HIP-0114 — ZAP Inter-VM Cognitive Transport (the ZAP-native default wire</li></ul>
|
|
<p> for plugins)</p>
|
|
<ul><li>HIP-0302 — Hanzo Replicate: Encrypted SQLite + ZapDB Durability (the</li></ul>
|
|
<p> per-org <code>{DataDir}/orgs/{org}/{svc}.db</code> layout fine sharding uses)</p>
|
|
<ul><li>Reference seams (grounded): <code>hanzoai/kms/replication.go</code> (<code>canPush</code>, line</li></ul>
|
|
<p> 452), <code>hanzoai/kms/lease.go</code> (interim <code>writerLease</code>), <code>hanzoai/kms/embed.go</code> (ZAP server), <code>hanzoai/cloud/build.go</code> (<code>Register</code>, line 351), <code>hanzoai/cloud/serve.go</code> (<code>IdentityMiddleware</code>, line 97), <code>github.com/luxfi/consensus/protocol/quasar</code> (<code>NewEngine</code>/<code>Submit</code>/<code>Finalized</code>/<code>IsFinalized</code>/<code>SetProfile</code>, <code>engine.go:59</code>; <code>Certifier</code>/<code>demandsTriple</code> strict-PQ gate; <code>pulsar/round_signer.go</code> <code>PulsarRoundSigner</code> — <code>Round1</code>/<code>Round2</code>/<code>Finalize</code> — importing <code>github.com/luxfi/pulsar/ref/go/pkg/pulsar</code> RoundSigner), <code>luxfi/validators</code> Manager</p>
|
|
<ul><li>Quasar + Pulsar (post-quantum BFT ordering + threshold certification): the</li></ul>
|
|
<p> control plane's consensus engine (<code>github.com/luxfi/consensus/protocol/quasar</code>) and its strict-PQ threshold signer (<code>github.com/luxfi/pulsar</code>, Pulsar threshold ML-DSA + Corona Ring-LWE threshold). <strong>Dependency prerequisite (unmet):</strong> the RoundSigner primitive is published (<code>luxfi/pulsar v1.1.5</code>), but the consensus cert-profile package <code>protocol/quasar/pulsar</code> is unpublished (0/268 tags; dev branch <code>feat/quasar-pulsar-profile</code> @<code>009ad013</code>, behind a <code>replace</code> Go ignores downstream). Needs one <code>luxfi/consensus</code> release tag carrying <code>protocol/quasar/pulsar</code>, requiring published pulsar v1.1.5, <code>replace</code> removed — forward-port in progress</p>
|
|
<ul><li>Prior art: M. Burrows, *The Chubby Lock Service for Loosely-Coupled</li></ul>
|
|
<p> Distributed Systems* (OSDI 2006); <em>etcd</em> Raft coordination kernel</p>
|
|
<h2>Copyright</h2>
|
|
<p>Copyright and related rights waived via <a href="https://creativecommons.org/publicdomain/zero/1.0/">CC0</a>.</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/hip-0125.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |