mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
410 lines
35 KiB
HTML
410 lines
35 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>hip-0065</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/hip-0065.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/hip-0065">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/hip-0065.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>hip-0065</h1><p class="lede">HIP-65: Backup & Disaster Recovery Standard. Status Final. Hanzo architectural specification.</p><h1>HIP-0065: Backup & Disaster Recovery Standard</h1>
|
|
<h2>Abstract</h2>
|
|
<p>This proposal defines the unified backup and disaster recovery (DR) standard for all stateful services in the Hanzo ecosystem. Every data store -- SQL (HIP-1104), Valkey/KV (HIP-1164), Hanzo Datastore (HIP-1241), MinIO/S3 (HIP-0405), model artifacts, training checkpoints, datasets, and configuration secrets -- MUST be backed up, verified, and recoverable through the single Hanzo Backup service defined here.</p>
|
|
<p><strong>Repository</strong>: none yet -- the backup controller described below is proposed, not built. <strong>Image</strong>: none yet. The <code>ghcr.io/hanzoai/backup</code> name is already taken by an unrelated service, so this controller needs a different one when it ships. <strong>Port</strong>: 8065 (backup controller API) <strong>License</strong>: Apache-2.0</p>
|
|
<h2>Motivation</h2>
|
|
<p>Hanzo operates 15+ stateful services across two Kubernetes clusters (hanzo-k8s, lux-k8s). Each service adopted its own backup approach:</p>
|
|
<ul><li><strong>PostgreSQL</strong> runs a CronJob with <code>pg_dump</code> every 6 hours (HIP-1104).</li><li><strong>KV</strong> relies on RDB snapshots that only persist to the local PVC.</li><li><strong>Hanzo Datastore</strong> has no automated backup; operators run manual <code>BACKUP</code> commands.</li><li><strong>MinIO</strong> replicates buckets between clusters but has no off-site copy.</li><li><strong>Model artifacts</strong> are stored in S3 buckets with no versioning policy.</li><li><strong>KMS secrets</strong> are backed up only through the KMS built-in export.</li></ul>
|
|
<p>This patchwork creates five problems:</p>
|
|
<ol><li><strong>No unified Recovery Point Objective (RPO).</strong> Some services can lose 6 hours</li></ol>
|
|
<p> of data (SQL) while others can lose days (Hanzo Datastore). There is no organizational agreement on acceptable data loss per service tier.</p>
|
|
<ol><li><strong>No tested Recovery Time Objective (RTO).</strong> Nobody has timed a full restore</li></ol>
|
|
<p> of any service from backup. We have backups but no proof they work. Untested backups are not backups.</p>
|
|
<ol><li><strong>No cross-region copies.</strong> All backups live in the same DigitalOcean region</li></ol>
|
|
<p> as the production clusters. A regional outage (datacenter fire, network partition) destroys both production data and backups simultaneously.</p>
|
|
<ol><li><strong>No AI-specific DR.</strong> Model weights, training checkpoints, and datasets are</li></ol>
|
|
<p> the most valuable and most expensive-to-reproduce assets in the organization. Recreating a fine-tuned model from scratch costs thousands of dollars in GPU time. Yet these artifacts have no formal backup or versioning strategy.</p>
|
|
<ol><li><strong>No encryption consistency.</strong> Some backups are encrypted; some are not. There</li></ol>
|
|
<p> is no standard for which KMS key encrypts what, or how to rotate backup encryption keys.</p>
|
|
<p>We need ONE backup and DR system that covers every data store with explicit RPO/RTO targets, automated verification, cross-region replication, and AI-aware artifact preservation.</p>
|
|
<h2>Design Philosophy</h2>
|
|
<p>This section explains the reasoning behind each major architectural decision. Every heading addresses a single decision and why the alternatives were rejected.</p>
|
|
<h3>Why Unified Backup Over Per-Service Scripts</h3>
|
|
<p>The status quo is per-service backup scripts: a CronJob for SQL, a ConfigMap-driven script for KV, nothing for Hanzo Datastore. This approach has three fundamental problems:</p>
|
|
<ol><li><strong>Inconsistent scheduling.</strong> Each team picks its own cron schedule. There is</li></ol>
|
|
<p> no way to answer "what is the most recent consistent snapshot of the entire system?" because backups are taken at different times.</p>
|
|
<ol><li><strong>No single recovery plan.</strong> Disaster recovery requires restoring multiple</li></ol>
|
|
<p> services in the correct order (KMS first, then SQL, then application services). Per-service scripts have no concept of orchestrated recovery.</p>
|
|
<ol><li><strong>Duplicated infrastructure.</strong> Every script independently implements S3</li></ol>
|
|
<p> upload, retention pruning, encryption, and alerting. This code is duplicated across 5+ CronJobs and tested nowhere.</p>
|
|
<p>A unified backup controller eliminates all three problems. It schedules backups across all stores, maintains a dependency graph for ordered recovery, and provides a single codebase for upload, encryption, verification, and alerting.</p>
|
|
<p>The trade-off is coupling: a bug in the backup controller affects all stores. We accept this because backup infrastructure is inherently cross-cutting. A single well-tested controller is more reliable than five untested scripts.</p>
|
|
<h2>Specification</h2>
|
|
<h3>Service Tiers and RPO/RTO Targets</h3>
|
|
<p>Every Hanzo service is assigned one of three tiers:</p>
|
|
<p>| Tier | RPO | RTO | Backup Frequency | Replication | Examples | |------|-----|-----|------------------|-------------|----------| | <strong>Critical</strong> | 1 minute | 5 minutes | Continuous (WAL/AOF streaming) | Synchronous cross-region | SQL (IAM, Cloud), KMS secrets | | <strong>Standard</strong> | 1 hour | 1 hour | Hourly snapshots | Async cross-region | Valkey/KV, Hanzo Datastore, MinIO buckets | | <strong>Archival</strong> | 24 hours | 4 hours | Daily snapshots | Async, single copy | Model artifacts, training datasets, logs |</p>
|
|
<p>RPO = Recovery Point Objective (maximum acceptable data loss). RTO = Recovery Time Objective (maximum acceptable downtime during recovery).</p>
|
|
<h3>Backup Targets</h3>
|
|
<p>The backup controller manages the following data stores:</p>
|
|
<pre><code>Backup Controller (:8065)
|
|
│
|
|
├── PostgreSQL (HIP-1104) ── pg_basebackup + WAL archiving
|
|
├── Valkey/KV (HIP-1164) ── RDB snapshot export
|
|
├── Hanzo Datastore (HIP-1241) ── BACKUP DATABASE ... TO S3
|
|
├── MinIO/S3 (HIP-0405) ── mc mirror (bucket replication)
|
|
├── Model Weights / Checkpoints / Datasets ── versioned S3
|
|
└── Config / Secrets ── Velero + KMS export
|
|
│
|
|
Encryption (KMS HIP-0027)
|
|
│
|
|
S3 Primary Region ──async──→ S3 Secondary Region</code></pre>
|
|
<h3>Per-Store Backup Methods</h3>
|
|
<h4>SQL (Critical Tier)</h4>
|
|
<p>Two complementary backup mechanisms run simultaneously:</p>
|
|
<ol><li><strong>Continuous WAL archiving.</strong> WAL segments are shipped to the backup S3</li></ol>
|
|
<p> bucket as they are produced. This provides point-in-time recovery (PITR) to any second within the WAL retention window (default: 7 days). Configuration:</p>
|
|
<p> ``<code>ini # postgresql.conf additions for PITR archive_mode = on archive_command = 'backup-wal-push %p --endpoint s3://hanzo-backups/wal/%f' archive_timeout = 60 </code>``</p>
|
|
<ol><li><strong>Periodic base backups.</strong> A full <code>pg_basebackup</code> runs every 24 hours. This</li></ol>
|
|
<p> establishes a restore baseline. PITR replays WAL on top of the most recent base backup.</p>
|
|
<p> To restore to a specific point in time:</p>
|
|
<p> ``<code>bash # Restore base backup backup-pg-restore --base-backup 20260223_000000 \ --target-time "2026-02-23 14:30:00 UTC" \ --endpoint s3://hanzo-backups </code>``</p>
|
|
<p>The existing <code>pg_dump</code> CronJob (HIP-1104) continues as a logical backup for selective per-database restore. It supplements but does not replace PITR.</p>
|
|
<h4>Valkey/KV (Standard Tier)</h4>
|
|
<p>KV supports two persistence formats:</p>
|
|
<ul><li><strong>RDB snapshots</strong>: Point-in-time binary dumps. Compact and fast to restore.</li><li><strong>AOF (Append Only File)</strong>: Write-ahead log of every command. Higher fidelity</li></ul>
|
|
<p> but larger and slower to replay.</p>
|
|
<p>The backup controller exports an RDB snapshot every hour and uploads it to S3. For critical deployments that require sub-hour RPO, AOF streaming to S3 can be enabled per-instance.</p>
|
|
<pre><code class="lang-bash"># Trigger RDB snapshot and upload
|
|
backup-kv-snapshot --host localhost:6379 \
|
|
--output s3://hanzo-backups/kv/$(date +%Y%m%d_%H%M%S).rdb</code></pre>
|
|
<h4>Hanzo Datastore (Standard Tier)</h4>
|
|
<p>Hanzo Datastore provides native <code>BACKUP TABLE ... TO S3(...)</code> syntax. The backup controller issues backup commands for each database on an hourly schedule.</p>
|
|
<pre><code class="lang-sql">BACKUP DATABASE insights TO S3(
|
|
'https://s3.hanzo-backups.svc/datastore/insights/20260223_140000',
|
|
'backup-access-key',
|
|
'backup-secret-key'
|
|
) SETTINGS compression_method = 'zstd';</code></pre>
|
|
<p>Incremental backups are supported via <code>BACKUP ... SETTINGS base_backup</code>. Each hourly backup is incremental against the most recent daily full backup.</p>
|
|
<h4>MinIO/S3 Object Storage (Standard Tier)</h4>
|
|
<p>MinIO buckets are replicated using <code>mc mirror</code> to a secondary S3 endpoint. This provides both backup and geographic redundancy.</p>
|
|
<pre><code class="lang-bash"># Mirror production bucket to backup region
|
|
mc mirror --watch --overwrite \
|
|
prod/hanzo-storage s3backup/hanzo-storage</code></pre>
|
|
<p>For versioned buckets (model weights, datasets), MinIO's built-in versioning preserves every object revision. The backup controller verifies that versioning is enabled on all critical buckets.</p>
|
|
<h4>Model Artifacts and Training Checkpoints (Archival Tier)</h4>
|
|
<p>AI artifacts are the most expensive data to reproduce. A single fine-tuning run can cost $500-10,000 in GPU compute. The backup strategy must preserve:</p>
|
|
<ol><li><strong>Model weights</strong>: Final trained parameters. Stored in MinIO with versioning.</li></ol>
|
|
<p> Each model version is tagged with its training run ID, dataset hash, and hyperparameter fingerprint.</p>
|
|
<ol><li><strong>Training checkpoints</strong>: Intermediate snapshots taken every N steps during</li></ol>
|
|
<p> training. These allow resuming a failed training run without starting over. Retained for 30 days after training completion, then pruned.</p>
|
|
<ol><li><strong>Datasets</strong>: Training and evaluation data. Versioned using content-addressed</li></ol>
|
|
<p> hashing (SHA-256 of the dataset manifest). Immutable once published.</p>
|
|
<p>All artifacts are stored in dedicated MinIO buckets with lifecycle rules:</p>
|
|
<p>| Artifact Type | Bucket | Versioning | Retention | |---------------|--------|------------|-----------| | Model weights (released) | <code>models-release</code> | Enabled | Permanent | | Model weights (experimental) | <code>models-dev</code> | Enabled | 90 days | | Training checkpoints | <code>training-checkpoints</code> | Disabled | 30 days post-run | | Datasets (published) | <code>dataset</code> | Enabled (content-addressed) | Permanent | | Datasets (staging) | <code>datasets-staging</code> | Disabled | 14 days |</p>
|
|
<h4>Configuration and Secrets (Critical Tier)</h4>
|
|
<p>Two mechanisms protect cluster configuration:</p>
|
|
<ol><li><strong>Velero</strong>: Backs up all Kubernetes API objects (Deployments, StatefulSets,</li></ol>
|
|
<p> ConfigMaps, CRDs, Services) to S3 every hour. Velero also coordinates PVC snapshot creation for volume-level backup.</p>
|
|
<p> ``<code>bash velero schedule create hanzo-cluster-backup \ --schedule="0 <em> </em> <em> </em>" \ --include-namespaces hanzo \ --storage-location default \ --ttl 720h </code>``</p>
|
|
<ol><li><strong>KMS export</strong>: KMS secrets are exported nightly as an encrypted JSON bundle.</li></ol>
|
|
<p> This bundle is encrypted with a separate backup-specific KMS key that is itself stored in a hardware security module (HSM) or offline cold storage.</p>
|
|
<h3>Cross-Region Replication</h3>
|
|
<p>Every backup is replicated to a secondary geographic region. The primary and secondary regions MUST be in different data centers with independent failure domains.</p>
|
|
<pre><code>Primary (NYC1/SFO3) Secondary (AMS3/SGP1)
|
|
s3://hanzo-backups ──async──→ s3://hanzo-backups-secondary
|
|
WAL, RDB, CH, Velero Full replica of all backup data</code></pre>
|
|
<p>Replication lag for async cross-region copy MUST stay below 15 minutes under normal operation. The backup controller monitors replication lag and alerts when it exceeds the threshold.</p>
|
|
<h3>Point-in-Time Recovery (PITR)</h3>
|
|
<p>PITR is available for SQL via WAL archiving. The recovery window is configurable per cluster:</p>
|
|
<p>| Cluster | PITR Window | WAL Retention | |---------|-------------|---------------| | hanzo-k8s | 7 days | 7 days of WAL segments | | lux-k8s | 7 days | 7 days of WAL segments |</p>
|
|
<p>To perform PITR:</p>
|
|
<pre><code class="lang-bash"># 1. Stop the target SQL instance
|
|
kubectl scale statefulset postgres --replicas=0
|
|
|
|
# 2. Restore base backup + replay WAL to target time
|
|
backup-pg-restore \
|
|
--cluster hanzo-k8s \
|
|
--target-time "2026-02-23 14:30:00 UTC" \
|
|
--output /var/lib/postgresql/data
|
|
|
|
# 3. Start SQL (it will replay WAL to the target time)
|
|
kubectl scale statefulset postgres --replicas=1</code></pre>
|
|
<p>For KV and Hanzo Datastore, PITR is not natively supported. Recovery is to the most recent snapshot. If sub-hour granularity is needed for KV, enable AOF streaming.</p>
|
|
<h3>Backup Encryption</h3>
|
|
<p>All backups MUST be encrypted at rest using AES-256-GCM. Encryption keys are managed by KMS (HIP-0027).</p>
|
|
<pre><code>Backup data → AES-256-GCM encryption → Encrypted blob → S3 upload
|
|
↑
|
|
Data Encryption Key (DEK)
|
|
↑
|
|
Key Encryption Key (KEK) from KMS</code></pre>
|
|
<p><strong>Key hierarchy</strong>:</p>
|
|
<ol><li><strong>KEK (Key Encryption Key)</strong>: Stored in KMS under path <code>/backup/kek</code>.</li></ol>
|
|
<p> Rotated every 90 days. Old KEKs are retained (but marked inactive) for decrypting historical backups.</p>
|
|
<ol><li><strong>DEK (Data Encryption Key)</strong>: Generated per backup operation. Encrypted</li></ol>
|
|
<p> with the current KEK and stored alongside the backup metadata.</p>
|
|
<ol><li><strong>Emergency recovery key</strong>: A copy of the KEK is stored offline (printed</li></ol>
|
|
<p> QR code in a physical safe) for scenarios where KMS itself is unavailable.</p>
|
|
<h3>Automated Backup Verification</h3>
|
|
<p>Backups that are never tested are not backups. The backup controller runs automated restore tests on every backup:</p>
|
|
<ol><li><strong>Integrity check</strong>: After upload, download the backup and verify SHA-256</li></ol>
|
|
<p> checksum matches. This catches S3 corruption and upload errors.</p>
|
|
<ol><li><strong>Restore test</strong>: Once per day, the backup controller spins up an ephemeral</li></ol>
|
|
<p> environment (a temporary Pod with no production access) and restores the most recent backup of each Critical-tier service. If the restore succeeds and basic health checks pass, the test passes.</p>
|
|
<ol><li><strong>Data consistency check</strong>: For PostgreSQL, run <code>pg_restore --list</code> to</li></ol>
|
|
<p> verify the dump TOC is valid. For Hanzo Datastore, run <code>CHECK TABLE</code> on restored tables. For KV, load the RDB and run <code>DBSIZE</code> to verify non-zero key count.</p>
|
|
<p>Verification runs as a CronJob at 04:00 UTC daily (<code>backup-verify --all-critical</code>). Failures trigger PagerDuty alerts at the same severity as a production outage.</p>
|
|
<h3>Retention Policy</h3>
|
|
<p>| Backup Type | Retention | Pruning | |-------------|-----------|---------| | WAL segments | 7 days | Automatic after base backup + WAL coverage | | SQL base backups | 30 days | Oldest pruned when count exceeds 30 | | KV RDB snapshots | 30 days | Oldest pruned when count exceeds 720 (hourly) | | Hanzo Datastore backups | 90 days | Oldest pruned when count exceeds 2160 | | MinIO bucket mirrors | Current + 1 previous | Continuous mirror, version history in bucket | | Velero cluster backups | 30 days | TTL-based (720h) | | KMS secret exports | 90 days | Oldest pruned on schedule | | Model weights (released) | Permanent | Never pruned | | Training checkpoints | 30 days post-run | Automatic after run completion + 30d |</p>
|
|
<h2>Implementation</h2>
|
|
<h3>Backup Controller</h3>
|
|
<p>The backup controller is a Go binary deployed as a single-replica Kubernetes Deployment in the <code>hanzo</code> namespace. It authenticates to all data stores using credentials from KMS and uploads encrypted backups to S3.</p>
|
|
<pre><code class="lang-yaml"># Key environment variables (all sourced from KMS secrets)
|
|
BACKUP_S3_ENDPOINT: # Primary S3 backup endpoint
|
|
BACKUP_S3_ACCESS_KEY: # S3 credentials
|
|
BACKUP_S3_SECRET_KEY: # S3 credentials
|
|
BACKUP_KMS_ENDPOINT: https://kms.hanzo.ai/api
|
|
BACKUP_SECONDARY_REGION: # Secondary S3 endpoint for cross-region copy</code></pre>
|
|
<p>Resource requests: 256Mi memory, 200m CPU. Limits: 512Mi memory, 1 CPU. The controller Service exposes port 8065 within the cluster.</p>
|
|
<h3>API Endpoints</h3>
|
|
<p>| Method | Path | Description | |--------|------|-------------| | <code>GET</code> | <code>/v1/backup/status</code> | Backup system health and last backup times | | <code>GET</code> | <code>/v1/backup/backups</code> | List all backups with metadata | | <code>POST</code> | <code>/v1/backup/backups</code> | Trigger an ad-hoc backup for a specific store | | <code>POST</code> | <code>/v1/backup/restore</code> | Initiate a restore operation | | <code>GET</code> | <code>/v1/backup/verify</code> | Last verification results | | <code>POST</code> | <code>/v1/backup/verify</code> | Trigger an ad-hoc verification | | <code>GET</code> | <code>/v1/backup/metrics</code> | Prometheus-compatible metrics |</p>
|
|
<h3>Disaster Recovery Runbooks</h3>
|
|
<h4>Runbook 1: Single-Service Database Corruption</h4>
|
|
<p>Scenario: A bad migration corrupts the <code>iam</code> database. RTO: 5 minutes.</p>
|
|
<ol><li>Identify corruption timestamp from application logs.</li><li><code>POST /v1/backup/restore</code> with <code>store=sql</code>,</li></ol>
|
|
<p> <code>target_time=<pre-corruption></code>, <code>method=pitr</code>.</p>
|
|
<ol><li>Controller stops IAM pods, restores base backup, replays WAL to target time.</li><li>Controller restarts IAM and runs health check. Verify login flow manually.</li></ol>
|
|
<h4>Runbook 2: Full Cluster Loss</h4>
|
|
<p>Scenario: hanzo-k8s is destroyed (provider outage). RTO: 1 hour.</p>
|
|
<ol><li>Provision new Kubernetes cluster in secondary region.</li><li><code>velero restore create --from-backup hanzo-cluster-backup-latest</code></li><li><code>backup-pg-restore --cluster hanzo-k8s --latest</code></li><li><code>backup-kv-restore --cluster hanzo-k8s --latest</code></li><li><code>backup-ch-restore --cluster hanzo-k8s --latest</code></li><li>Update DNS (hanzo.id, cloud.hanzo.ai, etc.) to new cluster IP.</li><li>Verify all services via <code>/healthz</code> endpoints.</li></ol>
|
|
<h4>Runbook 3: Model Artifact Recovery</h4>
|
|
<p>Scenario: Production model accidentally deleted. RTO: 15 minutes.</p>
|
|
<ol><li>Identify model version from inference error logs.</li><li><code>mc cp --version-id <ver> backup/models-release/<model> prod/models-release/</code></li><li>Restart inference pods. Verify via test prompt.</li></ol>
|
|
<h3>Monitoring and Alerting</h3>
|
|
<p>The backup controller exposes Prometheus metrics:</p>
|
|
<p>| Metric | Type | Description | |--------|------|-------------| | <code>backup_last_success_timestamp</code> | Gauge | Unix timestamp of last successful backup per store | | <code>backup_last_duration_seconds</code> | Gauge | Duration of last backup per store | | <code>backup_size_bytes</code> | Gauge | Size of last backup per store | | <code>backup_verification_success</code> | Gauge | 1 if last verification passed, 0 if failed | | <code>backup_replication_lag_seconds</code> | Gauge | Cross-region replication lag | | <code>backup_operations_total</code> | Counter | Total backup operations by store and status |</p>
|
|
<p><strong>Alert rules</strong>:</p>
|
|
<p>| Alert | Condition | Severity | |-------|-----------|----------| | BackupMissed | No successful backup in 2x the scheduled interval | Critical | | BackupVerificationFailed | <code>backup_verification_success == 0</code> | Critical | | ReplicationLagHigh | <code>backup_replication_lag_seconds > 900</code> | Warning | | BackupSizeAnomaly | Size differs > 50% from 7-day average | Warning |</p>
|
|
<h2>Security</h2>
|
|
<h3>Encryption at Rest</h3>
|
|
<p>All backup data MUST be encrypted before leaving the backup controller. The controller fetches the current KEK from KMS (HIP-0027), generates a per-backup DEK, encrypts the backup payload with AES-256-GCM, wraps the DEK with the KEK, and stores both the encrypted payload and wrapped DEK in S3.</p>
|
|
<h3>Network Isolation</h3>
|
|
<p>A NetworkPolicy restricts the backup controller's egress to only the required ports within the <code>hanzo</code> namespace (5432 SQL, 6379 KV, 8123 Hanzo Datastore, 9000 MinIO) and port 443 for external HTTPS (KMS API, secondary S3 endpoint). All other egress is denied.</p>
|
|
<h3>Access Control</h3>
|
|
<p>Backup and restore operations require the <code>backup-admin</code> KMS role. The backup controller authenticates via Universal Auth. Human operators MUST authenticate via KMS SSO and have explicit <code>backup-admin</code> membership to trigger manual restores. All backup and restore operations are logged to the audit trail in KMS.</p>
|
|
<h3>Backup Isolation</h3>
|
|
<p>Backup S3 buckets use separate credentials from production S3 buckets. A compromised production MinIO key cannot read or delete backups. Backup buckets have Object Lock enabled (WORM -- write once read many) for Critical-tier backups to prevent ransomware-style deletion.</p>
|
|
<h2>Future Work</h2>
|
|
<h3>Phase 4: Multi-Cloud DR</h3>
|
|
<p>Extend cross-region replication to cross-cloud. Secondary backups stored on a different cloud provider (AWS S3, GCS) ensure recovery even if the primary cloud provider experiences a global outage.</p>
|
|
<h2>References</h2>
|
|
<ol><li><a href="./hip-0027-secrets-management-standard.md">HIP-0027: Secrets Management Standard</a></li><li><a href="./hip-1164-provisioning-stores-on-demand.md">HIP-1164: Provisioning — Stores on Demand</a></li><li><a href="./hip-1104-base-hosted-backend.md">HIP-1104: Base — The Hosted Backend</a></li><li><a href="./hip-0405-s3-crd.md">HIP-0405: S3 CRD</a></li><li><a href="./hip-1241-metrics-one-store-three-signals.md">HIP-1241: Metrics — One Native Store, Three Signals</a></li><li><a href="https://velero.io/docs/">Velero Documentation</a></li><li><a href="https://www.postgresql.org/docs/16/continuous-archiving.html">PostgreSQL PITR</a></li><li><a href="https://clickhouse.com/docs/en/operations/backup">ClickHouse BACKUP</a></li><li><a href="https://min.io/docs/minio/linux/administration/bucket-replication.html">MinIO Replication</a></li></ol>
|
|
<h2>Copyright</h2>
|
|
<p>Copyright and related rights waived via <a href="https://creativecommons.org/publicdomain/zero/1.0/">CC0</a>.</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/hip-0065.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |