mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
713 lines
32 KiB
HTML
713 lines
32 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>distributed-systems-consensus-raft</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/distributed-systems-consensus-raft.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/distributed-systems-consensus-raft">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/distributed-systems-consensus-raft.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>distributed-systems-consensus-raft</h1><p class="lede">RAFT consensus algorithm including leader election, log replication, safety guarantees, and implementation patterns</p><h1>RAFT Consensus</h1>
|
|
<p><strong>Scope</strong>: RAFT algorithm, leader election, log replication, safety, implementation patterns <strong>Lines</strong>: ~380 <strong>Last Updated</strong>: 2025-10-27 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Building distributed consensus systems</li><li>Implementing replicated state machines</li><li>Designing highly available services</li><li>Understanding etcd, Consul internals</li><li>Implementing leader election</li><li>Building distributed databases</li><li>Ensuring strong consistency</li><li>Working with Kubernetes (uses etcd/RAFT)</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>RAFT Overview</h3>
|
|
<p><strong>Goal</strong>: Maintain replicated log across cluster with strong consistency</p>
|
|
<p><strong>Key properties</strong>:</p>
|
|
<ul><li>Leader-based (one leader at a time)</li><li>Strong consistency (linearizable)</li><li>Majority quorum (tolerates f failures with 2f+1 nodes)</li><li>Understandable (designed to be easier than Paxos)</li></ul>
|
|
<p><strong>Node states</strong>:</p>
|
|
<pre><code>Follower → Candidate → Leader
|
|
↑ ↓
|
|
└──────────────────────┘</code></pre>
|
|
<h3>Terms (Logical Clock)</h3>
|
|
<p><strong>Term</strong>: Monotonically increasing number representing election cycle</p>
|
|
<pre><code>Term 1: Term 2: Term 3:
|
|
Leader A ──X Election → Leader B ────→
|
|
|
|
Each term has at most one leader
|
|
Terms prevent conflicts from split-brain</code></pre>
|
|
<hr>
|
|
<h2>Leader Election</h2>
|
|
<h3>Election Process</h3>
|
|
<pre><code>1. Follower times out waiting for leader heartbeat
|
|
2. Increment term, become Candidate
|
|
3. Vote for self, request votes from all nodes
|
|
4. If majority votes received → become Leader
|
|
5. If another leader discovered → become Follower
|
|
6. If election timeout → start new election</code></pre>
|
|
<h3>Election Rules</h3>
|
|
<pre><code class="lang-python">class RaftNode:
|
|
def __init__(self):
|
|
self.state = "follower"
|
|
self.current_term = 0
|
|
self.voted_for = None
|
|
self.election_timeout = random.randint(150, 300) # ms
|
|
|
|
def on_election_timeout(self):
|
|
"""Start election"""
|
|
self.current_term += 1
|
|
self.state = "candidate"
|
|
self.voted_for = self.node_id
|
|
self.votes_received = {self.node_id}
|
|
|
|
# Request votes from all nodes
|
|
for node in self.cluster_nodes:
|
|
self.send_request_vote(node, self.current_term, self.node_id,
|
|
self.last_log_index, self.last_log_term)
|
|
|
|
def handle_request_vote(self, term, candidate_id, last_log_index, last_log_term):
|
|
"""Handle vote request"""
|
|
# Reject if term is old
|
|
if term < self.current_term:
|
|
return False
|
|
|
|
# Update term if newer
|
|
if term > self.current_term:
|
|
self.current_term = term
|
|
self.voted_for = None
|
|
self.state = "follower"
|
|
|
|
# Grant vote if:
|
|
# 1. Haven't voted this term
|
|
# 2. Candidate's log is at least as up-to-date
|
|
if self.voted_for is None or self.voted_for == candidate_id:
|
|
if self.is_log_up_to_date(last_log_index, last_log_term):
|
|
self.voted_for = candidate_id
|
|
return True
|
|
|
|
return False
|
|
|
|
def is_log_up_to_date(self, candidate_last_log_index, candidate_last_log_term):
|
|
"""Check if candidate's log is at least as up-to-date"""
|
|
my_last_log_term = self.log[-1].term if self.log else 0
|
|
my_last_log_index = len(self.log) - 1
|
|
|
|
# Candidate's last log term is more recent
|
|
if candidate_last_log_term > my_last_log_term:
|
|
return True
|
|
|
|
# Same term, candidate's log is at least as long
|
|
if candidate_last_log_term == my_last_log_term:
|
|
return candidate_last_log_index >= my_last_log_index
|
|
|
|
return False</code></pre>
|
|
<hr>
|
|
<h2>Log Replication</h2>
|
|
<h3>Replication Process</h3>
|
|
<pre><code>Client → Leader: Command
|
|
Leader:
|
|
1. Append to local log
|
|
2. Send AppendEntries to all followers
|
|
3. Wait for majority acknowledgment
|
|
4. Apply to state machine
|
|
5. Respond to client
|
|
6. Eventually notify followers to apply</code></pre>
|
|
<h3>AppendEntries RPC</h3>
|
|
<pre><code class="lang-python">class LogEntry:
|
|
def __init__(self, term, command):
|
|
self.term = term
|
|
self.command = command
|
|
|
|
class RaftNode:
|
|
def __init__(self):
|
|
self.log = [] # List of LogEntry
|
|
self.commit_index = 0 # Index of highest committed entry
|
|
self.last_applied = 0 # Index of highest applied entry
|
|
|
|
# Leader state (reinitialized after election)
|
|
self.next_index = {} # For each server, index of next log entry to send
|
|
self.match_index = {} # For each server, index of highest log entry known to be replicated
|
|
|
|
def append_entries(self, term, leader_id, prev_log_index, prev_log_term,
|
|
entries, leader_commit):
|
|
"""Handle AppendEntries RPC (heartbeat or log replication)"""
|
|
|
|
# Reply false if term < current_term
|
|
if term < self.current_term:
|
|
return False
|
|
|
|
# Update term and revert to follower if necessary
|
|
if term > self.current_term:
|
|
self.current_term = term
|
|
self.state = "follower"
|
|
self.voted_for = None
|
|
|
|
# Reset election timeout (received heartbeat from valid leader)
|
|
self.reset_election_timeout()
|
|
|
|
# Reply false if log doesn't contain entry at prev_log_index matching prev_log_term
|
|
if prev_log_index >= 0:
|
|
if prev_log_index >= len(self.log) or \
|
|
self.log[prev_log_index].term != prev_log_term:
|
|
return False
|
|
|
|
# Delete conflicting entries and append new ones
|
|
log_index = prev_log_index + 1
|
|
for i, entry in enumerate(entries):
|
|
if log_index + i < len(self.log):
|
|
if self.log[log_index + i].term != entry.term:
|
|
# Delete this and all following entries
|
|
self.log = self.log[:log_index + i]
|
|
self.log.append(entry)
|
|
else:
|
|
self.log.append(entry)
|
|
|
|
# Update commit index
|
|
if leader_commit > self.commit_index:
|
|
self.commit_index = min(leader_commit, len(self.log) - 1)
|
|
|
|
# Apply newly committed entries
|
|
self.apply_committed_entries()
|
|
|
|
return True
|
|
|
|
def apply_committed_entries(self):
|
|
"""Apply committed log entries to state machine"""
|
|
while self.last_applied < self.commit_index:
|
|
self.last_applied += 1
|
|
entry = self.log[self.last_applied]
|
|
self.state_machine.apply(entry.command)</code></pre>
|
|
<h3>Leader Logic</h3>
|
|
<pre><code class="lang-python">class RaftLeader(RaftNode):
|
|
def replicate_log(self):
|
|
"""Send AppendEntries to all followers"""
|
|
for follower_id in self.cluster_nodes:
|
|
if follower_id == self.node_id:
|
|
continue
|
|
|
|
# Get entries to send to this follower
|
|
next_index = self.next_index[follower_id]
|
|
prev_log_index = next_index - 1
|
|
prev_log_term = self.log[prev_log_index].term if prev_log_index >= 0 else 0
|
|
entries = self.log[next_index:]
|
|
|
|
success = self.send_append_entries(
|
|
follower_id,
|
|
self.current_term,
|
|
self.node_id,
|
|
prev_log_index,
|
|
prev_log_term,
|
|
entries,
|
|
self.commit_index
|
|
)
|
|
|
|
if success:
|
|
# Update next_index and match_index
|
|
self.next_index[follower_id] = next_index + len(entries)
|
|
self.match_index[follower_id] = self.next_index[follower_id] - 1
|
|
|
|
# Update commit_index if majority replicated
|
|
self.update_commit_index()
|
|
else:
|
|
# Decrement next_index and retry
|
|
self.next_index[follower_id] = max(0, self.next_index[follower_id] - 1)
|
|
|
|
def update_commit_index(self):
|
|
"""Advance commit_index if majority of servers have replicated entry"""
|
|
for n in range(self.commit_index + 1, len(self.log)):
|
|
# Only commit entries from current term
|
|
if self.log[n].term != self.current_term:
|
|
continue
|
|
|
|
# Count replicas
|
|
replicas = 1 # Leader has it
|
|
for follower_id in self.cluster_nodes:
|
|
if follower_id == self.node_id:
|
|
continue
|
|
if self.match_index[follower_id] >= n:
|
|
replicas += 1
|
|
|
|
# Commit if majority
|
|
if replicas > len(self.cluster_nodes) / 2:
|
|
self.commit_index = n
|
|
self.apply_committed_entries()</code></pre>
|
|
<hr>
|
|
<h2>Safety Guarantees</h2>
|
|
<h3>Election Safety</h3>
|
|
<p><strong>Guarantee</strong>: At most one leader per term</p>
|
|
<p><strong>How</strong>: Node votes for at most one candidate per term, majority quorum required</p>
|
|
<h3>Leader Append-Only</h3>
|
|
<p><strong>Guarantee</strong>: Leader never overwrites or deletes entries in its log</p>
|
|
<p><strong>How</strong>: Leader only appends new entries</p>
|
|
<h3>Log Matching</h3>
|
|
<p><strong>Guarantee</strong>: If two logs contain entry with same index and term, all preceding entries are identical</p>
|
|
<p><strong>How</strong>: AppendEntries consistency check (prev_log_index, prev_log_term)</p>
|
|
<h3>Leader Completeness</h3>
|
|
<p><strong>Guarantee</strong>: If entry is committed in term T, it will be present in all leader logs for terms > T</p>
|
|
<p><strong>How</strong>: Candidate with incomplete log won't get majority votes</p>
|
|
<h3>State Machine Safety</h3>
|
|
<p><strong>Guarantee</strong>: If server has applied log entry at index, no other server will apply different entry at same index</p>
|
|
<p><strong>How</strong>: Follows from other properties</p>
|
|
<hr>
|
|
<h2>Real-World Implementation (etcd)</h2>
|
|
<h3>Using etcd</h3>
|
|
<pre><code class="lang-bash"># Start etcd cluster (3 nodes)
|
|
etcd --name node1 --initial-cluster node1=http://10.0.1.1:2380,node2=http://10.0.1.2:2380,node3=http://10.0.1.3:2380</code></pre>
|
|
<p><strong>Python client</strong>:</p>
|
|
<pre><code class="lang-python">import etcd3
|
|
|
|
etcd = etcd3.client(host='localhost', port=2379)
|
|
|
|
# Write (goes through RAFT consensus)
|
|
etcd.put('/config/setting', 'value')
|
|
|
|
# Read (consistent read from leader)
|
|
value, metadata = etcd.get('/config/setting')
|
|
|
|
# Watch for changes
|
|
watch_id = etcd.add_watch_callback('/config/', lambda event: print(event))
|
|
|
|
# Lease (TTL)
|
|
lease = etcd.lease(ttl=60)
|
|
etcd.put('/ephemeral/key', 'value', lease=lease)
|
|
|
|
# Transactions (atomic operations)
|
|
etcd.transaction(
|
|
compare=[etcd.transactions.value('/counter') == b'5'],
|
|
success=[etcd.transactions.put('/counter', '6')],
|
|
failure=[etcd.transactions.get('/counter')]
|
|
)</code></pre>
|
|
<h3>Go client (native etcd client):</h3>
|
|
<pre><code class="lang-go">package main
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
clientv3 "go.etcd.io/etcd/client/v3"
|
|
)
|
|
|
|
func main() {
|
|
cli, _ := clientv3.New(clientv3.Config{
|
|
Endpoints: []string{"localhost:2379"},
|
|
DialTimeout: 5 * time.Second,
|
|
})
|
|
defer cli.Close()
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
|
|
// Put
|
|
cli.Put(ctx, "/config/setting", "value")
|
|
|
|
// Get
|
|
resp, _ := cli.Get(ctx, "/config/setting")
|
|
for _, kv := range resp.Kvs {
|
|
fmt.Printf("%s: %s\n", kv.Key, kv.Value)
|
|
}
|
|
|
|
// Watch
|
|
watchChan := cli.Watch(context.Background(), "/config/", clientv3.WithPrefix())
|
|
for watchResp := range watchChan {
|
|
for _, event := range watchResp.Events {
|
|
fmt.Printf("Event: %s %s\n", event.Type, event.Kv.Key)
|
|
}
|
|
}
|
|
}</code></pre>
|
|
<hr>
|
|
<h2>Performance Considerations</h2>
|
|
<h3>Cluster Size</h3>
|
|
<pre><code>3 nodes: Tolerates 1 failure (quorum = 2)
|
|
5 nodes: Tolerates 2 failures (quorum = 3)
|
|
7 nodes: Tolerates 3 failures (quorum = 4)
|
|
|
|
More nodes:
|
|
✅ Higher availability
|
|
❌ Slower writes (more nodes to replicate to)
|
|
❌ More network traffic
|
|
|
|
Recommendation: 3 or 5 nodes for most use cases</code></pre>
|
|
<h3>Latency</h3>
|
|
<pre><code>Write latency = Network RTT + Disk sync time
|
|
|
|
Typically:
|
|
- Same datacenter: 1-10ms
|
|
- Cross-region: 50-300ms
|
|
|
|
Optimization:
|
|
- Batch writes
|
|
- Use faster disks (SSD)
|
|
- Tune election timeout</code></pre>
|
|
<hr>
|
|
<h2>Common Patterns</h2>
|
|
<h3>Pattern 1: Configuration Management</h3>
|
|
<pre><code class="lang-python">class ConfigManager:
|
|
"""Use RAFT for distributed configuration"""
|
|
|
|
def __init__(self, etcd_client):
|
|
self.etcd = etcd_client
|
|
|
|
def set_config(self, key, value):
|
|
"""Set configuration value (strongly consistent)"""
|
|
self.etcd.put(f'/config/{key}', value)
|
|
|
|
def get_config(self, key):
|
|
"""Get configuration value"""
|
|
value, _ = self.etcd.get(f'/config/{key}')
|
|
return value
|
|
|
|
def watch_config(self, key, callback):
|
|
"""Watch for configuration changes"""
|
|
self.etcd.add_watch_callback(f'/config/{key}', callback)</code></pre>
|
|
<h3>Pattern 2: Leader Election</h3>
|
|
<pre><code class="lang-python">import etcd3
|
|
import time
|
|
|
|
class LeaderElection:
|
|
"""Distributed leader election using etcd"""
|
|
|
|
def __init__(self, etcd_client, name):
|
|
self.etcd = etcd_client
|
|
self.name = name
|
|
self.lease = None
|
|
|
|
def campaign(self):
|
|
"""Attempt to become leader"""
|
|
self.lease = self.etcd.lease(ttl=60)
|
|
|
|
# Try to create election key with our name
|
|
success, _ = self.etcd.transaction(
|
|
compare=[
|
|
self.etcd.transactions.version('/election/leader') == 0
|
|
],
|
|
success=[
|
|
self.etcd.transactions.put('/election/leader', self.name, lease=self.lease)
|
|
],
|
|
failure=[]
|
|
)
|
|
|
|
if success:
|
|
print(f"{self.name} became leader")
|
|
return True
|
|
return False
|
|
|
|
def resign(self):
|
|
"""Resign from leadership"""
|
|
if self.lease:
|
|
self.lease.revoke()
|
|
|
|
def keep_alive(self):
|
|
"""Keep leadership alive"""
|
|
if self.lease:
|
|
self.lease.refresh()</code></pre>
|
|
<hr>
|
|
<h2>Testing RAFT</h2>
|
|
<h3>Partition Testing</h3>
|
|
<pre><code class="lang-python">def test_network_partition():
|
|
"""Test RAFT behavior during network partition"""
|
|
cluster = RaftCluster(nodes=5)
|
|
|
|
# Normal operation
|
|
cluster.write('key1', 'value1')
|
|
assert cluster.read('key1') == 'value1'
|
|
|
|
# Partition: [1, 2] | [3, 4, 5]
|
|
cluster.partition([1, 2], [3, 4, 5])
|
|
|
|
# Majority side (3, 4, 5) should elect new leader and continue
|
|
cluster.write_to_node(3, 'key2', 'value2') # Should succeed
|
|
assert cluster.read_from_node(3, 'key2') == 'value2'
|
|
|
|
# Minority side (1, 2) should refuse writes
|
|
with pytest.raises(NoQuorumError):
|
|
cluster.write_to_node(1, 'key3', 'value3')
|
|
|
|
# Heal partition
|
|
cluster.heal_partition()
|
|
|
|
# Minority nodes should sync with majority
|
|
time.sleep(1)
|
|
assert cluster.read_from_node(1, 'key2') == 'value2'</code></pre>
|
|
<hr>
|
|
<h2>Level 3 Resources</h2>
|
|
<p>This skill includes executable scripts, comprehensive references, and examples for hands-on RAFT consensus work.</p>
|
|
<h3>Quick Start</h3>
|
|
<pre><code class="lang-bash"># Navigate to resources
|
|
cd skills/distributed-systems/consensus-raft/resources/scripts
|
|
|
|
# Start test cluster (Docker-based)
|
|
./test_etcd_cluster.sh --nodes 3
|
|
|
|
# Benchmark consensus performance
|
|
./benchmark_consensus.py --operations 5000 --concurrency 50
|
|
|
|
# Visualize RAFT state machine
|
|
./visualize_raft_state.py --type state-machine --format mermaid</code></pre>
|
|
<h3>Resources Structure</h3>
|
|
<pre><code>consensus-raft/resources/
|
|
├── REFERENCE.md # RAFT algorithm deep-dive, etcd internals, proofs
|
|
├── scripts/
|
|
│ ├── README.md # Scripts documentation
|
|
│ ├── test_etcd_cluster.sh # Docker-based etcd cluster setup
|
|
│ ├── benchmark_consensus.py # Performance benchmarking
|
|
│ └── visualize_raft_state.py # State machine visualizations
|
|
└── examples/
|
|
├── python/ # Python RAFT client examples
|
|
└── go/ # Go etcd integration examples</code></pre>
|
|
<h3>Available Scripts</h3>
|
|
<h4>test_etcd_cluster.sh</h4>
|
|
<p>Automated Docker-based etcd RAFT cluster for testing:</p>
|
|
<pre><code class="lang-bash"># 5-node cluster
|
|
./test_etcd_cluster.sh --nodes 5
|
|
|
|
# JSON output for CI/CD
|
|
./test_etcd_cluster.sh --nodes 3 --json
|
|
|
|
# Cleanup
|
|
./test_etcd_cluster.sh --cleanup</code></pre>
|
|
<h4>benchmark_consensus.py</h4>
|
|
<p>Measure consensus latency and throughput:</p>
|
|
<pre><code class="lang-bash"># Basic benchmark
|
|
./benchmark_consensus.py --operations 10000
|
|
|
|
# High concurrency
|
|
./benchmark_consensus.py --operations 50000 --concurrency 200 --json</code></pre>
|
|
<p><strong>Metrics</strong>: min/max/mean/p95/p99 latency, throughput, success rates</p>
|
|
<h4>visualize_raft_state.py</h4>
|
|
<p>Generate RAFT diagrams:</p>
|
|
<pre><code class="lang-bash"># State machine diagram
|
|
./visualize_raft_state.py --type state-machine --format mermaid
|
|
|
|
# Log replication sequence
|
|
./visualize_raft_state.py --type log-replication --format ascii
|
|
|
|
# Live cluster status
|
|
./visualize_raft_state.py --type cluster-status --endpoints localhost:2379</code></pre>
|
|
<h3>Reference Material</h3>
|
|
<p><strong>REFERENCE.md</strong> includes:</p>
|
|
<ul><li>RAFT paper key sections and safety proofs</li><li>Complete state machine specification</li><li>Leader election algorithm details</li><li>Log replication implementation</li><li>etcd architecture and internals</li><li>RAFT vs Paxos comparison</li><li>Performance characteristics</li><li>Common implementation pitfalls</li></ul>
|
|
<h3>Example Workflow</h3>
|
|
<pre><code class="lang-bash"># 1. Setup cluster
|
|
./test_etcd_cluster.sh --nodes 3
|
|
|
|
# 2. Run benchmarks
|
|
./benchmark_consensus.py --operations 5000 --json > results.json
|
|
|
|
# 3. Analyze results
|
|
cat results.json | jq '.results.put.latency.p99_ms'
|
|
|
|
# 4. Generate diagrams
|
|
./visualize_raft_state.py --type state-machine --output state.mmd
|
|
|
|
# 5. Test fault tolerance (in separate terminal)
|
|
docker stop etcd-node2 # Observe leader election
|
|
|
|
# 6. Cleanup
|
|
./test_etcd_cluster.sh --cleanup</code></pre>
|
|
<p><strong>See</strong>: <code>resources/scripts/README.md</code> for complete documentation</p>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>distributed-systems-consensus-paxos</code> - Alternative consensus algorithm</li><li><code>distributed-systems-leader-election</code> - Leader election patterns</li><li><code>distributed-systems-replication-strategies</code> - Data replication</li><li><code>distributed-systems-cap-theorem</code> - Consistency trade-offs</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-27</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/distributed-systems-consensus-raft.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |