mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
Every page now loads /webmcp.js, scripts/webmcp.ts with its types stripped (npm run webmcp). Where the browser has document.modelContext it registers two read-only tools: search_skills over the published skills index, and get_skill, which returns a skill's markdown checked against the index digest. The markdown alternate link moves from build.py to site.py, so CI holds every page to it. The API catalog gains the site itself (llms.txt and the skills index) and hanzo.ai/agents; the live discovery check now fetches every catalog link. tests/site.mjs drives the tools through a recording modelContext, and checks a browser without one loads clean.
918 lines
37 KiB
HTML
918 lines
37 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>debugging-distributed-systems-debugging</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/debugging-distributed-systems-debugging">
|
|
<link rel="alternate" type="text/markdown" href="/debugging-distributed-systems-debugging.md">
|
|
<script type="module" src="/webmcp.js"></script>
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/debugging-distributed-systems-debugging.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>debugging-distributed-systems-debugging</h1><p class="lede">Debugging distributed systems with trace correlation, cross-service workflows, request replay, clock skew handling, and chaos engineering</p><h1>Distributed Systems Debugging</h1>
|
|
<p><strong>Scope</strong>: Distributed tracing correlation, cross-service debugging, request replay, traffic shadowing, clock skew, chaos engineering, cascading failures</p>
|
|
<p><strong>Lines</strong>: 440</p>
|
|
<p><strong>Last Updated</strong>: 2025-10-26</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Use this skill when:</p>
|
|
<ul><li>Debugging failures that span multiple services</li><li>Investigating request flows across microservices</li><li>Tracking down distributed deadlocks or race conditions</li><li>Analyzing cascading failures or circuit breaker trips</li><li>Debugging eventual consistency issues</li><li>Troubleshooting distributed transactions (saga patterns)</li><li>Investigating clock skew or time synchronization problems</li><li>Using chaos engineering to reproduce production issues</li></ul>
|
|
<p><strong>Don't use</strong> for:</p>
|
|
<ul><li>Single-service debugging (use standard debugging)</li><li>Monolithic applications (use IDE debuggers)</li><li>Local development (use breakpoints)</li></ul>
|
|
<hr>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Distributed Systems Debugging Challenges</h3>
|
|
<p><strong>Unique challenges</strong>:</p>
|
|
<ol><li><strong>No global state</strong>: Each service has partial view</li><li><strong>Asynchronous communication</strong>: Request/response not always paired</li><li><strong>Network unreliability</strong>: Packets drop, timeout, reorder</li><li><strong>Clock skew</strong>: Timestamps from different hosts diverge</li><li><strong>Emergent behavior</strong>: Bugs appear only under specific interaction patterns</li><li><strong>Partial failures</strong>: Some services succeed, others fail</li><li><strong>Cascading failures</strong>: One slow service affects entire system</li></ol>
|
|
<h3>Debugging Layers in Distributed Systems</h3>
|
|
<pre><code>Layer 1: Request Tracing (follow single request)
|
|
├─ Trace ID correlation across services
|
|
├─ Span hierarchy (parent-child relationships)
|
|
└─ Timing analysis (latency breakdown)
|
|
|
|
Layer 2: Cross-Service Analysis (multi-request patterns)
|
|
├─ Request replay/shadowing
|
|
├─ Dependency mapping
|
|
└─ Circuit breaker states
|
|
|
|
Layer 3: System-Wide Behavior (emergent issues)
|
|
├─ Clock skew detection
|
|
├─ Distributed deadlock analysis
|
|
└─ Cascading failure patterns
|
|
|
|
Layer 4: Chaos Engineering (controlled failure injection)
|
|
├─ Network partition simulation
|
|
├─ Latency injection
|
|
└─ Service failure scenarios</code></pre>
|
|
<h3>Common Distributed Failure Patterns</h3>
|
|
<p>| Pattern | Symptom | Debugging Approach | |---------|---------|-------------------| | <strong>Cascading failure</strong> | One slow service causes timeouts everywhere | Trace latency, check circuit breakers | | <strong>Distributed deadlock</strong> | Services waiting on each other indefinitely | Analyze request dependencies, check for cycles | | <strong>Split brain</strong> | Multiple nodes think they're leader | Check consensus logs, clock skew | | <strong>Thundering herd</strong> | Simultaneous requests overwhelm service | Analyze request timestamps, check for jitter | | <strong>Data inconsistency</strong> | Different services see different data | Check event ordering, compare timestamps |</p>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Trace ID Correlation Across Services</h3>
|
|
<pre><code class="lang-python">import uuid
|
|
from opentelemetry import trace
|
|
from opentelemetry.propagate import inject, extract
|
|
from fastapi import FastAPI, Request, HTTPException
|
|
import httpx
|
|
import structlog
|
|
|
|
app = FastAPI()
|
|
tracer = trace.get_tracer(__name__)
|
|
logger = structlog.get_logger()
|
|
|
|
# Service A: Gateway
|
|
@app.post("/api/orders")
|
|
async def create_order(request: Request, order_data: dict):
|
|
# Extract trace context from incoming request (if exists)
|
|
ctx = extract(request.headers)
|
|
|
|
with tracer.start_as_current_span("create_order", context=ctx) as span:
|
|
# Get trace ID for logging
|
|
trace_id = format(span.get_span_context().trace_id, '032x')
|
|
span_id = format(span.get_span_context().span_id, '016x')
|
|
|
|
log = logger.bind(trace_id=trace_id, span_id=span_id)
|
|
log.info("Order creation started", order_data=order_data)
|
|
|
|
# Propagate trace context to downstream services
|
|
headers = {}
|
|
inject(headers) # Injects traceparent, tracestate headers
|
|
|
|
try:
|
|
# Call Service B (Inventory)
|
|
async with httpx.AsyncClient() as client:
|
|
inventory_response = await client.post(
|
|
"http://inventory-service/api/reserve",
|
|
json={"product_id": order_data["product_id"]},
|
|
headers=headers # Propagate trace context
|
|
)
|
|
|
|
if inventory_response.status_code != 200:
|
|
log.error(
|
|
"Inventory reservation failed",
|
|
status_code=inventory_response.status_code,
|
|
response=inventory_response.text
|
|
)
|
|
raise HTTPException(
|
|
status_code=503,
|
|
detail="Inventory service unavailable"
|
|
)
|
|
|
|
# Call Service C (Payment)
|
|
payment_response = await client.post(
|
|
"http://payment-service/api/charge",
|
|
json={"amount": order_data["amount"]},
|
|
headers=headers # Same trace context
|
|
)
|
|
|
|
if payment_response.status_code != 200:
|
|
# Compensating transaction: release inventory
|
|
await client.post(
|
|
"http://inventory-service/api/release",
|
|
json={"product_id": order_data["product_id"]},
|
|
headers=headers
|
|
)
|
|
log.error("Payment failed, inventory released")
|
|
raise HTTPException(
|
|
status_code=402,
|
|
detail="Payment processing failed"
|
|
)
|
|
|
|
log.info("Order created successfully", trace_id=trace_id)
|
|
return {
|
|
"order_id": str(uuid.uuid4()),
|
|
"trace_id": trace_id,
|
|
"status": "success"
|
|
}
|
|
|
|
except httpx.RequestError as e:
|
|
log.error("HTTP request failed", error=str(e), exc_info=True)
|
|
span.record_exception(e)
|
|
raise HTTPException(status_code=503, detail="Service unavailable")
|
|
|
|
# Service B: Inventory Service
|
|
@app.post("/api/reserve")
|
|
async def reserve_inventory(request: Request, data: dict):
|
|
# Extract trace context from upstream service
|
|
ctx = extract(request.headers)
|
|
|
|
with tracer.start_as_current_span("reserve_inventory", context=ctx) as span:
|
|
trace_id = format(span.get_span_context().trace_id, '032x')
|
|
log = logger.bind(trace_id=trace_id, service="inventory")
|
|
|
|
log.info("Inventory reservation", product_id=data["product_id"])
|
|
|
|
# Simulate inventory check
|
|
if data["product_id"] == "out-of-stock":
|
|
log.error("Product out of stock", product_id=data["product_id"])
|
|
return {"status": "out_of_stock"}, 409
|
|
|
|
log.info("Inventory reserved", product_id=data["product_id"])
|
|
return {"status": "reserved"}
|
|
|
|
# Service C: Payment Service
|
|
@app.post("/api/charge")
|
|
async def charge_payment(request: Request, data: dict):
|
|
ctx = extract(request.headers)
|
|
|
|
with tracer.start_as_current_span("charge_payment", context=ctx) as span:
|
|
trace_id = format(span.get_span_context().trace_id, '032x')
|
|
log = logger.bind(trace_id=trace_id, service="payment")
|
|
|
|
log.info("Payment processing", amount=data["amount"])
|
|
|
|
# Simulate payment processing
|
|
if data["amount"] > 10000:
|
|
log.error("Payment amount exceeds limit", amount=data["amount"])
|
|
return {"status": "declined"}, 402
|
|
|
|
log.info("Payment successful", amount=data["amount"])
|
|
return {"status": "charged"}</code></pre>
|
|
<p><strong>Query traces across services</strong> (Grafana Tempo):</p>
|
|
<pre><code class="lang-promql"># Find all spans for trace ID
|
|
{trace_id="a1b2c3d4e5f6g7h8"}
|
|
|
|
# Find traces with errors in any service
|
|
{status.code="ERROR"}
|
|
|
|
# Find slow traces (>1s)
|
|
{duration > 1s}
|
|
|
|
# Find traces involving specific service
|
|
{service.name="payment-service"}</code></pre>
|
|
<h3>Pattern 2: Request Replay for Debugging</h3>
|
|
<pre><code class="lang-python">import json
|
|
import asyncio
|
|
from typing import Dict, Any
|
|
from datetime import datetime
|
|
import httpx
|
|
|
|
class RequestRecorder:
|
|
"""
|
|
Record requests for later replay.
|
|
"""
|
|
|
|
def __init__(self, storage_path: str = "/tmp/requests.jsonl"):
|
|
self.storage_path = storage_path
|
|
|
|
async def record_request(
|
|
self,
|
|
method: str,
|
|
url: str,
|
|
headers: Dict[str, str],
|
|
body: Any,
|
|
response_status: int,
|
|
response_body: Any
|
|
):
|
|
"""
|
|
Record request/response for replay.
|
|
"""
|
|
record = {
|
|
"timestamp": datetime.utcnow().isoformat(),
|
|
"request": {
|
|
"method": method,
|
|
"url": url,
|
|
"headers": dict(headers),
|
|
"body": body
|
|
},
|
|
"response": {
|
|
"status": response_status,
|
|
"body": response_body
|
|
}
|
|
}
|
|
|
|
with open(self.storage_path, "a") as f:
|
|
f.write(json.dumps(record) + "\n")
|
|
|
|
class RequestReplayer:
|
|
"""
|
|
Replay recorded requests against test environment.
|
|
"""
|
|
|
|
def __init__(self, storage_path: str = "/tmp/requests.jsonl"):
|
|
self.storage_path = storage_path
|
|
|
|
async def replay_all(self, target_url: str):
|
|
"""
|
|
Replay all recorded requests.
|
|
"""
|
|
with open(self.storage_path, "r") as f:
|
|
for line in f:
|
|
record = json.loads(line)
|
|
await self.replay_request(record, target_url)
|
|
|
|
async def replay_request(self, record: Dict, target_url: str):
|
|
"""
|
|
Replay single request.
|
|
"""
|
|
req = record["request"]
|
|
|
|
# Replace production URL with target URL
|
|
url = req["url"].replace("https://api.example.com", target_url)
|
|
|
|
async with httpx.AsyncClient() as client:
|
|
try:
|
|
response = await client.request(
|
|
method=req["method"],
|
|
url=url,
|
|
headers=req["headers"],
|
|
json=req["body"]
|
|
)
|
|
|
|
# Compare response with recorded response
|
|
original_status = record["response"]["status"]
|
|
if response.status_code != original_status:
|
|
print(f"MISMATCH: Expected {original_status}, got {response.status_code}")
|
|
print(f"URL: {url}")
|
|
print(f"Response: {response.text}")
|
|
|
|
except Exception as e:
|
|
print(f"ERROR replaying request: {e}")
|
|
print(f"URL: {url}")
|
|
|
|
# Middleware to record requests
|
|
from fastapi import Request, Response
|
|
from starlette.middleware.base import BaseHTTPMiddleware
|
|
|
|
recorder = RequestRecorder()
|
|
|
|
class RecordingMiddleware(BaseHTTPMiddleware):
|
|
async def dispatch(self, request: Request, call_next):
|
|
# Record request
|
|
body = await request.body()
|
|
|
|
response = await call_next(request)
|
|
|
|
# Record response
|
|
response_body = b""
|
|
async for chunk in response.body_iterator:
|
|
response_body += chunk
|
|
|
|
await recorder.record_request(
|
|
method=request.method,
|
|
url=str(request.url),
|
|
headers=request.headers,
|
|
body=body.decode() if body else None,
|
|
response_status=response.status_code,
|
|
response_body=response_body.decode()
|
|
)
|
|
|
|
return Response(
|
|
content=response_body,
|
|
status_code=response.status_code,
|
|
headers=dict(response.headers)
|
|
)
|
|
|
|
# Usage
|
|
app.add_middleware(RecordingMiddleware)
|
|
|
|
# Replay requests in test environment
|
|
# python replay.py --target http://localhost:8000</code></pre>
|
|
<h3>Pattern 3: Traffic Shadowing (Dark Traffic)</h3>
|
|
<pre><code class="lang-yaml"># Kubernetes: Shadow traffic to debug environment
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: api-service
|
|
spec:
|
|
selector:
|
|
app: api
|
|
ports:
|
|
- port: 80
|
|
targetPort: 8080
|
|
|
|
---
|
|
# Envoy/Istio: Mirror traffic to debug instance
|
|
apiVersion: networking.istio.io/v1beta1
|
|
kind: VirtualService
|
|
metadata:
|
|
name: api-service
|
|
spec:
|
|
hosts:
|
|
- api-service
|
|
http:
|
|
- match:
|
|
- headers:
|
|
x-debug:
|
|
exact: "true"
|
|
route:
|
|
- destination:
|
|
host: api-service
|
|
subset: production
|
|
weight: 100
|
|
mirror:
|
|
host: api-service
|
|
subset: debug # Mirror to debug instance
|
|
mirrorPercentage:
|
|
value: 100 # Mirror 100% of matching requests
|
|
|
|
---
|
|
apiVersion: networking.istio.io/v1beta1
|
|
kind: DestinationRule
|
|
metadata:
|
|
name: api-service
|
|
spec:
|
|
host: api-service
|
|
subsets:
|
|
- name: production
|
|
labels:
|
|
version: v1
|
|
- name: debug
|
|
labels:
|
|
version: debug</code></pre>
|
|
<p><strong>Application-level traffic shadowing</strong>:</p>
|
|
<pre><code class="lang-python">from fastapi import Request, BackgroundTasks
|
|
import httpx
|
|
|
|
@app.post("/api/orders")
|
|
async def create_order(
|
|
request: Request,
|
|
order_data: dict,
|
|
background_tasks: BackgroundTasks
|
|
):
|
|
# Process request normally
|
|
result = await process_order(order_data)
|
|
|
|
# Shadow request to debug environment (async)
|
|
if should_shadow_request(request):
|
|
background_tasks.add_task(
|
|
shadow_request,
|
|
method=request.method,
|
|
url=str(request.url),
|
|
headers=dict(request.headers),
|
|
body=order_data
|
|
)
|
|
|
|
return result
|
|
|
|
async def shadow_request(method: str, url: str, headers: dict, body: dict):
|
|
"""
|
|
Send request to debug environment in background.
|
|
"""
|
|
# Replace production URL with debug URL
|
|
debug_url = url.replace("api.example.com", "debug.example.com")
|
|
|
|
async with httpx.AsyncClient() as client:
|
|
try:
|
|
response = await client.request(
|
|
method=method,
|
|
url=debug_url,
|
|
headers=headers,
|
|
json=body,
|
|
timeout=5.0 # Don't block production traffic
|
|
)
|
|
|
|
# Log differences
|
|
logger.info(
|
|
"Shadow request completed",
|
|
status_code=response.status_code,
|
|
url=debug_url
|
|
)
|
|
|
|
except Exception as e:
|
|
# Don't fail production traffic
|
|
logger.error("Shadow request failed", error=str(e))
|
|
|
|
def should_shadow_request(request: Request) -> bool:
|
|
"""
|
|
Determine if request should be shadowed.
|
|
"""
|
|
# Shadow 10% of traffic
|
|
import random
|
|
return random.random() < 0.10
|
|
|
|
# Or shadow specific users
|
|
# user_id = request.headers.get("X-User-ID")
|
|
# return user_id in ["debug_user_1", "debug_user_2"]</code></pre>
|
|
<h3>Pattern 4: Clock Skew Detection and Handling</h3>
|
|
<pre><code class="lang-python">import time
|
|
from datetime import datetime, timedelta
|
|
import structlog
|
|
|
|
logger = structlog.get_logger()
|
|
|
|
class ClockSkewDetector:
|
|
"""
|
|
Detect and handle clock skew in distributed systems.
|
|
"""
|
|
|
|
def __init__(self, max_skew_seconds: int = 300): # 5 minutes
|
|
self.max_skew_seconds = max_skew_seconds
|
|
|
|
def check_timestamp(
|
|
self,
|
|
remote_timestamp: str,
|
|
service_name: str
|
|
) -> bool:
|
|
"""
|
|
Check if remote timestamp is within acceptable range.
|
|
"""
|
|
try:
|
|
remote_time = datetime.fromisoformat(remote_timestamp)
|
|
local_time = datetime.utcnow()
|
|
skew = abs((remote_time - local_time).total_seconds())
|
|
|
|
if skew > self.max_skew_seconds:
|
|
logger.warning(
|
|
"Clock skew detected",
|
|
service=service_name,
|
|
remote_timestamp=remote_timestamp,
|
|
local_timestamp=local_time.isoformat(),
|
|
skew_seconds=skew
|
|
)
|
|
return False
|
|
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error("Invalid timestamp format", error=str(e))
|
|
return False
|
|
|
|
def sync_with_ntp(self):
|
|
"""
|
|
Sync local clock with NTP server.
|
|
"""
|
|
import ntplib
|
|
from datetime import datetime
|
|
|
|
ntp_client = ntplib.NTPClient()
|
|
|
|
try:
|
|
response = ntp_client.request('pool.ntp.org', version=3)
|
|
local_time = time.time()
|
|
ntp_time = response.tx_time
|
|
offset = ntp_time - local_time
|
|
|
|
logger.info(
|
|
"NTP sync completed",
|
|
offset_ms=offset * 1000,
|
|
ntp_server="pool.ntp.org"
|
|
)
|
|
|
|
return offset
|
|
|
|
except Exception as e:
|
|
logger.error("NTP sync failed", error=str(e))
|
|
return None
|
|
|
|
# Middleware to check clock skew
|
|
from fastapi import Request, HTTPException
|
|
|
|
clock_detector = ClockSkewDetector()
|
|
|
|
@app.middleware("http")
|
|
async def check_clock_skew(request: Request, call_next):
|
|
# Check if request has timestamp header
|
|
remote_timestamp = request.headers.get("X-Timestamp")
|
|
|
|
if remote_timestamp:
|
|
if not clock_detector.check_timestamp(
|
|
remote_timestamp,
|
|
request.headers.get("X-Service-Name", "unknown")
|
|
):
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="Clock skew detected. Check NTP sync."
|
|
)
|
|
|
|
response = await call_next(request)
|
|
return response</code></pre>
|
|
<p><strong>NTP sync commands</strong>:</p>
|
|
<pre><code class="lang-bash"># Check NTP status
|
|
timedatectl status
|
|
|
|
# Sync with NTP server (Linux)
|
|
sudo ntpdate pool.ntp.org
|
|
|
|
# Enable NTP sync (systemd)
|
|
sudo timedatectl set-ntp true
|
|
|
|
# Check clock offset
|
|
ntpq -p</code></pre>
|
|
<h3>Pattern 5: Chaos Engineering for Debugging</h3>
|
|
<pre><code class="lang-python">import random
|
|
import asyncio
|
|
from fastapi import HTTPException
|
|
|
|
class ChaosMonkey:
|
|
"""
|
|
Inject controlled failures for debugging.
|
|
"""
|
|
|
|
def __init__(self, enabled: bool = False):
|
|
self.enabled = enabled
|
|
|
|
async def inject_latency(
|
|
self,
|
|
min_ms: int = 100,
|
|
max_ms: int = 1000,
|
|
probability: float = 0.1
|
|
):
|
|
"""
|
|
Inject random latency.
|
|
"""
|
|
if not self.enabled:
|
|
return
|
|
|
|
if random.random() < probability:
|
|
delay = random.randint(min_ms, max_ms) / 1000
|
|
logger.warning("CHAOS: Injecting latency", delay_seconds=delay)
|
|
await asyncio.sleep(delay)
|
|
|
|
def inject_error(
|
|
self,
|
|
probability: float = 0.05,
|
|
error_code: int = 503
|
|
):
|
|
"""
|
|
Inject random errors.
|
|
"""
|
|
if not self.enabled:
|
|
return
|
|
|
|
if random.random() < probability:
|
|
logger.warning("CHAOS: Injecting error", error_code=error_code)
|
|
raise HTTPException(
|
|
status_code=error_code,
|
|
detail="Chaos engineering: simulated failure"
|
|
)
|
|
|
|
def inject_timeout(
|
|
self,
|
|
probability: float = 0.05,
|
|
timeout_seconds: int = 30
|
|
):
|
|
"""
|
|
Inject timeout by sleeping indefinitely.
|
|
"""
|
|
if not self.enabled:
|
|
return
|
|
|
|
if random.random() < probability:
|
|
logger.warning("CHAOS: Injecting timeout", timeout_seconds=timeout_seconds)
|
|
import time
|
|
time.sleep(timeout_seconds)
|
|
|
|
# Enable chaos in specific environment
|
|
import os
|
|
chaos = ChaosMonkey(enabled=os.getenv("CHAOS_ENABLED") == "true")
|
|
|
|
@app.get("/api/users")
|
|
async def get_users():
|
|
# Inject chaos
|
|
await chaos.inject_latency(probability=0.1)
|
|
chaos.inject_error(probability=0.05)
|
|
|
|
users = await db.get_users()
|
|
return users</code></pre>
|
|
<p><strong>Using Chaos Mesh</strong> (Kubernetes):</p>
|
|
<pre><code class="lang-yaml">apiVersion: chaos-mesh.org/v1alpha1
|
|
kind: NetworkChaos
|
|
metadata:
|
|
name: network-delay
|
|
spec:
|
|
action: delay
|
|
mode: one
|
|
selector:
|
|
namespaces:
|
|
- production
|
|
labelSelectors:
|
|
app: api-service
|
|
delay:
|
|
latency: "100ms"
|
|
jitter: "50ms"
|
|
duration: "30s"
|
|
|
|
---
|
|
apiVersion: chaos-mesh.org/v1alpha1
|
|
kind: PodChaos
|
|
metadata:
|
|
name: pod-failure
|
|
spec:
|
|
action: pod-failure
|
|
mode: one
|
|
selector:
|
|
namespaces:
|
|
- production
|
|
labelSelectors:
|
|
app: payment-service
|
|
duration: "60s"</code></pre>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Distributed Debugging Checklist</h3>
|
|
<pre><code class="lang-markdown">## Step 1: Identify Trace
|
|
- [ ] Extract trace ID from error logs
|
|
- [ ] Find all spans in trace (Grafana Tempo, Jaeger)
|
|
- [ ] Identify failed span and service
|
|
|
|
## Step 2: Analyze Dependencies
|
|
- [ ] Check which services were called
|
|
- [ ] Verify trace context propagation
|
|
- [ ] Look for missing spans (broken propagation)
|
|
|
|
## Step 3: Check Timing
|
|
- [ ] Compare timestamps across services
|
|
- [ ] Look for clock skew (>5min difference)
|
|
- [ ] Identify slow spans (>1s)
|
|
|
|
## Step 4: Correlate with Infrastructure
|
|
- [ ] Check CPU/memory on affected services
|
|
- [ ] Review network metrics (latency, packet loss)
|
|
- [ ] Check for recent deployments
|
|
|
|
## Step 5: Reproduce
|
|
- [ ] Replay request in staging
|
|
- [ ] Use traffic shadowing
|
|
- [ ] Inject chaos (latency, errors)</code></pre>
|
|
<h3>Trace Correlation Queries</h3>
|
|
<pre><code class="lang-promql"># Grafana Loki + Tempo
|
|
|
|
# Find logs for trace
|
|
{app="api-service"} | json | trace_id="abc123"
|
|
|
|
# Find errors in trace
|
|
{trace_id="abc123"} | json | level="error"
|
|
|
|
# Find slow requests
|
|
{app="api-service"} | json | duration_ms > 1000 | trace_id!=""</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>❌ Not Propagating Trace Context</h3>
|
|
<pre><code class="lang-python"># WRONG: Breaks trace chain
|
|
requests.get("http://service-b/api/data")
|
|
|
|
# CORRECT: Propagate trace context
|
|
headers = {}
|
|
inject(headers)
|
|
requests.get("http://service-b/api/data", headers=headers)</code></pre>
|
|
<h3>❌ Using Local Timestamps</h3>
|
|
<pre><code class="lang-python"># WRONG: Clock skew causes issues
|
|
event_time = datetime.now() # Local clock!
|
|
|
|
# CORRECT: Use monotonic clock or logical timestamps
|
|
event_time = time.monotonic() # Monotonic clock
|
|
# Or: Use Lamport/Vector clocks for ordering</code></pre>
|
|
<h3>❌ Ignoring Compensating Transactions</h3>
|
|
<pre><code class="lang-python"># WRONG: Leaves system in inconsistent state
|
|
await reserve_inventory()
|
|
await charge_payment() # Fails, but inventory not released!
|
|
|
|
# CORRECT: Compensate on failure
|
|
try:
|
|
await reserve_inventory()
|
|
await charge_payment()
|
|
except:
|
|
await release_inventory() # Compensating transaction
|
|
raise</code></pre>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><strong>observability/distributed-tracing.md</strong> - Trace context propagation</li><li><strong>debugging/production-debugging.md</strong> - Non-intrusive debugging</li><li><strong>testing/chaos-engineering.md</strong> - Controlled failure injection</li><li><strong>database/distributed-transactions.md</strong> - Saga patterns</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-26 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/debugging-distributed-systems-debugging.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |