mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
652 lines
29 KiB
HTML
652 lines
29 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>observability-production-incident-debugging</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/observability-production-incident-debugging.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/observability-production-incident-debugging">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/observability-production-incident-debugging.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>observability-production-incident-debugging</h1><p class="lede">Debugging production incidents using observability data (logs, metrics, traces)</p><h1>Production Incident Debugging</h1>
|
|
<p><strong>Scope</strong>: Incident triage workflow, root cause analysis, distributed tracing for debugging, correlation IDs, runbook creation, blameless postmortems</p>
|
|
<p><strong>Lines</strong>: 440</p>
|
|
<p><strong>Last Updated</strong>: 2025-10-26</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Use this skill when:</p>
|
|
<ul><li>Triaging production incidents with observability data</li><li>Performing root cause analysis (RCA) for system failures</li><li>Debugging distributed systems with multiple microservices</li><li>Correlating logs, metrics, and traces to diagnose issues</li><li>Creating runbooks from past incidents</li><li>Conducting blameless postmortems</li><li>Implementing time-travel debugging with distributed tracing</li></ul>
|
|
<p><strong>Don't use</strong> for:</p>
|
|
<ul><li>Local development debugging (use IDE debugger)</li><li>Performance optimization (use profiling tools)</li><li>Security incident response (use security-specific tools)</li></ul>
|
|
<p><strong>Context</strong>:</p>
|
|
<ul><li><strong>Mean Time to Detect (MTTD)</strong>: Average 15-30 minutes for mature teams</li><li><strong>Mean Time to Resolve (MTTR)</strong>: Reduced by 50% with proper observability</li><li><strong>Root cause accuracy</strong>: 80%+ with correlated telemetry vs 40% with logs alone</li></ul>
|
|
<hr>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Incident Triage Workflow</h3>
|
|
<pre><code>1. DETECT: Alert fires or user reports issue
|
|
↓
|
|
2. ASSESS: Check dashboards (RED metrics: Rate, Errors, Duration)
|
|
↓
|
|
3. INVESTIGATE: Logs → Metrics → Traces (in that order)
|
|
↓
|
|
4. ISOLATE: Identify failing component/service
|
|
↓
|
|
5. MITIGATE: Rollback, scale, or patch
|
|
↓
|
|
6. RESOLVE: Fix root cause
|
|
↓
|
|
7. LEARN: Postmortem, runbook updates</code></pre>
|
|
<h3>RED Method (Request-Driven Systems)</h3>
|
|
<p><strong>Rate</strong>: Requests per second</p>
|
|
<ul><li>Sudden drop? Upstream failure or routing issue</li><li>Sudden spike? Traffic surge or DDoS</li></ul>
|
|
<p><strong>Errors</strong>: Error rate (percentage)</p>
|
|
<ul><li>Increase in 5xx errors? Backend failure</li><li>Increase in 4xx errors? Client or API contract issue</li></ul>
|
|
<p><strong>Duration</strong>: Latency (p50, p95, p99)</p>
|
|
<ul><li>Increased latency? Database slow, network congestion, resource exhaustion</li></ul>
|
|
<h3>USE Method (Resource-Driven Systems)</h3>
|
|
<p><strong>Utilization</strong>: % time resource is busy <strong>Saturation</strong>: Degree of queued work <strong>Errors</strong>: Error count</p>
|
|
<h3>The Three Pillars</h3>
|
|
<p><strong>Logs</strong> → What happened?</p>
|
|
<ul><li>Discrete events with context</li><li>Use: Error messages, stack traces, business events</li></ul>
|
|
<p><strong>Metrics</strong> → How much/many?</p>
|
|
<ul><li>Aggregated measurements</li><li>Use: Trends, thresholds, alerting</li></ul>
|
|
<p><strong>Traces</strong> → Where is the bottleneck?</p>
|
|
<ul><li>Request flows across services</li><li>Use: Latency breakdown, dependency mapping</li></ul>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Incident Triage with RED Metrics</h3>
|
|
<pre><code class="lang-promql"># 1. CHECK RATE: Sudden drop or spike?
|
|
sum(rate(http_requests_total[5m])) by (service)
|
|
|
|
# 2. CHECK ERRORS: Error rate increase?
|
|
sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
|
|
/
|
|
sum(rate(http_requests_total[5m])) by (service)
|
|
* 100 # Percentage
|
|
|
|
# 3. CHECK DURATION: Latency spike?
|
|
histogram_quantile(0.95,
|
|
sum by (le, service) (
|
|
rate(http_request_duration_seconds_bucket[5m])
|
|
)
|
|
)
|
|
|
|
# 4. IDENTIFY FAILING SERVICE
|
|
# Look for:
|
|
# - Service with highest error rate
|
|
# - Service with highest latency increase
|
|
# - Service with rate drop (if upstream)</code></pre>
|
|
<h3>Pattern 2: Logs → Metrics → Traces Workflow</h3>
|
|
<p><strong>Step 1: Start with Logs (What happened?)</strong></p>
|
|
<pre><code class="lang-bash"># Find errors around incident time
|
|
kubectl logs -l app=order-service --since=30m | grep -i error
|
|
|
|
# Example output:
|
|
# 2025-10-26 10:15:32 ERROR [trace_id=abc123] Database connection timeout
|
|
# 2025-10-26 10:15:45 ERROR [trace_id=def456] Payment gateway unreachable
|
|
# 2025-10-26 10:16:12 ERROR [trace_id=ghi789] Order processing failed</code></pre>
|
|
<p><strong>Step 2: Check Metrics (How widespread?)</strong></p>
|
|
<pre><code class="lang-promql"># How many errors?
|
|
sum(rate(logs_total{level="error", service="order-service"}[5m]))
|
|
|
|
# Which endpoint?
|
|
sum by (endpoint) (rate(http_requests_total{status="500", service="order-service"}[5m]))
|
|
|
|
# When did it start?
|
|
delta(http_requests_total{status="500", service="order-service"}[1h])</code></pre>
|
|
<p><strong>Step 3: Trace Deep Dive (Where is the bottleneck?)</strong></p>
|
|
<pre><code># Use trace_id from logs (abc123)
|
|
# Open Jaeger/Tempo UI → Search trace_id=abc123
|
|
|
|
Trace View:
|
|
order-service [200ms]
|
|
├─ validate-order [10ms]
|
|
├─ check-inventory [50ms]
|
|
├─ process-payment [5000ms] ← SLOW!
|
|
│ ├─ call-payment-gateway [4950ms] ← BOTTLENECK!
|
|
│ └─ update-ledger [50ms]
|
|
└─ send-confirmation [20ms]</code></pre>
|
|
<p><strong>Diagnosis</strong>: Payment gateway timeout (4950ms vs expected 100ms)</p>
|
|
<h3>Pattern 3: Correlation IDs for Request Tracking</h3>
|
|
<pre><code class="lang-python">import uuid
|
|
import logging
|
|
from flask import Flask, request, g
|
|
|
|
app = Flask(__name__)
|
|
|
|
@app.before_request
|
|
def set_correlation_id():
|
|
# Extract or generate correlation ID
|
|
correlation_id = request.headers.get('X-Correlation-ID') or str(uuid.uuid4())
|
|
g.correlation_id = correlation_id
|
|
|
|
@app.after_request
|
|
def add_correlation_header(response):
|
|
response.headers['X-Correlation-ID'] = g.correlation_id
|
|
return response
|
|
|
|
# Configure logger with correlation ID
|
|
class CorrelationFilter(logging.Filter):
|
|
def filter(self, record):
|
|
record.correlation_id = getattr(g, 'correlation_id', 'N/A')
|
|
return True
|
|
|
|
logging.basicConfig(
|
|
format='%(asctime)s [correlation_id=%(correlation_id)s] %(levelname)s %(message)s'
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
logger.addFilter(CorrelationFilter())
|
|
|
|
@app.route('/api/orders', methods=['POST'])
|
|
def create_order():
|
|
logger.info("Creating order")
|
|
|
|
# Pass correlation ID to downstream services
|
|
headers = {'X-Correlation-ID': g.correlation_id}
|
|
response = requests.post('http://inventory-service/check', headers=headers)
|
|
|
|
logger.info("Order created successfully")
|
|
return {"order_id": "12345"}
|
|
|
|
# Log output:
|
|
# 2025-10-26 10:30:15 [correlation_id=abc-123-def] INFO Creating order
|
|
# 2025-10-26 10:30:16 [correlation_id=abc-123-def] INFO Order created successfully</code></pre>
|
|
<h3>Pattern 4: Time-Travel Debugging with Traces</h3>
|
|
<p><strong>Scenario</strong>: User reports checkout failed at 10:15 AM</p>
|
|
<pre><code>1. Find user's trace:
|
|
- Search by user_id in Jaeger: user_id=user_789 timestamp>2025-10-26T10:10:00
|
|
- Or search by correlation_id if logged
|
|
|
|
2. Examine trace timeline:
|
|
checkout-service [10:15:32 - 10:15:37] 5s
|
|
├─ cart-service [10:15:32 - 10:15:33] 1s
|
|
├─ payment [10:15:33 - 10:15:38] 5s ← TIMEOUT!
|
|
└─ inventory [not reached]
|
|
|
|
3. Drill into payment span:
|
|
- Status: ERROR
|
|
- Error message: "Connection refused: payment-gateway:8080"
|
|
- Attributes:
|
|
- payment.amount: 99.99
|
|
- payment.method: credit_card
|
|
- retry.count: 3
|
|
|
|
4. Cross-reference with logs:
|
|
grep "correlation_id=trace_xyz" logs/payment-service.log
|
|
→ "2025-10-26 10:15:33 ERROR Failed to connect to payment gateway after 3 retries"
|
|
|
|
5. Check metrics at 10:15:
|
|
- payment_gateway_connections{state="failed"} → spike at 10:15
|
|
- payment_gateway_health → DOWN at 10:14
|
|
|
|
6. Root cause: Payment gateway crash at 10:14, user checkout at 10:15 failed</code></pre>
|
|
<h3>Pattern 5: Database Query Debugging</h3>
|
|
<pre><code class="lang-python">import logging
|
|
from opentelemetry import trace
|
|
from sqlalchemy import event, create_engine
|
|
|
|
tracer = trace.get_tracer(__name__)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
engine = create_engine('postgresql://localhost/mydb')
|
|
|
|
@event.listens_for(engine, "before_cursor_execute")
|
|
def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
|
|
# Start span for SQL query
|
|
span = tracer.start_span("db.query")
|
|
span.set_attribute("db.statement", statement)
|
|
span.set_attribute("db.system", "postgresql")
|
|
context._span = span
|
|
context._query_start_time = time.time()
|
|
|
|
@event.listens_for(engine, "after_cursor_execute")
|
|
def after_cursor_execute(conn, cursor, statement, parameters, context, executemany):
|
|
duration = time.time() - context._query_start_time
|
|
context._span.set_attribute("db.duration_ms", duration * 1000)
|
|
context._span.end()
|
|
|
|
# Log slow queries
|
|
if duration > 1.0:
|
|
logger.warning(f"Slow query ({duration:.2f}s): {statement[:100]}")
|
|
|
|
# Usage
|
|
with tracer.start_as_current_span("get_user_orders"):
|
|
user = session.query(User).filter_by(id=user_id).first() # Auto-traced
|
|
orders = session.query(Order).filter_by(user_id=user_id).all() # Auto-traced
|
|
|
|
# Trace shows:
|
|
# get_user_orders [150ms]
|
|
# ├─ db.query [10ms] "SELECT * FROM users WHERE id = ?"
|
|
# └─ db.query [140ms] "SELECT * FROM orders WHERE user_id = ?" ← SLOW!</code></pre>
|
|
<h3>Pattern 6: Runbook Creation from Incidents</h3>
|
|
<p><strong>Runbook Template</strong>:</p>
|
|
<pre><code class="lang-markdown"># Incident: Payment Gateway Timeout (2025-10-26)
|
|
|
|
## Symptoms
|
|
- Users unable to complete checkout
|
|
- Error: "Payment processing failed, please try again"
|
|
- Alert: `payment_errors_rate > 5%`
|
|
|
|
## Detection
|
|
- Alert fired: `payment_gateway_health == 0`
|
|
- Dashboard: RED metrics show 50% error rate on `/api/checkout`
|
|
|
|
## Investigation
|
|
1. Check RED metrics:
|
|
```promql
|
|
sum(rate(http_requests_total{service="payment", status="500"}[5m]))
|
|
```
|
|
|
|
2. Find failing requests in logs:
|
|
```bash
|
|
kubectl logs -l app=payment-service --since=30m | grep ERROR
|
|
```
|
|
|
|
3. Trace sample failed request:
|
|
- Search Jaeger: service=payment status=error time>now-30m
|
|
- Identify span with error
|
|
|
|
## Diagnosis
|
|
- Payment gateway connection timeout (5s)
|
|
- Root cause: Payment gateway service crash
|
|
|
|
## Mitigation (Immediate)
|
|
1. Check payment gateway health:
|
|
```bash
|
|
kubectl get pods -l app=payment-gateway
|
|
```
|
|
|
|
2. Restart payment gateway:
|
|
```bash
|
|
kubectl rollout restart deployment/payment-gateway
|
|
```
|
|
|
|
3. Monitor recovery:
|
|
```promql
|
|
payment_gateway_health
|
|
```
|
|
|
|
## Resolution (Long-term)
|
|
- Implement circuit breaker pattern
|
|
- Add retry with exponential backoff
|
|
- Set up redundant payment gateway instances
|
|
- Add alerting for payment gateway health
|
|
|
|
## Prevention
|
|
- Deploy Istio circuit breaker config
|
|
- Add PodDisruptionBudget for payment gateway
|
|
- Implement graceful degradation (queue payments for retry)
|
|
|
|
## Related Incidents
|
|
- [2025-09-15] Payment gateway timeout (similar root cause)
|
|
- [2025-08-22] Database connection pool exhaustion
|
|
|
|
## Postmortem
|
|
- [Link to postmortem doc]</code></pre>
|
|
<h3>Pattern 7: Blameless Postmortem</h3>
|
|
<p><strong>Postmortem Template</strong>:</p>
|
|
<pre><code class="lang-markdown"># Postmortem: Payment Gateway Outage (2025-10-26)
|
|
|
|
## Incident Summary
|
|
- **Date**: 2025-10-26 10:14 - 10:45 UTC
|
|
- **Duration**: 31 minutes
|
|
- **Severity**: SEV-2 (Revenue impact)
|
|
- **Impact**: 15% of checkout attempts failed (~$50k revenue at risk)
|
|
|
|
## Timeline
|
|
- **10:14**: Payment gateway pod crash (OOM)
|
|
- **10:15**: First user reports checkout failure
|
|
- **10:16**: Alert fired: `payment_errors_rate > 5%`
|
|
- **10:18**: On-call engineer paged
|
|
- **10:22**: Investigation started
|
|
- **10:28**: Payment gateway restarted
|
|
- **10:30**: Health check passed
|
|
- **10:35**: Error rate returned to normal
|
|
- **10:45**: Incident marked resolved
|
|
|
|
## Root Cause
|
|
Payment gateway pod exceeded memory limit (2GB) due to connection leak in payment provider SDK v1.2.3. Under high load, SDK failed to close idle connections, leading to OOM crash.
|
|
|
|
## What Went Well
|
|
- Alert fired within 2 minutes of first error
|
|
- Correlation IDs enabled quick trace lookup
|
|
- Distributed tracing pinpointed exact failing span
|
|
- Runbook had clear mitigation steps
|
|
|
|
## What Went Wrong
|
|
- No memory usage alerting for payment gateway
|
|
- Single replica of payment gateway (no redundancy)
|
|
- Circuit breaker not configured (cascading failures)
|
|
- SDK version not pinned (auto-updated to buggy version)
|
|
|
|
## Action Items
|
|
1. **[P0]** Add memory usage alerts for all critical services (Owner: @alice, Due: 2025-10-28)
|
|
2. **[P0]** Scale payment gateway to 3 replicas with PDB (Owner: @bob, Due: 2025-10-27)
|
|
3. **[P1]** Implement circuit breaker with 10s timeout (Owner: @charlie, Due: 2025-11-02)
|
|
4. **[P1]** Pin SDK versions in requirements.txt (Owner: @alice, Due: 2025-10-27)
|
|
5. **[P2]** Add integration tests for payment flow (Owner: @bob, Due: 2025-11-10)
|
|
6. **[P2]** Update runbook with memory debugging steps (Owner: @charlie, Due: 2025-10-30)
|
|
|
|
## Lessons Learned
|
|
- **Observability gap**: Missing memory metrics for critical services
|
|
- **Architecture gap**: Single point of failure (payment gateway)
|
|
- **Process gap**: SDK updates without testing
|
|
|
|
## Follow-up
|
|
- Review all critical services for similar gaps
|
|
- Implement mandatory testing for dependency updates</code></pre>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Incident Response Commands</h3>
|
|
<pre><code class="lang-bash"># Check pod health
|
|
kubectl get pods -l app=service-name
|
|
|
|
# View recent logs
|
|
kubectl logs -l app=service-name --since=30m --tail=100
|
|
|
|
# Stream logs
|
|
kubectl logs -f -l app=service-name
|
|
|
|
# Get pod metrics
|
|
kubectl top pods -l app=service-name
|
|
|
|
# Describe pod (events)
|
|
kubectl describe pod service-name-abc123
|
|
|
|
# Execute into pod
|
|
kubectl exec -it service-name-abc123 -- /bin/bash
|
|
|
|
# Port forward for debugging
|
|
kubectl port-forward service-name-abc123 8080:8080</code></pre>
|
|
<h3>Useful PromQL Queries</h3>
|
|
<pre><code class="lang-promql"># Error rate (percentage)
|
|
sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
|
|
/
|
|
sum(rate(http_requests_total[5m])) by (service)
|
|
* 100
|
|
|
|
# Latency increase (current vs 1h ago)
|
|
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))
|
|
/
|
|
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m] offset 1h))
|
|
|
|
# Request rate change
|
|
sum(rate(http_requests_total[5m])) by (service)
|
|
/
|
|
sum(rate(http_requests_total[5m] offset 1h)) by (service)
|
|
|
|
# Memory usage
|
|
container_memory_usage_bytes{pod=~"service-.*"} / 1024 / 1024 # MB
|
|
|
|
# CPU usage
|
|
rate(container_cpu_usage_seconds_total{pod=~"service-.*"}[5m]) * 100 # %</code></pre>
|
|
<h3>Jaeger Search Queries</h3>
|
|
<pre><code># By service and status
|
|
service=order-service status=error
|
|
|
|
# By tag
|
|
http.status_code=500
|
|
user_id=user_789
|
|
|
|
# By time range
|
|
service=payment minDuration=1s maxDuration=10s
|
|
|
|
# By operation
|
|
service=checkout operation=/api/checkout</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>❌ Starting with Traces Instead of Metrics</h3>
|
|
<pre><code># WRONG: Jump straight to traces
|
|
1. Alert fires → 2. Search traces for errors
|
|
|
|
# CORRECT: Follow workflow
|
|
1. Alert fires → 2. Check RED metrics → 3. Find pattern → 4. Sample traces</code></pre>
|
|
<h3>❌ No Correlation IDs</h3>
|
|
<pre><code class="lang-python"># WRONG: No request tracking across services
|
|
def call_service_a():
|
|
requests.post('http://service-a/api')
|
|
|
|
def call_service_b():
|
|
requests.post('http://service-b/api')
|
|
|
|
# CORRECT: Propagate correlation ID
|
|
def call_service(url):
|
|
headers = {'X-Correlation-ID': g.correlation_id}
|
|
requests.post(url, headers=headers)</code></pre>
|
|
<h3>❌ Blaming Individuals in Postmortems</h3>
|
|
<pre><code class="lang-markdown"># WRONG: Blame-focused
|
|
## Root Cause
|
|
Bob deployed buggy code without testing.
|
|
|
|
# CORRECT: System-focused
|
|
## Root Cause
|
|
Deployment pipeline lacked integration tests, allowing buggy SDK update to reach production.
|
|
|
|
## Action Items
|
|
- Add integration tests to CI/CD pipeline
|
|
- Implement mandatory code review for dependency updates</code></pre>
|
|
<h3>❌ No Follow-up on Action Items</h3>
|
|
<pre><code class="lang-markdown"># WRONG: No ownership or deadlines
|
|
## Action Items
|
|
- Add alerts
|
|
- Fix bug
|
|
- Update runbook
|
|
|
|
# CORRECT: Clear ownership and deadlines
|
|
## Action Items
|
|
1. **[P0]** Add memory alerts (Owner: @alice, Due: 2025-10-28)
|
|
2. **[P1]** Fix connection leak (Owner: @bob, Due: 2025-11-02)
|
|
3. **[P2]** Update runbook (Owner: @charlie, Due: 2025-10-30)</code></pre>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><strong>distributed-tracing.md</strong> - Deep dive into distributed tracing</li><li><strong>metrics-instrumentation.md</strong> - Setting up RED metrics</li><li><strong>structured-logging.md</strong> - Effective log formatting for debugging</li><li><strong>opentelemetry-integration.md</strong> - Correlating traces, metrics, logs</li><li><strong>alerting-strategy.md</strong> - Setting up effective alerts</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-26 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/observability-production-incident-debugging.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |