mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
517 lines
30 KiB
HTML
517 lines
30 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>engineering-incident-response</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/engineering-incident-response.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/engineering-incident-response">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/engineering-incident-response.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>engineering-incident-response</h1><p class="lede">Comprehensive incident response practices including lifecycle management, severity classification, war rooms, runbooks, blameless postmortems, and SRE metrics (MTTR, MTTD, MTTA, MTBF)</p><h1>Incident Response</h1>
|
|
<p><strong>Scope</strong>: Complete incident lifecycle from detection through postmortem, including IC role, communication protocols, escalation policies, metrics-driven improvement, and blameless culture development</p>
|
|
<p><strong>Lines</strong>: ~700</p>
|
|
<p><strong>Last Updated</strong>: 2025-10-27</p>
|
|
<p><strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Responding to service outages or degradations</li><li>Managing production incidents (SEV-1 through SEV-4)</li><li>Setting up incident response processes</li><li>Defining severity classification criteria</li><li>Creating runbooks and playbooks</li><li>Conducting postmortem meetings</li><li>Implementing on-call rotations</li><li>Measuring incident response effectiveness (MTTR, MTTD, MTTA)</li><li>Building blameless culture</li></ul>
|
|
<p>Don't use this skill for:</p>
|
|
<ul><li>Proactive monitoring setup (see <code>monitoring-observability.md</code>)</li><li>Deployment strategies (see <code>deployment-strategies.md</code>)</li><li>Security incident response (see <code>security-incident-response</code>)</li><li>Disaster recovery planning (see <code>disaster-recovery</code>)</li></ul>
|
|
<hr>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Concept 1: Incident Lifecycle</h3>
|
|
<p><strong>Definition</strong>: Structured progression from detection through resolution and learning</p>
|
|
<p><strong>Lifecycle Stages</strong>:</p>
|
|
<pre><code>Detection → Response → Mitigation → Resolution → Postmortem
|
|
↓ ↓ ↓ ↓ ↓
|
|
Alert Assemble Implement Verify Document
|
|
Team Fix Normal Lessons</code></pre>
|
|
<p><strong>Key Principles</strong>:</p>
|
|
<ol><li><strong>Customer First</strong>: Restore service before finding root cause</li><li><strong>Clear Communication</strong>: Keep stakeholders informed</li><li><strong>Document Everything</strong>: Timeline, decisions, actions</li><li><strong>Learn and Improve</strong>: Every incident is learning opportunity</li><li><strong>Blameless</strong>: Focus on systems, not individuals</li></ol>
|
|
<p><strong>Stage Objectives</strong>:</p>
|
|
<ul><li><strong>Detection</strong>: Minimize MTTD (Mean Time To Detect)</li><li><strong>Response</strong>: Fast acknowledgment and team assembly</li><li><strong>Mitigation</strong>: Reduce customer impact quickly</li><li><strong>Resolution</strong>: Restore full service functionality</li><li><strong>Postmortem</strong>: Extract lessons, prevent recurrence</li></ul>
|
|
<hr>
|
|
<h3>Concept 2: Severity Classification</h3>
|
|
<p><strong>Definition</strong>: Categorize incidents by impact to determine response urgency</p>
|
|
<p><strong>Severity Levels</strong>:</p>
|
|
<pre><code>SEV-1 (Critical):
|
|
├─ Complete outage or severe degradation
|
|
├─ >50% users affected
|
|
├─ Business-critical functions unavailable
|
|
└─ Response: Immediate, 24/7, all hands
|
|
|
|
SEV-2 (High):
|
|
├─ Significant degradation
|
|
├─ 10-50% users affected
|
|
├─ Major features impaired
|
|
└─ Response: Urgent, specialized team
|
|
|
|
SEV-3 (Medium):
|
|
├─ Minor degradation
|
|
├─ 1-10% users affected
|
|
├─ Workarounds available
|
|
└─ Response: Standard on-call
|
|
|
|
SEV-4 (Low):
|
|
├─ Minimal/no user impact
|
|
├─ Internal tools affected
|
|
├─ Cosmetic issues
|
|
└─ Response: Business hours</code></pre>
|
|
<p><strong>Classification Matrix</strong>: | Users Affected | Revenue Impact | SLA Risk | Severity | |----------------|----------------|----------|----------| | >50% | >$10K/hr | Yes | SEV-1 | | 10-50% | $1-10K/hr | Maybe | SEV-2 | | 1-10% | <$1K/hr | No | SEV-3 | | <1% | None | No | SEV-4 |</p>
|
|
<p><strong>Escalation/De-escalation</strong>:</p>
|
|
<ul><li>Escalate when impact worse than assessed</li><li>De-escalate when mitigation reduces impact</li><li>Re-assess severity every 30 minutes</li></ul>
|
|
<hr>
|
|
<h3>Concept 3: Incident Commander (IC) Role</h3>
|
|
<p><strong>Definition</strong>: Single person responsible for coordinating incident response</p>
|
|
<p><strong>IC Responsibilities</strong>:</p>
|
|
<pre><code>Coordinate: Direct technical responders
|
|
Decide: Final call on mitigation strategies
|
|
Communicate: Ensure stakeholders informed
|
|
Document: Maintain timeline
|
|
Drive: Keep team focused on restoration
|
|
Declare: Determine when resolved</code></pre>
|
|
<p><strong>IC is NOT</strong>:</p>
|
|
<ul><li>The person fixing the technical issue</li><li>The person writing all updates</li><li>Responsible for root cause (during incident)</li><li>Working on multiple tasks simultaneously</li></ul>
|
|
<p><strong>IC Decision Framework</strong>:</p>
|
|
<pre><code>Rollback? → Recent deployment + High confidence issue? → YES
|
|
→ Unsure or no deployment? → Investigate 15 min → Still bad? → ROLLBACK
|
|
|
|
Escalate? → Need expertise not on-call? → Page specialist
|
|
→ Need management decision? → Page manager
|
|
→ Need vendor help? → Open critical ticket
|
|
|
|
Resolve? → All criteria met + 15+ min stable? → YES
|
|
→ Any criteria failed? → Continue monitoring</code></pre>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: War Room Coordination</h3>
|
|
<p><strong>Problem</strong>: Need structured communication during chaotic incidents</p>
|
|
<p><strong>War Room Structure</strong>:</p>
|
|
<pre><code>Channel: #incident-YYYYMMDD-description
|
|
├─ IC: Coordinates, makes decisions
|
|
├─ Tech Lead: Hands on keyboard, implements
|
|
├─ Comms Lead: External updates
|
|
├─ Scribe: Documents timeline
|
|
└─ SMEs: Domain experts as needed
|
|
|
|
Communication Format:
|
|
[HH:MM] [@person] [TYPE] Description
|
|
|
|
Types:
|
|
- ACTION: Something done
|
|
- DECISION: Choice made
|
|
- OBSERVATION: Data/status
|
|
- QUESTION: Need info
|
|
- ANSWER: Response</code></pre>
|
|
<p><strong>Example War Room Flow</strong>:</p>
|
|
<pre><code>[14:30] @ic [DECISION] Declaring SEV-1, creating war room
|
|
[14:32] @tech-lead [OBSERVATION] Error rate 35%, started 14:20
|
|
[14:35] @tech-lead [OBSERVATION] Correlates with deployment v2.3.1
|
|
[14:37] @ic [DECISION] Rollback deployment
|
|
[14:40] @tech-lead [ACTION] Rollback complete
|
|
[14:45] @tech-lead [OBSERVATION] Error rate declining to 8%
|
|
[14:50] @ic [DECISION] Monitoring 15 min before resolving</code></pre>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>Clear roles prevent confusion</li><li>Structured updates create timeline</li><li>Decision documentation for postmortem</li><li>Stakeholders get visibility</li></ul>
|
|
<hr>
|
|
<h3>Pattern 2: Blameless Postmortem</h3>
|
|
<p><strong>Problem</strong>: Need to learn from incidents without creating fear</p>
|
|
<p><strong>Blameless Principles</strong>:</p>
|
|
<pre><code>Focus on:
|
|
✓ Systems and processes
|
|
✓ "The monitoring didn't..."
|
|
✓ "The process allowed..."
|
|
✓ Why system permitted issue
|
|
|
|
Avoid:
|
|
✗ Individual blame
|
|
✗ "Alice deployed bad code"
|
|
✗ "Bob didn't notice"
|
|
✗ Who made mistake</code></pre>
|
|
<p><strong>Postmortem Structure</strong>:</p>
|
|
<pre><code class="lang-markdown"># Postmortem: [Title]
|
|
|
|
## Executive Summary
|
|
3-sentence summary: what happened, impact, resolution
|
|
|
|
## Impact
|
|
- User impact metrics
|
|
- Business impact
|
|
- Technical metrics
|
|
|
|
## Timeline
|
|
| Time | Event |
|
|
|------|-------|
|
|
| ... | ... |
|
|
|
|
## Root Cause Analysis (Five Whys)
|
|
1. Why did X fail? → Answer
|
|
2. Why did that happen? → Answer
|
|
3. Why did that happen? → Answer
|
|
4. Why did that happen? → Answer
|
|
5. Why did that happen? → Answer
|
|
|
|
## What Went Well
|
|
- Positives to reinforce
|
|
|
|
## What Went Poorly
|
|
- Areas for improvement (blameless!)
|
|
|
|
## Action Items
|
|
| Action | Owner | Deadline | Category |
|
|
|--------|-------|----------|----------|
|
|
| ... | ... | ... | Prevent |
|
|
|
|
## Lessons Learned
|
|
Key takeaways for organization</code></pre>
|
|
<p><strong>Action Item Categories</strong>:</p>
|
|
<ul><li><strong>Prevent</strong>: Stop recurrence</li><li><strong>Detect</strong>: Find faster</li><li><strong>Respond</strong>: Mitigate faster</li><li><strong>Process</strong>: Improve workflow</li></ul>
|
|
<hr>
|
|
<h3>Pattern 3: Metrics-Driven Improvement</h3>
|
|
<p><strong>Problem</strong>: Need objective measures of incident response effectiveness</p>
|
|
<p><strong>The Four Golden Metrics</strong>:</p>
|
|
<pre><code>MTTR (Mean Time To Repair):
|
|
├─ Time from incident start to resolution
|
|
├─ Target: <1 hour SEV-1, <4 hours SEV-2
|
|
└─ Improve: Better runbooks, automation, rollback
|
|
|
|
MTTD (Mean Time To Detect):
|
|
├─ Time from issue start to detection
|
|
├─ Target: <5 minutes
|
|
└─ Improve: Synthetic monitoring, better alerts
|
|
|
|
MTTA (Mean Time To Acknowledge):
|
|
├─ Time from alert to acknowledgment
|
|
├─ Target: <5 min SEV-1, <15 min SEV-2
|
|
└─ Improve: Clear on-call, alert quality
|
|
|
|
MTBF (Mean Time Between Failures):
|
|
├─ Time between incidents
|
|
├─ Target: Increasing over time
|
|
└─ Improve: Action item completion, testing</code></pre>
|
|
<p><strong>Measurement Example</strong>:</p>
|
|
<pre><code class="lang-python"># Calculate MTTR
|
|
incidents = load_resolved_incidents()
|
|
durations = [
|
|
(inc.resolved_at - inc.started_at).total_seconds() / 60
|
|
for inc in incidents
|
|
]
|
|
mttr_minutes = mean(durations)
|
|
|
|
# By severity
|
|
sev1_mttr = mean([d for inc, d in zip(incidents, durations)
|
|
if inc.severity == 'SEV-1'])</code></pre>
|
|
<p><strong>SLO and Error Budget</strong>:</p>
|
|
<pre><code class="lang-yaml">service: api
|
|
slo:
|
|
availability: 99.9% # Max 43.8 min downtime/month
|
|
error_rate: 0.1%
|
|
|
|
error_budget:
|
|
monthly_downtime: 43.8 minutes
|
|
used_this_month: 135 minutes # 308% - EXHAUSTED
|
|
|
|
actions_when_exhausted:
|
|
- Freeze non-critical deployments
|
|
- Focus on reliability
|
|
- Postmortem all incidents
|
|
- Mandatory chaos engineering</code></pre>
|
|
<hr>
|
|
<h2>Checklist</h2>
|
|
<h3>Incident Response Checklist</h3>
|
|
<p><strong>Initial Response (0-5 min)</strong>:</p>
|
|
<ul><li>[ ] Acknowledge alert</li><li>[ ] Assess severity and impact</li><li>[ ] Create incident ticket</li><li>[ ] Notify stakeholders</li><li>[ ] Assign Incident Commander</li><li>[ ] Create war room (#incident-YYYYMMDD-description)</li><li>[ ] Begin timeline documentation</li></ul>
|
|
<p><strong>Mitigation (5-60 min)</strong>:</p>
|
|
<ul><li>[ ] Identify recent changes (deployments, configs)</li><li>[ ] Determine mitigation strategy (rollback/scale/fix)</li><li>[ ] Implement mitigation</li><li>[ ] Verify impact reduced</li><li>[ ] Provide updates every 15-30 minutes</li><li>[ ] Document all actions and decisions</li></ul>
|
|
<p><strong>Resolution</strong>:</p>
|
|
<ul><li>[ ] Error rates back to baseline</li><li>[ ] Latency within SLOs</li><li>[ ] No active customer reports</li><li>[ ] System stable 15+ minutes</li><li>[ ] All health checks passing</li><li>[ ] Dependencies verified</li><li>[ ] Send final customer notification</li><li>[ ] Schedule postmortem (24-48 hours)</li></ul>
|
|
<p><strong>Postmortem (24-72 hours)</strong>:</p>
|
|
<ul><li>[ ] Compile timeline from war room</li><li>[ ] Gather metrics and graphs</li><li>[ ] Draft postmortem document</li><li>[ ] Conduct postmortem meeting (blameless)</li><li>[ ] Define action items with owners</li><li>[ ] Publish postmortem internally</li><li>[ ] Follow up on action items</li></ul>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<p><strong>Process Anti-Patterns</strong>:</p>
|
|
<pre><code>❌ No clear IC → Multiple people making conflicting decisions
|
|
❌ Skip severity classification → Wrong response urgency
|
|
❌ Investigate before mitigating → Customers suffer longer
|
|
❌ No communication updates → Stakeholders anxious, duplicate work
|
|
❌ Resolve too early → Issue recurs immediately
|
|
❌ Skip postmortem → No learning, same issue repeats</code></pre>
|
|
<p><strong>Communication Anti-Patterns</strong>:</p>
|
|
<pre><code>❌ Blame individuals → Fear, information hiding
|
|
❌ Vague updates → Stakeholders don't understand status
|
|
❌ Irregular updates → Anxiety and escalations
|
|
❌ Over-technical → Non-engineers confused
|
|
❌ No final notification → Unclear if resolved</code></pre>
|
|
<p><strong>Metrics Anti-Patterns</strong>:</p>
|
|
<pre><code>❌ Only measure MTTR → Ignore detection and acknowledgment
|
|
❌ Don't track by severity → Can't prioritize improvements
|
|
❌ No trend analysis → Miss systemic issues
|
|
❌ Ignore action item completion → Same incidents repeat
|
|
❌ Celebrate "hero" fixes → Discourage prevention</code></pre>
|
|
<hr>
|
|
<h2>Recovery</h2>
|
|
<p><strong>When Incident Gets Worse</strong>:</p>
|
|
<pre><code>1. STOP current mitigation if making things worse
|
|
2. ROLLBACK to last known good state
|
|
3. RE-ASSESS severity (may need to escalate)
|
|
4. PAGE additional help (specialists, manager)
|
|
5. COMMUNICATE change in status
|
|
6. CONSIDER maintenance mode to stop damage</code></pre>
|
|
<p><strong>When IC Needs to Hand Off</strong>:</p>
|
|
<pre><code>1. Brief new IC on:
|
|
- Current status and impact
|
|
- What's been tried
|
|
- Current strategy
|
|
- Key people and roles
|
|
- Pending decisions
|
|
|
|
2. New IC announces role in war room
|
|
3. Old IC stays available for questions
|
|
4. Document handoff in timeline</code></pre>
|
|
<p><strong>When Postmortem Becomes Blame</strong>:</p>
|
|
<pre><code>1. Facilitator intervenes immediately
|
|
2. Reframe: "What allowed this?" not "Who did this?"
|
|
3. Focus on systems: monitoring, processes, tools
|
|
4. If continues, pause meeting
|
|
5. Reset expectations: blameless or don't continue</code></pre>
|
|
<hr>
|
|
<h2>Level 3: Resources</h2>
|
|
<p><strong>Extended Documentation</strong>: <a href="resources/REFERENCE.md">REFERENCE.md</a> (3,200+ lines)</p>
|
|
<ul><li>Complete incident lifecycle with detailed procedures</li><li>Comprehensive severity classification framework</li><li>Incident Commander playbook with decision trees</li><li>Communication protocols and templates</li><li>Detection and alerting strategies</li><li>War room setup and coordination</li><li>Runbook and playbook structures</li><li>Blameless postmortem methodology</li><li>Metrics and measurement (MTTR, MTTD, MTTA, MTBF)</li><li>On-call management best practices</li><li>Escalation policies and decision matrices</li><li>Tool integrations (PagerDuty, Slack, Statuspage)</li><li>SRE practices (chaos engineering, error budgets)</li></ul>
|
|
<p><strong>Scripts</strong>: Production-ready tools in <code>resources/scripts/</code></p>
|
|
<ul><li><code>create_incident.py</code> (600 lines): Create and track incidents with structured templates, severity classification, war room setup, and timeline management</li><li><code>analyze_mttr.py</code> (550 lines): Comprehensive MTTR/MTTD/MTTA analysis, incident patterns, trends, recommendations, and service comparisons</li><li><code>generate_postmortem.py</code> (500 lines): Generate blameless postmortem documents from incidents with templates for common scenarios (deployment, resource exhaustion, dependency failures)</li></ul>
|
|
<p><strong>Examples</strong>: Production-ready examples in <code>resources/examples/</code></p>
|
|
<ul><li><strong>templates/</strong>:</li><li><code>sev1-incident-template.md</code>: Complete SEV-1 response template with checklists, communication formats, decision trees, and role definitions</li><li><code>escalation-policy.yaml</code>: Comprehensive escalation policies for services, severity levels, time-based rules, and vendor contacts</li><li><strong>runbooks/</strong>:</li><li><code>database-connection-pool-exhausted.md</code>: Production runbook with symptoms, diagnosis steps, mitigation strategies, and prevention actions</li><li><strong>integrations/</strong>:</li><li><code>pagerduty-integration.py</code>: Full PagerDuty API integration for incident creation, updates, escalation, on-call schedules, and Events API v2</li><li><code>slack-incident-bot.py</code>: Slack bot for war room creation, status updates, role assignment, timeline tracking, and resolution notifications</li><li><strong>workflows/</strong>:</li><li><code>sev1-response-workflow.md</code>: Complete SEV-1 walkthrough from detection through postmortem with real timeline, metrics, and lessons learned</li></ul>
|
|
<p>All scripts include:</p>
|
|
<ul><li><code>--help</code> for comprehensive usage documentation</li><li><code>--json</code> output for programmatic integration</li><li>Executable permissions and proper shebang lines</li><li>Type hints and docstrings</li><li>Error handling and validation</li><li>Example usage in main block</li></ul>
|
|
<p><strong>Usage</strong>:</p>
|
|
<pre><code class="lang-bash"># Create incident with tracking
|
|
./create_incident.py create --title "High API error rate" \
|
|
--severity SEV-1 --impact "50% of users affected" --service api
|
|
|
|
# Analyze incident metrics
|
|
./analyze_mttr.py --period 30 --verbose --compare-services
|
|
|
|
# Generate postmortem from incident
|
|
./generate_postmortem.py --incident-id INC-123 \
|
|
--authors alice,bob --template deployment --output postmortem.md
|
|
|
|
# PagerDuty integration
|
|
python pagerduty-integration.py --create --service-id PXXXXXX
|
|
|
|
# Slack war room
|
|
python slack-incident-bot.py create-war-room \
|
|
--incident-id INC-123 --title "API errors" --severity SEV-1</code></pre>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>monitoring-observability.md</code>: Alerting and detection</li><li><code>deployment-strategies.md</code>: Rollback procedures</li><li><code>on-call-management.md</code>: Rotation and compensation</li><li><code>sre-practices.md</code>: SLOs, error budgets, chaos engineering</li><li><code>technical-writing.md</code>: Documentation and runbooks</li><li><code>communication-stakeholder.md</code>: Update templates</li></ul>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/engineering-incident-response.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |