mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
666 lines
35 KiB
HTML
666 lines
35 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>debugging-memory-leak-debugging</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/debugging-memory-leak-debugging.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/debugging-memory-leak-debugging">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/debugging-memory-leak-debugging.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>debugging-memory-leak-debugging</h1><p class="lede">Comprehensive memory leak detection and debugging using heap profiling tools across multiple languages</p><h1>Memory Leak Debugging</h1>
|
|
<p><strong>Scope</strong>: Heap profiling, memory leak detection patterns, and tools for Python, Go, Rust, C/C++, and .NET <strong>Lines</strong>: ~440 <strong>Last Updated</strong>: 2025-10-26 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Application memory usage grows unbounded over time</li><li>Out-of-memory errors occur in production or long-running processes</li><li>Need to profile heap allocations and identify leak sources</li><li>Investigating memory bloat in microservices or servers</li><li>Validating memory cleanup in resource-intensive applications</li><li>Debugging reference cycles or unreleased resources</li><li>Optimizing memory footprint for containerized applications</li><li>Analyzing memory patterns in Python, Go, Rust, C/C++, or .NET</li></ul>
|
|
<hr>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Concept 1: Memory Leak Types</h3>
|
|
<p><strong>Leak Categories</strong>:</p>
|
|
<ul><li><strong>Direct leaks</strong>: Allocated memory never freed (no references)</li><li><strong>Indirect leaks</strong>: Memory reachable only through leaked blocks</li><li><strong>Still-reachable</strong>: Memory with valid references but not freed at exit</li><li><strong>Definitely lost</strong>: No pointers to allocated blocks</li><li><strong>Possibly lost</strong>: Interior pointers only (offset from allocation start)</li></ul>
|
|
<p><strong>Common Causes</strong>:</p>
|
|
<ul><li>Forgotten deallocations (missing free/delete)</li><li>Reference cycles in garbage-collected languages</li><li>Event listener leaks (unsubscribed callbacks)</li><li>Cache unbounded growth</li><li>Thread-local storage leaks</li><li>Static container growth</li></ul>
|
|
<pre><code class="lang-python"># ❌ Memory leak: cached data never evicted
|
|
cache = {}
|
|
|
|
def get_user(user_id):
|
|
if user_id not in cache:
|
|
cache[user_id] = fetch_user(user_id) # Cache grows forever
|
|
return cache[user_id]
|
|
|
|
# ✅ Fixed: use LRU cache with size limit
|
|
from functools import lru_cache
|
|
|
|
@lru_cache(maxsize=1000)
|
|
def get_user(user_id):
|
|
return fetch_user(user_id)</code></pre>
|
|
<h3>Concept 2: Heap Profiling Workflow</h3>
|
|
<p><strong>Standard Process</strong>:</p>
|
|
<ol><li><strong>Establish baseline</strong>: Measure initial memory usage</li><li><strong>Run workload</strong>: Execute representative operations</li><li><strong>Capture snapshot</strong>: Take heap dump at intervals</li><li><strong>Compare snapshots</strong>: Identify growing allocations</li><li><strong>Analyze backtraces</strong>: Find allocation call stacks</li><li><strong>Fix and verify</strong>: Patch leaks, re-profile to confirm</li></ol>
|
|
<p><strong>Sampling vs Tracking</strong>:</p>
|
|
<ul><li><strong>Sampling</strong>: Periodic snapshots, lower overhead (production-safe)</li><li><strong>Tracking</strong>: Record every allocation, high overhead (dev/staging only)</li></ul>
|
|
<pre><code class="lang-bash"># Production-safe sampling (low overhead)
|
|
py-spy record -o profile.svg --duration 60 python app.py
|
|
|
|
# Development tracking (high overhead)
|
|
valgrind --leak-check=full --track-origins=yes ./binary</code></pre>
|
|
<h3>Concept 3: Leak Detection Tools by Language</h3>
|
|
<p><strong>Tool Selection Matrix</strong>:</p>
|
|
<p>| Language | Primary Tool | Secondary Tool | Production-Safe | |----------|--------------|----------------|-----------------| | C/C++ | Valgrind | AddressSanitizer | ASan only | | Python | tracemalloc | memory_profiler | tracemalloc yes | | Go | pprof heap | heapster | pprof yes | | Rust | MIRI | valgrind | No (dev only) | | .NET | dotnet-dump | PerfView | dotnet-dump yes | | Java | VisualVM | Eclipse MAT | Both yes |</p>
|
|
<p><strong>Key Characteristics</strong>:</p>
|
|
<ul><li><strong>Valgrind</strong>: Zero code changes, high overhead (10-30x slowdown)</li><li><strong>ASan</strong>: Compile-time instrumentation, moderate overhead (2x slowdown)</li><li><strong>tracemalloc</strong>: Python stdlib, snapshot comparison, minimal overhead</li><li><strong>pprof</strong>: Go runtime profiling, live process inspection</li><li><strong>MIRI</strong>: Rust interpreter, catches undefined behavior</li></ul>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Valgrind Leak Detection (C/C++)</h3>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>C/C++ applications with suspected memory leaks</li><li>Need detailed leak reports without code changes</li><li>Development/staging environments (overhead acceptable)</li></ul>
|
|
<pre><code class="lang-bash"># Full leak check with origins tracking
|
|
valgrind --leak-check=full \
|
|
--show-leak-kinds=all \
|
|
--track-origins=yes \
|
|
--log-file=valgrind.log \
|
|
./myapp arg1 arg2
|
|
|
|
# Analyze log file
|
|
grep "definitely lost" valgrind.log
|
|
grep "indirectly lost" valgrind.log
|
|
|
|
# Example output interpretation:
|
|
# ==12345== LEAK SUMMARY:
|
|
# ==12345== definitely lost: 4,096 bytes in 1 blocks
|
|
# ==12345== indirectly lost: 8,192 bytes in 2 blocks
|
|
# ==12345== possibly lost: 0 bytes in 0 blocks
|
|
# ==12345== still reachable: 512 bytes in 4 blocks</code></pre>
|
|
<p><strong>Key Options</strong>:</p>
|
|
<ul><li><code>--leak-check=full</code>: Show detailed leak backtraces</li><li><code>--track-origins=yes</code>: Track uninitialized value origins</li><li><code>--show-leak-kinds=all</code>: Show all leak types (not just definite)</li><li><code>--suppressions=file.supp</code>: Suppress known library leaks</li></ul>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>No recompilation required</li><li>Precise leak location reporting</li><li>Handles C libraries transparently</li></ul>
|
|
<h3>Pattern 2: AddressSanitizer (ASan) for C/C++</h3>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Faster leak detection than Valgrind (2x vs 10-30x overhead)</li><li>Integration testing with moderate performance impact</li><li>Compile-time instrumentation acceptable</li></ul>
|
|
<pre><code class="lang-bash"># Compile with ASan
|
|
g++ -fsanitize=address -g -O1 app.cpp -o app
|
|
# or
|
|
clang++ -fsanitize=address -g -O1 app.cpp -o app
|
|
|
|
# Run instrumented binary
|
|
ASAN_OPTIONS=detect_leaks=1:log_path=asan.log ./app
|
|
|
|
# Example ASan report:
|
|
# =================================================================
|
|
# ==12345==ERROR: LeakSanitizer: detected memory leaks
|
|
#
|
|
# Direct leak of 4096 byte(s) in 1 object(s) allocated from:
|
|
# #0 0x7f1234567890 in malloc (/lib/x86_64-linux-gnu/libasan.so.5+0x...)
|
|
# #1 0x5678901234ab in allocate_buffer app.cpp:42
|
|
# #2 0x5678901234cd in main app.cpp:123</code></pre>
|
|
<p><strong>Configuration via ASAN_OPTIONS</strong>:</p>
|
|
<pre><code class="lang-bash"># Common ASan options
|
|
export ASAN_OPTIONS="detect_leaks=1:\
|
|
fast_unwind_on_malloc=0:\
|
|
malloc_context_size=30:\
|
|
log_path=asan.log"</code></pre>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>10-15x faster than Valgrind</li><li>Catches use-after-free, buffer overflows simultaneously</li><li>Integrates with CI/CD pipelines</li></ul>
|
|
<h3>Pattern 3: Python tracemalloc</h3>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Python applications with growing memory usage</li><li>Need allocation snapshots without external tools</li><li>Production environments (low overhead)</li></ul>
|
|
<pre><code class="lang-python">import tracemalloc
|
|
import linecache
|
|
|
|
def display_top(snapshot, key_type='lineno', limit=10):
|
|
"""Display top memory allocations."""
|
|
snapshot = snapshot.filter_traces((
|
|
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
|
|
tracemalloc.Filter(False, "<unknown>"),
|
|
))
|
|
top_stats = snapshot.statistics(key_type)
|
|
|
|
print(f"Top {limit} lines:")
|
|
for index, stat in enumerate(top_stats[:limit], 1):
|
|
frame = stat.traceback[0]
|
|
filename = frame.filename
|
|
lineno = frame.lineno
|
|
print(f"#{index}: {filename}:{lineno}: {stat.size / 1024:.1f} KiB")
|
|
line = linecache.getline(filename, lineno).strip()
|
|
if line:
|
|
print(f" {line}")
|
|
|
|
# Start tracing
|
|
tracemalloc.start()
|
|
|
|
# Take snapshot before operation
|
|
snapshot1 = tracemalloc.take_snapshot()
|
|
|
|
# Run workload
|
|
run_application()
|
|
|
|
# Take snapshot after operation
|
|
snapshot2 = tracemalloc.take_snapshot()
|
|
|
|
# Compare snapshots
|
|
top_stats = snapshot2.compare_to(snapshot1, 'lineno')
|
|
|
|
print("[ Top 10 differences ]")
|
|
for stat in top_stats[:10]:
|
|
print(stat)
|
|
|
|
# Stop tracing
|
|
tracemalloc.stop()</code></pre>
|
|
<p><strong>Snapshot Comparison</strong>:</p>
|
|
<pre><code class="lang-python"># Identify memory growth between snapshots
|
|
for stat in top_stats[:10]:
|
|
print(f"{stat.size_diff / 1024:.1f} KiB | {stat.count_diff:+} allocations")
|
|
print(f" {stat.traceback.format()[0]}")</code></pre>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>Built into Python stdlib (no dependencies)</li><li>Low overhead (suitable for production)</li><li>Precise line-level allocation tracking</li></ul>
|
|
<h3>Pattern 4: Python memory_profiler</h3>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Line-by-line memory profiling needed</li><li>Debugging specific function memory usage</li><li>Development environments (higher overhead)</li></ul>
|
|
<pre><code class="lang-python"># Install: pip install memory_profiler
|
|
|
|
from memory_profiler import profile
|
|
|
|
@profile
|
|
def process_data(data):
|
|
"""Process large dataset."""
|
|
result = []
|
|
for item in data:
|
|
# This line allocates heavily
|
|
result.append(expensive_transform(item))
|
|
return result
|
|
|
|
# Run with: python -m memory_profiler script.py
|
|
# Output:
|
|
# Line # Mem usage Increment Occurrences Line Contents
|
|
# ================================================================
|
|
# 5 38.816 MiB 38.816 MiB 1 @profile
|
|
# 6 def process_data(data):
|
|
# 7 38.816 MiB 0.000 MiB 1 result = []
|
|
# 8 156.234 MiB 117.418 MiB 1001 for item in data:
|
|
# 9 156.234 MiB 0.000 MiB 1000 result.append(expensive_transform(item))
|
|
# 10 156.234 MiB 0.000 MiB 1 return result</code></pre>
|
|
<p><strong>mprof Command-Line Tool</strong>:</p>
|
|
<pre><code class="lang-bash"># Record memory usage over time
|
|
mprof run python script.py
|
|
|
|
# Plot memory usage graph
|
|
mprof plot
|
|
|
|
# Show peak memory
|
|
mprof peak</code></pre>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>Line-by-line granularity</li><li>Visual plots of memory over time</li><li>Easy decorator-based profiling</li></ul>
|
|
<h3>Pattern 5: Go pprof Heap Profiling</h3>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Go services with memory growth issues</li><li>Live production profiling (minimal overhead)</li><li>Need allocation flame graphs</li></ul>
|
|
<pre><code class="lang-go">package main
|
|
|
|
import (
|
|
"net/http"
|
|
_ "net/http/pprof" // Registers /debug/pprof/* handlers
|
|
"runtime"
|
|
)
|
|
|
|
func main() {
|
|
// Enable pprof HTTP endpoint
|
|
go func() {
|
|
http.ListenAndServe("localhost:6060", nil)
|
|
}()
|
|
|
|
// Your application code
|
|
runServer()
|
|
}
|
|
|
|
// Programmatic heap profiling
|
|
func captureHeapProfile() {
|
|
runtime.GC() // Get up-to-date statistics
|
|
f, _ := os.Create("heap.prof")
|
|
defer f.Close()
|
|
runtime.WriteHeapProfile(f)
|
|
}</code></pre>
|
|
<p><strong>Analyzing Heap Profiles</strong>:</p>
|
|
<pre><code class="lang-bash"># Capture heap profile from running service
|
|
curl http://localhost:6060/debug/pprof/heap > heap.prof
|
|
|
|
# Interactive analysis
|
|
go tool pprof heap.prof
|
|
# Commands in interactive mode:
|
|
# > top10 # Show top 10 memory consumers
|
|
# > list funcName # Show source with allocations
|
|
# > web # Open browser with call graph
|
|
|
|
# Generate flame graph
|
|
go tool pprof -http=:8080 heap.prof
|
|
|
|
# Compare two heap snapshots (find memory growth)
|
|
curl http://localhost:6060/debug/pprof/heap > heap1.prof
|
|
# ... wait and let app run ...
|
|
curl http://localhost:6060/debug/pprof/heap > heap2.prof
|
|
go tool pprof -base=heap1.prof heap2.prof</code></pre>
|
|
<p><strong>Key pprof Commands</strong>:</p>
|
|
<ul><li><code>top</code>: Show functions with highest memory usage</li><li><code>list <func></code>: Source code with allocation annotations</li><li><code>web</code>: Visual call graph in browser</li><li><code>pdf</code>: Export call graph as PDF</li></ul>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>Production-safe (sampling, low overhead)</li><li>Live process inspection</li><li>Visual call graphs and flame graphs</li></ul>
|
|
<h3>Pattern 6: Rust MIRI for Memory Debugging</h3>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Rust unsafe code memory validation</li><li>Development/testing environments only</li><li>Need to catch undefined behavior</li></ul>
|
|
<pre><code class="lang-bash"># Install MIRI
|
|
rustup +nightly component add miri
|
|
|
|
# Run tests with MIRI
|
|
cargo +nightly miri test
|
|
|
|
# Run specific binary with MIRI
|
|
cargo +nightly miri run
|
|
|
|
# Example MIRI output for use-after-free:
|
|
# error: Undefined Behavior: pointer to alloc123 was dereferenced after this allocation got freed
|
|
# --> src/main.rs:10:13
|
|
# |
|
|
# 10| println!("{}", *ptr);
|
|
# | ^^^^^^^^^^^^^^^^^ pointer to alloc123 was dereferenced after this allocation got freed</code></pre>
|
|
<p><strong>MIRI Capabilities</strong>:</p>
|
|
<ul><li>Detects use-after-free</li><li>Detects invalid pointer arithmetic</li><li>Detects data races (limited)</li><li>Validates unsafe code invariants</li></ul>
|
|
<p><strong>Limitations</strong>:</p>
|
|
<ul><li>Slow (interpreter, not profiler)</li><li>Dev/test only (cannot run in production)</li><li>Cannot analyze external C libraries</li></ul>
|
|
<h3>Pattern 7: .NET Memory Diagnostics</h3>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>.NET Core/5+ applications with memory issues</li><li>Production memory dumps analysis</li><li>Cross-platform .NET diagnostics</li></ul>
|
|
<pre><code class="lang-bash"># Install dotnet-dump
|
|
dotnet tool install --global dotnet-dump
|
|
|
|
# Capture memory dump from running process
|
|
dotnet-dump collect --process-id 12345
|
|
|
|
# Analyze dump
|
|
dotnet-dump analyze core_20251026_123456
|
|
|
|
# Inside analysis session:
|
|
> dumpheap -stat # Heap statistics by type
|
|
> dumpheap -mt <MethodTable> # Objects of specific type
|
|
> gcroot <address> # Find roots keeping object alive
|
|
> eeheap -gc # GC heap statistics
|
|
> objsize <address> # Total size including referenced objects</code></pre>
|
|
<p><strong>Example Analysis</strong>:</p>
|
|
<pre><code class="lang-bash"># Find objects consuming most memory
|
|
> dumpheap -stat
|
|
# Output shows types sorted by total size:
|
|
# MT Count TotalSize Class Name
|
|
# 00007f1234567890 50000 12800000 System.String
|
|
# 00007f1234567891 10000 10240000 MyApp.LargeObject
|
|
|
|
# Investigate specific type
|
|
> dumpheap -mt 00007f1234567891
|
|
# Shows all instances of MyApp.LargeObject
|
|
|
|
# Find what's keeping them alive
|
|
> gcroot 00007f9876543210
|
|
# Shows reference chain from GC root to object</code></pre>
|
|
<p><strong>Eclipse MAT (Memory Analyzer Tool)</strong>:</p>
|
|
<ul><li>Cross-platform Java/.NET heap dump analysis</li><li>Leak suspects report (automatic leak detection)</li><li>Dominator tree (memory ownership visualization)</li></ul>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>Production dump analysis</li><li>No application restart needed</li><li>Cross-platform support</li></ul>
|
|
<h3>Pattern 8: Heaptrack for C/C++</h3>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Linux C/C++ memory profiling</li><li>Need GUI visualization of allocations</li><li>Alternative to Valgrind with better performance</li></ul>
|
|
<pre><code class="lang-bash"># Install heaptrack
|
|
sudo apt install heaptrack heaptrack-gui
|
|
|
|
# Record heap allocations
|
|
heaptrack ./myapp arg1 arg2
|
|
# Creates heaptrack.myapp.XXXXX.gz
|
|
|
|
# Analyze with GUI
|
|
heaptrack_gui heaptrack.myapp.12345.gz
|
|
|
|
# Command-line analysis
|
|
heaptrack_print heaptrack.myapp.12345.gz</code></pre>
|
|
<p><strong>Heaptrack Features</strong>:</p>
|
|
<ul><li>Flame graph visualization</li><li>Timeline of allocations</li><li>Peak memory analysis</li><li>Call stack attribution</li><li>Leak detection</li></ul>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>Lower overhead than Valgrind</li><li>Excellent GUI for visual analysis</li><li>Timeline view shows allocation patterns</li></ul>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Tool Selection by Use Case</h3>
|
|
<pre><code>Use Case | Tool | Overhead | Production-Safe
|
|
----------------------------|---------------------------|-----------|----------------
|
|
C/C++ leak detection | Valgrind | 10-30x | No
|
|
C/C++ faster detection | AddressSanitizer | 2x | Maybe (staging)
|
|
C/C++ with GUI | heaptrack | Low | Yes
|
|
Python stdlib | tracemalloc | <5% | Yes
|
|
Python line-by-line | memory_profiler | Medium | No
|
|
Go live profiling | pprof | <3% | Yes
|
|
Rust unsafe validation | MIRI | 100x+ | No (dev only)
|
|
.NET production dumps | dotnet-dump | N/A | Yes
|
|
Java heap analysis | Eclipse MAT, VisualVM | N/A | Yes (dumps)</code></pre>
|
|
<h3>Key Guidelines</h3>
|
|
<pre><code>✅ DO: Establish memory baseline before profiling
|
|
✅ DO: Use sampling profilers in production (pprof, tracemalloc)
|
|
✅ DO: Compare heap snapshots to identify growth
|
|
✅ DO: Run leak detectors in CI/CD (ASan, Valgrind in tests)
|
|
✅ DO: Set memory limits for caches and containers
|
|
✅ DO: Profile representative workloads (not synthetic tests)
|
|
✅ DO: Suppress known library leaks in Valgrind
|
|
|
|
❌ DON'T: Run Valgrind in production (too slow)
|
|
❌ DON'T: Ignore "still-reachable" leaks (may indicate issues)
|
|
❌ DON'T: Profile debug builds (optimizations affect allocations)
|
|
❌ DON'T: Assume GC languages never leak (reference cycles do leak)
|
|
❌ DON'T: Profile with insufficient workload (leaks may not appear)
|
|
❌ DON'T: Mix profiler data from different runs (inconsistent)</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>Critical Violations</h3>
|
|
<pre><code class="lang-python"># ❌ NEVER: Unbounded cache growth
|
|
class UserService:
|
|
def __init__(self):
|
|
self.cache = {} # Grows forever
|
|
|
|
def get_user(self, user_id):
|
|
if user_id not in self.cache:
|
|
self.cache[user_id] = self.fetch_user(user_id)
|
|
return self.cache[user_id]
|
|
|
|
# ✅ CORRECT: Use LRU cache with size limit
|
|
from functools import lru_cache
|
|
|
|
class UserService:
|
|
@lru_cache(maxsize=1000)
|
|
def get_user(self, user_id):
|
|
return self.fetch_user(user_id)</code></pre>
|
|
<p>❌ <strong>Unbounded caching</strong>: Cache grows indefinitely, causing OOM ✅ <strong>Correct approach</strong>: Use LRU cache with max size, TTL eviction, or weak references</p>
|
|
<h3>Common Mistakes</h3>
|
|
<pre><code class="lang-go">// ❌ Don't: Forget to close resources in goroutines
|
|
func processFiles(files []string) {
|
|
for _, file := range files {
|
|
go func(f string) {
|
|
data, _ := os.ReadFile(f) // Leaks if goroutine never exits
|
|
process(data)
|
|
}(file)
|
|
}
|
|
}
|
|
|
|
// ✅ Correct: Use context and cleanup
|
|
func processFiles(ctx context.Context, files []string) {
|
|
var wg sync.WaitGroup
|
|
for _, file := range files {
|
|
wg.Add(1)
|
|
go func(f string) {
|
|
defer wg.Done()
|
|
data, err := os.ReadFile(f)
|
|
if err != nil {
|
|
return
|
|
}
|
|
process(data)
|
|
}(file)
|
|
}
|
|
wg.Wait()
|
|
}</code></pre>
|
|
<p>❌ <strong>Unclosed resources in goroutines</strong>: File handles, connections leak ✅ <strong>Better</strong>: Use defer, WaitGroup, and context cancellation</p>
|
|
<pre><code class="lang-c++">// ❌ Don't: Raw pointers without ownership
|
|
class Service {
|
|
std::vector<Connection*> connections; // Who owns these?
|
|
|
|
void add_connection(Connection* conn) {
|
|
connections.push_back(conn); // Leak risk
|
|
}
|
|
};
|
|
|
|
// ✅ Correct: Use smart pointers with clear ownership
|
|
class Service {
|
|
std::vector<std::unique_ptr<Connection>> connections;
|
|
|
|
void add_connection(std::unique_ptr<Connection> conn) {
|
|
connections.push_back(std::move(conn)); // Clear ownership
|
|
}
|
|
};</code></pre>
|
|
<p>❌ <strong>Raw pointer ownership unclear</strong>: Memory leaks from ambiguous ownership ✅ <strong>Better</strong>: Use unique_ptr/shared_ptr for automatic cleanup</p>
|
|
<pre><code class="lang-python"># ❌ Don't: Create reference cycles without weak references
|
|
class Node:
|
|
def __init__(self, value):
|
|
self.value = value
|
|
self.parent = None # Strong reference
|
|
self.children = []
|
|
|
|
def add_child(self, child):
|
|
child.parent = self # Cycle: parent <-> child
|
|
self.children.append(child)
|
|
|
|
# ✅ Correct: Use weak references to break cycles
|
|
import weakref
|
|
|
|
class Node:
|
|
def __init__(self, value):
|
|
self.value = value
|
|
self.parent = None # Will be weakref
|
|
self.children = []
|
|
|
|
def add_child(self, child):
|
|
child.parent = weakref.ref(self) # Weak reference
|
|
self.children.append(child)</code></pre>
|
|
<p>❌ <strong>Reference cycles</strong>: Python GC can't collect cyclic references efficiently ✅ <strong>Better</strong>: Use weakref to break cycles in parent-child relationships</p>
|
|
<pre><code class="lang-bash"># ❌ Don't: Run production with tracking profilers
|
|
valgrind ./production-server # 10-30x slowdown!
|
|
|
|
# ✅ Correct: Use sampling profilers in production
|
|
# Go
|
|
curl http://localhost:6060/debug/pprof/heap > heap.prof
|
|
|
|
# Python
|
|
python -X tracemalloc=5 app.py # Minimal overhead</code></pre>
|
|
<p>❌ <strong>Tracking profilers in production</strong>: Valgrind, MIRI unusable in production ✅ <strong>Better</strong>: Use sampling profilers (pprof, tracemalloc, dotnet-dump)</p>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>testing/performance-testing.md</code> - Memory profiling integrates with performance tests</li><li><code>observability/metrics-instrumentation.md</code> - Export memory metrics to monitoring systems</li><li><code>containers/docker-optimization.md</code> - Container memory limits and OOM debugging</li><li><code>plt/rust-memory-safety.md</code> - Rust ownership model prevents many leaks</li><li><code>debugging/crash-debugging.md</code> - OOM crashes require memory analysis</li><li><code>cicd/testing-strategy.md</code> - Integrate ASan/Valgrind into CI pipelines</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-26 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/debugging-memory-leak-debugging.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |