mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
659 lines
28 KiB
HTML
659 lines
28 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>debugging-container-debugging</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/debugging-container-debugging.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/debugging-container-debugging">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/debugging-container-debugging.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>debugging-container-debugging</h1><p class="lede">Debugging applications inside containers using docker exec, kubectl debug, ephemeral containers, and distroless debugging techniques</p><h1>Container Debugging</h1>
|
|
<p><strong>Scope</strong>: Docker debugging, Kubernetes ephemeral containers, kubectl debug, debug vs production images, distroless debugging, sidecar patterns</p>
|
|
<p><strong>Lines</strong>: 410</p>
|
|
<p><strong>Last Updated</strong>: 2025-10-26</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Use this skill when:</p>
|
|
<ul><li>Debugging applications running in Docker containers</li><li>Troubleshooting Kubernetes pods without debug tools</li><li>Investigating networking issues in containerized environments</li><li>Analyzing resource constraints (CPU, memory limits)</li><li>Debugging distroless or minimal container images</li><li>Inspecting container filesystem and processes</li><li>Troubleshooting container startup failures</li><li>Debugging multi-container pods (sidecars, init containers)</li></ul>
|
|
<p><strong>Don't use</strong> for:</p>
|
|
<ul><li>Local development debugging (use IDE debuggers)</li><li>Applications running directly on host (use standard debugging)</li><li>Simple logging issues (check container logs first)</li></ul>
|
|
<hr>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Container Debugging Challenges</h3>
|
|
<p><strong>Why containers are harder to debug</strong>:</p>
|
|
<ol><li><strong>Immutable infrastructure</strong>: Can't install debug tools at runtime</li><li><strong>Minimal images</strong>: Production images lack shells, utilities</li><li><strong>Ephemeral nature</strong>: Containers restart, losing state</li><li><strong>Network isolation</strong>: Different networking namespace</li><li><strong>Resource limits</strong>: CPU/memory constraints affect behavior</li></ol>
|
|
<h3>Debug Image vs Production Image</h3>
|
|
<pre><code class="lang-dockerfile"># Production image: Minimal, secure
|
|
FROM gcr.io/distroless/python3-debian11
|
|
COPY app.py /app/
|
|
CMD ["/app/app.py"]
|
|
# No shell, no package manager, no debug tools!
|
|
|
|
# Debug image: Full tooling
|
|
FROM python:3.11-slim
|
|
RUN apt-get update && apt-get install -y \
|
|
curl vim strace tcpdump netcat
|
|
COPY app.py /app/
|
|
CMD ["/app/app.py"]</code></pre>
|
|
<h3>Container Debugging Techniques</h3>
|
|
<pre><code>Layer 1: Non-invasive (read-only inspection)
|
|
├─ docker logs / kubectl logs
|
|
├─ docker inspect / kubectl describe
|
|
└─ docker stats / kubectl top
|
|
|
|
Layer 2: Shell access
|
|
├─ docker exec -it <container> /bin/sh
|
|
├─ kubectl exec -it <pod> -- /bin/sh
|
|
└─ nsenter (host-level container access)
|
|
|
|
Layer 3: Ephemeral debugging
|
|
├─ kubectl debug (ephemeral container)
|
|
├─ Debug sidecar containers
|
|
└─ docker run with shared volumes
|
|
|
|
Layer 4: Advanced diagnostics
|
|
├─ strace (syscall tracing)
|
|
├─ tcpdump (network analysis)
|
|
└─ gdb (attach to running process)</code></pre>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Docker Container Debugging</h3>
|
|
<pre><code class="lang-bash"># 1. Check container status
|
|
docker ps -a # Show all containers (including stopped)
|
|
docker inspect <container_id> # Full container metadata
|
|
|
|
# 2. View logs
|
|
docker logs <container_id>
|
|
docker logs -f <container_id> # Follow logs
|
|
docker logs --tail 100 <container_id> # Last 100 lines
|
|
docker logs --since 10m <container_id> # Last 10 minutes
|
|
|
|
# 3. Execute commands inside container
|
|
docker exec -it <container_id> /bin/sh
|
|
docker exec -it <container_id> /bin/bash # If bash available
|
|
|
|
# 4. Inspect filesystem
|
|
docker exec <container_id> ls -la /app
|
|
docker exec <container_id> cat /app/config.yaml
|
|
|
|
# 5. Check resource usage
|
|
docker stats <container_id> # Real-time stats
|
|
docker top <container_id> # Running processes
|
|
|
|
# 6. Copy files from container
|
|
docker cp <container_id>:/app/logs/error.log ./error.log
|
|
|
|
# 7. Attach to running container (see stdout)
|
|
docker attach <container_id>
|
|
|
|
# 8. Run debug tools (if not in image)
|
|
# Mount debug tools from host
|
|
docker run -it --pid=container:<container_id> \
|
|
--net=container:<container_id> \
|
|
--cap-add sys_admin \
|
|
nicolaka/netshoot</code></pre>
|
|
<h3>Pattern 2: Kubernetes Pod Debugging</h3>
|
|
<pre><code class="lang-bash"># 1. Check pod status
|
|
kubectl get pods -n <namespace>
|
|
kubectl describe pod <pod_name> -n <namespace>
|
|
|
|
# 2. View logs
|
|
kubectl logs <pod_name> -n <namespace>
|
|
kubectl logs <pod_name> -c <container_name> -n <namespace> # Multi-container pod
|
|
kubectl logs -f <pod_name> -n <namespace> # Follow logs
|
|
kubectl logs --previous <pod_name> # Previous container (after crash)
|
|
|
|
# 3. Execute commands in pod
|
|
kubectl exec -it <pod_name> -n <namespace> -- /bin/sh
|
|
kubectl exec -it <pod_name> -c <container_name> -- /bin/sh # Specific container
|
|
|
|
# 4. Port forwarding (access pod from localhost)
|
|
kubectl port-forward <pod_name> 8080:80 -n <namespace>
|
|
|
|
# 5. Check resource usage
|
|
kubectl top pod <pod_name> -n <namespace>
|
|
kubectl top pods -n <namespace> --sort-by=memory
|
|
|
|
# 6. Copy files from pod
|
|
kubectl cp <namespace>/<pod_name>:/app/config.yaml ./config.yaml
|
|
kubectl cp <namespace>/<pod_name>:/app/logs ./logs -c <container_name>
|
|
|
|
# 7. View events
|
|
kubectl get events -n <namespace> --sort-by='.lastTimestamp'
|
|
kubectl describe pod <pod_name> -n <namespace> | grep -A 10 Events</code></pre>
|
|
<h3>Pattern 3: Ephemeral Debug Containers (Kubernetes 1.23+)</h3>
|
|
<pre><code class="lang-bash"># Create ephemeral debug container with all tools
|
|
kubectl debug <pod_name> -n <namespace> \
|
|
-it --image=nicolaka/netshoot \
|
|
--target=<container_name>
|
|
|
|
# Debug with specific debug image
|
|
kubectl debug <pod_name> -n <namespace> \
|
|
-it --image=busybox:1.35 \
|
|
--target=app-container
|
|
|
|
# Debug by copying pod and replacing image
|
|
kubectl debug <pod_name> -n <namespace> \
|
|
-it --copy-to=debug-pod \
|
|
--container=app \
|
|
--image=python:3.11-slim
|
|
|
|
# Debug distroless container
|
|
kubectl debug <pod_name> -n <namespace> \
|
|
-it --image=busybox:1.35 \
|
|
--target=distroless-app \
|
|
--share-processes
|
|
|
|
# Example: Debug container with full tooling
|
|
kubectl debug my-app-pod -n production \
|
|
-it --image=ubuntu:22.04 \
|
|
--target=app -- /bin/bash
|
|
|
|
# Inside debug container:
|
|
# - Shared PID namespace: can see target process
|
|
# - Shared network namespace: same network stack
|
|
# - Shared filesystem: can inspect volumes</code></pre>
|
|
<p><strong>Debug container example</strong> (Python app in distroless):</p>
|
|
<pre><code class="lang-yaml"># Original pod (distroless, no shell)
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: my-app
|
|
spec:
|
|
containers:
|
|
- name: app
|
|
image: gcr.io/distroless/python3
|
|
command: ["/app/main.py"]
|
|
|
|
# Debug with ephemeral container
|
|
# kubectl debug my-app -it --image=python:3.11-slim --target=app
|
|
# Now you can:
|
|
# - ps aux (see processes)
|
|
# - ls -la /proc/<pid>/fd (open file descriptors)
|
|
# - cat /proc/<pid>/environ (environment variables)
|
|
# - strace -p <pid> (trace syscalls)</code></pre>
|
|
<h3>Pattern 4: nsenter (Host-Level Container Access)</h3>
|
|
<pre><code class="lang-bash"># Get container PID
|
|
docker inspect --format '{{.State.Pid}}' <container_id>
|
|
# Or:
|
|
docker inspect <container_id> | jq '.[0].State.Pid'
|
|
|
|
# Enter container namespaces from host
|
|
PID=$(docker inspect --format '{{.State.Pid}}' <container_id>)
|
|
|
|
# Enter all namespaces
|
|
nsenter -t $PID -m -u -i -n -p /bin/sh
|
|
|
|
# Enter specific namespaces
|
|
nsenter -t $PID -n ip addr # Network namespace
|
|
nsenter -t $PID -m ls /app # Mount namespace
|
|
nsenter -t $PID -p ps aux # PID namespace
|
|
|
|
# Use nsenter with Kubernetes (requires node access)
|
|
# 1. Get node where pod is running
|
|
kubectl get pod <pod_name> -o wide
|
|
|
|
# 2. SSH to node
|
|
ssh <node_name>
|
|
|
|
# 3. Find container ID
|
|
crictl ps | grep <pod_name>
|
|
|
|
# 4. Get container PID
|
|
PID=$(crictl inspect <container_id> | jq '.info.pid')
|
|
|
|
# 5. Enter container
|
|
nsenter -t $PID -m -u -i -n -p /bin/sh</code></pre>
|
|
<h3>Pattern 5: Debug Sidecar Pattern</h3>
|
|
<pre><code class="lang-yaml"># Deployment with debug sidecar
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: my-app
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: my-app
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: my-app
|
|
spec:
|
|
shareProcessNamespace: true # Share PID namespace
|
|
containers:
|
|
# Main application (distroless)
|
|
- name: app
|
|
image: gcr.io/distroless/python3
|
|
command: ["/app/main.py"]
|
|
volumeMounts:
|
|
- name: app-data
|
|
mountPath: /app/data
|
|
|
|
# Debug sidecar (full tooling)
|
|
- name: debug
|
|
image: nicolaka/netshoot
|
|
command: ["/bin/sleep", "infinity"]
|
|
volumeMounts:
|
|
- name: app-data
|
|
mountPath: /app/data
|
|
securityContext:
|
|
capabilities:
|
|
add: ["SYS_PTRACE"] # Allow attaching to processes
|
|
|
|
volumes:
|
|
- name: app-data
|
|
emptyDir: {}
|
|
|
|
# Access debug sidecar
|
|
# kubectl exec -it <pod_name> -c debug -- /bin/bash
|
|
|
|
# From debug sidecar, you can:
|
|
# - ps aux (see main app process)
|
|
# - strace -p <pid> (trace main app)
|
|
# - tcpdump -i any (capture network traffic)
|
|
# - ls /app/data (inspect shared volumes)</code></pre>
|
|
<h3>Pattern 6: Distroless Container Debugging</h3>
|
|
<pre><code class="lang-bash"># Option 1: Ephemeral container (Kubernetes 1.23+)
|
|
kubectl debug <pod_name> \
|
|
-it --image=busybox:1.35 \
|
|
--target=distroless-app \
|
|
--share-processes
|
|
|
|
# Option 2: Debug image with same app
|
|
# Build debug variant of Dockerfile
|
|
FROM python:3.11-slim AS debug
|
|
RUN apt-get update && apt-get install -y \
|
|
curl vim strace tcpdump netcat procps
|
|
COPY app/ /app/
|
|
CMD ["/app/main.py"]
|
|
|
|
FROM gcr.io/distroless/python3 AS production
|
|
COPY app/ /app/
|
|
CMD ["/app/main.py"]
|
|
|
|
# Deploy debug image temporarily
|
|
kubectl set image deployment/my-app app=my-app:debug
|
|
|
|
# Rollback to production after debugging
|
|
kubectl rollout undo deployment/my-app
|
|
|
|
# Option 3: Copy files from running container
|
|
kubectl cp <pod_name>:/app/config.yaml ./config.yaml
|
|
kubectl exec <pod_name> -- cat /app/logs/error.log > error.log
|
|
|
|
# Option 4: Use docker multi-stage with debug target
|
|
docker build --target debug -t my-app:debug .
|
|
docker run -it my-app:debug /bin/bash</code></pre>
|
|
<h3>Pattern 7: Network Debugging in Containers</h3>
|
|
<pre><code class="lang-bash"># Debugging DNS
|
|
kubectl exec -it <pod_name> -- nslookup kubernetes.default
|
|
kubectl exec -it <pod_name> -- cat /etc/resolv.conf
|
|
|
|
# If no nslookup, use debug container
|
|
kubectl debug <pod_name> -it --image=nicolaka/netshoot
|
|
# Inside debug container:
|
|
dig kubernetes.default.svc.cluster.local
|
|
nslookup google.com
|
|
|
|
# Test connectivity
|
|
kubectl exec -it <pod_name> -- wget -O- http://service-name:8080/health
|
|
kubectl exec -it <pod_name> -- nc -zv service-name 8080
|
|
|
|
# Capture network traffic (requires privileged)
|
|
kubectl exec -it <pod_name> -- tcpdump -i any -w /tmp/capture.pcap
|
|
kubectl cp <pod_name>:/tmp/capture.pcap ./capture.pcap
|
|
|
|
# Analyze in Wireshark
|
|
wireshark capture.pcap
|
|
|
|
# Check network policies
|
|
kubectl get networkpolicies -n <namespace>
|
|
kubectl describe networkpolicy <policy_name>
|
|
|
|
# Test pod-to-pod connectivity
|
|
# Terminal 1: Start server
|
|
kubectl run test-server --image=nginx --port=80
|
|
|
|
# Terminal 2: Test from client
|
|
kubectl run test-client -it --rm --image=busybox -- wget -O- test-server</code></pre>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Essential Debug Images</h3>
|
|
<p>| Image | Use Case | Size | Tools | |-------|----------|------|-------| | <strong>nicolaka/netshoot</strong> | Network debugging | ~400MB | curl, tcpdump, nslookup, iperf, netcat | | <strong>busybox:1.35</strong> | Minimal debugging | ~5MB | sh, wget, nc (basic utils) | | <strong>ubuntu:22.04</strong> | Full debugging | ~80MB | apt, full shell, build tools | | <strong>alpine:3.18</strong> | Lightweight + package manager | ~7MB | apk, sh, basic utils |</p>
|
|
<h3>Docker Debug Commands</h3>
|
|
<pre><code class="lang-bash"># Logs
|
|
docker logs <container> [-f] [--tail N] [--since TIME]
|
|
|
|
# Stats
|
|
docker stats [<container>]
|
|
docker top <container>
|
|
|
|
# Filesystem
|
|
docker exec <container> ls /app
|
|
docker cp <container>:/path/to/file ./file
|
|
|
|
# Shell
|
|
docker exec -it <container> /bin/sh
|
|
docker run -it --entrypoint /bin/sh <image>
|
|
|
|
# Network
|
|
docker inspect <container> | jq '.[0].NetworkSettings'
|
|
docker network inspect <network>
|
|
|
|
# Processes
|
|
docker exec <container> ps aux
|
|
nsenter -t <pid> -p ps aux</code></pre>
|
|
<h3>Kubernetes Debug Commands</h3>
|
|
<pre><code class="lang-bash"># Logs
|
|
kubectl logs <pod> [-c <container>] [-f] [--previous]
|
|
|
|
# Stats
|
|
kubectl top pod <pod>
|
|
kubectl top nodes
|
|
|
|
# Filesystem
|
|
kubectl exec <pod> -- ls /app
|
|
kubectl cp <pod>:/path/to/file ./file
|
|
|
|
# Shell
|
|
kubectl exec -it <pod> -- /bin/sh
|
|
kubectl debug <pod> -it --image=busybox
|
|
|
|
# Network
|
|
kubectl port-forward <pod> 8080:80
|
|
kubectl get svc,endpoints
|
|
|
|
# Events
|
|
kubectl get events --sort-by='.lastTimestamp'
|
|
kubectl describe pod <pod></code></pre>
|
|
<h3>strace Cheat Sheet</h3>
|
|
<pre><code class="lang-bash"># Trace syscalls of running process
|
|
strace -p <pid>
|
|
|
|
# Trace specific syscalls
|
|
strace -e trace=open,read,write -p <pid>
|
|
|
|
# Trace with timestamps
|
|
strace -tt -p <pid>
|
|
|
|
# Count syscalls
|
|
strace -c -p <pid>
|
|
|
|
# Trace new process from start
|
|
strace -f python app.py
|
|
|
|
# Save to file
|
|
strace -o /tmp/trace.log -p <pid></code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>❌ Installing Debug Tools in Production Images</h3>
|
|
<pre><code class="lang-dockerfile"># WRONG: Bloated production image
|
|
FROM python:3.11-slim
|
|
RUN apt-get update && apt-get install -y \
|
|
vim curl tcpdump strace gdb # Increases attack surface!
|
|
COPY app.py /app/
|
|
CMD ["python", "/app/app.py"]
|
|
|
|
# CORRECT: Multi-stage with debug target
|
|
FROM python:3.11-slim AS base
|
|
COPY app.py /app/
|
|
|
|
FROM base AS debug
|
|
RUN apt-get update && apt-get install -y vim curl strace
|
|
|
|
FROM base AS production
|
|
CMD ["python", "/app/app.py"]</code></pre>
|
|
<h3>❌ Running Containers as Root</h3>
|
|
<pre><code class="lang-yaml"># WRONG: Root user in container
|
|
containers:
|
|
- name: app
|
|
image: my-app:latest
|
|
# Runs as root by default!
|
|
|
|
# CORRECT: Non-root user
|
|
containers:
|
|
- name: app
|
|
image: my-app:latest
|
|
securityContext:
|
|
runAsUser: 1000
|
|
runAsNonRoot: true
|
|
allowPrivilegeEscalation: false</code></pre>
|
|
<h3>❌ Debugging with Modified Code</h3>
|
|
<pre><code class="lang-bash"># WRONG: Edit code in running container
|
|
kubectl exec -it <pod> -- vi /app/main.py # Changes lost on restart!
|
|
|
|
# CORRECT: Rebuild image with changes
|
|
docker build -t my-app:debug .
|
|
kubectl set image deployment/my-app app=my-app:debug</code></pre>
|
|
<h3>❌ Forgetting Resource Limits</h3>
|
|
<pre><code class="lang-yaml"># WRONG: No resource limits (can OOM)
|
|
containers:
|
|
- name: app
|
|
image: my-app:latest
|
|
|
|
# CORRECT: Define limits
|
|
containers:
|
|
- name: app
|
|
image: my-app:latest
|
|
resources:
|
|
requests:
|
|
memory: "128Mi"
|
|
cpu: "100m"
|
|
limits:
|
|
memory: "256Mi"
|
|
cpu: "500m"</code></pre>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><strong>containers/dockerfile-optimization.md</strong> - Multi-stage builds for debug images</li><li><strong>debugging/network-debugging.md</strong> - tcpdump, DNS, connectivity testing</li><li><strong>infrastructure/kubernetes-basics.md</strong> - Pod lifecycle, namespaces</li><li><strong>observability/structured-logging.md</strong> - Container logging best practices</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-26 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/debugging-container-debugging.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |