mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
665 lines
29 KiB
HTML
665 lines
29 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>ebpf-tracing-observability</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/ebpf-tracing-observability.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/ebpf-tracing-observability">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/ebpf-tracing-observability.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>ebpf-tracing-observability</h1><p class="lede">Using eBPF for tracing with bpftrace, kprobes, uprobes, tracepoints, and performance analysis</p><h1>eBPF Tracing and Observability</h1>
|
|
<p><strong>Scope</strong>: System tracing with bpftrace, kprobes/uprobes, tracepoints, perf events, and latency analysis patterns <strong>Lines</strong>: ~360 <strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Analyzing system performance and latency issues</li><li>Tracing kernel function calls with kprobes</li><li>Tracing user-space function calls with uprobes</li><li>Using bpftrace for ad-hoc system analysis</li><li>Monitoring function call frequencies and durations</li><li>Debugging production issues with minimal overhead</li><li>Collecting stack traces for profiling</li><li>Creating custom observability tools</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Concept 1: bpftrace Language</h3>
|
|
<p><strong>One-liners</strong>: Quick system insights</p>
|
|
<pre><code class="lang-bash"># Trace all syscalls by process
|
|
bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); }'
|
|
|
|
# Show file opens
|
|
bpftrace -e 'tracepoint:syscalls:sys_enter_openat { printf("%s: %s\n", comm, str(args->filename)); }'
|
|
|
|
# Measure syscall latency
|
|
bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @start[tid] = nsecs; }
|
|
tracepoint:raw_syscalls:sys_exit /@start[tid]/ {
|
|
@ns = hist(nsecs - @start[tid]);
|
|
delete(@start[tid]);
|
|
}'
|
|
|
|
# Top processes by CPU
|
|
bpftrace -e 'profile:hz:99 { @[comm] = count(); }'
|
|
|
|
# Block I/O size distribution
|
|
bpftrace -e 'tracepoint:block:block_rq_issue { @bytes = hist(args->bytes); }'</code></pre>
|
|
<h3>Concept 2: kprobes and kretprobes</h3>
|
|
<p><strong>kprobe</strong>: Trace kernel function entry <strong>kretprobe</strong>: Trace kernel function exit (return value)</p>
|
|
<pre><code class="lang-bash"># bpftrace: Trace function entry
|
|
bpftrace -e 'kprobe:do_sys_open { printf("Process %s opening file\n", comm); }'
|
|
|
|
# Trace function with arguments
|
|
bpftrace -e 'kprobe:do_sys_open {
|
|
printf("%s called do_sys_open(dfd=%d, filename=%s)\n",
|
|
comm, arg0, str(arg1));
|
|
}'
|
|
|
|
# Trace function return value
|
|
bpftrace -e 'kretprobe:do_sys_open {
|
|
printf("do_sys_open returned: %d\n", retval);
|
|
}'
|
|
|
|
# Measure function duration
|
|
bpftrace -e 'kprobe:vfs_read { @start[tid] = nsecs; }
|
|
kretprobe:vfs_read /@start[tid]/ {
|
|
@duration_us = hist((nsecs - @start[tid]) / 1000);
|
|
delete(@start[tid]);
|
|
}'</code></pre>
|
|
<p><strong>C program</strong> (libbpf):</p>
|
|
<pre><code class="lang-c">#include <vmlinux.h>
|
|
#include <bpf/bpf_helpers.h>
|
|
#include <bpf/bpf_tracing.h>
|
|
|
|
SEC("kprobe/do_sys_open")
|
|
int BPF_KPROBE(trace_open_entry, int dfd, const char __user *filename) {
|
|
char fname[256];
|
|
u32 pid = bpf_get_current_pid_tgid() >> 32;
|
|
char comm[16];
|
|
|
|
bpf_get_current_comm(&comm, sizeof(comm));
|
|
bpf_probe_read_user_str(&fname, sizeof(fname), filename);
|
|
|
|
bpf_printk("PID %d (%s) opening: %s\n", pid, comm, fname);
|
|
return 0;
|
|
}
|
|
|
|
SEC("kretprobe/do_sys_open")
|
|
int BPF_KRETPROBE(trace_open_exit, int ret) {
|
|
u32 pid = bpf_get_current_pid_tgid() >> 32;
|
|
bpf_printk("PID %d open returned: %d\n", pid, ret);
|
|
return 0;
|
|
}
|
|
|
|
char LICENSE[] SEC("license") = "GPL";</code></pre>
|
|
<h3>Concept 3: uprobes for User-Space Tracing</h3>
|
|
<p><strong>Purpose</strong>: Trace user-space functions (applications, libraries)</p>
|
|
<pre><code class="lang-bash"># Trace malloc calls
|
|
bpftrace -e 'uprobe:/lib/x86_64-linux-gnu/libc.so.6:malloc {
|
|
printf("%s malloc(%d)\n", comm, arg0);
|
|
}'
|
|
|
|
# Trace application function
|
|
bpftrace -e 'uprobe:/usr/bin/python3.9:PyEval_EvalFrameEx {
|
|
printf("Python frame execution in %s\n", comm);
|
|
}'
|
|
|
|
# Trace with return value
|
|
bpftrace -e 'uretprobe:/lib/x86_64-linux-gnu/libc.so.6:malloc {
|
|
printf("malloc returned: %p\n", retval);
|
|
}'
|
|
|
|
# Trace custom application
|
|
bpftrace -e 'uprobe:./myapp:process_request {
|
|
printf("Request processing started\n");
|
|
}
|
|
uretprobe:./myapp:process_request {
|
|
printf("Request took: %d ns\n", retval);
|
|
}'</code></pre>
|
|
<p><strong>C program</strong> (libbpf):</p>
|
|
<pre><code class="lang-c">SEC("uprobe/lib/x86_64-linux-gnu/libc.so.6:malloc")
|
|
int BPF_KPROBE(trace_malloc, size_t size) {
|
|
u32 pid = bpf_get_current_pid_tgid() >> 32;
|
|
char comm[16];
|
|
bpf_get_current_comm(&comm, sizeof(comm));
|
|
|
|
bpf_printk("PID %d (%s) malloc(%lu)\n", pid, comm, size);
|
|
return 0;
|
|
}
|
|
|
|
SEC("uretprobe/lib/x86_64-linux-gnu/libc.so.6:malloc")
|
|
int BPF_KRETPROBE(trace_malloc_ret, void *ret) {
|
|
u32 pid = bpf_get_current_pid_tgid() >> 32;
|
|
bpf_printk("PID %d malloc returned: %p\n", pid, ret);
|
|
return 0;
|
|
}</code></pre>
|
|
<h3>Concept 4: Tracepoints - Stable ABI</h3>
|
|
<p><strong>Advantages</strong>: Stable across kernel versions, no symbol lookup</p>
|
|
<pre><code class="lang-bash"># List available tracepoints
|
|
bpftrace -l 'tracepoint:*' | grep syscalls
|
|
|
|
# Trace syscall entry
|
|
bpftrace -e 'tracepoint:syscalls:sys_enter_write {
|
|
printf("write(%d, ..., %d)\n", args->fd, args->count);
|
|
}'
|
|
|
|
# Trace scheduler events
|
|
bpftrace -e 'tracepoint:sched:sched_switch {
|
|
printf("%s -> %s\n", args->prev_comm, args->next_comm);
|
|
}'
|
|
|
|
# Trace network transmit
|
|
bpftrace -e 'tracepoint:net:netif_rx {
|
|
printf("RX %d bytes on %s\n", args->len, args->name);
|
|
}'</code></pre>
|
|
<p><strong>C program</strong>:</p>
|
|
<pre><code class="lang-c">#include <vmlinux.h>
|
|
#include <bpf/bpf_helpers.h>
|
|
|
|
SEC("tracepoint/syscalls/sys_enter_openat")
|
|
int trace_openat(struct trace_event_raw_sys_enter *ctx) {
|
|
char filename[256];
|
|
char comm[16];
|
|
|
|
bpf_get_current_comm(&comm, sizeof(comm));
|
|
bpf_probe_read_user_str(&filename, sizeof(filename),
|
|
(void *)ctx->args[1]);
|
|
|
|
bpf_printk("%s opening: %s\n", comm, filename);
|
|
return 0;
|
|
}
|
|
|
|
SEC("tracepoint/sched/sched_process_exec")
|
|
int trace_exec(struct trace_event_raw_sched_process_exec *ctx) {
|
|
char comm[16];
|
|
bpf_get_current_comm(&comm, sizeof(comm));
|
|
|
|
bpf_printk("Exec: %s (PID %d)\n", comm, ctx->pid);
|
|
return 0;
|
|
}
|
|
|
|
char LICENSE[] SEC("license") = "GPL";</code></pre>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Latency Analysis</h3>
|
|
<p><strong>When to use</strong>: Identify slow operations</p>
|
|
<pre><code class="lang-bash"># Histogram of read() latency
|
|
bpftrace -e '
|
|
tracepoint:syscalls:sys_enter_read {
|
|
@start[tid] = nsecs;
|
|
}
|
|
|
|
tracepoint:syscalls:sys_exit_read /@start[tid]/ {
|
|
@latency_us = hist((nsecs - @start[tid]) / 1000);
|
|
delete(@start[tid]);
|
|
}
|
|
|
|
interval:s:5 {
|
|
print(@latency_us);
|
|
clear(@latency_us);
|
|
}'</code></pre>
|
|
<p><strong>C implementation</strong> (more control):</p>
|
|
<pre><code class="lang-c">struct {
|
|
__uint(type, BPF_MAP_TYPE_HASH);
|
|
__uint(max_entries, 10240);
|
|
__type(key, u32);
|
|
__type(value, u64);
|
|
} start_times SEC(".maps");
|
|
|
|
struct {
|
|
__uint(type, BPF_MAP_TYPE_HISTOGRAM);
|
|
__uint(max_entries, 64);
|
|
} latency_hist SEC(".maps");
|
|
|
|
SEC("tracepoint/syscalls/sys_enter_read")
|
|
int trace_read_enter(struct trace_event_raw_sys_enter *ctx) {
|
|
u32 tid = bpf_get_current_pid_tgid() & 0xFFFFFFFF;
|
|
u64 ts = bpf_ktime_get_ns();
|
|
bpf_map_update_elem(&start_times, &tid, &ts, BPF_ANY);
|
|
return 0;
|
|
}
|
|
|
|
SEC("tracepoint/syscalls/sys_exit_read")
|
|
int trace_read_exit(struct trace_event_raw_sys_exit *ctx) {
|
|
u32 tid = bpf_get_current_pid_tgid() & 0xFFFFFFFF;
|
|
u64 *tsp = bpf_map_lookup_elem(&start_times, &tid);
|
|
|
|
if (tsp) {
|
|
u64 delta_us = (bpf_ktime_get_ns() - *tsp) / 1000;
|
|
bpf_map_update_elem(&latency_hist, &delta_us, NULL, BPF_ANY);
|
|
bpf_map_delete_elem(&start_times, &tid);
|
|
}
|
|
|
|
return 0;
|
|
}</code></pre>
|
|
<h3>Pattern 2: Function Counting</h3>
|
|
<p><strong>Use case</strong>: Find hot paths in code</p>
|
|
<pre><code class="lang-bash"># Count kernel function calls
|
|
bpftrace -e 'kprobe:vfs_* { @[probe] = count(); }'
|
|
|
|
# Count by process
|
|
bpftrace -e 'kprobe:tcp_sendmsg { @sends[comm] = count(); }'
|
|
|
|
# Top 10 callers
|
|
bpftrace -e 'kprobe:do_sys_open { @[comm] = count(); }
|
|
interval:s:10 { print(@, 10); clear(@); }'</code></pre>
|
|
<p><strong>C program</strong> (per-CPU for performance):</p>
|
|
<pre><code class="lang-c">struct {
|
|
__uint(type, BPF_MAP_TYPE_PERCPU_HASH);
|
|
__uint(max_entries, 10240);
|
|
__type(key, char[16]);
|
|
__type(value, u64);
|
|
} call_counts SEC(".maps");
|
|
|
|
SEC("kprobe/tcp_sendmsg")
|
|
int count_tcp_send(struct pt_regs *ctx) {
|
|
char comm[16];
|
|
u64 zero = 0, *count;
|
|
|
|
bpf_get_current_comm(&comm, sizeof(comm));
|
|
|
|
count = bpf_map_lookup_elem(&call_counts, &comm);
|
|
if (count) {
|
|
__sync_fetch_and_add(count, 1);
|
|
} else {
|
|
bpf_map_update_elem(&call_counts, &comm, &zero, BPF_NOEXIST);
|
|
}
|
|
|
|
return 0;
|
|
}</code></pre>
|
|
<h3>Pattern 3: Stack Trace Collection</h3>
|
|
<p><strong>When to use</strong>: Profiling, identifying code paths</p>
|
|
<pre><code class="lang-bash"># User stack traces
|
|
bpftrace -e 'profile:hz:99 /comm == "myapp"/ {
|
|
@[ustack] = count();
|
|
}'
|
|
|
|
# Kernel stack traces
|
|
bpftrace -e 'kprobe:tcp_sendmsg {
|
|
@[kstack] = count();
|
|
}'
|
|
|
|
# Combined user + kernel stacks
|
|
bpftrace -e 'profile:hz:99 {
|
|
@[ustack, kstack, comm] = count();
|
|
}'</code></pre>
|
|
<p><strong>C program</strong>:</p>
|
|
<pre><code class="lang-c">struct {
|
|
__uint(type, BPF_MAP_TYPE_STACK_TRACE);
|
|
__uint(max_entries, 1024);
|
|
__uint(key_size, sizeof(u32));
|
|
__uint(value_size, PERF_MAX_STACK_DEPTH * sizeof(u64));
|
|
} stack_traces SEC(".maps");
|
|
|
|
struct {
|
|
__uint(type, BPF_MAP_TYPE_HASH);
|
|
__uint(max_entries, 10240);
|
|
__type(key, u32);
|
|
__type(value, u64);
|
|
} stack_counts SEC(".maps");
|
|
|
|
SEC("kprobe/tcp_sendmsg")
|
|
int trace_tcp_send(struct pt_regs *ctx) {
|
|
u32 stack_id = bpf_get_stackid(ctx, &stack_traces, BPF_F_USER_STACK);
|
|
u64 *count, zero = 0;
|
|
|
|
count = bpf_map_lookup_elem(&stack_counts, &stack_id);
|
|
if (count) {
|
|
__sync_fetch_and_add(count, 1);
|
|
} else {
|
|
bpf_map_update_elem(&stack_counts, &stack_id, &zero, BPF_ANY);
|
|
}
|
|
|
|
return 0;
|
|
}</code></pre>
|
|
<h3>Pattern 4: Argument Inspection</h3>
|
|
<p><strong>Use case</strong>: Understand function inputs</p>
|
|
<pre><code class="lang-bash"># Inspect syscall arguments
|
|
bpftrace -e 'tracepoint:syscalls:sys_enter_write {
|
|
printf("write(fd=%d, count=%d) by %s\n",
|
|
args->fd, args->count, comm);
|
|
}'
|
|
|
|
# String arguments
|
|
bpftrace -e 'tracepoint:syscalls:sys_enter_execve {
|
|
printf("exec: %s %s\n", comm, str(args->filename));
|
|
}'
|
|
|
|
# Struct arguments (kernel functions)
|
|
bpftrace -e 'kprobe:tcp_sendmsg {
|
|
printf("tcp_sendmsg: size=%d\n", ((struct msghdr *)arg1)->msg_iter.count);
|
|
}'</code></pre>
|
|
<h3>Pattern 5: Filtering by PID/Process</h3>
|
|
<p><strong>Use case</strong>: Focus on specific application</p>
|
|
<pre><code class="lang-bash"># Single PID
|
|
bpftrace -e 'tracepoint:syscalls:sys_enter_read /pid == 1234/ {
|
|
printf("Read by PID 1234\n");
|
|
}'
|
|
|
|
# Process name
|
|
bpftrace -e 'kprobe:vfs_read /comm == "nginx"/ {
|
|
@reads = count();
|
|
}'
|
|
|
|
# Multiple PIDs
|
|
bpftrace -e '
|
|
BEGIN { @pids[1234] = 1; @pids[5678] = 1; }
|
|
|
|
kprobe:do_sys_open /@pids[pid]/ {
|
|
printf("Monitored PID %d opened file\n", pid);
|
|
}'</code></pre>
|
|
<p><strong>C program</strong>:</p>
|
|
<pre><code class="lang-c">struct {
|
|
__uint(type, BPF_MAP_TYPE_HASH);
|
|
__uint(max_entries, 1024);
|
|
__type(key, u32);
|
|
__type(value, u8);
|
|
} filter_pids SEC(".maps");
|
|
|
|
SEC("kprobe/do_sys_open")
|
|
int trace_open_filtered(struct pt_regs *ctx) {
|
|
u32 pid = bpf_get_current_pid_tgid() >> 32;
|
|
|
|
if (!bpf_map_lookup_elem(&filter_pids, &pid))
|
|
return 0; // Skip if not in filter
|
|
|
|
char comm[16];
|
|
bpf_get_current_comm(&comm, sizeof(comm));
|
|
bpf_printk("Filtered PID %d (%s) opening file\n", pid, comm);
|
|
|
|
return 0;
|
|
}</code></pre>
|
|
<h3>Pattern 6: Time-Based Sampling</h3>
|
|
<p><strong>When to use</strong>: Reduce overhead for high-frequency events</p>
|
|
<pre><code class="lang-bash"># Sample 1% of events
|
|
bpftrace -e 'kprobe:vfs_read /(rand() % 100) < 1/ {
|
|
@sampled = count();
|
|
}'
|
|
|
|
# Sample every 100th event
|
|
bpftrace -e '
|
|
kprobe:tcp_sendmsg {
|
|
@count++;
|
|
if (@count % 100 == 0) {
|
|
printf("Sample: %s sent TCP\n", comm);
|
|
}
|
|
}'</code></pre>
|
|
<h3>Pattern 7: Multi-Function Correlation</h3>
|
|
<p><strong>Use case</strong>: Track request flow across functions</p>
|
|
<pre><code class="lang-bash">bpftrace -e '
|
|
kprobe:tcp_sendmsg {
|
|
@start[tid] = nsecs;
|
|
printf("-> tcp_sendmsg (%s)\n", comm);
|
|
}
|
|
|
|
kprobe:ip_queue_xmit /@start[tid]/ {
|
|
printf(" -> ip_queue_xmit (+%d us)\n",
|
|
(nsecs - @start[tid]) / 1000);
|
|
}
|
|
|
|
kretprobe:tcp_sendmsg /@start[tid]/ {
|
|
printf("<- tcp_sendmsg (total: %d us)\n",
|
|
(nsecs - @start[tid]) / 1000);
|
|
delete(@start[tid]);
|
|
}'</code></pre>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>bpftrace Probe Types</h3>
|
|
<pre><code>Probe Type | Syntax | Example
|
|
---------------------|-------------------------------|---------------------------
|
|
Tracepoint | tracepoint:category:name | tracepoint:syscalls:sys_enter_read
|
|
kprobe | kprobe:function | kprobe:do_sys_open
|
|
kretprobe | kretprobe:function | kretprobe:do_sys_open
|
|
uprobe | uprobe:path:function | uprobe:/bin/bash:readline
|
|
uretprobe | uretprobe:path:function | uretprobe:/lib/libc.so:malloc
|
|
Software | software:event:count | software:page-faults:
|
|
Hardware | hardware:event:count | hardware:cache-misses:
|
|
Profile | profile:hz:rate | profile:hz:99
|
|
Interval | interval:s:seconds | interval:s:5</code></pre>
|
|
<h3>Common bpftrace Variables</h3>
|
|
<pre><code>Variable | Description
|
|
----------|----------------------------------
|
|
pid | Process ID
|
|
tid | Thread ID
|
|
uid | User ID
|
|
comm | Process name
|
|
nsecs | Nanosecond timestamp
|
|
arg0-argN | Function arguments (kprobe/uprobe)
|
|
retval | Return value (kretprobe/uretprobe)</code></pre>
|
|
<h3>Key Guidelines</h3>
|
|
<pre><code>✅ DO: Use tracepoints for stable production tracing
|
|
✅ DO: Sample high-frequency events to reduce overhead
|
|
✅ DO: Clean up maps after use (delete)
|
|
✅ DO: Use filters to minimize data collection
|
|
✅ DO: Test scripts on dev systems first
|
|
|
|
❌ DON'T: Trace every event without filtering
|
|
❌ DON'T: Forget to handle missing entries in maps
|
|
❌ DON'T: Use kprobes on unstable kernel internals in production</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>Critical Violations</h3>
|
|
<pre><code class="lang-bash"># ❌ NEVER: Trace everything without filtering
|
|
bpftrace -e 'kprobe:* { printf("Called: %s\n", probe); }'
|
|
# Massive overhead, system slowdown
|
|
|
|
# ✅ CORRECT: Filter to specific probes
|
|
bpftrace -e 'kprobe:vfs_* { @[probe] = count(); }'</code></pre>
|
|
<p>❌ <strong>Unbounded tracing</strong>: System performance degradation ✅ <strong>Correct approach</strong>: Specific probes with filters</p>
|
|
<h3>Common Mistakes</h3>
|
|
<pre><code class="lang-bash"># ❌ Don't: Forget to clean up maps
|
|
bpftrace -e '
|
|
kprobe:do_sys_open { @start[tid] = nsecs; }
|
|
kretprobe:do_sys_open { /* No delete! */ }'
|
|
|
|
# ✅ Correct: Always clean up
|
|
bpftrace -e '
|
|
kprobe:do_sys_open { @start[tid] = nsecs; }
|
|
kretprobe:do_sys_open /@start[tid]/ {
|
|
delete(@start[tid]); # Prevent map growth
|
|
}'</code></pre>
|
|
<p>❌ <strong>Map bloat</strong>: Memory exhaustion over time ✅ <strong>Better</strong>: Delete entries after use</p>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>ebpf-fundamentals.md</code> - Core eBPF concepts and verifier</li><li><code>ebpf-networking.md</code> - Packet processing with XDP/TC</li><li><code>ebpf-security-monitoring.md</code> - Security observability</li><li><code>performance-tuning.md</code> - System performance optimization</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/ebpf-tracing-observability.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |