mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
797 lines
33 KiB
HTML
797 lines
33 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>observability-opentelemetry-integration</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/observability-opentelemetry-integration.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/observability-opentelemetry-integration">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/observability-opentelemetry-integration.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>observability-opentelemetry-integration</h1><p class="lede">Integrating OpenTelemetry for unified observability (traces, metrics, logs)</p><h1>OpenTelemetry Integration</h1>
|
|
<p><strong>Scope</strong>: OTel Collector, auto-instrumentation, manual instrumentation, traces/metrics/logs correlation, backend integrations</p>
|
|
<p><strong>Lines</strong>: 460</p>
|
|
<p><strong>Last Updated</strong>: 2025-10-26</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Use this skill when:</p>
|
|
<ul><li>Adopting OpenTelemetry for unified observability (traces, metrics, logs)</li><li>Migrating from proprietary instrumentation (Datadog, New Relic, AppDynamics)</li><li>Implementing distributed tracing across microservices</li><li>Correlating traces, metrics, and logs with exemplars</li><li>Setting up OTel Collector pipelines and processors</li><li>Auto-instrumenting applications (Python, Java, Go, Node.js)</li><li>Integrating with observability backends (Grafana, Jaeger, Datadog)</li><li>Standardizing telemetry across polyglot environments</li></ul>
|
|
<p><strong>Don't use</strong> for:</p>
|
|
<ul><li>Simple metrics-only instrumentation (use Prometheus directly)</li><li>Single-process applications without distributed components</li><li>Legacy systems that can't run OTel agents</li></ul>
|
|
<p><strong>Context (2024-2025)</strong>:</p>
|
|
<ul><li><strong>89% of organizations</strong> investing in OpenTelemetry (CNCF survey)</li><li><strong>75% still use Prometheus</strong> alongside OTel (complementary, not replacement)</li><li><strong>100% YoY increase</strong> in OTel search volume (2024-2025)</li><li><strong>45% YoY growth</strong> in OTel adoption across enterprises</li></ul>
|
|
<hr>
|
|
<h2>Core Concepts</h2>
|
|
<h3>OpenTelemetry Architecture</h3>
|
|
<pre><code>Application Code
|
|
↓
|
|
OTel SDK (auto or manual instrumentation)
|
|
↓
|
|
OTel Collector (optional but recommended)
|
|
↓ (pipelines: receivers → processors → exporters)
|
|
Backend (Jaeger, Grafana Tempo, Datadog, etc.)</code></pre>
|
|
<p><strong>Key Components</strong>:</p>
|
|
<ol><li><strong>SDK</strong>: Libraries for instrumenting code (auto or manual)</li><li><strong>Collector</strong>: Pipeline for receiving, processing, exporting telemetry</li><li><strong>Backend</strong>: Storage and visualization (Jaeger, Tempo, Prometheus, Loki)</li></ol>
|
|
<h3>Signals: Traces, Metrics, Logs</h3>
|
|
<p><strong>Traces</strong>: Request flows across services</p>
|
|
<ul><li>Spans: Individual operations with start/end times</li><li>Context propagation: W3C Trace Context headers</li><li>Use: "Why is this request slow?"</li></ul>
|
|
<p><strong>Metrics</strong>: Aggregated measurements over time</p>
|
|
<ul><li>Counters, gauges, histograms (like Prometheus)</li><li>Exemplars: Link metrics to traces</li><li>Use: "How many errors occurred?"</li></ul>
|
|
<p><strong>Logs</strong>: Discrete events with context</p>
|
|
<ul><li>Structured logs with trace/span IDs</li><li>Correlation with traces and metrics</li><li>Use: "What happened in this request?"</li></ul>
|
|
<h3>OpenTelemetry vs Prometheus</h3>
|
|
<p>| Feature | OpenTelemetry | Prometheus | |---------|--------------|------------| | <strong>Purpose</strong> | Unified telemetry (traces/metrics/logs) | Metrics-only | | <strong>Adoption</strong> | 89% investment (2024) | 75% active usage (2024) | | <strong>Model</strong> | Push (via Collector) | Pull (scrape) | | <strong>Traces</strong> | Native support | None | | <strong>Logs</strong> | Native support | None (use Loki) | | <strong>Metrics</strong> | Compatible with Prometheus | Native | | <strong>Exemplars</strong> | Yes (link metrics → traces) | Yes (since v2.26) | | <strong>Use Case</strong> | Modern cloud-native observability | Metrics-focused monitoring |</p>
|
|
<p><strong>Key Insight</strong>: OTel complements Prometheus (not replaces). Use OTel for traces/logs, export to Prometheus for metrics.</p>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: OTel Collector Configuration</h3>
|
|
<pre><code class="lang-yaml"># otel-collector-config.yaml
|
|
receivers:
|
|
otlp:
|
|
protocols:
|
|
grpc:
|
|
endpoint: 0.0.0.0:4317
|
|
http:
|
|
endpoint: 0.0.0.0:4318
|
|
|
|
# Prometheus scraper (if needed)
|
|
prometheus:
|
|
config:
|
|
scrape_configs:
|
|
- job_name: 'otel-collector'
|
|
scrape_interval: 15s
|
|
static_configs:
|
|
- targets: ['localhost:8888']
|
|
|
|
processors:
|
|
# Batch telemetry for efficiency
|
|
batch:
|
|
timeout: 10s
|
|
send_batch_size: 1024
|
|
|
|
# Sample traces (90% reduction)
|
|
probabilistic_sampler:
|
|
sampling_percentage: 10
|
|
|
|
# Add resource attributes
|
|
resource:
|
|
attributes:
|
|
- key: environment
|
|
value: production
|
|
action: insert
|
|
- key: service.version
|
|
value: ${SERVICE_VERSION}
|
|
action: insert
|
|
|
|
# Remove PII from logs
|
|
attributes:
|
|
actions:
|
|
- key: email
|
|
action: delete
|
|
- key: ssn
|
|
action: delete
|
|
|
|
exporters:
|
|
# Jaeger for traces
|
|
otlp/jaeger:
|
|
endpoint: jaeger:4317
|
|
tls:
|
|
insecure: true
|
|
|
|
# Prometheus for metrics
|
|
prometheus:
|
|
endpoint: 0.0.0.0:8889
|
|
namespace: otel
|
|
|
|
# Loki for logs
|
|
loki:
|
|
endpoint: http://loki:3100/loki/api/v1/push
|
|
labels:
|
|
resource:
|
|
service.name: "service_name"
|
|
environment: "env"
|
|
|
|
# Datadog (if using commercial backend)
|
|
datadog:
|
|
api:
|
|
site: datadoghq.com
|
|
key: ${DD_API_KEY}
|
|
|
|
service:
|
|
pipelines:
|
|
traces:
|
|
receivers: [otlp]
|
|
processors: [batch, probabilistic_sampler, resource]
|
|
exporters: [otlp/jaeger, datadog]
|
|
|
|
metrics:
|
|
receivers: [otlp, prometheus]
|
|
processors: [batch, resource]
|
|
exporters: [prometheus, datadog]
|
|
|
|
logs:
|
|
receivers: [otlp]
|
|
processors: [batch, attributes, resource]
|
|
exporters: [loki, datadog]
|
|
|
|
telemetry:
|
|
logs:
|
|
level: info
|
|
metrics:
|
|
address: 0.0.0.0:8888</code></pre>
|
|
<h3>Pattern 2: Auto-Instrumentation (Python)</h3>
|
|
<pre><code class="lang-python"># Auto-instrumentation with opentelemetry-instrument CLI
|
|
# Install:
|
|
# pip install opentelemetry-distro opentelemetry-exporter-otlp
|
|
|
|
# Run with auto-instrumentation:
|
|
# opentelemetry-instrument \
|
|
# --traces_exporter otlp \
|
|
# --metrics_exporter otlp \
|
|
# --service_name my-service \
|
|
# --exporter_otlp_endpoint http://otel-collector:4318 \
|
|
# python app.py
|
|
|
|
from flask import Flask
|
|
import logging
|
|
|
|
app = Flask(__name__)
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
@app.route('/api/users/<user_id>')
|
|
def get_user(user_id):
|
|
# Auto-instrumented by OTel
|
|
logging.info(f"Fetching user {user_id}")
|
|
|
|
# Database query (auto-traced)
|
|
user = db.query(User).filter_by(id=user_id).first()
|
|
|
|
# HTTP call (auto-traced)
|
|
response = requests.get(f'http://auth-service/validate/{user_id}')
|
|
|
|
return {"user": user.to_dict()}
|
|
|
|
if __name__ == '__main__':
|
|
app.run()</code></pre>
|
|
<p><strong>Auto-instrumentation supports</strong>:</p>
|
|
<ul><li><strong>Python</strong>: Flask, Django, FastAPI, SQLAlchemy, requests, httpx</li><li><strong>Java</strong>: Spring Boot, JDBC, Hibernate, OkHttp, gRPC</li><li><strong>Node.js</strong>: Express, Fastify, HTTP, MongoDB, MySQL</li><li><strong>Go</strong>: net/http, database/sql, gRPC (requires manual setup)</li></ul>
|
|
<h3>Pattern 3: Manual Instrumentation (Python)</h3>
|
|
<pre><code class="lang-python">from opentelemetry import trace, metrics
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
|
|
from opentelemetry.sdk.resources import Resource
|
|
|
|
# Setup tracer
|
|
resource = Resource.create({
|
|
"service.name": "order-service",
|
|
"service.version": "1.0.0",
|
|
"deployment.environment": "production"
|
|
})
|
|
|
|
trace.set_tracer_provider(TracerProvider(resource=resource))
|
|
tracer = trace.get_tracer(__name__)
|
|
|
|
span_exporter = OTLPSpanExporter(endpoint="http://otel-collector:4317")
|
|
trace.get_tracer_provider().add_span_processor(
|
|
BatchSpanProcessor(span_exporter)
|
|
)
|
|
|
|
# Setup metrics
|
|
metric_reader = PeriodicExportingMetricReader(
|
|
OTLPMetricExporter(endpoint="http://otel-collector:4317")
|
|
)
|
|
metrics.set_meter_provider(MeterProvider(
|
|
resource=resource,
|
|
metric_readers=[metric_reader]
|
|
))
|
|
meter = metrics.get_meter(__name__)
|
|
|
|
# Create metrics
|
|
order_counter = meter.create_counter(
|
|
"orders.placed",
|
|
description="Number of orders placed",
|
|
unit="1"
|
|
)
|
|
|
|
order_value_histogram = meter.create_histogram(
|
|
"orders.value",
|
|
description="Order value in USD",
|
|
unit="USD"
|
|
)
|
|
|
|
# Manual instrumentation
|
|
def place_order(order_data: dict):
|
|
with tracer.start_as_current_span("place_order") as span:
|
|
# Add attributes to span
|
|
span.set_attribute("order.id", order_data["id"])
|
|
span.set_attribute("order.total", order_data["total"])
|
|
span.set_attribute("customer.id", order_data["customer_id"])
|
|
|
|
try:
|
|
# Validate order
|
|
with tracer.start_as_current_span("validate_order"):
|
|
validate(order_data)
|
|
|
|
# Save to database
|
|
with tracer.start_as_current_span("save_to_database") as db_span:
|
|
db_span.set_attribute("db.system", "postgresql")
|
|
db_span.set_attribute("db.operation", "INSERT")
|
|
order_id = db.save_order(order_data)
|
|
|
|
# Update metrics
|
|
order_counter.add(1, {"status": "success"})
|
|
order_value_histogram.record(order_data["total"])
|
|
|
|
# Add event to span
|
|
span.add_event("Order placed successfully", {
|
|
"order.id": order_id
|
|
})
|
|
|
|
return order_id
|
|
|
|
except Exception as e:
|
|
# Record exception in span
|
|
span.record_exception(e)
|
|
span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
|
|
|
|
order_counter.add(1, {"status": "error"})
|
|
raise</code></pre>
|
|
<h3>Pattern 4: Traces + Metrics Correlation (Exemplars)</h3>
|
|
<pre><code class="lang-python">from opentelemetry import trace, metrics
|
|
from opentelemetry.sdk.metrics.view import View
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
|
|
# Enable exemplars for histograms
|
|
meter_provider = MeterProvider(
|
|
views=[
|
|
View(
|
|
instrument_name="http.server.duration",
|
|
aggregation=ExplicitBucketHistogramAggregation(
|
|
boundaries=[0.01, 0.05, 0.1, 0.5, 1.0, 5.0],
|
|
record_min_max=True
|
|
)
|
|
)
|
|
]
|
|
)
|
|
|
|
metrics.set_meter_provider(meter_provider)
|
|
meter = metrics.get_meter(__name__)
|
|
|
|
duration_histogram = meter.create_histogram(
|
|
"http.server.duration",
|
|
description="HTTP request duration",
|
|
unit="s"
|
|
)
|
|
|
|
# Record metric with trace context (automatic exemplar)
|
|
def handle_request(request):
|
|
start = time.time()
|
|
|
|
# Trace is automatically linked to metric
|
|
with tracer.start_as_current_span("handle_request") as span:
|
|
try:
|
|
response = process(request)
|
|
return response
|
|
finally:
|
|
duration = time.time() - start
|
|
# Exemplar links this metric point to the trace
|
|
duration_histogram.record(duration, {
|
|
"http.method": request.method,
|
|
"http.status_code": response.status_code
|
|
})</code></pre>
|
|
<p><strong>Exemplar workflow</strong>:</p>
|
|
<ol><li>High latency metric detected: p95 = 2.5s</li><li>Click on metric point in Grafana</li><li>View exemplar trace IDs linked to that metric</li><li>Jump to trace view to see exact slow request</li></ol>
|
|
<h3>Pattern 5: Log + Trace Correlation</h3>
|
|
<pre><code class="lang-python">import logging
|
|
from opentelemetry import trace
|
|
|
|
# Configure structured logging with trace context
|
|
class TraceContextFilter(logging.Filter):
|
|
def filter(self, record):
|
|
span = trace.get_current_span()
|
|
if span:
|
|
ctx = span.get_span_context()
|
|
record.trace_id = format(ctx.trace_id, '032x')
|
|
record.span_id = format(ctx.span_id, '016x')
|
|
else:
|
|
record.trace_id = '0' * 32
|
|
record.span_id = '0' * 16
|
|
return True
|
|
|
|
logging.basicConfig(
|
|
format='%(asctime)s %(levelname)s [trace_id=%(trace_id)s span_id=%(span_id)s] %(message)s',
|
|
level=logging.INFO
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
logger.addFilter(TraceContextFilter())
|
|
|
|
# Use logger within traced functions
|
|
def process_payment(payment_data):
|
|
with tracer.start_as_current_span("process_payment") as span:
|
|
logger.info("Processing payment", extra={
|
|
"payment.amount": payment_data["amount"],
|
|
"payment.method": payment_data["method"]
|
|
})
|
|
|
|
try:
|
|
result = charge(payment_data)
|
|
logger.info("Payment successful", extra={"payment.id": result.id})
|
|
return result
|
|
except Exception as e:
|
|
logger.error("Payment failed", extra={"error": str(e)})
|
|
raise
|
|
|
|
# Log output includes trace/span IDs:
|
|
# 2025-10-26 10:30:15 INFO [trace_id=4bf92f3577b34da6a3ce929d0e0e4736 span_id=00f067aa0ba902b7] Processing payment</code></pre>
|
|
<h3>Pattern 6: Sampling Strategies</h3>
|
|
<pre><code class="lang-yaml"># Tail-based sampling (smart sampling after trace completes)
|
|
processors:
|
|
tail_sampling:
|
|
policies:
|
|
# Always sample errors
|
|
- name: errors
|
|
type: status_code
|
|
status_code:
|
|
status_codes: [ERROR]
|
|
|
|
# Sample slow requests (>1s)
|
|
- name: slow-requests
|
|
type: latency
|
|
latency:
|
|
threshold_ms: 1000
|
|
|
|
# Sample 10% of all other requests
|
|
- name: probabilistic
|
|
type: probabilistic
|
|
probabilistic:
|
|
sampling_percentage: 10
|
|
|
|
# Always sample specific endpoints
|
|
- name: critical-endpoints
|
|
type: string_attribute
|
|
string_attribute:
|
|
key: http.target
|
|
values: [/api/checkout, /api/payment]
|
|
|
|
service:
|
|
pipelines:
|
|
traces:
|
|
receivers: [otlp]
|
|
processors: [tail_sampling, batch]
|
|
exporters: [otlp/jaeger]</code></pre>
|
|
<h3>Pattern 7: Resource Attributes</h3>
|
|
<pre><code class="lang-python">from opentelemetry.sdk.resources import Resource, SERVICE_NAME, SERVICE_VERSION
|
|
|
|
# Standard semantic conventions
|
|
resource = Resource.create({
|
|
# Service attributes
|
|
SERVICE_NAME: "order-service",
|
|
SERVICE_VERSION: "1.2.3",
|
|
"service.namespace": "ecommerce",
|
|
"service.instance.id": "order-service-abc123",
|
|
|
|
# Deployment attributes
|
|
"deployment.environment": "production",
|
|
"deployment.region": "us-east-1",
|
|
|
|
# Cloud provider attributes
|
|
"cloud.provider": "aws",
|
|
"cloud.platform": "aws_eks",
|
|
"cloud.region": "us-east-1",
|
|
"cloud.availability_zone": "us-east-1a",
|
|
|
|
# Kubernetes attributes (if applicable)
|
|
"k8s.cluster.name": "prod-cluster",
|
|
"k8s.namespace.name": "ecommerce",
|
|
"k8s.pod.name": "order-service-7d8f5c9b4-xk2pl",
|
|
"k8s.container.name": "order-service",
|
|
})
|
|
|
|
# Use resource in tracer/meter providers
|
|
tracer_provider = TracerProvider(resource=resource)</code></pre>
|
|
<hr>
|
|
<h2>Backend Integrations</h2>
|
|
<h3>Grafana Stack (Open Source)</h3>
|
|
<pre><code class="lang-yaml"># docker-compose.yml
|
|
services:
|
|
otel-collector:
|
|
image: otel/opentelemetry-collector-contrib:latest
|
|
volumes:
|
|
- ./otel-config.yaml:/etc/otel-collector-config.yaml
|
|
command: ["--config=/etc/otel-collector-config.yaml"]
|
|
ports:
|
|
- "4317:4317" # OTLP gRPC
|
|
- "4318:4318" # OTLP HTTP
|
|
|
|
# Traces
|
|
tempo:
|
|
image: grafana/tempo:latest
|
|
ports:
|
|
- "3200:3200"
|
|
|
|
# Metrics
|
|
prometheus:
|
|
image: prom/prometheus:latest
|
|
volumes:
|
|
- ./prometheus.yml:/etc/prometheus/prometheus.yml
|
|
ports:
|
|
- "9090:9090"
|
|
|
|
# Logs
|
|
loki:
|
|
image: grafana/loki:latest
|
|
ports:
|
|
- "3100:3100"
|
|
|
|
# Visualization
|
|
grafana:
|
|
image: grafana/grafana:latest
|
|
ports:
|
|
- "3000:3000"
|
|
environment:
|
|
- GF_AUTH_ANONYMOUS_ENABLED=true</code></pre>
|
|
<h3>Datadog Integration</h3>
|
|
<pre><code class="lang-yaml"># otel-collector-config.yaml
|
|
exporters:
|
|
datadog:
|
|
api:
|
|
site: datadoghq.com
|
|
key: ${DD_API_KEY}
|
|
|
|
host_metadata:
|
|
enabled: true
|
|
hostname: ${HOSTNAME}
|
|
tags:
|
|
- env:production
|
|
- team:platform
|
|
|
|
service:
|
|
pipelines:
|
|
traces:
|
|
receivers: [otlp]
|
|
processors: [batch]
|
|
exporters: [datadog]
|
|
|
|
metrics:
|
|
receivers: [otlp]
|
|
processors: [batch]
|
|
exporters: [datadog]</code></pre>
|
|
<h3>Jaeger Integration</h3>
|
|
<pre><code class="lang-yaml">exporters:
|
|
otlp/jaeger:
|
|
endpoint: jaeger:4317
|
|
tls:
|
|
insecure: true
|
|
|
|
service:
|
|
pipelines:
|
|
traces:
|
|
receivers: [otlp]
|
|
processors: [batch]
|
|
exporters: [otlp/jaeger]</code></pre>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Installation</h3>
|
|
<pre><code class="lang-bash"># Python
|
|
pip install opentelemetry-distro opentelemetry-exporter-otlp
|
|
|
|
# Auto-instrumentation
|
|
pip install opentelemetry-instrumentation-flask # or django, fastapi, etc.
|
|
|
|
# Node.js
|
|
npm install @opentelemetry/sdk-node @opentelemetry/auto-instrumentations-node
|
|
|
|
# Go (manual only)
|
|
go get go.opentelemetry.io/otel
|
|
go get go.opentelemetry.io/otel/sdk
|
|
go get go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc
|
|
|
|
# Java
|
|
# Download opentelemetry-javaagent.jar
|
|
java -javaagent:opentelemetry-javaagent.jar \
|
|
-Dotel.service.name=my-service \
|
|
-Dotel.exporter.otlp.endpoint=http://otel-collector:4317 \
|
|
-jar myapp.jar</code></pre>
|
|
<h3>OTel Collector Deployment</h3>
|
|
<pre><code class="lang-bash"># Docker
|
|
docker run -v $(pwd)/config.yaml:/etc/otel-collector-config.yaml \
|
|
-p 4317:4317 -p 4318:4318 \
|
|
otel/opentelemetry-collector-contrib:latest \
|
|
--config=/etc/otel-collector-config.yaml
|
|
|
|
# Kubernetes
|
|
kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml</code></pre>
|
|
<h3>Key Environment Variables</h3>
|
|
<pre><code class="lang-bash"># Exporter endpoint
|
|
export OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
|
|
|
# Service name
|
|
export OTEL_SERVICE_NAME=my-service
|
|
|
|
# Resource attributes
|
|
export OTEL_RESOURCE_ATTRIBUTES=deployment.environment=production,service.version=1.0.0
|
|
|
|
# Traces, metrics, logs exporters
|
|
export OTEL_TRACES_EXPORTER=otlp
|
|
export OTEL_METRICS_EXPORTER=otlp
|
|
export OTEL_LOGS_EXPORTER=otlp</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>❌ Not Using OTel Collector</h3>
|
|
<pre><code># WRONG: Direct export to backend
|
|
App → Jaeger (vendor lock-in, no processing)
|
|
|
|
# CORRECT: Export through Collector
|
|
App → OTel Collector → Jaeger (portable, scalable)</code></pre>
|
|
<h3>❌ Over-Sampling in Production</h3>
|
|
<pre><code class="lang-yaml"># WRONG: 100% sampling (expensive, high cardinality)
|
|
processors:
|
|
probabilistic_sampler:
|
|
sampling_percentage: 100
|
|
|
|
# CORRECT: Tail-based sampling (smart sampling)
|
|
processors:
|
|
tail_sampling:
|
|
policies:
|
|
- name: errors
|
|
type: status_code
|
|
status_code: {status_codes: [ERROR]}
|
|
- name: probabilistic
|
|
type: probabilistic
|
|
probabilistic: {sampling_percentage: 10}</code></pre>
|
|
<h3>❌ Missing Resource Attributes</h3>
|
|
<pre><code class="lang-python"># WRONG: No context about service
|
|
resource = Resource.create({})
|
|
|
|
# CORRECT: Rich service metadata
|
|
resource = Resource.create({
|
|
"service.name": "order-service",
|
|
"service.version": "1.0.0",
|
|
"deployment.environment": "production"
|
|
})</code></pre>
|
|
<h3>❌ Not Correlating Signals</h3>
|
|
<pre><code class="lang-python"># WRONG: Logs without trace context
|
|
logger.info("Processing order")
|
|
|
|
# CORRECT: Logs with trace/span IDs
|
|
logger.info("Processing order", extra={
|
|
"trace_id": trace.get_current_span().get_span_context().trace_id
|
|
})</code></pre>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><strong>distributed-tracing.md</strong> - Deep dive into distributed tracing concepts</li><li><strong>metrics-instrumentation.md</strong> - Prometheus and metrics best practices</li><li><strong>structured-logging.md</strong> - Log formatting and correlation</li><li><strong>observability-cost-optimization.md</strong> - Reduce OTel costs with sampling/filtering</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-26 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/observability-opentelemetry-integration.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |