mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
634 lines
29 KiB
HTML
634 lines
29 KiB
HTML
<!doctype html>
|
||
<html lang="en"><head>
|
||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||
<title>observability-cost-optimization</title>
|
||
<meta name="color-scheme" content="dark">
|
||
<style>
|
||
/* Committed to one visual world: a dark instrument panel. No light variant —
|
||
the dot field and the hairline rules only read on black, and a naive
|
||
inversion would wash both out. Every colour is painted explicitly so the page
|
||
never borrows a host background. */
|
||
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
||
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
||
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
||
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
||
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
||
|
||
:root{
|
||
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
||
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
||
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
||
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
||
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
||
--mono:var(--font-zen-mono);
|
||
--sans:var(--font-zen-sans);
|
||
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
||
--r:999px; --r-soft:14px;
|
||
}
|
||
*{box-sizing:border-box}
|
||
html{-webkit-text-size-adjust:100%}
|
||
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
||
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
||
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
||
|
||
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
||
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
||
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
||
would cut off the copy-prompt menu. */
|
||
.hero{position:relative;border-bottom:1px solid var(--line)}
|
||
.hero::before{content:"";position:absolute;inset:0;
|
||
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
||
background-size:7px 7px;
|
||
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
||
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
||
pointer-events:none}
|
||
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
||
|
||
nav{border-bottom:1px solid var(--line)}
|
||
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
||
justify-content:space-between;align-items:center;gap:1rem}
|
||
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
||
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
||
display:inline-flex;align-items:center;gap:.6rem}
|
||
.brand i{font-style:normal;color:var(--faint)}
|
||
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
||
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
||
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
||
nav a.up:hover{color:var(--ink)}
|
||
|
||
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
||
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
||
h1 em{font-style:normal;color:var(--faint)}
|
||
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
||
|
||
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
||
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
||
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
||
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
||
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
||
line-height:1.2;font-weight:400}
|
||
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
||
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
||
.btn.solid:hover{background:#d8d8d4}
|
||
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
||
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
||
|
||
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
||
.copy{position:relative;display:inline-flex}
|
||
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
||
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
||
padding:.4rem;display:flex;flex-direction:column;
|
||
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
||
.menu[hidden]{display:none}
|
||
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
||
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
||
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
||
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
||
white-space:nowrap}
|
||
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
||
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
||
.mi:hover .mark{color:var(--ink)}
|
||
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
||
letter-spacing:.1em;text-transform:uppercase}
|
||
|
||
/* Works-with bar: one scrolling row, name and mark per agent. */
|
||
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
||
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
||
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
||
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
||
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
||
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
||
display:flex;align-items:center}
|
||
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
||
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
||
-webkit-overflow-scrolling:touch;
|
||
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
||
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
||
.bar::-webkit-scrollbar{display:none}
|
||
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
||
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
||
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
||
text-decoration:none;background:var(--ground)}
|
||
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
||
.agent .mark{width:.95rem;height:.95rem}
|
||
|
||
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
||
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
||
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
||
p{margin:0 0 1rem;max-width:70ch}
|
||
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
||
li{margin:.22rem 0}
|
||
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
||
text-decoration-color:var(--line-hi)}
|
||
a:hover{text-decoration-color:var(--ink)}
|
||
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
||
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
||
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
||
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
||
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
||
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
||
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
||
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
||
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
||
display:block;overflow-x:auto}
|
||
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
||
|
||
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
||
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
||
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
||
white-space:pre;line-height:1.9}
|
||
.curl b{color:var(--ink);font-weight:400}
|
||
.curl i{font-style:normal;color:var(--faint)}
|
||
|
||
/* index */
|
||
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
||
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
||
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
||
#q:focus{outline:none;border-color:var(--soft)}
|
||
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
||
text-transform:uppercase;margin:0 0 2rem}
|
||
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
||
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
||
padding-bottom:.45rem}
|
||
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
||
border-radius:var(--r-soft);text-decoration:none}
|
||
.row:hover{background:#0a0a0a}
|
||
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
||
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
||
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
||
color:var(--faint);font-size:.85rem}
|
||
footer a{color:var(--soft)}
|
||
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
||
margin:0 0 2rem;font-size:.85rem}
|
||
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
||
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
||
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
||
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
||
.fmap a:hover{text-decoration:underline}
|
||
.fmap div{color:var(--faint);line-height:1.5}
|
||
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
||
|
||
@media (max-width:640px){
|
||
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
||
.btns{gap:.5rem}
|
||
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
||
.copy,.copy .btn{width:100%;justify-content:center}
|
||
.menu{left:0;right:0;min-width:0}
|
||
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
||
.strip b{border-right:0;padding-right:0}
|
||
.bar{width:100%}
|
||
}
|
||
</style>
|
||
<link rel="alternate" type="text/markdown" href="/observability-cost-optimization.md">
|
||
|
||
<link rel="icon" href="/favicon.svg">
|
||
<link rel="canonical" href="https://hanzoskills.com/observability-cost-optimization">
|
||
</head>
|
||
<!-- Reading this with a machine? Every page here is also plain markdown.
|
||
This one: https://hanzoskills.com/observability-cost-optimization.md
|
||
Index: https://hanzoskills.com/llms.txt
|
||
Corpus: https://hanzoskills.com/all.md
|
||
Append .md to any path for the markdown twin. -->
|
||
<body>
|
||
<nav><div class="in">
|
||
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
||
<a class="up" href="/">All skills</a>
|
||
</div></nav>
|
||
|
||
<div class="wrap">
|
||
<h1>observability-cost-optimization</h1><p class="lede">Optimizing observability costs through sampling, cardinality management, and retention strategies</p><h1>Observability Cost Optimization</h1>
|
||
<p><strong>Scope</strong>: Cardinality management, sampling strategies, log reduction, metrics aggregation, retention policies, OTel Collector cost reduction</p>
|
||
<p><strong>Lines</strong>: 420</p>
|
||
<p><strong>Last Updated</strong>: 2025-10-26</p>
|
||
<hr>
|
||
<h2>When to Use This Skill</h2>
|
||
<p>Use this skill when:</p>
|
||
<ul><li>Observability costs are growing faster than infrastructure costs</li><li>Experiencing cardinality explosion in metrics or traces</li><li>Need to reduce log ingestion volume without losing critical data</li><li>Implementing sampling strategies for high-traffic systems</li><li>Optimizing retention policies for traces, metrics, and logs</li><li>Configuring OTel Collector processors for cost reduction</li><li>Balancing observability coverage with budget constraints</li></ul>
|
||
<p><strong>Don't use</strong> for:</p>
|
||
<ul><li>Initial observability setup (implement first, optimize later)</li><li>Low-traffic systems (<1000 req/sec)</li><li>Systems where observability budget is unconstrained</li></ul>
|
||
<p><strong>Context (2024-2025)</strong>:</p>
|
||
<ul><li><strong>50%+ of teams</strong> cite cost as biggest observability challenge (CNCF survey)</li><li><strong>Average 30-40% cost reduction</strong> achievable with proper sampling</li><li><strong>Cardinality is #1 cost driver</strong> for metrics-based systems</li><li><strong>Logs account for 60-70%</strong> of total observability costs</li></ul>
|
||
<hr>
|
||
<h2>Core Concepts</h2>
|
||
<h3>Cost Drivers</h3>
|
||
<p><strong>1. Cardinality</strong> (Metrics/Traces)</p>
|
||
<ul><li>Number of unique time series (metric + unique label combinations)</li><li>Example: <code>http_requests_total{method="GET", endpoint="/api/users", status="200"}</code></li><li><strong>Impact</strong>: 10x cardinality = 10x storage + 10x query cost</li></ul>
|
||
<p><strong>2. Ingestion Volume</strong> (Logs/Traces)</p>
|
||
<ul><li>Number of events per second</li><li><strong>Impact</strong>: 1M logs/day vs 1B logs/day = 1000x cost difference</li></ul>
|
||
<p><strong>3. Retention Period</strong></p>
|
||
<ul><li>How long data is stored</li><li><strong>Impact</strong>: 30d vs 90d retention = 3x storage cost</li></ul>
|
||
<p><strong>4. Query Complexity</strong></p>
|
||
<ul><li>Aggregations, joins, scans</li><li><strong>Impact</strong>: High-cardinality queries can timeout or spike costs</li></ul>
|
||
<h3>Cost Optimization Strategy</h3>
|
||
<pre><code>1. Identify: What's costing the most? (cardinality, volume, retention)
|
||
2. Reduce: Apply sampling, filtering, aggregation
|
||
3. Retain: Short retention for raw data, long retention for aggregates
|
||
4. Monitor: Track cost metrics (bytes ingested, cardinality growth)</code></pre>
|
||
<hr>
|
||
<h2>Patterns</h2>
|
||
<h3>Pattern 1: Cardinality Management (Metrics)</h3>
|
||
<pre><code class="lang-python">from prometheus_client import Counter, Histogram
|
||
|
||
# ❌ HIGH CARDINALITY: user_id has 1M+ unique values
|
||
requests_bad = Counter(
|
||
'http_requests_total',
|
||
'HTTP requests',
|
||
['method', 'endpoint', 'status', 'user_id'] # 10 × 50 × 5 × 1M = 2.5M time series!
|
||
)
|
||
|
||
# ✅ LOW CARDINALITY: bounded label values
|
||
requests_good = Counter(
|
||
'http_requests_total',
|
||
'HTTP requests',
|
||
['method', 'endpoint', 'status'] # 10 × 50 × 5 = 2,500 time series
|
||
)
|
||
|
||
# ✅ ALTERNATIVE: Use histograms to reduce cardinality
|
||
request_duration = Histogram(
|
||
'http_request_duration_seconds',
|
||
'Request duration',
|
||
['method', 'endpoint'], # No status code (use buckets instead)
|
||
buckets=[0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
|
||
)
|
||
|
||
# Usage
|
||
def handle_request(method, endpoint, user_id):
|
||
# Track request count (low cardinality)
|
||
requests_good.labels(method=method, endpoint=endpoint, status="200").inc()
|
||
|
||
# Track user_id in logs/traces (not metrics)
|
||
logger.info("Request processed", extra={"user_id": user_id})
|
||
|
||
# Track duration in histogram (bounded cardinality)
|
||
with request_duration.labels(method=method, endpoint=endpoint).time():
|
||
process_request()</code></pre>
|
||
<p><strong>Cardinality Reduction Techniques</strong>:</p>
|
||
<ol><li><strong>Remove high-cardinality labels</strong>: user_id, email, trace_id, session_id</li><li><strong>Group labels</strong>: Instead of 1000 endpoints, group into 10 categories</li><li><strong>Use histograms</strong>: Instead of per-bucket metrics, use histogram buckets</li><li><strong>Relabel in Prometheus</strong>: Drop or aggregate labels at scrape time</li></ol>
|
||
<h3>Pattern 2: Prometheus Relabeling</h3>
|
||
<pre><code class="lang-yaml"># prometheus.yml
|
||
scrape_configs:
|
||
- job_name: 'api-servers'
|
||
static_configs:
|
||
- targets: ['api-1:8080', 'api-2:8080']
|
||
|
||
metric_relabel_configs:
|
||
# Drop high-cardinality labels
|
||
- source_labels: [user_id]
|
||
action: labeldrop
|
||
|
||
# Group endpoints into categories
|
||
- source_labels: [endpoint]
|
||
regex: '/api/users/.*'
|
||
replacement: '/api/users/:id'
|
||
target_label: endpoint
|
||
|
||
# Drop low-value metrics entirely
|
||
- source_labels: [__name__]
|
||
regex: 'go_gc_.*'
|
||
action: drop
|
||
|
||
# Limit label values
|
||
- source_labels: [status_code]
|
||
regex: '2..'
|
||
replacement: '2xx'
|
||
target_label: status_code</code></pre>
|
||
<h3>Pattern 3: Tail-Based Sampling (Traces)</h3>
|
||
<pre><code class="lang-yaml"># otel-collector-config.yaml
|
||
processors:
|
||
tail_sampling:
|
||
# Wait 10s for trace to complete before sampling decision
|
||
decision_wait: 10s
|
||
|
||
# Limit in-memory traces
|
||
num_traces: 100000
|
||
|
||
policies:
|
||
# ALWAYS sample errors (0% sampling = keep all)
|
||
- name: errors
|
||
type: status_code
|
||
status_code:
|
||
status_codes: [ERROR]
|
||
|
||
# ALWAYS sample slow requests (>1s)
|
||
- name: slow-requests
|
||
type: latency
|
||
latency:
|
||
threshold_ms: 1000
|
||
|
||
# Sample 100% of critical endpoints
|
||
- name: critical-endpoints
|
||
type: string_attribute
|
||
string_attribute:
|
||
key: http.target
|
||
values: [/api/checkout, /api/payment]
|
||
enabled_regex_matching: true
|
||
invert_match: false
|
||
|
||
# Sample 5% of all other requests
|
||
- name: probabilistic
|
||
type: probabilistic
|
||
probabilistic:
|
||
sampling_percentage: 5
|
||
|
||
service:
|
||
pipelines:
|
||
traces:
|
||
receivers: [otlp]
|
||
processors: [tail_sampling, batch]
|
||
exporters: [otlp/jaeger]</code></pre>
|
||
<p><strong>Tail-Based Sampling Benefits</strong>:</p>
|
||
<ul><li><strong>Smart sampling</strong>: Keep errors and slow requests, sample rest</li><li><strong>Cost reduction</strong>: 95% reduction in trace volume → 95% cost savings</li><li><strong>No data loss</strong>: Critical traces always retained</li></ul>
|
||
<h3>Pattern 4: Probabilistic Sampling (Head-Based)</h3>
|
||
<pre><code class="lang-python">from opentelemetry.sdk.trace import TracerProvider
|
||
from opentelemetry.sdk.trace.sampling import ParentBasedTraceIdRatioBased
|
||
|
||
# Sample 10% of traces (decided at trace creation)
|
||
sampler = ParentBasedTraceIdRatioBased(0.1)
|
||
|
||
tracer_provider = TracerProvider(sampler=sampler)</code></pre>
|
||
<p><strong>Head-Based vs Tail-Based</strong>:</p>
|
||
<p>| Feature | Head-Based | Tail-Based | |---------|-----------|------------| | <strong>Decision Point</strong> | Trace start | Trace end | | <strong>Context</strong> | No visibility into trace outcome | Full trace visibility | | <strong>Latency</strong> | Low | Adds 10s buffer | | <strong>Cost</strong> | Lower (drops early) | Higher (buffers traces) | | <strong>Use Case</strong> | High-volume, uniform traffic | Error/latency-aware sampling |</p>
|
||
<h3>Pattern 5: Log Sampling and Filtering</h3>
|
||
<pre><code class="lang-python">import logging
|
||
import random
|
||
|
||
class SamplingFilter(logging.Filter):
|
||
"""Sample 10% of INFO logs, keep all WARNING+."""
|
||
|
||
def filter(self, record):
|
||
if record.levelno >= logging.WARNING:
|
||
return True # Keep all warnings/errors
|
||
return random.random() < 0.1 # Sample 10% of info logs
|
||
|
||
logger = logging.getLogger(__name__)
|
||
logger.addFilter(SamplingFilter())</code></pre>
|
||
<p><strong>OTel Collector Log Filtering</strong>:</p>
|
||
<pre><code class="lang-yaml">processors:
|
||
filter:
|
||
logs:
|
||
include:
|
||
# Keep errors
|
||
match_type: regexp
|
||
record_attributes:
|
||
- key: level
|
||
value: ERROR|FATAL
|
||
|
||
exclude:
|
||
# Drop noisy debug logs
|
||
match_type: regexp
|
||
record_attributes:
|
||
- key: message
|
||
value: '.*health check.*'
|
||
|
||
# Rate limiting (max 1000 logs/sec)
|
||
groupbytrace:
|
||
wait_duration: 1s
|
||
num_traces: 1000
|
||
|
||
service:
|
||
pipelines:
|
||
logs:
|
||
receivers: [otlp]
|
||
processors: [filter, groupbytrace, batch]
|
||
exporters: [loki]</code></pre>
|
||
<h3>Pattern 6: Adaptive Sampling</h3>
|
||
<pre><code class="lang-python">from opentelemetry.sdk.trace.sampling import Sampler, Decision, SamplingResult
|
||
import time
|
||
|
||
class AdaptiveSampler(Sampler):
|
||
"""Dynamically adjust sampling based on request rate."""
|
||
|
||
def __init__(self, target_rps=100):
|
||
self.target_rps = target_rps
|
||
self.request_count = 0
|
||
self.last_reset = time.time()
|
||
self.current_rate = 1.0
|
||
|
||
def should_sample(self, *args, **kwargs):
|
||
now = time.time()
|
||
|
||
# Reset counter every second
|
||
if now - self.last_reset > 1.0:
|
||
current_rps = self.request_count
|
||
self.request_count = 0
|
||
self.last_reset = now
|
||
|
||
# Adjust sampling rate
|
||
if current_rps > 0:
|
||
self.current_rate = min(1.0, self.target_rps / current_rps)
|
||
|
||
self.request_count += 1
|
||
|
||
# Sample based on current rate
|
||
if random.random() < self.current_rate:
|
||
return SamplingResult(Decision.RECORD_AND_SAMPLE)
|
||
return SamplingResult(Decision.DROP)
|
||
|
||
def get_description(self):
|
||
return f"AdaptiveSampler(target_rps={self.target_rps})"</code></pre>
|
||
<h3>Pattern 7: Metrics Aggregation and Recording Rules</h3>
|
||
<pre><code class="lang-yaml"># prometheus-rules.yml
|
||
groups:
|
||
- name: aggregation
|
||
interval: 1m
|
||
rules:
|
||
# Pre-aggregate high-cardinality metrics
|
||
- record: http_requests:rate5m
|
||
expr: |
|
||
sum by (method, status) (
|
||
rate(http_requests_total[5m])
|
||
)
|
||
|
||
# Drop detailed labels, keep aggregates
|
||
- record: http_errors:rate5m
|
||
expr: |
|
||
sum by (service) (
|
||
rate(http_requests_total{status=~"5.."}[5m])
|
||
)
|
||
|
||
# Compute p95 latency (cheaper to query pre-computed)
|
||
- record: http_request_duration:p95
|
||
expr: |
|
||
histogram_quantile(0.95,
|
||
sum by (le, service) (
|
||
rate(http_request_duration_seconds_bucket[5m])
|
||
)
|
||
)</code></pre>
|
||
<p><strong>Retention Strategy</strong>:</p>
|
||
<pre><code>Raw metrics: 7d retention
|
||
Aggregated metrics (5m): 30d retention
|
||
Aggregated metrics (1h): 90d retention</code></pre>
|
||
<h3>Pattern 8: Attribute Processor (OTel Collector)</h3>
|
||
<pre><code class="lang-yaml">processors:
|
||
# Remove high-cardinality attributes
|
||
attributes:
|
||
actions:
|
||
# Delete PII
|
||
- key: email
|
||
action: delete
|
||
- key: user_id
|
||
action: delete
|
||
|
||
# Hash high-cardinality IDs
|
||
- key: session_id
|
||
action: hash
|
||
|
||
# Limit attribute value length
|
||
- key: error_message
|
||
action: extract
|
||
pattern: '^(.{100}).*'
|
||
extracted_attributes: {short_message: '\1'}
|
||
|
||
# Group spans by endpoint pattern
|
||
spanmetrics:
|
||
metrics_exporter: prometheus
|
||
latency_histogram_buckets: [0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0]
|
||
dimensions:
|
||
- name: http.method
|
||
- name: http.status_code
|
||
|
||
service:
|
||
pipelines:
|
||
traces:
|
||
receivers: [otlp]
|
||
processors: [attributes, spanmetrics, batch]
|
||
exporters: [otlp/jaeger]</code></pre>
|
||
<hr>
|
||
<h2>Cost Optimization Checklist</h2>
|
||
<h3>Metrics</h3>
|
||
<pre><code>[ ] Identify high-cardinality metrics (>10k time series)
|
||
[ ] Remove user_id, email, trace_id from labels
|
||
[ ] Group endpoints into categories (e.g., /api/users/:id)
|
||
[ ] Use histograms instead of per-value metrics
|
||
[ ] Set up Prometheus relabeling rules
|
||
[ ] Create recording rules for expensive queries
|
||
[ ] Reduce scrape interval for low-priority metrics (30s → 1m)
|
||
[ ] Set retention: 7d raw, 30d aggregated</code></pre>
|
||
<h3>Traces</h3>
|
||
<pre><code>[ ] Implement tail-based sampling (sample 5-10% of normal traffic)
|
||
[ ] ALWAYS sample errors and slow requests
|
||
[ ] ALWAYS sample critical endpoints
|
||
[ ] Use adaptive sampling for high-traffic services
|
||
[ ] Remove PII from span attributes
|
||
[ ] Set trace retention: 7d for sampled traces
|
||
[ ] Enable span metrics for aggregate view</code></pre>
|
||
<h3>Logs</h3>
|
||
<pre><code>[ ] Sample INFO logs (10-20%), keep WARNING/ERROR (100%)
|
||
[ ] Filter out health checks, heartbeats
|
||
[ ] Limit log message length (max 1KB)
|
||
[ ] Use structured logging (cheaper to query)
|
||
[ ] Set retention: 3d raw, 30d errors only
|
||
[ ] Rate limit noisy services (max 1k logs/sec)</code></pre>
|
||
<h3>General</h3>
|
||
<pre><code>[ ] Monitor cost metrics (bytes ingested, cardinality growth)
|
||
[ ] Set up alerts for cost spikes
|
||
[ ] Review and prune unused metrics/logs
|
||
[ ] Use OTel Collector for preprocessing (cheaper than backend)
|
||
[ ] Negotiate volume discounts with observability vendor
|
||
[ ] Consider open-source alternatives (Grafana stack)</code></pre>
|
||
<hr>
|
||
<h2>Quick Reference</h2>
|
||
<h3>Cardinality Limits (Best Practices)</h3>
|
||
<pre><code>Metric label cardinality: <100 unique values per label
|
||
Total time series per metric: <10,000
|
||
Total time series per service: <100,000
|
||
Prometheus instance limit: <1M active time series</code></pre>
|
||
<h3>Sampling Rates by Traffic Volume</h3>
|
||
<pre><code>< 100 req/sec: 100% sampling
|
||
100-1k req/sec: 50% sampling
|
||
1k-10k req/sec: 10% sampling
|
||
10k+ req/sec: 1-5% sampling (with tail-based for errors)</code></pre>
|
||
<h3>Retention Guidelines</h3>
|
||
<pre><code>Traces:
|
||
- Raw: 7 days
|
||
- Aggregated (span metrics): 30 days
|
||
|
||
Metrics:
|
||
- Raw (15s): 7 days
|
||
- 5m aggregates: 30 days
|
||
- 1h aggregates: 90 days
|
||
|
||
Logs:
|
||
- INFO: 3 days
|
||
- WARNING: 14 days
|
||
- ERROR: 30 days</code></pre>
|
||
<h3>Cost Estimation Formula</h3>
|
||
<pre><code>Monthly Cost = (Ingestion Volume × Ingestion Rate) + (Storage Volume × Retention Days × Storage Rate)
|
||
|
||
Example:
|
||
Ingestion: 1TB/day × $0.50/GB = $500/day
|
||
Storage: 30TB × $0.02/GB/month = $600/month
|
||
Total: ~$15,600/month
|
||
|
||
After optimization (90% reduction):
|
||
Ingestion: 100GB/day × $0.50/GB = $50/day
|
||
Storage: 3TB × $0.02/GB/month = $60/month
|
||
Total: ~$1,560/month (90% savings)</code></pre>
|
||
<hr>
|
||
<h2>Anti-Patterns</h2>
|
||
<h3>❌ High-Cardinality Labels</h3>
|
||
<pre><code class="lang-python"># WRONG: Unbounded label values
|
||
requests = Counter('requests', 'Requests', ['user_id', 'email'])
|
||
|
||
# CORRECT: Bounded labels
|
||
requests = Counter('requests', 'Requests', ['method', 'status'])</code></pre>
|
||
<h3>❌ No Sampling Strategy</h3>
|
||
<pre><code class="lang-yaml"># WRONG: 100% sampling for high-traffic service
|
||
processors:
|
||
probabilistic_sampler:
|
||
sampling_percentage: 100 # 10k req/sec = 10k traces/sec!
|
||
|
||
# CORRECT: Tail-based sampling
|
||
processors:
|
||
tail_sampling:
|
||
policies:
|
||
- name: errors
|
||
type: status_code
|
||
status_code: {status_codes: [ERROR]}
|
||
- name: sample
|
||
type: probabilistic
|
||
probabilistic: {sampling_percentage: 5}</code></pre>
|
||
<h3>❌ Same Retention for All Data</h3>
|
||
<pre><code># WRONG: 90d retention for all logs (expensive)
|
||
retention_days: 90
|
||
|
||
# CORRECT: Tiered retention
|
||
retention:
|
||
info_logs: 3d
|
||
error_logs: 30d
|
||
raw_metrics: 7d
|
||
aggregated_metrics: 90d</code></pre>
|
||
<h3>❌ Not Monitoring Cost Metrics</h3>
|
||
<pre><code class="lang-python"># WRONG: No visibility into observability costs
|
||
|
||
# CORRECT: Track cost metrics
|
||
from prometheus_client import Counter, Gauge
|
||
|
||
logs_ingested_bytes = Counter('logs_ingested_bytes_total', 'Logs ingested')
|
||
metrics_cardinality = Gauge('metrics_active_series', 'Active time series')
|
||
traces_sampled = Counter('traces_sampled_total', 'Traces sampled', ['sampled'])</code></pre>
|
||
<hr>
|
||
<h2>Related Skills</h2>
|
||
<ul><li><strong>opentelemetry-integration.md</strong> - OTel Collector processors for cost reduction</li><li><strong>metrics-instrumentation.md</strong> - Cardinality management best practices</li><li><strong>distributed-tracing.md</strong> - Sampling strategies for traces</li><li><strong>structured-logging.md</strong> - Efficient log formatting</li></ul>
|
||
<hr>
|
||
<p><strong>Last Updated</strong>: 2025-10-26 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
||
<footer>
|
||
<div class="fmap">
|
||
<div><b>Build</b>
|
||
<a href="https://hanzo.app">hanzo.app</a> app builder
|
||
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
||
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
||
<div><b>Run</b>
|
||
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
||
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
||
<div><b>Account</b>
|
||
<a href="https://hanzo.id">hanzo.id</a> sign in
|
||
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
||
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
||
<div><b>Company</b>
|
||
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
||
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
||
</div>
|
||
<div class="fend"><a href="/observability-cost-optimization.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
||
</footer>
|
||
</div></body></html> |