mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
641 lines
30 KiB
HTML
641 lines
30 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>redpanda-streaming</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/redpanda-streaming.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/redpanda-streaming">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/redpanda-streaming.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>redpanda-streaming</h1><p class="lede">**Redpanda deployments**: Setting up Kafka-compatible streaming infrastructure</p><h1>Redpanda Streaming Platform</h1>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate <code>redpanda-streaming.md</code> when working with:</p>
|
|
<ul><li><strong>Redpanda deployments</strong>: Setting up Kafka-compatible streaming infrastructure</li><li><strong>Event streaming</strong>: Building real-time data pipelines and event-driven architectures</li><li><strong>Kafka migration</strong>: Migrating from Apache Kafka to Redpanda</li><li><strong>Stream processing</strong>: Producing/consuming events at high throughput</li><li><strong>Schema management</strong>: Using Schema Registry for data contracts</li><li><strong>HTTP streaming</strong>: REST APIs for Kafka topics via Pandaproxy</li><li><strong>rpk CLI operations</strong>: Managing topics, ACLs, clusters via command line</li><li><strong>Performance optimization</strong>: Tuning for low latency and high throughput</li></ul>
|
|
<p><strong>Technology focus</strong>: Redpanda 23.x+, Kafka API compatibility, rpk CLI, Schema Registry, Pandaproxy</p>
|
|
<h2>Core Concepts</h2>
|
|
<h3>What is Redpanda?</h3>
|
|
<p><strong>Redpanda</strong> is a Kafka-compatible streaming data platform written in C++ that eliminates complexity:</p>
|
|
<ul><li><strong>No ZooKeeper</strong>: Uses Raft consensus for coordination</li><li><strong>No JVM</strong>: C++ implementation with lower latency and memory usage</li><li><strong>Kafka API compatible</strong>: Drop-in replacement for Kafka clients</li><li><strong>Built-in features</strong>: Schema Registry, HTTP Proxy, tiered storage included</li></ul>
|
|
<h3>Architecture Components</h3>
|
|
<pre><code>┌─────────────────────────────────────────────────────────┐
|
|
│ Redpanda Cluster │
|
|
├─────────────────────────────────────────────────────────┤
|
|
│ Broker 1 Broker 2 Broker 3 │
|
|
│ ┌─────────┐ ┌─────────┐ ┌─────────┐ │
|
|
│ │ Topics │ │ Topics │ │ Topics │ │
|
|
│ │ Raft │ │ Raft │ │ Raft │ │
|
|
│ └─────────┘ └─────────┘ └─────────┘ │
|
|
└─────────────────────────────────────────────────────────┘
|
|
↑ ↑ ↑
|
|
│ │ │
|
|
┌────┴──────┬───────┴────┬─────────┴───┐
|
|
│ │ │ │
|
|
Kafka API Pandaproxy Schema Reg rpk CLI
|
|
(port 9092) (port 8082) (port 8081) (local)</code></pre>
|
|
<h3>Topics and Partitions</h3>
|
|
<p><strong>Topics</strong> are named streams of records (events):</p>
|
|
<pre><code class="lang-bash"># Create topic with 3 partitions, replication factor 3
|
|
rpk topic create orders -p 3 -r 3
|
|
|
|
# List topics
|
|
rpk topic list
|
|
|
|
# Describe topic
|
|
rpk topic describe orders
|
|
|
|
# Delete topic
|
|
rpk topic delete orders</code></pre>
|
|
<p><strong>Partitions</strong> enable parallelism:</p>
|
|
<ul><li>Events with same key → same partition (ordering guarantee)</li><li>Multiple partitions → parallel consumption</li><li>Replication factor → data durability (survive broker failures)</li></ul>
|
|
<h3>Producers and Consumers</h3>
|
|
<p><strong>Producer</strong> pattern (Python with kafka-python):</p>
|
|
<pre><code class="lang-python">from kafka import KafkaProducer
|
|
import json
|
|
|
|
producer = KafkaProducer(
|
|
bootstrap_servers=['localhost:9092'],
|
|
value_serializer=lambda v: json.dumps(v).encode('utf-8'),
|
|
acks='all', # Wait for all replicas
|
|
retries=3
|
|
)
|
|
|
|
# Produce event
|
|
producer.send('orders', value={'order_id': 123, 'amount': 99.99})
|
|
producer.flush()</code></pre>
|
|
<p><strong>Consumer</strong> pattern (Python):</p>
|
|
<pre><code class="lang-python">from kafka import KafkaConsumer
|
|
import json
|
|
|
|
consumer = KafkaConsumer(
|
|
'orders',
|
|
bootstrap_servers=['localhost:9092'],
|
|
group_id='order-processor',
|
|
value_deserializer=lambda m: json.loads(m.decode('utf-8')),
|
|
auto_offset_reset='earliest', # Start from beginning if no offset
|
|
enable_auto_commit=True
|
|
)
|
|
|
|
for message in consumer:
|
|
order = message.value
|
|
print(f"Processing order {order['order_id']}")</code></pre>
|
|
<h3>Consumer Groups</h3>
|
|
<p><strong>Consumer groups</strong> enable parallel processing:</p>
|
|
<pre><code class="lang-bash"># Each consumer in group processes subset of partitions
|
|
# Group: order-processor
|
|
# Consumer 1 → Partition 0
|
|
# Consumer 2 → Partition 1
|
|
# Consumer 3 → Partition 2
|
|
|
|
# View consumer groups
|
|
rpk group list
|
|
|
|
# Describe group (offsets, lag)
|
|
rpk group describe order-processor
|
|
|
|
# Seek to offset
|
|
rpk group seek order-processor --to start
|
|
rpk group seek order-processor --to end
|
|
rpk group seek order-processor --to 1000</code></pre>
|
|
<h2>Deployment Patterns</h2>
|
|
<h3>Docker Compose (Development)</h3>
|
|
<pre><code class="lang-yaml"># docker-compose.yml
|
|
version: '3.8'
|
|
services:
|
|
redpanda:
|
|
image: redpandadata/redpanda:latest
|
|
command:
|
|
- redpanda start
|
|
- --smp 1
|
|
- --memory 1G
|
|
- --overprovisioned
|
|
- --node-id 0
|
|
- --kafka-addr PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
|
|
- --advertise-kafka-addr PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092
|
|
ports:
|
|
- "9092:9092" # Kafka API
|
|
- "8081:8081" # Schema Registry
|
|
- "8082:8082" # Pandaproxy (HTTP)
|
|
- "9644:9644" # Admin API</code></pre>
|
|
<p>Start with:</p>
|
|
<pre><code class="lang-bash">docker-compose up -d
|
|
rpk cluster info --brokers localhost:9092</code></pre>
|
|
<h3>Kubernetes (Production)</h3>
|
|
<pre><code class="lang-bash"># Install Redpanda operator
|
|
helm repo add redpanda https://charts.redpanda.com
|
|
helm install redpanda-operator redpanda/redpanda-operator \
|
|
--namespace redpanda-system --create-namespace
|
|
|
|
# Deploy 3-broker cluster
|
|
kubectl apply -f - <<EOF
|
|
apiVersion: cluster.redpanda.com/v1alpha1
|
|
kind: Redpanda
|
|
metadata:
|
|
name: redpanda
|
|
spec:
|
|
replicas: 3
|
|
resources:
|
|
requests:
|
|
cpu: "2"
|
|
memory: "8Gi"
|
|
storage:
|
|
capacity: 100Gi
|
|
EOF
|
|
|
|
# Verify cluster
|
|
kubectl get redpanda
|
|
rpk cluster info --brokers redpanda-0.redpanda.default.svc.cluster.local:9092</code></pre>
|
|
<h2>Schema Registry</h2>
|
|
<p><strong>Schema Registry</strong> enforces data contracts using Avro/Protobuf/JSON Schema:</p>
|
|
<h3>Register Schema</h3>
|
|
<pre><code class="lang-bash"># Register Avro schema for topic 'orders-value'
|
|
curl -X POST http://localhost:8081/subjects/orders-value/versions \
|
|
-H 'Content-Type: application/vnd.schemaregistry.v1+json' \
|
|
-d '{
|
|
"schema": "{\"type\":\"record\",\"name\":\"Order\",\"fields\":[{\"name\":\"order_id\",\"type\":\"int\"},{\"name\":\"amount\",\"type\":\"double\"}]}"
|
|
}'</code></pre>
|
|
<h3>Python with Schema Registry</h3>
|
|
<pre><code class="lang-python">from confluent_kafka import SerializingProducer
|
|
from confluent_kafka.schema_registry import SchemaRegistryClient
|
|
from confluent_kafka.schema_registry.avro import AvroSerializer
|
|
|
|
schema_str = """
|
|
{
|
|
"type": "record",
|
|
"name": "Order",
|
|
"fields": [
|
|
{"name": "order_id", "type": "int"},
|
|
{"name": "amount", "type": "double"}
|
|
]
|
|
}
|
|
"""
|
|
|
|
schema_registry_client = SchemaRegistryClient({'url': 'http://localhost:8081'})
|
|
avro_serializer = AvroSerializer(schema_registry_client, schema_str)
|
|
|
|
producer = SerializingProducer({
|
|
'bootstrap.servers': 'localhost:9092',
|
|
'value.serializer': avro_serializer
|
|
})
|
|
|
|
producer.produce('orders', value={'order_id': 123, 'amount': 99.99})
|
|
producer.flush()</code></pre>
|
|
<h2>Pandaproxy (HTTP API)</h2>
|
|
<p><strong>Pandaproxy</strong> provides REST API for Kafka operations:</p>
|
|
<h3>Produce via HTTP</h3>
|
|
<pre><code class="lang-bash">curl -X POST http://localhost:8082/topics/orders \
|
|
-H 'Content-Type: application/vnd.kafka.json.v2+json' \
|
|
-d '{
|
|
"records": [
|
|
{"value": {"order_id": 123, "amount": 99.99}}
|
|
]
|
|
}'</code></pre>
|
|
<h3>Consume via HTTP</h3>
|
|
<pre><code class="lang-bash"># Create consumer instance
|
|
curl -X POST http://localhost:8082/consumers/order-group \
|
|
-H 'Content-Type: application/vnd.kafka.v2+json' \
|
|
-d '{
|
|
"name": "consumer1",
|
|
"format": "json",
|
|
"auto.offset.reset": "earliest"
|
|
}'
|
|
|
|
# Subscribe to topic
|
|
curl -X POST http://localhost:8082/consumers/order-group/instances/consumer1/subscription \
|
|
-H 'Content-Type: application/vnd.kafka.v2+json' \
|
|
-d '{"topics": ["orders"]}'
|
|
|
|
# Fetch records
|
|
curl http://localhost:8082/consumers/order-group/instances/consumer1/records \
|
|
-H 'Accept: application/vnd.kafka.json.v2+json'</code></pre>
|
|
<h2>rpk CLI Quick Reference</h2>
|
|
<h3>Cluster Operations</h3>
|
|
<pre><code class="lang-bash"># Cluster info
|
|
rpk cluster info
|
|
|
|
# Cluster health
|
|
rpk cluster health
|
|
|
|
# Cluster config
|
|
rpk cluster config get
|
|
rpk cluster config set log_retention_ms 604800000 # 7 days</code></pre>
|
|
<h3>Topic Operations</h3>
|
|
<pre><code class="lang-bash"># Create topic
|
|
rpk topic create <topic> -p <partitions> -r <replicas>
|
|
|
|
# List topics
|
|
rpk topic list
|
|
|
|
# Describe topic (partitions, replicas, config)
|
|
rpk topic describe <topic>
|
|
|
|
# Alter config
|
|
rpk topic alter-config <topic> --set retention.ms=3600000
|
|
|
|
# Delete topic
|
|
rpk topic delete <topic>
|
|
|
|
# Produce to topic
|
|
echo '{"key": "value"}' | rpk topic produce <topic>
|
|
|
|
# Consume from topic
|
|
rpk topic consume <topic> --format json
|
|
rpk topic consume <topic> --offset start # From beginning
|
|
rpk topic consume <topic> --offset end # Only new messages</code></pre>
|
|
<h3>ACL Management</h3>
|
|
<pre><code class="lang-bash"># Create ACL (allow user 'alice' to read topic 'orders')
|
|
rpk acl create --allow-principal User:alice \
|
|
--operation read --topic orders
|
|
|
|
# List ACLs
|
|
rpk acl list
|
|
|
|
# Delete ACL
|
|
rpk acl delete --allow-principal User:alice \
|
|
--operation read --topic orders</code></pre>
|
|
<h3>Consumer Groups</h3>
|
|
<pre><code class="lang-bash"># List groups
|
|
rpk group list
|
|
|
|
# Describe group (members, lag)
|
|
rpk group describe <group>
|
|
|
|
# Seek to offset
|
|
rpk group seek <group> --to start
|
|
rpk group seek <group> --to end
|
|
rpk group seek <group> --to 1000
|
|
|
|
# Delete group
|
|
rpk group delete <group></code></pre>
|
|
<h2>Performance Tuning</h2>
|
|
<h3>Producer Optimization</h3>
|
|
<pre><code class="lang-python"># High throughput configuration
|
|
producer = KafkaProducer(
|
|
bootstrap_servers=['localhost:9092'],
|
|
acks='1', # Only leader ack (faster than 'all')
|
|
compression_type='lz4', # Compress batches
|
|
batch_size=32768, # 32KB batches
|
|
linger_ms=10, # Wait 10ms to batch more records
|
|
buffer_memory=67108864 # 64MB buffer
|
|
)</code></pre>
|
|
<h3>Consumer Optimization</h3>
|
|
<pre><code class="lang-python"># High throughput configuration
|
|
consumer = KafkaConsumer(
|
|
'orders',
|
|
bootstrap_servers=['localhost:9092'],
|
|
fetch_min_bytes=1024, # Wait for 1KB before returning
|
|
fetch_max_wait_ms=500, # Max wait 500ms
|
|
max_partition_fetch_bytes=1048576, # 1MB per partition
|
|
enable_auto_commit=True,
|
|
auto_commit_interval_ms=5000 # Commit every 5s
|
|
)</code></pre>
|
|
<h3>Broker Tuning</h3>
|
|
<pre><code class="lang-bash"># Increase retention (7 days)
|
|
rpk cluster config set log_retention_ms 604800000
|
|
|
|
# Increase segment size (1GB)
|
|
rpk cluster config set log_segment_size 1073741824
|
|
|
|
# Enable compression
|
|
rpk cluster config set compression.type producer # Use producer's compression</code></pre>
|
|
<h2>Monitoring</h2>
|
|
<h3>Key Metrics (via Admin API)</h3>
|
|
<pre><code class="lang-bash"># Cluster metrics
|
|
curl http://localhost:9644/metrics | grep redpanda
|
|
|
|
# Important metrics:
|
|
# - redpanda_kafka_request_latency_seconds
|
|
# - redpanda_kafka_request_bytes_total
|
|
# - redpanda_storage_disk_free_bytes
|
|
# - redpanda_cluster_partition_count</code></pre>
|
|
<h3>rpk Monitoring</h3>
|
|
<pre><code class="lang-bash"># Cluster health
|
|
rpk cluster health
|
|
|
|
# Topic lag
|
|
rpk group describe <group>
|
|
|
|
# Disk usage
|
|
rpk cluster info</code></pre>
|
|
<h2>Kafka Compatibility</h2>
|
|
<p>Redpanda is <strong>100% Kafka API compatible</strong>:</p>
|
|
<pre><code class="lang-python"># Works with any Kafka client library
|
|
from kafka import KafkaProducer, KafkaConsumer # kafka-python
|
|
from confluent_kafka import Producer, Consumer # confluent-kafka-python
|
|
from aiokafka import AIOKafkaProducer, AIOKafkaConsumer # aiokafka
|
|
|
|
# Just point to Redpanda broker
|
|
bootstrap_servers = 'localhost:9092' # Redpanda instead of Kafka</code></pre>
|
|
<p><strong>Migration from Kafka</strong>:</p>
|
|
<ol><li>Update <code>bootstrap.servers</code> to Redpanda brokers</li><li>No code changes needed</li><li>Remove ZooKeeper configuration (Redpanda doesn't use it)</li></ol>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>❌ Don't: Produce without error handling</h3>
|
|
<pre><code class="lang-python"># Bad: Silent failures
|
|
producer.send('orders', value=order)</code></pre>
|
|
<p><strong>✅ Do: Handle errors</strong></p>
|
|
<pre><code class="lang-python">future = producer.send('orders', value=order)
|
|
try:
|
|
record_metadata = future.get(timeout=10)
|
|
print(f"Produced to {record_metadata.topic}:{record_metadata.partition}")
|
|
except Exception as e:
|
|
print(f"Failed to produce: {e}")</code></pre>
|
|
<h3>❌ Don't: Ignore consumer lag</h3>
|
|
<pre><code class="lang-python"># Bad: No lag monitoring
|
|
for message in consumer:
|
|
process(message.value)</code></pre>
|
|
<p><strong>✅ Do: Monitor lag and scale</strong></p>
|
|
<pre><code class="lang-bash"># Monitor lag regularly
|
|
rpk group describe order-processor
|
|
|
|
# High lag? Add consumers to group (up to partition count)</code></pre>
|
|
<h3>❌ Don't: Use auto-commit without understanding</h3>
|
|
<pre><code class="lang-python"># Bad: Auto-commit can lose messages on crashes
|
|
consumer = KafkaConsumer(enable_auto_commit=True)</code></pre>
|
|
<p><strong>✅ Do: Manual commit for critical data</strong></p>
|
|
<pre><code class="lang-python">consumer = KafkaConsumer(enable_auto_commit=False)
|
|
for message in consumer:
|
|
process(message.value)
|
|
consumer.commit() # Commit only after processing</code></pre>
|
|
<h3>❌ Don't: Create topics with 1 partition for scalability</h3>
|
|
<pre><code class="lang-bash"># Bad: Limits to 1 consumer
|
|
rpk topic create orders -p 1</code></pre>
|
|
<p><strong>✅ Do: Plan partitions based on throughput</strong></p>
|
|
<pre><code class="lang-bash"># Good: 3 partitions → up to 3 parallel consumers
|
|
rpk topic create orders -p 3 -r 3</code></pre>
|
|
<h3>❌ Don't: Hardcode broker addresses</h3>
|
|
<pre><code class="lang-python"># Bad: Breaks when brokers change
|
|
producer = KafkaProducer(bootstrap_servers=['redpanda-1:9092'])</code></pre>
|
|
<p><strong>✅ Do: Use service discovery or env vars</strong></p>
|
|
<pre><code class="lang-python">import os
|
|
brokers = os.getenv('REDPANDA_BROKERS', 'localhost:9092')
|
|
producer = KafkaProducer(bootstrap_servers=brokers.split(','))</code></pre>
|
|
<h2>Related Skills</h2>
|
|
<p><strong>Core Dependencies</strong>:</p>
|
|
<ul><li><code>docker-basics.md</code> - Container deployment patterns</li><li><code>kubernetes-deployment.md</code> - K8s operator deployment</li></ul>
|
|
<p><strong>Complementary Skills</strong>:</p>
|
|
<ul><li><code>kafka-streams.md</code> - Stream processing with Kafka Streams API</li><li><code>flink-streaming.md</code> - Apache Flink for complex event processing</li><li><code>postgres-partitioning.md</code> - Database patterns for event sourcing</li><li><code>observability-patterns.md</code> - Monitoring and alerting</li></ul>
|
|
<p><strong>Integration Patterns</strong>:</p>
|
|
<ul><li><code>fastapi-async.md</code> - Building async HTTP APIs over Redpanda</li><li><code>python-async-patterns.md</code> - Async producers/consumers with aiokafka</li><li><code>go-concurrency.md</code> - High-performance Go consumers</li></ul>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Common Commands</h3>
|
|
<pre><code class="lang-bash"># Cluster
|
|
rpk cluster info
|
|
rpk cluster health
|
|
rpk cluster config get
|
|
|
|
# Topics
|
|
rpk topic create <topic> -p 3 -r 3
|
|
rpk topic list
|
|
rpk topic describe <topic>
|
|
rpk topic delete <topic>
|
|
|
|
# Produce/Consume
|
|
echo 'message' | rpk topic produce <topic>
|
|
rpk topic consume <topic> --format json
|
|
|
|
# Consumer Groups
|
|
rpk group list
|
|
rpk group describe <group>
|
|
rpk group seek <group> --to start
|
|
|
|
# ACLs
|
|
rpk acl create --allow-principal User:alice --operation read --topic orders
|
|
rpk acl list</code></pre>
|
|
<h3>Client Configuration (Python)</h3>
|
|
<pre><code class="lang-python"># Producer (high throughput)
|
|
producer = KafkaProducer(
|
|
bootstrap_servers=['localhost:9092'],
|
|
acks='all',
|
|
compression_type='lz4',
|
|
batch_size=32768,
|
|
linger_ms=10
|
|
)
|
|
|
|
# Consumer (reliable processing)
|
|
consumer = KafkaConsumer(
|
|
'orders',
|
|
bootstrap_servers=['localhost:9092'],
|
|
group_id='order-processor',
|
|
enable_auto_commit=False,
|
|
auto_offset_reset='earliest'
|
|
)</code></pre>
|
|
<h3>Docker Compose Snippet</h3>
|
|
<pre><code class="lang-yaml">services:
|
|
redpanda:
|
|
image: redpandadata/redpanda:latest
|
|
command:
|
|
- redpanda start
|
|
- --kafka-addr PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
|
|
- --advertise-kafka-addr PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092
|
|
ports:
|
|
- "9092:9092"
|
|
- "8081:8081"
|
|
- "8082:8082"</code></pre>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/redpanda-streaming.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |