mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
775 lines
40 KiB
HTML
775 lines
40 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>protocols-kafka-streams</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/protocols-kafka-streams.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/protocols-kafka-streams">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/protocols-kafka-streams.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>protocols-kafka-streams</h1><p class="lede">Apache Kafka stream processing and event streaming platform</p><h1>Kafka Streams</h1>
|
|
<p><strong>Scope</strong>: Kafka architecture, producers, consumers, Kafka Streams API, exactly-once semantics, schema registry, ksqlDB <strong>Lines</strong>: ~300 <strong>Last Updated</strong>: 2025-10-27</p>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Building real-time data pipelines with Kafka</li><li>Implementing event-driven architectures</li><li>Processing streams with Kafka Streams or ksqlDB</li><li>Ensuring exactly-once message delivery</li><li>Using Avro/Protobuf with Schema Registry</li><li>Monitoring consumer lag and cluster health</li><li>Optimizing Kafka performance and throughput</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>What is Apache Kafka?</h3>
|
|
<p><strong>Apache Kafka</strong>: Distributed event streaming platform</p>
|
|
<ul><li><strong>High-throughput messaging</strong>: Millions of messages/sec</li><li><strong>Durable storage</strong>: Messages persisted to disk with replication</li><li><strong>Stream processing</strong>: Real-time data transformation (Kafka Streams, ksqlDB)</li><li><strong>Scalable</strong>: Horizontal scaling with partitions and brokers</li></ul>
|
|
<p><strong>Use cases</strong>:</p>
|
|
<ul><li>Event sourcing and CQRS</li><li>Real-time analytics and monitoring</li><li>Log aggregation and metrics collection</li><li>Microservices communication</li><li>Data integration (CDC, ETL)</li></ul>
|
|
<h3>Architecture Components</h3>
|
|
<p><strong>Topic</strong>: Logical stream of records (like a table)</p>
|
|
<ul><li>Messages organized into ordered log</li><li>Immutable append-only structure</li></ul>
|
|
<p><strong>Partition</strong>: Ordered sequence of messages within topic</p>
|
|
<ul><li>Enable parallelism (multiple consumers)</li><li>Messages in partition are ordered</li><li>Key-based routing ensures same key → same partition</li></ul>
|
|
<p><strong>Broker</strong>: Kafka server instance</p>
|
|
<ul><li>Stores data for partitions</li><li>Handles reads/writes</li><li>Cluster has multiple brokers (3-5+ for production)</li></ul>
|
|
<p><strong>Producer</strong>: Publishes messages to topics <strong>Consumer</strong>: Reads messages from topics <strong>Consumer Group</strong>: Set of consumers sharing topic partitions</p>
|
|
<pre><code>Producer → Topic (Partitions) → Consumer Group → Consumers
|
|
↓
|
|
Replication (ISR)</code></pre>
|
|
<hr>
|
|
<h2>Basic Producer/Consumer</h2>
|
|
<h3>Producer (Python)</h3>
|
|
<pre><code class="lang-python">from kafka import KafkaProducer
|
|
import json
|
|
|
|
# Create producer
|
|
producer = KafkaProducer(
|
|
bootstrap_servers=['localhost:9092'],
|
|
value_serializer=lambda v: json.dumps(v).encode('utf-8'),
|
|
acks='all', # Wait for all in-sync replicas
|
|
retries=3,
|
|
compression_type='lz4'
|
|
)
|
|
|
|
# Send message
|
|
producer.send(
|
|
'orders',
|
|
key='order-123',
|
|
value={'order_id': 'order-123', 'amount': 99.99}
|
|
)
|
|
|
|
# Flush and close
|
|
producer.flush()
|
|
producer.close()</code></pre>
|
|
<h3>Consumer (Python)</h3>
|
|
<pre><code class="lang-python">from kafka import KafkaConsumer
|
|
import json
|
|
|
|
# Create consumer
|
|
consumer = KafkaConsumer(
|
|
'orders',
|
|
bootstrap_servers=['localhost:9092'],
|
|
group_id='order-processor',
|
|
value_deserializer=lambda m: json.loads(m.decode('utf-8')),
|
|
auto_offset_reset='earliest',
|
|
enable_auto_commit=False # Manual commit for reliability
|
|
)
|
|
|
|
# Consume messages
|
|
for message in consumer:
|
|
print(f"Partition: {message.partition}, Offset: {message.offset}")
|
|
print(f"Value: {message.value}")
|
|
|
|
# Process message
|
|
process_order(message.value)
|
|
|
|
# Commit after processing (at-least-once)
|
|
consumer.commit()
|
|
|
|
consumer.close()</code></pre>
|
|
<hr>
|
|
<h2>Kafka Streams API</h2>
|
|
<h3>Stream Processing (Java)</h3>
|
|
<pre><code class="lang-java">import org.apache.kafka.streams.*;
|
|
import org.apache.kafka.streams.kstream.*;
|
|
|
|
Properties props = new Properties();
|
|
props.put(StreamsConfig.APPLICATION_ID_CONFIG, "word-count-app");
|
|
props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
|
|
|
|
StreamsBuilder builder = new StreamsBuilder();
|
|
|
|
// Input topic
|
|
KStream<String, String> textLines = builder.stream("text-input");
|
|
|
|
// Process: split words → count
|
|
KTable<String, Long> wordCounts = textLines
|
|
.flatMapValues(line -> Arrays.asList(line.split("\\s+")))
|
|
.groupBy((key, word) -> word)
|
|
.count();
|
|
|
|
// Output topic
|
|
wordCounts.toStream().to("word-count-output");
|
|
|
|
// Start streams app
|
|
KafkaStreams streams = new KafkaStreams(builder.build(), props);
|
|
streams.start();</code></pre>
|
|
<h3>Stateless Operations</h3>
|
|
<pre><code class="lang-java">// Filter
|
|
KStream<String, Integer> evens = stream.filter((k, v) -> v % 2 == 0);
|
|
|
|
// Map
|
|
KStream<String, String> upper = stream.mapValues(v -> v.toUpperCase());
|
|
|
|
// FlatMap
|
|
KStream<String, String> words = stream.flatMapValues(
|
|
line -> Arrays.asList(line.split("\\s+"))
|
|
);
|
|
|
|
// Branch (split stream)
|
|
KStream<String, Integer>[] branches = stream.branch(
|
|
(k, v) -> v > 0, // Positive
|
|
(k, v) -> v < 0, // Negative
|
|
(k, v) -> true // Zero
|
|
);</code></pre>
|
|
<h3>Stateful Operations</h3>
|
|
<pre><code class="lang-java">// Group and count
|
|
KTable<String, Long> counts = stream
|
|
.groupByKey()
|
|
.count();
|
|
|
|
// Aggregate
|
|
KTable<String, Integer> sums = stream
|
|
.groupByKey()
|
|
.reduce((aggValue, newValue) -> aggValue + newValue);
|
|
|
|
// Custom aggregation
|
|
KTable<String, Stats> stats = stream
|
|
.groupByKey()
|
|
.aggregate(
|
|
Stats::new, // Initializer
|
|
(key, value, aggregate) -> {
|
|
aggregate.sum += value;
|
|
aggregate.count++;
|
|
return aggregate;
|
|
}
|
|
);</code></pre>
|
|
<h3>Windowing</h3>
|
|
<pre><code class="lang-java">// Tumbling window (5 minutes)
|
|
KTable<Windowed<String>, Long> counts = stream
|
|
.groupByKey()
|
|
.windowedBy(TimeWindows.of(Duration.ofMinutes(5)))
|
|
.count();
|
|
|
|
// Hopping window
|
|
KTable<Windowed<String>, Long> counts = stream
|
|
.groupByKey()
|
|
.windowedBy(TimeWindows.of(Duration.ofMinutes(10))
|
|
.advanceBy(Duration.ofMinutes(5)))
|
|
.count();
|
|
|
|
// Session window (30-minute inactivity gap)
|
|
KTable<Windowed<String>, Long> sessions = stream
|
|
.groupByKey()
|
|
.windowedBy(SessionWindows.with(Duration.ofMinutes(30)))
|
|
.count();</code></pre>
|
|
<hr>
|
|
<h2>Exactly-Once Semantics</h2>
|
|
<h3>Transactional Producer (Python)</h3>
|
|
<pre><code class="lang-python">from kafka import KafkaProducer
|
|
|
|
# Create transactional producer
|
|
producer = KafkaProducer(
|
|
transactional_id='order-service-1', # Unique ID
|
|
enable_idempotence=True,
|
|
acks='all'
|
|
)
|
|
|
|
# Initialize transactions
|
|
producer.init_transactions()
|
|
|
|
try:
|
|
# Begin transaction
|
|
producer.begin_transaction()
|
|
|
|
# Send messages (atomic across topics)
|
|
producer.send('orders', order_data)
|
|
producer.send('payments', payment_data)
|
|
|
|
# Commit transaction
|
|
producer.commit_transaction()
|
|
|
|
except Exception as e:
|
|
# Rollback on error
|
|
producer.abort_transaction()</code></pre>
|
|
<h3>Exactly-Once Streams</h3>
|
|
<pre><code class="lang-java">// Enable exactly-once in Kafka Streams
|
|
props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG,
|
|
StreamsConfig.EXACTLY_ONCE_V2);
|
|
|
|
// Streams app now has exactly-once guarantees:
|
|
// - Output writes are transactional
|
|
// - State stores are consistent
|
|
// - Offset commits are atomic</code></pre>
|
|
<hr>
|
|
<h2>Schema Registry (Avro)</h2>
|
|
<h3>Producer with Avro</h3>
|
|
<pre><code class="lang-python">from confluent_kafka import avro
|
|
from confluent_kafka.avro import AvroProducer
|
|
|
|
# Define schema
|
|
user_schema = avro.loads('''
|
|
{
|
|
"type": "record",
|
|
"name": "User",
|
|
"fields": [
|
|
{"name": "id", "type": "string"},
|
|
{"name": "name", "type": "string"},
|
|
{"name": "email", "type": "string"},
|
|
{"name": "age", "type": ["null", "int"], "default": null}
|
|
]
|
|
}
|
|
''')
|
|
|
|
# Create producer
|
|
producer = AvroProducer({
|
|
'bootstrap.servers': 'localhost:9092',
|
|
'schema.registry.url': 'http://localhost:8081'
|
|
}, default_value_schema=user_schema)
|
|
|
|
# Send message (Avro serialization automatic)
|
|
user = {
|
|
'id': 'user-1',
|
|
'name': 'Alice',
|
|
'email': '[email protected]',
|
|
'age': 30
|
|
}
|
|
producer.produce(topic='users', value=user)
|
|
producer.flush()</code></pre>
|
|
<h3>Consumer with Avro</h3>
|
|
<pre><code class="lang-python">from confluent_kafka.avro import AvroConsumer
|
|
|
|
consumer = AvroConsumer({
|
|
'bootstrap.servers': 'localhost:9092',
|
|
'group.id': 'user-consumer',
|
|
'schema.registry.url': 'http://localhost:8081'
|
|
})
|
|
|
|
consumer.subscribe(['users'])
|
|
|
|
while True:
|
|
msg = consumer.poll(1.0)
|
|
if msg is None:
|
|
continue
|
|
|
|
# Value automatically deserialized
|
|
user = msg.value()
|
|
print(f"User: {user['name']} ({user['email']})")</code></pre>
|
|
<hr>
|
|
<h2>ksqlDB</h2>
|
|
<h3>Basic Queries</h3>
|
|
<pre><code class="lang-sql">-- Create stream from topic
|
|
CREATE STREAM user_events (
|
|
user_id VARCHAR,
|
|
action VARCHAR,
|
|
timestamp BIGINT
|
|
) WITH (
|
|
KAFKA_TOPIC='user-events',
|
|
VALUE_FORMAT='JSON'
|
|
);
|
|
|
|
-- Query stream (continuous)
|
|
SELECT * FROM user_events EMIT CHANGES;
|
|
|
|
-- Create table (aggregation)
|
|
CREATE TABLE event_counts AS
|
|
SELECT user_id, COUNT(*) AS count
|
|
FROM user_events
|
|
GROUP BY user_id
|
|
EMIT CHANGES;
|
|
|
|
-- Query table (point-in-time)
|
|
SELECT * FROM event_counts WHERE user_id='user123';</code></pre>
|
|
<h3>Windowing</h3>
|
|
<pre><code class="lang-sql">-- Tumbling window (5 minutes)
|
|
CREATE TABLE event_counts_5min AS
|
|
SELECT
|
|
user_id,
|
|
COUNT(*) AS count,
|
|
WINDOWSTART AS window_start
|
|
FROM user_events
|
|
WINDOW TUMBLING (SIZE 5 MINUTES)
|
|
GROUP BY user_id
|
|
EMIT CHANGES;
|
|
|
|
-- Session window (30-minute gap)
|
|
CREATE TABLE user_sessions AS
|
|
SELECT user_id, COUNT(*) AS event_count
|
|
FROM user_events
|
|
WINDOW SESSION (30 MINUTES)
|
|
GROUP BY user_id
|
|
EMIT CHANGES;</code></pre>
|
|
<h3>Joins</h3>
|
|
<pre><code class="lang-sql">-- Stream-stream join
|
|
CREATE STREAM order_payments AS
|
|
SELECT
|
|
o.order_id,
|
|
o.amount,
|
|
p.payment_id,
|
|
p.status
|
|
FROM orders o
|
|
INNER JOIN payments p WITHIN 1 HOUR
|
|
ON o.order_id = p.order_id
|
|
EMIT CHANGES;
|
|
|
|
-- Stream-table join (enrichment)
|
|
CREATE STREAM enriched_orders AS
|
|
SELECT
|
|
o.order_id,
|
|
o.amount,
|
|
u.name AS user_name,
|
|
u.email AS user_email
|
|
FROM orders o
|
|
LEFT JOIN users u ON o.user_id = u.user_id
|
|
EMIT CHANGES;</code></pre>
|
|
<hr>
|
|
<h2>Performance Tuning</h2>
|
|
<h3>Producer Tuning</h3>
|
|
<p><strong>High throughput</strong>:</p>
|
|
<pre><code class="lang-python">producer = KafkaProducer(
|
|
acks=1, # Leader only
|
|
compression_type='lz4',
|
|
batch_size=32768, # 32 KB
|
|
linger_ms=20, # Wait to batch
|
|
buffer_memory=67108864 # 64 MB
|
|
)</code></pre>
|
|
<p><strong>High reliability</strong>:</p>
|
|
<pre><code class="lang-python">producer = KafkaProducer(
|
|
acks='all', # All ISR
|
|
enable_idempotence=True,
|
|
retries=3,
|
|
max_in_flight_requests_per_connection=5
|
|
)</code></pre>
|
|
<h3>Consumer Tuning</h3>
|
|
<p><strong>High throughput</strong>:</p>
|
|
<pre><code class="lang-python">consumer = KafkaConsumer(
|
|
fetch_min_bytes=1048576, # 1 MB
|
|
max_poll_records=1000
|
|
)</code></pre>
|
|
<p><strong>Low latency</strong>:</p>
|
|
<pre><code class="lang-python">consumer = KafkaConsumer(
|
|
fetch_min_bytes=1,
|
|
fetch_max_wait_ms=0
|
|
)</code></pre>
|
|
<hr>
|
|
<h2>Monitoring</h2>
|
|
<h3>Key Metrics</h3>
|
|
<p><strong>Broker health</strong>:</p>
|
|
<ul><li><code>UnderReplicatedPartitions</code>: Should be 0</li><li><code>OfflinePartitionsCount</code>: Should be 0</li><li><code>ActiveControllerCount</code>: Should be 1</li></ul>
|
|
<p><strong>Producer</strong>:</p>
|
|
<ul><li><code>record-send-rate</code>: Messages/sec</li><li><code>record-error-rate</code>: Errors/sec</li><li><code>request-latency-avg</code>: Latency in ms</li></ul>
|
|
<p><strong>Consumer</strong>:</p>
|
|
<ul><li><code>records-lag-max</code>: Max lag across partitions</li><li><code>records-consumed-rate</code>: Messages/sec</li><li><code>fetch-latency-avg</code>: Fetch latency</li></ul>
|
|
<h3>Consumer Lag</h3>
|
|
<pre><code class="lang-bash"># Check consumer lag
|
|
kafka-consumer-groups.sh --bootstrap-server localhost:9092 \
|
|
--group my-group --describe
|
|
|
|
# Output:
|
|
# TOPIC PARTITION CURRENT-OFFSET LOG-END-OFFSET LAG
|
|
# orders 0 1000 1500 500
|
|
# orders 1 2000 2000 0</code></pre>
|
|
<p><strong>Thresholds</strong>:</p>
|
|
<ul><li>Lag < 1000: Healthy</li><li>Lag 1000-10000: Warning</li><li>Lag > 10000: Critical</li></ul>
|
|
<hr>
|
|
<h2>Common Patterns</h2>
|
|
<h3>Partitioning Strategies</h3>
|
|
<p><strong>Key-based</strong> (ordering guarantee):</p>
|
|
<pre><code class="lang-python">producer.send('orders', key='user123', value=data)
|
|
# Same key → same partition → ordering preserved</code></pre>
|
|
<p><strong>Custom partitioner</strong>:</p>
|
|
<pre><code class="lang-python">from kafka.partitioner import Partitioner
|
|
|
|
class VIPPartitioner(Partitioner):
|
|
def partition(self, key, all_partitions, available):
|
|
if key.startswith('vip_'):
|
|
return 0 # VIP users to partition 0
|
|
return hash(key) % len(all_partitions)</code></pre>
|
|
<h3>Offset Management</h3>
|
|
<p><strong>Auto-commit</strong> (at-most-once):</p>
|
|
<pre><code class="lang-python">consumer = KafkaConsumer(enable_auto_commit=True)
|
|
# Simple, but can lose data on crash</code></pre>
|
|
<p><strong>Manual commit</strong> (at-least-once):</p>
|
|
<pre><code class="lang-python">consumer = KafkaConsumer(enable_auto_commit=False)
|
|
for msg in consumer:
|
|
process(msg)
|
|
consumer.commit() # Commit after processing</code></pre>
|
|
<p><strong>Batch commit</strong> (throughput):</p>
|
|
<pre><code class="lang-python">for i, msg in enumerate(consumer):
|
|
process(msg)
|
|
if i % 100 == 0:
|
|
consumer.commit()</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<p>❌ <strong>Creating producer per message</strong>: Expensive connection overhead ✅ Create once, reuse for all messages</p>
|
|
<p>❌ <strong>Not setting timeouts</strong>: Can hang forever ✅ Always set <code>request_timeout_ms</code>, <code>timeout</code> parameters</p>
|
|
<p>❌ <strong>Large messages (> 1 MB)</strong>: Performance issues ✅ Split into smaller messages or use compression</p>
|
|
<p>❌ <strong>No replication</strong>: Data loss on broker failure ✅ Use <code>replication.factor=3</code>, <code>min.insync.replicas=2</code></p>
|
|
<p>❌ <strong>Too many partitions</strong>: Overhead, slow rebalancing ✅ Start with 10-100 partitions per topic</p>
|
|
<p>❌ <strong>Ignoring consumer lag</strong>: Consumers falling behind ✅ Monitor lag, alert on thresholds, scale consumers</p>
|
|
<hr>
|
|
<h2>Level 3: Resources</h2>
|
|
<h3>Overview</h3>
|
|
<p>This skill includes comprehensive Level 3 resources for deep Kafka knowledge and practical automation tools.</p>
|
|
<p><strong>Resources include</strong>:</p>
|
|
<ul><li><strong>REFERENCE.md</strong> (2,438 lines): Complete technical reference covering all Kafka concepts</li><li><strong>3 executable scripts</strong>: Config validation, consumer lag analysis, throughput benchmarking</li><li><strong>9 production examples</strong>: Complete implementations in Python, Java, with Docker stack</li></ul>
|
|
<h3>REFERENCE.md</h3>
|
|
<p><strong>Location</strong>: <code>skills/protocols/kafka-streams/resources/REFERENCE.md</code></p>
|
|
<p><strong>Comprehensive technical reference</strong> (2,438 lines) covering:</p>
|
|
<p><strong>Core Topics</strong>:</p>
|
|
<ul><li>Kafka fundamentals and architecture</li><li>Producer API (Python, Java) - sync, async, batching, transactions</li><li>Consumer API - offset management, rebalancing, partition control</li><li>Kafka Streams API - stateless/stateful operations, windowing, joins</li><li>Exactly-once semantics (EOS) - transactions, idempotence</li><li>Schema Registry - Avro, Protobuf, JSON Schema evolution</li><li>ksqlDB - streams, tables, windowing, joins, aggregations</li><li>Performance tuning - producer/consumer/broker optimization</li><li>Monitoring - JMX metrics, Prometheus, consumer lag</li><li>Security - SASL, TLS, ACLs</li><li>Testing - unit testing, integration testing, Testcontainers</li></ul>
|
|
<p><strong>Key Sections</strong>:</p>
|
|
<ol><li><strong>Fundamentals</strong>: Topics, partitions, brokers, replication, ISR</li><li><strong>Producer API</strong>: Configuration, partitioning, batching, compression</li><li><strong>Consumer API</strong>: Groups, offset management, rebalancing</li><li><strong>Kafka Streams</strong>: Topology, stateless/stateful ops, state stores</li><li><strong>Windowing</strong>: Tumbling, hopping, sliding, session windows</li><li><strong>Joins</strong>: Stream-stream, stream-table, table-table joins</li><li><strong>Exactly-Once</strong>: Transactional producer, idempotent writes, EOS v2</li><li><strong>Schema Registry</strong>: Avro schemas, backward/forward compatibility</li><li><strong>ksqlDB</strong>: SQL queries, push/pull queries, materialized views</li><li><strong>Performance</strong>: Throughput vs latency tuning, partition sizing</li><li><strong>Monitoring</strong>: Key metrics, alerting thresholds, lag monitoring</li><li><strong>Security</strong>: Authentication (SASL), encryption (TLS), authorization (ACLs)</li></ol>
|
|
<p><strong>Format</strong>: Markdown with extensive code examples in Python, Java</p>
|
|
<h3>Scripts</h3>
|
|
<p>Three production-ready executable scripts in <code>resources/scripts/</code>:</p>
|
|
<h4>1. validate_kafka_config.py (588 lines)</h4>
|
|
<p><strong>Purpose</strong>: Validate Kafka broker and topic configurations against best practices</p>
|
|
<p><strong>Features</strong>:</p>
|
|
<ul><li>Parse and validate broker configs (threads, buffers, replication)</li><li>Check topic settings (partitions, replication factor, ISR)</li><li>Detect misconfigurations (low replication, missing ISR)</li><li>Validate cluster health (active controller, broker count)</li><li>Output as JSON or human-readable text</li></ul>
|
|
<p><strong>Usage</strong>:</p>
|
|
<pre><code class="lang-bash"># Validate entire cluster
|
|
./validate_kafka_config.py --bootstrap-servers localhost:9092
|
|
|
|
# Validate specific topics
|
|
./validate_kafka_config.py --bootstrap-servers localhost:9092 --topics orders,payments
|
|
|
|
# JSON output for CI/CD
|
|
./validate_kafka_config.py --bootstrap-servers localhost:9092 --json > report.json</code></pre>
|
|
<p><strong>Categories checked</strong>:</p>
|
|
<ul><li>Cluster health (broker count, active controller)</li><li>Broker configs (network threads, I/O threads, buffers)</li><li>Topic configs (partitions, replication factor, min.insync.replicas)</li><li>Retention policies (log retention, cleanup policy)</li></ul>
|
|
<h4>2. analyze_consumer_lag.py (537 lines)</h4>
|
|
<p><strong>Purpose</strong>: Monitor and analyze consumer lag across consumer groups</p>
|
|
<p><strong>Features</strong>:</p>
|
|
<ul><li>Calculate lag per partition (current offset vs end offset)</li><li>Identify slow consumers and stuck partitions</li><li>Aggregate lag by consumer group</li><li>Classify groups (healthy, warning, critical)</li><li>Output lag statistics and trends</li></ul>
|
|
<p><strong>Usage</strong>:</p>
|
|
<pre><code class="lang-bash"># Analyze all consumer groups
|
|
./analyze_consumer_lag.py --bootstrap-servers localhost:9092
|
|
|
|
# Analyze specific group
|
|
./analyze_consumer_lag.py --bootstrap-servers localhost:9092 --group order-processor
|
|
|
|
# Custom lag threshold
|
|
./analyze_consumer_lag.py --bootstrap-servers localhost:9092 --threshold 5000
|
|
|
|
# JSON output for monitoring systems
|
|
./analyze_consumer_lag.py --bootstrap-servers localhost:9092 --json</code></pre>
|
|
<p><strong>Metrics</strong>:</p>
|
|
<ul><li>Total lag per group</li><li>Max partition lag</li><li>Stuck partitions (not consuming)</li><li>Slow consumers (high lag per consumer)</li></ul>
|
|
<p><strong>Thresholds</strong>:</p>
|
|
<ul><li>Healthy: lag < 1000</li><li>Warning: lag 1000-10000</li><li>Critical: lag > 10000</li></ul>
|
|
<h4>3. benchmark_throughput.py (620 lines)</h4>
|
|
<p><strong>Purpose</strong>: Benchmark Kafka producer/consumer throughput and latency</p>
|
|
<p><strong>Features</strong>:</p>
|
|
<ul><li>Producer benchmark (messages/sec, MB/sec)</li><li>Consumer benchmark (end-to-end latency)</li><li>Latency percentiles (p50, p95, p99)</li><li>Configurable message size and count</li><li>JSON output for automation</li></ul>
|
|
<p><strong>Usage</strong>:</p>
|
|
<pre><code class="lang-bash"># Producer benchmark (100k messages)
|
|
./benchmark_throughput.py --bootstrap-servers localhost:9092 --mode producer --messages 100000
|
|
|
|
# Consumer benchmark
|
|
./benchmark_throughput.py --bootstrap-servers localhost:9092 --mode consumer --topic orders
|
|
|
|
# End-to-end benchmark (producer + consumer)
|
|
./benchmark_throughput.py --bootstrap-servers localhost:9092 --mode both --messages 50000
|
|
|
|
# Custom message size (10 KB)
|
|
./benchmark_throughput.py --bootstrap-servers localhost:9092 --mode producer --messages 10000 --size 10240
|
|
|
|
# JSON output
|
|
./benchmark_throughput.py --bootstrap-servers localhost:9092 --mode producer --messages 10000 --json</code></pre>
|
|
<p><strong>Metrics</strong>:</p>
|
|
<ul><li>Throughput: messages/sec, MB/sec</li><li>Latency: min, avg, p50, p95, p99, max</li><li>Success rate and error count</li></ul>
|
|
<h3>Examples</h3>
|
|
<p>Nine production-ready examples in <code>resources/examples/</code>:</p>
|
|
<h4>1. python/basic_producer.py (200 lines)</h4>
|
|
<p>Complete Python producer demonstrating:</p>
|
|
<ul><li>Synchronous send (blocking, low throughput)</li><li>Asynchronous send (non-blocking, high throughput)</li><li>Batch send (highest throughput)</li><li>Send with headers (metadata)</li><li>Error handling and callbacks</li><li>Production-ready configuration</li></ul>
|
|
<h4>2. python/basic_consumer.py (280 lines)</h4>
|
|
<p>Complete Python consumer demonstrating:</p>
|
|
<ul><li>Auto-commit (at-most-once)</li><li>Manual commit (at-least-once)</li><li>Batch commit (throughput)</li><li>Manual partition assignment</li><li>Multiple topic subscription</li><li>Header access</li><li>Production-ready configuration</li></ul>
|
|
<h4>3. python/exactly_once_producer.py (180 lines)</h4>
|
|
<p>Transactional producer with exactly-once semantics:</p>
|
|
<ul><li>Transactional ID and initialization</li><li>Begin/commit/abort transactions</li><li>Atomic writes across multiple topics</li><li>Error handling and rollback</li><li>Multiple transaction sequence</li></ul>
|
|
<h4>4. python/avro_schema_registry.py (220 lines)</h4>
|
|
<p>Avro serialization with Schema Registry:</p>
|
|
<ul><li>Define Avro schemas (V1, V2 with evolution)</li><li>Producer with Avro serialization</li><li>Consumer with Avro deserialization</li><li>Complex schemas (enums, arrays, nested records)</li><li>Schema evolution (backward/forward compatible)</li></ul>
|
|
<h4>5. java/WordCountStreamsApp.java (200 lines)</h4>
|
|
<p>Kafka Streams application (Java):</p>
|
|
<ul><li>Complete word count topology</li><li>Stateless operations (filter, map, flatMap)</li><li>Stateful operations (groupBy, count, aggregate)</li><li>Advanced topology with windowing and joins</li><li>Exactly-once processing (EOS v2)</li><li>State stores and changelog topics</li></ul>
|
|
<h4>6. ksqldb/queries.sql (500 lines)</h4>
|
|
<p>Comprehensive ksqlDB query examples:</p>
|
|
<ul><li>Create streams and tables</li><li>Filter and transform data</li><li>Aggregations (count, sum, avg)</li><li>Windowing (tumbling, hopping, session)</li><li>Joins (stream-stream, stream-table, table-table)</li><li>Advanced functions (string, date/time, conditional)</li><li>Push and pull queries</li><li>Management commands</li></ul>
|
|
<h4>7. docker/docker-compose.yml (280 lines)</h4>
|
|
<p>Complete Kafka stack with Docker Compose:</p>
|
|
<ul><li>Kafka broker (with JMX monitoring)</li><li>ZooKeeper</li><li>Schema Registry</li><li>Kafka Connect</li><li>ksqlDB server and CLI</li><li>Control Center (Web UI)</li><li>REST Proxy</li><li>Production-ready configuration with health checks</li></ul>
|
|
<p><strong>Services</strong>:</p>
|
|
<ul><li>Kafka: localhost:9092</li><li>Schema Registry: http://localhost:8081</li><li>ksqlDB: http://localhost:8088</li><li>Control Center: http://localhost:9021</li><li>REST Proxy: http://localhost:8082</li></ul>
|
|
<h4>8. monitoring/prometheus_config.yml (250 lines)</h4>
|
|
<p>Prometheus configuration for Kafka monitoring:</p>
|
|
<ul><li>Scrape configs for broker, ZooKeeper, Schema Registry</li><li>Alert rules (under-replicated partitions, high lag, offline partitions)</li><li>Consumer lag monitoring</li><li>JMX metrics collection</li><li>Grafana dashboard integration</li></ul>
|
|
<p><strong>Key alerts</strong>:</p>
|
|
<ul><li>Broker down</li><li>Under-replicated partitions</li><li>Offline partitions</li><li>No active controller</li><li>High consumer lag</li><li>Critical consumer lag</li></ul>
|
|
<h4>9. examples/README.md (420 lines)</h4>
|
|
<p>Comprehensive guide with:</p>
|
|
<ul><li>Quick start instructions</li><li>Example usage for each pattern</li><li>Configuration tuning guidelines</li><li>Troubleshooting common issues</li><li>Best practices checklist</li></ul>
|
|
<h3>Quick Start</h3>
|
|
<p><strong>1. Validate Kafka cluster</strong>:</p>
|
|
<pre><code class="lang-bash">cd skills/protocols/kafka-streams/resources/scripts
|
|
./validate_kafka_config.py --bootstrap-servers localhost:9092 --json</code></pre>
|
|
<p><strong>2. Monitor consumer lag</strong>:</p>
|
|
<pre><code class="lang-bash">./analyze_consumer_lag.py --bootstrap-servers localhost:9092 --group my-group</code></pre>
|
|
<p><strong>3. Benchmark throughput</strong>:</p>
|
|
<pre><code class="lang-bash">./benchmark_throughput.py --bootstrap-servers localhost:9092 --mode both --messages 10000</code></pre>
|
|
<p><strong>4. Run examples with Docker</strong>:</p>
|
|
<pre><code class="lang-bash">cd ../examples/docker
|
|
docker-compose up -d
|
|
|
|
# Wait for services
|
|
docker-compose logs -f kafka
|
|
|
|
# Access Control Center: http://localhost:9021</code></pre>
|
|
<p><strong>5. Test producer/consumer</strong>:</p>
|
|
<pre><code class="lang-bash">cd ../examples/python
|
|
python basic_producer.py
|
|
python basic_consumer.py # In another terminal</code></pre>
|
|
<h3>File Structure</h3>
|
|
<pre><code>skills/protocols/kafka-streams/
|
|
├── kafka-streams.md (this file)
|
|
└── resources/
|
|
├── REFERENCE.md (2,438 lines)
|
|
├── scripts/
|
|
│ ├── validate_kafka_config.py (588 lines)
|
|
│ ├── analyze_consumer_lag.py (537 lines)
|
|
│ └── benchmark_throughput.py (620 lines)
|
|
└── examples/
|
|
├── README.md (420 lines)
|
|
├── python/
|
|
│ ├── basic_producer.py (200 lines)
|
|
│ ├── basic_consumer.py (280 lines)
|
|
│ ├── exactly_once_producer.py (180 lines)
|
|
│ └── avro_schema_registry.py (220 lines)
|
|
├── java/
|
|
│ └── WordCountStreamsApp.java (200 lines)
|
|
├── ksqldb/
|
|
│ └── queries.sql (500 lines)
|
|
├── docker/
|
|
│ └── docker-compose.yml (280 lines)
|
|
└── monitoring/
|
|
└── prometheus_config.yml (250 lines)</code></pre>
|
|
<h3>Resources Summary</h3>
|
|
<p>| Category | Item | Lines | Description | |----------|------|-------|-------------| | <strong>Reference</strong> | REFERENCE.md | 2,438 | Complete technical reference | | <strong>Scripts</strong> | validate_kafka_config.py | 588 | Config validator | | | analyze_consumer_lag.py | 537 | Lag analyzer | | | benchmark_throughput.py | 620 | Throughput benchmark | | <strong>Examples</strong> | basic_producer.py | 200 | Producer patterns | | | basic_consumer.py | 280 | Consumer patterns | | | exactly_once_producer.py | 180 | Transactions | | | avro_schema_registry.py | 220 | Schema Registry | | | WordCountStreamsApp.java | 200 | Kafka Streams | | | queries.sql | 500 | ksqlDB queries | | | docker-compose.yml | 280 | Docker stack | | | prometheus_config.yml | 250 | Monitoring | | | README.md | 420 | Examples guide |</p>
|
|
<p><strong>Total</strong>: 6,713 lines of production-ready resources</p>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-27 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/protocols-kafka-streams.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |