mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
725 lines
31 KiB
HTML
725 lines
31 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>data-differential-dataflow</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/data-differential-dataflow.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/data-differential-dataflow">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/data-differential-dataflow.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>data-differential-dataflow</h1><p class="lede">Differential computation for incremental updates, maintaining indexed collections and efficient joins</p><h1>Differential Dataflow</h1>
|
|
<p><strong>Scope</strong>: Incremental computation, arrangements, differential updates, efficient joins <strong>Lines</strong>: 390 <strong>Last Updated</strong>: 2025-10-27 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Use this skill when:</p>
|
|
<ul><li>Building systems that need incremental computation (avoid full recomputation)</li><li>Maintaining materialized views that update efficiently</li><li>Implementing real-time dashboards over changing data</li><li>Performing joins on large, slowly-changing datasets</li><li>Creating reactive systems that respond to data changes</li><li>Building graph algorithms that update incrementally</li><li>Optimizing computations that repeat with small data changes</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Differential vs Standard Dataflow</h3>
|
|
<pre><code>Standard Dataflow (Timely)
|
|
→ Process complete data at each timestamp
|
|
→ Full recomputation for each change
|
|
→ Good for: Stateless transformations
|
|
|
|
Differential Dataflow
|
|
→ Process only changes (deltas)
|
|
→ Maintains indexed state (arrangements)
|
|
→ Incrementally updates results
|
|
→ Good for: Joins, aggregations, iterative algorithms</code></pre>
|
|
<h3>Collections and Updates</h3>
|
|
<pre><code>Collection<G, D>
|
|
→ Multiset of data D at times G::Timestamp
|
|
→ Represented as (data, time, diff) triples
|
|
→ diff: +1 (insert), -1 (delete), +N (multiplicity)
|
|
|
|
Example:
|
|
("alice", 0, +1) → Insert "alice" at time 0
|
|
("alice", 5, -1) → Delete "alice" at time 5
|
|
("bob", 0, +2) → Insert "bob" twice at time 0</code></pre>
|
|
<h3>Arrangements (Indexed State)</h3>
|
|
<pre><code>Arrangement
|
|
→ Indexed collection for efficient lookup
|
|
→ Key-value pairs: (key, value, time, diff)
|
|
→ Enables fast joins without full scans
|
|
→ Shared across operators (zero-copy)
|
|
|
|
TraceHandle
|
|
→ Reference to arranged data
|
|
→ Multiple operators can share same arrangement
|
|
→ Compaction merges historical updates</code></pre>
|
|
<h3>Differential Operators</h3>
|
|
<pre><code>Map/Filter
|
|
→ Transform collections
|
|
→ diff propagates through transformations
|
|
|
|
Join
|
|
→ Efficient incremental join on arrangements
|
|
→ Updates only affected outputs
|
|
→ O(delta) complexity, not O(data)
|
|
|
|
Reduce
|
|
→ Group-by aggregation
|
|
→ Incremental update on changes
|
|
→ Maintains grouped state
|
|
|
|
Iterate
|
|
→ Fixed-point computation
|
|
→ Converges when no more changes
|
|
→ Efficient for graph algorithms</code></pre>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Basic Collections and Transformations</h3>
|
|
<pre><code class="lang-rust">// Cargo.toml
|
|
// [dependencies]
|
|
// timely = "0.12"
|
|
// differential-dataflow = "0.12"
|
|
|
|
use differential_dataflow::input::Input;
|
|
use differential_dataflow::operators::{Join, Reduce};
|
|
|
|
fn main() {
|
|
timely::execute_from_args(std::env::args(), |worker| {
|
|
let mut input = worker.dataflow::<u64, _, _>(|scope| {
|
|
let (input, data) = scope.new_collection();
|
|
|
|
// Transform collection
|
|
data.map(|x| x * 2)
|
|
.filter(|x| *x > 10)
|
|
.inspect(|x| println!("Filtered: {:?}", x));
|
|
|
|
input
|
|
});
|
|
|
|
// Insert data
|
|
input.insert(5); // (5, +1)
|
|
input.insert(10); // (10, +1)
|
|
input.advance_to(1);
|
|
worker.step_while(|| input.time().less_than(&1));
|
|
|
|
// Remove data
|
|
input.remove(5); // (5, -1)
|
|
input.advance_to(2);
|
|
worker.step_while(|| input.time().less_than(&2));
|
|
|
|
// Output shows only changes
|
|
}).expect("Execution failed");
|
|
}</code></pre>
|
|
<h3>Pattern 2: Incremental Join</h3>
|
|
<pre><code class="lang-rust">use differential_dataflow::input::Input;
|
|
use differential_dataflow::operators::Join;
|
|
|
|
#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
|
|
struct User {
|
|
id: u32,
|
|
name: String,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
|
|
struct Order {
|
|
user_id: u32,
|
|
product: String,
|
|
amount: f64,
|
|
}
|
|
|
|
fn main() {
|
|
timely::execute_from_args(std::env::args(), |worker| {
|
|
let (mut users_in, mut orders_in) = worker.dataflow::<u64, _, _>(|scope| {
|
|
let (users_in, users) = scope.new_collection();
|
|
let (orders_in, orders) = scope.new_collection();
|
|
|
|
// Join users with orders
|
|
// Convert to (key, value) format
|
|
let users_keyed = users.map(|u| (u.id, u.name.clone()));
|
|
let orders_keyed = orders.map(|o| (o.user_id, (o.product.clone(), o.amount)));
|
|
|
|
// Incremental join
|
|
users_keyed
|
|
.join(&orders_keyed)
|
|
.inspect(|(user_id, (name, (product, amount)))| {
|
|
println!("User {} ({}) ordered {} for ${}",
|
|
user_id, name, product, amount);
|
|
});
|
|
|
|
(users_in, orders_in)
|
|
});
|
|
|
|
// Initial data at time 0
|
|
users_in.insert(User { id: 1, name: "Alice".to_string() });
|
|
users_in.insert(User { id: 2, name: "Bob".to_string() });
|
|
orders_in.insert(Order { user_id: 1, product: "Laptop".to_string(), amount: 999.0 });
|
|
|
|
users_in.advance_to(1);
|
|
orders_in.advance_to(1);
|
|
worker.step_while(|| users_in.time().less_than(&1));
|
|
|
|
// Add order at time 1 - only processes this change
|
|
orders_in.insert(Order { user_id: 2, product: "Mouse".to_string(), amount: 25.0 });
|
|
|
|
users_in.advance_to(2);
|
|
orders_in.advance_to(2);
|
|
worker.step_while(|| orders_in.time().less_than(&2));
|
|
|
|
// Update user name at time 2 - join recomputes only affected records
|
|
users_in.remove(User { id: 1, name: "Alice".to_string() });
|
|
users_in.insert(User { id: 1, name: "Alice Smith".to_string() });
|
|
|
|
users_in.advance_to(3);
|
|
orders_in.advance_to(3);
|
|
worker.step_while(|| users_in.time().less_than(&3));
|
|
}).expect("Execution failed");
|
|
}</code></pre>
|
|
<h3>Pattern 3: Arrangements for Shared State</h3>
|
|
<pre><code class="lang-rust">use differential_dataflow::input::Input;
|
|
use differential_dataflow::operators::{Join, Reduce, Threshold, arrange::ArrangeBySelf};
|
|
use differential_dataflow::operators::arrange::ArrangeByKey;
|
|
|
|
fn main() {
|
|
timely::execute_from_args(std::env::args(), |worker| {
|
|
let mut input = worker.dataflow::<u64, _, _>(|scope| {
|
|
let (input, edges) = scope.new_collection::<(u32, u32)>();
|
|
|
|
// Arrange edges by source node (shared arrangement)
|
|
let edges_arranged = edges.arrange_by_key();
|
|
|
|
// Query 1: Count outgoing edges per node
|
|
edges_arranged
|
|
.reduce(|_src, input, output| {
|
|
let count: i32 = input.iter().map(|(_, diff)| diff).sum();
|
|
output.push((count, 1));
|
|
})
|
|
.inspect(|(node, count)| {
|
|
println!("Node {} has {} outgoing edges", node, count);
|
|
});
|
|
|
|
// Query 2: Find nodes with edges (using same arrangement)
|
|
edges_arranged
|
|
.as_collection(|&src, &_dst| src)
|
|
.distinct()
|
|
.inspect(|node| println!("Active node: {}", node));
|
|
|
|
// Query 3: Two-hop paths (self-join on arrangement)
|
|
edges_arranged
|
|
.join_core(&edges_arranged, |&src, &mid, &dst| {
|
|
Some((src, mid, dst))
|
|
})
|
|
.inspect(|(src, mid, dst)| {
|
|
println!("Path: {} -> {} -> {}", src, mid, dst);
|
|
});
|
|
|
|
input
|
|
});
|
|
|
|
// Insert graph edges
|
|
input.insert((1, 2));
|
|
input.insert((2, 3));
|
|
input.insert((1, 3));
|
|
input.advance_to(1);
|
|
worker.step_while(|| input.time().less_than(&1));
|
|
|
|
// Add new edge - only affected queries recompute
|
|
input.insert((3, 4));
|
|
input.advance_to(2);
|
|
worker.step_while(|| input.time().less_than(&2));
|
|
}).expect("Execution failed");
|
|
}</code></pre>
|
|
<h3>Pattern 4: Incremental Aggregation (Group-By Reduce)</h3>
|
|
<pre><code class="lang-rust">use differential_dataflow::input::Input;
|
|
use differential_dataflow::operators::Reduce;
|
|
|
|
#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
|
|
struct Sale {
|
|
category: String,
|
|
amount: i32,
|
|
}
|
|
|
|
fn main() {
|
|
timely::execute_from_args(std::env::args(), |worker| {
|
|
let mut input = worker.dataflow::<u64, _, _>(|scope| {
|
|
let (input, sales) = scope.new_collection();
|
|
|
|
// Group by category and sum amounts
|
|
sales
|
|
.map(|sale| (sale.category.clone(), sale.amount))
|
|
.reduce(|_category, amounts, output| {
|
|
let total: i32 = amounts.iter()
|
|
.map(|(amount, diff)| amount * diff)
|
|
.sum();
|
|
output.push((total, 1));
|
|
})
|
|
.inspect(|(category, total)| {
|
|
println!("Category {}: total = {}", category, total);
|
|
});
|
|
|
|
input
|
|
});
|
|
|
|
// Initial sales
|
|
input.insert(Sale { category: "Electronics".to_string(), amount: 100 });
|
|
input.insert(Sale { category: "Electronics".to_string(), amount: 200 });
|
|
input.insert(Sale { category: "Books".to_string(), amount: 50 });
|
|
input.advance_to(1);
|
|
worker.step_while(|| input.time().less_than(&1));
|
|
|
|
// Add sale - only Electronics group recomputes
|
|
input.insert(Sale { category: "Electronics".to_string(), amount: 150 });
|
|
input.advance_to(2);
|
|
worker.step_while(|| input.time().less_than(&2));
|
|
|
|
// Remove sale - incremental update
|
|
input.remove(Sale { category: "Books".to_string(), amount: 50 });
|
|
input.advance_to(3);
|
|
worker.step_while(|| input.time().less_than(&3));
|
|
}).expect("Execution failed");
|
|
}</code></pre>
|
|
<h3>Pattern 5: Iterative Computation (Connected Components)</h3>
|
|
<pre><code class="lang-rust">use differential_dataflow::input::Input;
|
|
use differential_dataflow::operators::{Join, Consolidate, Iterate};
|
|
|
|
fn main() {
|
|
timely::execute_from_args(std::env::args(), |worker| {
|
|
let mut input = worker.dataflow::<u64, _, _>(|scope| {
|
|
let (input, edges) = scope.new_collection::<(u32, u32)>();
|
|
|
|
// Compute connected components
|
|
let labels = edges.iterate(|inner| {
|
|
let edges = edges.enter(&inner.scope());
|
|
|
|
// Start with self-labels: node -> node
|
|
let nodes = edges.flat_map(|(src, dst)| vec![src, dst])
|
|
.distinct()
|
|
.map(|node| (node, node));
|
|
|
|
// Propagate minimum label along edges
|
|
inner.join(&edges.map(|(src, dst)| (src, dst)))
|
|
.map(|(_src, (label, dst))| (dst, label))
|
|
.concat(&nodes)
|
|
.reduce(|_node, labels, output| {
|
|
// Keep minimum label
|
|
let min_label = labels.iter()
|
|
.map(|(label, _)| *label)
|
|
.min()
|
|
.unwrap();
|
|
output.push((min_label, 1));
|
|
})
|
|
});
|
|
|
|
labels.inspect(|(node, component)| {
|
|
println!("Node {} in component {}", node, component);
|
|
});
|
|
|
|
input
|
|
});
|
|
|
|
// Graph with two components: {1,2,3} and {4,5}
|
|
input.insert((1, 2));
|
|
input.insert((2, 3));
|
|
input.insert((4, 5));
|
|
input.advance_to(1);
|
|
worker.step_while(|| input.time().less_than(&1));
|
|
|
|
// Bridge components - iterates until convergence
|
|
input.insert((3, 4));
|
|
input.advance_to(2);
|
|
worker.step_while(|| input.time().less_than(&2));
|
|
}).expect("Execution failed");
|
|
}</code></pre>
|
|
<h3>Pattern 6: External State Updates (Probing)</h3>
|
|
<pre><code class="lang-rust">use differential_dataflow::input::InputSession;
|
|
use differential_dataflow::operators::Join;
|
|
use timely::dataflow::operators::probe::Handle as ProbeHandle;
|
|
|
|
fn main() {
|
|
timely::execute_from_args(std::env::args(), |worker| {
|
|
let mut input = InputSession::new();
|
|
let mut probe = ProbeHandle::new();
|
|
|
|
worker.dataflow::<u64, _, _>(|scope| {
|
|
input.to_collection(scope)
|
|
.map(|x| x * 2)
|
|
.inspect(|x| println!("Result: {:?}", x))
|
|
.probe_with(&mut probe);
|
|
});
|
|
|
|
// External loop with synchronization
|
|
for round in 0..10 {
|
|
input.insert(round);
|
|
input.advance_to(round + 1);
|
|
input.flush();
|
|
|
|
// Wait for completion
|
|
while probe.less_than(input.time()) {
|
|
worker.step();
|
|
}
|
|
|
|
println!("Round {} complete", round);
|
|
}
|
|
}).expect("Execution failed");
|
|
}</code></pre>
|
|
<h2>Advanced Patterns</h2>
|
|
<h3>Custom Reduction Logic</h3>
|
|
<pre><code class="lang-rust">use differential_dataflow::operators::Reduce;
|
|
|
|
// Top-K items per group
|
|
collection
|
|
.map(|(group, value)| (group, value))
|
|
.reduce(move |_group, values, output| {
|
|
let k = 10;
|
|
let mut sorted: Vec<_> = values.iter()
|
|
.map(|(val, diff)| (*val, *diff))
|
|
.collect();
|
|
sorted.sort_by_key(|(val, _)| std::cmp::Reverse(*val));
|
|
|
|
for (val, diff) in sorted.into_iter().take(k) {
|
|
output.push((val, diff));
|
|
}
|
|
})</code></pre>
|
|
<h3>Threshold (Distinct)</h3>
|
|
<pre><code class="lang-rust">use differential_dataflow::operators::Threshold;
|
|
|
|
// Remove duplicates (keep diff = 1)
|
|
collection.distinct()
|
|
|
|
// Keep only positive diffs
|
|
collection.threshold(|_, count| if *count > 0 { 1 } else { 0 })
|
|
|
|
// Custom threshold logic
|
|
collection.threshold(|value, count| {
|
|
if *value > 100 && *count > 5 { 1 } else { 0 }
|
|
})</code></pre>
|
|
<h3>Multi-way Joins</h3>
|
|
<pre><code class="lang-rust">// Three-way join
|
|
let result = collection1
|
|
.join(&collection2)
|
|
.map(|(key, (val1, val2))| ((key, val1), val2))
|
|
.join(&collection3.map(|(key, val)| ((key, ()), val)))
|
|
.map(|((key, val1), ((), val2))| (key, val1, val2));</code></pre>
|
|
<h2>Performance Optimization</h2>
|
|
<h3>Arrangement Strategy</h3>
|
|
<pre><code class="lang-rust">// Arrange when:
|
|
// 1. Multiple operators read same data
|
|
// 2. Performing joins
|
|
// 3. Random access needed
|
|
|
|
let arranged = collection.arrange_by_key();
|
|
|
|
// Use arrangement multiple times (zero-copy)
|
|
arranged.as_collection(|k, v| (k.clone(), v.clone()));
|
|
arranged.join_core(&other_arranged, |k, v1, v2| ...);</code></pre>
|
|
<h3>Compaction Control</h3>
|
|
<pre><code class="lang-rust">// Configure trace compaction
|
|
use differential_dataflow::trace::implementations::ord::OrdValSpine;
|
|
|
|
let trace = collection
|
|
.arrange_by_key()
|
|
.trace;
|
|
|
|
// Compact aggressively (saves memory, costs CPU)
|
|
trace.set_physical_compaction(Duration::from_secs(1));
|
|
|
|
// Compact lazily (saves CPU, uses more memory)
|
|
trace.set_logical_compaction(Duration::from_secs(60));</code></pre>
|
|
<h3>Batch Size Tuning</h3>
|
|
<pre><code class="lang-rust">use differential_dataflow::input::InputSession;
|
|
|
|
let mut input = InputSession::new();
|
|
|
|
// Buffer insertions
|
|
for item in large_dataset {
|
|
input.insert(item);
|
|
|
|
// Periodic flush
|
|
if input.len() > 10000 {
|
|
input.flush();
|
|
}
|
|
}
|
|
|
|
input.advance_to(1);
|
|
input.flush();</code></pre>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Collection Operators</h3>
|
|
<pre><code class="lang-rust">// Transform
|
|
collection.map(|x| x * 2)
|
|
collection.flat_map(|x| vec![x, x * 2])
|
|
collection.filter(|x| *x > 10)
|
|
|
|
// Combine
|
|
collection1.concat(&collection2)
|
|
collection.distinct()
|
|
collection.negate() // Flip diff signs
|
|
|
|
// Join
|
|
keyed1.join(&keyed2)
|
|
keyed1.semijoin(&keyed2) // Keep keyed1 where key in keyed2
|
|
keyed1.antijoin(&keyed2) // Keep keyed1 where key NOT in keyed2
|
|
|
|
// Aggregate
|
|
keyed.reduce(|key, vals, output| { ... })
|
|
keyed.count() // Count occurrences
|
|
|
|
// Arrange
|
|
collection.arrange_by_key()
|
|
collection.arrange_by_self()</code></pre>
|
|
<h3>Input Management</h3>
|
|
<pre><code class="lang-rust">use differential_dataflow::input::{Input, InputSession};
|
|
|
|
// Simple input
|
|
let (mut input, collection) = scope.new_collection();
|
|
input.insert(data);
|
|
input.remove(data);
|
|
input.advance_to(time);
|
|
|
|
// Session input (buffered)
|
|
let mut input = InputSession::new();
|
|
input.insert(data);
|
|
input.flush(); // Send batched updates
|
|
input.advance_to(time);</code></pre>
|
|
<h3>Debugging</h3>
|
|
<pre><code class="lang-rust">// Inspect updates
|
|
collection.inspect(|(data, time, diff)| {
|
|
println!("Update: {:?} at {:?} with diff {}", data, time, diff);
|
|
});
|
|
|
|
// Consolidate before inspection (merge diffs)
|
|
collection.consolidate().inspect(|x| println!("{:?}", x));
|
|
|
|
// Count updates
|
|
collection.count().inspect(|x| println!("Counts: {:?}", x));</code></pre>
|
|
<h2>Anti-Patterns</h2>
|
|
<pre><code>❌ NEVER: Scan entire collection for point lookups
|
|
→ Use arrange_by_key for indexed access
|
|
|
|
❌ NEVER: Recompute from scratch on updates
|
|
→ Use differential operators (join, reduce) for incremental updates
|
|
|
|
❌ NEVER: Arrange data that's used only once
|
|
→ Arrangement overhead not worth it, use direct operators
|
|
|
|
❌ NEVER: Accumulate unbounded history
|
|
→ Configure compaction to bound memory usage
|
|
|
|
❌ NEVER: Join without arrangement
|
|
→ Direct join is O(n²), arranged join is O(delta)
|
|
|
|
❌ NEVER: Use distinct on high-cardinality data unnecessarily
|
|
→ Maintains full set in memory, expensive
|
|
|
|
❌ NEVER: Forget to consolidate before inspection
|
|
→ May see multiple updates for same data (inserts + deletes)
|
|
|
|
❌ NEVER: Use differential for stateless transformations
|
|
→ Use standard timely dataflow (less overhead)
|
|
|
|
❌ NEVER: Create arrangements inside iterations
|
|
→ Build arrangements outside, enter into iteration scope
|
|
|
|
❌ NEVER: Ignore diff values in custom reduce logic
|
|
→ Must handle negative diffs (deletions) correctly</code></pre>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>timely-dataflow.md</code> - Foundation for differential dataflow</li><li><code>dataflow-coordination.md</code> - Coordination patterns for complex computations</li><li><code>streaming-aggregations.md</code> - Window-based aggregations</li><li><code>stream-processing.md</code> - Higher-level stream processing abstractions</li></ul>
|
|
<p><strong>Last Updated</strong>: 2025-10-27 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/data-differential-dataflow.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |