mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
880 lines
35 KiB
HTML
880 lines
35 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>data-pipeline-orchestration</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/data-pipeline-orchestration.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/data-pipeline-orchestration">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/data-pipeline-orchestration.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>data-pipeline-orchestration</h1><p class="lede">Coordinating complex multi-step data workflows</p><h1>Pipeline Orchestration</h1>
|
|
<p><strong>Scope</strong>: Workflow engines, error handling, retries, monitoring <strong>Lines</strong>: 383 <strong>Last Updated</strong>: 2025-10-18 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Use this skill when:</p>
|
|
<ul><li>Coordinating complex multi-step data workflows</li><li>Managing dependencies between pipeline stages</li><li>Implementing robust error handling and retries</li><li>Monitoring pipeline health and performance</li><li>Handling partial failures and recovery</li><li>Setting up alerting for pipeline failures</li><li>Building idempotent and resumable pipelines</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Orchestration vs Execution</h3>
|
|
<pre><code>Orchestration (Control Plane)
|
|
→ Defines workflow, dependencies, schedules
|
|
→ Tools: Airflow, Prefect, Dagster, Step Functions
|
|
→ Manages: What runs, when, in what order
|
|
|
|
Execution (Data Plane)
|
|
→ Actual data processing work
|
|
→ Tools: Spark, dbt, custom Python/SQL
|
|
→ Manages: How data transforms</code></pre>
|
|
<h3>Orchestration Patterns</h3>
|
|
<pre><code>Static DAG
|
|
→ Fixed workflow structure
|
|
→ Use when: Predictable pipeline steps
|
|
→ Example: Daily ETL with known tables
|
|
|
|
Dynamic DAG
|
|
→ Generate tasks at runtime
|
|
→ Use when: Data-driven workflows
|
|
→ Example: Process all partitions found in S3
|
|
|
|
Event-Driven
|
|
→ Trigger on external events
|
|
→ Use when: Real-time or on-demand processing
|
|
→ Example: Process file on S3 upload
|
|
|
|
Hybrid Batch + Stream
|
|
→ Combine scheduled and event-triggered
|
|
→ Use when: Lambda architecture
|
|
→ Example: Hourly batch + real-time stream</code></pre>
|
|
<h3>Error Handling Strategies</h3>
|
|
<pre><code>Retry with Backoff
|
|
→ Exponential delays between retries
|
|
→ Use when: Transient failures expected
|
|
|
|
Circuit Breaker
|
|
→ Stop calling failing service
|
|
→ Use when: Cascading failures possible
|
|
|
|
Dead Letter Queue (DLQ)
|
|
→ Route failed items for later analysis
|
|
→ Use when: Partial processing acceptable
|
|
|
|
Compensating Transaction
|
|
→ Rollback on failure
|
|
→ Use when: Transactional consistency required</code></pre>
|
|
<h3>Idempotency</h3>
|
|
<pre><code>Why Important: Retries shouldn't duplicate data
|
|
|
|
Techniques:
|
|
→ Upsert instead of insert
|
|
→ Write to temp, then swap atomically
|
|
→ Use execution_date as partition key
|
|
→ Check watermarks before processing</code></pre>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Prefect Workflow with Error Handling</h3>
|
|
<pre><code class="lang-python">from prefect import flow, task
|
|
from prefect.task_runners import ConcurrentTaskRunner
|
|
from prefect.deployments import Deployment
|
|
from prefect.server.schemas.schedules import CronSchedule
|
|
from datetime import timedelta
|
|
import logging
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Configure retry behavior
|
|
@task(
|
|
retries=3,
|
|
retry_delay_seconds=60,
|
|
timeout_seconds=3600,
|
|
log_prints=True
|
|
)
|
|
def extract_data(source: str, date: str):
|
|
"""Extract data with automatic retries"""
|
|
logger.info(f"Extracting from {source} for {date}")
|
|
|
|
try:
|
|
# Extraction logic
|
|
data = perform_extraction(source, date)
|
|
|
|
if not data:
|
|
raise ValueError("No data extracted")
|
|
|
|
return data
|
|
|
|
except Exception as e:
|
|
logger.error(f"Extraction failed: {e}")
|
|
raise
|
|
|
|
@task(
|
|
retries=2,
|
|
retry_delay_seconds=30
|
|
)
|
|
def validate_data(data):
|
|
"""Validate extracted data"""
|
|
logger.info(f"Validating {len(data)} records")
|
|
|
|
# Validation logic
|
|
if len(data) < 100:
|
|
raise ValueError("Insufficient data volume")
|
|
|
|
return data
|
|
|
|
@task(retries=3)
|
|
def transform_data(data):
|
|
"""Transform data"""
|
|
logger.info("Transforming data")
|
|
|
|
# Transformation logic
|
|
transformed = apply_transformations(data)
|
|
|
|
return transformed
|
|
|
|
@task(
|
|
retries=5,
|
|
retry_delay_seconds=[60, 120, 300, 600, 1200] # Custom backoff
|
|
)
|
|
def load_data(data, target: str):
|
|
"""Load to target with aggressive retries"""
|
|
logger.info(f"Loading to {target}")
|
|
|
|
# Load logic
|
|
load_to_warehouse(data, target)
|
|
|
|
return {"status": "success", "rows": len(data)}
|
|
|
|
# Main flow
|
|
@flow(
|
|
name="etl_pipeline",
|
|
task_runner=ConcurrentTaskRunner(),
|
|
retries=1,
|
|
retry_delay_seconds=600
|
|
)
|
|
def etl_pipeline(date: str, sources: list[str]):
|
|
"""Main ETL pipeline with parallel extraction"""
|
|
results = []
|
|
|
|
# Extract from multiple sources in parallel
|
|
extract_futures = []
|
|
for source in sources:
|
|
future = extract_data.submit(source, date)
|
|
extract_futures.append(future)
|
|
|
|
# Wait for all extractions
|
|
extracted_data = [f.result() for f in extract_futures]
|
|
|
|
# Combine and validate
|
|
combined_data = combine_datasets(extracted_data)
|
|
validated_data = validate_data(combined_data)
|
|
|
|
# Transform and load sequentially
|
|
transformed_data = transform_data(validated_data)
|
|
result = load_data(transformed_data, "warehouse")
|
|
|
|
return result
|
|
|
|
# Deploy with schedule
|
|
deployment = Deployment.build_from_flow(
|
|
flow=etl_pipeline,
|
|
name="daily_etl",
|
|
schedule=CronSchedule(cron="0 2 * * *"), # 2 AM daily
|
|
parameters={"sources": ["db1", "db2", "api"]},
|
|
work_queue_name="data-pipelines"
|
|
)
|
|
|
|
if __name__ == "__main__":
|
|
deployment.apply()</code></pre>
|
|
<h3>Pattern 2: State Management and Recovery</h3>
|
|
<pre><code class="lang-python">import json
|
|
from pathlib import Path
|
|
from datetime import datetime
|
|
from typing import Dict, Any, Optional
|
|
from enum import Enum
|
|
|
|
class PipelineState(Enum):
|
|
PENDING = "pending"
|
|
RUNNING = "running"
|
|
SUCCESS = "success"
|
|
FAILED = "failed"
|
|
RETRYING = "retrying"
|
|
|
|
class StateManager:
|
|
def __init__(self, state_file: Path):
|
|
self.state_file = state_file
|
|
self.state = self._load_state()
|
|
|
|
def _load_state(self) -> Dict[str, Any]:
|
|
"""Load state from file"""
|
|
if self.state_file.exists():
|
|
return json.loads(self.state_file.read_text())
|
|
|
|
return {
|
|
"runs": {},
|
|
"checkpoints": {},
|
|
"watermarks": {}
|
|
}
|
|
|
|
def _save_state(self):
|
|
"""Persist state to file"""
|
|
self.state_file.parent.mkdir(parents=True, exist_ok=True)
|
|
self.state_file.write_text(
|
|
json.dumps(self.state, indent=2, default=str)
|
|
)
|
|
|
|
def start_run(self, run_id: str, metadata: Dict[str, Any]) -> None:
|
|
"""Mark run as started"""
|
|
self.state["runs"][run_id] = {
|
|
"state": PipelineState.RUNNING.value,
|
|
"started_at": datetime.now().isoformat(),
|
|
"metadata": metadata,
|
|
"checkpoints": []
|
|
}
|
|
self._save_state()
|
|
|
|
def checkpoint(self, run_id: str, stage: str, data: Any = None) -> None:
|
|
"""Save checkpoint for recovery"""
|
|
if run_id not in self.state["runs"]:
|
|
raise ValueError(f"Run {run_id} not found")
|
|
|
|
checkpoint = {
|
|
"stage": stage,
|
|
"timestamp": datetime.now().isoformat(),
|
|
"data": data
|
|
}
|
|
|
|
self.state["runs"][run_id]["checkpoints"].append(checkpoint)
|
|
self.state["checkpoints"][f"{run_id}:{stage}"] = checkpoint
|
|
self._save_state()
|
|
|
|
def complete_run(self, run_id: str, result: Any = None) -> None:
|
|
"""Mark run as complete"""
|
|
self.state["runs"][run_id]["state"] = PipelineState.SUCCESS.value
|
|
self.state["runs"][run_id]["completed_at"] = datetime.now().isoformat()
|
|
self.state["runs"][run_id]["result"] = result
|
|
self._save_state()
|
|
|
|
def fail_run(self, run_id: str, error: str) -> None:
|
|
"""Mark run as failed"""
|
|
self.state["runs"][run_id]["state"] = PipelineState.FAILED.value
|
|
self.state["runs"][run_id]["failed_at"] = datetime.now().isoformat()
|
|
self.state["runs"][run_id]["error"] = error
|
|
self._save_state()
|
|
|
|
def get_last_checkpoint(self, run_id: str) -> Optional[Dict[str, Any]]:
|
|
"""Get last checkpoint for recovery"""
|
|
if run_id not in self.state["runs"]:
|
|
return None
|
|
|
|
checkpoints = self.state["runs"][run_id].get("checkpoints", [])
|
|
return checkpoints[-1] if checkpoints else None
|
|
|
|
def set_watermark(self, key: str, value: Any) -> None:
|
|
"""Update watermark for incremental processing"""
|
|
self.state["watermarks"][key] = {
|
|
"value": value,
|
|
"updated_at": datetime.now().isoformat()
|
|
}
|
|
self._save_state()
|
|
|
|
def get_watermark(self, key: str) -> Optional[Any]:
|
|
"""Get watermark value"""
|
|
watermark = self.state["watermarks"].get(key)
|
|
return watermark["value"] if watermark else None
|
|
|
|
# Resumable pipeline
|
|
class ResumablePipeline:
|
|
def __init__(self, name: str, state_file: Path):
|
|
self.name = name
|
|
self.state_mgr = StateManager(state_file)
|
|
self.run_id = None
|
|
|
|
def run(self, params: Dict[str, Any], resume: bool = True):
|
|
"""Run pipeline with checkpointing"""
|
|
self.run_id = f"{self.name}_{datetime.now().isoformat()}"
|
|
|
|
try:
|
|
# Check for existing run to resume
|
|
if resume:
|
|
last_checkpoint = self.state_mgr.get_last_checkpoint(self.run_id)
|
|
if last_checkpoint:
|
|
print(f"Resuming from {last_checkpoint['stage']}")
|
|
return self._resume(last_checkpoint, params)
|
|
|
|
# Start new run
|
|
self.state_mgr.start_run(self.run_id, params)
|
|
|
|
# Stage 1: Extract
|
|
print("Stage 1: Extract")
|
|
data = self.extract(params)
|
|
self.state_mgr.checkpoint(self.run_id, "extract", {"row_count": len(data)})
|
|
|
|
# Stage 2: Transform
|
|
print("Stage 2: Transform")
|
|
transformed = self.transform(data)
|
|
self.state_mgr.checkpoint(self.run_id, "transform", {"row_count": len(transformed)})
|
|
|
|
# Stage 3: Load
|
|
print("Stage 3: Load")
|
|
result = self.load(transformed)
|
|
self.state_mgr.checkpoint(self.run_id, "load", result)
|
|
|
|
# Complete
|
|
self.state_mgr.complete_run(self.run_id, result)
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
self.state_mgr.fail_run(self.run_id, str(e))
|
|
raise
|
|
|
|
def _resume(self, checkpoint: Dict[str, Any], params: Dict[str, Any]):
|
|
"""Resume from last checkpoint"""
|
|
stage = checkpoint["stage"]
|
|
|
|
if stage == "extract":
|
|
# Re-extract (could load from checkpoint if saved)
|
|
data = self.extract(params)
|
|
transformed = self.transform(data)
|
|
result = self.load(transformed)
|
|
|
|
elif stage == "transform":
|
|
# Would need to restore data from checkpoint
|
|
# For simplicity, re-running from extract
|
|
data = self.extract(params)
|
|
transformed = self.transform(data)
|
|
result = self.load(transformed)
|
|
|
|
elif stage == "load":
|
|
# Transform complete, retry load only
|
|
# Would restore transformed data
|
|
result = checkpoint["data"]
|
|
|
|
self.state_mgr.complete_run(self.run_id, result)
|
|
return result
|
|
|
|
def extract(self, params):
|
|
# Extraction logic
|
|
return []
|
|
|
|
def transform(self, data):
|
|
# Transformation logic
|
|
return data
|
|
|
|
def load(self, data):
|
|
# Load logic
|
|
return {"status": "success"}
|
|
|
|
# Usage
|
|
pipeline = ResumablePipeline("sales_etl", Path("/tmp/pipeline_state.json"))
|
|
result = pipeline.run({"date": "2025-10-18"}, resume=True)</code></pre>
|
|
<h3>Pattern 3: Circuit Breaker for External Services</h3>
|
|
<pre><code class="lang-python">from datetime import datetime, timedelta
|
|
from typing import Callable, Any
|
|
from enum import Enum
|
|
import time
|
|
|
|
class CircuitState(Enum):
|
|
CLOSED = "closed" # Normal operation
|
|
OPEN = "open" # Rejecting calls
|
|
HALF_OPEN = "half_open" # Testing recovery
|
|
|
|
class CircuitBreaker:
|
|
def __init__(
|
|
self,
|
|
failure_threshold: int = 5,
|
|
recovery_timeout: int = 60,
|
|
expected_exception: type = Exception
|
|
):
|
|
self.failure_threshold = failure_threshold
|
|
self.recovery_timeout = recovery_timeout
|
|
self.expected_exception = expected_exception
|
|
|
|
self.failure_count = 0
|
|
self.last_failure_time = None
|
|
self.state = CircuitState.CLOSED
|
|
|
|
def call(self, func: Callable, *args, **kwargs) -> Any:
|
|
"""Execute function with circuit breaker protection"""
|
|
if self.state == CircuitState.OPEN:
|
|
if self._should_attempt_reset():
|
|
self.state = CircuitState.HALF_OPEN
|
|
else:
|
|
raise Exception("Circuit breaker is OPEN")
|
|
|
|
try:
|
|
result = func(*args, **kwargs)
|
|
self._on_success()
|
|
return result
|
|
|
|
except self.expected_exception as e:
|
|
self._on_failure()
|
|
raise
|
|
|
|
def _on_success(self):
|
|
"""Reset on successful call"""
|
|
self.failure_count = 0
|
|
self.state = CircuitState.CLOSED
|
|
|
|
def _on_failure(self):
|
|
"""Increment failure count and open if threshold reached"""
|
|
self.failure_count += 1
|
|
self.last_failure_time = datetime.now()
|
|
|
|
if self.failure_count >= self.failure_threshold:
|
|
self.state = CircuitState.OPEN
|
|
|
|
def _should_attempt_reset(self) -> bool:
|
|
"""Check if enough time has passed to try again"""
|
|
return (
|
|
self.last_failure_time is not None and
|
|
datetime.now() - self.last_failure_time >= timedelta(seconds=self.recovery_timeout)
|
|
)
|
|
|
|
# Usage in pipeline
|
|
class APIExtractor:
|
|
def __init__(self, api_url: str):
|
|
self.api_url = api_url
|
|
self.circuit_breaker = CircuitBreaker(
|
|
failure_threshold=3,
|
|
recovery_timeout=300, # 5 minutes
|
|
expected_exception=requests.RequestException
|
|
)
|
|
|
|
def extract(self, params: dict):
|
|
"""Extract with circuit breaker protection"""
|
|
def _extract():
|
|
response = requests.get(self.api_url, params=params)
|
|
response.raise_for_status()
|
|
return response.json()
|
|
|
|
try:
|
|
return self.circuit_breaker.call(_extract)
|
|
except Exception as e:
|
|
# Fallback to cached data or alternative source
|
|
logger.error(f"Circuit breaker prevented call: {e}")
|
|
return self._get_fallback_data(params)
|
|
|
|
def _get_fallback_data(self, params: dict):
|
|
"""Fallback strategy when circuit is open"""
|
|
# Could return cached data, empty dataset, or raise
|
|
return []</code></pre>
|
|
<h3>Pattern 4: Pipeline Monitoring and Alerting</h3>
|
|
<pre><code class="lang-python">from dataclasses import dataclass
|
|
from typing import List, Optional
|
|
from datetime import datetime, timedelta
|
|
import smtplib
|
|
from email.message import EmailMessage
|
|
|
|
@dataclass
|
|
class PipelineMetrics:
|
|
run_id: str
|
|
pipeline_name: str
|
|
started_at: datetime
|
|
completed_at: Optional[datetime]
|
|
duration_seconds: Optional[float]
|
|
rows_processed: int
|
|
rows_failed: int
|
|
state: str
|
|
error: Optional[str]
|
|
|
|
class PipelineMonitor:
|
|
def __init__(
|
|
self,
|
|
alert_email: str,
|
|
sla_minutes: int = 60,
|
|
min_rows_expected: int = 1000
|
|
):
|
|
self.alert_email = alert_email
|
|
self.sla_minutes = sla_minutes
|
|
self.min_rows_expected = min_rows_expected
|
|
self.metrics_history: List[PipelineMetrics] = []
|
|
|
|
def record_run(self, metrics: PipelineMetrics):
|
|
"""Record pipeline run metrics"""
|
|
self.metrics_history.append(metrics)
|
|
|
|
# Check SLAs and alert
|
|
self._check_sla(metrics)
|
|
self._check_data_volume(metrics)
|
|
self._check_error_rate()
|
|
|
|
def _check_sla(self, metrics: PipelineMetrics):
|
|
"""Alert if pipeline exceeded SLA"""
|
|
if metrics.duration_seconds is None:
|
|
return
|
|
|
|
if metrics.duration_seconds > self.sla_minutes * 60:
|
|
self._send_alert(
|
|
subject=f"SLA Violation: {metrics.pipeline_name}",
|
|
body=f"Pipeline took {metrics.duration_seconds/60:.1f} minutes "
|
|
f"(SLA: {self.sla_minutes} minutes)"
|
|
)
|
|
|
|
def _check_data_volume(self, metrics: PipelineMetrics):
|
|
"""Alert if data volume is unexpectedly low"""
|
|
if metrics.rows_processed < self.min_rows_expected:
|
|
self._send_alert(
|
|
subject=f"Low Data Volume: {metrics.pipeline_name}",
|
|
body=f"Only {metrics.rows_processed} rows processed "
|
|
f"(expected: {self.min_rows_expected})"
|
|
)
|
|
|
|
def _check_error_rate(self):
|
|
"""Alert if error rate is high"""
|
|
recent_runs = [
|
|
m for m in self.metrics_history[-10:]
|
|
if m.pipeline_name == self.metrics_history[-1].pipeline_name
|
|
]
|
|
|
|
if not recent_runs:
|
|
return
|
|
|
|
failed_runs = sum(1 for m in recent_runs if m.state == "failed")
|
|
error_rate = failed_runs / len(recent_runs)
|
|
|
|
if error_rate > 0.3: # 30% failure rate
|
|
self._send_alert(
|
|
subject=f"High Error Rate: {recent_runs[0].pipeline_name}",
|
|
body=f"Error rate: {error_rate*100:.1f}% over last {len(recent_runs)} runs"
|
|
)
|
|
|
|
def _send_alert(self, subject: str, body: str):
|
|
"""Send email alert"""
|
|
msg = EmailMessage()
|
|
msg['Subject'] = subject
|
|
msg['From'] = '[email protected]'
|
|
msg['To'] = self.alert_email
|
|
msg.set_content(body)
|
|
|
|
# Would use actual SMTP server
|
|
print(f"ALERT: {subject}\n{body}")
|
|
|
|
def get_stats(self, pipeline_name: str, days: int = 7) -> dict:
|
|
"""Get pipeline statistics"""
|
|
cutoff = datetime.now() - timedelta(days=days)
|
|
recent = [
|
|
m for m in self.metrics_history
|
|
if m.pipeline_name == pipeline_name and m.started_at >= cutoff
|
|
]
|
|
|
|
if not recent:
|
|
return {}
|
|
|
|
total_runs = len(recent)
|
|
successful_runs = sum(1 for m in recent if m.state == "success")
|
|
avg_duration = sum(
|
|
m.duration_seconds for m in recent if m.duration_seconds
|
|
) / len([m for m in recent if m.duration_seconds])
|
|
|
|
return {
|
|
"total_runs": total_runs,
|
|
"successful_runs": successful_runs,
|
|
"success_rate": successful_runs / total_runs,
|
|
"avg_duration_minutes": avg_duration / 60,
|
|
"avg_rows_processed": sum(m.rows_processed for m in recent) / total_runs
|
|
}
|
|
|
|
# Usage
|
|
monitor = PipelineMonitor(
|
|
alert_email="[email protected]",
|
|
sla_minutes=30,
|
|
min_rows_expected=5000
|
|
)
|
|
|
|
# After pipeline run
|
|
metrics = PipelineMetrics(
|
|
run_id="run123",
|
|
pipeline_name="daily_sales",
|
|
started_at=datetime.now() - timedelta(minutes=45),
|
|
completed_at=datetime.now(),
|
|
duration_seconds=2700, # 45 minutes
|
|
rows_processed=10000,
|
|
rows_failed=50,
|
|
state="success",
|
|
error=None
|
|
)
|
|
|
|
monitor.record_run(metrics)
|
|
stats = monitor.get_stats("daily_sales", days=7)
|
|
print(stats)</code></pre>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Common Orchestration Tools</h3>
|
|
<pre><code>Apache Airflow
|
|
→ Mature, Python-based, largest community
|
|
→ Use when: Complex dependencies, scheduling
|
|
|
|
Prefect
|
|
→ Modern, Python-native, better error handling
|
|
→ Use when: Dynamic workflows, developer-friendly
|
|
|
|
Dagster
|
|
→ Asset-oriented, testing-first
|
|
→ Use when: Data assets, strong typing
|
|
|
|
AWS Step Functions
|
|
→ Serverless, AWS-native
|
|
→ Use when: AWS ecosystem, event-driven
|
|
|
|
Temporal
|
|
→ Durable execution, microservices
|
|
→ Use when: Long-running workflows, reliability</code></pre>
|
|
<h3>Retry Strategies</h3>
|
|
<pre><code class="lang-python"># Exponential backoff
|
|
retry_delays = [60, 120, 300, 600, 1200] # 1m, 2m, 5m, 10m, 20m
|
|
|
|
# Linear backoff
|
|
retry_delays = [60] * 5 # 1m each time
|
|
|
|
# Fibonacci backoff
|
|
retry_delays = [60, 60, 120, 180, 300] # 1m, 1m, 2m, 3m, 5m</code></pre>
|
|
<h3>Monitoring Metrics</h3>
|
|
<pre><code>Pipeline Metrics:
|
|
- Duration (p50, p95, p99)
|
|
- Rows processed
|
|
- Success rate
|
|
- Error rate
|
|
- SLA compliance
|
|
|
|
Data Quality Metrics:
|
|
- Null rate
|
|
- Duplicate rate
|
|
- Schema validation failures
|
|
- Data freshness</code></pre>
|
|
<h2>Anti-Patterns</h2>
|
|
<pre><code>❌ NEVER: Run pipelines without idempotency
|
|
→ Use upserts, check watermarks, enable safe retries
|
|
|
|
❌ NEVER: Ignore partial failures
|
|
→ Track failed records, implement DLQ
|
|
|
|
❌ NEVER: Skip monitoring and alerting
|
|
→ Monitor duration, volume, errors
|
|
|
|
❌ NEVER: Use unbounded retries
|
|
→ Set max retries, implement circuit breakers
|
|
|
|
❌ NEVER: Hardcode dependencies
|
|
→ Use orchestrator's dependency management
|
|
|
|
❌ NEVER: Skip state persistence
|
|
→ Save checkpoints for resumability
|
|
|
|
❌ NEVER: Run long tasks in orchestrator
|
|
→ Offload heavy compute to workers
|
|
|
|
❌ NEVER: Ignore SLA requirements
|
|
→ Set timeouts, monitor compliance
|
|
|
|
❌ NEVER: Deploy without backfill strategy
|
|
→ Plan for historical data reprocessing
|
|
|
|
❌ NEVER: Skip testing failure scenarios
|
|
→ Test retries, timeouts, recovery paths</code></pre>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>batch-processing.md</code> - Airflow DAG patterns</li><li><code>etl-patterns.md</code> - Data pipeline logic</li><li><code>stream-processing.md</code> - Real-time orchestration</li><li><code>data-validation.md</code> - Quality gates in pipelines</li></ul>
|
|
<p><strong>Last Updated</strong>: 2025-10-18 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/data-pipeline-orchestration.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |