mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
794 lines
32 KiB
HTML
794 lines
32 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>data-batch-processing</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/data-batch-processing.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/data-batch-processing">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/data-batch-processing.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>data-batch-processing</h1><p class="lede">Orchestrating complex data pipelines with dependencies</p><h1>Batch Processing</h1>
|
|
<p><strong>Scope</strong>: Airflow, DAGs, scheduling, dependency management, backfills <strong>Lines</strong>: 391 <strong>Last Updated</strong>: 2025-10-18 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Use this skill when:</p>
|
|
<ul><li>Orchestrating complex data pipelines with dependencies</li><li>Scheduling batch jobs (hourly, daily, weekly)</li><li>Managing task retries and failure handling</li><li>Performing historical backfills of data</li><li>Coordinating multi-step ETL workflows</li><li>Monitoring pipeline health and SLAs</li><li>Working with Apache Airflow or similar orchestrators</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>DAG (Directed Acyclic Graph)</h3>
|
|
<pre><code>DAG Structure
|
|
→ Nodes: Tasks (Python functions, SQL, Bash)
|
|
→ Edges: Dependencies (task1 >> task2)
|
|
→ Properties: schedule, start_date, catchup, retries
|
|
|
|
Key Principle: No cycles allowed
|
|
→ Ensures deterministic execution order</code></pre>
|
|
<h3>Task Dependencies</h3>
|
|
<pre><code>Linear: task1 >> task2 >> task3
|
|
→ Sequential execution
|
|
|
|
Parallel: task1 >> [task2, task3] >> task4
|
|
→ task2 and task3 run in parallel
|
|
|
|
Cross-depends: [task1, task2] >> task3
|
|
→ task3 waits for both
|
|
|
|
Dynamic: task1 >> TaskGroup([...])
|
|
→ Generate tasks programmatically</code></pre>
|
|
<h3>Execution Concepts</h3>
|
|
<pre><code>Execution Date (logical date)
|
|
→ Data interval being processed (NOT run time)
|
|
→ Example: 2025-10-18 DAG processes 2025-10-17 data
|
|
|
|
Schedule Interval
|
|
→ @daily, @hourly, cron expressions
|
|
→ 0 0 * * * = daily at midnight
|
|
|
|
Catchup
|
|
→ True: Run missed intervals on deploy
|
|
→ False: Skip to present (default for new DAGs)
|
|
|
|
Backfill
|
|
→ Reprocess historical data intervals
|
|
→ airflow dags backfill -s START -e END dag_id</code></pre>
|
|
<h3>Task States</h3>
|
|
<pre><code>success: Completed successfully
|
|
failed: Error occurred
|
|
upstream_failed: Dependency failed
|
|
skipped: Condition not met (BranchOperator)
|
|
running: Currently executing
|
|
queued: Waiting for executor slot
|
|
retry: Failed, will retry</code></pre>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Basic Airflow DAG Structure</h3>
|
|
<pre><code class="lang-python">from airflow import DAG
|
|
from airflow.operators.python import PythonOperator
|
|
from airflow.operators.bash import BashOperator
|
|
from datetime import datetime, timedelta
|
|
|
|
# Default arguments for all tasks
|
|
default_args = {
|
|
'owner': 'data-team',
|
|
'depends_on_past': False, # Don't wait for previous interval
|
|
'email': ['[email protected]'],
|
|
'email_on_failure': True,
|
|
'email_on_retry': False,
|
|
'retries': 3,
|
|
'retry_delay': timedelta(minutes=5),
|
|
'execution_timeout': timedelta(hours=1), # Kill if exceeds
|
|
}
|
|
|
|
# Define DAG
|
|
with DAG(
|
|
'daily_sales_pipeline',
|
|
default_args=default_args,
|
|
description='Process daily sales data',
|
|
schedule_interval='0 2 * * *', # 2 AM daily
|
|
start_date=datetime(2025, 1, 1),
|
|
catchup=False, # Don't backfill automatically
|
|
tags=['sales', 'daily'],
|
|
max_active_runs=1, # Only one run at a time
|
|
) as dag:
|
|
|
|
# Task 1: Extract data
|
|
def extract_sales(**context):
|
|
"""Extract sales from previous day"""
|
|
execution_date = context['execution_date']
|
|
data_date = execution_date - timedelta(days=1)
|
|
|
|
print(f"Extracting sales for {data_date.date()}")
|
|
|
|
# Extract logic here
|
|
sales_count = 1250
|
|
|
|
# Push to XCom for downstream tasks
|
|
context['task_instance'].xcom_push(
|
|
key='sales_count',
|
|
value=sales_count
|
|
)
|
|
|
|
return f"Extracted {sales_count} sales"
|
|
|
|
extract = PythonOperator(
|
|
task_id='extract_sales',
|
|
python_callable=extract_sales,
|
|
provide_context=True
|
|
)
|
|
|
|
# Task 2: Validate data
|
|
def validate_sales(**context):
|
|
"""Validate extracted data"""
|
|
ti = context['task_instance']
|
|
sales_count = ti.xcom_pull(
|
|
task_ids='extract_sales',
|
|
key='sales_count'
|
|
)
|
|
|
|
if sales_count < 100:
|
|
raise ValueError(f"Too few sales: {sales_count}")
|
|
|
|
print(f"Validation passed: {sales_count} sales")
|
|
return True
|
|
|
|
validate = PythonOperator(
|
|
task_id='validate_sales',
|
|
python_callable=validate_sales,
|
|
provide_context=True
|
|
)
|
|
|
|
# Task 3: Transform
|
|
transform = BashOperator(
|
|
task_id='transform_sales',
|
|
bash_command='python /path/to/transform.py {{ ds }}', # ds = YYYY-MM-DD
|
|
)
|
|
|
|
# Task 4: Load to warehouse
|
|
def load_sales(**context):
|
|
"""Load to data warehouse"""
|
|
execution_date = context['execution_date']
|
|
|
|
print(f"Loading sales for {execution_date.date()}")
|
|
|
|
# Load logic here
|
|
return "Load complete"
|
|
|
|
load = PythonOperator(
|
|
task_id='load_sales',
|
|
python_callable=load_sales,
|
|
provide_context=True
|
|
)
|
|
|
|
# Task 5: Update metadata
|
|
metadata = BashOperator(
|
|
task_id='update_metadata',
|
|
bash_command='echo "Updated metadata for {{ ds }}"'
|
|
)
|
|
|
|
# Define dependencies
|
|
extract >> validate >> transform >> load >> metadata</code></pre>
|
|
<h3>Pattern 2: Dynamic Task Generation</h3>
|
|
<pre><code class="lang-python">from airflow import DAG
|
|
from airflow.operators.python import PythonOperator
|
|
from datetime import datetime
|
|
|
|
def process_partition(partition_id: str, **context):
|
|
"""Process a single partition"""
|
|
print(f"Processing partition {partition_id}")
|
|
# Processing logic
|
|
return f"Partition {partition_id} complete"
|
|
|
|
with DAG(
|
|
'parallel_partition_processing',
|
|
start_date=datetime(2025, 1, 1),
|
|
schedule_interval='@daily',
|
|
catchup=False
|
|
) as dag:
|
|
|
|
# Get partitions dynamically
|
|
partitions = ['2025-01', '2025-02', '2025-03', '2025-04']
|
|
|
|
start = PythonOperator(
|
|
task_id='start',
|
|
python_callable=lambda: print("Starting processing")
|
|
)
|
|
|
|
# Create task for each partition
|
|
partition_tasks = []
|
|
for partition in partitions:
|
|
task = PythonOperator(
|
|
task_id=f'process_{partition}',
|
|
python_callable=process_partition,
|
|
op_kwargs={'partition_id': partition},
|
|
provide_context=True
|
|
)
|
|
partition_tasks.append(task)
|
|
|
|
end = PythonOperator(
|
|
task_id='end',
|
|
python_callable=lambda: print("All partitions processed")
|
|
)
|
|
|
|
# Dependencies: start >> all partitions >> end
|
|
start >> partition_tasks >> end</code></pre>
|
|
<h3>Pattern 3: TaskGroups for Organization</h3>
|
|
<pre><code class="lang-python">from airflow import DAG
|
|
from airflow.operators.python import PythonOperator
|
|
from airflow.utils.task_group import TaskGroup
|
|
from datetime import datetime
|
|
|
|
with DAG(
|
|
'grouped_pipeline',
|
|
start_date=datetime(2025, 1, 1),
|
|
schedule_interval='@daily',
|
|
catchup=False
|
|
) as dag:
|
|
|
|
# Group 1: Extraction tasks
|
|
with TaskGroup('extraction', tooltip='Extract from multiple sources') as extract_group:
|
|
extract_db = PythonOperator(
|
|
task_id='extract_database',
|
|
python_callable=lambda: print("Extract from DB")
|
|
)
|
|
|
|
extract_api = PythonOperator(
|
|
task_id='extract_api',
|
|
python_callable=lambda: print("Extract from API")
|
|
)
|
|
|
|
extract_files = PythonOperator(
|
|
task_id='extract_files',
|
|
python_callable=lambda: print("Extract from files")
|
|
)
|
|
|
|
# Group 2: Transformation tasks
|
|
with TaskGroup('transformation', tooltip='Transform data') as transform_group:
|
|
clean = PythonOperator(
|
|
task_id='clean_data',
|
|
python_callable=lambda: print("Clean data")
|
|
)
|
|
|
|
enrich = PythonOperator(
|
|
task_id='enrich_data',
|
|
python_callable=lambda: print("Enrich data")
|
|
)
|
|
|
|
aggregate = PythonOperator(
|
|
task_id='aggregate_data',
|
|
python_callable=lambda: print("Aggregate data")
|
|
)
|
|
|
|
clean >> enrich >> aggregate
|
|
|
|
# Group 3: Loading tasks
|
|
with TaskGroup('loading', tooltip='Load to destinations') as load_group:
|
|
load_warehouse = PythonOperator(
|
|
task_id='load_warehouse',
|
|
python_callable=lambda: print("Load to warehouse")
|
|
)
|
|
|
|
load_cache = PythonOperator(
|
|
task_id='load_cache',
|
|
python_callable=lambda: print("Load to cache")
|
|
)
|
|
|
|
# Dependencies between groups
|
|
extract_group >> transform_group >> load_group</code></pre>
|
|
<h3>Pattern 4: Sensor for External Dependencies</h3>
|
|
<pre><code class="lang-python">from airflow import DAG
|
|
from airflow.sensors.filesystem import FileSensor
|
|
from airflow.sensors.external_task import ExternalTaskSensor
|
|
from airflow.sensors.python import PythonSensor
|
|
from airflow.operators.python import PythonOperator
|
|
from datetime import datetime, timedelta
|
|
|
|
def check_data_ready(**context):
|
|
"""Custom check for data availability"""
|
|
execution_date = context['execution_date']
|
|
|
|
# Check if data exists for this date
|
|
# Return True if ready, False otherwise
|
|
|
|
import random
|
|
return random.choice([True, False])
|
|
|
|
with DAG(
|
|
'sensor_pipeline',
|
|
start_date=datetime(2025, 1, 1),
|
|
schedule_interval='@hourly',
|
|
catchup=False
|
|
) as dag:
|
|
|
|
# Wait for file to appear
|
|
wait_for_file = FileSensor(
|
|
task_id='wait_for_input_file',
|
|
filepath='/data/input/{{ ds }}/data.csv',
|
|
poke_interval=60, # Check every 60 seconds
|
|
timeout=3600, # Fail after 1 hour
|
|
mode='poke' # or 'reschedule' to free worker slot
|
|
)
|
|
|
|
# Wait for upstream DAG to complete
|
|
wait_for_upstream = ExternalTaskSensor(
|
|
task_id='wait_for_extraction_dag',
|
|
external_dag_id='daily_extraction',
|
|
external_task_id='extract_complete',
|
|
execution_delta=timedelta(hours=1), # Adjust for schedule difference
|
|
timeout=7200,
|
|
mode='reschedule'
|
|
)
|
|
|
|
# Custom Python sensor
|
|
wait_for_data = PythonSensor(
|
|
task_id='wait_for_data_ready',
|
|
python_callable=check_data_ready,
|
|
provide_context=True,
|
|
poke_interval=300,
|
|
timeout=3600
|
|
)
|
|
|
|
# Process after all sensors succeed
|
|
process = PythonOperator(
|
|
task_id='process_data',
|
|
python_callable=lambda: print("Processing data")
|
|
)
|
|
|
|
[wait_for_file, wait_for_upstream, wait_for_data] >> process</code></pre>
|
|
<h3>Pattern 5: Branching and Conditional Logic</h3>
|
|
<pre><code class="lang-python">from airflow import DAG
|
|
from airflow.operators.python import PythonOperator, BranchPythonOperator
|
|
from airflow.operators.empty import EmptyOperator
|
|
from datetime import datetime
|
|
|
|
def decide_branch(**context):
|
|
"""Choose execution path based on conditions"""
|
|
execution_date = context['execution_date']
|
|
|
|
# Example: Different processing for weekends
|
|
if execution_date.weekday() >= 5: # Saturday or Sunday
|
|
return 'weekend_processing'
|
|
else:
|
|
return 'weekday_processing'
|
|
|
|
def check_data_quality(**context):
|
|
"""Return task ID based on data quality"""
|
|
# Simulate quality check
|
|
quality_score = 95
|
|
|
|
if quality_score >= 90:
|
|
return 'high_quality_path'
|
|
elif quality_score >= 70:
|
|
return 'medium_quality_path'
|
|
else:
|
|
return 'low_quality_path'
|
|
|
|
with DAG(
|
|
'branching_pipeline',
|
|
start_date=datetime(2025, 1, 1),
|
|
schedule_interval='@daily',
|
|
catchup=False
|
|
) as dag:
|
|
|
|
start = EmptyOperator(task_id='start')
|
|
|
|
# Branch based on day of week
|
|
branch_day = BranchPythonOperator(
|
|
task_id='check_day_of_week',
|
|
python_callable=decide_branch,
|
|
provide_context=True
|
|
)
|
|
|
|
weekday = PythonOperator(
|
|
task_id='weekday_processing',
|
|
python_callable=lambda: print("Weekday processing")
|
|
)
|
|
|
|
weekend = PythonOperator(
|
|
task_id='weekend_processing',
|
|
python_callable=lambda: print("Weekend processing")
|
|
)
|
|
|
|
join = EmptyOperator(
|
|
task_id='join',
|
|
trigger_rule='none_failed_min_one_success' # Continue if any branch succeeded
|
|
)
|
|
|
|
# Quality-based branching
|
|
branch_quality = BranchPythonOperator(
|
|
task_id='check_quality',
|
|
python_callable=check_data_quality,
|
|
provide_context=True
|
|
)
|
|
|
|
high_quality = PythonOperator(
|
|
task_id='high_quality_path',
|
|
python_callable=lambda: print("High quality - full processing")
|
|
)
|
|
|
|
medium_quality = PythonOperator(
|
|
task_id='medium_quality_path',
|
|
python_callable=lambda: print("Medium quality - partial processing")
|
|
)
|
|
|
|
low_quality = PythonOperator(
|
|
task_id='low_quality_path',
|
|
python_callable=lambda: print("Low quality - alert only")
|
|
)
|
|
|
|
end = EmptyOperator(
|
|
task_id='end',
|
|
trigger_rule='none_failed_min_one_success'
|
|
)
|
|
|
|
# Dependencies
|
|
start >> branch_day >> [weekday, weekend] >> join
|
|
join >> branch_quality >> [high_quality, medium_quality, low_quality] >> end</code></pre>
|
|
<h3>Pattern 6: Backfill with Error Handling</h3>
|
|
<pre><code class="lang-python">from airflow import DAG
|
|
from airflow.operators.python import PythonOperator
|
|
from airflow.exceptions import AirflowException
|
|
from datetime import datetime, timedelta
|
|
import logging
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
def process_date_range(start_date: str, end_date: str, **context):
|
|
"""Process data for date range with error handling"""
|
|
from datetime import datetime
|
|
|
|
start = datetime.strptime(start_date, '%Y-%m-%d')
|
|
end = datetime.strptime(end_date, '%Y-%m-%d')
|
|
|
|
current = start
|
|
failures = []
|
|
|
|
while current <= end:
|
|
try:
|
|
# Process single date
|
|
logger.info(f"Processing {current.date()}")
|
|
|
|
# Your processing logic here
|
|
# Could call extract, transform, load functions
|
|
|
|
logger.info(f"Successfully processed {current.date()}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to process {current.date()}: {e}")
|
|
failures.append((current.date(), str(e)))
|
|
|
|
current += timedelta(days=1)
|
|
|
|
if failures:
|
|
failure_msg = "\n".join([f"{date}: {error}" for date, error in failures])
|
|
raise AirflowException(f"Backfill had {len(failures)} failures:\n{failure_msg}")
|
|
|
|
return f"Successfully processed {(end - start).days + 1} dates"
|
|
|
|
with DAG(
|
|
'backfill_pipeline',
|
|
start_date=datetime(2025, 1, 1),
|
|
schedule_interval=None, # Manual trigger only
|
|
catchup=False
|
|
) as dag:
|
|
|
|
backfill = PythonOperator(
|
|
task_id='backfill_data',
|
|
python_callable=process_date_range,
|
|
op_kwargs={
|
|
'start_date': '2025-01-01',
|
|
'end_date': '2025-01-31'
|
|
},
|
|
provide_context=True
|
|
)
|
|
|
|
# Run with: airflow dags trigger backfill_pipeline</code></pre>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Airflow CLI Commands</h3>
|
|
<pre><code class="lang-bash"># List DAGs
|
|
airflow dags list
|
|
|
|
# Trigger DAG
|
|
airflow dags trigger my_dag
|
|
|
|
# Trigger with config
|
|
airflow dags trigger my_dag --conf '{"key": "value"}'
|
|
|
|
# Test task
|
|
airflow tasks test my_dag my_task 2025-10-18
|
|
|
|
# Backfill
|
|
airflow dags backfill my_dag \
|
|
--start-date 2025-01-01 \
|
|
--end-date 2025-01-31
|
|
|
|
# Clear task instances
|
|
airflow tasks clear my_dag --start-date 2025-10-18
|
|
|
|
# Pause/unpause DAG
|
|
airflow dags pause my_dag
|
|
airflow dags unpause my_dag
|
|
|
|
# Show DAG structure
|
|
airflow dags show my_dag
|
|
|
|
# List task instances
|
|
airflow tasks list my_dag --tree</code></pre>
|
|
<h3>Schedule Intervals</h3>
|
|
<pre><code class="lang-python"># Presets
|
|
'@once' # Run once
|
|
'@hourly' # 0 * * * *
|
|
'@daily' # 0 0 * * *
|
|
'@weekly' # 0 0 * * 0
|
|
'@monthly' # 0 0 1 * *
|
|
'@yearly' # 0 0 1 1 *
|
|
|
|
# Cron expressions
|
|
'0 */4 * * *' # Every 4 hours
|
|
'0 2 * * *' # Daily at 2 AM
|
|
'0 0 * * 1-5' # Weekdays at midnight
|
|
'0 0 1 * *' # First of month
|
|
'0 0 1 */3 *' # First day of quarter
|
|
|
|
# Timedelta
|
|
timedelta(hours=6) # Every 6 hours</code></pre>
|
|
<h3>Trigger Rules</h3>
|
|
<pre><code class="lang-python">'all_success' # All parents succeeded (default)
|
|
'all_failed' # All parents failed
|
|
'all_done' # All parents completed
|
|
'one_success' # At least one parent succeeded
|
|
'one_failed' # At least one parent failed
|
|
'none_failed' # No parents failed (some may be skipped)
|
|
'none_failed_min_one_success' # At least one succeeded, none failed
|
|
'none_skipped' # No parents skipped
|
|
'always' # Always run</code></pre>
|
|
<h3>XCom Patterns</h3>
|
|
<pre><code class="lang-python"># Push value
|
|
ti.xcom_push(key='my_key', value=my_value)
|
|
|
|
# Pull value
|
|
value = ti.xcom_pull(task_ids='upstream_task', key='my_key')
|
|
|
|
# Pull from multiple tasks
|
|
values = ti.xcom_pull(task_ids=['task1', 'task2'])
|
|
|
|
# Return value (auto-pushed to 'return_value' key)
|
|
return my_value</code></pre>
|
|
<h2>Anti-Patterns</h2>
|
|
<pre><code>❌ NEVER: Set depends_on_past=True without understanding
|
|
→ Can cause cascading failures across intervals
|
|
|
|
❌ NEVER: Use catchup=True for new DAGs with old start_date
|
|
→ Will trigger hundreds of backfill runs
|
|
|
|
❌ NEVER: Store large data in XCom
|
|
→ XCom for metadata only (<1MB), use external storage
|
|
|
|
❌ NEVER: Use sleep() in tasks
|
|
→ Use Sensors for waiting on conditions
|
|
|
|
❌ NEVER: Hardcode dates in tasks
|
|
→ Use {{ ds }}, {{ execution_date }}, context variables
|
|
|
|
❌ NEVER: Run tasks longer than schedule interval
|
|
→ Set execution_timeout, optimize task duration
|
|
|
|
❌ NEVER: Skip error handling in PythonOperator
|
|
→ Always wrap in try-except, raise AirflowException
|
|
|
|
❌ NEVER: Create tasks outside DAG context
|
|
→ All tasks must be defined within 'with DAG()' block
|
|
|
|
❌ NEVER: Use mutable default_args
|
|
→ Use new dict for each DAG
|
|
|
|
❌ NEVER: Ignore SLA monitoring
|
|
→ Set sla parameter for time-sensitive pipelines</code></pre>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>etl-patterns.md</code> - Data extraction and loading logic</li><li><code>stream-processing.md</code> - Real-time alternative to batch</li><li><code>pipeline-orchestration.md</code> - Advanced workflow patterns</li><li><code>data-validation.md</code> - Quality checks in pipelines</li></ul>
|
|
<p><strong>Last Updated</strong>: 2025-10-18 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/data-batch-processing.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |