mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
779 lines
33 KiB
HTML
779 lines
33 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>data-data-validation</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/data-data-validation.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/data-data-validation">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/data-data-validation.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>data-data-validation</h1><p class="lede">Validating data schema before processing</p><h1>Data Validation</h1>
|
|
<p><strong>Scope</strong>: Schema validation, data quality checks, anomaly detection <strong>Lines</strong>: 357 <strong>Last Updated</strong>: 2025-10-18 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Use this skill when:</p>
|
|
<ul><li>Validating data schema before processing</li><li>Implementing data quality gates in pipelines</li><li>Detecting anomalies and outliers in datasets</li><li>Ensuring referential integrity across tables</li><li>Monitoring data freshness and completeness</li><li>Building data quality dashboards</li><li>Working with Great Expectations, Pandera, or custom validators</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Validation Types</h3>
|
|
<pre><code>Schema Validation
|
|
→ Data types, column presence, nullable constraints
|
|
→ Use when: Incoming data format can vary
|
|
→ Tools: Pandera, Great Expectations, JSON Schema
|
|
|
|
Business Rules Validation
|
|
→ Domain-specific constraints (age > 0, email format)
|
|
→ Use when: Enforcing business logic
|
|
→ Tools: Custom validators, dbt tests
|
|
|
|
Statistical Validation
|
|
→ Distribution checks, outliers, anomalies
|
|
→ Use when: Data quality monitoring
|
|
→ Tools: Great Expectations, custom statistics
|
|
|
|
Referential Integrity
|
|
→ Foreign key constraints, cross-table checks
|
|
→ Use when: Multi-table relationships
|
|
→ Tools: SQL constraints, dbt tests</code></pre>
|
|
<h3>Validation Strategies</h3>
|
|
<pre><code>Fail Fast
|
|
→ Stop pipeline on first error
|
|
→ Use when: Downstream depends on quality
|
|
|
|
Collect All Errors
|
|
→ Continue validation, report all issues
|
|
→ Use when: Data profiling, quality reporting
|
|
|
|
Quarantine
|
|
→ Separate valid/invalid records
|
|
→ Use when: Partial data processing acceptable
|
|
|
|
Auto-Repair
|
|
→ Fix known issues automatically
|
|
→ Use when: Predictable, fixable problems</code></pre>
|
|
<h3>Data Quality Dimensions</h3>
|
|
<pre><code>Completeness: No missing required fields
|
|
Accuracy: Values match real-world entities
|
|
Consistency: Same value across systems
|
|
Timeliness: Data is fresh/current
|
|
Uniqueness: No duplicates where not expected
|
|
Validity: Values in expected format/range</code></pre>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Pandera Schema Validation (Python)</h3>
|
|
<pre><code class="lang-python">import pandas as pd
|
|
import pandera as pa
|
|
from pandera import Column, Check, DataFrameSchema
|
|
from datetime import datetime
|
|
|
|
# Define schema with constraints
|
|
user_schema = DataFrameSchema(
|
|
columns={
|
|
"user_id": Column(
|
|
int,
|
|
checks=[
|
|
Check.greater_than(0),
|
|
Check(lambda s: s.is_unique, error="user_id must be unique")
|
|
],
|
|
nullable=False
|
|
),
|
|
"email": Column(
|
|
str,
|
|
checks=Check.str_matches(r'^[\w\.-]+@[\w\.-]+\.\w+$'),
|
|
nullable=False
|
|
),
|
|
"age": Column(
|
|
int,
|
|
checks=[
|
|
Check.in_range(0, 120),
|
|
Check(lambda s: s >= 18, error="Users must be 18+")
|
|
],
|
|
nullable=True
|
|
),
|
|
"created_at": Column(
|
|
pa.DateTime,
|
|
checks=Check.less_than_or_equal_to(datetime.now()),
|
|
nullable=False
|
|
),
|
|
"status": Column(
|
|
str,
|
|
checks=Check.isin(['active', 'inactive', 'suspended']),
|
|
nullable=False
|
|
)
|
|
},
|
|
strict=True, # No extra columns allowed
|
|
coerce=True # Attempt type coercion
|
|
)
|
|
|
|
# Validate DataFrame
|
|
def validate_users(df: pd.DataFrame) -> tuple[pd.DataFrame, list]:
|
|
"""Validate user data, return valid rows and errors"""
|
|
errors = []
|
|
|
|
try:
|
|
# Validate entire DataFrame
|
|
validated_df = user_schema.validate(df, lazy=True)
|
|
return validated_df, errors
|
|
|
|
except pa.errors.SchemaErrors as e:
|
|
# Collect all validation errors
|
|
for error in e.failure_cases.itertuples():
|
|
errors.append({
|
|
'column': error.column,
|
|
'check': error.check,
|
|
'index': error.index,
|
|
'value': error.failure_case
|
|
})
|
|
|
|
# Return rows that passed validation
|
|
valid_indices = set(df.index) - set(e.failure_cases['index'])
|
|
valid_df = df.loc[list(valid_indices)]
|
|
|
|
return valid_df, errors
|
|
|
|
# Usage
|
|
df = pd.DataFrame({
|
|
'user_id': [1, 2, 3, 2], # Duplicate
|
|
'email': ['[email protected]', 'invalid-email', '[email protected]', '[email protected]'],
|
|
'age': [25, 15, 30, 200], # 15 too young, 200 out of range
|
|
'created_at': [datetime(2025, 1, 1)] * 4,
|
|
'status': ['active', 'pending', 'inactive', 'active'] # 'pending' not in allowed
|
|
})
|
|
|
|
valid_df, errors = validate_users(df)
|
|
print(f"Valid rows: {len(valid_df)}")
|
|
print(f"Errors: {errors}")</code></pre>
|
|
<h3>Pattern 2: Great Expectations Integration</h3>
|
|
<pre><code class="lang-python">import great_expectations as gx
|
|
from great_expectations.data_context import DataContext
|
|
from great_expectations.checkpoint import SimpleCheckpoint
|
|
|
|
class GXValidator:
|
|
def __init__(self, context_root_dir: str = None):
|
|
"""Initialize Great Expectations context"""
|
|
if context_root_dir:
|
|
self.context = DataContext(context_root_dir)
|
|
else:
|
|
self.context = gx.get_context()
|
|
|
|
def create_expectation_suite(self, suite_name: str) -> None:
|
|
"""Create expectation suite for a dataset"""
|
|
suite = self.context.add_expectation_suite(suite_name)
|
|
|
|
# Define expectations
|
|
suite.add_expectation(
|
|
gx.expectations.ExpectColumnValuesToNotBeNull(column="user_id")
|
|
)
|
|
suite.add_expectation(
|
|
gx.expectations.ExpectColumnValuesToBeUnique(column="user_id")
|
|
)
|
|
suite.add_expectation(
|
|
gx.expectations.ExpectColumnValuesToBeInSet(
|
|
column="status",
|
|
value_set=['active', 'inactive', 'suspended']
|
|
)
|
|
)
|
|
suite.add_expectation(
|
|
gx.expectations.ExpectColumnValuesToBeBetween(
|
|
column="age",
|
|
min_value=18,
|
|
max_value=120
|
|
)
|
|
)
|
|
suite.add_expectation(
|
|
gx.expectations.ExpectColumnValuesToMatchRegex(
|
|
column="email",
|
|
regex=r'^[\w\.-]+@[\w\.-]+\.\w+$'
|
|
)
|
|
)
|
|
|
|
self.context.save_expectation_suite(suite)
|
|
|
|
def validate_dataframe(
|
|
self,
|
|
df: pd.DataFrame,
|
|
suite_name: str,
|
|
datasource_name: str = "pandas_datasource"
|
|
) -> dict:
|
|
"""Validate DataFrame against expectation suite"""
|
|
# Add datasource
|
|
datasource = self.context.sources.add_pandas(datasource_name)
|
|
data_asset = datasource.add_dataframe_asset(name="df_asset")
|
|
|
|
# Create batch request
|
|
batch_request = data_asset.build_batch_request(dataframe=df)
|
|
|
|
# Create and run checkpoint
|
|
checkpoint = SimpleCheckpoint(
|
|
name="validation_checkpoint",
|
|
data_context=self.context,
|
|
validations=[
|
|
{
|
|
"batch_request": batch_request,
|
|
"expectation_suite_name": suite_name
|
|
}
|
|
]
|
|
)
|
|
|
|
result = checkpoint.run()
|
|
|
|
return {
|
|
'success': result.success,
|
|
'statistics': result.statistics,
|
|
'results': result.list_validation_results()
|
|
}
|
|
|
|
# Usage
|
|
validator = GXValidator()
|
|
validator.create_expectation_suite("user_suite")
|
|
|
|
result = validator.validate_dataframe(df, "user_suite")
|
|
print(f"Validation success: {result['success']}")
|
|
print(f"Statistics: {result['statistics']}")</code></pre>
|
|
<h3>Pattern 3: Custom Statistical Validation</h3>
|
|
<pre><code class="lang-python">import numpy as np
|
|
from scipy import stats
|
|
from typing import Dict, Any, List
|
|
import pandas as pd
|
|
|
|
class StatisticalValidator:
|
|
def __init__(self, baseline_stats: Dict[str, Any] = None):
|
|
"""Initialize with baseline statistics"""
|
|
self.baseline_stats = baseline_stats or {}
|
|
|
|
def calculate_baseline(self, df: pd.DataFrame) -> Dict[str, Any]:
|
|
"""Calculate baseline statistics for numeric columns"""
|
|
baseline = {}
|
|
|
|
for col in df.select_dtypes(include=[np.number]).columns:
|
|
baseline[col] = {
|
|
'mean': df[col].mean(),
|
|
'std': df[col].std(),
|
|
'median': df[col].median(),
|
|
'q1': df[col].quantile(0.25),
|
|
'q3': df[col].quantile(0.75),
|
|
'min': df[col].min(),
|
|
'max': df[col].max(),
|
|
'null_rate': df[col].isnull().mean()
|
|
}
|
|
|
|
self.baseline_stats = baseline
|
|
return baseline
|
|
|
|
def detect_outliers_iqr(
|
|
self,
|
|
df: pd.DataFrame,
|
|
column: str,
|
|
multiplier: float = 1.5
|
|
) -> pd.Series:
|
|
"""Detect outliers using IQR method"""
|
|
Q1 = df[column].quantile(0.25)
|
|
Q3 = df[column].quantile(0.75)
|
|
IQR = Q3 - Q1
|
|
|
|
lower_bound = Q1 - multiplier * IQR
|
|
upper_bound = Q3 + multiplier * IQR
|
|
|
|
is_outlier = (df[column] < lower_bound) | (df[column] > upper_bound)
|
|
return is_outlier
|
|
|
|
def detect_outliers_zscore(
|
|
self,
|
|
df: pd.DataFrame,
|
|
column: str,
|
|
threshold: float = 3.0
|
|
) -> pd.Series:
|
|
"""Detect outliers using Z-score method"""
|
|
z_scores = np.abs(stats.zscore(df[column].dropna()))
|
|
is_outlier = pd.Series([False] * len(df), index=df.index)
|
|
is_outlier[df[column].notna()] = z_scores > threshold
|
|
return is_outlier
|
|
|
|
def validate_distribution(
|
|
self,
|
|
df: pd.DataFrame,
|
|
column: str,
|
|
drift_threshold: float = 0.1
|
|
) -> Dict[str, Any]:
|
|
"""Check if distribution has drifted from baseline"""
|
|
if column not in self.baseline_stats:
|
|
return {'error': 'No baseline for column'}
|
|
|
|
baseline = self.baseline_stats[column]
|
|
current_mean = df[column].mean()
|
|
current_std = df[column].std()
|
|
|
|
# Calculate drift
|
|
mean_drift = abs(current_mean - baseline['mean']) / baseline['mean']
|
|
std_drift = abs(current_std - baseline['std']) / baseline['std']
|
|
|
|
return {
|
|
'column': column,
|
|
'mean_drift': mean_drift,
|
|
'std_drift': std_drift,
|
|
'drift_detected': mean_drift > drift_threshold or std_drift > drift_threshold,
|
|
'baseline_mean': baseline['mean'],
|
|
'current_mean': current_mean,
|
|
'baseline_std': baseline['std'],
|
|
'current_std': current_std
|
|
}
|
|
|
|
def validate_completeness(
|
|
self,
|
|
df: pd.DataFrame,
|
|
required_columns: List[str],
|
|
max_null_rate: float = 0.05
|
|
) -> Dict[str, Any]:
|
|
"""Check data completeness"""
|
|
results = {}
|
|
|
|
for col in required_columns:
|
|
if col not in df.columns:
|
|
results[col] = {'error': 'Column missing'}
|
|
continue
|
|
|
|
null_rate = df[col].isnull().mean()
|
|
results[col] = {
|
|
'null_rate': null_rate,
|
|
'passes': null_rate <= max_null_rate,
|
|
'null_count': df[col].isnull().sum(),
|
|
'total_count': len(df)
|
|
}
|
|
|
|
return results
|
|
|
|
# Usage
|
|
validator = StatisticalValidator()
|
|
|
|
# Calculate baseline from historical data
|
|
historical_df = pd.DataFrame({
|
|
'revenue': np.random.normal(1000, 200, 10000),
|
|
'quantity': np.random.poisson(50, 10000)
|
|
})
|
|
validator.calculate_baseline(historical_df)
|
|
|
|
# Validate new data
|
|
new_df = pd.DataFrame({
|
|
'revenue': np.random.normal(1200, 250, 1000), # Drifted
|
|
'quantity': np.random.poisson(51, 1000)
|
|
})
|
|
|
|
# Check distribution drift
|
|
drift = validator.validate_distribution(new_df, 'revenue')
|
|
print(f"Distribution drift: {drift}")
|
|
|
|
# Detect outliers
|
|
outliers = validator.detect_outliers_iqr(new_df, 'revenue')
|
|
print(f"Outliers detected: {outliers.sum()}")</code></pre>
|
|
<h3>Pattern 4: Cross-Table Validation</h3>
|
|
<pre><code class="lang-python">from sqlalchemy import create_engine, text
|
|
from typing import List, Dict
|
|
|
|
class CrossTableValidator:
|
|
def __init__(self, connection_string: str):
|
|
self.engine = create_engine(connection_string)
|
|
|
|
def validate_referential_integrity(
|
|
self,
|
|
child_table: str,
|
|
child_column: str,
|
|
parent_table: str,
|
|
parent_column: str
|
|
) -> Dict[str, Any]:
|
|
"""Check foreign key integrity"""
|
|
query = text(f"""
|
|
SELECT
|
|
COUNT(*) as orphan_count,
|
|
ARRAY_AGG(DISTINCT c.{child_column}) as orphan_values
|
|
FROM {child_table} c
|
|
LEFT JOIN {parent_table} p ON c.{child_column} = p.{parent_column}
|
|
WHERE p.{parent_column} IS NULL
|
|
AND c.{child_column} IS NOT NULL
|
|
""")
|
|
|
|
with self.engine.connect() as conn:
|
|
result = conn.execute(query).fetchone()
|
|
|
|
return {
|
|
'valid': result.orphan_count == 0,
|
|
'orphan_count': result.orphan_count,
|
|
'orphan_values': result.orphan_values[:10] # Sample
|
|
}
|
|
|
|
def validate_aggregates(
|
|
self,
|
|
detail_table: str,
|
|
summary_table: str,
|
|
group_column: str,
|
|
agg_column: str,
|
|
tolerance: float = 0.01
|
|
) -> Dict[str, Any]:
|
|
"""Validate summary table against detail table"""
|
|
query = text(f"""
|
|
WITH detail_agg AS (
|
|
SELECT
|
|
{group_column},
|
|
SUM({agg_column}) as detail_sum
|
|
FROM {detail_table}
|
|
GROUP BY {group_column}
|
|
),
|
|
comparison AS (
|
|
SELECT
|
|
d.{group_column},
|
|
d.detail_sum,
|
|
s.{agg_column} as summary_sum,
|
|
ABS(d.detail_sum - s.{agg_column}) / NULLIF(d.detail_sum, 0) as diff_pct
|
|
FROM detail_agg d
|
|
FULL OUTER JOIN {summary_table} s
|
|
ON d.{group_column} = s.{group_column}
|
|
)
|
|
SELECT
|
|
COUNT(*) as total_groups,
|
|
SUM(CASE WHEN diff_pct > :tolerance THEN 1 ELSE 0 END) as mismatches,
|
|
ARRAY_AGG(
|
|
{group_column} ORDER BY diff_pct DESC
|
|
) FILTER (WHERE diff_pct > :tolerance) as mismatch_groups
|
|
FROM comparison
|
|
""")
|
|
|
|
with self.engine.connect() as conn:
|
|
result = conn.execute(query, {'tolerance': tolerance}).fetchone()
|
|
|
|
return {
|
|
'valid': result.mismatches == 0,
|
|
'total_groups': result.total_groups,
|
|
'mismatch_count': result.mismatches,
|
|
'mismatch_groups': result.mismatch_groups[:10]
|
|
}
|
|
|
|
def validate_freshness(
|
|
self,
|
|
table: str,
|
|
timestamp_column: str,
|
|
max_age_hours: int = 24
|
|
) -> Dict[str, Any]:
|
|
"""Check data freshness"""
|
|
query = text(f"""
|
|
SELECT
|
|
MAX({timestamp_column}) as latest_timestamp,
|
|
EXTRACT(EPOCH FROM (NOW() - MAX({timestamp_column}))) / 3600 as age_hours
|
|
FROM {table}
|
|
""")
|
|
|
|
with self.engine.connect() as conn:
|
|
result = conn.execute(query).fetchone()
|
|
|
|
return {
|
|
'valid': result.age_hours <= max_age_hours,
|
|
'latest_timestamp': result.latest_timestamp,
|
|
'age_hours': result.age_hours,
|
|
'max_age_hours': max_age_hours
|
|
}
|
|
|
|
# Usage
|
|
validator = CrossTableValidator("postgresql://user:pass@localhost/db")
|
|
|
|
# Check foreign keys
|
|
fk_result = validator.validate_referential_integrity(
|
|
'orders', 'customer_id',
|
|
'customers', 'id'
|
|
)
|
|
print(f"Referential integrity: {fk_result}")
|
|
|
|
# Validate aggregates
|
|
agg_result = validator.validate_aggregates(
|
|
'order_items', 'order_totals',
|
|
'order_id', 'amount'
|
|
)
|
|
print(f"Aggregate validation: {agg_result}")
|
|
|
|
# Check freshness
|
|
fresh_result = validator.validate_freshness(
|
|
'events', 'created_at', max_age_hours=2
|
|
)
|
|
print(f"Data freshness: {fresh_result}")</code></pre>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Common Validation Checks</h3>
|
|
<pre><code class="lang-python"># Null checks
|
|
df['column'].isnull().any()
|
|
df['column'].notna().all()
|
|
|
|
# Uniqueness
|
|
df['column'].is_unique
|
|
df.duplicated(subset=['col1', 'col2']).any()
|
|
|
|
# Range checks
|
|
df['column'].between(min_val, max_val).all()
|
|
(df['column'] >= 0).all()
|
|
|
|
# Set membership
|
|
df['column'].isin(allowed_values).all()
|
|
|
|
# Regex pattern
|
|
df['column'].str.match(r'^pattern$').all()
|
|
|
|
# Type checks
|
|
df['column'].dtype == 'int64'
|
|
pd.api.types.is_numeric_dtype(df['column'])
|
|
|
|
# Relationships
|
|
df['col1'].sum() == df['col2'].sum() # Balance check</code></pre>
|
|
<h3>Pandera Quick Syntax</h3>
|
|
<pre><code class="lang-python">schema = pa.DataFrameSchema({
|
|
"col": pa.Column(int, checks=pa.Check.greater_than(0)),
|
|
"email": pa.Column(str, checks=pa.Check.str_matches(r'@')),
|
|
"status": pa.Column(str, checks=pa.Check.isin(['A', 'B']))
|
|
})
|
|
|
|
validated_df = schema.validate(df)</code></pre>
|
|
<h3>Great Expectations Quick Setup</h3>
|
|
<pre><code class="lang-bash"># Initialize
|
|
great_expectations init
|
|
|
|
# Create expectation suite
|
|
great_expectations suite new
|
|
|
|
# Edit expectations
|
|
great_expectations suite edit my_suite
|
|
|
|
# Run checkpoint
|
|
great_expectations checkpoint run my_checkpoint</code></pre>
|
|
<h2>Anti-Patterns</h2>
|
|
<pre><code>❌ NEVER: Validate after transformation
|
|
→ Validate raw data first, then transform
|
|
|
|
❌ NEVER: Silently drop invalid records
|
|
→ Log, alert, or quarantine for review
|
|
|
|
❌ NEVER: Use generic error messages
|
|
→ Specify column, check, and value that failed
|
|
|
|
❌ NEVER: Validate only in production
|
|
→ Run same validations in dev/staging
|
|
|
|
❌ NEVER: Skip statistical validation
|
|
→ Check distributions, not just schema
|
|
|
|
❌ NEVER: Ignore data drift
|
|
→ Monitor and alert on statistical changes
|
|
|
|
❌ NEVER: Fail pipeline without retry logic
|
|
→ Transient issues may resolve on retry
|
|
|
|
❌ NEVER: Hardcode validation rules in code
|
|
→ Use configuration files or database
|
|
|
|
❌ NEVER: Validate samples instead of full dataset
|
|
→ Sample for profiling, validate all for quality
|
|
|
|
❌ NEVER: Skip cross-table validation
|
|
→ Verify referential integrity and aggregates</code></pre>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>etl-patterns.md</code> - Integrating validation into ETL</li><li><code>batch-processing.md</code> - Adding validation tasks to DAGs</li><li><code>stream-processing.md</code> - Real-time validation patterns</li><li><code>pipeline-orchestration.md</code> - Orchestrating validation workflows</li></ul>
|
|
<p><strong>Last Updated</strong>: 2025-10-18 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/data-data-validation.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |