mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
Every page now loads /webmcp.js, scripts/webmcp.ts with its types stripped (npm run webmcp). Where the browser has document.modelContext it registers two read-only tools: search_skills over the published skills index, and get_skill, which returns a skill's markdown checked against the index digest. The markdown alternate link moves from build.py to site.py, so CI holds every page to it. The API catalog gains the site itself (llms.txt and the skills index) and hanzo.ai/agents; the live discovery check now fetches every catalog link. tests/site.mjs drives the tools through a recording modelContext, and checks a browser without one loads clean.
622 lines
29 KiB
HTML
622 lines
29 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>validation-patterns</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/validation-patterns">
|
|
<link rel="alternate" type="text/markdown" href="/validation-patterns.md">
|
|
<script type="module" src="/webmcp.js"></script>
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/validation-patterns.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>validation-patterns</h1><p class="lede">Reference guide for test patterns to validate hole resolutions including characterization tests, property tests, and integration tests. Use as reference when validating refactoring steps.</p><h1>Validation Patterns</h1>
|
|
<p>Test patterns for validating hole resolutions.</p>
|
|
<h2>Characterization Tests</h2>
|
|
<p>Capture current behavior as safety net.</p>
|
|
<h3>Pattern: API Contract Preservation</h3>
|
|
<pre><code class="lang-python"># tests/characterization/test_api_contracts.py
|
|
import pytest
|
|
import json
|
|
from pathlib import Path
|
|
|
|
class TestAPIContracts:
|
|
@pytest.fixture(scope="session")
|
|
def baseline_dir(self):
|
|
return Path("tests/characterization/baselines")
|
|
|
|
def test_all_public_apis(self, baseline_dir):
|
|
"""Every public API preserves exact behavior"""
|
|
apis = discover_public_apis()
|
|
|
|
for api in apis:
|
|
test_cases = load_test_cases(api)
|
|
|
|
for case in test_cases:
|
|
# Load baseline result
|
|
baseline_file = baseline_dir / f"{api}_{case.id}.json"
|
|
if baseline_file.exists():
|
|
baseline = json.loads(baseline_file.read_text())
|
|
else:
|
|
# First run - save baseline
|
|
result = execute_api(api, case)
|
|
baseline_file.write_text(json.dumps(result, indent=2))
|
|
baseline = result
|
|
|
|
# Compare current result to baseline
|
|
current = execute_api(api, case)
|
|
assert current == baseline, (
|
|
f"API {api} behavior changed for {case.id}\n"
|
|
f"Expected: {baseline}\n"
|
|
f"Got: {current}"
|
|
)</code></pre>
|
|
<h3>Pattern: Performance Baseline</h3>
|
|
<pre><code class="lang-python"># tests/characterization/test_performance.py
|
|
import pytest
|
|
import time
|
|
import statistics
|
|
|
|
class TestPerformanceBaselines:
|
|
ITERATIONS = 100
|
|
TOLERANCE = 1.2 # Allow 20% slowdown
|
|
|
|
@pytest.fixture(scope="session")
|
|
def baselines(self):
|
|
path = Path("tests/characterization/perf_baselines.json")
|
|
if path.exists():
|
|
return json.loads(path.read_text())
|
|
return {}
|
|
|
|
def measure_operation(self, operation, *args):
|
|
"""Measure operation timing over multiple iterations"""
|
|
times = []
|
|
for _ in range(self.ITERATIONS):
|
|
start = time.perf_counter()
|
|
operation(*args)
|
|
end = time.perf_counter()
|
|
times.append(end - start)
|
|
|
|
return {
|
|
"p50": statistics.median(times),
|
|
"p99": statistics.quantiles(times, n=100)[-1],
|
|
"mean": statistics.mean(times),
|
|
"stdev": statistics.stdev(times)
|
|
}
|
|
|
|
def test_critical_operations(self, baselines):
|
|
"""Ensure critical operations don't regress"""
|
|
operations = [
|
|
("parse_prompt", parse_prompt, sample_prompt),
|
|
("generate_ir", generate_ir, sample_input),
|
|
("execute_code", execute_code, sample_code),
|
|
]
|
|
|
|
for name, operation, test_data in operations:
|
|
current = self.measure_operation(operation, test_data)
|
|
|
|
if name in baselines:
|
|
baseline = baselines[name]
|
|
# Check p99 latency doesn't exceed tolerance
|
|
assert current["p99"] <= baseline["p99"] * self.TOLERANCE, (
|
|
f"{name} performance regression\n"
|
|
f"Baseline p99: {baseline['p99']:.4f}s\n"
|
|
f"Current p99: {current['p99']:.4f}s\n"
|
|
f"Increase: {(current['p99'] / baseline['p99']):.2%}"
|
|
)
|
|
else:
|
|
# First run - save baseline
|
|
baselines[name] = current</code></pre>
|
|
<h2>Resolution Validation Tests</h2>
|
|
<p>Verify hole resolution meets criteria.</p>
|
|
<h3>Pattern: Architecture Validation</h3>
|
|
<pre><code class="lang-python"># tests/refactor/test_r1_architecture.py
|
|
import ast
|
|
import importlib
|
|
from pathlib import Path
|
|
|
|
class TestArchitectureResolution:
|
|
"""Validate R1: target_architecture is resolved correctly"""
|
|
|
|
def test_layers_defined(self):
|
|
"""All layers exist with expected modules"""
|
|
layers = load_architecture_spec()["layers"]
|
|
|
|
for layer_name, expected_modules in layers.items():
|
|
layer_path = Path(f"src/{layer_name}")
|
|
assert layer_path.exists(), f"Layer {layer_name} not found"
|
|
|
|
actual_modules = {
|
|
f.stem for f in layer_path.glob("*.py")
|
|
if f.stem != "__init__"
|
|
}
|
|
|
|
assert actual_modules == set(expected_modules), (
|
|
f"Layer {layer_name} modules mismatch\n"
|
|
f"Expected: {expected_modules}\n"
|
|
f"Actual: {actual_modules}"
|
|
)
|
|
|
|
def test_no_layer_violations(self):
|
|
"""Verify no imports violate layer rules"""
|
|
arch = load_architecture_spec()
|
|
violations = []
|
|
|
|
for layer, rules in arch["rules"].items():
|
|
allowed = set(rules["can_import"])
|
|
forbidden = set(rules["cannot_import"])
|
|
|
|
for module_file in Path(f"src/{layer}").glob("**/*.py"):
|
|
tree = ast.parse(module_file.read_text())
|
|
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Import):
|
|
for alias in node.names:
|
|
imported_layer = alias.name.split(".")[0]
|
|
if imported_layer in forbidden:
|
|
violations.append({
|
|
"file": str(module_file),
|
|
"layer": layer,
|
|
"illegal_import": imported_layer
|
|
})
|
|
|
|
assert len(violations) == 0, (
|
|
f"Found {len(violations)} layer violations:\n" +
|
|
"\n".join(str(v) for v in violations)
|
|
)
|
|
|
|
def test_dependency_graph_acyclic(self):
|
|
"""Ensure no circular dependencies"""
|
|
graph = build_dependency_graph()
|
|
cycles = detect_cycles(graph)
|
|
|
|
assert len(cycles) == 0, (
|
|
f"Found circular dependencies:\n" +
|
|
"\n".join(" -> ".join(cycle) for cycle in cycles)
|
|
)</code></pre>
|
|
<h3>Pattern: Consolidation Validation</h3>
|
|
<pre><code class="lang-python"># tests/refactor/test_r4_consolidation.py
|
|
import pytest
|
|
|
|
class TestConsolidationResolution:
|
|
"""Validate R4: consolidation_targets resolved correctly"""
|
|
|
|
@pytest.fixture
|
|
def old_functions(self):
|
|
"""Old functions being consolidated"""
|
|
return [parse_v1, parse_v2, parse_v3]
|
|
|
|
@pytest.fixture
|
|
def test_cases(self):
|
|
"""Comprehensive test cases covering all variants"""
|
|
return load_test_cases("parsing")
|
|
|
|
def test_unified_function_equivalence(self, old_functions, test_cases):
|
|
"""New unified function produces same results as old functions"""
|
|
for old_func in old_functions:
|
|
# Get test cases specific to this variant
|
|
variant_cases = [
|
|
case for case in test_cases
|
|
if case.variant == old_func.__name__
|
|
]
|
|
|
|
for case in variant_cases:
|
|
old_result = old_func(case.input)
|
|
new_result = unified_parse(case.input, mode=old_func.__name__)
|
|
|
|
assert old_result == new_result, (
|
|
f"Consolidated function differs from {old_func.__name__}\n"
|
|
f"Input: {case.input}\n"
|
|
f"Expected: {old_result}\n"
|
|
f"Got: {new_result}"
|
|
)
|
|
|
|
def test_no_remaining_duplicates(self):
|
|
"""Verify duplicate code eliminated"""
|
|
from radon.complexity import cc_visit
|
|
|
|
duplicates = find_code_clones(
|
|
path="src/",
|
|
min_lines=6,
|
|
similarity_threshold=0.8
|
|
)
|
|
|
|
assert len(duplicates) == 0, (
|
|
f"Found {len(duplicates)} duplicate code blocks:\n" +
|
|
"\n".join(f"{d.file1}:{d.line1} <-> {d.file2}:{d.line2}"
|
|
for d in duplicates)
|
|
)
|
|
|
|
def test_complexity_reduced(self):
|
|
"""Verify cyclomatic complexity improved"""
|
|
old_complexity = sum(
|
|
cc_visit(open(f).read())[0].complexity
|
|
for f in ["parse_v1.py", "parse_v2.py", "parse_v3.py"]
|
|
)
|
|
|
|
new_complexity = cc_visit(
|
|
open("parser.py").read()
|
|
)[0].complexity
|
|
|
|
assert new_complexity < old_complexity, (
|
|
f"Complexity not reduced\n"
|
|
f"Old: {old_complexity}\n"
|
|
f"New: {new_complexity}"
|
|
)</code></pre>
|
|
<h3>Pattern: Performance Validation</h3>
|
|
<pre><code class="lang-python"># tests/refactor/test_r5_performance.py
|
|
import pytest
|
|
import time
|
|
|
|
class TestPerformanceImprovement:
|
|
"""Validate performance holes resolved correctly"""
|
|
|
|
ITERATIONS = 100
|
|
|
|
def measure_throughput(self, operation, test_data):
|
|
"""Measure operations per second"""
|
|
start = time.perf_counter()
|
|
for _ in range(self.ITERATIONS):
|
|
operation(test_data)
|
|
end = time.perf_counter()
|
|
|
|
return self.ITERATIONS / (end - start)
|
|
|
|
def test_throughput_improved(self):
|
|
"""Refactored code has better throughput"""
|
|
test_data = load_sample_data()
|
|
|
|
# Get baseline from characterization tests
|
|
baseline = load_baseline("throughput")
|
|
|
|
# Measure new throughput
|
|
current = self.measure_throughput(refactored_operation, test_data)
|
|
|
|
improvement = (current - baseline) / baseline
|
|
|
|
assert improvement > 0, (
|
|
f"Throughput regressed by {-improvement:.2%}\n"
|
|
f"Baseline: {baseline:.2f} ops/sec\n"
|
|
f"Current: {current:.2f} ops/sec"
|
|
)
|
|
|
|
print(f"Throughput improved by {improvement:.2%}")</code></pre>
|
|
<h2>Integration Tests</h2>
|
|
<p>End-to-end validation.</p>
|
|
<h3>Pattern: Workflow Equivalence</h3>
|
|
<pre><code class="lang-python"># tests/integration/test_workflows.py
|
|
import pytest
|
|
|
|
class TestWorkflowEquivalence:
|
|
"""Ensure end-to-end workflows produce identical results"""
|
|
|
|
@pytest.fixture
|
|
def scenarios(self):
|
|
"""Real-world scenarios with expected outputs"""
|
|
return load_scenarios("tests/integration/scenarios/")
|
|
|
|
def test_complete_workflows(self, scenarios):
|
|
"""Run full workflows on old and new code"""
|
|
for scenario in scenarios:
|
|
# Run on old code (from main branch)
|
|
old_result = run_workflow_old(scenario.input)
|
|
|
|
# Run on new code (refactored branch)
|
|
new_result = run_workflow_new(scenario.input)
|
|
|
|
# Results should be identical
|
|
assert old_result == new_result, (
|
|
f"Workflow {scenario.name} produced different results\n"
|
|
f"Input: {scenario.input}\n"
|
|
f"Old: {old_result}\n"
|
|
f"New: {new_result}"
|
|
)
|
|
|
|
# Performance should not regress
|
|
old_time = scenario.baseline_time
|
|
new_time = measure_workflow_time(scenario.input)
|
|
|
|
assert new_time <= old_time * 1.2, (
|
|
f"Workflow {scenario.name} performance regressed\n"
|
|
f"Old: {old_time:.2f}s\n"
|
|
f"New: {new_time:.2f}s"
|
|
)</code></pre>
|
|
<h2>Property-Based Tests</h2>
|
|
<p>Validate properties that should always hold.</p>
|
|
<h3>Pattern: Invariant Checking</h3>
|
|
<pre><code class="lang-python"># tests/refactor/test_invariants.py
|
|
from hypothesis import given, strategies as st
|
|
|
|
class TestInvariants:
|
|
"""Test properties that must always hold"""
|
|
|
|
@given(st.text(min_size=1, max_size=1000))
|
|
def test_parse_serialize_roundtrip(self, text):
|
|
"""Parse then serialize should return to original"""
|
|
parsed = parse(text)
|
|
serialized = serialize(parsed)
|
|
reparsed = parse(serialized)
|
|
|
|
assert parsed == reparsed
|
|
|
|
@given(st.lists(st.integers(), min_size=0, max_size=100))
|
|
def test_sort_idempotent(self, data):
|
|
"""Sorting twice gives same result as sorting once"""
|
|
sorted_once = custom_sort(data)
|
|
sorted_twice = custom_sort(sorted_once)
|
|
|
|
assert sorted_once == sorted_twice
|
|
|
|
@given(st.dictionaries(st.text(), st.integers()))
|
|
def test_cache_consistency(self, data):
|
|
"""Cache should return same value as uncached"""
|
|
for key, value in data.items():
|
|
# Uncached computation
|
|
uncached_result = expensive_operation(key, value)
|
|
|
|
# Cached computation
|
|
cached_result = cached_expensive_operation(key, value)
|
|
|
|
assert uncached_result == cached_result</code></pre>
|
|
<h2>Migration Tests</h2>
|
|
<p>Validate safe transition.</p>
|
|
<h3>Pattern: Feature Flag Testing</h3>
|
|
<pre><code class="lang-python"># tests/migration/test_feature_flags.py
|
|
import pytest
|
|
|
|
class TestFeatureFlags:
|
|
"""Validate feature flag rollout mechanism"""
|
|
|
|
def test_flag_off_uses_old_code(self):
|
|
"""When flag disabled, should use old implementation"""
|
|
set_feature_flag("use_refactored_code", False)
|
|
|
|
result = call_api("/endpoint", test_data)
|
|
|
|
# Should match old behavior exactly
|
|
assert result == old_baseline_result
|
|
assert get_active_implementation() == "old"
|
|
|
|
def test_flag_on_uses_new_code(self):
|
|
"""When flag enabled, should use new implementation"""
|
|
set_feature_flag("use_refactored_code", True)
|
|
|
|
result = call_api("/endpoint", test_data)
|
|
|
|
# Should match new behavior
|
|
assert result == new_expected_result
|
|
assert get_active_implementation() == "new"
|
|
|
|
def test_toggle_flag_mid_flight(self):
|
|
"""Can toggle flag without restart"""
|
|
set_feature_flag("use_refactored_code", False)
|
|
result1 = call_api("/endpoint", test_data)
|
|
|
|
set_feature_flag("use_refactored_code", True)
|
|
result2 = call_api("/endpoint", test_data)
|
|
|
|
# Both should work, possibly with different results
|
|
assert result1 is not None
|
|
assert result2 is not None
|
|
|
|
def test_percentage_rollout(self):
|
|
"""Percentage rollout distributes correctly"""
|
|
set_feature_flag_percentage("use_refactored_code", 50)
|
|
|
|
results = {"old": 0, "new": 0}
|
|
for user_id in range(1000):
|
|
impl = get_implementation_for_user(user_id)
|
|
results[impl] += 1
|
|
|
|
# Should be approximately 50/50
|
|
ratio = results["new"] / 1000
|
|
assert 0.45 <= ratio <= 0.55, f"Rollout ratio: {ratio}"</code></pre>
|
|
<h2>Summary</h2>
|
|
<p>Validation patterns ensure:</p>
|
|
<ol><li><strong>Characterization tests</strong> preserve existing behavior</li><li><strong>Resolution tests</strong> verify holes are correctly resolved</li><li><strong>Integration tests</strong> validate end-to-end workflows</li><li><strong>Property tests</strong> check invariants hold</li><li><strong>Migration tests</strong> ensure safe transitions</li></ol>
|
|
<p>Use these patterns throughout the refactoring process to maintain confidence and catch regressions early.</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/validation-patterns.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |