mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
730 lines
28 KiB
HTML
730 lines
28 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>cicd-ci-testing-strategy</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/cicd-ci-testing-strategy.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/cicd-ci-testing-strategy">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/cicd-ci-testing-strategy.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>cicd-ci-testing-strategy</h1><p class="lede">Designing test execution strategies for CI pipelines</p><h1>CI Testing Strategy</h1>
|
|
<p><strong>Scope</strong>: Test stages, parallel execution, test splitting, flaky test handling, and test result reporting in CI pipelines</p>
|
|
<p><strong>Lines</strong>: 365</p>
|
|
<p><strong>Last Updated</strong>: 2025-10-18</p>
|
|
<p><strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Use this skill when:</p>
|
|
<ul><li>Designing test execution strategies for CI pipelines</li><li>Implementing test stages (unit, integration, e2e)</li><li>Parallelizing tests for faster feedback</li><li>Handling flaky tests and test retries</li><li>Setting up test result reporting and analysis</li><li>Implementing test splitting across multiple runners</li><li>Optimizing test execution order for faster failure detection</li><li>Managing test data and fixtures in CI</li></ul>
|
|
<p>Don't use this skill for:</p>
|
|
<ul><li>Workflow syntax and configuration (see <code>github-actions-workflows.md</code>)</li><li>Deployment strategies (see <code>cd-deployment-patterns.md</code>)</li><li>General pipeline optimization (see <code>ci-optimization.md</code>)</li></ul>
|
|
<hr>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Test Pyramid in CI</h3>
|
|
<pre><code> /\
|
|
/E2E\ ← Slow, few, high-value
|
|
/──────\
|
|
/ Integ \ ← Medium speed, moderate count
|
|
/──────────\
|
|
/ Unit \ ← Fast, many, isolated
|
|
/──────────────\</code></pre>
|
|
<h3>Test Stages</h3>
|
|
<ol><li><strong>Unit Tests</strong>: Fast, isolated, no external dependencies</li><li><strong>Integration Tests</strong>: Test interactions between components</li><li><strong>E2E Tests</strong>: Full application flow, slowest but highest confidence</li><li><strong>Security Tests</strong>: SAST, dependency scanning</li><li><strong>Performance Tests</strong>: Load testing, benchmarks</li></ol>
|
|
<h3>Execution Strategy</h3>
|
|
<pre><code>Fast Feedback Loop:
|
|
Lint → Unit Tests → Build → Integration Tests → E2E Tests
|
|
↓ Fail fast at each stage
|
|
Stop pipeline on failure (unless configured otherwise)</code></pre>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Multi-Stage Testing</h3>
|
|
<pre><code class="lang-yaml">name: Test Pipeline
|
|
|
|
on: [push, pull_request]
|
|
|
|
jobs:
|
|
lint:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
- run: npm ci
|
|
- run: npm run lint
|
|
|
|
unit-tests:
|
|
needs: lint
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
- run: npm ci
|
|
- run: npm run test:unit
|
|
- uses: codecov/codecov-action@v4
|
|
with:
|
|
files: ./coverage/coverage-final.json
|
|
|
|
integration-tests:
|
|
needs: unit-tests
|
|
runs-on: ubuntu-latest
|
|
|
|
services:
|
|
postgres:
|
|
image: postgres:16
|
|
env:
|
|
POSTGRES_PASSWORD: postgres
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
ports:
|
|
- 5432:5432
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
- run: npm ci
|
|
- run: npm run test:integration
|
|
env:
|
|
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test
|
|
|
|
e2e-tests:
|
|
needs: integration-tests
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
- run: npm ci
|
|
- run: npx playwright install --with-deps
|
|
- run: npm run test:e2e
|
|
- uses: actions/upload-artifact@v4
|
|
if: failure()
|
|
with:
|
|
name: playwright-screenshots
|
|
path: test-results/</code></pre>
|
|
<h3>Parallel Test Execution</h3>
|
|
<pre><code class="lang-yaml">jobs:
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
# Split tests across 4 runners
|
|
shard: [1, 2, 3, 4]
|
|
fail-fast: false
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
- run: npm ci
|
|
|
|
# Playwright test sharding
|
|
- run: npx playwright test --shard=${{ matrix.shard }}/4
|
|
|
|
# Jest test sharding
|
|
- run: npm test -- --shard=${{ matrix.shard }}/4
|
|
|
|
# Upload results
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: test-results-${{ matrix.shard }}
|
|
path: test-results/</code></pre>
|
|
<h3>Test Splitting by Type</h3>
|
|
<pre><code class="lang-yaml">jobs:
|
|
test-unit:
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
package: [core, api, ui, utils]
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
- run: npm ci
|
|
- run: npm test -- packages/${{ matrix.package }}
|
|
|
|
test-browser:
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
browser: [chromium, firefox, webkit]
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
- run: npm ci
|
|
- run: npx playwright install --with-deps ${{ matrix.browser }}
|
|
- run: npx playwright test --project=${{ matrix.browser }}</code></pre>
|
|
<h3>Flaky Test Handling</h3>
|
|
<pre><code class="lang-yaml">jobs:
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
- run: npm ci
|
|
|
|
# Retry failed tests automatically
|
|
- name: Run tests with retry
|
|
run: npm test -- --retries=2
|
|
|
|
# Playwright retry configuration
|
|
- name: Run E2E tests
|
|
run: npx playwright test
|
|
env:
|
|
PLAYWRIGHT_RETRIES: 2
|
|
|
|
# Custom retry script
|
|
- name: Run tests with custom retry
|
|
run: |
|
|
for i in {1..3}; do
|
|
npm test && break || {
|
|
echo "Attempt $i failed"
|
|
sleep 5
|
|
}
|
|
done
|
|
|
|
# Quarantine flaky tests
|
|
- name: Run stable tests
|
|
run: npm test -- --testPathIgnorePatterns=flaky
|
|
|
|
- name: Run flaky tests (non-blocking)
|
|
run: npm test -- --testPathPattern=flaky
|
|
continue-on-error: true</code></pre>
|
|
<h3>Test Result Reporting</h3>
|
|
<pre><code class="lang-yaml">jobs:
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
- run: npm ci
|
|
|
|
# Generate test reports
|
|
- name: Run tests
|
|
run: npm test -- --coverage --reporters=default --reporters=jest-junit
|
|
env:
|
|
JEST_JUNIT_OUTPUT_DIR: ./test-results
|
|
|
|
# Publish test results
|
|
- name: Publish test results
|
|
uses: EnricoMi/publish-unit-test-result-action@v2
|
|
if: always()
|
|
with:
|
|
files: test-results/*.xml
|
|
|
|
# Code coverage
|
|
- name: Upload coverage to Codecov
|
|
uses: codecov/codecov-action@v4
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
files: ./coverage/coverage-final.json
|
|
fail_ci_if_error: true
|
|
|
|
# Comment PR with coverage
|
|
- name: Coverage comment
|
|
uses: romeovs/[email protected]
|
|
if: github.event_name == 'pull_request'
|
|
with:
|
|
lcov-file: ./coverage/lcov.info
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}</code></pre>
|
|
<h3>Test Data Management</h3>
|
|
<pre><code class="lang-yaml">jobs:
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
# Cache test fixtures
|
|
- name: Cache test data
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: test/fixtures/
|
|
key: test-fixtures-${{ hashFiles('scripts/generate-fixtures.sh') }}
|
|
|
|
# Generate test data if not cached
|
|
- name: Generate test fixtures
|
|
run: |
|
|
if [ ! -d "test/fixtures" ]; then
|
|
./scripts/generate-fixtures.sh
|
|
fi
|
|
|
|
# Database seeding
|
|
- name: Setup test database
|
|
run: |
|
|
npm run db:migrate
|
|
npm run db:seed:test
|
|
|
|
- run: npm test
|
|
|
|
# Cleanup
|
|
- name: Cleanup test data
|
|
if: always()
|
|
run: npm run db:reset</code></pre>
|
|
<h3>Performance Testing</h3>
|
|
<pre><code class="lang-yaml">jobs:
|
|
benchmark:
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
- run: npm ci
|
|
|
|
# Run benchmarks
|
|
- name: Run benchmarks
|
|
run: npm run bench
|
|
|
|
# Compare with baseline
|
|
- name: Store benchmark result
|
|
uses: benchmark-action/github-action-benchmark@v1
|
|
with:
|
|
tool: 'benchmarkjs'
|
|
output-file-path: bench-results.json
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
auto-push: true
|
|
alert-threshold: '150%'
|
|
comment-on-alert: true
|
|
fail-on-alert: true</code></pre>
|
|
<h3>Contract Testing</h3>
|
|
<pre><code class="lang-yaml">jobs:
|
|
contract-tests:
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
- run: npm ci
|
|
|
|
# Provider verification (API)
|
|
- name: Run Pact provider tests
|
|
run: npm run test:pact:provider
|
|
env:
|
|
PACT_BROKER_URL: ${{ secrets.PACT_BROKER_URL }}
|
|
PACT_BROKER_TOKEN: ${{ secrets.PACT_BROKER_TOKEN }}
|
|
|
|
# Consumer verification (Client)
|
|
- name: Run Pact consumer tests
|
|
run: npm run test:pact:consumer
|
|
|
|
# Publish contracts
|
|
- name: Publish Pact contracts
|
|
if: github.ref == 'refs/heads/main'
|
|
run: npm run pact:publish</code></pre>
|
|
<h3>Visual Regression Testing</h3>
|
|
<pre><code class="lang-yaml">jobs:
|
|
visual-tests:
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
- run: npm ci
|
|
|
|
# Chromatic visual testing
|
|
- name: Run visual tests
|
|
uses: chromaui/action@v1
|
|
with:
|
|
projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
|
|
exitZeroOnChanges: false
|
|
exitOnceUploaded: true
|
|
|
|
# Percy visual testing
|
|
- name: Percy visual tests
|
|
run: npx percy exec -- npm run test:visual
|
|
env:
|
|
PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}</code></pre>
|
|
<h3>Conditional Test Execution</h3>
|
|
<pre><code class="lang-yaml">jobs:
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
# Detect changed files
|
|
- name: Get changed files
|
|
id: changed-files
|
|
uses: tj-actions/changed-files@v40
|
|
with:
|
|
files: |
|
|
src/**
|
|
tests/**
|
|
|
|
# Run tests only if relevant files changed
|
|
- name: Run tests
|
|
if: steps.changed-files.outputs.any_changed == 'true'
|
|
run: npm test
|
|
|
|
# Run specific tests based on changes
|
|
- name: Run API tests
|
|
if: contains(steps.changed-files.outputs.modified_files, 'src/api/')
|
|
run: npm run test:api
|
|
|
|
# Always run critical tests
|
|
- name: Run smoke tests
|
|
run: npm run test:smoke</code></pre>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Test Execution Order</h3>
|
|
<pre><code class="lang-yaml">1. Lint/Format # Fastest, catches syntax errors
|
|
2. Unit Tests # Fast, isolated
|
|
3. Integration Tests # Medium, external deps
|
|
4. E2E Tests # Slow, full stack
|
|
5. Performance Tests # Slowest, resource-intensive</code></pre>
|
|
<h3>Test Parallelization Strategies</h3>
|
|
<pre><code class="lang-yaml">By Shard: --shard=1/4
|
|
By Package: packages/${{ matrix.package }}
|
|
By Browser: --project=${{ matrix.browser }}
|
|
By Test Suite: --testPathPattern=${{ matrix.suite }}</code></pre>
|
|
<h3>Common Test Runners</h3>
|
|
<pre><code class="lang-bash"># Jest
|
|
npm test -- --coverage --shard=1/4
|
|
|
|
# Playwright
|
|
npx playwright test --shard=1/4 --project=chromium
|
|
|
|
# Pytest
|
|
pytest --numprocesses=4 --dist=loadscope
|
|
|
|
# Go
|
|
go test -parallel=4 ./...
|
|
|
|
# Rust
|
|
cargo test --jobs=4</code></pre>
|
|
<h3>Coverage Thresholds</h3>
|
|
<pre><code class="lang-json">{
|
|
"coverageThreshold": {
|
|
"global": {
|
|
"branches": 80,
|
|
"functions": 80,
|
|
"lines": 80,
|
|
"statements": 80
|
|
}
|
|
}
|
|
}</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>❌ Running All Tests Sequentially</h3>
|
|
<pre><code class="lang-yaml"># WRONG: Sequential execution
|
|
- run: npm run test:unit
|
|
- run: npm run test:integration
|
|
- run: npm run test:e2e</code></pre>
|
|
<pre><code class="lang-yaml"># CORRECT: Parallel jobs
|
|
jobs:
|
|
test-unit:
|
|
steps: [...]
|
|
test-integration:
|
|
steps: [...]
|
|
test-e2e:
|
|
needs: [test-unit, test-integration]
|
|
steps: [...]</code></pre>
|
|
<h3>❌ No Test Isolation</h3>
|
|
<pre><code class="lang-yaml"># WRONG: Tests share state
|
|
- run: npm test
|
|
env:
|
|
DB_NAME: shared_test_db</code></pre>
|
|
<pre><code class="lang-yaml"># CORRECT: Isolated databases per runner
|
|
- run: npm test
|
|
env:
|
|
DB_NAME: test_db_${{ github.run_id }}_${{ matrix.shard }}</code></pre>
|
|
<h3>❌ Ignoring Flaky Tests</h3>
|
|
<pre><code class="lang-yaml"># WRONG: Hide failures
|
|
- run: npm test || true</code></pre>
|
|
<pre><code class="lang-yaml"># CORRECT: Track and fix flaky tests
|
|
- run: npm test -- --retries=2
|
|
- run: npm run test:flaky
|
|
continue-on-error: true
|
|
# File issue for flaky tests</code></pre>
|
|
<h3>❌ No Coverage Enforcement</h3>
|
|
<pre><code class="lang-yaml"># WRONG: No coverage check
|
|
- run: npm test</code></pre>
|
|
<pre><code class="lang-yaml"># CORRECT: Enforce coverage thresholds
|
|
- run: npm test -- --coverage
|
|
- run: |
|
|
if [ $(cat coverage/coverage-summary.json | jq '.total.lines.pct') -lt 80 ]; then
|
|
echo "Coverage below 80%"
|
|
exit 1
|
|
fi</code></pre>
|
|
<h3>❌ Slow E2E Tests on Every Commit</h3>
|
|
<pre><code class="lang-yaml"># WRONG: Full E2E suite on every push
|
|
on: [push]
|
|
jobs:
|
|
e2e:
|
|
steps: [run all 500 E2E tests]</code></pre>
|
|
<pre><code class="lang-yaml"># CORRECT: Smoke tests on push, full suite on main
|
|
jobs:
|
|
e2e-smoke:
|
|
if: github.ref != 'refs/heads/main'
|
|
steps: [run critical E2E tests]
|
|
|
|
e2e-full:
|
|
if: github.ref == 'refs/heads/main'
|
|
steps: [run all E2E tests]</code></pre>
|
|
<h3>❌ Not Retaining Test Artifacts</h3>
|
|
<pre><code class="lang-yaml"># WRONG: No artifacts on failure
|
|
- run: npx playwright test</code></pre>
|
|
<pre><code class="lang-yaml"># CORRECT: Upload screenshots/videos on failure
|
|
- run: npx playwright test
|
|
- uses: actions/upload-artifact@v4
|
|
if: failure()
|
|
with:
|
|
name: test-artifacts
|
|
path: |
|
|
test-results/
|
|
screenshots/
|
|
videos/</code></pre>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>github-actions-workflows.md</code> - Workflow configuration and syntax</li><li><code>ci-optimization.md</code> - Pipeline speed and efficiency</li><li><code>cd-deployment-patterns.md</code> - Post-test deployment strategies</li><li><code>ci-security.md</code> - Security testing and scanning</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-18</p>
|
|
<p><strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/cicd-ci-testing-strategy.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |