feat(ci): add fix-flakes workflow to auto-fix flaky tests (#41692)

.claude/skills/playwright-test-results/SKILL.md

@@ -25,12 +25,20 @@ The snapshot may be missing the newest runs. To top it up locally, run `update`:
 GITHUB_TOKEN=$(gh auth token) node utils/test-results-db/cli.ts update --lookback-days 3
 ```
 
-Query it with the `duckdb` CLI (or any DuckDB client):
+Query it through the bundled `@duckdb/node-api` binding — no separate DuckDB
+install needed, it ships in `node_modules` after `npm ci`:
 
 ```bash
-duckdb utils/test-results-db/test-results.duckdb "SELECT count(*) FROM test_results"
+node --input-type=module -e '
+import { DuckDBInstance } from "@duckdb/node-api";
+const conn = await (await DuckDBInstance.create("utils/test-results-db/test-results.duckdb")).connect();
+console.table((await conn.runAndReadAll(process.argv[1])).getRowObjectsJson());
+' "SELECT count(*) FROM test_results"
 ```
 
+Integer columns come back as strings (JSON-safe), so do ranking and filtering
+in SQL, not in JS.
+
 ## Schema
 
 Single table `test_results`, one row per test result (**one row per retry**).

.claude/skills/playwright-triage/SKILL.md

@@ -94,7 +94,7 @@ cannot-reproduce / not-a-bug; for a feature request or upstream/env issue: a sho
 the condensed repro and be exhaustive about **what you ran** — the full matrix of browsers,
 versions, and variations you tried, not just the one that worked — so the reader can trust the
 verdict and skip re-checking. Call out any browser-specific divergence. Write it in the
-[playwright-bot-voice](../playwright-bot-voice/SKILL.md) — maintainer voice, not AI-speak.
+Playwright bot voice (`.github/workflows/bot-voice.md`, relative to the repo root) — maintainer voice, not AI-speak.
 
 ## Watch out
 

.github/workflows/bot-voice.md

@@ -1,9 +1,3 @@
----
-name: playwright-bot-voice
-description: How the Playwright bot writes anything public — issue comments, PR descriptions and replies, release notes. Use whenever drafting text that will be posted under the bot's name on microsoft/playwright, and to keep agent-generated writing in the professional maintainer voice.
-user_invocable: true
----
-
 # Playwright Bot Voice
 
 You're posting in public on `microsoft/playwright` **as the Playwright bot**. You don't need

.github/workflows/fix-flakes-prompt.md

@@ -0,0 +1,111 @@
+# Playwright: Fix a Flaky or Red Test
+
+Turn CI test-results data into **one** concrete fix: pick a high-impact flaky-or-red test that is reproducible on your OS,
+confirm nobody's on it, fix the root cause *or* scope a skip, pick a reviewer, and hand off a
+single commit that becomes the PR. Fully autonomous — no approval stops.
+
+## 1. Pick one target
+
+Query the DB following the patterns in `.claude/skills/playwright-test-results/SKILL.md`.
+Two families:
+
+- **Cross-run flake** — the *final* verdict (after retries) flips between runs.
+- **Consistent red** — `expected_status = 'passed'` yet failing in ~every run.
+
+Rank by **impact**: fail %, run count (floor `runs >= 10` so it isn't a one-off), and how many
+bots/PRs it disrupts — **not** by "has a tidy error message".
+Pick a candidate whose failing `bot_name` OS matches yours so you can reproduce it.
+Keep the ranked list — you fall back down it in step 2. Say why you picked the top one.
+
+## 2. Check nobody's already on it
+
+Search open PRs/issues for the test title words and file path before touching anything.
+Also check any issue linked in the
+test's `annotation`.
+If an open PR already touches the test, **it's taken — drop it and go down your step-1 ranking**, re-checking each.
+Only stop and report if the whole shortlist is covered.
+
+## 3. Reproduce on this OS
+
+Read the test and its `error_message`.
+
+Build first (`npm run build`; watch is **not** running — if you touch generated-code files see
+`CLAUDE.md`). Then reproduce, scoped to the failing target.
+
+The DB `project_name` **is** the Playwright `--project`; `bot_name` encodes the OS. **Always
+pass a `<file>:<line>` filter** — a bare run launches the whole suite. The browser test scripts
+are project-locked, so use the one matching the failing bot:
+
+| Failing target | Command |
+|---|---|
+| chromium | `npm run ctest -- <file>:<line>` |
+| firefox | `npm run ftest -- <file>:<line>` |
+| webkit | `npm run wtest -- <file>:<line>` |
+| `tests/playwright-test/**` | `npm run ttest -- <file>:<line>` |
+| `tests/mcp/**` | `npm run test-mcp -- --project=<name> <file>:<line>` |
+
+Other suites (electron `etest`, etc.): see `package.json` scripts.
+Add `--repeat-each=N` to force a flake's flip.
+
+**You can only reproduce what this OS reaches.** A failure you can't reach here is itself
+evidence it's environment-specific — a skip candidate (step 6), not a blind-fix.
+Keep any OS handling keyed on the *current* OS, never hardcoded.
+Broader OS coverage comes from different agent runs on other OSes, not from you.
+
+## 4. Fix — root cause or scoped skip
+
+- **Tractable → fix the source.** Usually a test-side race: a missing `await`, waiting on the
+  wrong signal, an under-specified locator, leaked state. Fix the test (or the product bug).
+- **Engine/OS-specific, unreachable here, or genuinely hard → scope a skip**, narrowed to the
+  exact failing condition — never a whole file or browser:
+
+  ```ts
+  it.fixme(browserName === 'webkit' && isLinux, 'https://github.com/microsoft/playwright/issues/NNNNN');
+  ```
+
+  Link an issue if one exists or explain why the test is skipped. **Default to `fixme`** — it parks
+  the debt and stays greppable. Use `skip` only if reproduction shows the test is genuinely
+  mis-scoped for that config (`skip` claims "this failure is expected and correct," which a
+  flake-fighter rarely is). Never `skip` just to turn a red green.
+
+**Verify locally — your PR gets no CI (step 6), so this is the only proof:** flake → re-run with
+`--repeat-each` and confirm it's stable; skip → confirm it's skipped on the target config and
+**still runs elsewhere**. Then run `npm run flint`. Record exactly what you ran — it goes in the
+PR body.
+
+## 5. Pick a reviewer
+
+No CODEOWNERS. Derive one from the touched file(s) — recent authors and frequent committers:
+
+```bash
+git log --format='%an %ae' -n 20 -- <file>
+git log --format='%an' -n 200 -- <file> | sort | uniq -c | sort -rn
+```
+
+Also skim recent merged PRs on those paths for who reviewed them. Pick someone with real recency
++ ownership. Record it as a git trailer so it survives the handoff:
+
+```
+Suggested-reviewer: dgozman
+```
+
+## 6. Commit and hand off (you never open the PR in CI)
+
+You **never push or open a PR** in CI or locally. The harness does that for you, after you commit.
+
+- **Exactly one commit** on the current branch (`CLAUDE.md` conventions; no
+  co-author / "generated with" trailers; never amend).
+- **The commit message _is_ the PR** — the harness runs `gh pr create --fill`, mapping
+  subject→title and body→description. Write both in the Playwright bot voice
+  (`.github/workflows/bot-voice.md`) — verdict first, short, no slop. The
+  body must carry: the **DB evidence** (fail %, runs, bots) + any related issue; **what you
+  verified locally** and on which OS.
+- **Nothing actionable?** Make no commit — the harness then skips the PR.
+
+Report what you committed, the reviewer, and why.
+
+## Guardrails
+
+- **One target, one commit.** Don't batch.
+- **Scoped skips only** — narrow to the failing condition, never a whole file or browser matrix.
+- **No unverified PRs** — if you couldn't run it locally, prefer a documented fixme.

.github/workflows/fix-flakes.yml

@@ -0,0 +1,289 @@
+name: "Fix flaky tests"
+
+on:
+  workflow_dispatch:
+    inputs:
+      hint:
+        description: "Optional focus hint passed to the triage and fix agents (e.g. 'webkit flakes')"
+        type: string
+  schedule:
+    - cron: "17 6 * * 1-5"
+
+permissions: {}
+
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    if: github.repository == 'microsoft/playwright'
+    outputs:
+      runner: ${{ steps.validate.outputs.runner }}
+      has_target: ${{ steps.validate.outputs.has_target }}
+    permissions:
+      actions: read
+      copilot-requests: write
+    concurrency:
+      group: fix-flakes-triage
+      cancel-in-progress: false
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Download test-results DB
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: node utils/test-results-db/cli.ts download
+
+      - name: Install Copilot CLI
+        run: npm install -g @github/copilot
+
+      - name: Pick an eligible OS with Copilot CLI
+        shell: bash
+        env:
+          COPILOT_GITHUB_TOKEN: ${{ github.token }}
+          HINT: ${{ inputs.hint }}
+        run: |
+          mkdir -p output
+          cat > output/triage-prompt.md <<EOF
+          You are triage for the fix-flakes workflow. Pick the ONE operating system where booting
+          an expensive runner to fix a flaky or red test is most worthwhile right now, and hand
+          off only that runner label. Do NOT fix anything.
+
+          Optimise for the OS with the highest-impact actionable flakiness/reds that isn't
+          already being worked on (its top candidates aren't covered by an open fix PR — check
+          with the GitHub MCP tools). All else equal, favour variety so Linux, Windows and macOS
+          each get attention over time.
+
+          The DB is already downloaded at utils/test-results-db/test-results.duckdb — query it
+          via the playwright-test-results skill. \`bot_name\` encodes the OS.
+
+          Write your answer to output/runner.txt, and nothing else there: exactly one of these
+          six runner labels, or the single word \`none\` if no OS is worth a run.
+            ubuntu-22.04   ubuntu-22.04-arm   ubuntu-24.04
+            macos-14-xlarge   macos-15-xlarge   windows-latest
+          Map a candidate's \`bot_name\` to the closest of those (e.g. \`*-macos-15-*\` ->
+          macos-15-xlarge). Explain your reasoning briefly in your final message.
+
+          ${HINT:+Focus hint from the operator: $HINT}
+          EOF
+          copilot \
+            --allow-all-tools \
+            --allow-all-paths \
+            --no-ask-user \
+            --enable-all-github-mcp-tools \
+            --share=output/copilot-session.md \
+            --model claude-opus-4.8 \
+            --max-ai-credits 200 \
+            -p "$(cat output/triage-prompt.md)"
+
+      - name: Validate chosen runner
+        id: validate
+        shell: bash
+        run: |
+          runner="$(tr -d '[:space:]' < output/runner.txt 2>/dev/null || true)"
+          if [ -z "$runner" ] || [ "$runner" = "none" ]; then
+            echo "No eligible platform this run — skipping the fix job."
+            echo "has_target=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          case "$runner" in
+            ubuntu-22.04|ubuntu-22.04-arm|ubuntu-24.04|macos-14-xlarge|macos-15-xlarge|windows-latest) ;;
+            *) echo "::error::Triage chose a non-allowlisted runner: '$runner'." >&2; exit 1 ;;
+          esac
+          echo "Triage chose runner: $runner"
+          echo "runner=$runner" >> "$GITHUB_OUTPUT"
+          echo "has_target=true" >> "$GITHUB_OUTPUT"
+
+      - name: Add triage transcript to job summary
+        if: always()
+        shell: bash
+        run: |
+          {
+            echo "## Fix-flakes triage transcript"
+            echo ''
+            cat "output/copilot-session.md" 2>/dev/null || echo "(no transcript)"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  fix:
+    needs: triage
+    if: needs.triage.outputs.has_target == 'true'
+    runs-on: ${{ needs.triage.outputs.runner }}
+    timeout-minutes: 60
+    outputs:
+      has_fix: ${{ steps.export.outputs.has_fix }}
+    permissions:
+      actions: read
+      copilot-requests: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.sha }}
+          fetch-depth: 0
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build
+        run: npm run build
+
+      - name: Install browsers
+        shell: bash
+        run: |
+          if [ "$RUNNER_OS" = "Linux" ]; then
+            npx playwright install --with-deps
+          else
+            npx playwright install
+          fi
+
+      - name: Download test-results DB
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: node utils/test-results-db/cli.ts download
+
+      - name: Install Copilot CLI
+        run: npm install -g @github/copilot
+
+      - name: Configure git identity
+        shell: bash
+        run: |
+          git config user.name "Copilot"
+          git config user.email "[email protected]"
+
+      - name: Record base commit
+        id: base
+        shell: bash
+        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      - name: Fix a flaky/red test with Copilot CLI
+        shell: bash
+        env:
+          COPILOT_GITHUB_TOKEN: ${{ github.token }}
+          RUNNER: ${{ needs.triage.outputs.runner }}
+          HINT: ${{ inputs.hint }}
+          WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          mkdir -p output
+          cat > output/fix-prompt.md <<EOF
+          Read .github/workflows/fix-flakes-prompt.md and follow it.
+
+          You are running on $RUNNER.
+          Include the link to this run in your commit message: $WORKFLOW_URL.
+
+          ${HINT:+Operator focus hint: $HINT}
+          EOF
+          copilot \
+            --allow-all-tools \
+            --allow-all-paths \
+            --no-ask-user \
+            --enable-all-github-mcp-tools \
+            --share=output/copilot-session.md \
+            --model claude-opus-4.8 \
+            --max-ai-credits 1000 \
+            -p "$(cat output/fix-prompt.md)"
+
+      - name: Export the fix commit
+        id: export
+        shell: bash
+        run: |
+          base="${{ steps.base.outputs.sha }}"
+          count=$(git rev-list --count "$base"..HEAD)
+          if [ "$count" -eq 0 ]; then
+            echo "Agent produced no commit — nothing to open."
+            echo "has_fix=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          if [ "$count" -ne 1 ]; then
+            echo "::error::Expected exactly one commit, got $count. Refusing to hand off." >&2
+            exit 1
+          fi
+          mkdir -p handoff
+          git format-patch -1 HEAD --stdout > handoff/fix.patch
+          echo "has_fix=true" >> "$GITHUB_OUTPUT"
+
+      - name: Upload handoff (fix patch)
+        if: steps.export.outputs.has_fix == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: fix-flakes-handoff-${{ needs.triage.outputs.runner }}
+          path: handoff/
+          if-no-files-found: error
+
+      - name: Add session transcript to job summary
+        if: always()
+        shell: bash
+        run: |
+          {
+            echo "## Fix-flakes session transcript (${{ needs.triage.outputs.runner }})"
+            echo ''
+            cat "output/copilot-session.md" 2>/dev/null || echo "(no transcript)"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload session transcript
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: fix-flakes-session-${{ needs.triage.outputs.runner }}
+          path: output/copilot-session.md
+          if-no-files-found: warn
+
+  open_pr:
+    needs: [triage, fix]
+    if: needs.fix.outputs.has_fix == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Checkout base commit
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.sha }}
+          fetch-depth: 0
+
+      - name: Download handoff
+        uses: actions/download-artifact@v4
+        with:
+          name: fix-flakes-handoff-${{ needs.triage.outputs.runner }}
+          path: handoff
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          app-id: ${{ vars.PLAYWRIGHT_APP_ID }}
+          private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }}
+
+      - name: Apply commit and open PR
+        env:
+          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
+          RUNNER: ${{ needs.triage.outputs.runner }}
+          REVIEW_OVERRIDE: skn0tt
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git fetch --no-tags origin main:refs/remotes/origin/main
+          branch="fix-flakes/${RUNNER}-${{ github.run_id }}"
+          git checkout -b "$branch"
+          git am handoff/fix.patch
+          git push origin "$branch"
+          suggested=$(git log -1 --format='%(trailers:key=Suggested-reviewer,valueonly)' | head -n1 | tr -d '[:space:]@')
+          reviewer="${REVIEW_OVERRIDE:-$suggested}"
+          args=(--repo "${{ github.repository }}" --base main --head "$branch" --fill)
+          if [ -n "$reviewer" ]; then
+            args+=(--reviewer "$reviewer")
+          else
+            echo "::warning::No reviewer (empty override and no Suggested-reviewer trailer)."
+          fi
+          gh pr create "${args[@]}"

utils/test-results-db/github.ts

@@ -76,6 +76,7 @@ export class GitHubClient {
     const { ingested, lookbackDays, stopAfterSeen } = options;
     const cutoff = Date.now() - lookbackDays * 24 * 60 * 60 * 1000;
     const out: Artifact[] = [];
+    const queued = new Set<string>();
     let seen = 0;
     for await (const artifact of this._paginateArtifacts('/actions/artifacts?per_page=100')) {
       const createdAt = artifact.created_at ? Date.parse(artifact.created_at) : 0;
@@ -84,12 +85,15 @@ export class GitHubClient {
       if (artifact.expired || !artifact.name.startsWith(prefix))
         continue;
       const id = String(artifact.id);
+      if (queued.has(id))
+        continue;
       if (ingested.has(id)) {
         if (++seen >= stopAfterSeen)
           return out;
         continue;
       }
       seen = 0;
+      queued.add(id);
       out.push({ id, name: artifact.name });
     }
     return out;