Centralize test timings to fix shard divergence in CI (#90672)

## Summary For context we discovered jobs were not reliably always running in https://github.com/vercel/next.js/pull/90668 as addapters + turbopack group ran more tests than just turbopack group. - Each sharded test sub-job independently fetched test timings from KV via a turbo task. Because turbo cache keys can vary across job groups and KV can return slightly different data at different times, the sharding algorithm produced different test-to-shard assignments across groups — causing some tests to be missing entirely and others to run in multiple shards. - Adds a `--require-timings` flag to `run-tests.js` that fails loudly if `test-timings.json` can't be loaded from disk, preventing silent fallback to KV or round-robin. - Adds a `testTimingsArtifact` input to `build_reusable.yml` that downloads a pre-built timings artifact instead of fetching via turbo. When unset (default), existing behavior is preserved. - In both `test_e2e_deploy_release.yml` and `build_and_test.yml`, timings are now fetched once in a setup job, uploaded as a GitHub artifact, and downloaded by all sub-jobs — guaranteeing identical shard assignments across all job groups. ## Test plan - [ ] Verify `build_and_test.yml` jobs pick up the shared `test-timings` artifact - [ ] Verify `test_e2e_deploy_release.yml` deploy jobs pick up the shared `test-timings` artifact - [ ] Verify workflows that don't pass `testTimingsArtifact` (e.g. `integration_tests_reusable.yml`, `pull_request_stats.yml`) still use the turbo fallback path unchanged
2026-09-24 17:00:51 +00:00 · 2026-02-28 00:15:43 +00:00
parent 39e8fb2a48
commit 683180bfb3
4 changed files with 246 additions and 20 deletions
@@ -94,6 +94,53 @@ jobs:
      stepName: 'build-next'
    secrets: inherit

+  fetch-test-timings:
+    name: fetch test timings
+    runs-on: ubuntu-latest
+    needs: ['changes']
+    if: ${{ needs.changes.outputs.docs-only == 'false' }}
+    steps:
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_LTS_VERSION }}
+          check-latest: true
+
+      - name: Setup pnpm
+        run: |
+          npm i -g [email protected]
+          corepack enable
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 25
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Fetch test timings
+        run: node run-tests.js --timings --write-timings -g 1/1
+        continue-on-error: true
+        env:
+          KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }}
+          KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }}
+
+      - name: Ensure test timings file exists
+        run: |
+          if [ ! -f test-timings.json ]; then
+            echo "No timings fetched, creating empty timings file"
+            echo '{}' > test-timings.json
+          fi
+
+      - name: Upload test timings
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-timings
+          path: test-timings.json
+          retention-days: 1
+          if-no-files-found: error
+
  lint:
    name: lint
    needs: ['build-next']
@@ -236,7 +283,14 @@ jobs:

  test-turbopack-dev:
    name: test turbopack dev
-    needs: ['optimize-ci', 'changes', 'build-next', 'build-native']
+    needs:
+      [
+        'optimize-ci',
+        'changes',
+        'build-next',
+        'build-native',
+        'fetch-test-timings',
+      ]
    if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }}

    strategy:
@@ -261,13 +315,22 @@ jobs:
        node run-tests.js \
          --test-pattern '^(test\/(development|e2e))/.*\.test\.(js|jsx|ts|tsx)$' \
          --timings \
+          --require-timings \
          -g ${{ matrix.group }}
+      testTimingsArtifact: 'test-timings'
      stepName: 'test-turbopack-dev-react-${{ matrix.react }}-${{ matrix.group }}'
    secrets: inherit

  test-turbopack-integration:
    name: test turbopack integration
-    needs: ['optimize-ci', 'changes', 'build-native', 'build-next']
+    needs:
+      [
+        'optimize-ci',
+        'changes',
+        'build-native',
+        'build-next',
+        'fetch-test-timings',
+      ]
    if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }}

    strategy:
@@ -304,14 +367,23 @@ jobs:

        node run-tests.js \
          --timings \
+          --require-timings \
          -g ${{ matrix.group }} \
          --type integration
+      testTimingsArtifact: 'test-timings'
      stepName: 'test-turbopack-integration-react-${{ matrix.react }}-${{ matrix.group }}'
    secrets: inherit

  test-turbopack-production:
    name: test turbopack production
-    needs: ['optimize-ci', 'changes', 'build-next', 'build-native']
+    needs:
+      [
+        'optimize-ci',
+        'changes',
+        'build-next',
+        'build-native',
+        'fetch-test-timings',
+      ]
    if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }}

    strategy:
@@ -334,13 +406,21 @@ jobs:
        export __NEXT_EXPERIMENTAL_STRICT_ROUTE_TYPES=true
        export RUST_BACKTRACE=1

-        node run-tests.js --timings -g ${{ matrix.group }} --type production
+        node run-tests.js --timings --require-timings -g ${{ matrix.group }} --type production
+      testTimingsArtifact: 'test-timings'
      stepName: 'test-turbopack-production-react-${{ matrix.react }}-${{ matrix.group }}'
    secrets: inherit

  test-rspack-dev:
    name: test rspack dev
-    needs: ['optimize-ci', 'changes', 'build-next', 'build-native']
+    needs:
+      [
+        'optimize-ci',
+        'changes',
+        'build-next',
+        'build-native',
+        'fetch-test-timings',
+      ]
    if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' && needs.changes.outputs.rspack == 'true' }}
    strategy:
      fail-fast: false
@@ -370,13 +450,22 @@ jobs:
        node run-tests.js \
          --test-pattern '^(test\/(development|e2e))/.*\.test\.(js|jsx|ts|tsx)$' \
          --timings \
+          --require-timings \
          -g ${{ matrix.group }}
+      testTimingsArtifact: 'test-timings'
      stepName: 'test-rspack-dev-react-${{ matrix.react }}-${{ matrix.group }}'
    secrets: inherit

  test-rspack-integration:
    name: test rspack development integration
-    needs: ['optimize-ci', 'changes', 'build-next', 'build-native']
+    needs:
+      [
+        'optimize-ci',
+        'changes',
+        'build-next',
+        'build-native',
+        'fetch-test-timings',
+      ]
    if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' && needs.changes.outputs.rspack == 'true' }}
    strategy:
      fail-fast: false
@@ -406,14 +495,23 @@ jobs:

        node run-tests.js \
          --timings \
+          --require-timings \
          -g ${{ matrix.group }} \
          --type integration
+      testTimingsArtifact: 'test-timings'
      stepName: 'test-rspack-integration-react-${{ matrix.react }}-${{ matrix.group }}'
    secrets: inherit

  test-rspack-production:
    name: test rspack production
-    needs: ['optimize-ci', 'changes', 'build-next', 'build-native']
+    needs:
+      [
+        'optimize-ci',
+        'changes',
+        'build-next',
+        'build-native',
+        'fetch-test-timings',
+      ]
    if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' && needs.changes.outputs.rspack == 'true' }}
    strategy:
      fail-fast: false
@@ -441,13 +539,21 @@ jobs:
        # tests, so it's applicable to rspack
        export TURBOPACK_BUILD=1

-        node run-tests.js --timings -g ${{ matrix.group }} --type production
+        node run-tests.js --timings --require-timings -g ${{ matrix.group }} --type production
+      testTimingsArtifact: 'test-timings'
      stepName: 'test-rspack-production-react-${{ matrix.react }}-${{ matrix.group }}'
    secrets: inherit

  test-rspack-production-integration:
    name: test rspack production integration
-    needs: ['optimize-ci', 'changes', 'build-next', 'build-native']
+    needs:
+      [
+        'optimize-ci',
+        'changes',
+        'build-next',
+        'build-native',
+        'fetch-test-timings',
+      ]
    if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' && needs.changes.outputs.rspack == 'true' }}
    strategy:
      fail-fast: false
@@ -477,8 +583,10 @@ jobs:

        node run-tests.js \
          --timings \
+          --require-timings \
          -g ${{ matrix.group }} \
          --type integration
+      testTimingsArtifact: 'test-timings'
      stepName: 'test-rspack-production-integration-react-${{ matrix.react }}-${{ matrix.group }}'
    secrets: inherit

@@ -719,7 +827,14 @@ jobs:

  test-dev: # TODO: rename to include webpack
    name: test dev
-    needs: ['optimize-ci', 'changes', 'build-native', 'build-next']
+    needs:
+      [
+        'optimize-ci',
+        'changes',
+        'build-native',
+        'build-next',
+        'fetch-test-timings',
+      ]
    if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }}

    strategy:
@@ -741,8 +856,10 @@ jobs:

        node run-tests.js \
          --timings \
+          --require-timings \
          -g ${{ matrix.group }} \
          --type development
+      testTimingsArtifact: 'test-timings'
      stepName: 'test-dev-react-${{ matrix.react }}-${{ matrix.group }}'
    secrets: inherit

@@ -838,7 +955,14 @@ jobs:

  test-prod:
    name: test prod
-    needs: ['optimize-ci', 'changes', 'build-native', 'build-next']
+    needs:
+      [
+        'optimize-ci',
+        'changes',
+        'build-native',
+        'build-next',
+        'fetch-test-timings',
+      ]
    if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }}

    strategy:
@@ -858,13 +982,21 @@ jobs:
        export NEXT_TEST_REACT_VERSION="${{ matrix.react }}"
        export __NEXT_EXPERIMENTAL_STRICT_ROUTE_TYPES=true

-        node run-tests.js --timings -g ${{ matrix.group }} --type production
+        node run-tests.js --timings --require-timings -g ${{ matrix.group }} --type production
+      testTimingsArtifact: 'test-timings'
      stepName: 'test-prod-react-${{ matrix.react }}-${{ matrix.group }}'
    secrets: inherit

  test-integration:
    name: test integration
-    needs: ['optimize-ci', 'changes', 'build-native', 'build-next']
+    needs:
+      [
+        'optimize-ci',
+        'changes',
+        'build-native',
+        'build-next',
+        'fetch-test-timings',
+      ]
    if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }}

    strategy:
@@ -900,8 +1032,10 @@ jobs:

        node run-tests.js \
          --timings \
+          --require-timings \
          -g ${{ matrix.group }} \
          --type integration
+      testTimingsArtifact: 'test-timings'
      stepName: 'test-integration-${{ matrix.group }}-react-${{ matrix.react }}'
    secrets: inherit

@@ -958,7 +1092,14 @@ jobs:

  test-cache-components-dev:
    name: test cache components dev
-    needs: ['optimize-ci', 'changes', 'build-native', 'build-next']
+    needs:
+      [
+        'optimize-ci',
+        'changes',
+        'build-native',
+        'build-next',
+        'fetch-test-timings',
+      ]
    if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }}

    strategy:
@@ -976,14 +1117,23 @@ jobs:

        node run-tests.js \
          --timings \
+          --require-timings \
          -g ${{ matrix.group }} \
          --type development
+      testTimingsArtifact: 'test-timings'
      stepName: 'test-cache-components-dev-${{ matrix.group }}'
    secrets: inherit

  test-cache-components-prod:
    name: test cache components prod
-    needs: ['optimize-ci', 'changes', 'build-native', 'build-next']
+    needs:
+      [
+        'optimize-ci',
+        'changes',
+        'build-native',
+        'build-next',
+        'fetch-test-timings',
+      ]
    if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }}

    strategy:
@@ -1001,8 +1151,10 @@ jobs:

        node run-tests.js \
          --timings \
+          --require-timings \
          -g ${{ matrix.group }} \
          --type production
+      testTimingsArtifact: 'test-timings'
      stepName: 'test-cache-components-prod-${{ matrix.group }}'
    secrets: inherit

@@ -80,6 +80,12 @@ on:
        type: string
        default: ''

+      testTimingsArtifact:
+        description: 'Name of an uploaded artifact containing test-timings.json. When set, download it instead of fetching via turbo.'
+        required: false
+        type: string
+        default: ''
+
      browser:
        description: 'Browser to use for tests'
        required: false
@@ -307,7 +313,24 @@ jobs:
      - run: pnpm playwright install --with-deps ${{ inputs.browser }}
        if: ${{ inputs.skipInstallBuild != 'yes' }}

-      - run: pnpm dlx turbo@${TURBO_VERSION} run get-test-timings -- --build ${{ github.sha }}
+      - name: Download pre-built test timings
+        if: ${{ inputs.testTimingsArtifact != '' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.testTimingsArtifact }}
+
+      - name: Verify test timings
+        if: ${{ inputs.testTimingsArtifact != '' }}
+        run: |
+          if [ ! -f test-timings.json ]; then
+            echo "::error::test-timings.json not found"
+            exit 1
+          fi
+          echo "Test timings loaded ($(wc -c < test-timings.json) bytes)"
+
+      - name: Fetch test timings via turbo
+        if: ${{ inputs.testTimingsArtifact == '' }}
+        run: pnpm dlx turbo@${TURBO_VERSION} run get-test-timings -- --build ${{ github.sha }}

      - run: ${{ inputs.afterBuild }}
        # defaults.run.shell sets a stronger options (`-leo pipefail`)
@@ -76,6 +76,31 @@ jobs:
        name: Extract `next` version
        run: echo 'value=${{ fromJson(steps.nextPackageInfo.outputs.value).version }}' >> "$GITHUB_OUTPUT"

+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Fetch test timings
+        run: node run-tests.js --timings --write-timings -g 1/1
+        continue-on-error: true
+        env:
+          KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }}
+          KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }}
+
+      - name: Ensure test timings file exists
+        run: |
+          if [ ! -f test-timings.json ]; then
+            echo "No timings fetched, creating empty timings file"
+            echo '{}' > test-timings.json
+          fi
+
+      - name: Upload test timings
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-timings
+          path: test-timings.json
+          retention-days: 1
+          if-no-files-found: error
+
  test-deploy-webpack:
    name: Run Deploy Tests (Webpack)
    needs: setup
@@ -97,7 +122,8 @@ jobs:
        VERCEL_CLI_VERSION="${{ github.event.inputs.vercelCliVersion || 'vercel@latest' }}" \
        NEXT_TEST_DEPLOY_SCRIPT_PATH="${{ github.event.inputs.deployScriptPath || '' }}" \
        NEXT_TEST_DEPLOY_LOGS_SCRIPT_PATH="${{ github.event.inputs.deployLogsScriptPath || '' }}" \
-        node run-tests.js --timings -g ${{ matrix.group }} -c 2 --type e2e
+        node run-tests.js --timings --require-timings -g ${{ matrix.group }} -c 2 --type e2e
+      testTimingsArtifact: 'test-timings'
      skipNativeBuild: 'yes'
      skipNativeInstall: 'no'
      stepName: 'test-deploy-webpack-${{ matrix.group }}'
@@ -140,7 +166,8 @@ jobs:
        VERCEL_CLI_VERSION="${{ github.event.inputs.vercelCliVersion || 'vercel@latest' }}" \
        NEXT_TEST_DEPLOY_SCRIPT_PATH="${{ github.event.inputs.deployScriptPath || '' }}" \
        NEXT_TEST_DEPLOY_LOGS_SCRIPT_PATH="${{ github.event.inputs.deployLogsScriptPath || '' }}" \
-        node run-tests.js --timings -g ${{ matrix.group }} -c 2 --type e2e
+        node run-tests.js --timings --require-timings -g ${{ matrix.group }} -c 2 --type e2e
+      testTimingsArtifact: 'test-timings'
      skipNativeBuild: 'yes'
      skipNativeInstall: 'no'
      stepName: 'test-deploy-turbopack-${{ matrix.group }}'
@@ -182,7 +209,8 @@ jobs:
        NEXT_EXTERNAL_TESTS_FILTERS="test/deploy-tests-manifest.json" \
        NEXT_TEST_VERSION="${{ github.event.inputs.nextVersion || needs.setup.outputs.next-version || 'canary' }}" \
        VERCEL_CLI_VERSION="${{ github.event.inputs.vercelCliVersion || 'vercel@latest' }}" \
-        node run-tests.js --timings -g ${{ matrix.group }} -c 2 --type e2e
+        node run-tests.js --timings --require-timings -g ${{ matrix.group }} -c 2 --type e2e
+      testTimingsArtifact: 'test-timings'
      skipNativeBuild: 'yes'
      skipNativeInstall: 'no'
      stepName: 'test-deploy-deploy-${{ matrix.group }}'
@@ -32,6 +32,11 @@ let argv = require('yargs/yargs')(process.argv.slice(2))
  .boolean('dry')
  .boolean('print-tests')
  .describe('print-tests', 'Prints the test files that will be run')
+  .boolean('require-timings')
+  .describe(
+    'require-timings',
+    'Require test-timings.json from disk; fail if missing instead of falling back to KV or round-robin'
+  )
  .boolean('local')
  .alias('c', 'concurrency').argv

@@ -233,6 +238,7 @@ async function main() {
    testPattern: argv.testPattern ?? false,
    type: argv.type ?? false,
    retries: argv.retries ?? DEFAULT_NUM_RETRIES,
+    requireTimings: argv.requireTimings ?? false,
    dry: argv.dry ?? false,
    local: argv.local ?? false,
    printTests: argv.printTests ?? false,
@@ -322,13 +328,21 @@ async function main() {
      prevTimings = JSON.parse(await fsp.readFile(timingsFile, 'utf8'))
      console.log('Loaded test timings from disk successfully')
    } catch (_) {
+      if (options.requireTimings) {
+        console.error(
+          'ERROR: --require-timings is set but test-timings.json could not be loaded from disk:',
+          _
+        )
+        await cleanUpAndExit(1)
+        return
+      }
      console.error(
        'Failed to load test timings from disk. Proceeding to fetch from KV store. Original error: ',
        _
      )
    }

-    if (!prevTimings) {
+    if (!prevTimings && !options.requireTimings) {
      try {
        prevTimings = await getTestTimings()
        if (prevTimings) {
@@ -412,6 +426,15 @@ async function main() {
        Math.round(groupTimes[curGroupIdx]) + 's'
      )
    } else {
+      if (options.requireTimings) {
+        console.error(
+          'ERROR: --require-timings is set but no timing data is available for sharding. ' +
+            'Ensure test-timings.json is provided.'
+        )
+        await cleanUpAndExit(1)
+        return
+      }
+
      // assign every nth test "round-robin" to the group, so that similar slow
      // tests tend not to get clustered together
      tests = tests.filter((_value, idx) => idx % groupTotal === groupPos - 1)