[Turbopack] write route bundle stats to .next/diagnostics/route-bundle-stats.json (#90949)

After a successful, Turbopack production build, this writes
`.next/diagnostics/route-bundle-stats.json`
containing First Load JS stats for every route. 

## Why?

We used to have route bundle size statistics print after every build. It
was not very high-signal and could be misleading, often using heuristics
to give a best-effort estimate of what sizes may be at runtime. **These
kinds of metrics for route bundle sizes can be misleading, as pages can
be slow to visual/interactive completeness even if they have small first
load bytes.**

However, a minimal gauge of how much JavaScript a route ships during
initial page load can be a useful signal so long as users have
additional context and awareness of their app structure. Let's provide
just this number for now for those that could benefit.

Additionally, this does so in the form of writing a json document to the
build directory, rather than writing a table to stdout, where users have
needed to write extra tools to parse.

## Format

An array of route entries sorted descending by
`firstLoadUncompressedJsBytes`:

```json
[
  {
    "route": "/blog/[slug]",
    "firstLoadUncompressedJsBytes": 124928,
    "firstLoadChunkPaths": [
      ".next/static/chunks/framework.js",
      ".next/static/chunks/main.js",
      ".next/static/chunks/pages/blog/[slug].js"
    ]
  }
]
```

`firstLoadUncompressedJsBytes`: sum of uncompressed sizes of all JS
chunks loaded on first navigation to the route (shared chunks included),
for both Pages Router and App Router
`firstLoadChunkPaths`: paths to those chunks relative to the project
root

## Test Plan
Added an integration test. Built with a create-next-app and verified the
json document is written and the numbers align with what Chrome reports
during initial load of a page.

eslint.config.mjs

@@ -307,6 +307,7 @@ export default defineConfig([
             'retry',
             'itCI',
             'itHeaded',
+            'itTurbopack',
             'itTurbopackDev',
             'itOnlyTurbopack',
           ],

packages/next/src/build/index.ts

@@ -122,6 +122,7 @@ import { isWriteable } from './is-writeable'
 import * as Log from './output/log'
 import createSpinner from './spinner'
 import { trace, flushAllTraces, setGlobal, type Span } from '../trace'
+import { writeRouteBundleStats } from './route-bundle-stats'
 import {
   detectConflictingPaths,
   printCustomRoutes,
@@ -4189,6 +4190,14 @@ export default async function build(
         })
       )
 
+      if (bundler === Bundler.Turbopack) {
+        await nextBuildSpan
+          .traceChild('write-route-bundle-stats')
+          .traceAsyncFn(() =>
+            writeRouteBundleStats(pageKeys, buildManifest, distDir, dir)
+          )
+      }
+
       await nextBuildSpan
         .traceChild('telemetry-flush')
         .traceAsyncFn(() => telemetry.flush())

packages/next/src/build/route-bundle-stats.ts

@@ -0,0 +1,221 @@
+import path from 'path'
+import { promises as fs, statSync } from 'fs'
+import {
+  APP_PATHS_MANIFEST,
+  CLIENT_REFERENCE_MANIFEST,
+  SERVER_DIRECTORY,
+} from '../shared/lib/constants'
+import type { BuildManifest } from '../server/get-page-files'
+import { filterAndSortList } from './utils'
+
+const ROUTE_BUNDLE_STATS_FILE = 'route-bundle-stats.json'
+
+type RouteBundleStat = {
+  route: string
+  firstLoadUncompressedJsBytes: number
+  firstLoadChunkPaths: string[]
+}
+
+function sumFileSizes(
+  distDir: string,
+  files: string[],
+  cache: Map<string, number>
+): number {
+  let total = 0
+  for (const relPath of files) {
+    const cached = cache.get(relPath)
+    if (cached !== undefined) {
+      total += cached
+      continue
+    }
+    try {
+      const size = statSync(path.join(distDir, relPath)).size
+      cache.set(relPath, size)
+      total += size
+    } catch {
+      // ignore missing files
+    }
+  }
+  return total
+}
+
+function toProjectRelativePaths(
+  dir: string,
+  distDir: string,
+  relPaths: string[]
+): string[] {
+  return relPaths.map((f) => path.relative(dir, path.join(distDir, f)))
+}
+
+function buildRouteToAppPathsMap(
+  appPathsManifest: Record<string, string>
+): Map<string, string[]> {
+  const { normalizeAppPath } =
+    require('../shared/lib/router/utils/app-paths') as typeof import('../shared/lib/router/utils/app-paths')
+  // Keys in appPathsManifest are app paths like /blog/[slug]/page;
+  // values are server bundle file paths. Normalize the key to get the route.
+  const routeToAppPaths = new Map<string, string[]>()
+  for (const appPath of Object.keys(appPathsManifest)) {
+    const route = normalizeAppPath(appPath)
+    const existing = routeToAppPaths.get(route)
+    if (existing) {
+      existing.push(appPath)
+    } else {
+      routeToAppPaths.set(route, [appPath])
+    }
+  }
+  return routeToAppPaths
+}
+
+// Reads the manfiest file and gets the entry JS files. The manifest file is
+// a JavaScript file that sets a global variable (__RSC_MANIFEST). We require()
+// it with a save/restore of the global
+function readEntryJSFiles(
+  distDir: string,
+  pagePath: string,
+  appRoute: string
+): Record<string, string[]> | undefined {
+  const manifestFile = path.join(
+    distDir,
+    SERVER_DIRECTORY,
+    'app',
+    `${pagePath}_${CLIENT_REFERENCE_MANIFEST}.js`
+  )
+  try {
+    const g = global as Record<string, unknown>
+    const prev = g.__RSC_MANIFEST
+    g.__RSC_MANIFEST = undefined
+    require(manifestFile)
+    const rscManifest = g.__RSC_MANIFEST as
+      | Record<string, { entryJSFiles?: Record<string, string[]> }>
+      | undefined
+    g.__RSC_MANIFEST = prev
+
+    // The key in __RSC_MANIFEST is the app path (e.g. /blog/[slug]/page)
+    const manifestEntry = rscManifest?.[pagePath] ?? rscManifest?.[appRoute]
+    return manifestEntry?.entryJSFiles
+  } catch {
+    return undefined
+  }
+}
+
+function collectPagesRouterStats(
+  pages: ReadonlyArray<string>,
+  buildManifest: BuildManifest,
+  distDir: string,
+  dir: string,
+  cache: Map<string, number>
+): RouteBundleStat[] {
+  const rows: RouteBundleStat[] = []
+  const sharedFiles = buildManifest.pages['/_app'] ?? []
+  for (const page of filterAndSortList(pages, 'pages', false)) {
+    if (page === '/_app' || page === '/_document' || page === '/_error')
+      // Don't report on layouts directly
+      continue
+
+    const allFiles = (buildManifest.pages[page] ?? []).filter((f) =>
+      f.endsWith('.js')
+    )
+    const sharedJs = sharedFiles.filter((f) => f.endsWith('.js'))
+    const chunks = [...new Set([...allFiles, ...sharedJs])]
+    const firstLoadUncompressedJsBytes = sumFileSizes(distDir, chunks, cache)
+    rows.push({
+      route: page,
+      firstLoadUncompressedJsBytes,
+      firstLoadChunkPaths: toProjectRelativePaths(dir, distDir, chunks),
+    })
+  }
+  return rows
+}
+
+async function collectAppRouterStats(
+  appRoutes: ReadonlyArray<string>,
+  buildManifest: BuildManifest,
+  distDir: string,
+  dir: string,
+  cache: Map<string, number>
+): Promise<RouteBundleStat[]> {
+  let appPathsManifest: Record<string, string> = {}
+  try {
+    const manifestPath = path.join(
+      distDir,
+      SERVER_DIRECTORY,
+      APP_PATHS_MANIFEST
+    )
+    appPathsManifest = JSON.parse(await fs.readFile(manifestPath, 'utf8'))
+  } catch {
+    // App paths manifest not available; skip app router sizes
+    return []
+  }
+
+  const routeToAppPaths = buildRouteToAppPathsMap(appPathsManifest)
+  const sharedFiles = buildManifest.rootMainFiles ?? []
+  const rows: RouteBundleStat[] = []
+
+  for (const appRoute of filterAndSortList(appRoutes, 'app', false)) {
+    const appPaths = routeToAppPaths.get(appRoute) ?? []
+    // Find the /page entry (most specific, has the most chunks)
+    const pagePath = appPaths.find((p) => p.endsWith('/page'))
+    if (!pagePath) continue
+
+    const entryJSFiles = readEntryJSFiles(distDir, pagePath, appRoute)
+    if (!entryJSFiles) continue
+
+    // Union JS files across all segments (page, layout, etc.) so that
+    // layout code's contribution is included in the First Load JS total.
+    const allFiles = [
+      ...new Set(
+        Object.values(entryJSFiles)
+          .flat()
+          .filter((f) => f.endsWith('.js'))
+      ),
+    ]
+    const sharedJs = sharedFiles.filter((f) => f.endsWith('.js'))
+    const chunks = [...new Set([...allFiles, ...sharedJs])]
+    const firstLoadUncompressedJsBytes = sumFileSizes(distDir, chunks, cache)
+    rows.push({
+      route: appRoute,
+      firstLoadUncompressedJsBytes,
+      firstLoadChunkPaths: toProjectRelativePaths(dir, distDir, chunks),
+    })
+  }
+  return rows
+}
+
+export async function writeRouteBundleStats(
+  lists: {
+    pages: ReadonlyArray<string>
+    app: ReadonlyArray<string> | undefined
+  },
+  buildManifest: BuildManifest,
+  distDir: string,
+  dir: string
+): Promise<void> {
+  const cache = new Map<string, number>()
+
+  const rows = [
+    ...(lists.pages.length > 0
+      ? collectPagesRouterStats(lists.pages, buildManifest, distDir, dir, cache)
+      : []),
+    ...(lists.app && lists.app.length > 0
+      ? await collectAppRouterStats(
+          lists.app,
+          buildManifest,
+          distDir,
+          dir,
+          cache
+        )
+      : []),
+  ]
+
+  rows.sort(
+    (a, b) => b.firstLoadUncompressedJsBytes - a.firstLoadUncompressedJsBytes
+  )
+
+  const diagnosticsDir = path.join(distDir, 'diagnostics')
+  await fs.mkdir(diagnosticsDir, { recursive: true })
+  await fs.writeFile(
+    path.join(diagnosticsDir, ROUTE_BUNDLE_STATS_FILE),
+    JSON.stringify(rows, null, 2)
+  )
+}

packages/next/src/build/utils.ts

@@ -162,7 +162,7 @@ export function isInstrumentationHookFilename(file?: string | null) {
   )
 }
 
-const filterAndSortList = (
+export const filterAndSortList = (
   list: ReadonlyArray<string>,
   routeType: ROUTER_TYPE,
   hasCustomApp: boolean

test/lib/e2e-utils/index.ts

@@ -164,6 +164,8 @@ if (!process.env.NEXT_TEST_WASM && process.env.NEXT_TEST_WASM_AFTER_JEST) {
 
 export const isRspack = !!process.env.NEXT_RSPACK
 const isNextTestWasm = !!process.env.NEXT_TEST_WASM
+export const itTurbopack =
+  !isNextTestWasm && shouldUseTurbopack() ? it : it.skip
 
 if (!testMode) {
   throw new Error(

test/production/route-bundle-stats/app/blog/client.tsx

@@ -0,0 +1,5 @@
+'use client'
+
+export function BlogClient() {
+  return <span>client</span>
+}

test/production/route-bundle-stats/app/blog/page.tsx

@@ -0,0 +1,9 @@
+import { BlogClient } from './client'
+
+export default function BlogPage() {
+  return (
+    <p>
+      blog <BlogClient />
+    </p>
+  )
+}

test/production/route-bundle-stats/app/layout.tsx

@@ -0,0 +1,7 @@
+export default function Layout({ children }: { children: React.ReactNode }) {
+  return (
+    <html>
+      <body>{children}</body>
+    </html>
+  )
+}

test/production/route-bundle-stats/pages/index.tsx

@@ -0,0 +1,3 @@
+export default function Page() {
+  return <p>hello world</p>
+}

test/production/route-bundle-stats/route-bundle-stats.test.ts

@@ -0,0 +1,53 @@
+import { nextTestSetup, itTurbopack } from 'e2e-utils'
+
+describe('route-bundle-stats', () => {
+  const { next, isNextDeploy } = nextTestSetup({
+    files: __dirname,
+  })
+
+  if (isNextDeploy) {
+    it('should skip', () => {})
+    return
+  }
+
+  const STATS_PATH = '.next/diagnostics/route-bundle-stats.json'
+
+  itTurbopack(
+    'writes .next/diagnostics/route-bundle-stats.json after build',
+    async () => {
+      const raw = await next.readFile(STATS_PATH)
+      const stats = JSON.parse(raw)
+
+      expect(Array.isArray(stats)).toBe(true)
+      expect(stats.length).toBeGreaterThan(0)
+
+      for (const entry of stats) {
+        expect(typeof entry.route).toBe('string')
+        expect(entry.route).toMatch(/^\//)
+        expect(typeof entry.firstLoadUncompressedJsBytes).toBe('number')
+        expect(entry.firstLoadUncompressedJsBytes).toBeGreaterThan(0)
+        expect(Array.isArray(entry.firstLoadChunkPaths)).toBe(true)
+        expect(entry.firstLoadChunkPaths.length).toBeGreaterThan(0)
+
+        for (const chunkPath of entry.firstLoadChunkPaths) {
+          expect(chunkPath).toMatch(/^\.next[\\/]/)
+        }
+      }
+
+      // Entries are sorted descending by firstLoadUncompressedJsBytes.
+      for (let i = 1; i < stats.length; i++) {
+        expect(
+          stats[i - 1].firstLoadUncompressedJsBytes
+        ).toBeGreaterThanOrEqual(stats[i].firstLoadUncompressedJsBytes)
+      }
+    }
+  )
+
+  itTurbopack('includes both Pages Router and App Router routes', async () => {
+    const stats = JSON.parse(await next.readFile(STATS_PATH))
+    const routes = stats.map((e: { route: string }) => e.route)
+
+    expect(routes).toContain('/')
+    expect(routes).toContain('/blog')
+  })
+})