Add PPR evals and fix env loading for agent-eval 0.9.5 (#92063)

Upgrades `@vercel/agent-eval` from 0.8.0 to 0.9.5. The new version loads
`.env` / `.env.local` from its own cwd (`evals/`) rather than inheriting
from the parent process, so the old `process.loadEnvFile()` call no
longer works. Instead, `run-evals.js` now symlinks the root env files
into `evals/` before spawning the harness. `evals/.env` is added to
`.gitignore` to keep secrets out of the tree.

Two new evals exercise PPR knowledge: `agent-041-optimize-ppr-shell`
checks that the agent decomposes a monolithic `loading.tsx` into
granular `Suspense` boundaries, and `agent-042-enable-ppr` verifies the
agent knows PPR is enabled via `cacheComponents: true` (not the old
`experimental.ppr` flag).

.gitignore

@@ -50,6 +50,7 @@ examples/**/out/*
 
 # env
 .env*.local
+evals/.env
 
 pr-stats.md
 test-timings.json

evals/evals/agent-041-optimize-ppr-shell/EVAL.ts

@@ -0,0 +1,61 @@
+/**
+ * Optimize PPR Shell
+ *
+ * Tests whether the agent decomposes a monolithic loading.tsx (which creates
+ * a single implicit Suspense boundary around the entire page) into granular
+ * Suspense boundaries — one per dashboard section — so each section can
+ * stream independently and the PPR shell contains more static content.
+ *
+ * Tricky because the starting code uses Next.js's loading.tsx convention,
+ * which is an implicit Suspense boundary. Agents need to recognize that
+ * loading.tsx creates an all-or-nothing loading state, and that optimizing
+ * the PPR shell requires replacing it with per-section Suspense boundaries
+ * so each section can stream independently.
+ */
+
+import { expect, test } from 'vitest'
+import { readFileSync } from 'fs'
+import { join } from 'path'
+
+const appDir = join(process.cwd(), 'app')
+
+function readFile(name: string): string {
+  return readFileSync(join(appDir, name), 'utf-8')
+}
+
+test('Page has at least 3 Suspense boundaries', () => {
+  const page = readFile('page.tsx')
+
+  const suspenseCount = (page.match(/<Suspense[\s>]/g) || []).length
+  expect(suspenseCount).toBeGreaterThanOrEqual(3)
+})
+
+test('Each dashboard section has its own Suspense boundary in page.tsx', () => {
+  const page = readFile('page.tsx')
+
+  // Split page into Suspense blocks: text between each <Suspense and </Suspense>
+  const suspenseBlocks = page.split(/<Suspense[\s>]/).slice(1)
+
+  const components = ['CardStats', 'RevenueChart', 'LatestInvoices']
+  for (const component of components) {
+    const inOwnBlock = suspenseBlocks.some(
+      (block) => block.includes(component) && block.includes('</Suspense>')
+    )
+    expect(inOwnBlock, `${component} should be inside its own <Suspense>`).toBe(
+      true
+    )
+  }
+})
+
+test('Page does not await all data before rendering', () => {
+  const page = readFile('page.tsx')
+
+  // The page should not call getDashboardData() or fetch() at the top level.
+  // A simple check: the page shouldn't contain the original monolithic fetch.
+  expect(page).not.toMatch(/await\s+getDashboardData\s*\(/)
+
+  // The page component itself should not be async (data fetching moves into children)
+  // OR if it is async, it should not await a data fetch before returning JSX.
+  // We check the simpler signal: getDashboardData should not be called in page.tsx at all.
+  expect(page).not.toMatch(/getDashboardData\s*\(/)
+})

evals/evals/agent-041-optimize-ppr-shell/PROMPT.md

@@ -0,0 +1 @@
+Optimize the partial pre-rendering shell for this app.

evals/evals/agent-041-optimize-ppr-shell/app/CardStats.tsx

@@ -0,0 +1,20 @@
+export function CardStats({
+  totalRevenue,
+  totalInvoices,
+}: {
+  totalRevenue: number
+  totalInvoices: number
+}) {
+  return (
+    <div className="grid grid-cols-2 gap-4">
+      <div className="card">
+        <h2>Total Revenue</h2>
+        <p>${totalRevenue.toLocaleString()}</p>
+      </div>
+      <div className="card">
+        <h2>Total Invoices</h2>
+        <p>{totalInvoices}</p>
+      </div>
+    </div>
+  )
+}

evals/evals/agent-041-optimize-ppr-shell/app/LatestInvoices.tsx

@@ -0,0 +1,18 @@
+export function LatestInvoices({
+  invoices,
+}: {
+  invoices: { id: string; name: string; amount: number }[]
+}) {
+  return (
+    <div className="invoices">
+      <h2>Latest Invoices</h2>
+      <ul>
+        {invoices.map((invoice) => (
+          <li key={invoice.id}>
+            {invoice.name} - ${invoice.amount.toLocaleString()}
+          </li>
+        ))}
+      </ul>
+    </div>
+  )
+}

evals/evals/agent-041-optimize-ppr-shell/app/RevenueChart.tsx

@@ -0,0 +1,18 @@
+export function RevenueChart({
+  revenue,
+}: {
+  revenue: { month: string; amount: number }[]
+}) {
+  return (
+    <div className="chart">
+      <h2>Revenue</h2>
+      <ul>
+        {revenue.map((item) => (
+          <li key={item.month}>
+            {item.month}: ${item.amount.toLocaleString()}
+          </li>
+        ))}
+      </ul>
+    </div>
+  )
+}

evals/evals/agent-041-optimize-ppr-shell/app/layout.tsx

@@ -0,0 +1,18 @@
+import type { Metadata } from 'next'
+
+export const metadata: Metadata = {
+  title: 'Create Next App',
+  description: 'Generated by create next app',
+}
+
+export default function RootLayout({
+  children,
+}: Readonly<{
+  children: React.ReactNode
+}>) {
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  )
+}

evals/evals/agent-041-optimize-ppr-shell/app/loading.tsx

@@ -0,0 +1,8 @@
+export default function Loading() {
+  return (
+    <div className="loading">
+      <div className="spinner" />
+      <p>Loading dashboard...</p>
+    </div>
+  )
+}

evals/evals/agent-041-optimize-ppr-shell/app/page.tsx

@@ -0,0 +1,24 @@
+import { RevenueChart } from './RevenueChart'
+import { LatestInvoices } from './LatestInvoices'
+import { CardStats } from './CardStats'
+
+async function getDashboardData() {
+  const res = await fetch('https://api.example.com/dashboard')
+  return res.json()
+}
+
+export default async function Page() {
+  const data = await getDashboardData()
+
+  return (
+    <main>
+      <h1>Dashboard</h1>
+      <CardStats
+        totalRevenue={data.totalRevenue}
+        totalInvoices={data.totalInvoices}
+      />
+      <RevenueChart revenue={data.revenue} />
+      <LatestInvoices invoices={data.invoices} />
+    </main>
+  )
+}

evals/evals/agent-041-optimize-ppr-shell/next.config.ts

@@ -0,0 +1,7 @@
+import type { NextConfig } from 'next'
+
+const nextConfig: NextConfig = {
+  cacheComponents: true,
+}
+
+export default nextConfig

evals/evals/agent-041-optimize-ppr-shell/package.json

@@ -0,0 +1,23 @@
+{
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start"
+  },
+  "dependencies": {
+    "next": "^16",
+    "react": "19.1.0",
+    "react-dom": "19.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^5",
+    "vitest": "^3.1.3",
+    "@vitejs/plugin-react": "^4.4.1",
+    "vite-tsconfig-paths": "^5.1.4"
+  }
+}

evals/evals/agent-041-optimize-ppr-shell/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    "target": "ES2017",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "react-jsx",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}

evals/evals/agent-042-enable-ppr/EVAL.ts

@@ -0,0 +1,31 @@
+/**
+ * Enable PPR
+ *
+ * Tests whether the agent knows that partial pre-rendering is enabled via
+ * `cacheComponents: true` in next.config.ts, NOT the old
+ * `experimental: { ppr: true }` flag.
+ *
+ * Tricky because most training data and older docs reference the experimental
+ * flag. The current way to enable PPR in Next.js 16 is `cacheComponents: true`.
+ */
+
+import { expect, test } from 'vitest'
+import { readFileSync } from 'fs'
+import { join } from 'path'
+
+test('Enables PPR via cacheComponents in next.config', () => {
+  const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf-8')
+
+  expect(config).toMatch(/cacheComponents\s*:\s*true/)
+})
+
+test('Does not use the old experimental.ppr flag', () => {
+  const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf-8')
+
+  // Strip comments to avoid false positives from explanatory comments
+  const stripped = config
+    .replace(/\/\*[\s\S]*?\*\//g, '')
+    .replace(/\/\/.*$/gm, '')
+
+  expect(stripped).not.toMatch(/ppr\s*:\s*true/)
+})

evals/evals/agent-042-enable-ppr/PROMPT.md

@@ -0,0 +1 @@
+Enable partial pre-rendering for this app.

evals/evals/agent-042-enable-ppr/app/ProductList.tsx

@@ -0,0 +1,18 @@
+async function getProducts() {
+  const res = await fetch('https://api.example.com/products')
+  return res.json()
+}
+
+export async function ProductList() {
+  const products = await getProducts()
+
+  return (
+    <ul>
+      {products.map((p: { id: string; name: string; price: number }) => (
+        <li key={p.id}>
+          {p.name} - ${p.price}
+        </li>
+      ))}
+    </ul>
+  )
+}

evals/evals/agent-042-enable-ppr/app/Recommendations.tsx

@@ -0,0 +1,19 @@
+async function getRecommendations() {
+  const res = await fetch('https://api.example.com/recommendations')
+  return res.json()
+}
+
+export async function Recommendations() {
+  const items = await getRecommendations()
+
+  return (
+    <div>
+      <h2>Recommended for you</h2>
+      <ul>
+        {items.map((item: { id: string; name: string }) => (
+          <li key={item.id}>{item.name}</li>
+        ))}
+      </ul>
+    </div>
+  )
+}

evals/evals/agent-042-enable-ppr/app/layout.tsx

@@ -0,0 +1,18 @@
+import type { Metadata } from 'next'
+
+export const metadata: Metadata = {
+  title: 'Create Next App',
+  description: 'Generated by create next app',
+}
+
+export default function RootLayout({
+  children,
+}: Readonly<{
+  children: React.ReactNode
+}>) {
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  )
+}

evals/evals/agent-042-enable-ppr/app/page.tsx

@@ -0,0 +1,17 @@
+import { Suspense } from 'react'
+import { ProductList } from './ProductList'
+import { Recommendations } from './Recommendations'
+
+export default function Page() {
+  return (
+    <main>
+      <h1>Store</h1>
+      <Suspense fallback={<p>Loading products...</p>}>
+        <ProductList />
+      </Suspense>
+      <Suspense fallback={<p>Loading recommendations...</p>}>
+        <Recommendations />
+      </Suspense>
+    </main>
+  )
+}

evals/evals/agent-042-enable-ppr/next.config.ts

@@ -0,0 +1,5 @@
+import type { NextConfig } from 'next'
+
+const nextConfig: NextConfig = {}
+
+export default nextConfig

evals/evals/agent-042-enable-ppr/package.json

@@ -0,0 +1,23 @@
+{
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start"
+  },
+  "dependencies": {
+    "next": "^16",
+    "react": "19.1.0",
+    "react-dom": "19.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^5",
+    "vitest": "^3.1.3",
+    "@vitejs/plugin-react": "^4.4.1",
+    "vite-tsconfig-paths": "^5.1.4"
+  }
+}

evals/evals/agent-042-enable-ppr/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    "target": "ES2017",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "react-jsx",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}

package.json

@@ -177,7 +177,7 @@
     "@types/string-hash": "1.1.1",
     "@types/trusted-types": "2.0.3",
     "@types/yargs": "16.0.11",
-    "@vercel/agent-eval": "0.8.0",
+    "@vercel/agent-eval": "0.9.5",
     "@vercel/devlow-bench": "workspace:*",
     "@vercel/kv": "3.0.0",
     "@vercel/og": "0.11.1",

pnpm-lock.yaml

@@ -226,8 +226,8 @@ importers:
         specifier: 16.0.11
         version: 16.0.11
       '@vercel/agent-eval':
-        specifier: 0.8.0
-        version: 0.8.0
+        specifier: 0.9.5
+        version: 0.9.5
       '@vercel/devlow-bench':
         specifier: workspace:*
         version: link:turbopack/packages/devlow-bench
@@ -4138,14 +4138,6 @@ packages:
       '@types/node':
         optional: true
 
-  '@isaacs/[email protected]':
-    resolution: {integrity: sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==}
-    engines: {node: 20 || >=22}
-
-  '@isaacs/[email protected]':
-    resolution: {integrity: sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==}
-    engines: {node: 20 || >=22}
-
   '@isaacs/[email protected]':
     resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
     engines: {node: '>=12'}
@@ -7165,8 +7157,8 @@ packages:
   '@upstash/[email protected]':
     resolution: {integrity: sha512-hSjv66NOuahW3MisRGlSgoszU2uONAY2l5Qo3Sae8OT3/Tng9K+2/cBRuyPBX8egwEGcNNCF9+r0V6grNnhL+w==}
 
-  '@vercel/agent-eval@0.8.0':
-    resolution: {integrity: sha512-Efv6EPUUBc0I0fkw5964O4rGxGDS/j6vYJ2IspZ5sFVdrLdDlqI1llJdkTvuc+cg4EuqXsOhEQhJlaNG9eZ8zQ==}
+  '@vercel/agent-eval@0.9.5':
+    resolution: {integrity: sha512-wSs0K39zZFQDNniG0UNeZMy3GgVDSuTyhVKZVqxm33nu3sq2KJjOSJCpvoFaXKUWkk2Ea3DTaxJsVgp8b1Xq5w==}
     engines: {node: '>=18.0.0'}
     hasBin: true
 
@@ -13870,10 +13862,6 @@ packages:
   [email protected]:
     resolution: {integrity: sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==}
 
-  [email protected]:
-    resolution: {integrity: sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==}
-    engines: {node: 20 || >=22}
-
   [email protected]:
     resolution: {integrity: sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==}
     engines: {node: 18 || 20 || >=22}
@@ -21284,12 +21272,6 @@ snapshots:
     optionalDependencies:
       '@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu)
 
-  '@isaacs/[email protected]': {}
-
-  '@isaacs/[email protected]':
-    dependencies:
-      '@isaacs/balanced-match': 4.0.1
-
   '@isaacs/[email protected]':
     dependencies:
       string-width: 5.1.2
@@ -22108,7 +22090,7 @@ snapshots:
       hosted-git-info: 9.0.2
       json-stringify-nice: 1.1.4
       lru-cache: 11.2.1
-      minimatch: 10.1.1
+      minimatch: 10.2.4
       nopt: 8.1.0
       npm-install-checks: 7.1.2
       npm-package-arg: 13.0.1
@@ -22206,7 +22188,7 @@ snapshots:
       '@npmcli/name-from-folder': 4.0.0
       '@npmcli/package-json': 7.0.2
       glob: 13.0.0
-      minimatch: 10.1.1
+      minimatch: 10.2.4
 
   '@npmcli/[email protected]':
     dependencies:
@@ -24889,7 +24871,7 @@ snapshots:
     dependencies:
       uncrypto: 0.1.3
 
-  '@vercel/agent-eval@0.8.0':
+  '@vercel/agent-eval@0.9.5':
     dependencies:
       '@ai-sdk/anthropic': 1.2.12([email protected])
       '@vercel/sandbox': 1.8.0
@@ -29922,7 +29904,7 @@ snapshots:
 
   [email protected]:
     dependencies:
-      minimatch: 10.1.1
+      minimatch: 10.2.4
       minipass: 7.1.2
       path-scurry: 2.0.0
 
@@ -30637,7 +30619,7 @@ snapshots:
 
   [email protected]:
     dependencies:
-      minimatch: 10.1.1
+      minimatch: 10.2.4
 
   [email protected]: {}
 
@@ -33804,10 +33786,6 @@ snapshots:
 
   [email protected]: {}
 
-  [email protected]:
-    dependencies:
-      '@isaacs/brace-expansion': 5.0.0
-
   [email protected]:
     dependencies:
       brace-expansion: 5.0.4

run-evals.js

@@ -25,13 +25,6 @@ const { execFileSync, spawnSync } = require('child_process')
 
 const ROOT = __dirname
 
-// Sandbox + agent API keys. agent-eval looks for .env.local in its own cwd
-// (evals/), but `vc env pull` writes to the repo root. Load it here so the
-// vars are already in process.env when we spawn the child.
-try {
-  process.loadEnvFile(path.join(ROOT, '.env.local'))
-} catch {}
-
 const EVALS_DIR = path.join(ROOT, 'evals')
 const FIXTURES_DIR = path.join(EVALS_DIR, 'evals')
 const EXPERIMENTS_DIR = path.join(EVALS_DIR, 'experiments')
@@ -162,6 +155,20 @@ function main() {
     console.log(`  ${TARBALL} (${mb} MB)`)
   }
 
+  // agent-eval loads .env / .env.local from its own cwd (evals/). `vc env pull`
+  // writes to the repo root, so symlink them into evals/ for agent-eval to find.
+  for (const envFile of ['.env', '.env.local']) {
+    const src = path.join(ROOT, envFile)
+    const dest = path.join(EVALS_DIR, envFile)
+    try {
+      // Remove stale symlink or file before creating a fresh one.
+      fs.rmSync(dest, { force: true })
+      if (fs.existsSync(src)) {
+        fs.symlinkSync(src, dest)
+      }
+    } catch {}
+  }
+
   writeExperiments(evalName)
   console.log(
     evalName