mirror of
https://github.com/hanzoai/next.js.git
synced 2026-09-24 17:00:51 +00:00
Jude GaoandGitHub
0921733a94
Fixtures now live next to the code they test, like e2e. `pnpm eval <name>` packs the local `next` build, generates baseline + agents-md experiment configs on the fly, and runs both in a sandbox. The agents-md variant drops an `AGENTS.md` that points the agent at the bundled docs in `node_modules/next/dist/docs/` — comparing the two variants tells you whether shipping a doc actually changes agent behavior. `run-evals.js` mirrors `run-tests.js`: pack once, pass the tarball path to the child via `NEXT_EVAL_TARBALL` env, forward flags. We only pack `next`, not the whole workspace — the sandbox is remote Linux, so a local `@next/swc` darwin binary wouldn't run there anyway; the sandbox downloads the right one at runtime. The experiment config uses `sandbox: 'auto'`, which picks Vercel sandboxes when credentials are present and falls back to local Docker otherwise, so external contributors can run the same evals with just Docker + `ANTHROPIC_API_KEY`. `experiments/` is generated fresh each run and gitignored so we don't maintain N committed config files that differ by one line. Fixture code is excluded from eslint since it's deliberately imperfect code for agents to fix, and `EVAL.ts` uses vitest rather than jest. Fixture `package.json` files use `"next": "^16"` rather than a pinned canary so agents reading `package.json` to infer capabilities aren't misled by a stale version string; the tarball install overlays it regardless. next-evals-oss stays as the full benchmark runner for nextjs.org/evals; it'll pull fixtures from here instead of keeping its own copy.
53 lines
1.7 KiB
TypeScript
53 lines
1.7 KiB
TypeScript
import { readFileSync } from 'node:fs'
|
|
import type { Sandbox } from '@vercel/agent-eval'
|
|
|
|
/**
|
|
* Install the locally-built Next.js into the sandbox.
|
|
*
|
|
* The tarball path comes from run-evals.js via NEXT_EVAL_TARBALL, the same
|
|
* env-var handoff that run-tests.js uses for NEXT_TEST_PKG_PATHS. We hard-fail
|
|
* if it's missing rather than falling back to npm — silently testing the
|
|
* published canary instead of your local build defeats the point.
|
|
*/
|
|
export async function installNextJs(sandbox: Sandbox): Promise<void> {
|
|
const tarball = process.env.NEXT_EVAL_TARBALL
|
|
if (!tarball) {
|
|
throw new Error(
|
|
'NEXT_EVAL_TARBALL not set. Run evals via `pnpm eval` from the repo root.'
|
|
)
|
|
}
|
|
await sandbox.writeFiles({
|
|
// @ts-expect-error — upstream types writeFiles as Record<string, string>
|
|
// but the runtime accepts Buffer. Tarballs are binary; can't send as string.
|
|
'next.tgz': readFileSync(tarball),
|
|
})
|
|
const { exitCode, stderr } = await sandbox.runCommand('npm', [
|
|
'install',
|
|
'./next.tgz',
|
|
])
|
|
if (exitCode !== 0) {
|
|
throw new Error(
|
|
`npm install ./next.tgz failed (exit ${exitCode}):\n${stderr}`
|
|
)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Write AGENTS.md (and aliases) to the sandbox root, directing agents to read
|
|
* bundled docs from node_modules/next/dist/docs/.
|
|
*/
|
|
export async function writeAgentsMd(sandbox: Sandbox): Promise<void> {
|
|
const body = `<!-- BEGIN:nextjs-agent-rules -->
|
|
|
|
# Next.js: ALWAYS read docs before coding
|
|
|
|
Before any Next.js work, find and read the relevant doc in \`node_modules/next/dist/docs/\`. Your training data is outdated — the docs are the source of truth.
|
|
|
|
<!-- END:nextjs-agent-rules -->
|
|
`
|
|
await sandbox.writeFiles({
|
|
'AGENTS.md': body,
|
|
'CLAUDE.md': '@AGENTS.md\n',
|
|
})
|
|
}
|