feat(benchmark)!: rewrite the public API (#10113)

.gitignore

@@ -26,6 +26,7 @@ docs/.vitepress/cache/
 !test/e2e/fixtures/dotted-files/**/.cache
 test/**/__screenshots__/**/*
 test/**/__traces__/**/*
+test/browser/**/__benchmarks__
 test/browser/fixtures/update-snapshot/basic.test.ts
 test/e2e/fixtures/browser-multiple/basic-*
 *.tsbuildinfo

.vscode/settings.json

@@ -23,7 +23,7 @@
   // ],
 
   "vitest.ignoreWorkspace": true,
-  "vitest.configSearchPatternInclude": "test/{core,cli,config,browser,reporters}/{vitest,vite}.{config.ts,config.unit.mts}",
+  "vitest.configSearchPatternInclude": "test/{unit,e2e,config,browser,reporters}/{vitest,vite}.{config.ts,config.unit.mts}",
   "testing.automaticallyOpenTestResults": "neverOpen",
 
   // Enable eslint for all supported languages

docs/.vitepress/config.ts

@@ -901,6 +901,10 @@ export default ({ mode }: { mode: string }) => {
                 text: 'Testing Types',
                 link: '/guide/testing-types',
               },
+              {
+                text: 'Benchmarking',
+                link: '/guide/benchmarking',
+              },
               {
                 text: 'In-Source Testing',
                 link: '/guide/in-source',

docs/api/advanced/vitest.md

@@ -5,35 +5,9 @@ title: Vitest API
 
 # Vitest
 
-Vitest instance requires the current test mode. It can be either:
+## mode <Deprecated /> {#mode}
 
-- `test` when running runtime tests
-- `benchmark` when running benchmarks <Experimental />
-
-::: details New in Vitest 4
-Vitest 4 added several new APIs (they are marked with a "4.0.0+" badge) and removed deprecated APIs:
-
-- `invalidates`
-- `changedTests` (use [`onFilterWatchedSpecification`](#onfilterwatchedspecification) instead)
-- `server` (use [`vite`](#vite) instead)
-- `getProjectsByTestFile` (use [`getModuleSpecifications`](#getmodulespecifications) instead)
-- `getFileWorkspaceSpecs` (use [`getModuleSpecifications`](#getmodulespecifications) instead)
-- `getModuleProjects` (filter by [`this.projects`](#projects) yourself)
-- `updateLastChanged` (renamed to [`invalidateFile`](#invalidatefile))
-- `globTestSpecs` (use [`globTestSpecifications`](#globtestspecifications) instead)
-- `globTestFiles` (use [`globTestSpecifications`](#globtestspecifications) instead)
-- `listFile` (use [`getRelevantTestSpecifications`](#getrelevanttestspecifications) instead)
-:::
-
-## mode
-
-### test
-
-Test mode will only call functions inside `test` or `it`, and throws an error when `bench` is encountered. This mode uses `include` and `exclude` options in the config to find test files.
-
-### benchmark <Experimental /> {#benchmark}
-
-Benchmark mode calls `bench` functions and throws an error, when it encounters `test` or `it`. This mode uses `benchmark.include` and `benchmark.exclude` options in the config to find benchmark files.
+Since Vitest 5, this property is always `'test'`.
 
 ## config
 

docs/api/test.md

@@ -670,235 +670,8 @@ Scoped `aroundAll` hook that inherits types from [`test.extend`](#test-extend).
 
 ## bench <Experimental /> {#bench}
 
-- **Type:** `(name: string | Function, fn: BenchFunction, options?: BenchOptions) => void`
+::: warning Updated in Vitest 5
+The benchmarking API has been rewritten. `bench` is no longer a top-level import from `vitest`, and the `bench.skip` / `bench.only` / `bench.todo` helpers have been removed. `bench` is now a [test-context fixture](/guide/test-context#bench) accessed from inside a `test()`.
 
-::: danger
-Benchmarking is experimental and does not follow SemVer.
+See the [Benchmarking guide](/guide/benchmarking) for the new API.
 :::
-
-`bench` defines a benchmark. In Vitest terms, benchmark is a function that defines a series of operations. Vitest runs this function multiple times to display different performance results.
-
-Vitest uses the [`tinybench`](https://github.com/tinylibs/tinybench) library under the hood, inheriting all its options that can be used as a third argument.
-
-```ts
-import { bench } from 'vitest'
-
-bench('normal sorting', () => {
-  const x = [1, 5, 4, 2, 3]
-  x.sort((a, b) => {
-    return a - b
-  })
-}, { time: 1000 })
-```
-
-```ts
-export interface Options {
-  /**
-   * time needed for running a benchmark task (milliseconds)
-   * @default 500
-   */
-  time?: number
-
-  /**
-   * number of times that a task should run if even the time option is finished
-   * @default 10
-   */
-  iterations?: number
-
-  /**
-   * function to get the current timestamp in milliseconds
-   */
-  now?: () => number
-
-  /**
-   * An AbortSignal for aborting the benchmark
-   */
-  signal?: AbortSignal
-
-  /**
-   * Throw if a task fails (events will not work if true)
-   */
-  throws?: boolean
-
-  /**
-   * warmup time (milliseconds)
-   * @default 100ms
-   */
-  warmupTime?: number
-
-  /**
-   * warmup iterations
-   * @default 5
-   */
-  warmupIterations?: number
-
-  /**
-   * setup function to run before each benchmark task (cycle)
-   */
-  setup?: Hook
-
-  /**
-   * teardown function to run after each benchmark task (cycle)
-   */
-  teardown?: Hook
-}
-```
-After the test case is run, the output structure information is as follows:
-
-```
-  name                      hz     min     max    mean     p75     p99    p995    p999     rme  samples
-· normal sorting  6,526,368.12  0.0001  0.3638  0.0002  0.0002  0.0002  0.0002  0.0004  ±1.41%   652638
-```
-```ts
-export interface TaskResult {
-  /*
-   * the last error that was thrown while running the task
-   */
-  error?: unknown
-
-  /**
-   * The amount of time in milliseconds to run the benchmark task (cycle).
-   */
-  totalTime: number
-
-  /**
-   * the minimum value in the samples
-   */
-  min: number
-  /**
-   * the maximum value in the samples
-   */
-  max: number
-
-  /**
-   * the number of operations per second
-   */
-  hz: number
-
-  /**
-   * how long each operation takes (ms)
-   */
-  period: number
-
-  /**
-   * task samples of each task iteration time (ms)
-   */
-  samples: number[]
-
-  /**
-   * samples mean/average (estimate of the population mean)
-   */
-  mean: number
-
-  /**
-   * samples variance (estimate of the population variance)
-   */
-  variance: number
-
-  /**
-   * samples standard deviation (estimate of the population standard deviation)
-   */
-  sd: number
-
-  /**
-   * standard error of the mean (a.k.a. the standard deviation of the sampling distribution of the sample mean)
-   */
-  sem: number
-
-  /**
-   * degrees of freedom
-   */
-  df: number
-
-  /**
-   * critical value of the samples
-   */
-  critical: number
-
-  /**
-   * margin of error
-   */
-  moe: number
-
-  /**
-   * relative margin of error
-   */
-  rme: number
-
-  /**
-   * median absolute deviation
-   */
-  mad: number
-
-  /**
-   * p50/median percentile
-   */
-  p50: number
-
-  /**
-   * p75 percentile
-   */
-  p75: number
-
-  /**
-   * p99 percentile
-   */
-  p99: number
-
-  /**
-   * p995 percentile
-   */
-  p995: number
-
-  /**
-   * p999 percentile
-   */
-  p999: number
-}
-```
-
-### bench.skip
-
-- **Type:** `(name: string | Function, fn: BenchFunction, options?: BenchOptions) => void`
-
-You can use `bench.skip` syntax to skip running certain benchmarks.
-
-```ts
-import { bench } from 'vitest'
-
-bench.skip('normal sorting', () => {
-  const x = [1, 5, 4, 2, 3]
-  x.sort((a, b) => {
-    return a - b
-  })
-})
-```
-
-### bench.only
-
-- **Type:** `(name: string | Function, fn: BenchFunction, options?: BenchOptions) => void`
-
-Use `bench.only` to only run certain benchmarks in a given suite. This is useful when debugging.
-
-```ts
-import { bench } from 'vitest'
-
-bench.only('normal sorting', () => {
-  const x = [1, 5, 4, 2, 3]
-  x.sort((a, b) => {
-    return a - b
-  })
-})
-```
-
-### bench.todo
-
-- **Type:** `(name: string | Function) => void`
-
-Use `bench.todo` to stub benchmarks to be implemented later.
-
-```ts
-import { bench } from 'vitest'
-
-bench.todo('unimplemented test')
-```

docs/config/benchmark.md

@@ -9,6 +9,13 @@ outline: deep
 
 Options used when running `vitest bench`.
 
+## benchmark.enabled
+
+- **Type:** `boolean`
+- **Default:** `false`
+
+Enables the benchmark project. When set, Vitest creates a dedicated benchmark project alongside your regular test project, runs files matching [`benchmark.include`](#benchmark-include) in it, and exposes the [`bench` fixture](/guide/test-context#bench) to those files. Running `vitest bench` enables this automatically.
+
 ## benchmark.include
 
 - **Type:** `string[]`
@@ -32,39 +39,18 @@ Include globs for in-source benchmark test files. This option is similar to [`in
 
 When defined, Vitest will run all matched files with `import.meta.vitest` inside.
 
-## benchmark.reporters
+## benchmark.retainSamples
 
-- **Type:** `Arrayable<BenchmarkBuiltinReporters | Reporter>`
-- **Default:** `'default'`
+- **Type:** `boolean`
+- **Default:** `false`
 
-Custom reporter for output. Can contain one or more built-in report names, reporter instances, and/or paths to custom reporters.
+Include the `samples` array of per-iteration timings on every benchmark result. Disabled by default to reduce memory usage; enable when a custom reporter or API consumer needs the raw samples.
 
-## benchmark.outputFile
 
-Deprecated in favor of `benchmark.outputJson`.
+## benchmark.suppressExportGetterWarnings
 
-## benchmark.outputJson {#benchmark-outputJson}
+- **Type:** `boolean`
+- **Default:** `false`
 
-- **Type:** `string | undefined`
-- **Default:** `undefined`
+Suppress the warning printed when a benchmark accesses module export getters too many times. Vitest tracks getter access during benchmark runs because Vite's module runner wraps every export in a getter, and excessive access can dominate the measurement (see [Module Runner Overhead](/guide/benchmarking#module-runner-overhead)). Enable this when you've intentionally accepted the overhead, or when the warning is noisy for benchmarks where the getter cost is negligible.
 
-A file path to store the benchmark result, which can be used for `--compare` option later.
-
-For example:
-
-```sh
-# save main branch's result
-git checkout main
-vitest bench --outputJson main.json
-
-# change a branch and compare against main
-git checkout feature
-vitest bench --compare main.json
-```
-
-## benchmark.compare {#benchmark-compare}
-
-- **Type:** `string | undefined`
-- **Default:** `undefined`
-
-A file path to a previous benchmark result to compare against current runs.

docs/config/index.md

@@ -8,7 +8,7 @@ If you are using Vite and have a `vite.config` file, Vitest will read it to matc
 
 - Create `vitest.config.ts`, which will have the higher priority and will **override** the configuration from `vite.config.ts` (Vitest supports all conventional JS and TS extensions, but doesn't support `json`) - it means all options in your `vite.config` will be **ignored**
 - Pass `--config` option to CLI, e.g. `vitest --config ./path/to/vitest.config.ts`
-- Use `process.env.VITEST` or `mode` property on `defineConfig` (will be set to `test`/`benchmark` if not overridden with `--mode`) to conditionally apply different configuration in `vite.config.ts`. Note that like any other environment variable, `VITEST` is also exposed on `import.meta.env` in your tests
+- Use `process.env.VITEST` or `mode` property on `defineConfig` (will be set to `test` if not overridden with `--mode`) to conditionally apply different configuration in `vite.config.ts`. Note that like any other environment variable, `VITEST` is also exposed on `import.meta.env` in your tests
 
 To configure `vitest` itself, add `test` property in your Vite config. You'll also need to add a reference to Vitest types using a [triple slash command](https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html#-reference-types-) at the top of your config file, if you are importing `defineConfig` from `vite` itself.
 

docs/config/runner.md

@@ -6,6 +6,5 @@ outline: deep
 # runner
 
 - **Type:** `VitestRunnerConstructor`
-- **Default:** `node`, when running tests, or `benchmark`, when running benchmarks
 
 Path to a custom test runner. This is an advanced feature and should be used with custom library runners. You can read more about it in [the documentation](/api/advanced/runner).

docs/guide/advanced/index.md

@@ -14,7 +14,6 @@ You can import any method from the `vitest/node` entry-point.
 
 ```ts
 function startVitest(
-  mode: VitestRunMode,
   cliFilters: string[] = [],
   options: CliOptions = {},
   viteOverrides?: ViteUserConfig,
@@ -27,7 +26,7 @@ You can start running Vitest tests using its Node API:
 ```js
 import { startVitest } from 'vitest/node'
 
-const vitest = await startVitest('test')
+const vitest = await startVitest()
 
 await vitest.close()
 ```
@@ -47,7 +46,7 @@ After running the tests, you can get the results from the [`state.getTestModules
 ```ts
 import type { TestModule } from 'vitest/node'
 
-const vitest = await startVitest('test')
+const vitest = await startVitest()
 
 console.log(vitest.state.getTestModules()) // [TestModule]
 ```
@@ -60,7 +59,6 @@ The ["Running Tests"](/guide/advanced/tests#startvitest) guide has a usage examp
 
 ```ts
 function createVitest(
-  mode: VitestRunMode,
   options: CliOptions,
   viteOverrides: ViteUserConfig = {},
   vitestOptions: VitestOptions = {},

docs/guide/benchmarking.md

@@ -0,0 +1,480 @@
+---
+title: Benchmarking | Guide
+---
+
+# Benchmarking
+
+Vitest lets you write benchmarks alongside your tests using the `bench` fixture from the [test context](/guide/test-context). Benchmarks are powered by [Tinybench](https://github.com/tinylibs/tinybench) and are defined inside regular `test()` calls, giving you access to the full power of Vitest's test runner: retries, lifecycle hooks, filtering, and assertions.
+
+## Defining a Benchmark
+
+Use the `bench` fixture to define a benchmark. Call `.run()` to execute it:
+
+```ts
+import { expect, test } from 'vitest'
+
+test('parsing performance', async ({ bench }) => {
+  const result = await bench('parse', () => {
+    JSON.parse('{"key":"value"}')
+  }).run()
+})
+```
+
+The `bench()` function registers a benchmark without executing it. Calling `.run()` runs the benchmark and returns the result. After the test completes, Vitest prints a single-row version of the [comparison table](#comparing-benchmarks) (ops/sec, mean time, percentiles, etc.), so you get the same output for a one-off benchmark as you do for `bench.compare()`.
+
+::: warning
+The `bench` fixture is only available in files matched by [`benchmark.include`](/config/#benchmark-include) (default: `**/*.{bench,benchmark}.?(c|m)[jt]s?(x)`). Using `{ bench }` inside a regular test file will throw an error.
+
+Whether a file participates in the benchmark run is decided by the filename, not by whether the test uses the `bench` fixture. Renaming `parser.test.ts` to `parser.bench.ts` (or adjusting `benchmark.include`) is what moves it into the benchmark project.
+:::
+
+## Running Benchmarks
+
+Benchmark files are matched by [`benchmark.include`](/config/#benchmark-include) (default: `**/*.{bench,benchmark}.?(c|m)[jt]s?(x)`) and run in their own project, separate from your regular tests. There are three ways to run them, depending on whether you want to skip them, run them alongside tests, or run them on their own.
+
+### `vitest` (default)
+
+Without [`benchmark.enabled`](/config/#benchmark-enabled), the `vitest` command only runs regular tests. Benchmark files are ignored entirely. This is the default and the right choice for day-to-day development, since benchmarks are slow and noisy and shouldn't run on every save.
+
+### `vitest` with `benchmark.enabled`
+
+Set `benchmark.enabled: true` in your config to run benchmarks together with regular tests:
+
+```ts [vitest.config.ts]
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    benchmark: {
+      enabled: true,
+    },
+  },
+})
+```
+
+With this config, `vitest` runs your regular tests first, then runs the benchmarks in a separate isolated group (so benchmark execution never overlaps with test execution and adds noise to results). Useful in CI when you want a single command to validate correctness and performance.
+
+### `vitest bench`
+
+The `bench` subcommand runs only benchmarks and skips regular tests:
+
+```bash
+vitest bench
+```
+
+This implicitly enables `benchmark.enabled` for the run, so you don't need to set it in the config. Like the `vitest` command, it accepts filename filters and `-t`/`--testNamePattern` to narrow the run:
+
+```bash
+# only benchmarks in files matching "parser"
+vitest bench parser
+
+# only benchmarks whose test name matches "JSON"
+vitest bench -t JSON
+```
+
+## Comparing Benchmarks
+
+Use `bench.compare()` to compare multiple benchmarks against each other:
+
+```ts
+import { expect, test } from 'vitest'
+
+test('compare JSON libraries', async ({ bench }) => {
+  const input = '{"key":"value","nested":{"a":1}}'
+
+  const result = await bench.compare(
+    bench('JSON.parse', () => {
+      JSON.parse(input)
+    }),
+    bench('custom parser', () => {
+      customParse(input)
+    }),
+  )
+})
+```
+
+When comparing benchmarks, Vitest runs them using interleaved iterations to reduce environmental bias (CPU throttling, GC pressure, etc.) and prints a comparison table after the test completes:
+
+<<< ./snippets/benchmark-table.ansi
+
+### Options
+
+You can pass [options](https://tinylibs.github.io/tinybench/interfaces/BenchOptions.html) as the last argument to `bench.compare()`:
+
+```ts
+test('compare with options', async ({ bench }) => {
+  const result = await bench.compare(
+    bench('lib1', () => { lib1() }),
+    bench('lib2', () => { lib2() }),
+    {
+      iterations: 100,
+      time: 1000,
+    },
+  )
+})
+```
+
+You can also pass per-benchmark [options](https://tinylibs.github.io/tinybench/interfaces/FnOptions.html) as the second argument, matching how `test()` accepts options:
+
+```ts
+test('benchmarks with setup', async ({ bench }) => {
+  const result = await bench.compare(
+    bench('with-cache', () => {
+      readFromCache()
+    }),
+    bench(
+      'without-cache',
+      { beforeEach: () => clearCache() },
+      () => { readFromDisk() },
+    ),
+  )
+})
+```
+
+## Comparing Across Projects
+
+When your workspace defines multiple projects (e.g., different browsers or runtimes), pass `perProject: true` in the bench options to compare how the same benchmark performs across all of them. Vitest still prints the result inline for the current project, and additionally collects per-project results into a single comparison table at the end of the test run.
+
+```ts
+import { test } from 'vitest'
+
+test('simple example', async ({ bench }) => {
+  await bench('1 + 1', { perProject: true }, () => {
+    1 + 1
+  }).run()
+})
+```
+
+The same test file runs in each project (chromium, firefox, webkit, etc.), and Vitest groups the results:
+
+<<< ./snippets/benchmark-per-project.ansi
+
+You can also mix `perProject` benchmarks with regular ones inside `bench.compare()`:
+
+```ts
+test('compare implementations across browsers', async ({ bench }) => {
+  await bench.compare(
+    bench('JSON.parse', { perProject: true }, () => {
+      JSON.parse('{"key":"value"}')
+    }),
+    bench('custom parser', () => {
+      customParse('{"key":"value"}')
+    }),
+  )
+})
+```
+
+In this case, `custom parser` appears in the normal inline comparison table per project, while `JSON.parse` is additionally collected into the cross-project comparison table at the end.
+
+## Asserting Performance
+
+Use `toBeFasterThan()` and `toBeSlowerThan()` matchers to assert relative performance between benchmarks:
+
+```ts
+import { expect, test } from 'vitest'
+
+test('lib1 is faster than lib2', async ({ bench }) => {
+  const result = await bench.compare(
+    bench('lib1', () => { lib1() }),
+    bench('lib2', () => { lib2() }),
+  )
+
+  expect(result.get('lib1')).toBeFasterThan(result.get('lib2'))
+})
+```
+
+The `delta` option specifies the minimum relative difference required for the assertion to pass. This helps avoid flaky tests caused by benchmark noise:
+
+```ts
+// lib1 must be at least 10% faster than lib2
+expect(result.get('lib1')).toBeFasterThan(result.get('lib2'), {
+  delta: 0.1,
+})
+
+// lib2 must be at least 20% slower than lib1
+expect(result.get('lib2')).toBeSlowerThan(result.get('lib1'), {
+  delta: 0.2,
+})
+```
+
+You can also assert absolute performance using standard matchers:
+
+```ts
+test('parsing is fast enough', async ({ bench }) => {
+  const result = await bench('parse', () => {
+    parse(largeInput)
+  }).run()
+
+  expect(result.throughput.mean).toBeGreaterThan(10_000)
+})
+```
+
+## Retries
+
+Since benchmarks can be noisy, use the `retry` option to automatically retry failing benchmark tests:
+
+```ts
+test('performance comparison', { retry: 3 }, async ({ bench }) => {
+  const result = await bench.compare(
+    bench('lib1', () => { lib1() }),
+    bench('lib2', () => { lib2() }),
+  )
+
+  expect(result.get('lib1')).toBeFasterThan(result.get('lib2'))
+})
+```
+
+## Storing and Replaying Results
+
+Two primitives let you persist benchmark results to disk and compare against them in future runs: the `writeResult` option saves a result, and `bench.from()` reads one back.
+
+### `writeResult`
+
+Pass `writeResult` as a per-bench option to write the result to a JSON file every time the benchmark runs. The path is resolved against the project root:
+
+```ts
+test('parse', async ({ bench }) => {
+  await bench(
+    'parse',
+    { writeResult: './benchmarks/parse.json' },
+    () => parse(largeInput),
+  ).run()
+})
+```
+
+- The benchmark always runs. There is no skip-when-cached behaviour and no CLI flag, the file is overwritten on every successful run.
+- If the function throws, the file is not written.
+- Commit these files alongside your code so reviewers and CI share the same reference points.
+
+::: warning
+If you commit these files, keep in mind that benchmark results vary significantly between environments (developer machines, CI runners, different OSes). Designate a single environment (typically CI) to generate the file, and avoid regenerating it locally.
+:::
+
+### `bench.from()`
+
+`bench.from(name, source)` is a registration that doesn't execute a function. It reads a previously stored result and feeds it into `bench.compare()` (or returns it directly when you call `.run()`).
+
+The source can be a path (relative to the project root) or a function that returns the result data, including a Promise:
+
+```ts
+test('compare against the stored baseline', async ({ bench }) => {
+  const result = await bench.compare(
+    bench(
+      'current',
+      { writeResult: './benchmarks/parse.json' },
+      () => parse(largeInput),
+    ),
+    bench.from('previous', './benchmarks/parse.json'),
+    bench.from('remote', () => fetch('https://path/to/external/file.json').then(r => r.json())),
+  )
+
+  expect(result.get('current')).toBeFasterThan(result.get('previous'))
+})
+```
+
+You can keep historical artifacts for older versions and compare them against the current implementation. Because `bench.from()` never invokes the function that produced the file, the original benchmark code can be deleted once the artifact is committed:
+
+```ts
+test('compare parser versions', async ({ bench }) => {
+  const input = '{"key":"value"}'
+
+  await bench.compare(
+    bench.from('v1', './benchmarks/parse.v1.json'),
+    bench.from('v2', './benchmarks/parse.v2.json'),
+    bench(
+      'current',
+      { writeResult: './benchmarks/parse.current.json' },
+      () => customParser(input),
+    ),
+  )
+})
+```
+
+To produce a new historical artifact, point a fresh `bench()` at that version's implementation, set `writeResult` to a versioned path (`./benchmarks/parse.v3.json`), run it once, then replace the call with `bench.from('v3', './benchmarks/parse.v3.json')`.
+
+To regenerate the baseline on demand, gate the write behind an environment variable so the same test either refreshes the artifact or compares against it:
+
+```ts
+test('compare parser versions', async ({ bench }) => {
+  if (import.meta.env.VITE_WRITE_BENCH) {
+    const baseline = bench('baseline', { writeResult: './my-bench.json' }, () => fn())
+    await baseline.run()
+  }
+  else {
+    const baseline = bench.from('baseline', './my-bench.json')
+    await bench.compare(bench('current', () => fn()), baseline)
+  }
+})
+```
+
+Run `VITE_WRITE_BENCH=1 vitest bench` to refresh the stored result, and `vitest bench` to compare the current implementation against it.
+
+### Per-project artifacts
+
+In a multi-project workspace (different browsers, different runtimes), share one benchmark file across projects by including `${projectName}` in the path. The placeholder is substituted with the current project name at write time:
+
+```ts
+test('cross-project baseline', async ({ bench }) => {
+  await bench(
+    'parse',
+    // eslint-disable-next-line no-template-curly-in-string
+    { perProject: true, writeResult: './benchmarks/parse.${projectName}.json' },
+    () => parse(largeInput),
+  ).run()
+})
+```
+
+Use the same template in `bench.from()` so each project reads its own artifact.
+
+## Stability
+
+Benchmarks are inherently flaky: CPU load, thermal throttling, GC pressure, and background processes all affect results. Vitest takes several steps to minimize this noise:
+
+- **Separate project**: Benchmark files are grouped into their own project based on the [`benchmark.include`](/config/#benchmark-include) pattern. The `bench` fixture is only exposed in files matched by that pattern. Using it inside a regular test file will throw an error.
+- **No concurrency**: Tests within a benchmark file always run sequentially. Benchmark files themselves also run one at a time, never in parallel. This prevents benchmarks from interfering with each other.
+
+To further improve stability:
+
+- Use the [`retry`](#retries) option to automatically rerun flaky benchmark assertions.
+- Use the [`delta`](#asserting-performance) option in `toBeFasterThan` / `toBeSlowerThan` to allow for acceptable variance.
+- Avoid running benchmarks alongside CPU-intensive processes.
+- Close browsers, IDEs, and other applications that compete for CPU time.
+
+### Dead Code Elimination
+
+JavaScript engines can optimize away code that has no observable side effects. If your benchmark function doesn't use its result, the engine may skip the computation entirely, producing misleadingly fast numbers:
+
+```ts
+test('parsing', async ({ bench }) => {
+  // BAD: the engine may eliminate the work
+  await bench('parse', () => {
+    JSON.parse(input)
+  }).run()
+
+  // GOOD: the result is consumed
+  await bench('parse', () => {
+    const result = JSON.parse(input)
+    doSomething(result)
+  }).run()
+})
+```
+
+This applies to all engines (V8, JavaScriptCore, SpiderMonkey) but is especially aggressive in V8's TurboFan and JavaScriptCore's FTL compiler tiers.
+
+### Module Runner Overhead
+
+By default, Vitest runs tests in Node.js using Vite's module runner (configured by [`experimental.viteModuleRunner`](/config/experimental#experimental-vitemodulerunner)). This transforms all module exports into getters, so every access to an imported binding goes through something like `__vite_ssr_module__.value`. In regular tests this overhead is negligible, but in benchmarks where a function is called millions of times, the getter call itself can dominate the measurement.
+
+Vitest will print a warning if it detects excessive getter calls (which you can silence via [`benchmark.suppressExportGetterWarnings`](/config/benchmark#benchmark-suppressexportgetterwarnings)), but you should be aware of this when benchmarking imported functions:
+
+```ts
+import { parse } from './parser.js'
+
+const _parse = parse
+
+test('parsing', async ({ bench }) => {
+  // BAD: every call to `parse` goes through a getter
+  await bench('parse', () => {
+    parse(input)
+  }).run()
+
+  // GOOD: store the reference locally to bypass the getter
+  await bench('parse', () => {
+    _parse(input)
+  }).run()
+})
+```
+
+If you are the library author, the same overhead applies inside the library you are benchmarking: every cross-module call within its source goes through the same getter wrapper. If you are benchmarking your own library, you have two ways to remove this:
+
+**Benchmark the pre-built artifact.** Import the library through its package name (which resolves to its built output) instead of reaching into its source. The built file has already collapsed internal imports into direct references, so Vite's module runner sees a single module with no internal getters:
+
+```ts
+// BAD: every internal call inside the library goes through a getter
+import { parse } from '../src/index.ts'
+
+// GOOD: the published entry has no internal getters
+import { parse } from 'my-library'
+```
+
+If you compare your library against other packages, benchmark the same kind of artifact for every implementation. For workspace packages, make sure the package name resolves to the built output instead of source, for example by externalizing the package in Vite or by importing from `dist`.
+
+**Disable the module runner for the benchmark.** If the benchmark does not need Vite transforms, mocks, or Vitest module interception, disable [`experimental.viteModuleRunner`](/config/experimental#experimental-vitemodulerunner) for the benchmark project so Node runs native ESM directly.
+
+This only affects Node.js mode. Browser mode uses native ESM imports and does not have this overhead.
+
+### Engine-Specific Considerations
+
+#### V8 (Node.js, Chrome)
+
+- **JIT tiering**: V8 compiles functions through multiple optimization tiers (Sparkplug → Maglev → TurboFan). A function may run at different speeds during warmup vs. steady-state. Tinybench handles warmup automatically, but very short benchmark runs may not reach the highest optimization tier.
+- **Deoptimization**: V8 can "bail out" of optimized code mid-benchmark if it encounters unexpected types or shapes. Keep the types consistent in your benchmark function:
+
+  ```ts
+  test('process items', async ({ bench }) => {
+    // BAD: mixed shapes cause deoptimization
+    await bench('process', () => {
+      for (const item of items) {
+        // some items have { name: string }, others have { name: string, id: number }
+        process(item)
+      }
+    }).run()
+
+    // GOOD: consistent object shapes
+    await bench('process', () => {
+      for (const item of items) {
+        // all items have the same shape { name: string, id: number }
+        process(item)
+      }
+    }).run()
+  })
+  ```
+
+- **Garbage collection**: Large allocations inside the benchmark loop add GC noise. If you're measuring computation, pre-allocate data in a `setup` hook rather than inside the benchmarked function:
+
+  ```ts
+  test('sorting', async ({ bench }) => {
+    const original = Array.from({ length: 10000 }, () => Math.random())
+    let data: number[]
+
+    // BAD: allocates a new array every iteration, GC adds noise
+    await bench('sort', () => {
+      const data = Array.from({ length: 10000 }, () => Math.random())
+      data.sort()
+    }).run()
+
+    // GOOD: pre-allocate, copy in beforeEach
+    await bench(
+      'sort',
+      () => { data.sort() },
+      {
+        beforeEach() {
+          data = [...original]
+        },
+      },
+    ).run()
+  })
+  ```
+
+#### JavaScriptCore (Bun, Safari)
+
+- **Different optimization thresholds**: JSC uses its own JIT tiers (LLInt → Baseline → DFG → FTL) with different inlining and optimization heuristics. A benchmark that is fast on V8 may behave very differently on JSC.
+- **Async benchmarks**: Bun's event loop implementation differs from Node.js. If your benchmark involves async operations or timers, results may not be directly comparable across runtimes.
+
+#### Browser
+
+- **Timer resolution**: Browsers may reduce `performance.now()` precision (e.g., to 100μs or even 1ms) as a security mechanism. This makes very fast operations difficult to measure accurately, so increase iterations to compensate:
+
+  ```ts
+  test('fast operations', async ({ bench }) => {
+    await bench.compare(
+      bench('fast-op', () => { fastOp() }),
+      bench('other-op', () => { otherOp() }),
+      {
+        // more iterations help overcome low timer resolution
+        iterations: 1000,
+      },
+    )
+  })
+  ```
+- **Cross-browser differences**: V8 (Chrome), SpiderMonkey (Firefox), and JSC (Safari) optimize different patterns differently. A benchmark that shows one library winning in Chrome may show the opposite in Firefox.

docs/guide/cli-generated.md

@@ -311,7 +311,7 @@ Track coverage of the `node:child_process` and `node:worker_threads` spawned dur
 - **CLI:** `--mode <name>`
 - **Config:** [mode](/config/mode)
 
-Override Vite mode (default: `test` or `benchmark`)
+Override Vite mode (default: `test`)
 
 ### isolate
 

docs/guide/migration.md

@@ -13,6 +13,36 @@ outline: deep
 Vitest 5.0 is currently in beta. This section tracks breaking changes as they are merged and may change before the stable release.
 :::
 
+### Benchmarking API Rewrite
+
+The benchmarking API has been rewritten. `bench` is no longer a top-level import from `vitest`; it is a [test-context fixture](/guide/test-context#bench) accessed from inside a regular `test()`. See the [Benchmarking guide](/guide/benchmarking) for the new API.
+
+Removed, with replacements where applicable:
+
+- **`bench(name, fn)` at module scope**: destructure `bench` from the test context instead.
+
+```ts
+// v4
+import { bench } from 'vitest' // [!code --]
+
+bench('sort', () => { // [!code --]
+  [3, 1, 2].sort() // [!code --]
+}) // [!code --]
+
+// v5
+import { test } from 'vitest' // [!code ++]
+
+test('sort', async ({ bench }) => { // [!code ++]
+  await bench('sort', () => { [3, 1, 2].sort() }).run() // [!code ++]
+}) // [!code ++]
+```
+
+- **`bench.skip`, `bench.only`, `bench.todo`** are removed. Use the regular `test.skip`, `test.only`, `test.todo` on the surrounding `test()` instead.
+- **`benchmark.reporters` / `benchmark.outputFile`** are removed. Benchmark output is now part of the default reporter and the `json` reporter; configure those at the top level via `test.reporters` instead.
+- **`benchmark.compare` config and the `--compare` CLI flag** are removed. Pass [`writeResult`](/guide/benchmarking#storing-and-replaying-results) as a per-bench option to persist a result, and read it back with [`bench.from()`](/guide/benchmarking#bench-from) inside `bench.compare()`.
+- **`benchmark.outputJson` config and the `--outputJson` CLI flag** are removed. Use `--reporter=json --outputFile=<path>` to capture benchmark results; the JSON reporter now includes a `benchmarks` field on each test case.
+- **`Vitest` instance `mode` property** is now always `'test'`. The previous `'benchmark'` value is no longer used; benchmarks run inside a dedicated project of the same `Vitest` instance.
+
 ### Removed `test.sequential`, `describe.sequential`, and `sequential` Options
 
 Vitest 5.0 removes the deprecated `test.sequential`, `describe.sequential`, and `sequential` test options. Use `concurrent: false` when you need a test or suite to opt out of inherited or globally configured concurrency.

docs/guide/snippets/benchmark-per-project.ansi

@@ -0,0 +1,7 @@
+ Cross-Project Benchmark Comparison
+
+  test/parser.bench.ts > regular parsing > parsing
+   name                        hz     min     max    mean     p75     p99    p995    p999     rme  samples
+   webkit (bench)    6,730,808.15  0.0000  1.0000  0.0001  0.0000  0.0000  0.0000  0.0000  ±6.20%  6731808   fastest
+   chromium (bench)  3,841,599.95  0.0000  0.2000  0.0003  0.0000  0.0000  0.0000  0.1000  ±1.96%  3851571
+   firefox (bench)   3,546,066.28  0.0000  5.0000  0.0003  0.0000  0.0000  0.0000  0.0000  ±6.26%  3547062   slowest

docs/guide/snippets/benchmark-table.ansi

@@ -0,0 +1,9 @@
+ ✓  bench  test/basic.bench.ts (1 test) 3446ms
+   ✓ different libraries 3445ms
+     name              hz     min     max    mean     p75     p99    p995    p999     rme  samples
+     lib 1  40,678,348.31  0.0000  0.0002  0.0000  0.0000  0.0000  0.0000  0.0000  ±0.95%    53989   fastest
+     lib 3  39,152,132.23  0.0000  0.0002  0.0000  0.0000  0.0000  0.0000  0.0000  ±0.93%    52080
+     lib 2  38,088,138.97  0.0000  0.0066  0.0000  0.0000  0.0000  0.0000  0.0000  ±1.58%    50618   slowest
+
+ Test Files  1 passed (1)
+      Tests  1 passed (1)

docs/guide/test-context.md

@@ -117,6 +117,29 @@ it('stop request when test times out', async ({ signal }) => {
 }, 2000)
 ```
 
+### `bench` <Version>5.0.0</Version> {#bench}
+
+The `bench` fixture lets you define and run benchmarks inside regular tests. You can measure throughput, compare implementations, and assert relative performance:
+
+```ts
+import { expect, test } from 'vitest'
+
+test('compare parsers', async ({ bench }) => {
+  const result = await bench.compare(
+    bench('JSON.parse', () => {
+      JSON.parse('{"key":"value"}')
+    }),
+    bench('custom parser', () => {
+      customParse('{"key":"value"}')
+    }),
+  )
+
+  expect(result.get('JSON.parse')).toBeFasterThan(result.get('custom parser'))
+})
+```
+
+See the [Benchmarks guide](/guide/benchmarking) for full documentation on comparisons, baselines, and assertion matchers.
+
 ### `onTestFailed`
 
 The [`onTestFailed`](/api/hooks#ontestfailed) hook bound to the current test. This API is useful if you are running tests concurrently and need to have a special handling only for this specific test.

docs/guide/test-tags.md

@@ -233,7 +233,7 @@ If you are using a programmatic API, you can pass down a `tagsFilter` option to
 ```ts
 import { startVitest } from 'vitest/node'
 
-await startVitest('test', [], {
+await startVitest([], {
   tagsFilter: ['frontend and backend'],
 })
 ```

eslint.config.js

@@ -122,6 +122,7 @@ export default antfu(
       'no-self-compare': 'off',
       'import/no-mutable-exports': 'off',
       'no-throw-literal': 'off',
+      'import/no-duplicates': 'off',
     },
   },
   {

packages/browser/src/client/tester/runner.ts

@@ -8,15 +8,15 @@ import type {
   Test,
   TestAnnotation,
   TestArtifact,
+  TestTryOptions,
   VitestRunner,
 } from '@vitest/runner'
 import type { SerializedConfig, TestExecutionMethod, WorkerGlobalState } from 'vitest'
-import type { Traces } from 'vitest/internal/traces'
 import type { VitestBrowserClientMocker } from './mocker'
 import type { CommandsManager } from './tester-utils'
 import { globalChannel, onCancel } from '@vitest/browser/client'
 import { getTestName } from '@vitest/runner/utils'
-import { BenchmarkRunner, recordArtifact, TestRunner } from 'vitest'
+import { recordArtifact, TestRunner } from 'vitest'
 import { page, userEvent } from 'vitest/browser'
 import {
   DecodedMap,
@@ -48,18 +48,16 @@ interface BrowserVitestRunner extends VitestRunner {
 }
 
 export function createBrowserRunner(
-  runnerClass: { new (config: SerializedConfig): VitestRunner },
   mocker: VitestBrowserClientMocker,
   state: WorkerGlobalState,
-  coverageModule: CoverageHandler | null,
+  coverageModule: CoverageHandler,
 ): { new (options: BrowserRunnerOptions): BrowserVitestRunner } {
-  return class BrowserTestRunner extends runnerClass implements VitestRunner {
+  return class BrowserTestRunner extends TestRunner implements VitestRunner {
     public config: SerializedConfig
-    hashMap = browserHashMap
+    public hashMap = browserHashMap
     public sourceMapCache = new Map<string, any>()
     public method = 'run' as TestExecutionMethod
     private commands: CommandsManager
-    private _otel!: Traces
 
     constructor(options: BrowserRunnerOptions) {
       super(options.config)
@@ -75,12 +73,11 @@ export function createBrowserRunner(
 
     private traces = new Map<string, string[]>()
 
-    onBeforeTryTask: VitestRunner['onBeforeTryTask'] = async (...args) => {
+    async onBeforeTryTask(test: Test, options: TestTryOptions) {
       await userEvent.cleanup()
-      await super.onBeforeTryTask?.(...args)
+      super.onBeforeTryTask?.(test, options)
       const trace = this.config.browser.trace
-      const test = args[0]
-      const { retry, repeats } = args[1]
+      const { retry, repeats } = options
       const shouldTrace = trace !== 'off'
         && !(trace === 'on-all-retries' && retry === 0)
         && !(trace === 'on-first-retry' && retry !== 1)
@@ -152,7 +149,7 @@ export function createBrowserRunner(
     }
 
     onAfterRunTask = async (task: Test) => {
-      await super.onAfterRunTask?.(task)
+      super.onAfterRunTask?.(task)
       const trace = this.config.browser.trace
       const traces = this.traces.get(task.id) || []
       if (traces.length) {
@@ -230,10 +227,11 @@ export function createBrowserRunner(
     }
 
     onAfterRunFiles = async (files: File[]) => {
+      super.onAfterRunFiles(files)
+
       const [coverage] = await Promise.all([
-        coverageModule?.takeCoverage?.(),
+        coverageModule.takeCoverage(),
         mocker.invalidate(),
-        super.onAfterRunFiles?.(files),
       ])
 
       if (coverage) {
@@ -357,10 +355,7 @@ export async function initiateRunner(
   if (cachedRunner) {
     return cachedRunner
   }
-  const runnerClass
-    = config.mode === 'test' ? TestRunner : BenchmarkRunner
-
-  const BrowserRunner = createBrowserRunner(runnerClass, mocker, state, {
+  const BrowserRunner = createBrowserRunner(mocker, state, {
     takeCoverage: () =>
       takeCoverageInsideWorker(config.coverage, moduleRunner),
   })
@@ -377,10 +372,10 @@ export async function initiateRunner(
   })
 
   const [diffOptions] = await Promise.all([
-    loadDiffConfig(config, moduleRunner as any),
-    loadSnapshotSerializers(config, moduleRunner as any),
+    loadDiffConfig(config, moduleRunner),
+    loadSnapshotSerializers(config, moduleRunner),
   ])
-  runner.config.diffOptions = diffOptions
+  runner.config._diffOptions = diffOptions
   getWorkerState().onFilterStackTrace = (stack: string) => {
     const stacks = parseStacktrace(stack, {
       getSourceMap(file) {
@@ -400,7 +395,7 @@ async function getTraceMap(file: string, sourceMaps: Map<string, any>) {
   if (!result) {
     return null
   }
-  return new DecodedMap(result as any, file)
+  return new DecodedMap(result, file)
 }
 
 async function updateTestFilesLocations(files: File[], sourceMaps: Map<string, any>) {

packages/browser/src/node/plugin.ts

@@ -168,7 +168,27 @@ export default (parentServer: ParentBrowserProject, base = '/'): Plugin[] => {
         // this plugin can be used in different projects, but all of them
         // have the same `include` pattern, so it doesn't matter which project we use
         const project = parentServer.project
-        const { testFiles: browserTestFiles } = await project.globTestFiles()
+        // only glob benchmarks when a browser-enabled bench project exists in
+        // the workspace — keeps the optimizeDeps entries symmetrical across
+        // the test/bench project clones so the optimizer doesn't re-scan when
+        // the user switches modes
+        const hasBrowserBenchProject = parentServer.vitest.projects.some(p =>
+          p.config.browser.enabled && p.config.benchmark.enabled,
+        )
+        const benchInclude = hasBrowserBenchProject
+          ? project.config.benchmark.include
+          : []
+        const dir = project.config.dir || project.config.root
+        const [{ testFiles: browserTestFiles }, browserBenchFiles] = await Promise.all([
+          project.globTestFiles(),
+          benchInclude.length > 0
+            ? project.globFiles(
+                benchInclude,
+                project.config.benchmark.exclude ?? project.config.exclude,
+                dir,
+              )
+            : [],
+        ])
         const setupFiles = toArray(project.config.setupFiles)
 
         // replace env values - cannot be reassign at runtime
@@ -179,7 +199,7 @@ export default (parentServer: ParentBrowserProject, base = '/'): Plugin[] => {
         }
 
         const entries: string[] = [
-          ...browserTestFiles,
+          ...new Set([...browserTestFiles, ...browserBenchFiles]),
           ...setupFiles,
           resolve(vitestDist, 'index.js'),
           resolve(vitestDist, 'browser.js'),

packages/browser/src/node/rpc.ts

@@ -202,6 +202,23 @@ export function setupBrowserRpc(globalServer: ParentBrowserProject, defaultMocke
 
           return vitest._testRun.recordArtifact(id, artifact)
         },
+        async onTestBenchmark(testId, benchmark) {
+          return vitest._testRun.recordBenchmark(testId, benchmark)
+        },
+        async readBenchmarkResult(relativePath) {
+          checkFileAccess(project.benchmark.resolve(relativePath))
+          return project.benchmark.readResult(relativePath)
+        },
+        async writeBenchmarkResult(relativePath, data) {
+          if (!canWrite(project)) {
+            vitest.logger.error(
+              `[vitest] Cannot write benchmark artifact "${relativePath}" because file writing is disabled. See https://vitest.dev/config/browser/api.`,
+            )
+            return
+          }
+          checkFileAccess(project.benchmark.resolve(relativePath))
+          return project.benchmark.writeResult(relativePath, data)
+        },
         async onTaskUpdate(method, packs, events) {
           if (method === 'collect') {
             vitest.state.updateTasks(packs)

packages/browser/src/types.ts

@@ -1,5 +1,5 @@
 import type { MockedModuleSerialized, ServerIdResolution, ServerMockResolution } from '@vitest/mocker'
-import type { TaskEventPack, TaskResultPack, TestArtifact } from '@vitest/runner'
+import type { BaselineData, TaskEventPack, TaskResultPack, TestArtifact } from '@vitest/runner'
 import type { BirpcReturn } from 'birpc'
 import type {
   AfterSuiteRunMeta,
@@ -8,6 +8,7 @@ import type {
   RunnerTestFile,
   SerializedTestSpecification,
   SnapshotResult,
+  TestBenchmark,
   TestExecutionMethod,
   UserConsoleLog,
 } from 'vitest'
@@ -21,6 +22,9 @@ export interface WebSocketBrowserHandlers {
   onCollected: (method: TestExecutionMethod, files: RunnerTestFile[]) => Promise<void>
   onTaskArtifactRecord: <Artifact extends TestArtifact>(testId: string, artifact: Artifact) => Promise<Artifact>
   onTaskUpdate: (method: TestExecutionMethod, packs: TaskResultPack[], events: TaskEventPack[]) => void
+  onTestBenchmark: (testId: string, benchmark: TestBenchmark) => void
+  readBenchmarkResult: (relativePath: string) => Promise<BaselineData | null>
+  writeBenchmarkResult: (relativePath: string, data: BaselineData) => Promise<void>
   onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void
   cancelCurrentRun: (reason: CancelReason) => void
   getCountOfFailedTests: () => number

packages/runner/package.json

@@ -48,6 +48,7 @@
   },
   "dependencies": {
     "@vitest/utils": "workspace:*",
-    "pathe": "catalog:"
+    "pathe": "catalog:",
+    "tinybench": "catalog:"
   }
 }

packages/runner/src/collect.ts

@@ -112,8 +112,8 @@ export async function collectTests(
         }
         catch (e) {
           const errors = e instanceof AggregateError
-            ? e.errors.map(e => processError(e, runner.config.diffOptions))
-            : [processError(e, runner.config.diffOptions)]
+            ? e.errors.map(e => processError(e, runner.config._diffOptions))
+            : [processError(e, runner.config._diffOptions)]
           file.result = {
             state: 'fail',
             errors,

packages/runner/src/fixture.ts

@@ -32,6 +32,7 @@ export class TestFixtures {
     'onTestFinished',
     'skip',
     'annotate',
+    'bench',
   ] satisfies (keyof TestContext)[]
 
   private static _fixtureOptionKeys: string[] = ['auto', 'injected', 'scope']

packages/runner/src/run.ts

@@ -146,7 +146,7 @@ async function callTestHooks(
       await Promise.all(hooks.map(fn => limitMaxConcurrency(() => fn(test.context))))
     }
     catch (e) {
-      failTask(test.result!, e, runner.config.diffOptions)
+      failTask(test.result!, e, runner.config._diffOptions)
     }
   }
   else {
@@ -155,7 +155,7 @@ async function callTestHooks(
         await limitMaxConcurrency(() => fn(test.context))
       }
       catch (e) {
-        failTask(test.result!, e, runner.config.diffOptions)
+        failTask(test.result!, e, runner.config._diffOptions)
       }
     }
   }
@@ -662,14 +662,14 @@ export async function runTest(test: Test, runner: VitestRunner): Promise<void> {
           }
         }
         catch (e) {
-          failTask(test.result!, e, runner.config.diffOptions)
+          failTask(test.result!, e, runner.config._diffOptions)
         }
 
         try {
           await runner.onTaskFinished?.(test)
         }
         catch (e) {
-          failTask(test.result!, e, runner.config.diffOptions)
+          failTask(test.result!, e, runner.config._diffOptions)
         }
 
         try {
@@ -685,7 +685,7 @@ export async function runTest(test: Test, runner: VitestRunner): Promise<void> {
           await callFixtureCleanupFrom(test.context, fixtureCheckpoint)
         }
         catch (e) {
-          failTask(test.result!, e, runner.config.diffOptions)
+          failTask(test.result!, e, runner.config._diffOptions)
         }
 
         if (test.onFinished?.length) {
@@ -709,7 +709,7 @@ export async function runTest(test: Test, runner: VitestRunner): Promise<void> {
           repeats: repeatCount,
         })
       }).catch((error) => {
-        failTask(test.result!, error, runner.config.diffOptions)
+        failTask(test.result!, error, runner.config._diffOptions)
       })
 
       // Clean up fixtures that were created for aroundEach (before the checkpoint)
@@ -718,7 +718,7 @@ export async function runTest(test: Test, runner: VitestRunner): Promise<void> {
         await callFixtureCleanup(test.context)
       }
       catch (e) {
-        failTask(test.result!, e, runner.config.diffOptions)
+        failTask(test.result!, e, runner.config._diffOptions)
       }
 
       // skipped with new PendingError
@@ -886,7 +886,7 @@ export async function runSuite(suite: Suite, runner: VitestRunner): Promise<void
             ))
           }
           catch (e) {
-            failTask(suite.result!, e, runner.config.diffOptions)
+            failTask(suite.result!, e, runner.config._diffOptions)
             markTasksAsSkipped(suite, runner)
             return
           }
@@ -934,7 +934,7 @@ export async function runSuite(suite: Suite, runner: VitestRunner): Promise<void
             }
           }
           catch (e) {
-            failTask(suite.result!, e, runner.config.diffOptions)
+            failTask(suite.result!, e, runner.config._diffOptions)
           }
         }
       })
@@ -944,7 +944,7 @@ export async function runSuite(suite: Suite, runner: VitestRunner): Promise<void
       if (!suiteRan) {
         markTasksAsSkipped(suite, runner)
       }
-      failTask(suite.result!, e, runner.config.diffOptions)
+      failTask(suite.result!, e, runner.config._diffOptions)
     }
 
     if (suite.mode === 'run' || suite.mode === 'queued') {

packages/runner/src/suite.ts

@@ -398,6 +398,7 @@ function createSuiteCollector(
       meta: testMeta,
       annotations: [],
       artifacts: [],
+      benchmarks: [],
       tags: testTags,
     }
     const handler = options.handler

packages/runner/src/types.ts

@@ -2,6 +2,7 @@ export type {
   CancelReason,
   FileSpecification,
   TestTagDefinition,
+  TestTryOptions,
   VitestRunner,
   VitestRunnerConfig,
   VitestRunnerConstructor,
@@ -12,6 +13,7 @@ export type {
   AfterEachListener,
   AroundAllListener,
   AroundEachListener,
+  BaselineData,
   BeforeAllListener,
   BeforeEachListener,
   BrowserTraceArtifact,
@@ -40,6 +42,7 @@ export type {
   Task,
   TaskBase,
   TaskCustomOptions,
+  TaskEventData,
   TaskEventPack,
   TaskHook,
   TaskMeta,
@@ -58,6 +61,8 @@ export type {
   TestArtifactLocation,
   TestArtifactRegistry,
   TestAttachment,
+  TestBenchmark,
+  TestBenchmarkTask,
   TestContext,
   TestFunction,
   TestOptions,

packages/runner/src/types/runner.ts

@@ -42,10 +42,14 @@ export interface VitestRunnerConfig {
   hookTimeout: number
   retry: SerializableRetry
   includeTaskLocation: boolean | undefined
-  diffOptions?: DiffOptions
   tags: TestTagDefinition[]
   tagsFilter: string[] | undefined
   strictTags: boolean
+
+  /**
+   * @internal
+   */
+  _diffOptions?: DiffOptions
   mergeReportsLabel: string | undefined
 }
 
@@ -93,6 +97,11 @@ export type CancelReason
     | 'test-failure'
     | (string & Record<string, never>)
 
+export interface TestTryOptions {
+  retry: number
+  repeats: number
+}
+
 export interface VitestRunner {
   /**
    * First thing that's getting called before actually collecting and running tests.
@@ -123,7 +132,7 @@ export interface VitestRunner {
    */
   onBeforeTryTask?: (
     test: Test,
-    options: { retry: number; repeats: number },
+    options: TestTryOptions,
   ) => unknown
   /**
    * When the task has finished running, but before cleanup hooks are called
@@ -138,7 +147,7 @@ export interface VitestRunner {
    */
   onAfterTryTask?: (
     test: Test,
-    options: { retry: number; repeats: number },
+    options: TestTryOptions,
   ) => unknown
   /**
    * Called after the retry resolution happened. Unlike `onAfterTryTask`, the test now has a new state.
@@ -146,7 +155,7 @@ export interface VitestRunner {
    */
   onAfterRetryTask?: (
     test: Test,
-    options: { retry: number; repeats: number },
+    options: TestTryOptions,
   ) => unknown
 
   /**

packages/runner/src/types/tasks.ts

@@ -1,4 +1,5 @@
 import type { Awaitable, TestError } from '@vitest/utils'
+import type { Statistics } from 'tinybench'
 import type { TestFixtures } from '../fixture'
 import type { afterAll, afterEach, aroundAll, aroundEach, beforeAll, beforeEach } from '../hooks'
 import type { kChainableContext, TypedChainableFunction } from '../utils/chain'
@@ -340,6 +341,43 @@ export interface Test<ExtraContext = object> extends TaskPopulated {
    */
   artifacts: TestArtifact[]
   fullTestName: string
+  /**
+   * An array of benchmark results generated by `context.bench` function.
+   * The benchmark is added only after `bench().run` or `bench.compare` is resolved.
+   *
+   * @experimental
+   */
+  benchmarks: TestBenchmark[]
+}
+
+export interface TestBenchmark {
+  name: string
+  tasks: TestBenchmarkTask[]
+}
+
+export type TestBenchmarkStatistics = Statistics
+
+export interface BaselineData {
+  latency: TestBenchmarkStatistics
+  throughput: TestBenchmarkStatistics
+  period: number
+  totalTime: number
+}
+
+export interface TestBenchmarkTask {
+  name: string
+  latency: TestBenchmarkStatistics
+  throughput: TestBenchmarkStatistics
+  period: number
+  totalTime: number
+  rank: number
+  perProject?: boolean
+  /**
+   * `true` when the task was produced by `bench.from()` rather than by
+   * executing a function. Reporters can use this to render the row as a
+   * static reference (no margin of error, no samples).
+   */
+  fromStore?: boolean
 }
 
 export type Task = Test | Suite | File

packages/runner/tsconfig.json

@@ -4,6 +4,7 @@
     "target": "ESNext",
     "module": "ESNext",
     "moduleResolution": "Bundler",
+    "types": ["node"],
     "isolatedDeclarations": true
   },
   "include": ["./src/**/*.ts"],

packages/vitest/package.json

@@ -180,7 +180,7 @@
     "pathe": "catalog:",
     "picomatch": "^4.0.3",
     "std-env": "catalog:",
-    "tinybench": "^2.9.0",
+    "tinybench": "catalog:",
     "tinyexec": "^1.0.2",
     "tinyglobby": "catalog:",
     "tinyrainbow": "catalog:",

packages/vitest/src/defaults.ts

@@ -14,14 +14,16 @@ export const defaultExclude: string[] = [
   '**/node_modules/**',
   '**/.git/**',
 ]
-export const benchmarkConfigDefaults: Required<
-  Omit<BenchmarkUserOptions, 'outputFile' | 'compare' | 'outputJson'>
-> = {
+export const benchmarkConfigDefaults: Required<BenchmarkUserOptions> = {
+  enabled: false,
   include: ['**/*.{bench,benchmark}.?(c|m)[jt]s?(x)'],
   exclude: defaultExclude,
   includeSource: [],
-  reporters: ['default'],
-  includeSamples: false,
+  retainSamples: false,
+  suppressExportGetterWarnings: false,
+  // Populated automatically when Vitest clones the parent project; the default
+  // here applies to the (unused) raw config that's never run as a benchmark.
+  projectName: '',
 }
 
 // These are the generic defaults for coverage. Providers may also set some provider specific defaults.

packages/vitest/src/integrations/chai/bench.ts

@@ -0,0 +1,83 @@
+import type { MatchersObject } from '@vitest/expect'
+import type { BenchResult } from '../../runtime/benchmark'
+
+function isBenchResult(value: unknown): value is BenchResult {
+  return (
+    typeof value === 'object'
+    && value !== null
+    && 'latency' in value
+    && typeof (value as any).latency?.mean === 'number'
+  )
+}
+
+function formatOps(ops: number): string {
+  return ops.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })
+}
+
+export const benchMatchers: MatchersObject = {
+  toBeFasterThan(actual: unknown, expected: unknown, options?: { delta?: number }) {
+    const { matcherHint, RECEIVED_COLOR, EXPECTED_COLOR } = this.utils
+    const delta = options?.delta ?? 0
+
+    if (!isBenchResult(actual)) {
+      throw new TypeError(
+        `${matcherHint('.toBeFasterThan')} expects the actual value to be a benchmark result.`,
+      )
+    }
+    if (!isBenchResult(expected)) {
+      throw new TypeError(
+        `${matcherHint('.toBeFasterThan')} expects the expected value to be a benchmark result.`,
+      )
+    }
+
+    const threshold = expected.latency.mean * (1 - delta)
+    const pass = actual.latency.mean < threshold
+
+    return {
+      pass,
+      message: () => {
+        const relation = ((actual.latency.mean - expected.latency.mean) / expected.latency.mean * 100).toFixed(2)
+        return pass
+          ? `${matcherHint('.not.toBeFasterThan')}\n\nExpected to not be faster, but was ${Math.abs(Number(relation))}% faster.\n\n`
+          + `Received: ${RECEIVED_COLOR(formatOps(actual.throughput.mean))} ops/sec\n`
+          + `Expected: ${EXPECTED_COLOR(formatOps(expected.throughput.mean))} ops/sec\n`
+          : `${matcherHint('.toBeFasterThan')}\n\nExpected to be faster${delta > 0 ? ` by at least ${(delta * 100).toFixed(0)}%` : ''}, but was ${Number(relation) > 0 ? `${relation}% slower` : `only ${Math.abs(Number(relation))}% faster`}.\n\n`
+            + `Received: ${RECEIVED_COLOR(formatOps(actual.throughput.mean))} ops/sec\n`
+            + `Expected: ${EXPECTED_COLOR(formatOps(expected.throughput.mean))} ops/sec\n`
+      },
+    }
+  },
+
+  toBeSlowerThan(actual: unknown, expected: unknown, options?: { delta?: number }) {
+    const { matcherHint, RECEIVED_COLOR, EXPECTED_COLOR } = this.utils
+    const delta = options?.delta ?? 0
+
+    if (!isBenchResult(actual)) {
+      throw new TypeError(
+        `${matcherHint('.toBeSlowerThan')} expects the actual value to be a benchmark result.`,
+      )
+    }
+    if (!isBenchResult(expected)) {
+      throw new TypeError(
+        `${matcherHint('.toBeSlowerThan')} expects the expected value to be a benchmark result.`,
+      )
+    }
+
+    const threshold = expected.latency.mean * (1 + delta)
+    const pass = actual.latency.mean > threshold
+
+    return {
+      pass,
+      message: () => {
+        const relation = ((actual.latency.mean - expected.latency.mean) / expected.latency.mean * 100).toFixed(2)
+        return pass
+          ? `${matcherHint('.not.toBeSlowerThan')}\n\nExpected to not be slower, but was ${relation}% slower.\n\n`
+          + `Received: ${RECEIVED_COLOR(formatOps(actual.throughput.mean))} ops/sec\n`
+          + `Expected: ${EXPECTED_COLOR(formatOps(expected.throughput.mean))} ops/sec\n`
+          : `${matcherHint('.toBeSlowerThan')}\n\nExpected to be slower${delta > 0 ? ` by at least ${(delta * 100).toFixed(0)}%` : ''}, but was ${Number(relation) < 0 ? `${Math.abs(Number(relation))}% faster` : `only ${relation}% slower`}.\n\n`
+            + `Received: ${RECEIVED_COLOR(formatOps(actual.throughput.mean))} ops/sec\n`
+            + `Expected: ${EXPECTED_COLOR(formatOps(expected.throughput.mean))} ops/sec\n`
+      },
+    }
+  },
+}

packages/vitest/src/integrations/chai/index.ts

@@ -11,6 +11,7 @@ import {
 } from '@vitest/expect'
 import { getCurrentTest } from '@vitest/runner'
 import { getWorkerState } from '../../runtime/utils'
+import { benchMatchers } from './bench'
 import { createExpectPoll } from './poll'
 import './setup'
 
@@ -108,6 +109,7 @@ export function createExpect(test?: Test | TaskPopulated): ExpectStatic {
   chai.util.addMethod(expect, 'hasAssertions', hasAssertions)
 
   expect.extend(customMatchers)
+  expect.extend(benchMatchers)
 
   return expect
 }

packages/vitest/src/node/ast-collect.ts

@@ -487,6 +487,7 @@ function createFileTask(
         timeout: 0,
         annotations: [],
         artifacts: [],
+        benchmarks: [],
         tags: taskTags,
       }
       definition.task = task

packages/vitest/src/node/benchmark.ts

@@ -0,0 +1,42 @@
+import type { BaselineData } from '@vitest/runner'
+import type { TestProject } from './project'
+import { existsSync } from 'node:fs'
+import { mkdir, readFile, writeFile } from 'node:fs/promises'
+import { dirname, isAbsolute, resolve } from 'pathe'
+
+export class BenchmarkManager {
+  constructor(private project: TestProject) {}
+
+  // Resolve a user-supplied path against the project root. Reject paths that
+  // escape the project root: `bench.from()` accepts arbitrary input, and we
+  // never want a benchmark file to be able to read or clobber files outside
+  // the workspace.
+  public resolve(relativePath: string): string {
+    const root = this.project.config.root
+    const absolute = isAbsolute(relativePath)
+      ? resolve(relativePath)
+      : resolve(root, relativePath)
+    const rootWithSep = root.endsWith('/') ? root : `${root}/`
+    if (absolute !== root && !absolute.startsWith(rootWithSep)) {
+      throw new Error(
+        `Benchmark artifact path "${relativePath}" resolves outside the project root (${root}). `
+        + `Paths passed to \`writeResult\` and \`bench.from()\` must point inside the project.`,
+      )
+    }
+    return absolute
+  }
+
+  async readResult(relativePath: string): Promise<BaselineData | null> {
+    const path = this.resolve(relativePath)
+    if (!existsSync(path)) {
+      return null
+    }
+    return JSON.parse(await readFile(path, 'utf-8')) as BaselineData
+  }
+
+  async writeResult(relativePath: string, data: BaselineData): Promise<void> {
+    const absolute = this.resolve(relativePath)
+    await mkdir(dirname(absolute), { recursive: true })
+    await writeFile(absolute, `${JSON.stringify(data, null, 2)}\n`, 'utf-8')
+  }
+}

packages/vitest/src/node/cli/cac.ts

@@ -1,14 +1,13 @@
 import type { CAC, Command } from 'cac'
-import type { VitestRunMode } from '../types/config'
 import type { CliOptions } from './cli-api'
 import type { CLIOption, CLIOptions as CLIOptionsConfig } from './cli-config'
 import { toArray } from '@vitest/utils/helpers'
 import cac from 'cac'
 import { normalize } from 'pathe'
-import c, { disableDefaultColors } from 'tinyrainbow'
+import { disableDefaultColors } from 'tinyrainbow'
 import { version } from '../../../package.json' with { type: 'json' }
 import { isAgent, isForceColor } from '../../utils/env'
-import { benchCliOptionsConfig, cliOptionsConfig, collectCliOptionsConfig } from './cli-config'
+import { cliOptionsConfig, collectCliOptionsConfig } from './cli-config'
 import { setupTabCompletions } from './completions'
 
 function addCommand(cli: CAC | Command, name: string, option: CLIOption<any>) {
@@ -179,12 +178,9 @@ export function createCLI(options: CliParseOptions = {}): CAC {
     .command('dev [...filters]', undefined, options)
     .action(watch)
 
-  addCliOptions(
-    cli
-      .command('bench [...filters]', undefined, options)
-      .action(benchmark),
-    benchCliOptionsConfig,
-  )
+  cli
+    .command('bench [...filters]', undefined, options)
+    .action(benchmark)
 
   cli
     .command('init <project>', undefined, options)
@@ -193,13 +189,13 @@ export function createCLI(options: CliParseOptions = {}): CAC {
   addCliOptions(
     cli
       .command('list [...filters]', undefined, options)
-      .action((filters, options) => collect('test', filters, options)),
+      .action((filters, options) => collect(filters, options)),
     collectCliOptionsConfig,
   )
 
   cli
     .command('[...filters]', undefined, options)
-    .action((filters, options) => start('test', filters, options))
+    .action((filters, options) => start(filters, options))
 
   setupTabCompletions(cli)
   return cli
@@ -263,24 +259,26 @@ export function parseCLI(argv: string | string[], config: CliParseOptions = {}):
 async function runRelated(relatedFiles: string[] | string, argv: CliOptions): Promise<void> {
   argv.related = relatedFiles
   argv.passWithNoTests ??= true
-  await start('test', [], argv)
+  await start([], argv)
 }
 
 async function watch(cliFilters: string[], options: CliOptions): Promise<void> {
   options.watch = true
-  await start('test', cliFilters, options)
+  await start(cliFilters, options)
 }
 
 async function run(cliFilters: string[], options: CliOptions): Promise<void> {
   // "vitest run --watch" should still be watch mode
   options.run = !options.watch
 
-  await start('test', cliFilters, options)
+  await start(cliFilters, options)
 }
 
 async function benchmark(cliFilters: string[], options: CliOptions): Promise<void> {
-  console.warn(c.yellow('Benchmarking is an experimental feature.\nBreaking changes might not follow SemVer, please pin Vitest\'s version when using it.'))
-  await start('benchmark', cliFilters, options)
+  options.benchmarkOnly = true
+  options.coverage ??= {}
+  options.coverage.enabled = false
+  await start(cliFilters, options)
 }
 
 function normalizeCliOptions(cliFilters: string[], argv: CliOptions): CliOptions {
@@ -303,10 +301,10 @@ function normalizeCliOptions(cliFilters: string[], argv: CliOptions): CliOptions
   return argv
 }
 
-async function start(mode: VitestRunMode, cliFilters: string[], options: CliOptions): Promise<void> {
+async function start(cliFilters: string[], options: CliOptions): Promise<void> {
   try {
     const { startVitest } = await import('./cli-api')
-    const ctx = await startVitest(mode, cliFilters.map(normalize), normalizeCliOptions(cliFilters, options))
+    const ctx = await startVitest(cliFilters.map(normalize), normalizeCliOptions(cliFilters, options))
     if (!ctx.shouldKeepServer()) {
       await ctx.exit()
     }
@@ -335,10 +333,10 @@ async function init(project: string) {
   await create()
 }
 
-async function collect(mode: VitestRunMode, cliFilters: string[], options: CliOptions): Promise<void> {
+async function collect(cliFilters: string[], options: CliOptions): Promise<void> {
   try {
     const { prepareVitest, processCollected, outputFileList } = await import('./cli-api')
-    const ctx = await prepareVitest(mode, {
+    const ctx = await prepareVitest({
       ...normalizeCliOptions(cliFilters, options),
       watch: false,
       run: true,

packages/vitest/src/node/cli/cli-api.ts

@@ -46,6 +46,13 @@ export interface CliOptions extends UserConfig {
    * @experimental
    */
   configLoader?: ViteInlineConfig extends { configLoader?: infer T } ? T : never
+
+  /**
+   * Only run benchmark projects, filtering out all other projects.
+   * Set automatically by `vitest bench`.
+   * @internal
+   */
+  benchmarkOnly?: boolean
 }
 
 /**
@@ -54,23 +61,54 @@ export interface CliOptions extends UserConfig {
  * Returns a Vitest instance if initialized successfully.
  */
 export async function startVitest(
-  mode: VitestRunMode,
-  cliFilters: string[] = [],
-  options: CliOptions = {},
+  cliFilters?: string[],
+  options?: CliOptions,
   viteOverrides?: ViteUserConfig,
   vitestOptions?: VitestOptions,
+): Promise<Vitest>
+/**
+ * @deprecated The `mode` argument is no longer used. Use `startVitest(cliFilters?, options?, viteOverrides?, vitestOptions?)` instead.
+ */
+export async function startVitest(
+  mode: VitestRunMode,
+  cliFilters?: string[],
+  options?: CliOptions,
+  viteOverrides?: ViteUserConfig,
+  vitestOptions?: VitestOptions,
+): Promise<Vitest>
+export async function startVitest(
+  modeOrCliFilters?: VitestRunMode | string[],
+  cliFiltersOrOptions?: string[] | CliOptions,
+  optionsOrViteOverrides?: CliOptions | ViteUserConfig,
+  viteOverridesOrVitestOptions?: ViteUserConfig | VitestOptions,
+  maybeVitestOptions?: VitestOptions,
 ): Promise<Vitest> {
+  let cliFilters: string[]
+  let options: CliOptions
+  let viteOverrides: ViteUserConfig | undefined
+  let vitestOptions: VitestOptions | undefined
+  if (typeof modeOrCliFilters === 'string') {
+    cliFilters = (cliFiltersOrOptions as string[] | undefined) ?? []
+    options = (optionsOrViteOverrides as CliOptions | undefined) ?? {}
+    viteOverrides = viteOverridesOrVitestOptions as ViteUserConfig | undefined
+    vitestOptions = maybeVitestOptions
+  }
+  else {
+    cliFilters = modeOrCliFilters ?? []
+    options = (cliFiltersOrOptions as CliOptions | undefined) ?? {}
+    viteOverrides = optionsOrViteOverrides as ViteUserConfig | undefined
+    vitestOptions = viteOverridesOrVitestOptions as VitestOptions | undefined
+  }
   const root = resolve(options.root || process.cwd())
 
   const ctx = await prepareVitest(
-    mode,
     options,
     viteOverrides,
     vitestOptions,
     cliFilters,
   )
 
-  if (mode === 'test' && ctx._coverageOptions.enabled) {
+  if (ctx._coverageOptions.enabled) {
     const provider = ctx._coverageOptions.provider || 'v8'
     const requiredPackages = CoverageProviderMap[provider]
 
@@ -151,12 +189,44 @@ export async function startVitest(
 }
 
 export async function prepareVitest(
-  mode: VitestRunMode,
-  options: CliOptions = {},
+  options?: CliOptions,
   viteOverrides?: ViteUserConfig,
   vitestOptions?: VitestOptions,
   cliFilters?: string[],
+): Promise<Vitest>
+/**
+ * @deprecated The `mode` argument is no longer used. Use `prepareVitest(options?, viteOverrides?, vitestOptions?, cliFilters?)` instead.
+ */
+export async function prepareVitest(
+  mode: VitestRunMode,
+  options?: CliOptions,
+  viteOverrides?: ViteUserConfig,
+  vitestOptions?: VitestOptions,
+  cliFilters?: string[],
+): Promise<Vitest>
+export async function prepareVitest(
+  modeOrOptions?: VitestRunMode | CliOptions,
+  optionsOrViteOverrides?: CliOptions | ViteUserConfig,
+  viteOverridesOrVitestOptions?: ViteUserConfig | VitestOptions,
+  vitestOptionsOrCliFilters?: VitestOptions | string[],
+  maybeCliFilters?: string[],
 ): Promise<Vitest> {
+  let options: CliOptions
+  let viteOverrides: ViteUserConfig | undefined
+  let vitestOptions: VitestOptions | undefined
+  let cliFilters: string[] | undefined
+  if (typeof modeOrOptions === 'string') {
+    options = (optionsOrViteOverrides as CliOptions | undefined) ?? {}
+    viteOverrides = viteOverridesOrVitestOptions as ViteUserConfig | undefined
+    vitestOptions = vitestOptionsOrCliFilters as VitestOptions | undefined
+    cliFilters = maybeCliFilters
+  }
+  else {
+    options = modeOrOptions ?? {}
+    viteOverrides = optionsOrViteOverrides as ViteUserConfig | undefined
+    vitestOptions = viteOverridesOrVitestOptions as VitestOptions | undefined
+    cliFilters = vitestOptionsOrCliFilters as string[] | undefined
+  }
   process.env.TEST = 'true'
   process.env.VITEST = 'true'
   process.env.NODE_ENV ??= 'test'
@@ -172,7 +242,7 @@ export async function prepareVitest(
   // this shouldn't affect _application root_ that can be changed inside config
   const root = resolve(options.root || process.cwd())
 
-  const ctx = await createVitest(mode, options, viteOverrides, vitestOptions)
+  const ctx = await createVitest(options, viteOverrides, vitestOptions)
 
   const environmentPackage = getEnvPackageName(ctx.config.environment)
 

packages/vitest/src/node/cli/cli-config.ts

@@ -332,7 +332,7 @@ export const cliOptionsConfig: VitestCLIOptions = {
     },
   },
   mode: {
-    description: 'Override Vite mode (default: `test` or `benchmark`)',
+    description: 'Override Vite mode (default: `test`)',
     argument: '<name>',
   },
   isolate: {
@@ -976,8 +976,6 @@ export const cliOptionsConfig: VitestCLIOptions = {
   deps: null,
   name: null,
   snapshotEnvironment: null,
-  compare: null,
-  outputJson: null,
   json: null,
   provide: null,
   filesOnly: null,
@@ -986,23 +984,10 @@ export const cliOptionsConfig: VitestCLIOptions = {
   projects: null,
   watchTriggerPatterns: null,
   tags: null,
+  benchmarkOnly: null,
   taskTitleValueFormatTruncate: null,
 }
 
-export const benchCliOptionsConfig: Pick<
-  VitestCLIOptions,
-  'compare' | 'outputJson'
-> = {
-  compare: {
-    description: 'Benchmark output file to compare against',
-    argument: '<filename>',
-  },
-  outputJson: {
-    description: 'Benchmark output file',
-    argument: '<filename>',
-  },
-}
-
 export const collectCliOptionsConfig: VitestCLIOptions = {
   ...cliOptionsConfig,
   json: {

packages/vitest/src/node/config/resolveConfig.ts

@@ -1,7 +1,6 @@
 import type { ResolvedConfig as ResolvedViteConfig } from 'vite'
 import type { Vitest } from '../core'
 import type { Logger } from '../logger'
-import type { BenchmarkBuiltinReporters } from '../reporters'
 import type { ResolvedBrowserOptions } from '../types/browser'
 import type {
   ApiConfig,
@@ -155,7 +154,6 @@ export function resolveConfig(
   options: UserConfig,
   viteConfig: ResolvedViteConfig,
 ): ResolvedConfig {
-  const mode = vitest.mode
   const logger = vitest.logger
   if (options.dom) {
     if (
@@ -178,9 +176,10 @@ export function resolveConfig(
     ...configDefaults,
     ...options,
     root: viteConfig.root,
-    mode,
   } as any as ResolvedConfig
 
+  resolved.mode ??= viteConfig.mode ?? 'test'
+
   if (resolved.retry && typeof resolved.retry === 'object' && typeof resolved.retry.condition === 'function') {
     logger.console.warn(
       c.yellow('Warning: retry.condition function cannot be used inside a config file. '
@@ -245,6 +244,11 @@ export function resolveConfig(
     throw new Error(`Looks like you set "test.environment" to "browser". To enable Browser Mode, use "test.browser.enabled" instead.`)
   }
 
+  resolved.benchmark = {
+    ...benchmarkConfigDefaults,
+    ...resolved.benchmark,
+  }
+
   const inspector = resolved.inspect || resolved.inspectBrk
 
   resolved.inspector = {
@@ -295,8 +299,7 @@ export function resolveConfig(
     resolved.maxWorkers = resolveInlineWorkerOption(resolved.maxWorkers)
   }
 
-  // run benchmark sequentially by default
-  const fileParallelism = options.fileParallelism ?? mode !== 'benchmark'
+  const fileParallelism = options.fileParallelism ?? true
 
   if (!fileParallelism) {
     // ignore user config, parallelism cannot be implemented without limiting workers
@@ -634,44 +637,6 @@ export function resolveConfig(
     resolved.maxWorkers = Number.parseInt(process.env.VITEST_MAX_WORKERS)
   }
 
-  if (mode === 'benchmark') {
-    resolved.benchmark = {
-      ...benchmarkConfigDefaults,
-      ...resolved.benchmark,
-    }
-    // override test config
-    resolved.coverage.enabled = false
-    resolved.typecheck.enabled = false
-    resolved.include = resolved.benchmark.include
-    resolved.exclude = resolved.benchmark.exclude
-    resolved.includeSource = resolved.benchmark.includeSource
-    const reporters = Array.from(
-      new Set<BenchmarkBuiltinReporters>([
-        ...toArray(resolved.benchmark.reporters),
-        // @ts-expect-error reporter is CLI flag
-        ...toArray(options.reporter),
-      ]),
-    ).filter(Boolean)
-    if (reporters.length) {
-      resolved.benchmark.reporters = reporters
-    }
-    else {
-      resolved.benchmark.reporters = ['default']
-    }
-
-    if (options.outputFile) {
-      resolved.benchmark.outputFile = options.outputFile
-    }
-
-    // --compare from cli
-    if (options.compare) {
-      resolved.benchmark.compare = options.compare
-    }
-    if (options.outputJson) {
-      resolved.benchmark.outputJson = options.outputJson
-    }
-  }
-
   if (typeof resolved.diff === 'string') {
     resolved.diff = resolvePath(resolved.diff, resolved.root)
     resolved.forceRerunTriggers.push(resolved.diff)
@@ -729,39 +694,37 @@ export function resolveConfig(
     }
   }
 
-  if (mode !== 'benchmark') {
-    // @ts-expect-error "reporter" is from CLI, should be absolute to the running directory
-    // it is passed down as "vitest --reporter ../reporter.js"
-    const reportersFromCLI = resolved.reporter
+  // @ts-expect-error "reporter" is from CLI, should be absolute to the running directory
+  // it is passed down as "vitest --reporter ../reporter.js"
+  const reportersFromCLI = resolved.reporter
 
-    const cliReporters = toArray(reportersFromCLI || []).map(
-      (reporter: string) => {
-        // ./reporter.js || ../reporter.js, but not .reporters/reporter.js
-        if (/^\.\.?\//.test(reporter)) {
-          return resolve(process.cwd(), reporter)
-        }
-        return reporter
-      },
-    )
+  const cliReporters = toArray(reportersFromCLI || []).map(
+    (reporter: string) => {
+      // ./reporter.js || ../reporter.js, but not .reporters/reporter.js
+      if (/^\.\.?\//.test(reporter)) {
+        return resolve(process.cwd(), reporter)
+      }
+      return reporter
+    },
+  )
 
-    if (cliReporters.length) {
-      // When CLI reporters are specified, preserve options from config file
-      const configReportersMap = new Map<string, Record<string, unknown>>()
+  if (cliReporters.length) {
+    // When CLI reporters are specified, preserve options from config file
+    const configReportersMap = new Map<string, Record<string, unknown>>()
 
-      // Build a map of reporter names to their options from the config
-      for (const reporter of resolved.reporters) {
-        if (Array.isArray(reporter)) {
-          const [reporterName, reporterOptions] = reporter
-          if (typeof reporterName === 'string') {
-            configReportersMap.set(reporterName, reporterOptions as Record<string, unknown>)
-          }
+    // Build a map of reporter names to their options from the config
+    for (const reporter of resolved.reporters) {
+      if (Array.isArray(reporter)) {
+        const [reporterName, reporterOptions] = reporter
+        if (typeof reporterName === 'string') {
+          configReportersMap.set(reporterName, reporterOptions as Record<string, unknown>)
         }
       }
-
-      resolved.reporters = Array.from(new Set(toArray(cliReporters)))
-        .filter(Boolean)
-        .map(reporter => [reporter, configReportersMap.get(reporter) || {}])
     }
+
+    resolved.reporters = Array.from(new Set(toArray(cliReporters)))
+      .filter(Boolean)
+      .map(reporter => [reporter, configReportersMap.get(reporter) || {}])
   }
 
   resolved.mergeReportsLabel = process.env.VITEST_BLOB_LABEL
@@ -842,7 +805,7 @@ export function resolveConfig(
   }
   resolved.browser.isolate ??= resolved.isolate ?? true
   resolved.browser.fileParallelism
-    ??= options.fileParallelism ?? mode !== 'benchmark'
+    ??= options.fileParallelism ?? true
   // disable in headless mode by default, and if CI is detected
   resolved.browser.ui ??= resolved.browser.headless === true ? false : !isCI
   resolved.browser.commands ??= {}

packages/vitest/src/node/config/serializeConfig.ts

@@ -13,7 +13,6 @@ export function serializeConfig(project: TestProject): SerializedConfig {
   return {
     // TODO: remove functions from environmentOptions
     environmentOptions: config.environmentOptions,
-    mode: config.mode,
     isolate: config.isolate,
     maxWorkers: config.maxWorkers,
     base: config.base,
@@ -134,8 +133,11 @@ export function serializeConfig(project: TestProject): SerializedConfig {
     standalone: config.standalone,
     printConsoleTrace:
       config.printConsoleTrace ?? globalConfig.printConsoleTrace,
-    benchmark: config.benchmark && {
-      includeSamples: config.benchmark.includeSamples,
+    benchmark: {
+      enabled: config.benchmark.enabled,
+      retainSamples: config.benchmark.retainSamples,
+      suppressExportGetterWarnings: config.benchmark.suppressExportGetterWarnings,
+      projectName: config.benchmark.projectName,
     },
     // the browser initialized them via `@vite/env` import
     serializedDefines: config.browser.enabled

packages/vitest/src/node/core.ts

@@ -44,11 +44,11 @@ import { collectModuleDurationsDiagnostic, collectSourceModulesLocations } from
 import { VitestPackageInstaller } from './packageInstaller'
 import { createPool } from './pool'
 import { TestProject } from './project'
-import { getDefaultTestProject, resolveBrowserProjects, resolveProjects } from './projects/resolveProjects'
+import { getDefaultTestProject, resolveDefaultProjects, resolveProjects } from './projects/resolveProjects'
 import { BlobReporter, readBlobs } from './reporters/blob'
 import { HangingProcessReporter } from './reporters/hanging-process'
 import { createReport } from './reporters/report'
-import { createBenchmarkReporters, createReporters } from './reporters/utils'
+import { createReporters } from './reporters/utils'
 import { VitestResolver } from './resolver'
 import { VitestSpecifications } from './specifications'
 import { StateManager } from './state'
@@ -77,7 +77,7 @@ export class Vitest {
    * The logger instance used to log messages. It's recommended to use this logger instead of `console`.
    * It's possible to override stdout and stderr streams when initiating Vitest.
    * @example
-   * new Vitest('test', {
+   * new Vitest({
    *   stdout: new Writable(),
    * })
    */
@@ -143,11 +143,38 @@ export class Vitest {
   private _snapshot?: SnapshotManager
   private _coverageProvider?: CoverageProvider | null | undefined
 
+  /**
+   * @deprecated Do not rely on this property, it's always `test`. Scheduled to be removed in the next major.
+   */
+  public readonly mode = 'test'
+
   constructor(
-    public readonly mode: VitestRunMode,
     cliOptions: UserConfig,
-    options: VitestOptions = {},
+    options?: VitestOptions,
+  )
+  /**
+   * @deprecated The `mode` argument is no longer used. Use `new Vitest(cliOptions, options)` instead.
+   */
+  constructor(
+    mode: VitestRunMode,
+    cliOptions: UserConfig,
+    options?: VitestOptions,
+  )
+  constructor(
+    modeOrCliOptions: VitestRunMode | UserConfig,
+    cliOptionsOrOptions?: UserConfig | VitestOptions,
+    maybeOptions?: VitestOptions,
   ) {
+    let cliOptions: UserConfig
+    let options: VitestOptions
+    if (typeof modeOrCliOptions === 'string') {
+      cliOptions = cliOptionsOrOptions as UserConfig
+      options = maybeOptions ?? {}
+    }
+    else {
+      cliOptions = modeOrCliOptions
+      options = (cliOptionsOrOptions as VitestOptions) ?? {}
+    }
     this._cliOptions = cliOptions
     this.logger = new Logger(this, options.stdout, options.stderr)
     this.packageInstaller = options.packageInstaller || new VitestPackageInstaller()
@@ -306,10 +333,12 @@ export class Vitest {
     }
     catch { }
 
-    const projects = await this.resolveProjects(this._cliOptions)
-    this.projects = projects
+    this.projects = await this.resolveProjects(this._cliOptions)
+    if (this._cliOptions.benchmarkOnly) {
+      this.projects = this.projects.filter(c => c.config.benchmark.enabled)
+    }
 
-    await Promise.all(projects.flatMap((project) => {
+    await Promise.all(this.projects.flatMap((project) => {
       const hooks = project.vite.config.getSortedPluginHooks('configureVitest')
       return hooks.map(hook => hook({
         project,
@@ -356,9 +385,7 @@ export class Vitest {
       populateProjectsTags(this.coreWorkspaceProject, this.projects)
     }
 
-    this.reporters = resolved.mode === 'benchmark'
-      ? await createBenchmarkReporters(toArray(resolved.benchmark?.reporters), this.runner)
-      : await createReporters(resolved.reporters, this)
+    this.reporters = await createReporters(resolved.reporters, this)
 
     await this._fsCache.ensureCacheIntegrity()
 
@@ -567,7 +594,7 @@ export class Vitest {
     if (!project) {
       return []
     }
-    return resolveBrowserProjects(this, new Set([project.name]), [project])
+    return resolveDefaultProjects(this, new Set([project.name]), [project])
   }
 
   /**
@@ -797,7 +824,7 @@ export class Vitest {
         })
 
         if (!this.config.watch || !(this.config.changed || this.config.related?.length)) {
-          throw new FilesNotFoundError(this.mode)
+          throw new FilesNotFoundError()
         }
       }
 
@@ -1042,9 +1069,6 @@ export class Vitest {
     /** @default os.availableParallelism() */
     concurrency?: number
   }): Promise<TestModule[]> {
-    if (this.mode !== 'test') {
-      throw new Error(`The \`experimental_parseSpecifications\` does not support "${this.mode}" mode.`)
-    }
     const concurrency = options?.concurrency ?? (typeof os.availableParallelism === 'function'
       ? os.availableParallelism()
       : os.cpus().length)
@@ -1084,9 +1108,6 @@ export class Vitest {
   }
 
   public async experimental_parseSpecification(specification: TestSpecification): Promise<TestModule> {
-    if (this.mode !== 'test') {
-      throw new Error(`The \`experimental_parseSpecification\` does not support "${this.mode}" mode.`)
-    }
     const file = await astCollectTests(specification.project, specification.moduleId).catch((error) => {
       return createFailedFileTask(specification.project, specification.moduleId, error)
     })

packages/vitest/src/node/create.ts

@@ -15,13 +15,40 @@ import { Vitest } from './core'
 import { VitestPlugin } from './plugins'
 import { createViteServer } from './vite'
 
+export async function createVitest(
+  options: CliOptions,
+  viteOverrides?: ViteUserConfig,
+  vitestOptions?: VitestOptions,
+): Promise<Vitest>
+/**
+ * @deprecated The `mode` argument is no longer used. Use `createVitest(options, viteOverrides?, vitestOptions?)` instead.
+ */
 export async function createVitest(
   mode: VitestRunMode,
   options: CliOptions,
-  viteOverrides: ViteUserConfig = {},
-  vitestOptions: VitestOptions = {},
+  viteOverrides?: ViteUserConfig,
+  vitestOptions?: VitestOptions,
+): Promise<Vitest>
+export async function createVitest(
+  modeOrOptions: VitestRunMode | CliOptions,
+  optionsOrViteOverrides: CliOptions | ViteUserConfig = {},
+  viteOverridesOrVitestOptions: ViteUserConfig | VitestOptions = {},
+  maybeVitestOptions: VitestOptions = {},
 ): Promise<Vitest> {
-  const ctx = new Vitest(mode, deepClone(options), vitestOptions)
+  let options: CliOptions
+  let viteOverrides: ViteUserConfig
+  let vitestOptions: VitestOptions
+  if (typeof modeOrOptions === 'string') {
+    options = optionsOrViteOverrides as CliOptions
+    viteOverrides = viteOverridesOrVitestOptions as ViteUserConfig
+    vitestOptions = maybeVitestOptions
+  }
+  else {
+    options = modeOrOptions
+    viteOverrides = optionsOrViteOverrides as ViteUserConfig
+    vitestOptions = viteOverridesOrVitestOptions as VitestOptions
+  }
+  const ctx = new Vitest(deepClone(options), vitestOptions)
   const root = slash(resolve(options.root || process.cwd()))
 
   const configPath
@@ -38,8 +65,7 @@ export async function createVitest(
   const config: ViteInlineConfig = {
     configFile: configPath,
     configLoader: options.configLoader,
-    // this will make "mode": "test" | "benchmark" inside defineConfig
-    mode: options.mode || mode,
+    mode: options.mode || 'test',
     plugins: await VitestPlugin(restOptions, ctx),
   }
 

packages/vitest/src/node/errors.ts

@@ -1,8 +1,8 @@
 export class FilesNotFoundError extends Error {
   code = 'VITEST_FILES_NOT_FOUND'
 
-  constructor(mode: 'test' | 'benchmark') {
-    super(`No ${mode} files found`)
+  constructor() {
+    super(`No test files found`)
   }
 }
 

packages/vitest/src/node/logger.ts

@@ -169,20 +169,20 @@ export class Logger {
     const config = this.ctx.config
 
     if (config.watch && (config.changed || config.related?.length)) {
-      this.log(`No affected ${config.mode} files found\n`)
+      this.log(`No affected test files found\n`)
     }
     else if (config.watch) {
       this.log(
-        c.red(`No ${config.mode} files found. You can change the file name pattern by pressing "p"\n`),
+        c.red(`No test files found. You can change the file name pattern by pressing "p"\n`),
       )
     }
     else {
       if (config.passWithNoTests) {
-        this.log(`No ${config.mode} files found, exiting with code 0\n`)
+        this.log(`No test files found, exiting with code 0\n`)
       }
       else {
         this.error(
-          c.red(`No ${config.mode} files found, exiting with code 1\n`),
+          c.red(`No test files found, exiting with code 1\n`),
         )
       }
     }

packages/vitest/src/node/plugins/index.ts

@@ -25,7 +25,7 @@ import { VitestCoreResolver } from './vitestResolver'
 
 export async function VitestPlugin(
   options: UserConfig = {},
-  vitest: Vitest = new Vitest('test', deepClone(options)),
+  vitest: Vitest = new Vitest(deepClone(options)),
 ): Promise<VitePlugin[]> {
   const userConfig = deepMerge({}, options) as UserConfig
 
@@ -150,6 +150,11 @@ export async function VitestPlugin(
           }
         }
 
+        if (vitest._cliOptions.benchmarkOnly) {
+          config.test!.benchmark ??= {}
+          config.test!.benchmark.enabled = true
+        }
+
         // inherit so it's available in VitestOptimizer
         // I cannot wait to rewrite all of this in Vitest 4
         if (options.cache != null) {

packages/vitest/src/node/plugins/publicConfig.ts

@@ -27,12 +27,11 @@ export async function resolveConfig(
         : find.any(configFiles, { cwd: root })
   options.config = configPath
 
-  const vitest = new Vitest('test', deepClone(options))
+  const vitest = new Vitest(deepClone(options))
   const config = await resolveViteConfig(
     mergeConfig(
       {
         configFile: configPath,
-        // this will make "mode": "test" | "benchmark" inside defineConfig
         mode: options.mode || 'test',
         plugins: [
           await VitestPlugin(options, vitest),

packages/vitest/src/node/plugins/workspace.ts

@@ -37,7 +37,9 @@ export function WorkspaceVitestPlugin(
       name: 'vitest:project:name',
       enforce: 'post',
       config(viteConfig) {
-        const testConfig = viteConfig.test || {}
+        viteConfig.test ??= {}
+
+        const testConfig = viteConfig.test
 
         let { label: name, color } = typeof testConfig.name === 'string'
           ? { label: testConfig.name }
@@ -62,6 +64,11 @@ export function WorkspaceVitestPlugin(
           }
         }
 
+        if (project.vitest._cliOptions.benchmarkOnly) {
+          viteConfig.test.benchmark ??= {}
+          viteConfig.test.benchmark.enabled = true
+        }
+
         const isUserBrowserEnabled = viteConfig.test?.browser?.enabled
         const isBrowserEnabled = isUserBrowserEnabled ?? (viteConfig.test?.browser && project.vitest._cliOptions.browser?.enabled)
         // keep project names to potentially filter it out
@@ -79,6 +86,9 @@ export function WorkspaceVitestPlugin(
             workspaceNames.push(instance.name)
           }
         })
+        if (viteConfig.test?.benchmark?.enabled) {
+          workspaceNames.push(name ? `${name} (bench)` : 'bench')
+        }
 
         const filters = project.vitest.config.project
         // if there is `--project=...` filter, check if any of the potential projects match

packages/vitest/src/node/pools/poolRunner.ts

@@ -367,7 +367,7 @@ export class PoolRunner {
   }
 
   private emitUnexpectedExit = (): void => {
-    const error = new Error('Worker exited unexpectedly')
+    const error = new Error(`Worker exited unexpectedly during ${this._state} state`)
 
     this._eventEmitter.emit('error', error)
   }

packages/vitest/src/node/pools/rpc.ts

@@ -121,6 +121,15 @@ export function createMethodsRPC(project: TestProject, methodsOptions: MethodsOp
     onAfterSuiteRun(meta) {
       vitest.coverageProvider?.onAfterSuiteRun(meta)
     },
+    async onTestBenchmark(testId, benchmark) {
+      return vitest._testRun.recordBenchmark(testId, benchmark)
+    },
+    async readBenchmarkResult(relativePath) {
+      return project.benchmark.readResult(relativePath)
+    },
+    async writeBenchmarkResult(relativePath, data) {
+      return project.benchmark.writeResult(relativePath, data)
+    },
     async onTaskArtifactRecord(testId, artifact) {
       return vitest._testRun.recordArtifact(testId, artifact)
     },

packages/vitest/src/node/project.ts

@@ -28,6 +28,7 @@ import { isRunnableDevEnvironment } from 'vite'
 import { setup } from '../api/setup'
 import { createDefinesScript } from '../utils/config-helpers'
 import { NativeModuleRunner } from '../utils/nativeModuleRunner'
+import { BenchmarkManager } from './benchmark'
 import { isBrowserEnabled, resolveConfig } from './config/resolveConfig'
 import { serializeConfig } from './config/serializeConfig'
 import { createFetchModuleFunction } from './environments/fetchModule'
@@ -63,6 +64,8 @@ export class TestProject {
    */
   public readonly tmpDir: string
 
+  public readonly benchmark: BenchmarkManager = new BenchmarkManager(this)
+
   /** @internal */ typechecker?: Typechecker
   /** @internal */ _config?: ResolvedConfig
   /** @internal */ _vite?: ViteDevServer
@@ -731,7 +734,7 @@ export class TestProject {
   }
 
   /** @internal */
-  static _cloneBrowserProject(parent: TestProject, config: ResolvedConfig): TestProject {
+  static _cloneTestProject(parent: TestProject, config: ResolvedConfig): TestProject {
     const clone = new TestProject(parent.vitest, undefined, parent.tmpDir)
     clone.runner = parent.runner
     clone._vite = parent._vite

packages/vitest/src/node/projects/resolveProjects.ts

@@ -197,10 +197,80 @@ export async function resolveProjects(
     names.add(name)
   }
 
-  return resolveBrowserProjects(vitest, names, resolvedProjects)
+  return resolveDefaultProjects(vitest, names, resolvedProjects)
 }
 
-export async function resolveBrowserProjects(
+export async function resolveDefaultProjects(
+  vitest: Vitest,
+  names: Set<string>,
+  resolvedProjects: TestProject[],
+): Promise<TestProject[]> {
+  const newProjects = await resolveBrowserProjects(vitest, names, resolvedProjects)
+
+  let lastGroupOrder = Math.max(0, ...newProjects.map(p => p.config.sequence.groupOrder))
+
+  newProjects.forEach((project) => {
+    const benchmark = project.config.benchmark
+    if (!benchmark.enabled) {
+      return
+    }
+
+    if (vitest.isExcludedByProjectFilter(project.config.name)) {
+      benchmark.enabled = false
+      return
+    }
+
+    const name = project.config.name ? `${project.config.name} (bench)` : 'bench'
+    if (!vitest.matchesProjectFilter(name)) {
+      benchmark.enabled = false
+      return
+    }
+
+    if (names.has(name)) {
+      throw new Error(`Cannot create a benchmark project because the name "${name}" is already in use.`)
+    }
+    names.add(name)
+
+    const benchmarkProject = TestProject._cloneTestProject(project, {
+      ...project.config,
+      name,
+      include: benchmark.include,
+      exclude: benchmark.exclude,
+      includeSource: benchmark.includeSource,
+      coverage: {
+        ...project.config.coverage,
+        enabled: false,
+      },
+      maxWorkers: 1,
+      maxConcurrency: 1,
+      testTimeout: project.config.testTimeout < 60_000 ? 60_000 : project.config.testTimeout,
+      hookTimeout: project.config.hookTimeout < 120_000 ? 120_000 : project.config.hookTimeout,
+      // Spread because we disable it in the original project. `projectName`
+      // carries the parent's name so the runtime can substitute it into
+      // `${projectName}` placeholders inside `writeResult` / `bench.from()`
+      // paths — `project.config.name` already excludes the ` (bench)` suffix
+      // we add to the cloned project's own name above.
+      benchmark: { ...benchmark, projectName: project.config.name ?? '' },
+      sequence: {
+        ...project.config.sequence,
+        concurrent: false,
+        // benchmarks should always run in a separate isolated group
+        groupOrder: ++lastGroupOrder,
+      },
+      typecheck: {
+        ...project.config.typecheck,
+        enabled: false,
+      },
+      // TODO: mark if benchmark project?
+    })
+    // disable benchmark in the original project
+    benchmark.enabled = false
+    newProjects.push(benchmarkProject)
+  })
+  return newProjects
+}
+
+async function resolveBrowserProjects(
   vitest: Vitest,
   names: Set<string>,
   resolvedProjects: TestProject[],
@@ -260,7 +330,7 @@ export async function resolveBrowserProjects(
       names.add(name)
       const clonedConfig = cloneConfig(project, config)
       clonedConfig.name = name
-      const clone = TestProject._cloneBrowserProject(project, clonedConfig)
+      const clone = TestProject._cloneTestProject(project, clonedConfig)
       resolvedProjects.push(clone)
     })
 
@@ -481,11 +551,15 @@ export function getDefaultTestProject(vitest: Vitest): TestProject | null {
 
 function getPotentialProjectNames(project: TestProject) {
   const names = [project.name]
+  // TODO: include benchmarks in browsers
   if (project.config.browser.instances) {
     names.push(...project.config.browser.instances.map(i => i.name!))
   }
   else if (project.config.browser.name) {
     names.push(project.config.browser.name)
   }
+  if (project.config.benchmark.enabled) {
+    names.push(project.name ? `${project.name} (bench)` : 'bench')
+  }
   return names
 }

packages/vitest/src/node/reporters/base.ts

@@ -1,4 +1,4 @@
-import type { File, Task, TestAnnotation } from '@vitest/runner'
+import type { File, Task, TestAnnotation, TestBenchmark, TestBenchmarkTask } from '@vitest/runner'
 import type { ParsedStack, SerializedError } from '@vitest/utils'
 import type { AsyncLeak, TestError, UserConsoleLog } from '../../types/general'
 import type { Vitest } from '../core'
@@ -16,6 +16,7 @@ import { groupBy } from '../../utils/base'
 import { isTTY } from '../../utils/env'
 import { hasFailedSnapshot } from '../../utils/tasks'
 import { generateCodeFrame, printStack } from '../printError'
+import { BENCH_TABLE_HEAD, computeBenchColumnWidths, padBenchRow, renderBenchmarkRow } from './renderers/benchmark-table'
 import { F_CHECK, F_DOWN_RIGHT, F_POINTER } from './renderers/figures'
 import {
   countTestErrors,
@@ -54,6 +55,7 @@ export abstract class BaseReporter implements Reporter {
 
   private _filesInWatchMode = new Map<string, number>()
   private _timeStart = formatTimeString(new Date())
+  private _perProjectBenchmarks = new Map<string, Map<string, TestBenchmarkTask>>()
 
   constructor(options: BaseOptions = {}) {
     this.isTTY = options.isTTY ?? isTTY
@@ -82,6 +84,7 @@ export abstract class BaseReporter implements Reporter {
   onTestRunStart(_specifications: ReadonlyArray<TestSpecification>): void {
     this.start = performance.now()
     this._timeStart = formatTimeString(new Date())
+    this._perProjectBenchmarks.clear()
   }
 
   onTestRunEnd(
@@ -97,6 +100,7 @@ export abstract class BaseReporter implements Reporter {
       this.ctx.logger.printNoTestFound(this.ctx.filenamePattern)
     }
     else {
+      this.printPerProjectBenchmarks()
       this.reportSummary(files, errors)
     }
   }
@@ -107,6 +111,22 @@ export abstract class BaseReporter implements Reporter {
     }
   }
 
+  onTestCaseBenchmark(testCase: TestCase, benchmark: TestBenchmark): void {
+    const projectName = testCase.project.name || ''
+    for (const task of benchmark.tasks) {
+      if (!task.perProject) {
+        continue
+      }
+      const benchKey = `${testCase.module.relativeModuleId} > ${testCase.fullName} > ${task.name}`
+      let projectMap = this._perProjectBenchmarks.get(benchKey)
+      if (!projectMap) {
+        projectMap = new Map()
+        this._perProjectBenchmarks.set(benchKey, projectMap)
+      }
+      projectMap.set(projectName, task)
+    }
+  }
+
   onTestSuiteResult(testSuite: TestSuite): void {
     if (testSuite.state() === 'failed') {
       this.logFailedTask(testSuite.task)
@@ -206,6 +226,10 @@ export abstract class BaseReporter implements Reporter {
     const { duration = 0 } = test.diagnostic() || {}
     const padding = this.getTestIndentation(test.task)
     const suffix = this.getTestCaseSuffix(test)
+    const benchmarks = test.benchmarks()
+    // perProject tasks still appear in the inline table — they're additionally
+    // aggregated in the cross-project section at the end of the run
+    const inlineBenchmarks: TestBenchmark[] = benchmarks.filter(b => b.tasks.length > 0)
 
     if (testResult.state === 'failed') {
       this.log(c.red(` ${padding}${taskFail} ${this.getTestName(test.task, separator)}`) + suffix)
@@ -213,16 +237,20 @@ export abstract class BaseReporter implements Reporter {
 
     // also print slow tests
     else if (duration > this.ctx.config.slowTestThreshold) {
-      this.log(` ${padding}${c.yellow(c.dim(F_CHECK))} ${this.getTestName(test.task, separator)} ${suffix}`)
+      this.log(` ${padding}${c.yellow(c.dim(F_CHECK))} ${this.getTestName(test.task, separator)}${suffix}`)
     }
 
     else if (this.ctx.config.hideSkippedTests && testResult.state === 'skipped' && test.options.mode !== 'todo') {
       // Skipped tests are hidden when --hideSkippedTests
     }
 
-    else if (this.renderSucceed || moduleState === 'failed') {
+    else if (this.renderSucceed || moduleState === 'failed' || inlineBenchmarks.length) {
       this.log(` ${padding}${this.getStateSymbol(test)} ${this.getTestName(test.task, separator)}${suffix}`)
     }
+
+    if (inlineBenchmarks.length > 0) {
+      this.printBenchmarkTable(inlineBenchmarks, padding)
+    }
   }
 
   private getModuleLog(testModule: TestModule, counts: {
@@ -545,12 +573,7 @@ export abstract class BaseReporter implements Reporter {
 
     const leakCount = this.printLeaksSummary()
 
-    if (this.ctx.config.mode === 'benchmark') {
-      this.reportBenchmarkSummary(files)
-    }
-    else {
-      this.reportTestSummary(files, errors, leakCount)
-    }
+    this.reportTestSummary(files, errors, leakCount)
   }
 
   reportTestSummary(files: File[], errors: unknown[], leakCount: number): void {
@@ -873,34 +896,91 @@ export abstract class BaseReporter implements Reporter {
     return leakWithStacks.size
   }
 
-  reportBenchmarkSummary(files: File[]): void {
-    const benches = getTests(files)
-    const topBenches = benches.filter(i => i.result?.benchmark?.rank === 1)
+  protected printPerProjectBenchmarks(): void {
+    if (this._perProjectBenchmarks.size === 0) {
+      return
+    }
 
-    this.log(`\n${withLabel('cyan', 'BENCH', 'Summary\n')}`)
+    let hasComparable = false
+    for (const projectMap of this._perProjectBenchmarks.values()) {
+      if (projectMap.size > 1) {
+        hasComparable = true
+        break
+      }
+    }
+    if (!hasComparable) {
+      return
+    }
 
-    for (const bench of topBenches) {
-      const group = bench.suite || bench.file
+    this.log('')
+    this.log(divider(c.bold(c.bgBlue(` Cross-Project Benchmark Comparison `)), null, null, c.blue))
 
-      if (!group) {
+    for (const [benchName, projectMap] of this._perProjectBenchmarks) {
+      const tasks = [...projectMap.entries()]
+        .sort((a, b) => a[1].latency.mean - b[1].latency.mean)
+        .map(([projectName, task], index) => ({ ...task, name: projectName, rank: index + 1 }))
+
+      if (tasks.length <= 1) {
         continue
       }
 
-      const groupName = this.getFullName(group, separator)
-      const project = this.ctx.projects.find(p => p.name === bench.file.projectName)
+      this.log('')
+      this.log(`  ${c.dim(benchName)}`)
+      this.printBenchmarkTable([{ name: benchName, tasks }], '', 'project')
+    }
 
-      this.log(`  ${formatProjectName(project)}${bench.name}${c.dim(` - ${groupName}`)}`)
+    this.log('')
+  }
 
-      const siblings = group.tasks
-        .filter(i => i.meta.benchmark && i.result?.benchmark && i !== bench)
-        .sort((a, b) => a.result!.benchmark!.rank - b.result!.benchmark!.rank)
-
-      for (const sibling of siblings) {
-        const number = (sibling.result!.benchmark!.mean / bench.result!.benchmark!.mean).toFixed(2)
-        this.log(c.green(`    ${number}x `) + c.gray('faster than ') + sibling.name)
+  protected printBenchmarkTable(benchmarks: readonly TestBenchmark[], basePadding: string, columnName = 'name'): void {
+    let printedCount = 0
+    for (const benchmark of benchmarks) {
+      const { tasks } = benchmark
+      if (tasks.length === 0) {
+        continue
       }
 
-      this.log('')
+      if (printedCount > 0) {
+        this.log('')
+      }
+
+      const rows = tasks.map(t => renderBenchmarkRow(t))
+      const tableHead = [
+        columnName,
+        ...BENCH_TABLE_HEAD,
+      ]
+      const widths = computeBenchColumnWidths(tableHead, rows)
+      const indent = ` ${basePadding}  `
+
+      this.log(`${indent}${padBenchRow(tableHead, widths).map(c.bold).join('  ')}`)
+      printedCount++
+
+      for (const task of tasks) {
+        const padded = padBenchRow(renderBenchmarkRow(task), widths)
+        let row = [
+          padded[0],
+          c.blue(padded[1]),
+          c.cyan(padded[2]),
+          c.cyan(padded[3]),
+          c.cyan(padded[4]),
+          c.cyan(padded[5]),
+          c.cyan(padded[6]),
+          c.cyan(padded[7]),
+          c.cyan(padded[8]),
+          c.dim(padded[9]),
+          c.dim(padded[10]),
+        ].join('  ')
+
+        if (task.rank === 1 && tasks.length > 1) {
+          row += c.bold(c.green('   fastest'))
+        }
+
+        if (task.rank === tasks.length && tasks.length > 2) {
+          row += c.bold(c.gray('   slowest'))
+        }
+
+        this.log(`${indent}${row}`)
+      }
     }
   }
 

packages/vitest/src/node/reporters/benchmark/index.ts

@@ -1,17 +0,0 @@
-import { BenchmarkReporter } from './reporter'
-import { VerboseBenchmarkReporter } from './verbose'
-
-export {
-  BenchmarkReporter,
-  VerboseBenchmarkReporter,
-}
-
-export const BenchmarkReportsMap: {
-  default: typeof BenchmarkReporter
-  verbose: typeof VerboseBenchmarkReporter
-} = {
-  default: BenchmarkReporter,
-  verbose: VerboseBenchmarkReporter,
-}
-
-export type BenchmarkBuiltinReporters = keyof typeof BenchmarkReportsMap

packages/vitest/src/node/reporters/benchmark/json-formatter.ts

@@ -1,69 +0,0 @@
-import type { File } from '@vitest/runner'
-import type { BenchmarkResult } from '../../../runtime/types/benchmark'
-import { getFullName, getTasks } from '@vitest/runner/utils'
-
-interface Report {
-  files: {
-    filepath: string
-    groups: Group[]
-  }[]
-}
-
-interface Group {
-  fullName: string
-  benchmarks: FormattedBenchmarkResult[]
-}
-
-export type FormattedBenchmarkResult = BenchmarkResult & {
-  id: string
-}
-
-export function createBenchmarkJsonReport(files: File[]): Report {
-  const report: Report = { files: [] }
-
-  for (const file of files) {
-    const groups: Group[] = []
-
-    for (const task of getTasks(file)) {
-      if (task?.type === 'suite') {
-        const benchmarks: FormattedBenchmarkResult[] = []
-
-        for (const t of task.tasks) {
-          const benchmark = t.meta.benchmark && t.result?.benchmark
-
-          if (benchmark) {
-            benchmarks.push({ id: t.id, ...benchmark, samples: [] })
-          }
-        }
-
-        if (benchmarks.length) {
-          groups.push({
-            fullName: getFullName(task, ' > '),
-            benchmarks,
-          })
-        }
-      }
-    }
-
-    report.files.push({
-      filepath: file.filepath,
-      groups,
-    })
-  }
-
-  return report
-}
-
-export function flattenFormattedBenchmarkReport(report: Report): Record<string, FormattedBenchmarkResult> {
-  const flat: Record<FormattedBenchmarkResult['id'], FormattedBenchmarkResult> = {}
-
-  for (const file of report.files) {
-    for (const group of file.groups) {
-      for (const t of group.benchmarks) {
-        flat[t.id] = t
-      }
-    }
-  }
-
-  return flat
-}

packages/vitest/src/node/reporters/benchmark/reporter.ts

@@ -1,114 +0,0 @@
-import type { TaskResultPack } from '@vitest/runner'
-import type { SerializedError } from '@vitest/utils'
-import type { Vitest } from '../../core'
-import type { TestRunEndReason } from '../../types/reporter'
-import type { TestModule, TestSuite } from '../reported-tasks'
-import fs from 'node:fs'
-import { getFullName } from '@vitest/runner/utils'
-import * as pathe from 'pathe'
-import c from 'tinyrainbow'
-import { DefaultReporter } from '../default'
-import { formatProjectName, getStateSymbol, separator } from '../renderers/utils'
-import { createBenchmarkJsonReport, flattenFormattedBenchmarkReport } from './json-formatter'
-import { renderTable } from './tableRender'
-
-export class BenchmarkReporter extends DefaultReporter {
-  compare?: Parameters<typeof renderTable>[0]['compare']
-
-  async onInit(ctx: Vitest): Promise<void> {
-    super.onInit(ctx)
-
-    if (this.ctx.config.benchmark?.compare) {
-      const compareFile = pathe.resolve(
-        this.ctx.config.root,
-        this.ctx.config.benchmark?.compare,
-      )
-      try {
-        this.compare = flattenFormattedBenchmarkReport(
-          JSON.parse(await fs.promises.readFile(compareFile, 'utf-8')),
-        )
-      }
-      catch (e) {
-        this.error(`Failed to read '${compareFile}'`, e)
-      }
-    }
-  }
-
-  onTaskUpdate(packs: TaskResultPack[]): void {
-    for (const pack of packs) {
-      const task = this.ctx.state.idMap.get(pack[0])
-
-      if (task?.type === 'suite' && task.result?.state !== 'run') {
-        task.tasks.filter(task => task.result?.benchmark)
-          .sort((benchA, benchB) => benchA.result!.benchmark!.mean - benchB.result!.benchmark!.mean)
-          .forEach((bench, idx) => {
-            bench.result!.benchmark!.rank = Number(idx) + 1
-          })
-      }
-    }
-  }
-
-  onTestSuiteResult(testSuite: TestSuite): void {
-    super.onTestSuiteResult(testSuite)
-    this.printSuiteTable(testSuite)
-  }
-
-  protected printTestModule(testModule: TestModule): void {
-    this.printSuiteTable(testModule)
-  }
-
-  private printSuiteTable(testTask: TestModule | TestSuite): void {
-    const state = testTask.state()
-    if (state === 'pending' || state === 'queued') {
-      return
-    }
-
-    const benches = testTask.task.tasks.filter(t => t.meta.benchmark)
-    const duration = testTask.task.result?.duration || 0
-
-    if (benches.length > 0 && benches.every(t => t.result?.state !== 'run' && t.result?.state !== 'queued')) {
-      let title = `\n ${getStateSymbol(testTask.task)} ${formatProjectName(testTask.project)}${getFullName(testTask.task, separator)}`
-
-      if (duration != null && duration > this.ctx.config.slowTestThreshold) {
-        title += c.yellow(` ${Math.round(duration)}${c.dim('ms')}`)
-      }
-
-      this.log(title)
-      this.log(renderTable({
-        tasks: benches,
-        level: 1,
-        shallow: true,
-        columns: this.ctx.logger.getColumns(),
-        compare: this.compare,
-        showHeap: this.ctx.config.logHeapUsage,
-        slowTestThreshold: this.ctx.config.slowTestThreshold,
-      }))
-    }
-  }
-
-  async onTestRunEnd(
-    testModules: ReadonlyArray<TestModule>,
-    unhandledErrors: ReadonlyArray<SerializedError>,
-    reason: TestRunEndReason,
-  ): Promise<void> {
-    super.onTestRunEnd(testModules, unhandledErrors, reason)
-
-    // write output for future comparison
-    let outputFile = this.ctx.config.benchmark?.outputJson
-
-    if (outputFile) {
-      outputFile = pathe.resolve(this.ctx.config.root, outputFile)
-      const outputDirectory = pathe.dirname(outputFile)
-
-      if (!fs.existsSync(outputDirectory)) {
-        await fs.promises.mkdir(outputDirectory, { recursive: true })
-      }
-
-      const files = testModules.map(t => t.task.file)
-      const output = createBenchmarkJsonReport(files)
-
-      await fs.promises.writeFile(outputFile, JSON.stringify(output, null, 2))
-      this.log(`Benchmark report written to ${outputFile}`)
-    }
-  }
-}

packages/vitest/src/node/reporters/benchmark/tableRender.ts

@@ -1,221 +0,0 @@
-import type { Task } from '@vitest/runner'
-import type { BenchmarkResult } from '../../../runtime/types/benchmark'
-import type { FormattedBenchmarkResult } from './json-formatter'
-import { stripVTControlCharacters } from 'node:util'
-import { getTests } from '@vitest/runner/utils'
-import { notNullish } from '@vitest/utils/helpers'
-import c from 'tinyrainbow'
-import { F_RIGHT } from '../renderers/figures'
-import { getStateSymbol, truncateString } from '../renderers/utils'
-
-const outputMap = new WeakMap<Task, string>()
-
-function formatNumber(number: number) {
-  const res = String(number.toFixed(number < 100 ? 4 : 2)).split('.')
-
-  return res[0].replace(/(?=(?:\d{3})+$)\B/g, ',') + (res[1] ? `.${res[1]}` : '')
-}
-
-const tableHead = [
-  'name',
-  'hz',
-  'min',
-  'max',
-  'mean',
-  'p75',
-  'p99',
-  'p995',
-  'p999',
-  'rme',
-  'samples',
-]
-
-function renderBenchmarkItems(result: BenchmarkResult) {
-  return [
-    result.name,
-    formatNumber(result.hz || 0),
-    formatNumber(result.min || 0),
-    formatNumber(result.max || 0),
-    formatNumber(result.mean || 0),
-    formatNumber(result.p75 || 0),
-    formatNumber(result.p99 || 0),
-    formatNumber(result.p995 || 0),
-    formatNumber(result.p999 || 0),
-    `±${(result.rme || 0).toFixed(2)}%`,
-    (result.sampleCount || 0).toString(),
-  ]
-}
-
-function computeColumnWidths(results: BenchmarkResult[]): number[] {
-  const rows = [tableHead, ...results.map(v => renderBenchmarkItems(v))]
-
-  return Array.from(tableHead, (_, i) =>
-    Math.max(...rows.map(row => stripVTControlCharacters(row[i]).length)))
-}
-
-function padRow(row: string[], widths: number[]) {
-  return row.map(
-    (v, i) => (i ? v.padStart(widths[i], ' ') : v.padEnd(widths[i], ' ')), // name
-  )
-}
-
-function renderTableHead(widths: number[]) {
-  return ' '.repeat(3) + padRow(tableHead, widths).map(c.bold).join('  ')
-}
-
-function renderBenchmark(result: BenchmarkResult, widths: number[]) {
-  const padded = padRow(renderBenchmarkItems(result), widths)
-  return [
-    padded[0], // name
-    c.blue(padded[1]), // hz
-    c.cyan(padded[2]), // min
-    c.cyan(padded[3]), // max
-    c.cyan(padded[4]), // mean
-    c.cyan(padded[5]), // p75
-    c.cyan(padded[6]), // p99
-    c.cyan(padded[7]), // p995
-    c.cyan(padded[8]), // p999
-    c.dim(padded[9]), // rem
-    c.dim(padded[10]), // sample
-  ].join('  ')
-}
-
-export function renderTable(
-  options: {
-    tasks: Task[]
-    level: number
-    shallow?: boolean
-    showHeap: boolean
-    columns: number
-    slowTestThreshold: number
-    compare?: Record<Task['id'], FormattedBenchmarkResult>
-  },
-): string {
-  const output: string[] = []
-
-  const benchMap: Record<string, { current: BenchmarkResult; baseline?: BenchmarkResult }> = {}
-
-  for (const task of options.tasks) {
-    if (task.meta.benchmark && task.result?.benchmark) {
-      benchMap[task.id] = {
-        current: task.result.benchmark,
-        baseline: options.compare?.[task.id],
-      }
-    }
-  }
-
-  const benchCount = Object.entries(benchMap).length
-
-  const columnWidths = computeColumnWidths(
-    Object.values(benchMap)
-      .flatMap(v => [v.current, v.baseline])
-      .filter(notNullish),
-  )
-
-  let idx = 0
-  const padding = '  '.repeat(options.level ? 1 : 0)
-
-  for (const task of options.tasks) {
-    const duration = task.result?.duration
-    const bench = benchMap[task.id]
-
-    let prefix = ''
-
-    if (idx === 0 && task.meta?.benchmark) {
-      prefix += `${renderTableHead(columnWidths)}\n${padding}`
-    }
-
-    prefix += ` ${getStateSymbol(task)} `
-
-    let suffix = ''
-
-    if (task.type === 'suite') {
-      suffix += c.dim(` (${getTests(task).length})`)
-    }
-
-    if (task.mode === 'skip' || task.mode === 'todo') {
-      suffix += c.dim(c.gray(' [skipped]'))
-    }
-
-    if (duration != null) {
-      const color = duration > options.slowTestThreshold ? c.yellow : c.green
-
-      suffix += color(` ${Math.round(duration)}${c.dim('ms')}`)
-    }
-
-    if (options.showHeap && task.result?.heap != null) {
-      suffix += c.magenta(` ${Math.floor(task.result.heap / 1024 / 1024)} MB heap used`)
-    }
-
-    if (bench) {
-      let body = renderBenchmark(bench.current, columnWidths)
-
-      if (options.compare && bench.baseline) {
-        if (bench.current.hz) {
-          const diff = bench.current.hz / bench.baseline.hz
-          const diffFixed = diff.toFixed(2)
-
-          if (diffFixed === '1.0.0') {
-            body += c.gray(`  [${diffFixed}x]`)
-          }
-
-          if (diff > 1) {
-            body += c.blue(`  [${diffFixed}x] ⇑`)
-          }
-          else {
-            body += c.red(`  [${diffFixed}x] ⇓`)
-          }
-        }
-        output.push(padding + prefix + body + suffix)
-
-        const bodyBaseline = renderBenchmark(bench.baseline, columnWidths)
-        output.push(`${padding}   ${bodyBaseline}  ${c.dim('(baseline)')}`)
-      }
-
-      else {
-        if (bench.current.rank === 1 && benchCount > 1) {
-          body += c.bold(c.green('   fastest'))
-        }
-
-        if (bench.current.rank === benchCount && benchCount > 2) {
-          body += c.bold(c.gray('   slowest'))
-        }
-
-        output.push(padding + prefix + body + suffix)
-      }
-    }
-    else {
-      output.push(padding + prefix + task.name + suffix)
-    }
-
-    if (task.result?.state !== 'pass' && outputMap.get(task) != null) {
-      let data: string | undefined = outputMap.get(task)
-
-      if (typeof data === 'string') {
-        data = stripVTControlCharacters(data.trim().split('\n').filter(Boolean).pop()!)
-        if (data === '') {
-          data = undefined
-        }
-      }
-
-      if (data != null) {
-        const out = `   ${'  '.repeat(options.level)}${F_RIGHT} ${data}`
-        output.push(c.gray(truncateString(out, options.columns)))
-      }
-    }
-
-    if (!options.shallow && task.type === 'suite' && task.tasks.length > 0) {
-      if (task.result?.state) {
-        output.push(renderTable({
-          ...options,
-          tasks: task.tasks,
-          level: options.level + 1,
-          shallow: false,
-        }))
-      }
-    }
-    idx++
-  }
-
-  return output.filter(Boolean).join('\n')
-}

packages/vitest/src/node/reporters/benchmark/verbose.ts

@@ -1,5 +0,0 @@
-import { BenchmarkReporter } from './reporter'
-
-export class VerboseBenchmarkReporter extends BenchmarkReporter {
-  protected verbose = true
-}

packages/vitest/src/node/reporters/index.ts

@@ -35,12 +35,6 @@ export {
 }
 export type { BaseReporter, Reporter, TestRunEndReason }
 
-export type { BenchmarkBuiltinReporters } from './benchmark'
-export {
-  BenchmarkReporter,
-  BenchmarkReportsMap,
-  VerboseBenchmarkReporter,
-} from './benchmark'
 export type {
   JsonAssertionResult,
   JsonTestResult,

packages/vitest/src/node/reporters/json.ts

@@ -1,4 +1,4 @@
-import type { Suite, TaskMeta, TaskState } from '@vitest/runner'
+import type { Suite, TaskMeta, TaskState, TestBenchmark } from '@vitest/runner'
 import type { SnapshotSummary } from '@vitest/snapshot'
 import type { CoverageMap } from 'istanbul-lib-coverage'
 import type { Vitest } from '../core'
@@ -40,6 +40,7 @@ export interface JsonAssertionResult {
   failureMessages: Array<string> | null
   location?: Callsite | null
   tags: string[]
+  benchmarks: TestBenchmark[]
 }
 
 export interface JsonTestResult {
@@ -177,6 +178,7 @@ export class JsonReporter implements Reporter {
               })()
             : t.meta,
           tags: t.tags || [],
+          benchmarks: t.benchmarks,
         } satisfies JsonAssertionResult
       })
 

packages/vitest/src/node/reporters/junit.ts

@@ -11,6 +11,7 @@ import { stripVTControlCharacters } from 'node:util'
 import { getSuites } from '@vitest/runner/utils'
 import { basename, dirname, relative, resolve } from 'pathe'
 import { getOutputFile } from '../../utils/config-helpers'
+import { renderBenchmarkTableText } from './renderers/benchmark-table'
 import { IndentedLogger } from './renderers/indented-logger'
 
 export interface ClassnameTemplateVariables {
@@ -319,6 +320,30 @@ export class JUnitReporter implements Reporter {
     })
   }
 
+  async writeSystemOut(task: Task): Promise<void> {
+    const logs
+      = this.options.includeConsoleOutput && task.logs
+        ? task.logs.filter(log => log.type === 'stdout')
+        : []
+    const benchmarks = task.type === 'test' ? task.benchmarks : []
+
+    if (logs.length === 0 && benchmarks.length === 0) {
+      return
+    }
+
+    await this.writeElement('system-out', {}, async () => {
+      for (const log of logs) {
+        await this.baseLog(escapeXML(log.content))
+      }
+      if (benchmarks.length > 0) {
+        if (logs.length > 0) {
+          await this.baseLog('')
+        }
+        await this.baseLog(escapeXML(renderBenchmarkTableText(benchmarks)))
+      }
+    })
+  }
+
   private applyTemplate(
     template: string | ((vars: ClassnameTemplateVariables) => string),
     vars: ClassnameTemplateVariables,
@@ -368,8 +393,8 @@ export class JUnitReporter implements Reporter {
           time: getDuration(task),
         },
         async () => {
+          await this.writeSystemOut(task)
           if (this.options.includeConsoleOutput) {
-            await this.writeLogs(task, 'out')
             await this.writeLogs(task, 'err')
           }
 
@@ -569,6 +594,7 @@ export class JUnitReporter implements Reporter {
           file: null as any,
           annotations: [],
           artifacts: [],
+          benchmarks: [],
         } satisfies Task)
       }
 

packages/vitest/src/node/reporters/renderers/benchmark-table.ts

@@ -0,0 +1,80 @@
+import type { TestBenchmark, TestBenchmarkTask } from '@vitest/runner'
+import { stripVTControlCharacters } from 'node:util'
+
+export const BENCH_TABLE_HEAD: string[] = [
+  'hz',
+  'min',
+  'max',
+  'mean',
+  'p75',
+  'p99',
+  'p995',
+  'p999',
+  'rme',
+  'samples',
+]
+
+function formatBenchNumber(number: number): string {
+  const res = String(number.toFixed(number < 100 ? 4 : 2)).split('.')
+  return res[0].replace(/(?=(?:\d{3})+$)\B/g, ',') + (res[1] ? `.${res[1]}` : '')
+}
+
+// Plain-text rendering of the benchmark table (no ANSI colors, no indent).
+// Used by the junit reporter to embed benchmark data in <system-out>.
+export function renderBenchmarkTableText(
+  benchmarks: readonly TestBenchmark[],
+  columnName = 'name',
+): string {
+  const lines: string[] = []
+  for (const benchmark of benchmarks) {
+    const { tasks } = benchmark
+    if (tasks.length === 0) {
+      continue
+    }
+    if (lines.length > 0) {
+      lines.push('')
+    }
+    const rows = tasks.map(renderBenchmarkRow)
+    const head = [columnName, ...BENCH_TABLE_HEAD]
+    const widths = computeBenchColumnWidths(head, rows)
+    lines.push(padBenchRow(head, widths).join('  '))
+    for (const task of tasks) {
+      let row = padBenchRow(renderBenchmarkRow(task), widths).join('  ')
+      if (task.rank === 1 && tasks.length > 1) {
+        row += '   fastest'
+      }
+      if (task.rank === tasks.length && tasks.length > 2) {
+        row += '   slowest'
+      }
+      lines.push(row)
+    }
+  }
+  return lines.join('\n')
+}
+
+export function renderBenchmarkRow(task: TestBenchmarkTask): string[] {
+  return [
+    task.name,
+    formatBenchNumber(task.throughput.mean || 0),
+    formatBenchNumber(task.latency.min || 0),
+    formatBenchNumber(task.latency.max || 0),
+    formatBenchNumber(task.latency.mean || 0),
+    formatBenchNumber(task.latency.p75 || 0),
+    formatBenchNumber(task.latency.p99 || 0),
+    formatBenchNumber(task.latency.p995 || 0),
+    formatBenchNumber(task.latency.p999 || 0),
+    `\u00B1${(task.latency.rme || 0).toFixed(2)}%`,
+    String(task.latency.samplesCount || 0),
+  ]
+}
+
+export function computeBenchColumnWidths(header: string[], rows: string[][]): number[] {
+  const allRows = [header, ...rows]
+  return Array.from(header, (_, i) => Math.max(...allRows.map(row => stripVTControlCharacters(row[i]).length)))
+}
+
+export function padBenchRow(row: string[], widths: number[]): string[] {
+  return row.map(
+    (v, i) => (i === 0 ? v.padEnd(widths[i]) : v.padStart(widths[i])),
+  )
+}

packages/vitest/src/node/reporters/reported-tasks.ts

@@ -8,6 +8,7 @@ import type {
   TaskMeta,
   TestAnnotation,
   TestArtifact,
+  TestBenchmark,
 } from '@vitest/runner'
 import type { SerializedError, TestError } from '@vitest/utils'
 import type { DevEnvironment } from 'vite'
@@ -213,6 +214,15 @@ export class TestCase extends ReportedTaskImplementation {
     return [...this.task.artifacts]
   }
 
+  /**
+   * @experimental
+   *
+   * A list of benchmarks performed during the test.
+   */
+  public benchmarks(): ReadonlyArray<TestBenchmark> {
+    return [...this.task.benchmarks]
+  }
+
   /**
    * Useful information about the test like duration, memory usage, etc.
    * Diagnostic is only available after the test has finished.

packages/vitest/src/node/reporters/summary.ts

@@ -2,7 +2,7 @@ import type { Vitest } from '../core'
 import type { TestSpecification } from '../test-specification'
 import type { Reporter } from '../types/reporter'
 import type { Options as WindowRendererOptions } from './renderers/windowedRenderer'
-import type { ReportedHookContext, TestCase, TestModule } from './reported-tasks'
+import type { ReportedHookContext, TestCase, TestModule, TestSuite } from './reported-tasks'
 import c from 'tinyrainbow'
 import { F_POINTER, F_TREE_NODE_END, F_TREE_NODE_MIDDLE } from './renderers/figures'
 import { formatProjectName, formatTime, formatTimeString, padSummaryTitle } from './renderers/utils'
@@ -36,14 +36,14 @@ interface SlowTask {
   visible: boolean
   startTime: number
   onFinish: () => void
-  hook?: Omit<SlowTask, 'hook'>
+  step?: Omit<SlowTask, 'step'>
 }
 
 interface RunningModule extends Pick<Counter, 'total' | 'completed'> {
   filename: TestModule['task']['name']
   projectName: TestModule['project']['name']
   projectColor: TestModule['project']['color']
-  hook?: Omit<SlowTask, 'hook'>
+  step?: Omit<SlowTask, 'step'>
   tests: Map<TestCase['id'], SlowTask>
   meta: TestModule['task']['meta']
 }
@@ -139,42 +139,44 @@ export class SummaryReporter implements Reporter {
     this.renderer.schedule()
   }
 
-  onHookStart(options: ReportedHookContext): void {
-    const stats = this.getHookStats(options)
-
-    if (!stats) {
-      return
-    }
-
-    const hook = {
-      name: options.name,
+  private startStep(stats: RunningModule | SlowTask, name: string) {
+    const step = {
+      name,
       visible: false,
       startTime: performance.now(),
       onFinish: () => {},
     }
-    stats.hook?.onFinish?.()
-    stats.hook = hook
+    stats.step?.onFinish?.()
+    stats.step = step
 
     if (!Number.isFinite(this.ctx.config.slowTestThreshold)) {
       return
     }
 
     const timeout = setTimeout(() => {
-      hook.visible = true
+      step.visible = true
     }, this.ctx.config.slowTestThreshold).unref()
 
-    hook.onFinish = () => clearTimeout(timeout)
+    step.onFinish = () => clearTimeout(timeout)
+  }
+
+  onHookStart(options: ReportedHookContext): void {
+    const stats = this.getStepStats(options.entity)
+
+    if (stats) {
+      this.startStep(stats, options.name)
+    }
   }
 
   onHookEnd(options: ReportedHookContext): void {
-    const stats = this.getHookStats(options)
+    const stats = this.getStepStats(options.entity)
 
-    if (stats?.hook?.name !== options.name) {
+    if (stats?.step?.name !== options.name) {
       return
     }
 
-    stats.hook.onFinish()
-    stats.hook.visible = false
+    stats.step.onFinish()
+    stats.step.visible = false
   }
 
   onTestCaseReady(test: TestCase): void {
@@ -203,7 +205,7 @@ export class SummaryReporter implements Reporter {
       : undefined
 
     slowTest.onFinish = () => {
-      slowTest.hook?.onFinish()
+      slowTest.step?.onFinish()
       clearTimeout(timeout)
     }
 
@@ -283,7 +285,7 @@ export class SummaryReporter implements Reporter {
     this.renderer.schedule()
   }
 
-  private getHookStats({ entity }: ReportedHookContext) {
+  private getStepStats(entity: TestSuite | TestModule | TestCase) {
     // Track slow running hooks only on verbose mode
     if (!this.options.verbose) {
       return
@@ -317,7 +319,7 @@ export class SummaryReporter implements Reporter {
       )
 
       const slowTasks = [
-        testFile.hook,
+        testFile.step,
         ...testFile.tests.values(),
       ].filter((t): t is SlowTask => t != null && t.visible)
 
@@ -331,8 +333,8 @@ export class SummaryReporter implements Reporter {
           + c.bold(c.yellow(` ${formatTime(Math.max(0, elapsed))}`)),
         )
 
-        if (task.hook?.visible) {
-          summary.push(c.bold(c.yellow(`      ${F_TREE_NODE_END} `)) + task.hook.name)
+        if (task.step?.visible) {
+          summary.push(c.bold(c.yellow(`      ${F_TREE_NODE_END} `)) + task.step.name)
         }
       }
     }
@@ -367,7 +369,7 @@ export class SummaryReporter implements Reporter {
     }
 
     const testFile = this.runningModules.get(id)
-    testFile?.hook?.onFinish()
+    testFile?.step?.onFinish()
     testFile?.tests?.forEach(test => test.onFinish())
 
     this.runningModules.delete(id)

packages/vitest/src/node/reporters/utils.ts

@@ -3,8 +3,8 @@ import type { Vitest } from '../core'
 import type { ResolvedConfig } from '../types/config'
 import type { Reporter } from '../types/reporter'
 import type { BlobReporter } from './blob'
-import type { BenchmarkBuiltinReporters, BenchmarkReporter, BuiltinReporters, DefaultReporter, DotReporter, GithubActionsReporter, HangingProcessReporter, JsonReporter, JUnitReporter, TapReporter } from './index'
-import { BenchmarkReportsMap, ReportersMap } from './index'
+import type { BuiltinReporters, DefaultReporter, DotReporter, GithubActionsReporter, HangingProcessReporter, JsonReporter, JUnitReporter, TapReporter } from './index'
+import { ReportersMap } from './index'
 
 async function loadCustomReporterModule<C extends Reporter>(
   path: string,
@@ -70,32 +70,4 @@ function createReporters(
   return Promise.all(promisedReporters)
 }
 
-function createBenchmarkReporters(
-  reporterReferences: Array<string | Reporter | BenchmarkBuiltinReporters>,
-  runner: ModuleRunner,
-): Promise<(Reporter | BenchmarkReporter)[]> {
-  const promisedReporters = reporterReferences.map(
-    async (referenceOrInstance) => {
-      if (typeof referenceOrInstance === 'string') {
-        if (referenceOrInstance in BenchmarkReportsMap) {
-          const BuiltinReporter
-            = BenchmarkReportsMap[
-              referenceOrInstance as BenchmarkBuiltinReporters
-            ]
-          return new BuiltinReporter()
-        }
-        else {
-          const CustomReporter = await loadCustomReporterModule(
-            referenceOrInstance,
-            runner,
-          )
-          return new CustomReporter()
-        }
-      }
-      return referenceOrInstance
-    },
-  )
-  return Promise.all(promisedReporters)
-}
-
-export { createBenchmarkReporters, createReporters }
+export { createReporters }

packages/vitest/src/node/reporters/verbose.ts

@@ -44,5 +44,11 @@ export class VerboseReporter extends DefaultReporter {
       this.printAnnotations(test, 'log', 3)
       this.log()
     }
+
+    const benchmarks = test.benchmarks()
+    const inlineBenchmarks = benchmarks.filter(b => b.tasks.length > 0)
+    if (inlineBenchmarks.length > 0) {
+      this.printBenchmarkTable(inlineBenchmarks, '')
+    }
   }
 }

packages/vitest/src/node/test-run.ts

@@ -1,11 +1,13 @@
 import type {
   File as RunnerTestFile,
+  TaskEventData,
   TaskEventPack,
   TaskResultPack,
   TaskUpdateEvent,
+  TestArtifact,
   TestAttachment,
+  TestBenchmark,
 } from '@vitest/runner'
-import type { TaskEventData, TestArtifact } from '@vitest/runner/types/tasks'
 import type { SerializedError } from '@vitest/utils'
 import type { UserConsoleLog } from '../types/general'
 import type { Vitest } from './core'
@@ -55,33 +57,35 @@ export class TestRun {
     await this.vitest.report('onUserConsoleLog', log)
   }
 
-  async recordArtifact<Artifact extends TestArtifact>(testId: string, artifact: Artifact): Promise<Artifact> {
-    const task = this.vitest.state.idMap.get(testId)
-    const entity = task && this.vitest.state.getReportedEntity(task)
+  async recordBenchmark(testId: string, benchmark: TestBenchmark): Promise<void> {
+    const testCase = this.getTestCaseById(testId, 'Benchmark')
+    testCase.task.benchmarks.push(benchmark)
+    await this.vitest.report('onTestCaseBenchmark', testCase, benchmark)
+  }
 
-    assert(task && entity, `Entity must be found for task ${task?.name || testId}`)
-    assert(entity.type === 'test', `Artifacts can only be recorded on a test, instead got ${entity.type}`)
+  async recordArtifact<Artifact extends TestArtifact>(testId: string, artifact: Artifact): Promise<Artifact> {
+    const testCase = this.getTestCaseById(testId, 'Artifact')
 
     // annotations won't resolve as artifacts for backwards compatibility until next major
     if (artifact.type === 'internal:annotation') {
-      await this.resolveTestAttachment(entity, artifact.annotation.attachment, artifact.annotation.message)
+      await this.resolveTestAttachment(testCase, artifact.annotation.attachment, artifact.annotation.message)
 
-      entity.task.annotations.push(artifact.annotation)
+      testCase.task.annotations.push(artifact.annotation)
 
-      await this.vitest.report('onTestCaseAnnotate', entity, artifact.annotation)
+      await this.vitest.report('onTestCaseAnnotate', testCase, artifact.annotation)
 
       return artifact
     }
 
     if (Array.isArray(artifact.attachments)) {
       await Promise.all(
-        artifact.attachments.map(attachment => this.resolveTestAttachment(entity, attachment)),
+        artifact.attachments.map(attachment => this.resolveTestAttachment(testCase, attachment)),
       )
     }
 
-    entity.task.artifacts.push(artifact)
+    testCase.task.artifacts.push(artifact)
 
-    await this.vitest.report('onTestCaseArtifactRecord', entity, artifact)
+    await this.vitest.report('onTestCaseArtifactRecord', testCase, artifact)
 
     return artifact
   }
@@ -102,6 +106,15 @@ export class TestRun {
     await this.vitest.report('onTaskUpdate', update, events)
   }
 
+  private getTestCaseById(testId: string, recordType: string) {
+    const task = this.vitest.state.idMap.get(testId)
+    const entity = task && this.vitest.state.getReportedEntity(task)
+
+    assert(task && entity, `Entity must be found for task ${task?.name || testId}`)
+    assert(entity.type === 'test', `${recordType} can only be recorded on a test, instead got ${entity.type}`)
+    return entity
+  }
+
   async end(specifications: TestSpecification[], errors: unknown[], coverage?: unknown): Promise<void> {
     if (coverage) {
       await this.vitest.report('onCoverage', coverage)

packages/vitest/src/node/types/benchmark.ts

@@ -1,9 +1,6 @@
-import type { Arrayable } from '@vitest/utils'
-
-import type { BenchmarkBuiltinReporters } from '../reporters'
-import type { Reporter } from './reporter'
-
 export interface BenchmarkUserOptions {
+  enabled?: boolean
+
   /**
    * Include globs for benchmark test files
    *
@@ -13,7 +10,7 @@ export interface BenchmarkUserOptions {
 
   /**
    * Exclude globs for benchmark test files
-   * @default ['**\/node_modules/**', '**\/dist/**', '**\/cypress/**', '**\/.{idea,git,cache,output,temp}/**', '**\/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build,eslint,prettier}.config.*']
+   * @default []
    */
   exclude?: string[]
 
@@ -24,36 +21,25 @@ export interface BenchmarkUserOptions {
    */
   includeSource?: string[]
 
-  /**
-   * Custom reporters to use for output. Can contain one or more built-in reporter names, reporter instances,
-   * and/or paths to custom reporter files to import.
-   *
-   * @default ['default']
-   */
-  reporters?: Arrayable<BenchmarkBuiltinReporters | Reporter | (string & {})>
-
-  /**
-   * @deprecated Use `benchmark.outputJson` instead
-   */
-  outputFile?:
-    | string
-    | (Partial<Record<BenchmarkBuiltinReporters, string>>
-      & Record<string, string>)
-
-  /**
-   * benchmark output file to compare against
-   */
-  compare?: string
-
-  /**
-   * benchmark output file
-   */
-  outputJson?: string
-
   /**
    * Include `samples` array of benchmark results for API or custom reporter usages.
    * This is disabled by default to reduce memory usage.
    * @default false
    */
-  includeSamples?: boolean
+  retainSamples?: boolean
+
+  /**
+   * Disable warnings when a benchmark accesses module export getters too many times.
+   * @default false
+   */
+  suppressExportGetterWarnings?: boolean
+
+  /**
+   * The name of the parent project that this benchmark project was cloned
+   * from. Populated automatically when Vitest creates the dedicated benchmark
+   * project for a parent project. Used by the runtime as the value for the
+   * `${projectName}` placeholder in `writeResult` / `bench.from()` paths.
+   * @internal
+   */
+  projectName?: string
 }

packages/vitest/src/node/types/config.ts

@@ -73,7 +73,10 @@ export interface EnvironmentOptions {
 
 export type { HappyDOMOptions, JSDOMOptions }
 
-export type VitestRunMode = 'test' | 'benchmark'
+/**
+ * @deprecated
+ */
+export type VitestRunMode = 'test'
 
 export interface ProjectName {
   label: string
@@ -1093,16 +1096,6 @@ export interface UserConfig extends InlineConfig {
    */
   clearScreen?: boolean
 
-  /**
-   * benchmark.compare option exposed at the top level for cli
-   */
-  compare?: string
-
-  /**
-   * benchmark.outputJson option exposed at the top level for cli
-   */
-  outputJson?: string
-
   /**
    * Directory of blob reports to merge
    * @default '.vitest/blob'
@@ -1159,8 +1152,6 @@ export interface ResolvedConfig
     | 'fileParallelism'
     | 'tagsFilter'
   > {
-  mode: VitestRunMode
-
   name: ProjectName['label']
   color?: ProjectName['color']
   base?: string
@@ -1191,10 +1182,7 @@ export interface ResolvedConfig
   cliExclude?: string[]
 
   project: string[]
-  benchmark?: Required<
-    Omit<BenchmarkUserOptions, 'outputFile' | 'compare' | 'outputJson'>
-  >
-  & Pick<BenchmarkUserOptions, 'outputFile' | 'compare' | 'outputJson'>
+  benchmark: Required<BenchmarkUserOptions>
   shard?: {
     index: number
     count: number

packages/vitest/src/node/types/reporter.ts

@@ -1,4 +1,4 @@
-import type { File, TaskEventPack, TaskResultPack, TestAnnotation, TestArtifact } from '@vitest/runner'
+import type { File, TaskEventPack, TaskResultPack, TestAnnotation, TestArtifact, TestBenchmark } from '@vitest/runner'
 import type { Awaitable, SerializedError } from '@vitest/utils'
 import type { UserConsoleLog } from '../../types/general'
 import type { Vitest } from '../core'
@@ -96,4 +96,10 @@ export interface Reporter {
   onHookEnd?: (hook: ReportedHookContext) => Awaitable<void>
 
   onCoverage?: (coverage: unknown) => Awaitable<void>
+
+  /**
+   * @experimental
+   * Called after the benchmark is finished.
+   */
+  onTestCaseBenchmark?: (testCase: TestCase, benchmark: TestBenchmark) => Awaitable<void>
 }

packages/vitest/src/public/index.ts

@@ -34,7 +34,15 @@ export { Snapshots } from '../integrations/snapshot/chai'
 
 export { vi, vitest } from '../integrations/vi'
 export type { VitestUtils } from '../integrations/vi'
-export { bench } from '../runtime/benchmark'
+export type {
+  Bench,
+  BenchCompareOptions,
+  BenchFnOptions,
+  BenchFromSource,
+  BenchRegistration,
+  BenchResult,
+  BenchStorage,
+} from '../runtime/benchmark'
 
 export type {
   RuntimeConfig,
@@ -45,18 +53,7 @@ export type {
 
 export { VitestEvaluatedModules as EvaluatedModules } from '../runtime/moduleRunner/evaluatedModules'
 
-export { NodeBenchmarkRunner as BenchmarkRunner } from '../runtime/runners/benchmark'
 export { TestRunner } from '../runtime/runners/test'
-export type {
-  BenchFactory,
-  BenchFunction,
-  Benchmark,
-  BenchmarkAPI,
-  BenchmarkResult,
-  BenchOptions,
-  BenchTask,
-  BenchTaskResult,
-} from '../runtime/types/benchmark'
 export { assertType } from '../typecheck/assertType'
 
 export type { AssertType } from '../typecheck/assertType'
@@ -117,6 +114,7 @@ export {
   test,
 } from '@vitest/runner'
 export type {
+  BaselineData,
   ImportDuration,
   OnTestFailedHandler,
   OnTestFinishedHandler,
@@ -144,6 +142,8 @@ export type {
   TestArtifactLocation,
   TestArtifactRegistry,
   TestAttachment,
+  TestBenchmark,
+  TestBenchmarkTask,
   TestContext,
   TestFunction,
   TestOptions,

packages/vitest/src/public/node.ts

@@ -44,8 +44,6 @@ export type { SerializedTestProject, TestProject } from '../node/project'
 
 export {
   AgentReporter,
-  BenchmarkReporter,
-  BenchmarkReportsMap,
   DefaultReporter,
   DotReporter,
   GithubActionsReporter,
@@ -56,12 +54,10 @@ export {
   ReportersMap,
   TapFlatReporter,
   TapReporter,
-  VerboseBenchmarkReporter,
   VerboseReporter,
 } from '../node/reporters'
 export type {
   BaseReporter,
-  BenchmarkBuiltinReporters,
   BuiltinReporterOptions,
   BuiltinReporters,
   JsonAssertionResult,

packages/vitest/src/runtime/benchmark.ts

@@ -1,70 +1,517 @@
-import type { Test } from '@vitest/runner'
-import type { BenchFunction, BenchmarkAPI, BenchOptions } from './types/benchmark'
-import { getCurrentSuite } from '@vitest/runner'
-import { createChainable } from '@vitest/runner/utils'
-import { noop } from '@vitest/utils/helpers'
+import type { BaselineData, Test, TestBenchmark, TestBenchmarkTask } from '@vitest/runner'
+import type {
+  BenchOptions as BenchCompareOptions,
+  Fn,
+  FnOptions,
+  TaskResultCompleted,
+  TaskResultRuntimeInfo,
+  TaskResultTimestampProviderInfo,
+  Task as TinybenchTask,
+} from 'tinybench'
+import type { SerializedConfig } from './config'
+import { isAbsolute, relative } from 'pathe'
+import { Bench as Tinybench } from 'tinybench'
+import c from 'tinyrainbow'
+import { rpc } from './rpc'
+import { TestRunner } from './runners/test'
 import { getWorkerState } from './utils'
 
-const benchFns = new WeakMap<Test, BenchFunction>()
-const benchOptsMap = new WeakMap()
+const now = globalThis.performance
+  ? globalThis.performance.now.bind(globalThis.performance)
+  : Date.now
 
-export function getBenchOptions(key: Test): BenchOptions {
-  return benchOptsMap.get(key)
+const kRegistration: unique symbol = Symbol('registration')
+const kFromSource: unique symbol = Symbol('fromSource')
+const kPerProject: unique symbol = Symbol('perProject')
+const kWriteResult: unique symbol = Symbol('writeResult')
+export const kFinalize: unique symbol = Symbol('finalize')
+
+type ExtractBenchNames<T extends BenchRegistration<any>[]> = Exclude<{
+  [K in keyof T]: T[K] extends BenchRegistration<infer N> ? N : never
+}[number], never>
+
+// We throw an error if benchmark did not complete, so it will always be TaskResultCompleted
+export type BenchResult = TaskResultCompleted & TaskResultRuntimeInfo & TaskResultTimestampProviderInfo
+
+export interface BenchStorage<T extends string> {
+  get: (name: T) => BenchResult
 }
 
-export function getBenchFn(key: Test): BenchFunction {
-  return benchFns.get(key)!
+export type { BenchOptions as BenchCompareOptions } from 'tinybench'
+
+/**
+ * Options accepted by `bench(name, options, fn)`. Extends tinybench's
+ * `FnOptions` with Vitest-specific fields.
+ */
+export interface BenchFnOptions extends FnOptions {
+  /**
+   * Path (relative to the project root) where the benchmark result is written
+   * after a successful run. The string `${projectName}` is substituted with
+   * the current project name. Absolute paths are accepted as long as they
+   * resolve inside the project root.
+   */
+  writeResult?: string
+  /**
+   * Mark this benchmark as a per-project entry. Per-project tasks still appear
+   * in the inline comparison table for the current run, and Vitest additionally
+   * collects them across projects and prints a single cross-project table at
+   * the end of the run.
+   */
+  perProject?: boolean
 }
 
-export const bench: BenchmarkAPI = createBenchmark(function (
-  name,
-  fn: BenchFunction = noop,
-  options: BenchOptions = {},
-) {
-  if (getWorkerState().config.mode !== 'benchmark') {
-    throw new Error('`bench()` is only available in benchmark mode.')
+export interface BenchRegistration<Name extends string> {
+  name: Name
+  /**
+   * The benchmark function. Absent for registrations created via `bench.from()`.
+   */
+  fn?: Fn
+  /**
+   * Per-benchmark options (`beforeEach`, `beforeAll`, etc.). Absent for
+   * registrations created via `bench.from()`.
+   */
+  fnOpts?: FnOptions
+  /**
+   * @internal
+   */
+  [kRegistration]: true
+  run: (options?: BenchCompareOptions) => Promise<BenchResult>
+}
+
+interface BenchCompare {
+  <Args extends BenchRegistration<any>[]>(...args: Args): Promise<BenchStorage<ExtractBenchNames<Args>>>
+  <Args extends BenchRegistration<any>[]>(...args: [...Args, BenchCompareOptions]): Promise<BenchStorage<ExtractBenchNames<Args>>>
+}
+
+interface BenchFactory {
+  <Name extends string>(name: Name | Function, fn: Fn): BenchRegistration<Name>
+  <Name extends string>(name: Name | Function, options: BenchFnOptions, fn: Fn): BenchRegistration<Name>
+}
+
+export interface BenchFromSource {
+  (): BaselineData | Promise<BaselineData>
+}
+
+interface BenchFrom {
+  <Name extends string>(name: Name | Function, source: string | BenchFromSource): BenchRegistration<Name>
+}
+
+export interface Bench extends BenchFactory {
+  compare: BenchCompare
+  from: BenchFrom
+  /** @internal */
+  [kFinalize]: () => void
+}
+
+interface RunnableRegistration<Name extends string> extends BenchRegistration<Name> {
+  fn: Fn
+  fnOpts?: FnOptions
+  [kWriteResult]?: string
+  [kPerProject]?: true
+}
+
+interface FromRegistration<Name extends string> extends BenchRegistration<Name> {
+  [kFromSource]: string | BenchFromSource
+}
+
+function isFromRegistration(reg: BenchRegistration<any>): reg is FromRegistration<any> {
+  return kFromSource in reg
+}
+
+function substitutePath(template: string, projectName: string | undefined): string {
+  return template.replace(/\$\{projectName\}/g, projectName ?? '')
+}
+
+export function createBench(test: Test, config: SerializedConfig): Bench {
+  let benchIdx = 0
+  const pending = new Set<BenchRegistration<any>>()
+  const createTinybench = (options?: BenchCompareOptions) => {
+    const currentIndex = ++benchIdx
+    return new Tinybench({
+      signal: test.context.signal,
+      name: `${test.fullTestName} ${currentIndex}`,
+      retainSamples: config.benchmark.retainSamples,
+      ...options,
+      now,
+    })
   }
 
-  const task = getCurrentSuite().task(formatName(name), {
-    ...this,
-    meta: {
-      benchmark: true,
-    },
+  const resolveTemplate = (template: string) => substitutePath(template, config.benchmark.projectName)
+
+  const resolveFromSource = async (source: string | BenchFromSource): Promise<BaselineData> => {
+    if (typeof source === 'function') {
+      return source()
+    }
+    const resolved = resolveTemplate(source)
+    const data = await rpc().readBenchmarkResult(resolved)
+    if (data == null) {
+      throw new Error(`\`bench.from()\` could not find a result file at "${resolved}". Run the source benchmark first to create it.`)
+    }
+    return data
+  }
+
+  const taskFromBaseline = (
+    name: string,
+    data: BaselineData,
+  ): TestBenchmarkTask => ({
+    name,
+    latency: data.latency,
+    throughput: data.throughput,
+    period: data.period,
+    totalTime: data.totalTime,
+    rank: 0,
+    fromStore: true,
   })
-  benchFns.set(task, fn)
-  benchOptsMap.set(task, options)
-  // vitest runner sets mode to `todo` if handler is not passed down
-  // but we store handler separately
-  if (!this.todo && task.mode === 'todo') {
-    task.mode = 'run'
+
+  const createCompareStorage = <T extends string>(
+    bench: Tinybench,
+    fromResults?: Map<string, BaselineData>,
+  ): BenchStorage<T> => {
+    return {
+      get(name: T) {
+        const stored = fromResults?.get(name)
+        if (stored) {
+          return stored as BenchResult
+        }
+        const task = bench.getTask(name)
+        if (!task) {
+          throw new Error(`task "${name}" was not defined`)
+        }
+        return task.result as BenchResult
+      },
+    }
   }
-})
 
-function createBenchmark(
-  fn: (
-    this: Record<'skip' | 'only' | 'todo', boolean | undefined>,
-    name: string | Function,
-    fn?: BenchFunction,
-    options?: BenchOptions,
-  ) => void,
-) {
-  const benchmark = createChainable(
-    ['skip', 'only', 'todo'],
-    fn,
-  ) as BenchmarkAPI
+  interface TaskMeta { perProject?: true }
 
-  benchmark.skipIf = (condition: any) =>
-    (condition ? benchmark.skip : benchmark) as BenchmarkAPI
-  benchmark.runIf = (condition: any) =>
-    (condition ? benchmark : benchmark.skip) as BenchmarkAPI
+  const serializeBenchmark = (
+    tinybenchTasks: TinybenchTask[],
+    name: string | undefined,
+    taskMeta?: Map<string, TaskMeta>,
+    fromTasks?: TestBenchmarkTask[],
+  ): TestBenchmark => {
+    const tasks: TestBenchmarkTask[] = tinybenchTasks.map((t) => {
+      const result = t.result
+      if (result.state === 'errored') {
+        throw result.error
+      }
+      if (result.state !== 'completed') {
+        throw new Error(`task "${t.name}" did not complete: received "${result.state}"`)
+      }
+      return {
+        name: t.name,
+        latency: result.latency,
+        throughput: result.throughput,
+        period: result.period,
+        totalTime: result.totalTime,
+        rank: 0,
+        ...taskMeta?.get(t.name),
+      }
+    })
+    if (fromTasks) {
+      tasks.push(...fromTasks)
+    }
+    tasks.sort((a, b) => a.latency.mean - b.latency.mean)
+    tasks.forEach((task, idx) => {
+      task.rank = idx + 1
+    })
+    return {
+      name: name || test.fullTestName,
+      tasks,
+    }
+  }
 
-  return benchmark as BenchmarkAPI
+  const recordBenchmark = async (
+    tinybenchTasks: TinybenchTask[],
+    name: string | undefined,
+    taskMeta?: Map<string, TaskMeta>,
+    fromTasks?: TestBenchmarkTask[],
+  ) => {
+    const serializedBenchmark = serializeBenchmark(tinybenchTasks, name, taskMeta, fromTasks)
+    test.benchmarks.push(serializedBenchmark)
+    await rpc().onTestBenchmark(test.id, serializedBenchmark)
+  }
+
+  const writeResultArtifact = async (template: string, result: BenchResult) => {
+    const resolved = resolveTemplate(template)
+    const data: BaselineData = {
+      latency: result.latency,
+      throughput: result.throughput,
+      period: result.period,
+      totalTime: result.totalTime,
+    }
+    await rpc().writeBenchmarkResult(resolved, data)
+  }
+
+  const runBenchmarks = async (tinybench: Tinybench) => {
+    const workerState = getWorkerState()
+    const getterTracker = workerState.getterTracker
+    getterTracker?.resetInvocations()
+    try {
+      return await TestRunner.runBenchmarks(tinybench)
+    }
+    finally {
+      const excessiveInvocations = config.benchmark.suppressExportGetterWarnings
+        ? undefined
+        : getterTracker?.getExcessiveInvocations()
+      if (excessiveInvocations?.length) {
+        const entries = excessiveInvocations
+          .map(({ moduleId, exportName }) => `  - ${formatModuleId(moduleId, workerState.config.root)} > ${exportName}`)
+          .join('\n')
+        console.warn(
+          [
+            c.yellow(c.bold('Benchmark Warning')),
+            `Benchmark ${c.bold(`"${tinybench.name}"`)} accessed module export getters too many times.`,
+            '',
+            'This can make results unreliable because export getters add overhead.',
+            'See https://vitest.dev/guide/benchmarking#module-runner-overhead',
+            '',
+            'Tracked exports:',
+            entries,
+          ].join('\n'),
+        )
+      }
+    }
+  }
+
+  const runSingle = async (
+    name: string,
+    fn: Fn,
+    fnOpts: FnOptions | undefined,
+    options: BenchCompareOptions | undefined,
+    meta: TaskMeta | undefined,
+    writeResult: string | undefined,
+  ): Promise<BenchResult> => {
+    const tinybench = createTinybench(options).add(name, fn, fnOpts)
+    const tasks = await runBenchmarks(tinybench)
+    const task = tinybench.getTask(name)!
+    if (task.result.state === 'errored') {
+      throw task.result.error
+    }
+    await recordBenchmark(tasks, tinybench.name, meta ? new Map([[name, meta]]) : undefined)
+    if (writeResult) {
+      await writeResultArtifact(writeResult, task.result as BenchResult)
+    }
+    return task.result as BenchResult
+  }
+
+  const runFrom = async (
+    name: string,
+    source: string | BenchFromSource,
+  ): Promise<BenchResult> => {
+    const data = await resolveFromSource(source)
+    const benchmark: TestBenchmark = {
+      name: test.fullTestName,
+      tasks: [{ ...taskFromBaseline(name, data), rank: 1 }],
+    }
+    test.benchmarks.push(benchmark)
+    await rpc().onTestBenchmark(test.id, benchmark)
+    return data as BenchResult
+  }
+
+  const bench: Bench = (nameOrFunction: string | Function, a: Fn | BenchFnOptions, b?: Fn | BenchFnOptions) => {
+    validateBenchmarkProject(config)
+    const { fn, fnOpts, writeResult, perProject } = normalizeBenchArgs(a, b)
+    const name = typeof nameOrFunction === 'function' ? nameOrFunction.name || '<anonymous>' : nameOrFunction
+    const meta: TaskMeta | undefined = perProject ? { perProject: true } : undefined
+    const registration: RunnableRegistration<string> = {
+      [kRegistration]: true,
+      name,
+      fn,
+      fnOpts,
+      run: (options?: BenchCompareOptions) => {
+        pending.delete(registration)
+        return runSingle(name, fn, fnOpts, options, meta, writeResult)
+      },
+    }
+    if (perProject) {
+      registration[kPerProject] = true
+    }
+    if (writeResult) {
+      registration[kWriteResult] = writeResult
+    }
+    pending.add(registration)
+    return registration
+  }
+
+  bench.from = <Name extends string>(nameOrFunction: Name | Function, source: string | BenchFromSource): BenchRegistration<Name> => {
+    validateBenchmarkProject(config)
+    if (typeof nameOrFunction !== 'string' && typeof nameOrFunction !== 'function') {
+      throw new TypeError('`bench.from()` requires a name (string or named function) as its first argument.')
+    }
+    if (typeof source !== 'string' && typeof source !== 'function') {
+      throw new TypeError('`bench.from()` expects a string path or a function returning the result data as its second argument.')
+    }
+    const name = (typeof nameOrFunction === 'function' ? nameOrFunction.name || '<anonymous>' : nameOrFunction) as Name
+    const registration: FromRegistration<Name> = {
+      [kRegistration]: true,
+      [kFromSource]: source,
+      name,
+      run: () => {
+        pending.delete(registration)
+        return runFrom(name, source)
+      },
+    }
+    pending.add(registration)
+    return registration
+  }
+
+  bench.compare = async (...args) => {
+    validateBenchmarkProject(config)
+
+    // extract optional trailing BenchCompareOptions argument
+    const lastArg = args[args.length - 1]
+    const isOptions = lastArg != null && typeof lastArg === 'object' && !(kRegistration in lastArg)
+    const benchOptions = isOptions ? args.pop() as BenchCompareOptions : undefined
+    const registrations = args as BenchRegistration<any>[]
+
+    // Mark every passed-in registration as consumed before validation so a
+    // throwing `bench.compare()` (wrong arity, wrong shape) doesn't also
+    // trigger the unrun-bench warning — the user's intent was to consume them.
+    for (const reg of registrations) {
+      if (reg != null && typeof reg === 'object' && kRegistration in reg) {
+        pending.delete(reg)
+      }
+    }
+
+    if (registrations.length < 2) {
+      throw new SyntaxError(`\`bench.compare()\` requires at least 2 benchmarks, received ${registrations.length} instead. ${registrations.length === 1 ? 'Consider calling `bench().run()`. ' : 'Define benchmarks by calling `bench()`. '}See https://vitest.dev/guide/benchmarking#comparing-benchmarks`)
+    }
+    for (const reg of registrations) {
+      if (reg == null || typeof reg !== 'object' || !(kRegistration in reg)) {
+        throw new SyntaxError('`bench.compare()` expects every argument to be the return value of `bench` or `bench.from`.')
+      }
+    }
+
+    const runnable: RunnableRegistration<any>[] = []
+    const fromEntries: FromRegistration<any>[] = []
+    for (const reg of registrations) {
+      if (isFromRegistration(reg)) {
+        fromEntries.push(reg)
+      }
+      else {
+        runnable.push(reg as RunnableRegistration<any>)
+      }
+    }
+
+    const taskMeta = new Map<string, TaskMeta>()
+    for (const reg of runnable) {
+      if (reg[kPerProject]) {
+        taskMeta.set(reg.name, { perProject: true })
+      }
+    }
+
+    const fromResults = new Map<string, BaselineData>()
+    const fromTasks: TestBenchmarkTask[] = []
+    if (fromEntries.length > 0) {
+      const resolved = await Promise.all(
+        fromEntries.map(async (reg) => {
+          const data = await resolveFromSource(reg[kFromSource])
+          return { reg, data }
+        }),
+      )
+      for (const { reg, data } of resolved) {
+        fromResults.set(reg.name, data)
+        fromTasks.push(taskFromBaseline(reg.name, data))
+      }
+    }
+
+    const tinybench = createTinybench(benchOptions)
+    runnable.forEach((reg) => {
+      tinybench.add(reg.name, reg.fn, reg.fnOpts)
+    })
+
+    let tasks: TinybenchTask[] = []
+    if (runnable.length > 0) {
+      tasks = await runBenchmarks(tinybench)
+      const errors = tinybench.tasks
+        .filter(task => task.result.state === 'errored')
+        .map(task => (task.result as any).error)
+      if (errors.length > 0) {
+        throw new AggregateError(errors, 'Some benchmarks failed')
+      }
+    }
+
+    await recordBenchmark(tasks, tinybench.name, taskMeta, fromTasks)
+
+    // write artifacts for every runnable registration that requested it. We
+    // do this after recording so a write failure can't be confused with a
+    // benchmark failure in the reporter output.
+    await Promise.all(
+      runnable
+        .filter(reg => reg[kWriteResult] != null)
+        .map((reg) => {
+          const task = tinybench.getTask(reg.name)!
+          return writeResultArtifact(reg[kWriteResult]!, task.result as BenchResult)
+        }),
+    )
+
+    return createCompareStorage(tinybench, fromResults)
+  }
+
+  bench[kFinalize] = () => {
+    if (pending.size === 0) {
+      return
+    }
+    const names = [...pending].map(reg => `"${reg.name}"`).join(', ')
+    pending.clear()
+    console.warn(
+      [
+        c.yellow(c.bold('Benchmark Warning')),
+        `Test ${c.bold(`"${test.fullTestName}"`)} registered benchmarks that never ran: ${names}.`,
+        '',
+        'Call `.run()` on the registration, or pass it to `bench.compare()`.',
+        'See https://vitest.dev/guide/benchmarking#defining-a-benchmark',
+      ].join('\n'),
+    )
+  }
+
+  return bench
 }
 
-function formatName(name: string | Function) {
-  return typeof name === 'string'
-    ? name
-    : typeof name === 'function'
-      ? name.name || '<anonymous>'
-      : String(name)
+function formatModuleId(moduleId: string, root: string): string {
+  if (!root || !isAbsolute(moduleId)) {
+    return moduleId
+  }
+  return relative(root, moduleId)
+}
+
+function normalizeBenchArgs(
+  a: Fn | BenchFnOptions,
+  b: Fn | BenchFnOptions | undefined,
+): { fn: Fn; fnOpts: FnOptions | undefined; writeResult: string | undefined; perProject: boolean } {
+  if (typeof a === 'function') {
+    if (b !== undefined) {
+      throw new TypeError('`bench()` does not accept options as the third argument. Pass options as the second argument instead: `bench(name, options, fn)`.')
+    }
+    return { fn: a, fnOpts: undefined, writeResult: undefined, perProject: false }
+  }
+  if (typeof b !== 'function') {
+    throw new TypeError('`bench()` expects a benchmark function. Call `bench(name, fn)` or `bench(name, options, fn)`.')
+  }
+  // Strip vitest-specific fields only when present so we don't allocate a new
+  // object — preserving referential identity matters: users inspect
+  // `registration.fnOpts` and tinybench's `add` sees the same object the
+  // caller passed in.
+  if (a.writeResult === undefined && a.perProject === undefined) {
+    return { fn: b, fnOpts: a as FnOptions, writeResult: undefined, perProject: false }
+  }
+  const { writeResult, perProject, ...fnOpts } = a
+  return {
+    fn: b,
+    fnOpts: Object.keys(fnOpts).length > 0 ? fnOpts as FnOptions : undefined,
+    writeResult,
+    perProject: perProject ?? false,
+  }
+}
+
+function validateBenchmarkProject(config: SerializedConfig) {
+  if (!config.benchmark.enabled) {
+    throw new Error(
+      `Cannot use the \`bench\` test-context fixture within a regular test run. `
+      + `Benchmarks are inherently flaky, so Vitest runs them in a dedicated project based on the \`benchmark.include\` pattern (default \`**/*.{bench,benchmark}.?(c|m)[jt]s?(x)\`). `
+      + `Move this code to a file matched by \`benchmark.include\`, and make sure \`bench\` is destructured from the test context (\`test('...', async ({ bench }) => { ... })\`) — it is not a top-level export of \`vitest\`. `
+      + `See https://vitest.dev/guide/benchmarking#stability`,
+    )
+  }
 }

packages/vitest/src/runtime/config.ts

@@ -1,6 +1,6 @@
 import type { Config as FakeTimersConfig } from '@sinonjs/fake-timers'
 import type { PrettyFormatOptions } from '@vitest/pretty-format'
-import type { SequenceHooks, SequenceSetupFiles, SerializableRetry, TestTagDefinition } from '@vitest/runner'
+import type { SequenceHooks, VitestRunnerConfig } from '@vitest/runner'
 import type { SnapshotEnvironment, SnapshotUpdateState } from '@vitest/snapshot'
 import type { SerializedDiffOptions } from '@vitest/utils/diff'
 import type { LabelColor } from '../types/general'
@@ -8,8 +8,7 @@ import type { LabelColor } from '../types/general'
 /**
  * Config that tests have access to.
  */
-export interface SerializedConfig {
-  name: string | undefined
+export interface SerializedConfig extends VitestRunnerConfig {
   color?: LabelColor
   globals: boolean
   base: string | undefined
@@ -18,16 +17,8 @@ export interface SerializedConfig {
   runner: string | undefined
   isolate: boolean
   maxWorkers: number
-  mode: 'test' | 'benchmark'
   bail: number | undefined
   environmentOptions?: Record<string, any>
-  root: string
-  setupFiles: string[]
-  passWithNoTests: boolean
-  testNamePattern: RegExp | undefined
-  allowOnly: boolean
-  testTimeout: number
-  hookTimeout: number
   clearMocks: boolean
   mockReset: boolean
   restoreMocks: boolean
@@ -35,7 +26,6 @@ export interface SerializedConfig {
   unstubEnvs: boolean
   // TODO: make optional
   fakeTimers: FakeTimersConfig
-  maxConcurrency: number
   defines: Record<string, any>
   expect: {
     requireAssertions?: boolean
@@ -45,13 +35,6 @@ export interface SerializedConfig {
     }
   }
   printConsoleTrace: boolean | undefined
-  sequence: {
-    shuffle?: boolean
-    concurrent?: boolean
-    seed: number
-    hooks: SequenceHooks
-    setupFiles: SequenceSetupFiles
-  }
   deps: {
     web: {
       transformAssets?: boolean
@@ -84,8 +67,6 @@ export interface SerializedConfig {
     allowWrite: boolean | undefined
   }
   diff: string | SerializedDiffOptions | undefined
-  retry: SerializableRetry
-  includeTaskLocation: boolean | undefined
   inspect: boolean | string | undefined
   inspectBrk: boolean | string | undefined
   inspector: {
@@ -130,8 +111,11 @@ export interface SerializedConfig {
   detectAsyncLeaks: boolean
   coverage: SerializedCoverageConfig
   benchmark: {
-    includeSamples: boolean
-  } | undefined
+    enabled: boolean
+    retainSamples: boolean
+    suppressExportGetterWarnings: boolean
+    projectName: string
+  }
   serializedDefines: string
   experimental: {
     fsModuleCache: boolean
@@ -152,9 +136,6 @@ export interface SerializedConfig {
       browserSdkPath?: string
     } | undefined
   }
-  tags: TestTagDefinition[]
-  tagsFilter: string[] | undefined
-  strictTags: boolean
   mergeReportsLabel: string | undefined
   slowTestThreshold: number | undefined
   disableColors: boolean

packages/vitest/src/runtime/getter-tracker.ts

@@ -0,0 +1,37 @@
+export class GetterTracker {
+  static EXPORTS_MAX_INVOCATIONS = 1_000_000
+
+  private invocations = new Map<string, number>()
+  private excessiveInvocations = new Map<string, GetterTrackerExport>()
+
+  public createTracker(
+    moduleId: string,
+    defineExport: (name: string, getter: () => unknown) => void,
+  ): (name: string, getter: () => unknown) => void {
+    return (name, getter) => {
+      const key = `${moduleId}:${name}`
+      defineExport(name, () => {
+        const count = (this.invocations.get(key) || 0) + 1
+        this.invocations.set(key, count)
+        if (count > GetterTracker.EXPORTS_MAX_INVOCATIONS && !this.excessiveInvocations.has(key)) {
+          this.excessiveInvocations.set(key, { moduleId, exportName: name })
+        }
+        return getter()
+      })
+    }
+  }
+
+  public resetInvocations(): void {
+    this.invocations.clear()
+    this.excessiveInvocations.clear()
+  }
+
+  public getExcessiveInvocations(): GetterTrackerExport[] {
+    return [...this.excessiveInvocations.values()]
+  }
+}
+
+export interface GetterTrackerExport {
+  moduleId: string
+  exportName: string
+}

packages/vitest/src/runtime/moduleRunner/moduleEvaluator.ts

@@ -5,6 +5,7 @@ import type {
   ModuleRunnerContext,
   ModuleRunnerImportMeta,
 } from 'vite/module-runner'
+import type { GetterTracker } from '../getter-tracker'
 import type { VitestEvaluatedModules } from './evaluatedModules'
 import type { ModuleExecutionInfo } from './moduleDebug'
 import type { VitestVmOptions } from './moduleRunner'
@@ -30,6 +31,7 @@ export interface VitestModuleEvaluatorOptions {
   getCurrentTestFilepath?: () => string | undefined
   compiledFunctionArgumentsNames?: string[]
   compiledFunctionArgumentsValues?: unknown[]
+  getterTracker?: GetterTracker
   /**
    * @internal
    */
@@ -43,6 +45,9 @@ export class VitestModuleEvaluator implements ModuleEvaluator {
 
   private compiledFunctionArgumentsNames?: string[]
   private compiledFunctionArgumentsValues: unknown[] = []
+  private getterTracker: GetterTracker | undefined
+
+  static EXPORTS_MAX_INVOCATIONS = 1_000_000
 
   private primitives: {
     Object: typeof Object
@@ -81,6 +86,7 @@ export class VitestModuleEvaluator implements ModuleEvaluator {
         Reflect,
       }
     }
+    this.getterTracker = options.getterTracker
   }
 
   private convertIdToImportUrl(id: string) {
@@ -296,6 +302,7 @@ export class VitestModuleEvaluator implements ModuleEvaluator {
     argumentsList.push(
       // TODO@discuss deprecate in Vitest 5, remove in Vitest 6(?)
       // backwards compat for vite-node
+      // https://github.com/vitest-dev/vitest/issues/10292
       '__filename',
       '__dirname',
       'module',
@@ -333,19 +340,26 @@ export class VitestModuleEvaluator implements ModuleEvaluator {
         ? vm.runInContext(wrappedCode, this.vm.context, options)
         : vm.runInThisContext(wrappedCode, options)
 
+      const __vite_ssr_exportName__ = context.__vite_ssr_exportName__
+        || ((name: string, getter: () => unknown) => Object.defineProperty(context[ssrModuleExportsKey], name, {
+          enumerable: true,
+          configurable: true,
+          get: getter,
+        }))
+
+      let __vite_track_exportName__: ((name: string, getter: () => unknown) => void) | undefined
+      const getterTracker = this.getterTracker
+      if (getterTracker) {
+        __vite_track_exportName__ = getterTracker.createTracker(module.id, __vite_ssr_exportName__)
+      }
+
       await initModule(
         context[ssrModuleExportsKey],
         context[ssrImportMetaKey],
         context[ssrImportKey],
         context[ssrDynamicImportKey],
         context[ssrExportAllKey],
-        // vite 7 support, remove when vite 7+ is supported
-        context.__vite_ssr_exportName__
-        || ((name: string, getter: () => unknown) => Object.defineProperty(context[ssrModuleExportsKey], name, {
-          enumerable: true,
-          configurable: true,
-          get: getter,
-        })),
+        __vite_track_exportName__ || __vite_ssr_exportName__,
 
         cjsGlobals.__filename,
         cjsGlobals.__dirname,

packages/vitest/src/runtime/moduleRunner/startVitestModuleRunner.ts

@@ -66,6 +66,7 @@ export function startVitestModuleRunner(options: ContextModuleRunnerOptions): Vi
         return state().config.deps.interopDefault
       },
       getCurrentTestFilepath: () => state().filepath,
+      getterTracker: state().getterTracker,
     },
   )
 

packages/vitest/src/runtime/runners/benchmark.ts

@@ -1,180 +0,0 @@
-import type {
-  Suite,
-  Task,
-  TaskUpdateEvent,
-  VitestRunner,
-  VitestRunnerImportSource,
-} from '@vitest/runner'
-import type { ModuleRunner } from 'vite/module-runner'
-import type { SerializedConfig } from '../config'
-// import type { VitestExecutor } from '../execute'
-import type {
-  Benchmark,
-  BenchmarkResult,
-  BenchTask,
-} from '../types/benchmark'
-import { updateTask as updateRunnerTask } from '@vitest/runner'
-import { createDefer } from '@vitest/utils/helpers'
-import { getSafeTimers } from '@vitest/utils/timers'
-import { getBenchFn, getBenchOptions } from '../benchmark'
-import { getWorkerState } from '../utils'
-
-function createBenchmarkResult(name: string): BenchmarkResult {
-  return {
-    name,
-    rank: 0,
-    rme: 0,
-    samples: [] as number[],
-  } as BenchmarkResult
-}
-
-const benchmarkTasks = new WeakMap<Benchmark, import('tinybench').Task>()
-
-async function runBenchmarkSuite(suite: Suite, runner: NodeBenchmarkRunner) {
-  const { Task, Bench } = await runner.importTinybench()
-
-  const start = performance.now()
-
-  const benchmarkGroup: Benchmark[] = []
-  const benchmarkSuiteGroup = []
-  for (const task of suite.tasks) {
-    if (task.mode !== 'run' && task.mode !== 'queued') {
-      continue
-    }
-
-    if (task.meta?.benchmark) {
-      benchmarkGroup.push(task as Benchmark)
-    }
-    else if (task.type === 'suite') {
-      benchmarkSuiteGroup.push(task)
-    }
-  }
-
-  // run sub suites sequentially
-  for (const subSuite of benchmarkSuiteGroup) {
-    await runBenchmarkSuite(subSuite, runner)
-  }
-
-  if (benchmarkGroup.length) {
-    const defer = createDefer()
-    suite.result = {
-      state: 'run',
-      startTime: start,
-      benchmark: createBenchmarkResult(suite.name),
-    }
-    updateTask('suite-prepare', suite)
-
-    const addBenchTaskListener = (
-      task: InstanceType<typeof Task>,
-      benchmark: Benchmark,
-    ) => {
-      task.addEventListener(
-        'complete',
-        (e) => {
-          const task = e.task
-          const taskRes = task.result!
-          const result = benchmark.result!.benchmark!
-          benchmark.result!.state = 'pass'
-          Object.assign(result, taskRes)
-          // compute extra stats and free raw samples as early as possible
-          const samples = result.samples
-          result.sampleCount = samples.length
-          result.median = samples.length % 2
-            ? samples[Math.floor(samples.length / 2)]
-            : (samples[samples.length / 2] + samples[samples.length / 2 - 1]) / 2
-          if (!runner.config.benchmark?.includeSamples) {
-            result.samples.length = 0
-          }
-          updateTask('test-finished', benchmark)
-        },
-        {
-          once: true,
-        },
-      )
-      task.addEventListener(
-        'error',
-        (e) => {
-          const task = e.task
-          defer.reject(benchmark ? task.result!.error : e)
-        },
-        {
-          once: true,
-        },
-      )
-    }
-
-    benchmarkGroup.forEach((benchmark) => {
-      const options = getBenchOptions(benchmark)
-      const benchmarkInstance = new Bench(options)
-
-      const benchmarkFn = getBenchFn(benchmark)
-
-      benchmark.result = {
-        state: 'run',
-        startTime: start,
-        benchmark: createBenchmarkResult(benchmark.name),
-      }
-
-      const task = new Task(benchmarkInstance, benchmark.name, benchmarkFn)
-      benchmarkTasks.set(benchmark, task)
-      addBenchTaskListener(task, benchmark)
-    })
-
-    const { setTimeout } = getSafeTimers()
-    const tasks: [BenchTask, Benchmark][] = []
-
-    for (const benchmark of benchmarkGroup) {
-      const task = benchmarkTasks.get(benchmark)!
-      updateTask('test-prepare', benchmark)
-      await task.warmup()
-      tasks.push([
-        await new Promise<BenchTask>(resolve =>
-          setTimeout(async () => {
-            resolve(await task.run())
-          }),
-        ),
-        benchmark,
-      ])
-    }
-
-    suite.result!.duration = performance.now() - start
-    suite.result!.state = 'pass'
-
-    updateTask('suite-finished', suite)
-    defer.resolve(null)
-
-    await defer
-  }
-
-  function updateTask(event: TaskUpdateEvent, task: Task) {
-    updateRunnerTask(event, task, runner)
-  }
-}
-
-export class NodeBenchmarkRunner implements VitestRunner {
-  private moduleRunner!: ModuleRunner
-
-  constructor(public config: SerializedConfig) {}
-
-  async importTinybench(): Promise<typeof import('tinybench')> {
-    return await import('tinybench')
-  }
-
-  importFile(filepath: string, source: VitestRunnerImportSource): unknown {
-    if (source === 'setup') {
-      const moduleNode = getWorkerState().evaluatedModules.getModuleById(filepath)
-      if (moduleNode) {
-        getWorkerState().evaluatedModules.invalidateModule(moduleNode)
-      }
-    }
-    return this.moduleRunner.import(filepath)
-  }
-
-  async runSuite(suite: Suite): Promise<void> {
-    await runBenchmarkSuite(suite, this)
-  }
-
-  async runTask(): Promise<void> {
-    throw new Error('`test()` and `it()` is only available in test mode.')
-  }
-}

packages/vitest/src/runtime/runners/index.ts

@@ -6,7 +6,6 @@ import { takeCoverageInsideWorker } from '../../integrations/coverage'
 import { rpc } from '../rpc'
 import { loadDiffConfig, loadSnapshotSerializers } from '../setup-common'
 import { getWorkerState } from '../utils'
-import { NodeBenchmarkRunner } from './benchmark'
 import { TestRunner } from './test'
 
 async function getTestRunnerConstructor(
@@ -14,9 +13,7 @@ async function getTestRunnerConstructor(
   moduleRunner: TestModuleRunner,
 ): Promise<VitestRunnerConstructor> {
   if (!config.runner) {
-    return (
-      config.mode === 'test' ? TestRunner : NodeBenchmarkRunner
-    ) as any as VitestRunnerConstructor
+    return TestRunner as any as VitestRunnerConstructor
   }
   const mod = await moduleRunner.import(config.runner)
   if (!mod.default && typeof mod.default !== 'function') {
@@ -60,7 +57,7 @@ export async function resolveTestRunner(
     loadDiffConfig(config, moduleRunner),
     loadSnapshotSerializers(config, moduleRunner),
   ])
-  testRunner.config.diffOptions = diffOptions
+  testRunner.config._diffOptions = diffOptions
 
   // patch some methods, so custom runners don't need to call RPC
   const originalOnTaskUpdate = testRunner.onTaskUpdate

packages/vitest/src/runtime/runners/test.ts

@@ -8,11 +8,14 @@ import type {
   Task,
   Test,
   TestContext,
+  TestTryOptions,
   VitestRunnerImportSource,
   VitestRunner as VitestTestRunner,
 } from '@vitest/runner'
+import type { Bench as Tinybench, Task as TinybenchTask } from 'tinybench'
 import type { ModuleRunner } from 'vite/module-runner'
 import type { Traces } from '../../utils/traces'
+import type { Bench } from '../benchmark'
 import type { SerializedConfig } from '../config'
 import { getState, GLOBAL_EXPECT, setState } from '@vitest/expect'
 import {
@@ -29,7 +32,7 @@ import { createExpect } from '../../integrations/chai/index'
 import { inject } from '../../integrations/inject'
 import { getSnapshotClient } from '../../integrations/snapshot/chai'
 import { vi } from '../../integrations/vi'
-import { getBenchFn, getBenchOptions } from '../benchmark'
+import { createBench, kFinalize } from '../benchmark'
 import { rpc } from '../rpc'
 import { getWorkerState } from '../utils'
 
@@ -40,9 +43,13 @@ export class TestRunner implements VitestTestRunner {
   private cancelRun = false
 
   private assertionsErrors = new WeakMap<Readonly<Task>, Error>()
+  private benchInstances = new WeakMap<Readonly<Task>, Bench>()
 
   public pool: string = this.workerState.ctx.pool
-  private _otel!: Traces
+  /**
+   * @internal
+   */
+  public _otel!: Traces
   public viteEnvironment: string
   private viteModuleRunner: boolean
 
@@ -83,7 +90,7 @@ export class TestRunner implements VitestTestRunner {
     this.workerState.onCleanup(listener)
   }
 
-  onAfterRunFiles(): void {
+  onAfterRunFiles(_files: File[]): void {
     this.snapshotClient.clear()
     this.workerState.current = undefined
   }
@@ -167,7 +174,7 @@ export class TestRunner implements VitestTestRunner {
     this.workerState.current = suite
   }
 
-  onBeforeTryTask(test: Task): void {
+  onBeforeTryTask(test: Task, _options: TestTryOptions): void {
     clearModuleMocks(this.config)
     this.snapshotClient.clearTest(test.file.filepath, test.id)
     setState(
@@ -185,6 +192,7 @@ export class TestRunner implements VitestTestRunner {
   }
 
   onAfterTryTask(test: Test): void {
+    this.benchInstances.get(test)?.[kFinalize]()
     const {
       assertionCalls,
       expectedAssertionsNumber,
@@ -231,6 +239,18 @@ export class TestRunner implements VitestTestRunner {
         return _expect != null
       },
     })
+    let _bench: Bench | undefined
+    const runnerConfig = this.config
+    const benchInstances = this.benchInstances
+    Object.defineProperty(context, 'bench', {
+      get() {
+        if (!_bench) {
+          _bench = createBench(context.task, runnerConfig)
+          benchInstances.set(context.task, _bench)
+        }
+        return _bench
+      },
+    })
     return context
   }
 
@@ -281,13 +301,13 @@ export class TestRunner implements VitestTestRunner {
   static matchesTags: typeof matchesTags = matchesTags
 
   /**
-   * @deprecated
+   * @experimental
+   * A function that runs tinybench tasks.
+   * Can be overriden to run tasks in a special environment.
    */
-  static getBenchFn: typeof getBenchFn = getBenchFn
-  /**
-   * @deprecated
-   */
-  static getBenchOptions: typeof getBenchOptions = getBenchOptions
+  static async runBenchmarks(tinybench: Tinybench): Promise<TinybenchTask[]> {
+    return await tinybench.run()
+  }
 }
 
 function clearModuleMocks(config: SerializedConfig) {

packages/vitest/src/runtime/types/benchmark.ts

@@ -1,35 +0,0 @@
-import type { Test } from '@vitest/runner'
-import type { ChainableFunction } from '@vitest/runner/utils'
-import type {
-  Bench as BenchFactory,
-  Options as BenchOptions,
-  Task as BenchTask,
-  TaskResult as BenchTaskResult,
-  TaskResult as TinybenchResult,
-} from 'tinybench'
-
-export interface Benchmark extends Test {
-  meta: {
-    benchmark: true
-    result?: BenchTaskResult
-  }
-}
-
-export interface BenchmarkResult extends TinybenchResult {
-  name: string
-  rank: number
-  sampleCount: number
-  median: number
-}
-
-export type BenchFunction = (this: BenchFactory) => Promise<void> | void
-type ChainableBenchmarkAPI = ChainableFunction<
-  'skip' | 'only' | 'todo',
-  (name: string | Function, fn?: BenchFunction, options?: BenchOptions) => void
->
-export type BenchmarkAPI = ChainableBenchmarkAPI & {
-  skipIf: (condition: any) => ChainableBenchmarkAPI
-  runIf: (condition: any) => ChainableBenchmarkAPI
-}
-
-export { BenchFactory, BenchOptions, BenchTask, BenchTaskResult }

packages/vitest/src/runtime/worker.ts

@@ -2,6 +2,7 @@ import type { ContextRPC, WorkerGlobalState } from '../types/worker'
 import type { Traces } from '../utils/traces'
 import type { VitestWorker } from './workers/types'
 import { createStackString, parseStacktrace } from '@vitest/utils/source-map'
+import { GetterTracker } from './getter-tracker'
 import { setupInspect } from './inspector'
 import * as listeners from './listeners'
 import { VitestEvaluatedModules } from './moduleRunner/evaluatedModules'
@@ -47,6 +48,9 @@ async function execute(method: 'run' | 'collect', ctx: ContextRPC, worker: Vites
         return createStackString(parseStacktrace(stack))
       },
       metaEnv: createImportMetaEnvProxy(),
+      getterTracker: ctx.config.benchmark.enabled && !ctx.config.benchmark.suppressExportGetterWarnings
+        ? new GetterTracker()
+        : undefined,
     } satisfies WorkerGlobalState
 
     const methodName = method === 'collect' ? 'collectTests' : 'runTests'

packages/vitest/src/types/global.ts

@@ -2,7 +2,7 @@ import type { ExpectStatic, PromisifyAssertion, Tester } from '@vitest/expect'
 import type { Plugin as PrettyFormatPlugin } from '@vitest/pretty-format'
 import type { Test } from '@vitest/runner'
 import type { SnapshotState } from '@vitest/snapshot'
-import type { BenchmarkResult } from '../runtime/types/benchmark'
+import type { Bench, BenchResult } from '../runtime/benchmark'
 import type { UserConsoleLog } from './general'
 
 interface SnapshotMatcher<T> {
@@ -92,6 +92,40 @@ declare module 'vitest' {
      * await expect(largeData).toMatchFileSnapshot('path/to/snapshot.json');
      */
     toMatchFileSnapshot: (filepath: string, hint?: string) => Promise<void>
+
+    /**
+     * Asserts that a benchmark result is faster than another benchmark result.
+     * Compares mean latency — lower is faster.
+     *
+     * @example
+     * const result = await bench.compare(
+     *   bench('lib1', () => { lib1() }),
+     *   bench('lib2', () => { lib2() }),
+     * )
+     * expect(result.get('lib1')).toBeFasterThan(result.get('lib2'))
+     * expect(result.get('lib1')).toBeFasterThan(result.get('lib2'), { delta: 0.1 })
+     */
+    toBeFasterThan: (
+      expected: BenchResult,
+      options?: { delta?: number },
+    ) => void
+
+    /**
+     * Asserts that a benchmark result is slower than another benchmark result.
+     * Compares mean latency — higher is slower.
+     *
+     * @example
+     * const result = await bench.compare(
+     *   bench('lib1', () => { lib1() }),
+     *   bench('lib2', () => { lib2() }),
+     * )
+     * expect(result.get('lib2')).toBeSlowerThan(result.get('lib1'))
+     * expect(result.get('lib2')).toBeSlowerThan(result.get('lib1'), { delta: 0.2 })
+     */
+    toBeSlowerThan: (
+      expected: BenchResult,
+      options?: { delta?: number },
+    ) => void
   }
 }
 
@@ -103,6 +137,11 @@ declare module '@vitest/runner' {
      * This API is useful for running snapshot tests concurrently because global expect cannot track them.
      */
     readonly expect: ExpectStatic
+    /**
+     * Create a benchmark to run. It will be reported after the test is finished.
+     * @see {@link https://vitest.dev/guide/benchmarking}
+     */
+    readonly bench: Bench
     /** @internal */
     _local: boolean
   }
@@ -121,8 +160,4 @@ declare module '@vitest/runner' {
   interface TaskBase {
     logs?: UserConsoleLog[]
   }
-
-  interface TaskResult {
-    benchmark?: BenchmarkResult
-  }
 }

packages/vitest/src/types/rpc.ts

@@ -1,4 +1,4 @@
-import type { CancelReason, File, TaskEventPack, TaskResultPack, TestArtifact } from '@vitest/runner'
+import type { BaselineData, CancelReason, File, TaskEventPack, TaskResultPack, TestArtifact, TestBenchmark } from '@vitest/runner'
 import type { SnapshotResult } from '@vitest/snapshot'
 import type { FetchFunctionOptions, FetchResult } from 'vite/module-runner'
 import type { OTELCarrier } from '../utils/traces'
@@ -21,6 +21,7 @@ export interface RuntimeRPC {
   onQueued: (file: File) => void
   onCollected: (files: File[]) => Promise<void>
   onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void
+  onTestBenchmark: (testId: string, bench: TestBenchmark) => void
   onTaskArtifactRecord: <Artifact extends TestArtifact>(testId: string, artifact: Artifact) => Promise<Artifact>
   onTaskUpdate: (pack: TaskResultPack[], events: TaskEventPack[]) => Promise<void>
   onCancel: (reason: CancelReason) => void
@@ -29,6 +30,9 @@ export interface RuntimeRPC {
   snapshotSaved: (snapshot: SnapshotResult) => void
   resolveSnapshotPath: (testPath: string) => string
 
+  readBenchmarkResult: (relativePath: string) => Promise<BaselineData | null>
+  writeBenchmarkResult: (relativePath: string, data: BaselineData) => Promise<void>
+
   ensureModuleGraphEntry: (id: string, importer: string) => void
 }
 

packages/vitest/src/types/worker.ts

@@ -2,6 +2,7 @@ import type { CancelReason, FileSpecification, Task } from '@vitest/runner'
 import type { BirpcReturn } from 'birpc'
 import type { EvaluatedModules } from 'vite/module-runner'
 import type { SerializedConfig } from '../runtime/config'
+import type { GetterTracker } from '../runtime/getter-tracker'
 import type { Traces } from '../utils/traces'
 import type { Environment } from './environment'
 import type { RunnerRPC, RuntimeRPC } from './rpc'
@@ -74,6 +75,7 @@ export interface WorkerGlobalState {
   evaluatedModules: EvaluatedModules
   resolvingModules: Set<string>
   moduleExecutionInfo: Map<string, any>
+  getterTracker?: GetterTracker
   onCancel: (listener: (reason: CancelReason) => unknown) => void
   onCleanup: (listener: () => unknown) => void
   providedContext: Record<string, any>

packages/vitest/src/utils/config-helpers.ts

@@ -1,5 +1,4 @@
 import type {
-  BenchmarkBuiltinReporters,
   BuiltinReporters,
 } from '../node/reporters'
 
@@ -9,7 +8,7 @@ interface PotentialConfig {
 
 export function getOutputFile(
   config: PotentialConfig | undefined,
-  reporter: BuiltinReporters | BenchmarkBuiltinReporters | 'html',
+  reporter: BuiltinReporters | 'html',
 ): string | undefined {
   if (!config?.outputFile) {
     return

pnpm-lock.yaml

@@ -132,6 +132,9 @@ catalogs:
     strip-literal:
       specifier: ^3.1.0
       version: 3.1.0
+    tinybench:
+      specifier: ^6.0.1
+      version: 6.0.1
     tinyexec:
       specifier: ^1.0.2
       version: 1.0.2
@@ -835,6 +838,9 @@ importers:
       pathe:
         specifier: 'catalog:'
         version: 2.0.3
+      tinybench:
+        specifier: 'catalog:'
+        version: 6.0.1
 
   packages/snapshot:
     dependencies:
@@ -1072,8 +1078,8 @@ importers:
         specifier: 'catalog:'
         version: 4.0.0-rc.1
       tinybench:
-        specifier: ^2.9.0
-        version: 2.9.0
+        specifier: 'catalog:'
+        version: 6.0.1
       tinyexec:
         specifier: ^1.0.2
         version: 1.0.2
@@ -9709,8 +9715,9 @@ packages:
   [email protected]:
     resolution: {integrity: sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==}
 
-  tinybench@2.9.0:
-    resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==}
+  tinybench@6.0.1:
+    resolution: {integrity: sha512-cMdWsxmysdg8mNWf1pujiWl3TW0cU6m8QuNw55QlnP3I6N96Grb0wnu5N0syHIu3LbiVZCNqlfWzWDq84HZphA==}
+    engines: {node: '>=20.0.0'}
 
   [email protected]:
     resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==}
@@ -19042,7 +19049,7 @@ snapshots:
     dependencies:
       real-require: 0.2.0
 
-  tinybench@2.9.0: {}
+  tinybench@6.0.1: {}
 
   [email protected]: {}
 

pnpm-workspace.yaml

@@ -89,6 +89,7 @@ catalog:
   sirv: ^3.0.2
   std-env: ^4.0.0-rc.1
   strip-literal: ^3.1.0
+  tinybench: ^6.0.1
   tinyexec: ^1.0.2
   tinyglobby: ^0.2.15
   tinyhighlight: ^0.3.2

test/browser/fixtures/benchmark/basic.bench.ts

@@ -1,32 +0,0 @@
-import { bench, describe } from 'vitest'
-
-describe('suite-a', () => {
-  bench('good', async () => {
-    await sleep(10)
-  }, options)
-
-  bench('bad', async () => {
-    await sleep(300)
-  }, options)
-})
-
-describe('suite-b', () => {
-  bench('good', async () => {
-    await sleep(25)
-  }, options)
-
-  describe('suite-b-nested', () => {
-    bench('good', async () => {
-      await sleep(50)
-    }, options)
-  })
-})
-
-const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
-
-const options = {
-  time: 0,
-  iterations: 2,
-  warmupIterations: 0,
-  warmupTime: 0,
-}

test/browser/fixtures/benchmark/vitest.config.ts

@@ -1,15 +0,0 @@
-import { fileURLToPath } from 'node:url'
-import { defineConfig } from 'vitest/config'
-import { instances, provider } from '../../settings'
-
-export default defineConfig({
-  test: {
-    browser: {
-      enabled: true,
-      headless: true,
-      provider,
-      instances,
-    },
-  },
-  cacheDir: fileURLToPath(new URL("./node_modules/.vite", import.meta.url)),
-})

test/browser/package.json

@@ -9,6 +9,7 @@
     "test:playwright": "PROVIDER=playwright pnpm run test:unit",
     "test:safaridriver": "PROVIDER=webdriverio BROWSER=safari pnpm run test:unit",
     "test-fixtures": "vitest",
+    "bench-fixtures": "vitest bench",
     "test-expect-dom": "vitest --root ./fixtures/expect-dom",
     "test-mocking": "vitest --root ./fixtures/mocking",
     "test-logs": "vitest --root ./fixtures/print-logs",

test/browser/specs/benchmark.test.ts

@@ -1,21 +0,0 @@
-import { expect, test } from 'vitest'
-import { runVitest } from '../../test-utils'
-
-const IS_PLAYWRIGHT = process.env.PROVIDER === 'playwright'
-
-test('benchmark', async () => {
-  const result = await runVitest({ root: 'fixtures/benchmark' }, [], { mode: 'benchmark' })
-  expect(result.stderr).toReportNoErrors()
-
-  if (IS_PLAYWRIGHT) {
-    expect(result.stdout).toContain('✓ |chromium| basic.bench.ts > suite-a')
-    expect(result.stdout).toContain('✓ |firefox| basic.bench.ts > suite-a')
-    expect(result.stdout).toContain('✓ |webkit| basic.bench.ts > suite-a')
-  }
-  else {
-    expect(result.stdout).toContain('✓ |chrome| basic.bench.ts > suite-a')
-    expect(result.stdout).toContain('✓ |firefox| basic.bench.ts > suite-a')
-  }
-
-  expect(result.exitCode).toBe(0)
-})

test/browser/specs/runner.test.ts

@@ -1,4 +1,5 @@
-import type { JsonTestResults, Vitest } from 'vitest/node'
+import type { TestBenchmark } from 'vitest'
+import type { JsonTestResult, JsonTestResults, Vitest } from 'vitest/node'
 import { readdirSync } from 'node:fs'
 import { readFile } from 'node:fs/promises'
 import { beforeAll, describe, expect, onTestFailed, test } from 'vitest'
@@ -12,10 +13,11 @@ describe('running browser tests', async () => {
   let stderr: string
   let stdout: string
   let browserResultJson: JsonTestResults
-  let passedTests: any[]
-  let failedTests: any[]
+  let passedTests: JsonTestResult[]
+  let failedTests: JsonTestResult[]
   let vitest: Vitest
   const events: string[] = []
+  const emittedBenchmarks: Array<{ projectName: string; testName: string; benchmark: TestBenchmark }> = []
 
   beforeAll(async () => {
     ({
@@ -29,6 +31,13 @@ describe('running browser tests', async () => {
           onBrowserInit(project) {
             events.push(`onBrowserInit ${project.name}`)
           },
+          onTestCaseBenchmark(testCase, benchmark) {
+            emittedBenchmarks.push({
+              projectName: testCase.project.name || '',
+              testName: testCase.fullName,
+              benchmark,
+            })
+          },
         },
         'json',
         {
@@ -47,9 +56,9 @@ describe('running browser tests', async () => {
     }))
 
     const browserResult = await readFile('./browser.json', 'utf-8')
-    browserResultJson = JSON.parse(browserResult)
-    const getPassed = results => results.filter(result => result.status === 'passed' && !result.message)
-    const getFailed = results => results.filter(result => result.status === 'failed')
+    browserResultJson = JSON.parse(browserResult) as JsonTestResults
+    const getPassed = (results: JsonTestResult[]) => results.filter(result => result.status === 'passed' && !result.message)
+    const getFailed = (results: JsonTestResult[]) => results.filter(result => result.status === 'failed')
     passedTests = getPassed(browserResultJson.testResults)
     failedTests = getFailed(browserResultJson.testResults)
   })
@@ -70,7 +79,7 @@ describe('running browser tests', async () => {
       .toEqual(vitest.projects.map(() => expect.arrayContaining(runtimeTestFiles)))
 
     const testFilesCount = readdirSync('./test')
-      .filter(n => n.includes('.test.') || n.includes('.test-d.'))
+      .filter(n => n.includes('.test.') || n.includes('.test-d.') || n.includes('.bench.'))
       .length + 1 // 1 is in-source-test
 
     expect(browserResultJson.testResults).toHaveLength(testFilesCount * instances.length)
@@ -78,6 +87,50 @@ describe('running browser tests', async () => {
     expect(failedTests).toHaveLength(0)
   })
 
+  test('benchmarks run in a dedicated `(bench)` project per browser instance', () => {
+    const benchProjects = vitest.projects.filter(p => p.name.endsWith('(bench)'))
+    expect(benchProjects.map(p => p.name).sort()).toEqual(
+      instances.map(({ browser }) => `${browser} (bench)`).sort(),
+    )
+  })
+
+  test('perProject benchmarks emit tasks with the perProject flag in every browser', () => {
+    const records = emittedBenchmarks.filter(e =>
+      e.testName === 'perProject registrations flow through the browser RPC (onTestBenchmark)',
+    )
+    // the test calls `.run()` twice, so each browser produces 2 benchmark records
+    expect(records.length, `perProject emitted: ${records.length}`).toBe(2 * instances.length)
+    for (const record of records) {
+      expect(record.benchmark.tasks, `empty tasks for ${record.projectName}`).toHaveLength(1)
+      const [task] = record.benchmark.tasks
+      expect(task.perProject, `missing perProject flag on ${record.projectName}/${task.name}`).toBe(true)
+      expect(task.fromStore).toBeUndefined()
+    }
+  })
+
+  test('bench.compare emits one benchmark with both registrations ranked', () => {
+    const records = emittedBenchmarks.filter(e =>
+      e.testName === 'bench.compare resolves a BenchStorage in the browser',
+    )
+    expect(records.length).toBe(instances.length)
+    for (const record of records) {
+      expect(record.benchmark.tasks.map(t => t.name).sort(), `unexpected tasks for ${record.projectName}`).toEqual(['a', 'b'])
+      expect(record.benchmark.tasks.map(t => t.rank).sort()).toEqual([1, 2])
+    }
+  })
+
+  test('writeResult flows through the write-artifact RPC in every browser', () => {
+    const records = emittedBenchmarks.filter(e =>
+      e.testName === 'writeResult exercises the writeBenchmarkResult RPC round-trip',
+    )
+    expect(records.length).toBe(instances.length)
+    for (const record of records) {
+      expect(record.benchmark.tasks, `empty tasks for ${record.projectName}`).toHaveLength(1)
+      const [task] = record.benchmark.tasks
+      expect(task.name).toBe('with-write')
+    }
+  })
+
   test('tags are collected', () => {
     expect(vitest.config.tags).toEqual([
       { name: 'e2e', priority: 10 },
@@ -110,7 +163,7 @@ describe('running browser tests', async () => {
   test('runs in-source tests', () => {
     expect(stdout).toContain('src/actions.ts')
     const actionsTest = passedTests.find(t => t.name.includes('/actions.ts'))
-    expect(actionsTest).toBeDefined()
+    expect.assert(actionsTest)
     expect(actionsTest.assertionResults).toHaveLength(1)
   })
 
@@ -231,7 +284,7 @@ test(`stack trace points to correct file in every browser when failed`, async ()
             return
           }
           if (testCase.project.name === 'chromium' || testCase.project.name === 'chrome') {
-            expect(testCase.result().errors[0].stacks).toEqual([
+            expect(testCase.result().errors?.[0].stacks).toEqual([
               {
                 line: 11,
                 column: 12,
@@ -380,7 +433,7 @@ test.runIf(provider.name === 'playwright')('timeout hooks', async ({ onTestFaile
   })
 
   const lines = stderr.split('\n')
-  const timeoutErrorsIndexes = []
+  const timeoutErrorsIndexes: number[] = []
   lines.forEach((line, index) => {
     if (line.includes('TimeoutError:')) {
       timeoutErrorsIndexes.push(index)

test/browser/specs/utils.ts

@@ -43,5 +43,9 @@ export async function runBrowserTests(
     $viteConfig: viteOverrides,
   }, include, runnerOptions)
 
-  return { ...result, stderr: result.stderr.replace('Testing types with tsc and vue-tsc is an experimental feature.\nBreaking changes might not follow SemVer, please pin Vitest\'s version when using it.\n', '') }
+  return {
+    ...result,
+    ctx: result.ctx!,
+    stderr: result.stderr.replace('Testing types with tsc and vue-tsc is an experimental feature.\nBreaking changes might not follow SemVer, please pin Vitest\'s version when using it.\n', ''),
+  }
 }

test/browser/test/browser.bench.ts

@@ -0,0 +1,43 @@
+import { expect, test } from 'vitest'
+
+// Keep runs tiny — this file smoke-tests the browser RPC path
+// (onTestBenchmark, readBenchmarkResult / writeBenchmarkResult).
+// It is not a measurement harness.
+const fastBenchOptions = {
+  time: 0,
+  iterations: 2,
+  warmupTime: 0,
+  warmupIterations: 0,
+}
+
+test('perProject registrations flow through the browser RPC (onTestBenchmark)', async ({ bench }) => {
+  await bench('1 + 1', { perProject: true, ...fastBenchOptions }, () => {
+    const result = 1 + 1
+    expect.assert(result === 2)
+  }).run()
+  await bench('1 + 2', { perProject: true, ...fastBenchOptions }, () => {
+    const result = 1 + 2
+    expect.assert(result === 3)
+  }).run()
+})
+
+test('bench.compare resolves a BenchStorage in the browser', async ({ bench }) => {
+  const storage = await bench.compare(
+    bench('a', () => { const _ = 1 + 1 }),
+    bench('b', () => { const _ = 1 + 2 }),
+    fastBenchOptions,
+  )
+  // runtime smoke — every registration is accessible with a valid BenchResult
+  expect.assert(typeof storage.get('a').latency.mean === 'number')
+  expect.assert(typeof storage.get('b').latency.mean === 'number')
+})
+
+test('writeResult exercises the writeBenchmarkResult RPC round-trip', async ({ bench }) => {
+  // The browser worker forwards writeResult through the WebSocket RPC to the
+  // node side. We don't assert on the file contents here (the spec layer can
+  // do that), just that the round-trip completes without throwing.
+  const result = await bench('with-write', { writeResult: './out/with-write.json', ...fastBenchOptions }, () => {
+    const _ = 1 + 1
+  }).run()
+  expect.assert(typeof result.latency.mean === 'number')
+})

test/browser/vitest.config.mts

@@ -72,6 +72,9 @@ export default defineConfig({
       { name: 'test', priority: 5 },
       { name: 'browser', priority: 1 },
     ],
+    benchmark: {
+      enabled: true,
+    },
     alias: {
       '#src': resolve(dir, './src'),
     },

test/e2e/fixtures/benchmarking/basic/base.bench.ts

@@ -1,67 +0,0 @@
-import { bench, describe } from 'vitest'
-
-describe('sort', () => {
-  bench('normal', () => {
-    const x = [1, 5, 4, 2, 3]
-    x.sort((a, b) => {
-      return a - b
-    })
-  }, { iterations: 5, time: 0 })
-
-  bench('reverse', () => {
-    const x = [1, 5, 4, 2, 3]
-    x.reverse().sort((a, b) => {
-      return a - b
-    })
-  }, { iterations: 5, time: 0 })
-
-  // TODO: move to failed tests
-  // should not be collected
-  // it('test', () => {
-  //   expect(1 + 1).toBe(3)
-  // })
-})
-
-function timeout(time: number) {
-  return new Promise((resolve) => {
-    setTimeout(resolve, time)
-  })
-}
-
-describe('timeout', () => {
-  bench('timeout100', async () => {
-    await timeout(100)
-  }, {
-    setup() {
-
-    },
-    teardown() {
-
-    },
-    ...benchOptions
-  })
-
-  bench('timeout75', async () => {
-    await timeout(75)
-  }, benchOptions)
-
-  bench('timeout50', async () => {
-    await timeout(50)
-  }, benchOptions)
-
-  bench('timeout25', async () => {
-    await timeout(25)
-  }, benchOptions)
-
-  // TODO: move to failed tests
-  // test('reduce', () => {
-  //   expect(1 - 1).toBe(2)
-  // })
-})
-
-const benchOptions = {
-  time: 0,
-  iterations: 3,
-  warmupIterations: 0,
-  warmupTime: 0,
-}

test/e2e/fixtures/benchmarking/basic/mode.bench.ts

@@ -1,15 +0,0 @@
-import { bench, describe } from 'vitest'
-
-describe.todo('unimplemented suite')
-
-describe.skip('skipped', () => {
-  bench('skipped', () => {
-    throw new Error('should be skipped')
-  })
-
-  bench.todo('unimplemented test')
-})
-
-bench.skip('skipped', () => {
-  throw new Error('should be skipped')
-})

test/e2e/fixtures/benchmarking/basic/only.bench.ts

@@ -1,75 +0,0 @@
-import { bench, describe, expect, assert } from 'vitest'
-
-const run = [false, false, false, false, false]
-
-describe('a0', () => {
-  bench.only('0', () => {
-    run[0] = true
-  }, { iterations: 1, time: 0 })
-  bench('s0', () => {
-    expect(true).toBe(false)
-  })
-})
-
-describe('a1', () => {
-  describe('b1', () => {
-    describe('c1', () => {
-      bench.only('1', () => {
-        run[1] = true
-      }, { iterations: 1, time: 0 })
-    })
-    bench('s1', () => {
-      expect(true).toBe(false)
-    })
-  })
-})
-
-describe.only('a2', () => {
-  bench('2', () => {
-    run[2] = true
-  }, { iterations: 1, time: 0 })
-})
-
-bench('s2', () => {
-  expect(true).toBe(false)
-})
-
-describe.only('a3', () => {
-  describe('b3', () => {
-    bench('3', () => {
-      run[3] = true
-    }, { iterations: 1, time: 0 })
-  })
-  bench.skip('s3', () => {
-    expect(true).toBe(false)
-  })
-})
-
-describe('a4', () => {
-  describe.only('b4', () => {
-    bench('4', () => {
-      run[4] = true
-    }, { iterations: 1, time: 0 })
-  })
-  describe('sb4', () => {
-    bench('s4', () => {
-      expect(true).toBe(false)
-    })
-  })
-})
-
-bench.only(
-  'visited',
-  () => {
-    assert.deepEqual(run, [true, true, true, true, true])
-  },
-  { iterations: 1, time: 0 },
-)
-
-bench.only(
-  'visited2',
-  () => {
-    assert.deepEqual(run, [true, true, true, true, true])
-  },
-  { iterations: 1, time: 0 },
-)

test/e2e/fixtures/benchmarking/basic/should-not-run.test-d.ts

@@ -1,7 +0,0 @@
-import { describe, expectTypeOf, test } from 'vitest'
-
-describe('test', () => {
-  test('some-test', () => {
-    expectTypeOf({ a: 1 }).toEqualTypeOf({ a: "should not match" })
-  })
-})