vitest/test/browser/specs/runner.test.ts

import type { TestBenchmark } from 'vitest'
import type { JsonTestResult, JsonTestResults, Vitest } from 'vitest/node'
import { readdirSync } from 'node:fs'
import { readFile } from 'node:fs/promises'
import { beforeAll, describe, expect, onTestFailed, test } from 'vitest'
import { buildTestTree } from '../../test-utils'
import { instances, provider, runBrowserTests } from './utils'

function noop() {}

describe('running browser tests', async () => {
  let stderr: string
  let stdout: string
  let browserResultJson: JsonTestResults
  let passedTests: JsonTestResult[]
  let failedTests: JsonTestResult[]
  let vitest: Vitest
  const events: string[] = []
  const emittedBenchmarks: Array<{ projectName: string; testName: string; benchmark: TestBenchmark }> = []

  beforeAll(async () => {
    ({
      stderr,
      stdout,
      ctx: vitest,
    } = await runBrowserTests({
      allowOnly: true,
      reporters: [
        {
          onBrowserInit(project) {
            events.push(`onBrowserInit ${project.name}`)
          },
          onTestCaseBenchmark(testCase, benchmark) {
            emittedBenchmarks.push({
              projectName: testCase.project.name || '',
              testName: testCase.fullName,
              benchmark,
            })
          },
        },
        'json',
        {
          onInit: noop,
          onTestRunStart: noop,
          onTestModuleCollected: noop,
          onTestRunEnd: noop,
          onTestRemoved: noop,
          onWatcherStart: noop,
          onWatcherRerun: noop,
          onServerRestart: noop,
          onUserConsoleLog: noop,
        },
        'default',
      ],
    }))

    const browserResult = await readFile('./browser.json', 'utf-8')
    browserResultJson = JSON.parse(browserResult) as JsonTestResults
    const getPassed = (results: JsonTestResult[]) => results.filter(result => result.status === 'passed' && !result.message)
    const getFailed = (results: JsonTestResult[]) => results.filter(result => result.status === 'failed')
    passedTests = getPassed(browserResultJson.testResults)
    failedTests = getFailed(browserResultJson.testResults)
  })

  test('tests are actually running', () => {
    expect(stderr).toBe('')

    const testFiles = Array.from(new Set(browserResultJson.testResults.map(t => t.name)))

    vitest.projects.forEach((project) => {
      // the order is non-deterministic
      expect(events).toContain(`onBrowserInit ${project.name}`)
    })

    // test files are optimized automatically (type-check-only files are excluded)
    const runtimeTestFiles = testFiles.filter(f => !f.endsWith('.test-d.ts'))
    expect(vitest.projects.map(p => p.vite.config.optimizeDeps.entries))
      .toEqual(vitest.projects.map(() => expect.arrayContaining(runtimeTestFiles)))

    const testFilesCount = readdirSync('./test')
      .filter(n => n.includes('.test.') || n.includes('.test-d.') || n.includes('.bench.'))
      .length + 1 // 1 is in-source-test

    expect(browserResultJson.testResults).toHaveLength(testFilesCount * instances.length)
    expect(passedTests).toHaveLength(browserResultJson.testResults.length)
    expect(failedTests).toHaveLength(0)
  })

  test('benchmarks run in a dedicated `(bench)` project per browser instance', () => {
    const benchProjects = vitest.projects.filter(p => p.name.endsWith('(bench)'))
    expect(benchProjects.map(p => p.name).sort()).toEqual(
      instances.map(({ browser }) => `${browser} (bench)`).sort(),
    )
  })

  test('perProject benchmarks emit tasks with the perProject flag in every browser', () => {
    const records = emittedBenchmarks.filter(e =>
      e.testName === 'perProject registrations flow through the browser RPC (onTestBenchmark)',
    )
    // the test calls `.run()` twice, so each browser produces 2 benchmark records
    expect(records.length, `perProject emitted: ${records.length}`).toBe(2 * instances.length)
    for (const record of records) {
      expect(record.benchmark.tasks, `empty tasks for ${record.projectName}`).toHaveLength(1)
      const [task] = record.benchmark.tasks
      expect(task.perProject, `missing perProject flag on ${record.projectName}/${task.name}`).toBe(true)
      expect(task.fromStore).toBeUndefined()
    }
  })

  test('bench.compare emits one benchmark with both registrations ranked', () => {
    const records = emittedBenchmarks.filter(e =>
      e.testName === 'bench.compare resolves a BenchStorage in the browser',
    )
    expect(records.length).toBe(instances.length)
    for (const record of records) {
      expect(record.benchmark.tasks.map(t => t.name).sort(), `unexpected tasks for ${record.projectName}`).toEqual(['a', 'b'])
      expect(record.benchmark.tasks.map(t => t.rank).sort()).toEqual([1, 2])
    }
  })

  test('writeResult flows through the write-artifact RPC in every browser', () => {
    const records = emittedBenchmarks.filter(e =>
      e.testName === 'writeResult exercises the writeBenchmarkResult RPC round-trip',
    )
    expect(records.length).toBe(instances.length)
    for (const record of records) {
      expect(record.benchmark.tasks, `empty tasks for ${record.projectName}`).toHaveLength(1)
      const [task] = record.benchmark.tasks
      expect(task.name).toBe('with-write')
    }
  })

  test('tags are collected', () => {
    expect(vitest.config.tags).toEqual([
      { name: 'e2e', priority: 10 },
      { name: 'test', priority: 5 },
      { name: 'browser', priority: 1 },
    ])

    const testModule = vitest.state.getTestModules().find(m => m.moduleId.includes('tags.test.ts'))
    expect.assert(testModule)
    expect(buildTestTree([testModule], t => t.tags)).toMatchInlineSnapshot(`
      {
        "test/tags.test.ts": {
          "suite 1": {
            "suite 2": {
              "test 2": [
                "browser",
                "e2e",
              ],
            },
            "test 1": [
              "browser",
              "test",
            ],
          },
        },
      }
    `)
  })

  test('runs in-source tests', () => {
    expect(stdout).toContain('src/actions.ts')
    const actionsTest = passedTests.find(t => t.name.includes('/actions.ts'))
    expect.assert(actionsTest)
    expect(actionsTest.assertionResults).toHaveLength(1)
  })

  test('unsubscribes cancel listeners after run', async () => {
    // should not throw birpc closing errors
    await expect(vitest.cancelCurrentRun('keyboard-input')).resolves.not.toThrow()
  })
})

describe('console logging tests', async () => {
  let stderr: string
  let stdout: string
  beforeAll(async () => {
    ({
      stderr,
      stdout,
    } = await runBrowserTests({
      root: './fixtures/print-logs',
      // assert on Vitest's own console forwarding rather than Vite's client
      // relay, which does not run when the tester loads the stubbed @vite/client
      reporters: ['default'],
    }))
  })

  test('logs are redirected to stdout', () => {
    expect(stdout).toContain('stdout | test/logs.test.ts > logging to stdout')
    expect(stdout).toContain('hello from console.log')
    expect(stdout).toContain('hello from console.info')
    expect(stdout).toContain('hello from console.debug')
    expect(stdout).toContain(`
{
  hello: 'from dir',
}
      `.trim())
    expect(stdout).toContain(`
{
  hello: 'from dirxml',
}
      `.trim())
    expect(stdout).toContain('dom <div />')
    expect(stdout).toContain('default: 1')
    expect(stdout).toContain('default: 2')
    expect(stdout).toContain('default: 3')
    expect(stdout).toContain('count: 1')
    expect(stdout).toContain('count: 2')
    expect(stdout).toContain('count: 3')
    expect(stdout).toMatch(/default: [\d.]+ ms/)
    expect(stdout).toMatch(/time: [\d.]+ ms/)
    expect(stdout).toMatch(/\[console-time-fake\]: [\d.]+ ms/)
    expect(stdout).not.toContain('[console-time-fake]: 0 ms')
    expect(stdout).toContain('hello from one')
    expect(stdout).toContain(`hello from two {
  hello: 'object',
}`)
  })

  test('logs are redirected to stderr', () => {
    expect(stderr).toContain('stderr | test/logs.test.ts > logging to stderr')
    expect(stderr).toContain('hello from console.error')
    expect(stderr).toContain('hello from console.warn')
    expect(stderr).toContain('Timer "invalid timeLog" does not exist')
    expect(stderr).toContain('Timer "invalid timeEnd" does not exist')
    // safari logs the stack files with @https://...
    expect(stderr).toMatch(/hello from console.trace\s+(\w+|@)/)
  })

  test(`logs have stack traces`, () => {
    expect(stdout).toMatch(`
log with a stack
 ❯ test/logs.test.ts:58:11
    `.trim())
    expect(stderr).toMatch(`
error with a stack
 ❯ test/logs.test.ts:59:11
    `.trim())
    // console.trace processes the stack trace correctly
    expect(stderr).toMatch('test/logs.test.ts:60:11')
  })

  test('popup apis should log a warning', () => {
    expect(stderr).toContain('Vitest encountered a `alert("test")`')
    expect(stderr).toContain('Vitest encountered a `confirm("test")`')
    expect(stderr).toContain('Vitest encountered a `print()`')
    expect(stderr).toContain('Vitest encountered a `prompt("test")`')
  })
})

test('disableConsoleIntercept', async () => {
  const result = await runBrowserTests({
    root: './fixtures/print-logs',
    project: [instances[0].browser],
    disableConsoleIntercept: true,
  })
  expect(result.stderr).toBe('')
  expect(result.stdout).not.toContain('logging to stdout')
  expect(result.stdout).not.toContain('hello from console.log')
})

test(`stack trace points to correct file in every browser when failed`, async () => {
  expect.assertions(30)
  const { stderr } = await runBrowserTests({
    root: './fixtures/failing',
    reporters: [
      'default',
      {
        onTestCaseReady(testCase) {
          if (testCase.name !== 'correctly fails and prints a diff') {
            return
          }
          if (testCase.project.name === 'chromium' || testCase.project.name === 'chrome') {
            expect(testCase.result().errors?.[0].stacks).toEqual([
              {
                line: 11,
                column: 13,
                file: testCase.module.moduleId,
                method: '',
              },
            ])
          }
        },
      },
    ],
  })

  expect(stderr).toContain('expected 1 to be 2')
  expect(stderr).toMatch(/- 2\s+\+ 1/)
  // expect(stderr).toContain('Expected to be')
  // expect(stderr).toContain('But got')
  expect(stderr).toContain('Failure screenshot')
  expect(stderr).toContain('__screenshots__/failing')

  expect(stderr).toContain('Access denied to "/inaccessible/path".')

  expect(stderr).toMatch(/failing.test.ts:11:13/)

  // column is 13 in safari, 9 in others
  expect(stderr).toMatch(/throwError src\/error.ts:8:(13|9)/)

  expect(stderr).toContain('The call was not awaited. This method is asynchronous and must be awaited; otherwise, the call will not start to avoid unhandled rejections.')
  expect(stderr).toMatch(/failing.test.ts:19:28/)
  expect(stderr).toMatch(/failing.test.ts:20:28/)
  expect(stderr).toMatch(/failing.test.ts:21:28/)
  expect(stderr).toMatch(/failing.test.ts:22:13/)
  expect(stderr).toMatch(/failing.test.ts:23:13/)
  expect(stderr).toMatch(/failing.test.ts:24:13/)
  expect(stderr).toMatch(/failing.test.ts:25:13/)
  expect(stderr).toMatch(/failing.test.ts:26:13/)
  expect(stderr).toMatch(/failing.test.ts:27:13/)
  expect(stderr).toMatch(/failing.test.ts:28:13/)
  expect(stderr).toMatch(/failing.test.ts:29:13/)
  expect(stderr).toMatch(/failing.test.ts:30:13/)
  expect(stderr).toMatch(/failing.test.ts:31:13/)
  expect(stderr).toMatch(/failing.test.ts:32:13/)
  expect(stderr).toMatch(/failing.test.ts:33:13/)
  expect(stderr).toMatch(/failing.test.ts:34:13/)
  expect(stderr).toMatch(/failing.test.ts:35:13/)

  expect(stderr).toMatch(/bundled-lib\/src\/b.js:2:(9|18)/)
  expect(stderr).toMatch(/bundled-lib\/src\/index.js:5:(16|17)/)

  // index() is called from a bundled file
  expect(stderr).toMatch(/failing.test.ts:39:3/)

  // "not awaited but with then/catch/finally" test should not produce warnings
  expect(stderr).not.toMatch(/failing.test.ts:4[3-8]/)
})

test('user-event', async () => {
  const { stdout, stderr } = await runBrowserTests({
    root: './fixtures/user-event',
  })
  expect(stderr).toBe('')
  onTestFailed(() => console.error(stderr))
  instances.forEach(({ browser }) => {
    expect(stdout).toReportPassedTest('cleanup-retry.test.ts', browser)
    expect(stdout).toReportPassedTest('cleanup1.test.ts', browser)
    expect(stdout).toReportPassedTest('cleanup2.test.ts', browser)
    expect(stdout).toReportPassedTest('keyboard.test.ts', browser)
    expect(stdout).toReportPassedTest('clipboard.test.ts', browser)
  })
})

test('timeout settings', async () => {
  const { stderr } = await runBrowserTests({
    root: './fixtures/timeout',
  })
  expect(stderr).toContain('Matcher did not succeed in time.')
  if (provider.name === 'playwright') {
    expect(stderr).toContain('locator.click: Timeout 500ms exceeded.')
    expect(stderr).toContain('locator.click: Timeout 345ms exceeded.')
  }
})

test('viewport', async () => {
  const { stdout, stderr } = await runBrowserTests({
    root: './fixtures/viewport',
  })
  expect(stderr).toBe('')
  instances.forEach(({ browser }) => {
    expect(stdout).toReportPassedTest('basic.test.ts', browser)
  })
})

test('in-source tests don\'t run when the module is imported by the test', async () => {
  const { stderr, stdout } = await runBrowserTests({}, ['mocking.test.ts'])
  expect(stderr).toBe('')

  instances.forEach(({ browser }) => {
    expect(stdout).toReportPassedTest('test/mocking.test.ts', browser)
  })

  // there is only one file with one test inside
  // if this stops working, it will report twice as much tests
  expect(stdout).toContain(`Test Files  ${instances.length} passed`)
  expect(stdout).toContain(`Tests  ${instances.length} passed`)
})

test('in-source tests run correctly when filtered', async () => {
  const { stderr, stdout } = await runBrowserTests({}, ['actions.ts'])
  expect(stderr).toBe('')

  instances.forEach(({ browser }) => {
    expect(stdout).toReportPassedTest('src/actions.ts', browser)
  })

  // there is only one file with one test inside
  expect(stdout).toContain(`Test Files  ${instances.length} passed`)
  expect(stdout).toContain(`Tests  ${instances.length} passed`)
})

test('re-evaluate setupFiles on each test run even when isolate is false', async () => {
  const { exitCode, stderr, stdout } = await runBrowserTests({
    root: './fixtures/isolate-and-setup-file',
  })

  expect(stderr).toBe('')
  expect(exitCode).toBe(0)
  instances.forEach(({ browser }) => {
    expect(stdout).toReportPassedTest('a.test.ts', browser)
    expect(stdout).toReportPassedTest('b.test.ts', browser)
  })
})

test.runIf(provider.name === 'playwright')('timeout hooks', async ({ onTestFailed }) => {
  const { stderr } = await runBrowserTests({
    root: './fixtures/timeout-hooks',
  })

  onTestFailed(() => {
    console.error(stderr)
  })

  const lines = stderr.split('\n')
  const timeoutErrorsIndexes: number[] = []
  lines.forEach((line, index) => {
    if (line.includes('TimeoutError:')) {
      timeoutErrorsIndexes.push(index)
    }
  })

  const snapshot = timeoutErrorsIndexes.map((index) => {
    return [
      lines[index - 1],
      lines[index].replace(/Timeout \d+ms exceeded/, 'Timeout <ms> exceeded'),
      lines[index + 4],
    ].join('\n')
  }).sort().join('\n\n')

  expect(snapshot).toMatchInlineSnapshot(`
    " FAIL  |chromium| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:39:46

     FAIL  |chromium| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:23:46

     FAIL  |chromium| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:31:46

     FAIL  |chromium| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:15:46

     FAIL  |chromium| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:6:34

     FAIL  |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:62:48

     FAIL  |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:70:48

     FAIL  |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:48:48

     FAIL  |chromium| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:54:48

     FAIL  |firefox| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:39:46

     FAIL  |firefox| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:23:46

     FAIL  |firefox| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:31:46

     FAIL  |firefox| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:15:46

     FAIL  |firefox| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:6:34

     FAIL  |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:62:48

     FAIL  |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:70:48

     FAIL  |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:48:48

     FAIL  |firefox| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:54:48

     FAIL  |webkit| hooks-timeout.test.ts > timeouts are failing correctly > afterAll
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:39:46

     FAIL  |webkit| hooks-timeout.test.ts > timeouts are failing correctly > afterEach > skipped
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:23:46

     FAIL  |webkit| hooks-timeout.test.ts > timeouts are failing correctly > beforeAll
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:31:46

     FAIL  |webkit| hooks-timeout.test.ts > timeouts are failing correctly > beforeEach > skipped
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:15:46

     FAIL  |webkit| hooks-timeout.test.ts > timeouts are failing correctly > click on non-existing element fails
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:6:34

     FAIL  |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:62:48

     FAIL  |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFailed > fails global
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:70:48

     FAIL  |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:48:48

     FAIL  |webkit| hooks-timeout.test.ts > timeouts are failing correctly > onTestFinished > fails global
    TimeoutError: locator.click: Timeout <ms> exceeded.
     ❯ hooks-timeout.test.ts:54:48"
  `)

  // page.getByRole('code').click()
  expect(stderr).toContain('locator.click: Timeout')
  // playwright error is proxied from the server to the client and back correctly
  expect(stderr).toContain('waiting for locator(\'[data-vitest="true"]\').contentFrame().getByRole(\'code\')')
  expect(stderr).toMatch(/hooks-timeout.test.ts:6:34/)
  // await expect.element().toBeVisible()
  expect(stderr).toContain('Cannot find element with locator: getByRole(\'code\')')
  expect(stderr).toMatch(/hooks-timeout.test.ts:10:50/)
}, 120_000 * 3)