hanzo.ai/e2e/chat-send.spec.ts

// /chat: the composer takes a message, and the room answers in its own words.
//
// The honest outcome is the assertion. A send can end several ways on this
// surface — a streamed reply, a refusal the gateway worded,
// the free-tier quota, or simply the reader's own turn posted to a channel —
// and MOST OF THOSE ARE CORRECT. A spec that demanded a model reply would fail
// the day the account runs out of credit, or the day /chat became a team
// channel rather than a model composer (which is what it now is), and report a
// broken composer both times. That is worse than no coverage, because it is
// believed.
//
// What the test refuses is SILENCE: a draft that disappears with no turn, no
// reply and no error is the one outcome that means the
// composer is broken, and it is the outcome a naive spec cannot tell from a
// slow one.
//
//   BASE_URL=http://localhost:3000 pnpm exec playwright test --project=chromium e2e/chat-send.spec.ts

import { test, expect } from '@playwright/test'
import { api, token, open, field, sendButton, compose, shot, ready } from './app'

test.describe.configure({ timeout: 240_000 })

test.describe('chat', () => {
  test('the composer takes a draft and the room answers in its own words', async ({ page }) => {
    await open(page, '/chat')

    const draft = field(page)
    const send = sendButton(page)
    await expect(draft, 'the room must offer somewhere to write').toBeVisible({ timeout: 30_000 })
    await expect(send, 'the room must offer a way to send').toBeAttached({ timeout: 30_000 })

    // What the reader will see posted, whatever shape the room gives it.
    const said = 'reply with the single word ok'
    const posted = page.getByText(said, { exact: false })
    const reply = page.locator('[data-slot="message"][data-role="assistant"]')
    // A refusal is drawn by the room itself rather than by @hanzo/ui's Failure,
    // so it may carry no slot at all. Its retry control is what always
    // identifies it, and the quota wording is the one refusal worth naming.
    const refused = page
      .locator('[data-slot="failure"]')
      .or(page.getByRole('button', { name: 'Try again' }))
      .or(page.getByText(/free messages|insufficient|quota|Add credit/i))

    // A dev server rebuilding under the page takes the whole thread with it —
    // measured mid-stream, an answer that had begun arriving was replaced by an
    // empty room. That is the server's behaviour, not the composer's, so the
    // send is made again rather than reported as a swallowed message.
    let navigations = 0
    page.on('framenavigated', (f) => {
      if (f === page.mainFrame()) navigations++
    })

    let landed = false
    for (let attempt = 1; attempt <= 2 && !landed; attempt++) {
      const before = navigations
      await compose(draft, send, said)
      await expect(draft).toHaveValue(said)
      if (attempt === 1) await shot(page, 'chat-draft')

      if (await ready(send)) await send.click()
      else await draft.press('Enter')

      await expect
        .poll(
          async () => (await refused.count()) > 0 || (await posted.count()) > 0,
          { timeout: 60_000 },
        )
        .toBe(true)
        .catch(() => {})

      // Wait for CONTENT, not for an element. An assistant bubble mounts empty
      // and fills as the stream arrives, so a poll that stops at `count() > 0`
      // reads it in the instant before its first token and reports an empty
      // answer — which is what a poll on the element did.
      landed = await expect
        .poll(
          async () =>
            (await refused.count()) > 0 ||
            (await posted.count()) > 0 ||
            ((await reply.count()) > 0 && (await reply.first().innerText()).trim().length > 0),
          { timeout: 60_000 },
        )
        .toBe(true)
        .then(
          () => true,
          () => false,
        )

      if (!landed && navigations === before) break // a real silence, not a reload
    }

    expect(
      landed,
      navigations > 0
        ? `the send never completed, and the page reloaded ${navigations} times underneath it — ` +
          'the dev server is rebuilding, so this is not a verdict on the composer'
        : 'the composer swallowed the draft: no turn of the reader’s own, no reply, no refusal',
    ).toBe(true)

    if (await refused.count()) {
      // A refusal is a PASS, and its words are the gateway's. The free lane
      // running out is the expected one — `402 public_allowance_spent`, "You've
      // used today's free messages" — and a page that says so is doing its job.
      const words = (await page.locator('main, body').first().innerText()).replace(/\s+/g, ' ')
      expect(words.length, 'a refusal must say something').toBeGreaterThan(0)
      await shot(page, 'chat-refused')
      test.info().annotations.push({ type: 'refusal', description: words.slice(0, 200) })
      return
    }

    // Whatever came back, the reader must SEE what they asked. A composer that
    // clears the draft and shows nothing has lost the message.
    await expect(posted.first(), 'the reader must see what they wrote').toBeVisible({ timeout: 30_000 })
    await shot(page, 'chat-sent')
  })

  test('the model surface names something real', async ({ page }) => {
    await open(page, '/chat')

    // The gateway's catalog is the agreement that makes any label more than
    // decoration, and it is checked FIRST because it is true whether or not
    // this room offers a choice.
    const key = token()
    const res = await fetch(`${api()}/v1/models`, key ? { headers: { Authorization: `Bearer ${key}` } } : undefined)
    test.skip(!res.ok, `the gateway did not serve /v1/models (${res.status}) — the catalog cannot be checked`)

    const body = (await res.json()) as { data?: { id: string }[] }
    const ids = (body.data ?? []).map((m) => m.id)
    expect(ids.length, 'the gateway must serve a real catalog, not an empty envelope').toBeGreaterThan(0)
    expect(
      ids.every((id) => typeof id === 'string' && id.length > 0),
      'every row in the catalog must be a named model',
    ).toBe(true)

    // The trigger is whatever the room offers — a menu when there is a choice,
    // a fixed label when the model is decided for you. Both are honest. NEITHER
    // is also honest, and it is where this surface currently sits: /chat became
    // a team channel, and a channel does not pick a model. Asserting a menu
    // here would report a missing feature as a broken one.
    const menu = page.getByRole('combobox')
    const fixed = page.getByText('Enso Free', { exact: true })
    await expect
      .poll(async () => (await fixed.count()) > 0 || (await menu.count()) > 0, { timeout: 20_000 })
      .toBe(true)
      .catch(() => {})

    if (!(await menu.count()) && !(await fixed.count())) {
      await shot(page, 'chat-no-model-surface')
      test.skip(true, 'this room offers no model control — the catalog above is asserted, the picker is not here')
    }

    if (await menu.count()) {
      await menu.first().click()
      // An open menu must list SOMETHING. The rows are addressed by role
      // rather than by slot, because the slot vocabulary here has already
      // changed three times under this spec.
      const rows = page.getByRole('option').or(page.locator('[data-slot="command-item"]'))
      // 45s, not 30: the menu reads the gateway catalog, and on a COLD /chat
      // route (first hit after a while) the compile plus that fetch was measured
      // at ~35s while the warm run is ~3s. The data is real — it just arrives
      // slow on a cold server — so give the poll room rather than read a false
      // empty. (Warm, it returns on the first interval.)
      await expect
        .poll(async () => await rows.count(), { timeout: 45_000, message: 'the model menu opened onto nothing' })
        .toBeGreaterThan(0)
      await shot(page, 'chat-model-menu')
    } else {
      await shot(page, 'chat-model-fixed')
    }

    // `free` is the id a signed-out composer sends, and it is deliberately not
    // asserted to be in this list: the public lane may route an alias the
    // catalog does not publish, and demanding it would assert a contract
    // nothing states. Recorded so a reader can see which it is.
    test.info().annotations.push({
      type: 'catalog',
      description: `${ids.length} models served; the free alias is ${ids.includes('free') ? '' : 'not '}among them`,
    })
  })
})