hanzo.ai/e2e/agent-calls-audio.spec.ts

import { test, expect } from '@playwright/test'
import { credential } from './credential'

test.describe('Agent Voice, Audio, and Meeting Calls', () => {
  async function ensureWorkspace(page: import('@playwright/test').Page) {
    await page.waitForLoadState('domcontentloaded')
    const orgPicker = page.getByRole('region', { name: 'Choose an organization' })
    try {
      if (await orgPicker.isVisible({ timeout: 2000 })) {
        await orgPicker.getByRole('button', { name: 'hanzo', exact: true }).click()
        await expect(orgPicker).toBeHidden({ timeout: 5000 })
      }
    } catch {}
  }

  test.beforeEach(async ({ page }) => {
    const TOKEN = credential()
    if (TOKEN) {
      await page.addInitScript((t) => {
        localStorage.setItem('hanzo_iam_access_token', t)
        localStorage.setItem('hanzo_iam_id_token', t)
        try {
          const c = JSON.parse(atob(t.split('.')[1].replace(/-/g, '+').replace(/_/g, '/')))
          localStorage.setItem('hanzo_iam_expires_at', String(c.exp * 1000))
          if (c.sub) localStorage.setItem('hanzo:who', c.sub)
        } catch {}
        localStorage.setItem('hanzo_iam_current_org', 'hanzo')
        localStorage.setItem('hanzo_iam_org', 'hanzo')
        localStorage.setItem('hanzo_org', 'hanzo')
        localStorage.setItem('hanzo:org:picked', '1')
      }, TOKEN)
    }

    // Attach spy on browser native SpeechSynthesis
    await page.addInitScript(() => {
      ;(window as any).__spokenUtterances = []
      if ('speechSynthesis' in window) {
        const origSpeak = window.speechSynthesis.speak.bind(window.speechSynthesis)
        window.speechSynthesis.speak = function (utterance: SpeechSynthesisUtterance) {
          ;(window as any).__spokenUtterances.push({
            text: utterance.text,
            pitch: utterance.pitch,
            rate: utterance.rate,
            voice: utterance.voice ? utterance.voice.name : null,
          })
          try {
            origSpeak(utterance)
          } catch {}
          if (utterance.onstart) setTimeout(() => utterance.onstart?.(new Event('start') as any), 5)
          if (utterance.onend) setTimeout(() => utterance.onend?.(new Event('end') as any), 150)
        }
      }
    })
  })

  test('Hanzo Meet lobby allows inviting agents and tests audio output', async ({ page }) => {
    await page.goto('/meet')
    await ensureWorkspace(page)

    // 1. Check Header and Brand
    await expect(page.getByText('Hanzo Meet', { exact: true })).toBeVisible({ timeout: 15000 })
    await expect(page.getByText('LIVE · meet.hanzo.ai')).toBeVisible()

    // 2. Hardware Readiness Check and Audio Speaker Test
    const testAudioBtn = page.getByRole('button', { name: /test audio/i })
    await expect(testAudioBtn).toBeVisible()

    // Click Test Audio to verify agent voice speech works
    await testAudioBtn.click()

    // Verify speech synthesis received audio utterance
    const spoken = await page.evaluate(() => (window as any).__spokenUtterances)
    expect(spoken.length).toBeGreaterThan(0)
    expect(spoken[0].text).toContain('Welcome to Hanzo Meet')

    // 3. Agent Invitation Panel
    const inviteBtn = page.getByRole('button', { name: /invite agents/i })
    await expect(inviteBtn).toBeVisible()
    await inviteBtn.click()

    // Panel should open showing available meeting agents
    await expect(page.getByText('Select AI Agents to Attend Meetings')).toBeVisible()
    await expect(page.getByText('Hanzo Coder', { exact: true })).toBeVisible()
    await expect(page.getByText('Hanzo Researcher', { exact: true })).toBeVisible()
    await expect(page.getByText('Des', { exact: true })).toBeVisible()
    await expect(page.getByText('Maya', { exact: true })).toBeVisible()
  })

  test('Instant meeting starts with video iframe, agent grid, live co-pilot briefing, and Q&A', async ({ page }) => {
    await page.goto('/meet')
    await ensureWorkspace(page)

    // 1. Start Instant Meeting
    const startBtn = page.getByRole('button', { name: /start instant meeting/i })
    await expect(startBtn).toBeVisible()
    await startBtn.click()

    // 2. Verify in-call controls
    await expect(page.getByRole('button', { name: /leave call/i })).toBeVisible({ timeout: 10000 })
    await expect(page.getByRole('button', { name: /copy room link/i })).toBeVisible()

    // 3. Verify SFU stage iframe exists with WebRTC camera & microphone permissions
    const iframe = page.locator('iframe')
    await expect(iframe).toBeVisible()
    const allowAttr = await iframe.getAttribute('allow')
    expect(allowAttr).toContain('camera')
    expect(allowAttr).toContain('microphone')
    expect(allowAttr).toContain('display-capture')
    expect(allowAttr).toContain('autoplay')

    // 4. Switch to Agent Grid view
    const agentGridBtn = page.getByRole('button', { name: /agent grid/i })
    await expect(agentGridBtn).toBeVisible()
    await agentGridBtn.click()

    // Verify agent tiles appear in grid
    await expect(page.getByText('AI Agent · In Call').first()).toBeVisible()
    await expect(page.getByText('Hanzo Coder', { exact: true }).first()).toBeVisible()

    // 5. AI Co-pilot Sidebar & Live Voice Briefing
    await expect(page.getByText('AI Co-pilot · Voice: ash')).toBeVisible()
    const briefingBtn = page.getByRole('button', { name: /read meeting briefing aloud|briefing/i })
    await expect(briefingBtn).toBeVisible()
    await briefingBtn.click()

    // Verify spoken briefing was synthesized
    const spoken = await page.evaluate(() => (window as any).__spokenUtterances)
    const briefingUtterance = spoken.find((u: any) => u.text.toLowerCase().includes('briefing'))
    expect(briefingUtterance).toBeDefined()

    // 6. Ask Agent in Meeting
    const askInput = page.getByPlaceholder(/ask hanzo coder in meeting/i)
    await expect(askInput).toBeVisible()
    await askInput.fill('What are the action items?')
    await page.locator('form button[type="submit"]').click()

    // Verify question and bot response are added to the live notes transcript
    await expect(page.getByText('You: What are the action items?')).toBeVisible()
    await expect(page.getByText(/Hanzo Coder \((Bot|AI)\):/)).toBeVisible()

    // 7. Leave Call returns to lobby
    await page.getByRole('button', { name: /leave call/i }).click()
    await expect(page.getByText('Secure, High-Definition Video Calls')).toBeVisible()
  })

  test('Chat interface has agent speech buttons and handles voice synthesis', async ({ page }) => {
    await page.goto('/chat')
    await ensureWorkspace(page)

    // Verify mic button exists for voice dictation
    const micBtn = page.locator('[data-slot="prism-control"]').or(page.getByRole('button', { name: /mic|voice/i }))
    await expect(micBtn.first()).toBeVisible()

    // Crew scroller has agents with assigned voices
    const crewScroller = page.locator('[data-slot="crew"]')
    await expect(crewScroller).toBeVisible()

    // Click on Feynman to open DM chat with Feynman
    const feynmanBtn = crewScroller.getByRole('button', { name: /feynman/i })
    if (await feynmanBtn.isVisible()) {
      await feynmanBtn.click()
      await expect(page.getByRole('heading', { name: /feynman/i })).toBeVisible()
    }
  })

  test('In-call Hands-Free voice toggle enables real-time turn-taking and handles barge-in', async ({ page }) => {
    await page.goto('/meet')
    await ensureWorkspace(page)

    // Start instant meeting
    const startBtn = page.getByRole('button', { name: /start instant meeting/i })
    await expect(startBtn).toBeVisible()
    await startBtn.click()

    // Hands-Free toggle button should be visible in sidebar
    const handsFreeBtn = page.getByTestId('hands-free-voice-toggle')
    await expect(handsFreeBtn).toBeVisible({ timeout: 10000 })
    await expect(handsFreeBtn).toContainText('Hands-Free')

    // Click to enable Hands-Free voice mode
    await handsFreeBtn.click()
    await expect(handsFreeBtn).toContainText('Hands-Free On')

    // Click to disable Hands-Free voice mode
    await handsFreeBtn.click()
    await expect(handsFreeBtn).toContainText('Hands-Free')

    // Leave call
    await page.getByRole('button', { name: /leave call/i }).click()
  })
})