hanzo.ai/e2e/gates/sitemap.spec.ts

import { test, expect } from '@playwright/test'
import { readFileSync } from 'node:fs'
import oss from '../../lib/data/oss.json' with { type: 'json' }
import held from '../../lib/data/oss-repos.json' with { type: 'json' }
import families from '../../lib/data/oss-organizations.json' with { type: 'json' }
import { answer } from '../../lib/edge'
import { forward } from '../../lib/moved'
import { NOINDEX, indexable } from '../../lib/publish'
import { copy } from '../../lib/routes'
import { pages, read, serveExport } from './export'

/**
 * The sitemap and the pages agree, in both directions, and nothing links a
 * repository a reader cannot open.
 *
 *   every URL offered renders     served, with a heading and words to read
 *   every public page is offered  the dynamic segments included
 *   every offered page is linked  from some other page, or it is an orphan
 *   every GitHub link opens       a repository of ours must be public
 *   every URL offered is final    one self-canonical, no noindex, no redirect, a lastmod
 *   no link redirects             the export links each page at its one address
 */

/** Every route sitemap.xml offers. */
function offered(): string[] {
  return [...read('sitemap.xml').matchAll(/<loc>https:\/\/hanzo\.ai([^<]*)<\/loc>/g)].map((m) => m[1] || '/')
}

/** Every same-site path the export's pages link to, as the path it names. */
function linked(): Map<string, Set<string>> {
  const into = new Map<string, Set<string>>()
  for (const { route, file } of pages()) {
    for (const m of readFileSync(file, 'utf8').matchAll(/href="((?:https:\/\/hanzo\.ai)?\/[^"#?]*)/g)) {
      const to = m[1].replace(/^https:\/\/hanzo\.ai/, '').replace(/\/+$/, '') || '/'
      if (to === route) continue
      if (!into.has(to)) into.set(to, new Set())
      into.get(to)!.add(route)
    }
  }
  return into
}

test('every URL the sitemap offers renders', async ({ request }) => {
  const site = await serveExport()
  try {
    const urls = offered()
    expect(urls.length, 'the sitemap offers the site').toBeGreaterThan(500)
    const broken: string[] = []
    for (const route of urls) {
      const res = await request.get(site.url + encodeURI(route))
      const html = await res.text()
      if (res.status() !== 200) broken.push(`${route} → ${res.status()}`)
      else if (!/<h1[\s>]/.test(html)) broken.push(`${route} → no <h1>`)
      else if (copy(html).length < 200) broken.push(`${route} → ${copy(html).length} characters`)
    }
    expect(broken, 'offered in sitemap.xml and does not render a page').toEqual([])
  } finally {
    await site.close()
  }
})

test('every public page is in the sitemap', () => {
  const inMap = new Set(offered())
  const missing = pages()
    .map(({ route }) => route)
    .filter((route) => indexable(route) && !inMap.has(route))
  expect(missing, 'public, shipped, and not offered to a crawler').toEqual([])
})

test('every page the sitemap offers is linked from another page', () => {
  const into = linked()
  const orphans = offered().filter((route) => route !== '/' && !into.get(route)?.size)
  expect(orphans, 'offered in sitemap.xml and linked from nowhere').toEqual([])
})

test('no page links a repository of ours that a reader cannot open', () => {
  // Every org a group counts, Hanzo's, Lux's and Zoo's; open is Hanzo's every public
  // repository and each group's own, as scripts/sync-oss.ts listed them.
  const ours = new Set(Object.values(families).flatMap((f) => f.orgs))
  const open = new Set([
    ...oss.public,
    ...Object.entries(held.orgs as Record<string, { repos: { name: string }[] }>).flatMap(([org, o]) => o.repos.map((r) => `${org}/${r.name}`)),
  ])
  const closed = new Map<string, Set<string>>()
  for (const { route, file } of pages()) {
    for (const m of readFileSync(file, 'utf8').matchAll(/href="https:\/\/github\.com\/([^/"#?]+)\/([^/"#?]+)/g)) {
      const repo = `${m[1]}/${m[2].replace(/\.git$/, '')}`
      if (!ours.has(m[1]) || open.has(repo)) continue
      if (!closed.has(repo)) closed.set(repo, new Set())
      closed.get(repo)!.add(route)
    }
  }
  const report = [...closed].map(([repo, from]) => `${repo} <- ${[...from].slice(0, 3).join(', ')}`)
  expect(report, 'links to repositories that are private or gone').toEqual([])
})

test('every URL the sitemap offers is final: one self-canonical, no noindex, no redirect, a lastmod', () => {
  const files = new Map(pages().map(({ route, file }) => [route, file]))
  const entries = [...read('sitemap.xml').matchAll(/<url>([\s\S]*?)<\/url>/g)].map((m) => m[1])
  expect(entries.length, 'the sitemap offers the site').toBeGreaterThan(500)
  const wrong: string[] = []
  for (const entry of entries) {
    const loc = entry.match(/<loc>([^<]+)<\/loc>/)?.[1] ?? ''
    const route = loc.replace(/^https:\/\/hanzo\.ai/, '') || '/'
    const lastmod = entry.match(/<lastmod>([^<]+)<\/lastmod>/)?.[1]
    if (!lastmod || Number.isNaN(Date.parse(lastmod))) wrong.push(`${loc}: no lastmod`)
    if (forward(route)) wrong.push(`${loc}: retired, the edge sends it to ${forward(route)}`)
    const a = answer(route === '/' ? 'https://hanzo.ai/' : loc)
    if ('location' in a) wrong.push(`${loc}: redirects to ${a.location}`)
    const file = files.get(route)
    if (!file) {
      wrong.push(`${loc}: no page`)
      continue
    }
    const html = readFileSync(file, 'utf8')
    const head = html.slice(0, html.indexOf('</head>'))
    const canonical = [...head.matchAll(/<link[^>]+rel="canonical"[^>]*href="([^"]+)"/g)].map((m) => m[1])
    if (canonical.length !== 1 || canonical[0] !== loc) wrong.push(`${loc}: canonical ${canonical.join(' ') || 'missing'}`)
    if (html.includes(NOINDEX) || /<meta[^>]+name="robots"[^>]+noindex/.test(head)) wrong.push(`${loc}: noindex`)
  }
  expect(wrong, 'offered in sitemap.xml and not a final, indexable, canonical page').toEqual([])
})

test('no page links an address the edge redirects', () => {
  // Every same-site href in the export, deduplicated, asked of the edge's own
  // rules: a trailing slash, a .html, www, http or a retired address is a
  // redirect a crawler follows and reports, from a link we wrote.
  const from = new Map<string, string>()
  for (const { route, file } of pages()) {
    for (const m of readFileSync(file, 'utf8').matchAll(/href="((?:https?:\/\/(?:www\.)?hanzo\.ai)?\/(?!\/)[^"#]*)"/g)) {
      if (!from.has(m[1])) from.set(m[1], route)
    }
  }
  const redirected = [...from]
    .map(([href, route]) => {
      const a = answer(href.startsWith('/') ? `https://hanzo.ai${href}` : href.replace(/^http:/, 'https:'))
      if (href.startsWith('http://')) return `${href} on ${route}: http`
      return 'location' in a ? `${href} on ${route} -> ${a.location}` : null
    })
    .filter(Boolean)
  expect(redirected, 'links to an address that answers with a redirect').toEqual([])
})