hanzo.ai/scripts/crawl.mjs

#!/usr/bin/env node
/**
 * Ask the live site what it answers for a list of URLs, the way a crawler would.
 *
 * A tool you RUN, not a gate: it fetches production, throttled. Sources, mixed
 * freely:
 *
 *   sitemap        the live sitemap.xml
 *   <file>.csv     a Search Console export (Page indexing -> a reason -> Export
 *                  -> CSV gives a zip; pass the zip or its Table.csv)
 *   <file>.zip     that zip as downloaded
 *   <file>.txt     one URL or path per line
 *
 * For each URL it records the first answer (status, Location) and follows up to
 * five hops to the last one. For a 200 page it reads the canonical, the robots
 * meta and X-Robots-Tag. Each URL is then judged against lib/moved and
 * lib/publish:
 *
 *   ok        answers what the policy says it should
 *   moved     retired, and the edge 301s it to its target in one hop
 *   gone      404 and nothing names a target: the rows to extend MOVED from
 *   wrong     anything else, with the reason
 *
 * Usage:  node scripts/crawl.mjs [--base https://hanzo.ai] [--rate 4] [--json out.json] [--gone gone.txt] <source>...
 *         node scripts/crawl.mjs --replay out.json   judge a saved run again, fetching nothing
 */
import { execFileSync } from 'node:child_process'
import { readFileSync, writeFileSync } from 'node:fs'
import { forward } from '../lib/moved.ts'
import { ALTERNATE, HOSTED, policy } from '../lib/publish.ts'

const argv = process.argv.slice(2)
const flag = (name, fallback) => {
  const at = argv.indexOf(name)
  if (at < 0) return fallback
  const [, value] = argv.splice(at, 2)
  return value
}
const BASE = flag('--base', 'https://hanzo.ai').replace(/\/$/, '')
const RATE = Number(flag('--rate', '4'))
const JSON_OUT = flag('--json', null)
const REPLAY = flag('--replay', null)
const GONE_OUT = flag('--gone', null)
if (!(RATE > 0 && RATE <= 5)) throw new Error('--rate is requests per second, at most 5')
if (argv.length === 0 && !REPLAY) {
  console.error('usage: node scripts/crawl.mjs [--base URL] [--rate N] [--json FILE] sitemap|<csv|zip|txt>...')
  process.exit(2)
}

const UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36'
const HOST = new URL(BASE).host

/** The URLs in one CSV document: every field that is a URL on our hosts. */
function csv(text) {
  const found = []
  for (const line of text.split(/\r?\n/)) {
    for (const field of line.match(/"(?:[^"]|"")*"|[^,]+/g) ?? []) {
      const value = field.replace(/^"|"$/g, '').replace(/""/g, '"').trim()
      if (/^https?:\/\//.test(value)) found.push(value)
    }
  }
  return found
}

async function source(arg) {
  if (arg === 'sitemap') {
    const xml = await (await fetch(`${BASE}/sitemap.xml`, { headers: { 'user-agent': UA } })).text()
    return [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1])
  }
  if (arg.endsWith('.zip')) return csv(execFileSync('unzip', ['-p', arg, 'Table.csv'], { encoding: 'utf8' }))
  const text = readFileSync(arg, 'utf8')
  if (arg.endsWith('.csv')) return csv(text)
  return text
    .split(/\r?\n/)
    .map((line) => line.trim())
    .filter((line) => line && !line.startsWith('#'))
}

/** A URL as fetch and the judge both read it: `https://hanzo.ai` is `https://hanzo.ai/`. */
const absolute = (u) => new URL(u.startsWith('/') ? BASE + u : u).href
const urls = [...new Set((await Promise.all(argv.map(source))).flat().map(absolute))]

const sleep = (ms) => new Promise((ok) => setTimeout(ok, ms))
let last = 0
async function get(url) {
  const wait = last + 1000 / RATE - Date.now()
  if (wait > 0) await sleep(wait)
  last = Date.now()
  for (let attempt = 0; ; attempt++) {
    try {
      return await fetch(url, { redirect: 'manual', headers: { 'user-agent': UA }, signal: AbortSignal.timeout(30_000) })
    } catch (err) {
      if (attempt === 2) return { status: 0, headers: new Headers(), text: async () => String(err) }
      await sleep(2000)
    }
  }
}

const attr = (html, re) => [...html.matchAll(re)].map((m) => m[1])

async function probe(url) {
  const hops = []
  let at = url
  for (let i = 0; i < 6; i++) {
    const res = await get(at)
    const hop = { url: at, status: res.status }
    const location = res.headers.get('location')
    if (res.status >= 300 && res.status < 400 && location) {
      hop.location = new URL(location, at).href
      hops.push(hop)
      at = hop.location
      if (new URL(at).host !== HOST) break
      continue
    }
    const type = res.headers.get('content-type') ?? ''
    if (type.includes('text/html')) {
      const html = await res.text()
      const head = html.slice(0, html.indexOf('</head>') + 1 || 200_000)
      hop.canonical = attr(head, /<link[^>]+rel=["']canonical["'][^>]*href=["']([^"']+)["']/gi)
      hop.robots = attr(head, /<meta[^>]+name=["']robots["'][^>]*content=["']([^"']+)["']/gi)
      hop.title = (head.match(/<title>([^<]*)<\/title>/) ?? [])[1]
    }
    const xrobots = res.headers.get('x-robots-tag')
    if (xrobots) hop.robots = [...(hop.robots ?? []), xrobots]
    hops.push(hop)
    break
  }
  return hops
}

/** The path of a URL on this site, or null for another host. */
function path(url) {
  const u = new URL(url)
  if (u.host !== HOST && u.host !== `www.${HOST}`) return null
  return u.pathname
}

/** The canonical form of a path: one slash between segments, no trailing slash, no .html, no index. */
const clean = (p) =>
  p
    .replace(/\/{2,}/g, '/')
    .replace(/\/index(\.html)?$/, '/')
    .replace(/\.html$/, '')
    .replace(/(.)\/+$/, '$1')

/** A path as its segments name it, `%3A` as `:`; a malformed escape stays as it came. */
const decoded = (p) => {
  try {
    return decodeURI(p)
  } catch {
    return p
  }
}

/** Next's own not-found files, which the edge answers 404 by name. */
const MISSING = new Set(['/404', '/_not-found'])

function judge(url, hops) {
  const first = hops[0]
  const final = hops[hops.length - 1]
  const p = path(url)
  if (p === null) return { verdict: 'other-host' }
  const bare = clean(p)
  const query = new URL(url).search
  const to = forward(decoded(bare))
  if (to && !new URLSearchParams(query).has('go-get')) {
    const want = new URL(to, BASE).href + query
    if (first.status !== 301) return { verdict: 'wrong', why: `MOVED to ${to}, answered ${first.status}` }
    if (first.location !== want) return { verdict: 'wrong', why: `MOVED to ${want}, went to ${first.location}` }
    return { verdict: 'moved', to }
  }
  if (HOSTED[bare]) {
    const want = new URL(HOSTED[bare]).href
    return first.status === 301 && first.location === want ? { verdict: 'moved', to: want } : { verdict: 'wrong', why: `hosted at ${want}, answered ${first.status}` }
  }
  if (MISSING.has(bare)) return first.status === 404 ? { verdict: 'gone' } : { verdict: 'wrong', why: `a not-found page answered ${first.status}` }
  const self = `${BASE}${bare}${query}`
  // The ingress cleans `//x` to `/x` before any rule reads it: the same request.
  if (url.replace(/^(https?:\/\/[^/]+)\/{2,}/, '$1/') !== self) {
    if (first.status !== 301 || first.location !== self) return { verdict: 'wrong', why: `a second address for ${self}: ${first.status} ${first.location ?? ''}` }
    return final.status === 200 ? { verdict: 'ok', why: 'canonical form' } : { verdict: 'gone' }
  }
  if (first.status === 404 || first.status === 410) return { verdict: 'gone' }
  if (first.status !== 200) return { verdict: 'wrong', why: `answered ${first.status}${first.location ? ` -> ${first.location}` : ''}` }
  // A file, not a page: nothing below applies.
  if (!final.canonical) return { verdict: 'ok' }
  if (policy(bare) === 'private') return { verdict: 'ok' }
  const withheld = policy(bare) !== 'public'
  const noindex = (final.robots ?? []).some((r) => /noindex/i.test(r))
  if (withheld !== noindex) return { verdict: 'wrong', why: withheld ? 'withheld and indexable' : 'public and noindex' }
  const owner = `${BASE}${ALTERNATE[bare] ?? (bare === '/' ? '' : bare)}`
  if (!withheld && (final.canonical?.length !== 1 || final.canonical[0] !== owner)) {
    return { verdict: 'wrong', why: `canonical ${final.canonical?.join(' ') || 'missing'}, want ${owner}` }
  }
  return { verdict: 'ok' }
}

const rows = []
const started = Date.now()
if (REPLAY) {
  for (const { url, hops } of JSON.parse(readFileSync(REPLAY, 'utf8'))) rows.push({ url, hops, ...judge(url, hops) })
}
for (const [i, url] of urls.entries()) {
  const hops = await probe(url)
  rows.push({ url, hops, ...judge(url, hops) })
  if ((i + 1) % 100 === 0) console.error(`${i + 1}/${urls.length} in ${Math.round((Date.now() - started) / 1000)}s`)
}

const bucket = (s) => (s >= 500 || s === 0 ? '5xx' : String(s))
const tally = (key) => rows.reduce((m, r) => m.set(key(r), (m.get(key(r)) ?? 0) + 1), new Map())
const show = (title, m) => console.log(`${title}: ${[...m].sort().map(([k, v]) => `${k} ${v}`).join(', ')}`)

console.log(`${rows.length} URLs from ${REPLAY ?? argv.join(' ')} against ${BASE}`)
show('first answer', tally((r) => bucket(r.hops[0].status)))
show('last answer', tally((r) => bucket(r.hops[r.hops.length - 1].status)))
show('verdict', tally((r) => r.verdict))
for (const r of rows.filter((r) => r.verdict === 'wrong')) console.log(`wrong  ${r.url}  ${r.why}`)
const gone = rows.filter((r) => r.verdict === 'gone').map((r) => new URL(r.url).pathname)
const shelves = tally((r) => (r.verdict === 'gone' ? new URL(r.url).pathname.split('/').slice(0, 2).join('/') : null))
shelves.delete(null)
if (gone.length) show('gone, by first segment', shelves)
if (JSON_OUT) writeFileSync(JSON_OUT, JSON.stringify(rows, null, 1))
if (GONE_OUT) writeFileSync(GONE_OUT, gone.map((p) => `${p}\n`).join(''))