hanzo.ai/e2e/gates/deploy.spec.ts

import { test, expect } from '@playwright/test'
import { existsSync, readFileSync, readdirSync } from 'node:fs'
import { join } from 'node:path'
import { ROOT } from './export'

/**
 * The pipeline, as a subject a gate can read.
 *
 * Every other spec here asserts against `out/`, because the bytes that ship are
 * the only honest subject. These three assert against the workflows that ship
 * them, for the reason `publish.spec.ts` reads `app/robots.ts`: some defects are
 * a NAME that resolves to nothing, and nothing downstream can see the
 * difference between a name that resolves to nothing and a value that is
 * legitimately absent.
 *
 * Both defects below were live in `.github/workflows/deploy.yml` and both were
 * silent. Actions substitutes an unknown `steps.<id>.outputs.<name>` with the
 * empty string and BuildKit warns-and-ignores an unknown `--build-arg`, so a run
 * carrying neither the key it fetched nor the argument it meant to pass is a
 * green run. The KMS step goes to some length to make an ABSENT key loud — three
 * `::error::` branches — and then the value it fetched was dropped twice on one
 * line. A control that reports success while carrying nothing is the failure
 * mode; these say the wiring resolves.
 */
const DIR = join(ROOT, '.github', 'workflows')

type Workflow = { name: string; text: string }

const workflows = (): Workflow[] =>
  readdirSync(DIR)
    .filter((name) => name.endsWith('.yml'))
    .map((name) => ({ name, text: readFileSync(join(DIR, name), 'utf8') }))

/** A job's steps. Six spaces then `- ` is a step; a `run:` body is indented deeper. */
const steps = (text: string): string[] => text.split(/\n {6}- /)

test('every step output a workflow reads is one some step writes', () => {
  const wrote = new Map<string, Set<string>>()
  const read: { where: string; ref: string }[] = []
  const anonymous: string[] = []
  for (const { name, text } of workflows()) {
    for (const step of steps(text)) {
      const id = /^\s*id:\s*(\S+)/m.exec(step)?.[1]
      const written = [...step.matchAll(/([A-Za-z_]\w*)=[^\n]*>>\s*"?\$GITHUB_OUTPUT/g)].map((m) => m[1])
      if (written.length === 0) continue
      // An output written by a step with no `id:` is unreachable by definition —
      // there is no name to read it under.
      if (!id) anonymous.push(`${name}: writes ${written.join(', ')} in a step with no id`)
      else wrote.set(`${name}#${id}`, new Set([...(wrote.get(`${name}#${id}`) ?? []), ...written]))
    }
    for (const m of text.matchAll(/steps\.([A-Za-z_][\w-]*)\.outputs\.([A-Za-z_][\w-]*)/g)) {
      read.push({ where: `${name}#${m[1]}`, ref: m[2] })
    }
  }
  expect(anonymous, anonymous.join('\n')).toEqual([])
  // The floor, scoped to when it can mean something. "No reference is
  // unresolved" is satisfied perfectly by a workflow set that reads nothing, so
  // a set that PASSES values between steps has to prove it still reads them.
  // A set that passes none has nothing to lose track of: the publishable key is
  // now declared rather than fetched, so no step writes an output at all, and
  // demanding a read here would fail a pipeline whose wiring is simply gone.
  // What replaced this control is stronger and runs in `deploy.yml` before
  // publish: it greps the built `out/` for a `pk-` key and refuses to ship a
  // bundle carrying none, then asks the ingest endpoint whether that key still
  // resolves. Both hold however the key reaches the build.
  if (wrote.size > 0) {
    expect(read.length, 'a workflow writes a step output that nothing reads').toBeGreaterThan(0)
  }
  const unresolved = read
    .filter(({ where, ref }) => !wrote.get(where)?.has(ref))
    .map(({ where, ref }) => `${where}.outputs.${ref} — written by no step (Actions substitutes "")`)
  expect(unresolved, unresolved.join('\n')).toEqual([])
})

/**
 * How the apex PUBLISHES: a Sites-plane deploy through hanzoai/ci's `site`
 * action, matched by its `uses:` line, which only a real step can carry.
 * `v1` floats, so this name is the one place the file says which action.
 */
const PUBLISH = 'hanzoai/ci/.github/actions/site'

/** The forge that serves the action may be named in the ref; both forms are it. */
const uses = (ref = '') =>
  new RegExp(`^\\s*uses:\\s*(?:https://\\S+/)?${PUBLISH}@${ref}`, 'm')

test('the gates run on every push to main, every pull request and every day, beside the publish', () => {
  // The gates are their own workflow (gates.yml) and do not hold the publish:
  // deploy.yml ships what main holds, and gates.yml checks the same export on
  // every push to main, on every pull request, and in full once a day. What this
  // pins is that the gates are never off — a trigger dropped from gates.yml would
  // leave main shipping with nothing reading it.
  const gates = workflows().find(({ name }) => name === 'gates.yml')
  expect(gates, 'no gates.yml workflow runs the gates').toBeTruthy()
  const text = gates!.text
  expect(text, 'gates.yml never runs `pnpm gates`').toMatch(/^\s*run:\s*pnpm gates\b/m)
  expect(text, 'gates.yml does not run on push to main').toMatch(/^\s{2}push:\s*\n\s{4}branches:\s*\[main\]/m)
  expect(text, 'gates.yml does not run on pull requests').toMatch(/^\s{2}pull_request:/m)
  expect(text, 'gates.yml has no daily full run').toMatch(/^\s{2}schedule:\s*\n\s{4}- cron:/m)
})

test('the apex publish pins the action and carries the one credential it needs', () => {
  const [apex] = workflows().filter(({ text }) => uses().test(text))
  // Say which action is missing. Reading `.text` off nothing throws
  // "Cannot read properties of undefined", which is how a rename upstream
  // reported itself as a broken test rather than as an unpublishable site.
  expect(apex, `no workflow publishes the apex via ${PUBLISH}`).toBeDefined()
  // A floating action ref is the same defect as a floating image tag: the build
  // that ships becomes unnameable. bin/site resolves the action ref as its
  // OWN script ref, so @v1 is what makes "which publisher ran" answerable.
  expect(apex.text, `${apex.name} must pin the publish action to a ref`).toMatch(uses('v\\d'))
  // ONE credential. The 202 returns a prefix-scoped, short-lived presigned POST
  // grant, so CI needs no bucket key — SITES_S3_* is the standing shared-bucket
  // credential the grant replaced, and any repo holding it could overwrite every
  // other org's site.
  //
  // What this file passes is the KMS CLIENT IDENTITY, and the key itself is the
  // action's to fetch. The gate has now demanded three different things, each
  // time following the credential to where it actually lives:
  // `secrets.HANZO_DEPLOY_TOKEN` pinned it to a forge secret that cannot say
  // whether the key inside it is alive — it resolved to no principal and cloud
  // answered 403 under every green gate including this one; a KMS read in THIS
  // file fixed the apex and left hips, hanzo.app and agency on the dead secret.
  // The read belongs with the publish it serves, so one reseal rotates them all.
  expect(apex.text, `${apex.name} publishes without the KMS client identity`).toMatch(
    /KMS_CLIENT_ID:\s*\$\{\{\s*secrets\.KMS_CLIENT_ID\s*\}\}/,
  )
  expect(apex.text, `${apex.name} passes no KMS client secret, so the key cannot be read`).toMatch(
    /KMS_CLIENT_SECRET:\s*\$\{\{\s*secrets\.KMS_CLIENT_SECRET\s*\}\}/,
  )
  // The interpolation form, not the bare name — for the reason spelled out below
  // about SITES_S3_*. This workflow's comments carry the bare name, in the lines
  // that explain where the key moved and why.
  expect(apex.text, `${apex.name} takes the deploy key from a forge secret again`).not.toMatch(
    /\$\{\{\s*secrets\.HANZO_DEPLOY_TOKEN\s*\}\}/,
  )
  // Nor does it hand the action a token of its own. A second way to supply the
  // credential is a second place it can be stale.
  expect(apex.text, `${apex.name} passes its own deploy token to the action`).not.toMatch(
    /HANZO_DEPLOY_TOKEN:\s*\$\{\{/,
  )
  // `secrets.SITES_S3_`, not the bare name. The bare name is how the previous
  // version of the test above went wrong: it keyed on a string that a COMMENT can
  // legitimately contain — and this file's own workflow contains it, in the line
  // telling the next reader not to add it. A gate must match the thing, not the
  // word for the thing, or documenting a rule becomes a way to break it.
  expect(apex.text, `${apex.name} reintroduces SITES_S3_* — the grant replaced it`).not.toMatch(
    /\$\{\{\s*secrets\.SITES_S3_/,
  )
})