hanzo.ai/e2e/gates/backing.spec.ts

import { test, expect } from '@playwright/test'
import { readdirSync, readFileSync } from 'node:fs'
import { extname, join, relative } from 'node:path'
import { requireExport } from './export'
import { scan } from './backing'
import snapshot from '../../lib/data/benchmarks-kai.json' with { type: 'json' }

/**
 * NO FILE THE SITE SHIPS SAYS WHAT BACKS ENSO, ZEN OR KAI.
 *
 * Customer pages name Hanzo's models by their own names; other labs' models
 * stay named as competitors on a board and as entries in the gateway's
 * catalog, never as something inside ours. A name can reach the export by
 * more doors than the page a reader sees: a sentence typed into a page, a
 * data file or a dependency bundled into a client chunk, a filter that lists
 * the names it hides, flight data, the generated llms.txt and markdown twins.
 * So this reads every text file in `out/`, the bytes that ship, as e2e/gates/
 * backing reads them, and fails on each rule it finds held. It reads files
 * rather than rendered pages, so it needs no server and no browser.
 */

/** What the export serves as text: pages, flight data, scripts, styles, data, markdown, images drawn in XML, and the files with no extension. */
const TEXT = new Set(['.html', '.txt', '.md', '.js', '.mjs', '.css', '.json', '.xml', '.svg', '.webmanifest', '.sh', ''])

function* files(dir: string): Generator<string> {
  for (const entry of readdirSync(dir, { withFileTypes: true })) {
    const path = join(dir, entry.name)
    if (entry.isDirectory()) yield* files(path)
    else if (entry.isFile() && TEXT.has(extname(entry.name))) yield path
  }
}

const rules = (text: string, kind: string) => scan(text, kind).map((f) => f.rule)

test('the check can fail', () => {
  // A model inside ours, in one sentence, a script string, a flight value or a markdown line.
  expect(rules('<p>Enso picks among models from deepseek and nvidia, and Zen.</p>', '.html')).toContain('inside')
  expect(rules('x={fullName:"Zen Nano",d:"Zen Nano is Qwen3-0.6B, renamed."}', '.js')).toContain('inside')
  expect(rules('self.__next_f.push([1,"5:[\\"$\\",\\"p\\",null,{\\"children\\":\\"Zen 6 is the Qwen3.8 generation, served by Hanzo Engine and tuned by Zoo Labs for long context, tools and code; this sentence is long enough to be read as a document.\\"}]"])', '.js')).toContain('inside')
  expect(rules('| zen-eco | Zen Eco | MiniMax lineage |', '.md')).toContain('inside')
  // Across inline markup, across a record's fields, in any case, under any Zen id.
  expect(rules('<p>Enso routes to <strong>Qwen 3.5</strong> for code.</p>', '.html')).toContain('inside')
  expect(rules('{"system":"enso","arms":["qwen3.5-397b"]}', '.json')).toContain('inside')
  expect(rules('<p>ENSO ROUTES TO KIMI</p>', '.html')).toContain('inside')
  expect(rules('"zen-coder is a Wan2.1 checkpoint"', '.js')).toContain('inside')
  expect(rules('Zen Eco is a Tongyi model.', '.md')).toContain('inside')
  // What a router's record says it called, or what a model was made from.
  expect(rules("notes: \"Routed arm per item: {'solo': 1}\"", '.js')).toContain('routing')
  expect(rules('`routed to ${arms}`', '.js')).toContain('routing')
  expect(rules('"Open-weight repackage of Lab/Model-8B (MIT)."', '.js')).toContain('routing')
  expect(rules('"version":"anthropic-claude-fable-5+openai-gpt-5.6-sol"', '.json')).toContain('version')
  // A filter that lists the names it hides, and a vendor router read as ours.
  expect(rules('let l=/qwen|kimi|glm/i', '.js')).toContain('alternation')
  expect(rules('s="Enso Free",d=new Set(["free","enso-free","openrouter/free"])', '.js')).toContain('alias')
  expect(rules('n="openrouter/auto"===t?"enso-auto":t', '.js')).toContain('alias')
  // A link whose address names a model inside ours and whose words name ours.
  expect(rules('<a href="https://example.com/runs/spark-cuda-qwen3-4b">CUDA<!-- --> <!-- -->Zen-4B</a>', '.html')).toContain('link')
  // Kai's base, anywhere.
  expect(rules('{"system":"laya"}', '.json')).toContain('base')
  expect(rules('convaiinnovations/laya-multilingual@e4e9', '.txt')).toContain('base')
})

test('the check leaves competitors, the catalog and the typeface alone', () => {
  for (const [text, kind] of [
    ['<p>Enso 92.9% GPQA Diamond — 10.9× less than Opus 4.8 (86.4%, $3.66).</p>', '.html'],
    ['<title>Enso vs OpenRouter, Portkey, LiteLLM — Hanzo</title>', '.html'],
    ['x={deepseek:"DeepSeek",qwen:"Qwen",zen:"zoo"}', '.js'],
    ['<link rel="preload" href="/_next/static/media/Zen-Variable.woff2"/><link rel="canonical" href="https://hanzo.ai/models/qwen"/>', '.html'],
    ['x=["DeepSeek-AI: DeepSeekMath & GRPO","Lab 3: a GRPO loop on Zen 6 Flash"]', '.js'],
    ['<p>The simplest way to get free inference. openrouter/free is a router that selects free models.</p>', '.html'],
    ['every run visible and replayable', '.md'],
    ['{"system":"opus-4.8","version":"anthropic-opus"}', '.json'],
    // A route tree names a page's sibling segments; /models/[org] beside /models/enso is routing, not content.
    [':HL["/_next/static/css/a.css","style"]\n0:{"tree":{"name":"org","param":{"type":"d","key":"deepseek","siblings":["enso"]}}}', '.txt'],
    ['<p>Each run is priced at the list. <strong>DeepSeek</strong> V4 Pro scored 74.2%.</p>', '.html'],
    ['Third-party models (OpenAI, Anthropic, Google, DeepSeek) are billed from your pre-paid credit topup. First-party models (Enso, Zen, Kai) are covered by your plan.', '.js'],
  ] as const)
    expect(rules(text, kind), text).toEqual([])
})

/**
 * The Kai snapshot is read here as well as in the export because it is the one
 * subject this gate has that arrives by data rather than by source: a push that
 * changes only `lib/data/` imports nothing a spec's dependency graph already
 * knew, so `--only-changed` finds no test to run.
 */
test('the bundled Kai snapshot names no base', () => {
  expect(rules(JSON.stringify(snapshot), '.json')).not.toContain('base')
})

/**
 * The export is some 7,800 text files and 400 MB, which one worker reads in a
 * minute. So the sorted listing is dealt into PARTS, one test each, and the
 * tests run on as many workers as the runner gives: file i is part i % PARTS's,
 * so every file is read, and by exactly one part.
 */
const PARTS = 8

for (let part = 0; part < PARTS; part++) {
  test(`no file in the export says what backs Enso, Zen or Kai (${part + 1} of ${PARTS})`, () => {
    const out = requireExport()
    const mine = [...files(out)].sort().filter((_, i) => i % PARTS === part)
    const found: string[] = []
    for (const path of mine) {
      for (const f of scan(readFileSync(path, 'utf8'), extname(path))) found.push(`${relative(out, path)} [${f.rule}]: …${f.at}…`)
    }
    expect(mine.length, 'text files read from the export').toBeGreaterThan(0)
    expect(found).toEqual([])
  })
}