mirror of
https://github.com/hanzo-apps/hanzo.ai.git
synced 2026-10-28 07:32:52 +00:00
zeekay
1c71e7e200
noindex, the font preload, the canonical and the markdown link were each a process that read the pages from disk and wrote back what it changed: 8.6 s of every deploy (run 39431048221). scripts/export.mjs reads every page at once, runs the same passes in the same order over memory, and writes each changed page once; wellknown and skill follow as before. On the export rebuilt from the live pages the output is byte for byte what the four processes wrote, .md files, llms.txt and skill.md included, in 4.8 s where they took 12.7 on the same box.
62 lines
2.9 KiB
JavaScript
62 lines
2.9 KiB
JavaScript
/**
|
|
* Control 3, written into the export.
|
|
*
|
|
* `lib/publish` says which routes a crawler may fetch and must not index; this
|
|
* puts the tag that says so into every page it names, and into no other. It
|
|
* runs as the second half of `pnpm build` — the export is not finished until
|
|
* the pages that must not be indexed say so.
|
|
*
|
|
* Loud, never partial: it counts what it stamped against what the policy asked
|
|
* for and exits non-zero on any disagreement. A control that quietly covers
|
|
* nineteen of twenty pages is worse than no control, because the twentieth is
|
|
* indexed and the build is green.
|
|
*
|
|
* It also refuses to run over a page that already carries a robots tag. That is
|
|
* the second spelling appearing, and two sources for one control is how the
|
|
* first one drifted.
|
|
*/
|
|
// It imports the TypeScript source directly, which Node strips types from on
|
|
// its own from 22.18. That is the floor `.nvmrc` and `engines` both name, and
|
|
// it is the same floor wrangler already forced on the deploy. The alternative
|
|
// was a second copy of the list in JavaScript, which is the thing this file
|
|
// exists to prevent.
|
|
import { join } from 'node:path'
|
|
import { EMPTY, NOINDEX, UNAPPROVED, policy } from '../lib/publish.ts'
|
|
import { shipped } from '../lib/routes.ts'
|
|
|
|
const OUT = join(import.meta.dirname, '..', 'out')
|
|
// At the END of the head, not the start: the character encoding declaration is
|
|
// supposed to be the first thing in there, and a tag inserted ahead of it pushes
|
|
// `<meta charSet>` down for no reason. Anywhere in the head is equally read.
|
|
const HEAD = '</head>'
|
|
const EXISTING = /<meta[^>]+name=["']robots["']/i
|
|
|
|
// A pass of scripts/export.mjs, which reads each page once and writes it once.
|
|
export default function noindex({ read, write }) {
|
|
const pages = shipped(OUT)
|
|
if (pages.length === 0) {
|
|
throw new Error(`no export at ${OUT}: \`next build\` writes it, and this is the rest of that build`)
|
|
}
|
|
|
|
// A route named by a list and absent from the export is a typo, and the tag it
|
|
// asked for lands on nothing. Silence there is the whole failure mode, so it is
|
|
// the first thing checked.
|
|
const withheld = new Set(pages.filter(({ route }) => policy(route) === 'noindex').map(({ route }) => route))
|
|
const absent = [...UNAPPROVED, ...EMPTY].filter((route) => !withheld.has(route))
|
|
if (absent.length > 0) {
|
|
throw new Error(`withheld from indexing but not in the export: ${absent.join(', ')}`)
|
|
}
|
|
|
|
for (const { route, file } of pages) {
|
|
if (!withheld.has(route)) continue
|
|
const html = read(file)
|
|
if (html.includes(NOINDEX)) continue
|
|
if (EXISTING.test(html)) {
|
|
throw new Error(`${route} already declares a robots tag: lib/publish is the one place that decides this`)
|
|
}
|
|
if (!html.includes(HEAD)) throw new Error(`${route} has no <head> to carry the tag`)
|
|
write(file, html.replace(HEAD, NOINDEX + HEAD))
|
|
}
|
|
console.log(`noindex: ${withheld.size} of ${pages.length} exported pages withheld from indexing`)
|
|
}
|