mirror of
https://github.com/hanzo-apps/hanzo.ai.git
synced 2026-10-28 07:32:52 +00:00
zeekay
1c71e7e200
noindex, the font preload, the canonical and the markdown link were each a process that read the pages from disk and wrote back what it changed: 8.6 s of every deploy (run 39431048221). scripts/export.mjs reads every page at once, runs the same passes in the same order over memory, and writes each changed page once; wellknown and skill follow as before. On the export rebuilt from the live pages the output is byte for byte what the four processes wrote, .md files, llms.txt and skill.md included, in 4.8 s where they took 12.7 on the same box.
74 lines
2.5 KiB
JavaScript
74 lines
2.5 KiB
JavaScript
/**
|
|
* The second half of `pnpm build`: every pass over the exported pages, with
|
|
* each page read once and written once.
|
|
*
|
|
* Four passes edit the pages Next wrote — noindex, the font preload, the
|
|
* canonical and the markdown link — and each used to be a process of its own
|
|
* that read the pages from disk and wrote back the ones it changed, so most of
|
|
* the export's 300 MB of HTML was read four times and written three: 8.6 s of
|
|
* every deploy (run 39431048221). The passes are the same and run in the same
|
|
* order; they read and write the store below instead of the disk, and it writes
|
|
* each changed page once at the end. A pass that throws fails the build before
|
|
* anything is written.
|
|
*
|
|
* Then the two that write files of their own, wellknown and skill, as before.
|
|
*/
|
|
import { readFile, writeFile } from 'node:fs/promises'
|
|
import { readdirSync } from 'node:fs'
|
|
import { join, resolve } from 'node:path'
|
|
import noindex from './noindex.mjs'
|
|
import preloadFont from './preload-font.mjs'
|
|
import canonical from './canonical.mjs'
|
|
import llms from './llms.mjs'
|
|
|
|
const OUT = join(import.meta.dirname, '..', 'out')
|
|
|
|
/** Every page of the export, read at once: the passes then read memory. */
|
|
const files = []
|
|
const walk = (dir) => {
|
|
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
|
const p = join(dir, e.name)
|
|
if (e.isDirectory()) walk(p)
|
|
else if (e.name.endsWith('.html')) files.push(p)
|
|
}
|
|
}
|
|
walk(OUT)
|
|
|
|
/** At most this many files open at once. */
|
|
const WIDTH = 64
|
|
const each = async (list, fn) => {
|
|
let next = 0
|
|
await Promise.all(
|
|
Array.from({ length: Math.min(WIDTH, list.length) }, async () => {
|
|
while (next < list.length) await fn(list[next++])
|
|
}),
|
|
)
|
|
}
|
|
|
|
const held = new Map()
|
|
await each(files, async (file) => held.set(file, await readFile(file, 'utf8')))
|
|
|
|
// Keyed by the absolute path: a pass may name a page relative to the working
|
|
// directory (llms does) or to its own file.
|
|
const changed = new Set()
|
|
const store = {
|
|
read: (file) => {
|
|
const html = held.get(resolve(file))
|
|
if (html === undefined) throw new Error(`${file} is not a page of the export`)
|
|
return html
|
|
},
|
|
write: (file, html) => {
|
|
const key = resolve(file)
|
|
if (!held.has(key)) throw new Error(`${file} is not a page of the export`)
|
|
held.set(key, html)
|
|
changed.add(key)
|
|
},
|
|
}
|
|
|
|
for (const pass of [noindex, preloadFont, canonical, llms]) pass(store)
|
|
|
|
await each([...changed], (file) => writeFile(file, held.get(file)))
|
|
|
|
await import('./wellknown.mjs')
|
|
await import('./skill.mjs')
|