Press: batch file reads

.changeset/little-areas-brush.md

@@ -0,0 +1,5 @@
+---
+'fumapress': patch
+---
+
+batch file reads for perf

packages/press/src/cli/commands.ts

@@ -6,7 +6,7 @@ import { baseDir } from '@/constants';
 
 export async function runStart({
   host,
-  dirs = [''],
+  dirs,
   port: defaultPort = '8080',
 }: {
   dirs?: string[];
@@ -22,6 +22,9 @@ export async function runStart({
   }
   process.env.PORT = String(port);
   process.env.ROOT_DIR = process.cwd();
+  if (!dirs || dirs.length === 0) {
+    dirs = [''];
+  }
   process.env.DEFAULT_RPOJECT_DIR = JSON.stringify(dirs.map((v) => path.resolve(v)));
   process.env.HOT_RELOAD = '1';
   process.chdir(baseDir);

packages/press/src/lib/env.ts

@@ -1,6 +1,14 @@
+import { z } from 'zod/mini';
+
+const stringArraySchema = z.array(z.string());
+
 export function getDefaultProjectDirectories(): string[] {
   const v = process.env.DEFAULT_RPOJECT_DIR;
-  if (!v) return [];
 
-  return JSON.parse(v) as string[];
+  if (v) {
+    const parsed = stringArraySchema.safeParse(JSON.parse(v));
+    if (parsed.success) return parsed.data;
+  }
+
+  return [process.env.ROOT_DIR ?? process.cwd()];
 }

packages/press/src/lib/revalidable.ts

@@ -1,4 +1,5 @@
 import { hash } from './hash';
+import { cache } from 'react';
 
 export interface RevalidableConfig<Args extends unknown[], O> {
   staleTime?: number;
@@ -24,7 +25,7 @@ export function revalidable<Args extends unknown[], O>(
       revalidating: boolean;
     }
   >();
-  const out: WithRevalidate<Args, O> = function (...args: Args) {
+  const out = cache((...args: Args) => {
     const key = cacheKey(...args);
     const cache = cacheMap.get(key);
 
@@ -60,9 +61,9 @@ export function revalidable<Args extends unknown[], O>(
     }
 
     return cache.lastResult;
-  };
+  }) as WithRevalidate<Args, O>;
 
-  out.revalidate = (keepStale = true) => {
+  out.revalidate = function revalidate(keepStale = true) {
     if (keepStale) {
       for (const value of cacheMap.values()) {
         value.lastValidated = 0;

packages/press/src/lib/source/config.ts

@@ -7,7 +7,10 @@ export type NormalizedProjectConfig = Pick<Required<ProjectConfig>, 'include' |
 
 export function normalizeProjects(projects?: ProjectConfig[]): NormalizedProjectConfig[] {
   const baseDir = process.env.ROOT_DIR ?? process.cwd();
-  projects ??= getDefaultProjectDirectories().map((dir) => ({ dir, name: path.basename(dir) }));
+
+  if (!projects || projects.length === 0) {
+    projects = getDefaultProjectDirectories().map((dir) => ({ dir, name: path.basename(dir) }));
+  }
 
   return projects.map((project) => ({
     ...project,

packages/press/src/lib/source/storage.ts

@@ -32,8 +32,84 @@ export interface RawMeta {
   } & Record<string, unknown>;
 }
 
+type BuildFileOutput = RawPage | RawMeta | undefined;
+
+const CHUNK_SIZE = 100;
 export const filesCache = new Map<string, RawPage | RawMeta>();
 
+export async function getPages(config: ContentConfig): Promise<{
+  pages: RawPage[];
+  metas: RawMeta[];
+}> {
+  const projects = normalizeProjects(config.projects);
+  return (await Promise.all(projects.map(buildProject))).reduce(
+    (a, b) => {
+      a.metas.push(...b.metas);
+      a.pages.push(...b.pages);
+      return a;
+    },
+    { pages: [], metas: [] },
+  );
+}
+
+async function buildFile(project: NormalizedProjectConfig, file: string): Promise<BuildFileOutput> {
+  const absolutePath = path.resolve(project.dir, file);
+  const cached = filesCache.get(absolutePath);
+  if (cached) return cached;
+
+  const ext = path.extname(file);
+
+  try {
+    let out: BuildFileOutput;
+    switch (ext) {
+      case '.json':
+        out = await json(project, absolutePath, file);
+        break;
+      case '.mdx':
+      case '.md':
+        out = await md(project, absolutePath, file);
+        break;
+    }
+
+    if (out === undefined) filesCache.delete(absolutePath);
+    else filesCache.set(absolutePath, out);
+
+    return out;
+  } catch (e) {
+    console.error(`error when parsing ${file}`, e);
+    filesCache.delete(absolutePath);
+  }
+}
+
+async function buildProject(project: NormalizedProjectConfig) {
+  const files = await glob(project.include, {
+    cwd: project.dir,
+  });
+  const chunks: Promise<BuildFileOutput[]>[] = [];
+
+  for (let i = 0; i < files.length; i += CHUNK_SIZE) {
+    const promises: Promise<BuildFileOutput>[] = [];
+    const L = Math.min(files.length, i + CHUNK_SIZE);
+
+    for (let j = i; j < L; j++) {
+      promises.push(buildFile(project, files[j]!));
+    }
+
+    chunks.push(Promise.all(promises));
+  }
+
+  const pages: RawPage[] = [];
+  const metas: RawMeta[] = [];
+  for await (const chunk of chunks) {
+    for (const item of chunk) {
+      if (!item) continue;
+      if (item.type === 'page') pages.push(item);
+      else if (item.type === 'meta') metas.push(item);
+    }
+  }
+  return { pages, metas };
+}
+
 async function md(
   project: NormalizedProjectConfig,
   absolutePath: string,
@@ -77,53 +153,3 @@ async function json(
     data: result.data,
   };
 }
-
-export async function getPages(config: ContentConfig): Promise<{
-  pages: RawPage[];
-  metas: RawMeta[];
-}> {
-  async function project(project: NormalizedProjectConfig) {
-    const files = await glob(project.include, {
-      cwd: project.dir,
-    });
-
-    return await Promise.all(
-      files.map(async (file) => {
-        const absolutePath = path.resolve(project.dir, file);
-        const cached = filesCache.get(absolutePath);
-        if (cached) return cached;
-
-        const ext = path.extname(file);
-
-        try {
-          let out: RawMeta | RawPage | undefined;
-          switch (ext) {
-            case '.json':
-              out = await json(project, absolutePath, file);
-              break;
-            case '.mdx':
-            case '.md':
-              out = await md(project, absolutePath, file);
-              break;
-          }
-
-          if (out === undefined) filesCache.delete(absolutePath);
-          else filesCache.set(absolutePath, out);
-
-          return out;
-        } catch (e) {
-          console.error(`error when parsing ${file}`, e);
-          filesCache.delete(absolutePath);
-        }
-      }),
-    );
-  }
-
-  const projects = normalizeProjects(config.projects);
-  const all = (await Promise.all(projects.map(project))).flat();
-
-  return {
-    metas: all.filter((item) => item?.type === 'meta'),
-    pages: all.filter((item) => item?.type === 'page'),
-  };
-}

packages/press/src/pages/[...slugs].tsx

@@ -4,7 +4,7 @@ import { getPageImage, getSource, type SourcePage } from '@/lib/source';
 import { layoutConfig } from '@/layouts/config';
 import { getConfigRuntime } from '@/config/load-runtime';
 import { Card, Cards } from 'fumadocs-ui/components/card';
-import { CompileResult, createMarkdownCompiler, plugin } from '@/lib/md';
+import { type CompileResult, createMarkdownCompiler, plugin } from '@/lib/md';
 import { remarkHeading } from 'fumadocs-core/mdx-plugins/remark-heading';
 import { remarkGfm } from 'fumadocs-core/mdx-plugins/remark-gfm';
 import { remarkCodeTab } from 'fumadocs-core/mdx-plugins/remark-code-tab';
@@ -18,7 +18,7 @@ import rehypeKatex from 'rehype-katex';
 import { CodeBlock, Pre } from 'fumadocs-ui/components/codeblock';
 import { remarkMdxMermaid } from 'fumadocs-core/mdx-plugins/remark-mdx-mermaid';
 import { Mermaid } from '@/components/mermaid';
-import type { ComponentProps } from 'react';
+import type { ComponentProps, ReactNode } from 'react';
 import { Image } from '@/components/image';
 
 const compiler = createMarkdownCompiler({
@@ -71,7 +71,6 @@ interface MdPresetComponents {
 export default async function DocPage({ slugs }: PageProps<'/docs/[...slugs]'>) {
   const config = await getConfigRuntime();
   const source = await getSource(config);
-  const layout = layoutConfig(config);
   const page = source.getPage(slugs);
   const mdPreset = config.layout?.presets?.md ?? 'docs';
   let mdPresetComponents: MdPresetComponents;
@@ -88,12 +87,7 @@ export default async function DocPage({ slugs }: PageProps<'/docs/[...slugs]'>)
     };
   }
 
-  const { DocsLayout } = mdPresetComponents.layout;
-  return (
-    <DocsLayout {...await layout.docs()}>
-      <MdContent slugs={slugs} page={page} components={mdPresetComponents} />
-    </DocsLayout>
-  );
+  return <MdContent slugs={slugs} page={page} components={mdPresetComponents} />;
 }
 
 async function MdContent({
@@ -107,11 +101,17 @@ async function MdContent({
 }) {
   const config = await getConfigRuntime();
   const source = await getSource(config);
+  const layout = layoutConfig(config);
+  const { DocsLayout } = components.layout;
   const { DocsBody, DocsTitle, DocsPage, DocsDescription } = components.page;
 
+  async function renderContainer(children: ReactNode) {
+    return <DocsLayout {...await layout.docs()}>{children}</DocsLayout>;
+  }
+
   if (!page) {
     if (slugs.length === 0) {
-      return (
+      return renderContainer(
         <DocsPage>
           <DocsTitle>Home</DocsTitle>
           <DocsDescription>
@@ -126,15 +126,15 @@ async function MdContent({
               );
             })}
           </Cards>
-        </DocsPage>
+        </DocsPage>,
       );
     }
 
-    return (
+    return renderContainer(
       <DocsPage>
         <DocsTitle>Not Found</DocsTitle>
         <DocsDescription>The page you are looking for does not exist.</DocsDescription>
-      </DocsPage>
+      </DocsPage>,
     );
   }
 
@@ -146,7 +146,7 @@ async function MdContent({
       value: page.data.content,
     });
   } catch (e) {
-    return (
+    return renderContainer(
       <DocsPage>
         <DocsTitle>Failed to Compile</DocsTitle>
         {e instanceof Error ? (
@@ -161,7 +161,7 @@ async function MdContent({
         ) : (
           <DocsDescription>{String(e)}</DocsDescription>
         )}
-      </DocsPage>
+      </DocsPage>,
     );
   }
 
@@ -180,7 +180,7 @@ async function MdContent({
     }),
   );
 
-  return (
+  return renderContainer(
     <DocsPage toc={toc}>
       <title>{page.data.title}</title>
       <meta property="og:image" content={getPageImage(slugs).url} />
@@ -190,7 +190,7 @@ async function MdContent({
       <DocsBody>
         <Fragment key={page.absolutePath}>{compiled.render(mdxComponents)}</Fragment>
       </DocsBody>
-    </DocsPage>
+    </DocsPage>,
   );
 }