Core: Simplify Source API storage

.changeset/kind-geckos-lay.md

@@ -0,0 +1,16 @@
+---
+"fumadocs-core": major
+---
+
+Simplify Source API virtual storage.
+
+why: Improve performance
+
+migrate:
+
+```diff
+- storage.write('path.mdx', { type: 'page', ... })
+- storage.readPage('page')
++ storage.write('path.mdx', 'page', { ... })
++ storage.read('page', 'page')
+```

packages/core/src/source/file-system.ts

@@ -1,4 +1,3 @@
-import type { MetaData, PageData } from './types';
 import {
   parseFilePath,
   parseFolderPath,
@@ -6,27 +5,16 @@ import {
   type FileInfo,
 } from './path';
 
-export interface Meta<Data extends MetaData = MetaData> {
-  type: 'meta';
+export interface File {
+  path: string;
   file: FileInfo;
-  data: Data;
+  format: 'meta' | 'page';
+  data: Record<string, unknown>;
 }
 
-export interface Page<Data extends PageData = PageData> {
-  type: 'page';
+export interface Folder {
   file: FileInfo;
-  slugs: string[];
-  url: string;
-  data: Data;
-}
-
-export interface Folder<
-  MD extends MetaData = MetaData,
-  PD extends PageData = PageData,
-> {
-  type: 'folder';
-  file: FileInfo;
-  children: (Meta<MD> | Page<PD> | Folder<MD, PD>)[];
+  children: (File | Folder)[];
 }
 
 /**
@@ -35,10 +23,9 @@ export interface Folder<
  * Some source providers may not provide the full file structure, this will cause inconsistent outputs for page builder and other transformers
  */
 export class Storage {
-  files = new Map<string, Page | Meta>();
+  files = new Map<string, File>();
   folders = new Map<string, Folder>();
   private rootFolder: Folder = {
-    type: 'folder',
     file: parseFolderPath(''),
     children: [],
   };
@@ -50,28 +37,8 @@ export class Storage {
   /**
    * @param path - flattened path
    */
-  private read(path: string, type: string): Page | Meta | undefined {
-    return this.files.get(`${path}.${type}`);
-  }
-
-  /**
-   * @param path - flattened path
-   */
-  readPage(path: string): Page | undefined {
-    const result = this.read(path, 'page');
-    if (result?.type !== 'page') return;
-
-    return result;
-  }
-
-  /**
-   * @param path - flattened path
-   */
-  readMeta(path: string): Meta | undefined {
-    const result = this.read(path, 'meta');
-    if (result?.type !== 'meta') return;
-
-    return result;
+  read(path: string, format: string): File | undefined {
+    return this.files.get(`${path}.${format}`);
   }
 
   readDir(path: string): Folder | undefined {
@@ -82,18 +49,24 @@ export class Storage {
     return this.rootFolder;
   }
 
-  write(path: string, file: Omit<Page, 'file'> | Omit<Meta, 'file'>): void {
-    const node: Page | Meta = {
+  write(
+    path: string,
+    format: 'meta' | 'page',
+    data: Record<string, unknown>,
+  ): void {
+    const node: File = {
+      path,
+      format,
       file: parseFilePath(path),
-      ...file,
+      data,
     };
 
     this.makeDir(node.file.dirname);
     this.readDir(node.file.dirname)?.children.push(node);
-    this.files.set(`${node.file.flattenedPath}.${node.type}`, node);
+    this.files.set(`${node.file.flattenedPath}.${node.format}`, node);
   }
 
-  list(): (Page | Meta)[] {
+  list(): File[] {
     return [...this.files.values()];
   }
 
@@ -105,7 +78,6 @@ export class Storage {
       if (this.folders.has(segment)) continue;
 
       const folder: Folder = {
-        type: 'folder',
         file: parseFolderPath(segment),
         children: [],
       };

packages/core/src/source/index.ts

@@ -2,6 +2,6 @@ export * from './page-tree-builder';
 export * from './loader';
 export * as FileSystem from './file-system';
 
-export type { LoadOptions, Transformer, VirtualFile } from './load';
+export type { LoadOptions, Transformer, VirtualFile } from './load-files';
 export type * from './types';
 export { type FileInfo, parseFilePath, parseFolderPath } from './path';

packages/core/src/source/load-files.ts

@@ -1,6 +1,4 @@
-import * as path from 'node:path';
-import type { MetaData, PageData } from './types';
-import { parseFilePath, type FileInfo } from './path';
+import { parseFilePath, type FileInfo, normalizePath } from './path';
 import { Storage } from './file-system';
 
 export interface LoadOptions {
@@ -12,6 +10,11 @@ export interface LoadOptions {
 }
 
 export interface VirtualFile {
+  /**
+   * Relative path
+   *
+   * @example `docs/page.mdx`
+   */
   path: string;
   type: 'page' | 'meta';
   data: unknown;
@@ -46,32 +49,28 @@ export function load(options: LoadOptions): LoadResult {
 
 function buildStorage(options: LoadOptions): Storage {
   const storage = new Storage();
+  const rootDir = normalizePath(options.rootDir);
 
   for (const file of options.files) {
-    const relativePath = path.join(
-      path.relative(options.rootDir, path.join('./', path.dirname(file.path))),
-      path.basename(file.path),
-    );
+    const normalizedPath = normalizePath(file.path);
+    if (!normalizedPath.startsWith(rootDir)) continue;
 
-    if (relativePath.startsWith('..') || path.isAbsolute(relativePath))
-      continue;
+    const relativePath = normalizePath(normalizedPath.slice(rootDir.length));
 
     if (file.type === 'page') {
       const parsedPath = parseFilePath(relativePath);
       const slugs = options.getSlugs(parsedPath);
 
-      storage.write(relativePath, {
+      storage.write(relativePath, file.type, {
         slugs,
         url: options.getUrl(slugs, parsedPath.locale),
-        type: file.type,
-        data: file.data as PageData,
+        data: file.data,
       });
     }
 
     if (file.type === 'meta') {
-      storage.write(relativePath, {
-        type: file.type,
-        data: file.data as MetaData,
+      storage.write(relativePath, file.type, {
+        data: file.data,
       });
     }
   }

packages/core/src/source/loader.ts

@@ -4,12 +4,12 @@ import {
   type LoadOptions,
   type VirtualFile,
   type Transformer,
-} from './load';
-import type { MetaData, PageData } from './types';
+} from './load-files';
+import type { FileData, MetaData, PageData } from './types';
 import type { CreatePageTreeBuilderOptions } from './page-tree-builder';
 import { createPageTreeBuilder } from './page-tree-builder';
 import { joinPaths, splitPath, type FileInfo } from './path';
-import type { Meta, Page, Storage } from './file-system';
+import type { File, Storage } from './file-system';
 
 export interface LoaderConfig {
   source: SourceConfig;
@@ -47,15 +47,24 @@ export interface Source<Config extends SourceConfig> {
   files: VirtualFile[] | ((rootDir: string) => VirtualFile[]);
 }
 
+export interface Page<Data = PageData> {
+  file: FileInfo;
+  slugs: string[];
+  url: string;
+  data: Data;
+}
+
+export interface Meta<Data = MetaData> {
+  file: FileInfo;
+  data: Data;
+}
+
 export interface LoaderOutput<Config extends LoaderConfig> {
   pageTree: Config['i18n'] extends true
     ? Record<string, PageTree.Root>
     : PageTree.Root;
 
-  files: (
-    | Meta<Config['source']['metaData']>
-    | Page<Config['source']['pageData']>
-  )[];
+  files: File[];
 
   /**
    * Get list of pages from language, empty if language hasn't specified
@@ -80,14 +89,14 @@ function groupByLanguages(
 
   langMap.set('', []);
   for (const node of storage.list()) {
-    if (node.type !== 'page' || node.file.locale) continue;
-    langMap.get('')?.push(node);
+    if (node.format !== 'page' || node.file.locale) continue;
+    langMap.get('')?.push(fileToPage(node));
 
     for (const lang of languages) {
       const list = langMap.get(lang) ?? [];
       const page =
-        storage.readPage(`${node.file.flattenedPath}.${lang}`) ?? node;
-      list.push(page);
+        storage.read(`${node.file.flattenedPath}.${lang}`, 'page') ?? node;
+      list.push(fileToPage(page));
       langMap.set(lang, list);
     }
   }
@@ -166,3 +175,14 @@ function createOutput({
     },
   };
 }
+
+function fileToPage<Data>(file: File): Page<Data> {
+  const data = file.data as FileData['file'];
+
+  return {
+    file: file.file,
+    url: data.url,
+    slugs: data.slugs,
+    data: data.data as Data,
+  };
+}

packages/core/src/source/page-tree-builder.ts

@@ -1,7 +1,8 @@
 import type { ReactElement } from 'react';
 import type * as PageTree from '../server/page-tree';
-import type { Folder, Meta, Page, Storage } from './file-system';
+import type { File, Folder, Storage } from './file-system';
 import { joinPaths } from './path';
+import { type FileData } from './types';
 
 interface PageTreeBuilderContext {
   storage: Storage;
@@ -41,7 +42,7 @@ const extractor = /\.\.\.(?<name>.+)/;
  * @returns Nodes with specified locale in context (sorted)
  */
 function buildAll(
-  nodes: (Folder | Page | Meta)[],
+  nodes: (Folder | File)[],
   ctx: PageTreeBuilderContext,
   skipIndex: boolean,
 ): PageTree.Node[] {
@@ -50,8 +51,9 @@ function buildAll(
   for (const node of [...nodes].sort((a, b) =>
     a.file.name.localeCompare(b.file.name),
   )) {
-    if (node.type === 'page') {
-      if (node.file.locale) continue;
+    if (node.file.locale) continue;
+
+    if ('data' in node && node.format === 'page') {
       const treeNode = buildFileNode(node, ctx);
 
       if (node.file.name === 'index') {
@@ -60,9 +62,7 @@ function buildAll(
       }
 
       output.push(treeNode);
-    }
-
-    if (node.type === 'folder') {
+    } else if ('children' in node) {
       output.push(buildFolderNode(node, false, ctx));
     }
   }
@@ -70,21 +70,6 @@ function buildAll(
   return output;
 }
 
-function getFolderMeta(
-  folder: Folder,
-  ctx: PageTreeBuilderContext,
-): Meta | undefined {
-  let meta = ctx.storage.readMeta(joinPaths([folder.file.path, 'meta']));
-
-  if (ctx.lang) {
-    meta =
-      ctx.storage.readMeta(joinPaths([folder.file.path, `meta.${ctx.lang}`])) ??
-      meta;
-  }
-
-  return meta;
-}
-
 function resolveFolderItem(
   folder: Folder,
   item: string,
@@ -121,13 +106,13 @@ function resolveFolderItem(
     folder.file.path,
     extractResult?.groups?.name ?? item,
   ]);
-  const itemNode = ctx.storage.readDir(path) ?? ctx.storage.readPage(path);
+  const itemNode = ctx.storage.readDir(path) ?? ctx.storage.read(path, 'page');
 
   if (!itemNode) return [];
 
   addedNodePaths.add(itemNode.file.path);
 
-  if (itemNode.type === 'folder') {
+  if ('children' in itemNode) {
     const node = buildFolderNode(itemNode, false, ctx);
 
     return extractResult ? node.children : [node];
@@ -141,22 +126,28 @@ function buildFolderNode(
   defaultIsRoot: boolean,
   ctx: PageTreeBuilderContext,
 ): PageTree.Folder {
-  const indexFile = ctx.storage.readPage(
+  let meta = ctx.storage.read(joinPaths([folder.file.path, 'meta']), 'meta');
+  const indexFile = ctx.storage.read(
     joinPaths([folder.file.flattenedPath, 'index']),
+    'page',
   );
 
+  if (meta) {
+    meta = findLocalizedFile(meta, ctx) ?? meta;
+  }
+
+  const metadata = meta?.data.data as FileData['meta']['data'] | undefined;
   const index = indexFile ? buildFileNode(indexFile, ctx) : undefined;
-  const meta = getFolderMeta(folder, ctx)?.data;
 
   let children: PageTree.Node[];
 
   if (!meta) {
     children = buildAll(folder.children, ctx, !defaultIsRoot);
   } else {
-    const isRoot = meta.root ?? defaultIsRoot;
+    const isRoot = metadata?.root ?? defaultIsRoot;
     const addedNodePaths = new Set<string>();
 
-    const resolved = meta.pages?.flatMap<PageTree.Node | '...'>((item) => {
+    const resolved = metadata?.pages?.flatMap<PageTree.Node | '...'>((item) => {
       return resolveFolderItem(folder, item, ctx, addedNodePaths);
     });
 
@@ -179,30 +170,24 @@ function buildFolderNode(
 
   return removeUndefined({
     type: 'folder',
-    name: meta?.title ?? index?.name ?? pathToName(folder.file.name),
-    icon: ctx.resolveIcon(meta?.icon),
-    root: meta?.root,
-    defaultOpen: meta?.defaultOpen,
+    name: metadata?.title ?? index?.name ?? pathToName(folder.file.name),
+    icon: ctx.resolveIcon(metadata?.icon),
+    root: metadata?.root,
+    defaultOpen: metadata?.defaultOpen,
     index,
     children,
   });
 }
 
-function buildFileNode(page: Page, ctx: PageTreeBuilderContext): PageTree.Item {
-  let localePage = page;
-  if (ctx.lang) {
-    const result = ctx.storage.readPage(
-      `${page.file.flattenedPath}.${ctx.lang}`,
-    );
-
-    if (result) localePage = result;
-  }
+function buildFileNode(file: File, ctx: PageTreeBuilderContext): PageTree.Item {
+  const localized = findLocalizedFile(file, ctx) ?? file;
+  const data = localized.data as FileData['file'];
 
   return removeUndefined({
     type: 'page',
-    name: localePage.data.title,
-    icon: ctx.resolveIcon(localePage.data.icon),
-    url: localePage.url,
+    name: data.data.title,
+    icon: ctx.resolveIcon(data.data.icon),
+    url: data.url,
   });
 }
 
@@ -251,6 +236,18 @@ export function createPageTreeBuilder({
   };
 }
 
+function findLocalizedFile(
+  file: File,
+  ctx: PageTreeBuilderContext,
+): File | undefined {
+  if (!ctx.lang) return file;
+
+  return ctx.storage.read(
+    `${file.file.flattenedPath}.${ctx.lang}`,
+    file.format,
+  );
+}
+
 function pathToName(path: string): string {
   return path.slice(0, 1).toUpperCase() + path.slice(1);
 }

packages/core/src/source/path.ts

@@ -51,6 +51,18 @@ export function parseFolderPath(path: string): FileInfo {
   };
 }
 
+/**
+ * @param path - Relative path
+ * @returns Normalized path, with no trailing/leading slashes
+ * @throws Throws error if path starts with `./` or `../`
+ */
+export function normalizePath(path: string): string {
+  const segments = splitPath(slash(path));
+  if (segments[0] === '.' || segments[0] === '..')
+    throw new Error("It must not start with './' or '../'");
+  return joinPaths(segments);
+}
+
 /**
  * Split path into segments, trailing/leading slashes are removed
  */

packages/core/src/source/types.ts

@@ -1,5 +1,4 @@
-import type { LoaderOutput } from './loader';
-import type { Meta, Page } from './file-system';
+import type { LoaderOutput, Meta, Page } from './loader';
 
 export interface MetaData {
   icon?: string;
@@ -25,3 +24,17 @@ export type InferMetaType<Utils extends LoaderOutput<any>> =
   Utils extends LoaderOutput<infer Config>
     ? Meta<Config['source']['metaData']>
     : never;
+
+/**
+ * @internal
+ */
+export interface FileData {
+  meta: {
+    data: MetaData;
+  };
+  file: {
+    url: string;
+    slugs: string[];
+    data: PageData;
+  };
+}

packages/core/test/source.test.ts

@@ -150,6 +150,13 @@ test('Internationalized Routing', () => {
             title: 'Nested Page Chinese',
           },
         },
+        {
+          type: 'meta',
+          path: '/nested/meta.json',
+          data: {
+            title: 'Nested',
+          },
+        },
         {
           type: 'meta',
           path: '/nested/meta.cn.json',
@@ -171,6 +178,10 @@ test('Internationalized Routing', () => {
         },
         {
           "children": [
+            {
+              "name": "Nested",
+              "type": "page",
+            },
             {
               "name": "Nested Page",
               "type": "page",
@@ -195,6 +206,10 @@ test('Internationalized Routing', () => {
         },
         {
           "children": [
+            {
+              "name": "Nested Chinese",
+              "type": "page",
+            },
             {
               "name": "Nested Page Chinese",
               "type": "page",

packages/core/test/storage.test.ts

@@ -1,37 +1,31 @@
 import { Storage } from '@/source/file-system';
 import { parseFilePath, parseFolderPath } from '@/source/path';
 import { describe, expect, test } from 'vitest';
-import { load } from '@/source/load';
+import { load } from '@/source/load-files';
 
 describe('Virtual Storage', () => {
   const storage = new Storage();
 
   test('Writing', () => {
-    storage.write('test.mdx', {
-      type: 'page',
-      data: { title: 'Hello' },
-      slugs: ['test'],
-      url: '/test',
-    });
-    storage.write('meta.json', { type: 'meta', data: { pages: ['test'] } });
+    storage.write('test.mdx', 'page', {});
+    storage.write('meta.json', 'meta', {});
 
     expect(storage.list().length).toBe(2);
   });
 
   test('Reading', () => {
-    expect(storage.readPage('test')).toBeDefined();
-    expect(storage.readMeta('meta')).toBeDefined();
+    expect(storage.read('test', 'page')).toBeDefined();
+    expect(storage.read('meta', 'meta')).toBeDefined();
   });
 
   test('Nested Directories', () => {
-    storage.write('dir1/dir2/meta.json', {
-      type: 'meta',
-      data: { pages: ['test'] },
+    storage.write('dir1/dir2/meta.json', 'meta', {
+      data: 'Hello',
     });
 
     expect(storage.readDir('dir1')).toBeDefined();
     expect(storage.readDir('dir1/dir2')).toBeDefined();
-    expect(storage.readMeta('dir1/dir2/meta')).toBeDefined();
+    expect(storage.read('dir1/dir2/meta', 'meta')).toBeDefined();
   });
 });
 
@@ -67,23 +61,21 @@ test('Building File Graph', () => {
 
   expect(result.storage.root().children).toEqual([
     expect.objectContaining({
-      type: 'page',
+      format: 'page',
       file: parseFilePath('test.mdx'),
     }),
     expect.objectContaining({
-      type: 'folder',
       file: parseFolderPath('nested'),
       children: [
         expect.objectContaining({
-          type: 'page',
+          format: 'page',
           file: parseFilePath('nested/test.mdx'),
         }),
         expect.objectContaining({
-          type: 'folder',
           file: parseFolderPath('nested/nested'),
           children: [
             expect.objectContaining({
-              type: 'page',
+              format: 'page',
               file: parseFilePath('nested/nested/test.mdx'),
             }),
           ],