MDX Async mode: read file content on load (#2394)

* MDX: read file content on load

* Use path.resolve in .source/index.ts

* Add absolutePath to force nft file tracing

* improve performance

---------

Co-authored-by: Fuma Nama <[email protected]>

.changeset/thin-moose-follow.md

@@ -0,0 +1,5 @@
+---
+'fumadocs-mdx': patch
+---
+
+MDX Async mode: read file content on load

packages/mdx/src/next/map/file-cache.ts

@@ -1,21 +1,32 @@
 import { LRUCache } from 'lru-cache';
+import fs from 'node:fs/promises';
+import path from 'node:path';
 
-const map = new LRUCache<string, string | object>({
-  max: 200,
+const map = new LRUCache<string, Promise<string>>({
+  max: 100,
 });
 
-export const fileCache = {
-  read<Data>(namespace: string, path: string) {
-    return map.get(`${namespace}.${path}`) as Data;
-  },
-  write(namespace: string, path: string, data: unknown) {
-    map.set(`${namespace}.${path}`, data as object);
-  },
-  removeCache(path: string) {
-    for (const key of map.keys()) {
-      const keyPath = key.slice(key.indexOf('.') + 1);
+/**
+ * make file paths relative to cwd
+ */
+function toFullPath(file: string) {
+  if (path.isAbsolute(file)) {
+    return path.relative(process.cwd(), file);
+  }
 
-      if (keyPath === path) map.delete(key);
-    }
-  },
-};
+  return file;
+}
+
+export async function readFileWithCache(file: string): Promise<string> {
+  const fullPath = toFullPath(file);
+  const cached = map.get(fullPath);
+  if (cached) return cached;
+
+  const read = fs.readFile(fullPath).then((s) => s.toString());
+  map.set(fullPath, read);
+  return read;
+}
+
+export function removeFileCache(file: string) {
+  map.delete(toFullPath(file));
+}

packages/mdx/src/next/map/generate.ts

@@ -1,11 +1,10 @@
 import * as path from 'node:path';
-import * as fs from 'node:fs/promises';
+import { createHash } from 'node:crypto';
 import { glob } from 'tinyglobby';
 import type { LoadedConfig } from '@/loaders/config';
 import type { DocCollection, MetaCollection } from '@/config';
 import { validate } from '@/utils/validation';
-import { fileCache } from '@/next/map/file-cache';
-import type { AsyncRuntimeFile } from '@/runtime/next/types';
+import { readFileWithCache } from '@/next/map/file-cache';
 import { load } from 'js-yaml';
 import { getGitTimestamp } from '@/utils/git-timestamp';
 import { fumaMatter } from '@/utils/fuma-matter';
@@ -17,13 +16,6 @@ import {
 import { getGlobPatterns, isFileSupported } from '@/utils/collections';
 import type { FileInfo } from '@/runtime/shared';
 
-async function readFileWithCache(file: string): Promise<string> {
-  const cached = fileCache.read<string>('read-file', file);
-  if (cached) return cached;
-
-  return (await fs.readFile(file)).toString();
-}
-
 export async function generateJS(
   configPath: string,
   config: LoadedConfig,
@@ -104,15 +96,19 @@ export async function generateJS(
           names: ['_runtimeAsync', 'buildConfig'],
         }),
         'const _sourceConfig = buildConfig(_source)',
+        getImportCode({
+          type: 'default',
+          name: 'path',
+          specifier: 'node:path',
+        }),
       );
 
       asyncInit = true;
     }
 
     const entries = files.map(async (file) => {
-      const parsed = fumaMatter(
-        await readFileWithCache(file.fullPath).catch(() => ''),
-      );
+      const content = await readFileWithCache(file.fullPath).catch(() => '');
+      const parsed = fumaMatter(content);
       let data = parsed.data;
 
       if (collection.schema) {
@@ -129,12 +125,16 @@ export async function generateJS(
         lastModified = await getGitTimestamp(file.fullPath);
       }
 
-      return JSON.stringify({
-        info: file,
-        lastModified,
-        data: data as Record<string, unknown>,
-        content: { body: parsed.content, matter: parsed.matter },
-      } satisfies AsyncRuntimeFile);
+      const hash = createHash('md5').update(content).digest('hex');
+      const infoStr: string[] = [];
+      for (const [k, v] of Object.entries({ ...file, hash })) {
+        infoStr.push(`${k}: ${JSON.stringify(v)}`);
+      }
+      infoStr.push(
+        `absolutePath: path.resolve(${JSON.stringify(file.fullPath)})`,
+      );
+
+      return `{ info: { ${infoStr.join(', ')} }, lastModified: ${JSON.stringify(lastModified)}, data: ${JSON.stringify(data)} }`;
     });
 
     return Promise.all(entries);

packages/mdx/src/next/map/index.ts

@@ -2,7 +2,7 @@ import * as path from 'node:path';
 import * as fs from 'node:fs/promises';
 import { getConfigHash, loadConfig } from '@/loaders/config/load';
 import { generateJS } from '@/next/map/generate';
-import { fileCache } from '@/next/map/file-cache';
+import { removeFileCache } from '@/next/map/file-cache';
 import { ValidationError } from '@/utils/validation';
 
 /**
@@ -66,7 +66,7 @@ export async function start(
           config = await loadConfig(configPath, outDir, configHash, true);
         }
 
-        if (event === 'change') fileCache.removeCache(absolutePath);
+        if (event === 'change') removeFileCache(absolutePath);
 
         await updateMapFile();
       };

packages/mdx/src/runtime/next/async.ts

@@ -7,6 +7,8 @@ import { executeMdx } from '@fumadocs/mdx-remote/client';
 import { pathToFileURL } from 'node:url';
 import { type DocData, missingProcessedMarkdown } from '@/runtime/shared';
 import type { DocCollection } from '@/config';
+import { readFileWithCache } from '@/next/map/file-cache';
+import { fumaMatter } from '@/utils/fuma-matter';
 
 function getDocCollection(config: LoadedConfig, collection: string) {
   const col = config.collections.get(collection);
@@ -28,13 +30,15 @@ export const _runtimeAsync: RuntimeAsync = {
     const collection = getDocCollection(config, collectionName);
     const initMdxOptions = getOptions(config, collection);
 
-    return files.map(({ info, data, content, lastModified }) => {
+    return files.map(({ info, data, lastModified }) => {
       let cachedResult: CompiledMDXProperties | undefined;
 
       async function compileAndLoad() {
         if (cachedResult) return cachedResult;
         const mdxOptions = await initMdxOptions;
-        const compiled = await buildMDX(collectionName, content.body, {
+        const raw = await readFileWithCache(info.fullPath);
+        const { content } = fumaMatter(raw);
+        const compiled = await buildMDX(collectionName, content, {
           ...mdxOptions,
           development: false,
           frontmatter: data,
@@ -56,7 +60,7 @@ export const _runtimeAsync: RuntimeAsync = {
         info,
         async getText(type) {
           if (type === 'raw') {
-            return `${content.matter}${content.body}`;
+            return readFileWithCache(info.fullPath);
           }
 
           const out = await compileAndLoad();

packages/mdx/src/runtime/next/types.ts

@@ -15,9 +15,8 @@ export interface RuntimeFile {
 }
 
 export interface AsyncRuntimeFile {
-  info: FileInfo;
+  info: FileInfo & { hash?: string; absolutePath?: string };
   data: Record<string, unknown>;
-  content: { matter: string; body: string };
   lastModified?: Date;
 }
 

packages/mdx/test/fixtures/index-async.output.js

@@ -1,6 +1,7 @@
 // @ts-nocheck -- skip type checking
 import { _runtimeAsync, buildConfig } from "fumadocs-mdx/runtime/async"
 const _sourceConfig = buildConfig(_source)
+import path from "node:path"
 import { _runtime } from "fumadocs-mdx"
 import * as _source from "./config"
-export const docs = _runtimeAsync.doc<typeof _source.docs>([{"info":{"path":"index.mdx","fullPath":"$cwd/packages/mdx/test/fixtures/generate-index/index.mdx"},"data":{},"content":{"body":"# Hello World\n","matter":""}}, {"info":{"path":"folder/test.mdx","fullPath":"$cwd/packages/mdx/test/fixtures/generate-index/folder/test.mdx"},"data":{},"content":{"body":"","matter":""}}], "docs", _sourceConfig)
+export const docs = _runtimeAsync.doc<typeof _source.docs>([{ info: { path: "index.mdx", fullPath: "packages/mdx/test/fixtures/generate-index/index.mdx", hash: "b12f02f44f5ed3318104c095c455e5ee", absolutePath: path.resolve("packages/mdx/test/fixtures/generate-index/index.mdx") }, lastModified: undefined, data: {} }, { info: { path: "folder/test.mdx", fullPath: "packages/mdx/test/fixtures/generate-index/folder/test.mdx", hash: "d41d8cd98f00b204e9800998ecf8427e", absolutePath: path.resolve("packages/mdx/test/fixtures/generate-index/folder/test.mdx") }, lastModified: undefined, data: {} }], "docs", _sourceConfig)

packages/mdx/test/fixtures/index-sync.output.js

@@ -3,4 +3,4 @@ import * as docs_1 from "./generate-index/folder/test.mdx?collection=docs&hash=h
 import * as docs_0 from "./generate-index/index.mdx?collection=docs&hash=hash"
 import { _runtime } from "fumadocs-mdx"
 import * as _source from "./config"
-export const docs = _runtime.doc<typeof _source.docs>([{ info: {"path":"index.mdx","fullPath":"$cwd/packages/mdx/test/fixtures/generate-index/index.mdx"}, data: docs_0 }, { info: {"path":"folder/test.mdx","fullPath":"$cwd/packages/mdx/test/fixtures/generate-index/folder/test.mdx"}, data: docs_1 }]);
+export const docs = _runtime.doc<typeof _source.docs>([{ info: {"path":"index.mdx","fullPath":"packages/mdx/test/fixtures/generate-index/index.mdx"}, data: docs_0 }, { info: {"path":"folder/test.mdx","fullPath":"packages/mdx/test/fixtures/generate-index/folder/test.mdx"}, data: docs_1 }]);

packages/mdx/test/index.test.ts

@@ -38,20 +38,23 @@ test('format errors', async () => {
   }
 });
 
-const dir = path.dirname(fileURLToPath(import.meta.url));
+const baseDir = path.relative(
+  process.cwd(),
+  path.dirname(fileURLToPath(import.meta.url)),
+);
 const cases = [
   {
     name: 'sync',
     collection: defineCollections({
       type: 'doc',
-      dir: path.join(dir, './fixtures/generate-index'),
+      dir: path.join(baseDir, './fixtures/generate-index'),
     }),
   },
   {
     name: 'async',
     collection: defineCollections({
       type: 'doc',
-      dir: path.join(dir, './fixtures/generate-index'),
+      dir: path.join(baseDir, './fixtures/generate-index'),
       async: true,
     }),
   },
@@ -60,7 +63,7 @@ const cases = [
 for (const { name, collection } of cases) {
   test(`generate JS index file: ${name}`, async () => {
     const out = await generateJS(
-      path.join(dir, './fixtures/config.ts'),
+      path.join(baseDir, './fixtures/config.ts'),
       {
         // @ts-expect-error -- test file
         _runtime: {
@@ -69,7 +72,7 @@ for (const { name, collection } of cases) {
         collections: new Map([['docs', collection]]),
       },
       {
-        relativeTo: path.join(dir, './fixtures'),
+        relativeTo: path.join(baseDir, './fixtures'),
       },
       'hash',
     );