MDX: Improve performance

.changeset/ninety-games-sip.md

@@ -0,0 +1,5 @@
+---
+'fumadocs-mdx': patch
+---
+
+Improve performance

apps/docs/content/docs/mdx/async.mdx

@@ -75,4 +75,4 @@ if (page) {
 }
 ```
 
-When using Async Mode, we highly recommend to use 3rd services to implement search, which usually has a better capability to handle massive amount of content to index.
+When using Async Mode, we highly recommend to use 3rd party services to implement search, which usually has a better capability to handle massive amount of content to index.

packages/core/src/utils/use-shiki.tsx

@@ -94,7 +94,8 @@ export function useShiki(
   });
 
   if (typeof window === 'undefined') {
-    return use(highlight(code, shikiOptions));
+    // return promise on server
+    return highlight(code, shikiOptions) as ReactNode;
   }
 
   // on change

packages/mdx/src/config/build.ts

@@ -1,6 +1,6 @@
 import type { GlobalConfig } from '@/config/types';
 import type { ProcessorOptions } from '@mdx-js/mdx';
-import type { LoadedConfig } from '@/utils/load-config';
+import type { LoadedConfig } from '@/utils/config';
 import type {
   DocsCollection,
   DocCollection,

packages/mdx/src/loader-mdx.ts

@@ -2,7 +2,7 @@ import * as path from 'node:path';
 import { parse } from 'node:querystring';
 import grayMatter from 'gray-matter';
 import { type LoaderContext } from 'webpack';
-import { getConfigHash, loadConfigCached } from '@/utils/config-cache';
+import { getConfigHash, loadConfig } from '@/utils/config';
 import { buildMDX } from '@/utils/build-mdx';
 import { formatError } from '@/utils/format-error';
 import { getGitTimestamp } from './utils/git-timestamp';
@@ -54,7 +54,7 @@ export default async function loader(
     hash: configHash = await getConfigHash(_ctx.configPath),
     collection: collectionId,
   } = parseQuery(this.resourceQuery);
-  const config = await loadConfigCached(_ctx.configPath, configHash);
+  const config = await loadConfig(_ctx.configPath, configHash);
 
   let collection =
     collectionId !== undefined
@@ -76,8 +76,7 @@ export default async function loader(
       schema = schema({
         async buildMDX(v, options = mdxOptions) {
           const res = await buildMDX(
-            collectionId ?? 'global',
-            configHash,
+            `${configHash}:${collectionId ?? 'global'}`,
             v,
             options,
           );
@@ -116,8 +115,7 @@ export default async function loader(
     );
 
     const file = await buildMDX(
-      collectionId ?? 'global',
-      configHash,
+      `${configHash}:${collectionId ?? 'global'}`,
       lineOffset + matter.content,
       {
         development: this.mode === 'development',

packages/mdx/src/map/generate.ts

@@ -2,7 +2,7 @@ import * as path from 'node:path';
 import fg from 'fast-glob';
 import { getTypeFromPath } from '@/utils/get-type-from-path';
 import type { FileInfo } from '@/config/types';
-import { type LoadedConfig } from '@/utils/load-config';
+import { type LoadedConfig } from '@/utils/config';
 import type { DocCollection, MetaCollection } from '@/config';
 
 export async function generateJS(

packages/mdx/src/map/index.ts

@@ -1,7 +1,6 @@
 import * as path from 'node:path';
-import * as fs from 'node:fs';
-import { writeFile, rm } from 'node:fs/promises';
-import { getConfigHash, loadConfigCached } from '@/utils/config-cache';
+import * as fs from 'node:fs/promises';
+import { getConfigHash, loadConfig } from '@/utils/config';
 import { generateJS } from '@/map/generate';
 import { readFrontmatter } from '@/utils/read-frontmatter';
 
@@ -16,31 +15,44 @@ export async function start(
   outDir: string,
 ): Promise<void> {
   // delete previous output
-  void rm(path.resolve(outDir, `index.js`), { force: true });
-  void rm(path.resolve(outDir, `index.d.ts`), { force: true });
+  void fs.rm(path.resolve(outDir, `index.js`), { force: true });
+  void fs.rm(path.resolve(outDir, `index.d.ts`), { force: true });
 
+  // init
+  await fs.mkdir(outDir, { recursive: true });
   let configHash = await getConfigHash(configPath);
-  let config = await loadConfigCached(configPath, configHash);
+  let config = await loadConfig(configPath, configHash, true);
   const outPath = path.resolve(outDir, `index.ts`);
 
   const frontmatterCache = new Map<string, unknown>();
   let hookUpdate = false;
 
-  fs.mkdirSync(outDir, { recursive: true });
-  fs.writeFileSync(
-    outPath,
-    await generateJS(configPath, config, outPath, configHash, (file) => {
-      hookUpdate = true;
-      const cached = frontmatterCache.get(file);
-      if (cached) return cached;
+  async function readFrontmatterWithCache(file: string) {
+    hookUpdate = true;
+    const cached = frontmatterCache.get(file);
+    if (cached) return cached;
 
-      return readFrontmatter(file).then((res) => {
-        frontmatterCache.set(file, res);
-        return res;
-      });
-    }),
-  );
-  console.log('[MDX] initialized map file');
+    const res = await readFrontmatter(file);
+    frontmatterCache.set(file, res);
+    return res;
+  }
+
+  async function updateMapFile() {
+    await fs.writeFile(
+      outPath,
+      await generateJS(
+        configPath,
+        config,
+        outPath,
+        configHash,
+        readFrontmatterWithCache,
+      ),
+    );
+  }
+
+  console.time(`[MDX] initialize map file`);
+  await updateMapFile();
+  console.timeEnd(`[MDX] initialize map file`);
 
   if (dev) {
     const { watcher } = await import('@/map/watcher');
@@ -53,28 +65,18 @@ export async function start(
     instance.on('all', (event, file) => {
       if (typeof file !== 'string') return;
 
-      const onUpdate = async (): Promise<void> => {
+      const onUpdate = async () => {
         const isConfigFile = path.resolve(file) === configPath;
 
         if (isConfigFile) {
           configHash = await getConfigHash(configPath);
-          config = await loadConfigCached(configPath, configHash);
+          config = await loadConfig(configPath, configHash, true);
         }
 
         if (isConfigFile || event !== 'change' || hookUpdate) {
           if (event === 'change') frontmatterCache.delete(file);
 
-          await writeFile(
-            outPath,
-            await generateJS(
-              configPath,
-              config,
-              outPath,
-              configHash,
-              readFrontmatter,
-            ),
-          );
-
+          await updateMapFile();
           console.log('[MDX] Updated map file');
         }
       };

packages/mdx/src/map/watcher.ts

@@ -1,5 +1,5 @@
 import { type FSWatcher, watch } from 'chokidar';
-import { type LoadedConfig } from '@/utils/load-config';
+import { type LoadedConfig } from '@/utils/config';
 
 export function watcher(configPath: string, config: LoadedConfig): FSWatcher {
   const deps: string[] = [configPath];

packages/mdx/src/next/create.ts

@@ -1,7 +1,7 @@
 import path from 'node:path';
 import type { NextConfig } from 'next';
 import type { Configuration } from 'webpack';
-import { findConfigFile } from '@/utils/load-config';
+import { findConfigFile } from '@/utils/config';
 import { start } from '@/map';
 import { type Options as MDXLoaderOptions } from '../loader-mdx';
 

packages/mdx/src/postinstall.ts

@@ -1,16 +1,15 @@
 import * as path from 'node:path';
 import * as fs from 'node:fs';
-import { findConfigFile, loadConfig } from '@/utils/load-config';
+import { findConfigFile, getConfigHash, loadConfig } from '@/utils/config';
 import { generateJS } from '@/map/generate';
-import { getConfigHash } from '@/utils/config-cache';
 import { readFrontmatter } from '@/utils/read-frontmatter';
 
 export async function postInstall(
   configPath = findConfigFile(),
 ): Promise<void> {
   const jsOut = path.resolve('.source/index.ts');
-  const config = await loadConfig(configPath);
   const hash = await getConfigHash(configPath);
+  const config = await loadConfig(configPath, hash, true);
 
   fs.mkdirSync(path.dirname(jsOut), { recursive: true });
   fs.writeFileSync(

packages/mdx/src/runtime/async.ts

@@ -1,7 +1,7 @@
 import { type BaseCollectionEntry, type MarkdownProps } from '@/config/types';
 import { createCompiler, type MDXOptions } from '@fumadocs/mdx-remote';
 import * as fs from 'node:fs/promises';
-import type { LoadedConfig } from '@/utils/load-config';
+import type { LoadedConfig } from '@/utils/config';
 import { remarkInclude } from '@/mdx-plugins/remark-include';
 import type { defineCollections } from '@/config/define';
 import {

packages/mdx/src/utils/build-mdx.ts

@@ -4,7 +4,7 @@ import { remarkInclude } from '@/mdx-plugins/remark-include';
 
 type Processor = ReturnType<typeof createProcessor>;
 
-const cache = new Map<string, { processor: Processor; configHash: string }>();
+const cache = new Map<string, Processor>();
 
 export interface MDXOptions extends ProcessorOptions {
   /**
@@ -31,10 +31,6 @@ export interface CompilerOptions {
   addDependency: (file: string) => void;
 }
 
-function cacheKey(group: string, format: string): string {
-  return `${group}:${format}`;
-}
-
 declare module 'vfile' {
   interface DataMap {
     /**
@@ -45,14 +41,12 @@ declare module 'vfile' {
 }
 
 /**
- * @param group - The cache group of MDX content, usually the collection name
- * @param configHash - config hash
+ * @param cacheKey -- key to cache processor
  * @param source - mdx content
  * @param options - MDX options
  */
-export function buildMDX(
-  group: string,
-  configHash: string,
+export async function buildMDX(
+  cacheKey: string,
   source: string,
   options: MDXOptions = {},
 ): Promise<VFile> {
@@ -63,27 +57,22 @@ export function buildMDX(
     format = filePath.endsWith('.mdx') ? 'mdx' : 'md';
   }
   format ??= 'mdx';
-
-  const key = cacheKey(group, format);
+  const key = `${cacheKey}:${format}`;
   let cached = cache.get(key);
 
-  if (cached === undefined || cached.configHash !== configHash) {
-    cached = {
-      processor: createProcessor({
-        outputFormat: 'program',
-        development: process.env.NODE_ENV === 'development',
-        ...rest,
-        remarkPlugins: [remarkInclude, ...(rest.remarkPlugins ?? [])],
-        format,
-      }),
-
-      configHash,
-    };
+  if (cached === undefined) {
+    cached = createProcessor({
+      outputFormat: 'program',
+      development: process.env.NODE_ENV === 'development',
+      ...rest,
+      remarkPlugins: [remarkInclude, ...(rest.remarkPlugins ?? [])],
+      format,
+    });
 
     cache.set(key, cached);
   }
 
-  return cached.processor.process({
+  return cached.process({
     value: source,
     path: filePath,
     data: {

packages/mdx/src/utils/config-cache.ts

@@ -1,38 +0,0 @@
-import { createHash } from 'node:crypto';
-import * as fs from 'node:fs';
-import { loadConfig, type LoadedConfig } from '@/utils/load-config';
-
-const cache = new Map<
-  string,
-  {
-    hash: string;
-    config: Promise<LoadedConfig>;
-  }
->();
-
-export async function loadConfigCached(
-  configPath: string,
-  hash: string,
-): Promise<LoadedConfig> {
-  const cached = cache.get(configPath);
-  if (cached && cached.hash === hash) {
-    return await cached.config;
-  }
-  const config = loadConfig(configPath);
-  cache.set(configPath, { config, hash });
-  return await config;
-}
-
-/**
- * Generate hash based on the content of config
- */
-export async function getConfigHash(configPath: string): Promise<string> {
-  const hash = createHash('md5');
-  const rs = fs.createReadStream(configPath);
-
-  for await (const chunk of rs) {
-    hash.update(chunk as string);
-  }
-
-  return hash.digest('hex');
-}

packages/mdx/src/utils/config.ts

@@ -0,0 +1,110 @@
+import { createHash } from 'node:crypto';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import type {
+  DocCollection,
+  DocsCollection,
+  MetaCollection,
+} from '@/config/define';
+import { type GlobalConfig } from '@/config/types';
+import type { ProcessorOptions } from '@mdx-js/mdx';
+import { pathToFileURL } from 'node:url';
+import { buildConfig } from '@/config/build';
+
+export function findConfigFile(): string {
+  return path.resolve('source.config.ts');
+}
+
+export interface LoadedConfig {
+  collections: Map<string, DocCollection | MetaCollection | DocsCollection>;
+  getDefaultMDXOptions: () => Promise<ProcessorOptions>;
+  global?: GlobalConfig;
+
+  _runtime: {
+    /**
+     * Absolute file path and their associated collections
+     */
+    files: Map<string, string>;
+  };
+}
+
+const cache = new Map<
+  string,
+  {
+    hash: string;
+    config: Promise<LoadedConfig>;
+  }
+>();
+
+async function compileConfig(configPath: string) {
+  const { build } = await import('esbuild');
+
+  const transformed = await build({
+    entryPoints: [{ in: configPath, out: 'source.config' }],
+    bundle: true,
+    outdir: '.source',
+    target: 'node18',
+    write: true,
+    platform: 'node',
+    format: 'esm',
+    packages: 'external',
+    outExtension: {
+      '.js': '.mjs',
+    },
+    allowOverwrite: true,
+  });
+
+  if (transformed.errors.length > 0) {
+    throw new Error('failed to compile configuration file');
+  }
+}
+
+/**
+ * Load config
+ *
+ * @param configPath - config path
+ * @param hash - hash of config content
+ * @param build - By default, it assumes the TypeScript config has been compiled to `.source/source.config.mjs`. Set this `true` to compile the config first.
+ */
+export async function loadConfig(
+  configPath: string,
+  hash: string,
+  build = false,
+): Promise<LoadedConfig> {
+  const cached = cache.get(configPath);
+  if (cached && cached.hash === hash) {
+    return await cached.config;
+  }
+
+  if (build) await compileConfig(configPath);
+
+  const url = pathToFileURL(path.resolve('.source/source.config.mjs'));
+
+  // every call to `loadConfig` should cause the previous cache to be ignored
+  const config = import(`${url.href}?hash=${configPath}`).then((loaded) => {
+    const [err, config] = buildConfig(
+      // every call to `loadConfig` will cause the previous cache to be ignored
+      loaded as Record<string, unknown>,
+    );
+
+    if (err !== null) throw new Error(err);
+    return config;
+  });
+
+  cache.set(configPath, { config, hash });
+  return await config;
+}
+
+/**
+ * Generate hash based on the content of config
+ */
+export async function getConfigHash(configPath: string): Promise<string> {
+  const hash = createHash('md5');
+  const rs = fs.createReadStream(configPath);
+
+  for await (const chunk of rs) {
+    hash.update(chunk as string);
+  }
+
+  return hash.digest('hex');
+}

packages/mdx/src/utils/load-config.ts

@@ -1,62 +0,0 @@
-import * as path from 'node:path';
-import type {
-  DocCollection,
-  DocsCollection,
-  MetaCollection,
-} from '@/config/define';
-import { type GlobalConfig } from '@/config/types';
-import type { ProcessorOptions } from '@mdx-js/mdx';
-import { pathToFileURL } from 'node:url';
-import { buildConfig } from '@/config/build';
-
-export function findConfigFile(): string {
-  return path.resolve('source.config.ts');
-}
-
-export interface LoadedConfig {
-  collections: Map<string, DocCollection | MetaCollection | DocsCollection>;
-  getDefaultMDXOptions: () => Promise<ProcessorOptions>;
-  global?: GlobalConfig;
-
-  _runtime: {
-    /**
-     * Absolute file path and their associated collections
-     */
-    files: Map<string, string>;
-  };
-}
-
-export async function loadConfig(configPath: string): Promise<LoadedConfig> {
-  const { build } = await import('esbuild');
-
-  const url = pathToFileURL(path.resolve('.source/source.config.mjs'));
-  const transformed = await build({
-    entryPoints: [{ in: configPath, out: 'source.config' }],
-    bundle: true,
-    outdir: '.source',
-    target: 'node18',
-    write: true,
-    platform: 'node',
-    format: 'esm',
-    packages: 'external',
-    outExtension: {
-      '.js': '.mjs',
-    },
-    allowOverwrite: true,
-  });
-
-  if (transformed.errors.length > 0) {
-    throw new Error('failed to compile configuration file');
-  }
-
-  // every call to `loadConfig` should cause the previous cache to be ignored
-  const loaded = await import(`${url.href}?hash=${Date.now().toString()}`);
-
-  const [err, config] = buildConfig(
-    // every call to `loadConfig` will cause the previous cache to be ignored
-    loaded as Record<string, unknown>,
-  );
-
-  if (err !== null) throw new Error(err);
-  return config;
-}