MDX: Lazy update index files on Next.js

.changeset/upset-rivers-shine.md

@@ -0,0 +1,5 @@
+---
+'fumadocs-mdx': patch
+---
+
+Lazy update index files on Next.js

packages/mdx/package.json

@@ -92,7 +92,6 @@
     "picocolors": "^1.1.1",
     "picomatch": "^4.0.3",
     "remark-mdx": "^3.1.1",
-    "remark-parse": "^11.0.0",
     "tinyexec": "^1.0.1",
     "tinyglobby": "^0.2.15",
     "unified": "^11.0.5",

packages/mdx/src/config/define.ts

@@ -3,7 +3,7 @@ import type { DefaultMDXOptions } from '@/loaders/mdx/preset';
 import type { ProcessorOptions } from '@mdx-js/mdx';
 import { frontmatterSchema, metaSchema } from '@/config/zod-4';
 import type { PostprocessOptions } from '@/loaders/mdx/remark-postprocess';
-import type { PluginOption } from '@/plugins';
+import type { PluginOption } from '@/core';
 
 export type CollectionSchema<Schema extends StandardSchemaV1, Context> =
   | Schema

packages/mdx/src/config/zod-4.ts

@@ -1,5 +1,8 @@
 import { z } from 'zod';
 
+/**
+ * Zod 4 schema
+ */
 export const metaSchema = z.object({
   title: z.string().optional(),
   pages: z.array(z.string()).optional(),
@@ -9,6 +12,9 @@ export const metaSchema = z.object({
   icon: z.string().optional(),
 });
 
+/**
+ * Zod 4 schema
+ */
 export const frontmatterSchema = z.object({
   title: z.string(),
   description: z.string().optional(),

packages/mdx/src/core.ts

@@ -1,10 +1,55 @@
+import type { ConfigLoader, LoadedConfig } from '@/loaders/config';
 import path from 'node:path';
 import fs from 'node:fs/promises';
-import type { ConfigLoader, LoadedConfig } from '@/loaders/config';
-import type { EmitEntry, Plugin, PluginOption } from '@/plugins';
 import type { FSWatcher } from 'chokidar';
 
+type Awaitable<T> = T | Promise<T>;
+
+export interface EmitEntry {
+  /**
+   * path relative to output directory
+   */
+  path: string;
+  content: string;
+}
+
+export interface PluginContext extends CoreOptions {
+  core: Core;
+}
+
+export interface Plugin {
+  name?: string;
+
+  /**
+   * on config loaded/updated
+   */
+  config?: (
+    this: PluginContext,
+    config: LoadedConfig,
+  ) => Awaitable<void | LoadedConfig>;
+
+  /**
+   * Generate files (e.g. types, index file, or JSON schemas)
+   */
+  emit?: (this: PluginContext) => Awaitable<EmitEntry[]>;
+
+  /**
+   * Configure Fumadocs dev server
+   */
+  configureServer?: (
+    this: PluginContext,
+    server: ServerContext,
+  ) => Awaitable<void>;
+}
+
+export type PluginOption = Awaitable<Plugin | Plugin[] | false>;
+
 export interface ServerContext {
+  /**
+   * the file watcher, by default all content files are watched, along with other files.
+   *
+   * make sure to filter when listening to events
+   */
   watcher?: FSWatcher;
 }
 
@@ -14,23 +59,35 @@ export interface CoreOptions {
   outDir: string;
 }
 
+export interface EmitOptions {
+  /**
+   * filter the plugins to run emit
+   */
+  filterPlugin?: (plugin: Plugin) => boolean;
+}
+
 export function createCore(
-  context: CoreOptions,
+  options: CoreOptions,
   defaultPlugins: PluginOption[] = [],
 ) {
   let config: LoadedConfig;
   let plugins: Plugin[];
 
-  async function write(entry: EmitEntry) {
-    const file = path.join(context.outDir, entry.path);
-    await fs.mkdir(path.dirname(file), { recursive: true });
-    await fs.writeFile(file, entry.content);
-  }
-
   return {
-    _options: context,
-    async init(options: { config: LoadedConfig | Promise<LoadedConfig> }) {
-      config = await options.config;
+    _options: options,
+    getPluginContext(): PluginContext {
+      return {
+        core: this,
+        ...options,
+      };
+    },
+    /**
+     * Convenient cache store, reset when config changes
+     */
+    cache: new Map<string, unknown>(),
+    async init({ config: newConfig }: { config: Awaitable<LoadedConfig> }) {
+      config = await newConfig;
+      this.cache.clear();
       plugins = [];
 
       for await (const option of [
@@ -43,7 +100,7 @@ export function createCore(
       }
 
       for (const plugin of plugins) {
-        const out = await plugin.config?.call(context, config);
+        const out = await plugin.config?.call(this.getPluginContext(), config);
         if (out) config = out;
       }
 
@@ -61,22 +118,31 @@ export function createCore(
     },
     async initServer(server: ServerContext) {
       for (const plugin of plugins) {
-        await plugin.configureServer?.call(context, server);
+        await plugin.configureServer?.call(this.getPluginContext(), server);
       }
     },
-    async emit(): Promise<EmitEntry[]> {
+    async emitAndWrite({
+      filterPlugin = () => true,
+    }: EmitOptions = {}): Promise<void> {
+      const start = performance.now();
+
       const out = await Promise.all(
         plugins.map((plugin) => {
-          return plugin.emit?.call(context) ?? [];
+          if (!filterPlugin(plugin) || !plugin.emit) return [];
+
+          return plugin.emit.call(this.getPluginContext());
         }),
       );
 
-      return out.flat();
-    },
-    async emitAndWrite(): Promise<void> {
-      const entries = await this.emit();
+      await Promise.all(
+        out.flat().map(async (entry) => {
+          const file = path.join(options.outDir, entry.path);
 
-      await Promise.all(entries.map(write));
+          await fs.mkdir(path.dirname(file), { recursive: true });
+          await fs.writeFile(file, entry.content);
+        }),
+      );
+      console.log(`[MDX] generated files in ${performance.now() - start}ms`);
     },
   };
 }

packages/mdx/src/index.ts

@@ -1,3 +1,3 @@
 export type * from './runtime/shared';
-
+export type * from './core';
 export type { ExtractedReference } from '@/loaders/mdx/remark-postprocess';

packages/mdx/src/loaders/config/index.ts

@@ -7,7 +7,7 @@ import type {
 } from '@/config';
 import type { ProcessorOptions } from '@mdx-js/mdx';
 import fs from 'node:fs/promises';
-import { Core } from '@/core';
+import type { Core } from '@/core';
 
 export interface ConfigLoader {
   getConfig: () => LoadedConfig | Promise<LoadedConfig>;

packages/mdx/src/next/index.ts

@@ -11,8 +11,7 @@ import { loadConfig } from '@/loaders/config/load';
 import { removeFileCache } from '@/next/file-cache';
 import { ValidationError } from '@/utils/validation';
 import next from '@/plugins/next';
-import type { EventName } from 'chokidar/handler.js';
-import { createCore } from '@/core';
+import { type Core, createCore } from '@/core';
 
 export interface CreateMDXOptions {
   /**
@@ -97,26 +96,11 @@ async function init(
 ): Promise<void> {
   const core = createNextCore(options);
 
-  async function updateConfig() {
+  async function initOrReload() {
     await core.init({
       config: loadConfig(options.configPath, options.outDir, true),
     });
-  }
-
-  async function emitFiles() {
-    const start = performance.now();
-
-    try {
-      await core.emitAndWrite();
-    } catch (err) {
-      if (err instanceof ValidationError) {
-        console.error(err.toStringFormatted());
-      } else {
-        console.error(err);
-      }
-    }
-
-    console.log(`[MDX] updated files in ${performance.now() - start}ms`);
+    await core.emitAndWrite();
   }
 
   async function devServer() {
@@ -137,38 +121,36 @@ async function init(
       }
     }
 
-    await core.initServer({ watcher });
-
-    async function onUpdate(event: EventName, file: string) {
-      const absolutePath = path.resolve(file);
-      if (event === 'change') removeFileCache(absolutePath);
-
-      if (absolutePath === path.resolve(options.configPath)) {
-        await updateConfig();
-        console.log('[MDX] restarting dev server');
-        await watcher.close();
-        void devServer();
-      }
-
-      await emitFiles();
-    }
-
     watcher.on('ready', () => {
       console.log('[MDX] started dev server');
     });
 
-    watcher.on('all', (event, file) => {
-      void onUpdate(event, file);
+    watcher.on('all', async (event, file) => {
+      const absolutePath = path.resolve(file);
+      if (event === 'change') removeFileCache(absolutePath);
+
+      if (absolutePath === path.resolve(options.configPath)) {
+        // skip plugin listeners
+        watcher.removeAllListeners();
+
+        await watcher.close();
+        await initOrReload();
+        console.log('[MDX] restarting dev server');
+        await devServer();
+      }
     });
 
     process.on('exit', () => {
+      if (watcher.closed) return;
+
       console.log('[MDX] closing dev server');
       void watcher.close();
     });
+
+    await core.initServer({ watcher });
   }
 
-  await updateConfig();
-  await emitFiles();
+  await initOrReload();
   if (dev) {
     await devServer();
   }
@@ -186,7 +168,6 @@ export async function postInstall(
   });
 
   await core.emitAndWrite();
-  console.log('[MDX] generated');
 }
 
 function applyDefaults(options: CreateMDXOptions): Required<CreateMDXOptions> {
@@ -196,8 +177,11 @@ function applyDefaults(options: CreateMDXOptions): Required<CreateMDXOptions> {
   };
 }
 
-function createNextCore({ outDir, configPath }: Required<CreateMDXOptions>) {
-  return createCore(
+function createNextCore({
+  outDir,
+  configPath,
+}: Required<CreateMDXOptions>): Core {
+  const core = createCore(
     {
       environment: 'next',
       outDir,
@@ -205,4 +189,19 @@ function createNextCore({ outDir, configPath }: Required<CreateMDXOptions>) {
     },
     [next()],
   );
+
+  return {
+    ...core,
+    async emitAndWrite(...args) {
+      try {
+        await core.emitAndWrite(...args);
+      } catch (err) {
+        if (err instanceof ValidationError) {
+          console.error(err.toStringFormatted());
+        } else {
+          console.error(err);
+        }
+      }
+    },
+  };
 }

packages/mdx/src/plugins/index.ts

@@ -1,39 +0,0 @@
-import type { LoadedConfig } from '@/loaders/config';
-import type { CoreOptions, ServerContext } from '@/core';
-
-type Awaitable<T> = T | Promise<T>;
-
-export interface EmitEntry {
-  /**
-   * path relative to output directory
-   */
-  path: string;
-  content: string;
-}
-
-export type PluginContext = CoreOptions;
-
-export interface Plugin {
-  /**
-   * on config loaded/updated
-   */
-  config?: (
-    this: PluginContext,
-    config: LoadedConfig,
-  ) => Awaitable<void | LoadedConfig>;
-
-  /**
-   * Generate files (e.g. types, index file, or JSON schemas)
-   */
-  emit?: (this: PluginContext) => Awaitable<EmitEntry[]>;
-
-  /**
-   * Configure Fumadocs dev server
-   */
-  configureServer?: (
-    this: PluginContext,
-    server: ServerContext,
-  ) => Awaitable<void>;
-}
-
-export type PluginOption = Awaitable<Plugin | Plugin[] | false>;

packages/mdx/src/plugins/json-schema.ts

@@ -1,7 +1,7 @@
-import type { EmitEntry, Plugin } from '@/plugins';
+import type { EmitEntry, Plugin } from '@/core';
 import type { LoadedConfig } from '@/loaders/config';
 import { z } from 'zod';
-import { isFileInCollection } from '@/utils/collections';
+import { createCollectionMatcher } from '@/utils/collections';
 import fs from 'node:fs/promises';
 import path from 'node:path';
 
@@ -34,37 +34,34 @@ export default function jsonSchema({
     },
     configureServer(server) {
       if (!server.watcher || !insert) return;
+      const matcher = createCollectionMatcher(this.core);
 
       server.watcher.on('add', async (file) => {
-        for (const [name, collection] of config.collections) {
-          const single =
-            collection.type === 'docs' ? collection.meta : collection;
+        const match = matcher.getFileCollection(file);
+        if (!match || match.collection.type !== 'meta') return;
 
-          if (single.type !== 'meta' || !isFileInCollection(file, single))
-            continue;
+        const { name } = match;
+        const parent = config.collections.get(name);
 
-          let obj: object;
-          try {
-            const content = (await fs.readFile(file)).toString();
-            obj = content.length > 0 ? JSON.parse(content) : {};
-          } catch {
-            return;
-          }
-
-          if ('$schema' in obj) return;
-          const schemaPath = path.join(
-            this.outDir,
-            getSchemaPath(collection.type === 'docs' ? `${name}.meta` : name),
-          );
-          const updated = {
-            $schema: path.relative(path.dirname(file), schemaPath),
-            ...obj,
-          };
-
-          // TODO: try persist formatting?
-          await fs.writeFile(file, JSON.stringify(updated, null, 2));
+        let obj: object;
+        try {
+          const content = (await fs.readFile(file)).toString();
+          obj = content.length > 0 ? JSON.parse(content) : {};
+        } catch {
           return;
         }
+
+        if ('$schema' in obj) return;
+        const schemaPath = path.join(
+          this.outDir,
+          getSchemaPath(parent?.type === 'docs' ? `${name}.meta` : name),
+        );
+        const updated = {
+          $schema: path.relative(path.dirname(file), schemaPath),
+          ...obj,
+        };
+
+        await fs.writeFile(file, JSON.stringify(updated, null, 2));
       });
     },
     emit() {

packages/mdx/src/plugins/next.ts

@@ -1,6 +1,5 @@
 import * as path from 'node:path';
 import { createHash } from 'node:crypto';
-import { glob } from 'tinyglobby';
 import type { LoadedConfig } from '@/loaders/config';
 import type { DocCollection, MetaCollection } from '@/config';
 import { validate } from '@/utils/validation';
@@ -13,16 +12,40 @@ import {
   type ImportPathConfig,
   toImportPath,
 } from '@/utils/import-formatter';
-import { getGlobPatterns, isFileSupported } from '@/utils/collections';
+import { getCollectionFiles } from '@/utils/collections';
 import type { FileInfo } from '@/runtime/shared';
-import type { Plugin } from '@/plugins/index';
+import type { Plugin } from '@/core';
 
 export default function next(): Plugin {
   let config: LoadedConfig;
+  let shouldEmitOnChange = false;
 
   return {
+    name: 'next',
     config(v) {
       config = v;
+
+      // always emit again when async mode enabled
+      shouldEmitOnChange = false;
+      for (const collection of config.collections.values()) {
+        if (
+          (collection.type === 'doc' && collection.async) ||
+          (collection.type === 'docs' && collection.docs.async)
+        ) {
+          shouldEmitOnChange = true;
+        }
+      }
+    },
+    configureServer(server) {
+      if (!server.watcher) return;
+
+      server.watcher.on('all', async () => {
+        if (!shouldEmitOnChange) return;
+
+        await this.core.emitAndWrite({
+          filterPlugin: (plugin) => plugin.name === 'next',
+        });
+      });
     },
     async emit() {
       return [
@@ -200,36 +223,6 @@ export async function indexFile(
   ].join('\n');
 }
 
-async function getCollectionFiles(
-  collection: DocCollection | MetaCollection,
-): Promise<FileInfo[]> {
-  const files = new Map<string, FileInfo>();
-  const dirs = Array.isArray(collection.dir)
-    ? collection.dir
-    : [collection.dir];
-  const patterns = getGlobPatterns(collection);
-
-  await Promise.all(
-    dirs.map(async (dir) => {
-      const result = await glob(patterns, {
-        cwd: path.resolve(dir),
-      });
-
-      for (const item of result) {
-        if (!isFileSupported(item, collection)) continue;
-        const fullPath = path.join(dir, item);
-
-        files.set(fullPath, {
-          path: item,
-          fullPath,
-        });
-      }
-    }),
-  );
-
-  return Array.from(files.values());
-}
-
 function parseMetaEntry(file: string, content: string) {
   const extname = path.extname(file);
   try {

packages/mdx/src/plugins/vite.ts

@@ -11,7 +11,7 @@ import {
   generateGlobImport,
   type GlobImportOptions,
 } from '@/utils/glob-import';
-import type { Plugin } from '@/plugins';
+import type { Plugin } from '@/core';
 import path from 'node:path';
 
 export interface IndexFileOptions {
@@ -37,8 +37,6 @@ export default function vite(options: IndexFileOptions): Plugin {
       config = v;
     },
     emit() {
-      console.log('[Fumadocs MDX] Generating index files');
-
       return [
         {
           path: 'index.ts',

packages/mdx/src/utils/collections.ts

@@ -1,43 +1,112 @@
 import type { DocCollection, MetaCollection } from '@/config/define';
 import picomatch from 'picomatch';
+import type { LoadedConfig } from '@/loaders/config';
+import type { FileInfo } from '@/runtime/shared';
+import { glob } from 'tinyglobby';
+import path from 'node:path';
+import type { Core } from '@/core';
 
-export function getSupportedFormats(
-  collection: DocCollection | MetaCollection,
-) {
-  return {
-    doc: ['mdx', 'md'],
-    meta: ['json', 'yaml'],
-  }[collection.type];
-}
+const SupportedFormats = {
+  doc: ['mdx', 'md'],
+  meta: ['json', 'yaml'],
+};
 
 export function getGlobPatterns(
   collection: MetaCollection | DocCollection,
 ): string[] {
   if (collection.files) return collection.files;
-  return [`**/*.{${getSupportedFormats(collection).join(',')}}`];
+  return [`**/*.{${SupportedFormats[collection.type].join(',')}}`];
 }
 
-export function isFileSupported(
+function isFileSupported(
   filePath: string,
   collection: MetaCollection | DocCollection,
 ) {
-  for (const format of getSupportedFormats(collection)) {
-    if (filePath.endsWith(`.${format}`)) return true;
-  }
-
-  return false;
+  return SupportedFormats[collection.type].some((format) =>
+    filePath.endsWith(`.${format}`),
+  );
 }
 
-export function isFileInCollection(
-  file: string,
+interface ScanIndex {
+  name: string;
+  collection: DocCollection | MetaCollection;
+  matcher: picomatch.Matcher;
+}
+
+export function createCollectionMatcher(core: Core) {
+  const CacheKey = 'collection-matcher';
+  return {
+    scan(config: LoadedConfig) {
+      const scanned: ScanIndex[] = [];
+
+      function scan(name: string, collection: DocCollection | MetaCollection) {
+        const patterns = getGlobPatterns(collection);
+
+        for (const dir of Array.isArray(collection.dir)
+          ? collection.dir
+          : [collection.dir]) {
+          scanned.push({
+            name,
+            collection,
+            matcher: picomatch(patterns, {
+              cwd: dir,
+            }),
+          });
+        }
+      }
+
+      for (const [name, collection] of config.collections) {
+        if (collection.type === 'docs') {
+          scan(name, collection.meta);
+          scan(name, collection.docs);
+        } else {
+          scan(name, collection);
+        }
+      }
+
+      return scanned;
+    },
+    getFileCollection(file: string) {
+      const scanned =
+        (core.cache.get(CacheKey) as ScanIndex[]) ??
+        this.scan(core.getConfig());
+
+      core.cache.set(CacheKey, scanned);
+
+      for (const item of scanned) {
+        if (isFileSupported(file, item.collection) && item.matcher(file))
+          return { name: item.name, collection: item.collection };
+      }
+    },
+  };
+}
+
+export async function getCollectionFiles(
   collection: DocCollection | MetaCollection,
-): boolean {
-  if (!isFileSupported(file, collection)) return false;
-
+) {
+  const files = new Map<string, FileInfo>();
+  const dirs = Array.isArray(collection.dir)
+    ? collection.dir
+    : [collection.dir];
   const patterns = getGlobPatterns(collection);
-  const isMatch = picomatch(patterns, {
-    cwd: Array.isArray(collection.dir) ? collection.dir[0] : collection.dir,
-  });
 
-  return isMatch(file);
+  await Promise.all(
+    dirs.map(async (dir) => {
+      const result = await glob(patterns, {
+        cwd: path.resolve(dir),
+      });
+
+      for (const item of result) {
+        if (!isFileSupported(item, collection)) continue;
+        const fullPath = path.join(dir, item);
+
+        files.set(fullPath, {
+          path: item,
+          fullPath,
+        });
+      }
+    }),
+  );
+
+  return Array.from(files.values());
 }

packages/mdx/src/vite/index.ts

@@ -160,7 +160,6 @@ export async function postInstall(
   });
 
   await core.emitAndWrite();
-  console.log('[MDX] generated');
 }
 
 function createViteCore({

pnpm-lock.yaml

@@ -327,7 +327,7 @@ importers:
         version: 0.2.2(@content-collections/[email protected]([email protected]))([email protected]([email protected]))([email protected])
       '@content-collections/next':
         specifier: ^0.2.8
-        version: 0.2.8(@content-collections/[email protected]([email protected]))([email protected](@babel/[email protected])(@opentelemetry/[email protected])([email protected]([email protected]))([email protected]))
+        version: 0.2.8(@content-collections/[email protected]([email protected]))([email protected](@babel/[email protected])(@opentelemetry/[email protected])([email protected])([email protected]([email protected]))([email protected]))
       '@types/mdx':
         specifier: ^2.0.13
         version: 2.0.13
@@ -1636,7 +1636,7 @@ importers:
         version: 0.2.2(@content-collections/[email protected]([email protected]))([email protected]([email protected]))([email protected])
       '@content-collections/next':
         specifier: ^0.2.8
-        version: 0.2.8(@content-collections/[email protected]([email protected]))([email protected](@babel/[email protected])(@opentelemetry/[email protected])([email protected]([email protected]))([email protected]))
+        version: 0.2.8(@content-collections/[email protected]([email protected]))([email protected](@babel/[email protected])(@opentelemetry/[email protected])([email protected])([email protected]([email protected]))([email protected]))
       '@orama/core':
         specifier: ^1.2.13
         version: 1.2.13
@@ -1823,9 +1823,6 @@ importers:
       remark-mdx:
         specifier: ^3.1.1
         version: 3.1.1
-      remark-parse:
-        specifier: ^11.0.0
-        version: 11.0.0
       tinyexec:
         specifier: ^1.0.1
         version: 1.0.1
@@ -6332,7 +6329,6 @@ packages:
 
   [email protected]:
     resolution: {integrity: sha512-YI7mFs7iWc/VsGsh2aw6eAPD2cjzn1j+LKdYVk09x1CrdTWKYIHyd+dG5iQoN9//3hCDoZj8U6vKpZzEf5UARA==}
-    cpu: [arm64, x64]
     os: [darwin, linux, win32]
     hasBin: true
 
@@ -12198,7 +12194,7 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  '@content-collections/[email protected](@content-collections/[email protected]([email protected]))([email protected](@babel/[email protected])(@opentelemetry/[email protected])([email protected]([email protected]))([email protected]))':
+  '@content-collections/[email protected](@content-collections/[email protected]([email protected]))([email protected](@babel/[email protected])(@opentelemetry/[email protected])([email protected])([email protected]([email protected]))([email protected]))':
     dependencies:
       '@content-collections/core': 0.11.1([email protected])
       '@content-collections/integrations': 0.3.0(@content-collections/[email protected]([email protected]))