MDX: Support generate search indexes in build time

.changeset/swift-lamps-allow.md

@@ -0,0 +1,5 @@
+---
+'fumadocs-mdx': minor
+---
+
+Support generate search indexes in build time

apps/docs/next.config.mjs

@@ -23,6 +23,7 @@ const config = {
 };
 
 const withMDX = createMDX({
+  buildSearchIndex: {},
   mdxOptions: {
     rehypeCodeOptions: {
       transformers: [

apps/docs/scripts/update-index.mts

@@ -3,13 +3,15 @@ import { resolve } from 'node:path';
 import env from '@next/env';
 import algosearch from 'algoliasearch';
 import { sync } from 'fumadocs-core/search-algolia/server';
-import type { Index } from '@/utils/source';
+import type { SearchIndex } from 'fumadocs-mdx';
 
 env.loadEnvConfig(process.cwd());
 
 const indexes = JSON.parse(
-  readFileSync(resolve('./.next/_map_indexes.json')).toString(),
-) as Index[];
+  readFileSync(
+    resolve('./.next/server/chunks/fumadocs_search.json'),
+  ).toString(),
+) as SearchIndex[];
 
 const client = algosearch(
   process.env.ALGOLIA_APP_ID || '',

apps/docs/utils/source.ts

@@ -1,8 +1,5 @@
-import { writeFileSync } from 'node:fs';
-import path from 'node:path';
 import { createMDXSource, defaultSchemas } from 'fumadocs-mdx';
 import type { StructuredData } from 'fumadocs-core/mdx-plugins';
-import { PHASE_PRODUCTION_BUILD } from 'next/constants';
 import { z } from 'zod';
 import type { InferMetaType, InferPageType } from 'fumadocs-core/source';
 import { loader } from 'fumadocs-core/source';
@@ -36,31 +33,3 @@ export interface Index {
   url: string;
   structuredData: StructuredData;
 }
-
-// Access and export MDX pages data to json file
-// So that we can update search indexes after the build
-const g = globalThis as unknown as {
-  __NEXT_DOCS_INDEX_UPDATED?: boolean;
-};
-
-if (
-  process.env.NEXT_PHASE === PHASE_PRODUCTION_BUILD &&
-  !g.__NEXT_DOCS_INDEX_UPDATED
-) {
-  const mapPath = path.resolve('./.next/_map_indexes.json');
-  const indexes: Index[] = utils.files.flatMap((file) => {
-    if (file.type !== 'page') return [];
-
-    return {
-      id: file.url,
-      title: file.data.title,
-      description: file.data.description,
-      url: file.url,
-      structuredData: file.data.exports.structuredData,
-    };
-  });
-
-  writeFileSync(mapPath, JSON.stringify(indexes));
-
-  g.__NEXT_DOCS_INDEX_UPDATED = true;
-}

packages/core/src/source/index.ts

@@ -1,5 +1,7 @@
 export * from './page-tree-builder';
-export * from './types';
 export * from './loader';
 export * as FileSystem from './file-system';
-export * from './load';
+
+export type { LoadOptions, Transformer, VirtualFile } from './load';
+export type * from './types';
+export { type FileInfo, parseFilePath, parseFolderPath } from './path';

packages/core/src/source/loader.ts

@@ -95,7 +95,7 @@ function groupByLanguages(
   return langMap;
 }
 
-function createGetUrl(
+export function createGetUrl(
   baseUrl: string,
 ): (slugs: string[], locale?: string) => string {
   return (slugs, locale) => {
@@ -106,7 +106,7 @@ function createGetUrl(
   };
 }
 
-function getSlugs(info: FileInfo): string[] {
+export function getSlugs(info: FileInfo): string[] {
   const result = [...splitPath(info.dirname), info.name];
 
   return result[result.length - 1] === 'index' ? result.slice(0, -1) : result;

packages/mdx/.eslintrc.cjs

@@ -1,3 +1,7 @@
 module.exports = {
   extends: ['custom/library'],
+  rules: {
+    'import/no-named-as-default': 'off',
+    'import/no-named-as-default-member': 'off',
+  },
 };

packages/mdx/src/config.ts

@@ -15,6 +15,10 @@ import { MapWebpackPlugin } from './webpack-plugins/map-plugin';
 import remarkMdxExport from './mdx-plugins/remark-exports';
 import type { LoaderOptions } from './loader';
 import type { Options as MDXLoaderOptions } from './loader-mdx';
+import {
+  SearchIndexPlugin,
+  type Options as SearchIndexPluginOptions,
+} from './webpack-plugins/search-index-plugin';
 
 type MDXOptions = Omit<
   NonNullable<MDXLoaderOptions>,
@@ -39,6 +43,8 @@ export interface CreateMDXOptions {
 
   mdxOptions?: MDXOptions;
 
+  buildSearchIndex?: Partial<SearchIndexPluginOptions> | false;
+
   /**
    * Where the root map.ts should be, relative to cwd
    *
@@ -119,9 +125,11 @@ const createMDX =
     cwd = process.cwd(),
     rootMapPath = './.map.ts',
     rootContentPath = './content',
+    buildSearchIndex = false,
   }: CreateMDXOptions = {}) =>
   (nextConfig: NextConfig = {}) => {
-    const _mapPath = path.resolve(cwd, rootMapPath);
+    const rootMapFile = path.resolve(cwd, rootMapPath);
+    const rootContentDir = path.resolve(cwd, rootContentPath);
     const mdxLoaderOptions = getMDXLoaderOptions(mdxOptions);
 
     return {
@@ -153,13 +161,12 @@ const createMDX =
               ],
             },
             {
-              test: _mapPath,
+              test: rootMapFile,
               use: {
                 loader: 'fumadocs-mdx/loader',
                 options: {
-                  cwd,
-                  rootContentPath,
-                  mapPath: _mapPath,
+                  rootContentDir,
+                  rootMapFile,
                 } satisfies LoaderOptions,
               },
             },
@@ -167,7 +174,16 @@ const createMDX =
 
           config.plugins ||= [];
 
-          config.plugins.push(new MapWebpackPlugin({ rootMapFile: _mapPath }));
+          config.plugins.push(
+            new MapWebpackPlugin({
+              rootMapFile,
+            }),
+          );
+
+          if (buildSearchIndex !== false)
+            config.plugins.push(
+              new SearchIndexPlugin({ rootContentDir, ...buildSearchIndex }),
+            );
 
           // eslint-disable-next-line @typescript-eslint/no-unsafe-return -- not provided
           return nextConfig.webpack?.(config, options) ?? config;

packages/mdx/src/loader-mdx.ts

@@ -13,8 +13,8 @@ export interface Options extends ProcessorOptions {
   lastModifiedTime?: 'git' | 'none';
 }
 
-interface InternalBuildInfo {
-  __next_docs?: {
+export interface InternalBuildInfo {
+  __fumadocs?: {
     path: string;
     /**
      * `vfile.data` parsed from file
@@ -71,7 +71,7 @@ export default async function loader(
       (file) => {
         const info = this._module?.buildInfo as InternalBuildInfo;
 
-        info.__next_docs = {
+        info.__fumadocs = {
           path: filePath,
           data: file.data,
         };

packages/mdx/src/loader.ts

@@ -3,9 +3,8 @@ import fg from 'fast-glob';
 import type { LoaderContext } from 'webpack';
 
 export interface LoaderOptions {
-  rootContentPath: string;
-  mapPath: string;
-  cwd: string;
+  rootContentDir: string;
+  rootMapFile: string;
 }
 
 /**
@@ -19,18 +18,17 @@ export default function loader(
   const options = this.getOptions();
 
   this.cacheable(true);
-  this.addContextDependency(path.resolve(options.cwd, options.rootContentPath));
+  this.addContextDependency(options.rootContentDir);
 
   callback(null, buildMap(options));
 }
 
-function buildMap({ cwd, rootContentPath, mapPath }: LoaderOptions): string {
-  const mapDir = path.dirname(mapPath);
-  const absoluteContentPath = path.resolve(cwd, rootContentPath);
+function buildMap({ rootContentDir, rootMapFile }: LoaderOptions): string {
+  const mapDir = path.dirname(rootMapFile);
 
   // eslint-disable-next-line import/no-named-as-default-member -- commom.js
   const files = fg.sync('./**/*.{md,mdx,json}', {
-    cwd: absoluteContentPath,
+    cwd: rootContentDir,
   });
 
   const imports: string[] = [];
@@ -38,7 +36,7 @@ function buildMap({ cwd, rootContentPath, mapPath }: LoaderOptions): string {
 
   files.forEach((file, i) => {
     let importPath = path
-      .relative(mapDir, path.join(absoluteContentPath, file))
+      .relative(mapDir, path.join(rootContentDir, file))
       .replaceAll(path.sep, '/');
 
     if (!importPath.startsWith('.')) {

packages/mdx/src/types.ts

@@ -32,3 +32,5 @@ export type MDXPageData<Frontmatter extends PageData = PageData> = Omit<
 > & {
   exports: Omit<MDXExport<Frontmatter>, 'frontmatter'>;
 };
+
+export type { SearchIndex } from './webpack-plugins/search-index-plugin';

packages/mdx/src/webpack-plugins/map-plugin.ts

@@ -1,9 +1,12 @@
-import fs from 'node:fs';
-import type { Compiler } from 'webpack';
+import * as fs from 'node:fs';
+import { type Compiler } from 'webpack';
 
 let firstLoad = true;
 
 interface Options {
+  /**
+   * Absolute path of .map file
+   */
   rootMapFile: string;
 }
 

packages/mdx/src/webpack-plugins/search-index-plugin.ts

@@ -0,0 +1,109 @@
+/* eslint-disable @typescript-eslint/consistent-type-imports -- Webpack */
+import * as path from 'node:path';
+import { createRequire } from 'node:module';
+import type { StructuredData } from 'fumadocs-core/mdx-plugins';
+import { createGetUrl } from 'fumadocs-core/source';
+import type { Compiler } from 'webpack';
+
+const require = createRequire(import.meta.url);
+
+const pkg = require('next/dist/compiled/webpack/webpack.js') as {
+  sources: typeof import('webpack').sources;
+  webpack: typeof import('webpack');
+};
+
+export interface SearchIndex {
+  id: string;
+  title: string;
+  description?: string;
+  url: string;
+  structuredData: StructuredData;
+}
+
+export interface Options {
+  /**
+   * Only enable at production builds, default: `true`
+   */
+  productionOnly?: boolean;
+
+  /**
+   * Absolute path of root content directory
+   */
+  rootContentDir: string;
+
+  /**
+   * @param path - MDX file path relative to root content dir
+   * @returns page URL
+   */
+  getUrl?: (path: string) => string;
+}
+
+export class SearchIndexPlugin {
+  options: Options;
+
+  constructor(options: Options) {
+    this.options = options;
+  }
+
+  apply(compiler: Compiler): void {
+    const {
+      rootContentDir,
+      productionOnly = true,
+      getUrl = (file) => {
+        const parsedPath = path.parse(file);
+        const flattenedPath = path.join(parsedPath.dir, parsedPath.name);
+
+        return createGetUrl('/')(flattenedPath.split(path.sep));
+      },
+    } = this.options;
+    const logger = compiler.getInfrastructureLogger(SearchIndexPlugin.name);
+    const isProduction = process.env.NODE_ENV === 'production';
+
+    if (productionOnly && !isProduction) return;
+
+    compiler.hooks.compilation.tap(SearchIndexPlugin.name, (compilation) => {
+      compilation.hooks.processAssets.tap(
+        {
+          name: SearchIndexPlugin.name,
+          stage: pkg.webpack.Compilation.PROCESS_ASSETS_STAGE_ADDITIONS,
+        },
+        () => {
+          const indexFiles: SearchIndex[] = [];
+
+          for (const m of compilation.modules.values()) {
+            if (!m.buildInfo || !('__fumadocs' in m.buildInfo)) continue;
+
+            const searchData = m.buildInfo.__fumadocs as {
+              path: string;
+              data: {
+                structuredData: StructuredData;
+                frontmatter: {
+                  title: string;
+                  description?: string;
+                };
+              };
+            };
+
+            const relativePath = path.relative(rootContentDir, searchData.path);
+
+            const data = searchData.data;
+            indexFiles.push({
+              id: searchData.path,
+              structuredData: data.structuredData,
+              title: data.frontmatter.title,
+              description: data.frontmatter.description,
+              url: getUrl(relativePath),
+            });
+          }
+
+          compilation.emitAsset(
+            'fumadocs_search.json',
+            new pkg.sources.RawSource(JSON.stringify(indexFiles)),
+          );
+
+          logger.info('Generated Search Indexes');
+        },
+      );
+    });
+  }
+}

packages/mdx/tsup.config.ts

@@ -3,7 +3,7 @@ import { defineConfig } from 'tsup';
 export default defineConfig({
   entry: ['./src/{config,index,loader,loader-mdx}.ts'],
   format: 'esm',
-  external: ['fumadocs-core', 'webpack'],
+  external: ['fumadocs-core', 'webpack', 'next'],
   dts: true,
-  target: 'es6',
+  target: 'node18',
 });