MDX: Improve llms.txt generation via remark-llms plugin

.changeset/breezy-kids-battle.md

@@ -0,0 +1,6 @@
+---
+'fumadocs-core': patch
+'fumadocs-mdx': patch
+---
+
+Improve `llms.txt` generation via `remark-llms` plugin

.vscode/settings.json

@@ -15,5 +15,6 @@
   "editor.formatOnSave": true,
   "editor.defaultFormatter": "oxc.oxc-vscode",
   "oxc.fmt.configPath": ".oxfmtrc.jsonc",
-  "terminal.integrated.scrollback": 10000
+  "terminal.integrated.scrollback": 10000,
+  "typescript.tsdk": "node_modules/typescript/lib"
 }

packages/base-ui/tsdown.config.ts

@@ -24,6 +24,6 @@ export default defineConfig({
     console.log('CSS import updated');
   },
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
   },
 });

packages/cli/tsdown.config.ts

@@ -15,7 +15,7 @@ export default defineConfig({
   fixedExtension: false,
   target: 'node22',
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
   },
   exports: {
     enabled: true,

packages/content-collections/tsdown.config.ts

@@ -7,6 +7,6 @@ export default defineConfig({
   format: 'esm',
   entry: ['src/{index,configuration}.ts'],
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
   },
 });

packages/content/tsdown.config.ts

@@ -7,7 +7,7 @@ export default defineConfig({
   fixedExtension: false,
   target: 'es2023',
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
   },
   exports: {
     enabled: true,

packages/core/src/mdx-plugins/remark-feedback-block.ts

@@ -90,6 +90,9 @@ export function remarkFeedbackBlock({
             value: id,
           },
         ],
+        data: {
+          _stringify: 'children-only',
+        },
         children: [node as BlockContent],
       };
       if (generateBody)

packages/core/src/mdx-plugins/remark-llms.ts

@@ -1,12 +1,74 @@
-import { Transformer } from 'unified';
+import type { Processor, Transformer } from 'unified';
 import { toMdxExport } from './mdast-utils';
-import { Root } from 'mdast';
-import { defaultStringifier } from './stringifier';
-import { StringifyOptions } from './remark-structure';
+import type { Heading, Root } from 'mdast';
+import { defaultStringifier, type StringifyOptions } from './stringifier';
 
-export function remarkLlms(options: StringifyOptions): Transformer<Root, Root> {
-  return () => {
-    defaultStringifier({});
-    toMdxExport();
+export interface LLMsOptions extends StringifyOptions {
+  /**
+   * export name for output Markdown.
+   *
+   * @default _markdown
+   */
+  as?: string;
+
+  /**
+   * Explicit heading IDs in output.
+   *
+   * @default true
+   */
+  headingIds?: boolean;
+
+  /**
+   * Filter the elements to be included in the output:
+   *
+   * - `true`: include element & its children.
+   * - `children-only`: exclude element but keep its children.
+   * - `false`: exclude element & its children.
+   *
+   * Default:
+   *
+   * ```ts
+   * filterElement = (node) => {
+   *   switch (node.type) {
+   *     case 'mdxjsEsm':
+   *       return false;
+   *     default:
+   *       return true;
+   *   }
+   * },
+   * ```
+   */
+  filterElement?: StringifyOptions['filterElement'];
+}
+
+/**
+ * generate `llms.txt` for markdown.
+ */
+export function remarkLLMs(
+  this: Processor,
+  { as = '_markdown', headingIds = true, ...stringify }: LLMsOptions = {},
+): Transformer<Root, Root> {
+  const stringifier = defaultStringifier({
+    ...stringify,
+    filterElement(node) {
+      switch (node.type) {
+        case 'mdxjsEsm':
+          return false;
+        default:
+          return true;
+      }
+    },
+    handlers: {
+      heading(node: Heading, _p, state, info) {
+        const id = node.data?.hProperties?.id;
+        const content = state.containerPhrasing(node, info);
+        return headingIds && id ? `${content} [#${id}]` : content;
+      },
+      ...stringify.handlers,
+    },
+  });
+
+  return (node) => {
+    node.children.unshift(toMdxExport(as, stringifier.call(this, node, undefined)));
   };
 }

packages/core/src/mdx-plugins/stringifier.ts

@@ -12,9 +12,12 @@ import type { Processor } from 'unified';
 declare module 'mdast' {
   interface Data {
     /**
-     * [Fumadocs: stringify] The stringified form of node, used for generating search index, `llms.txt`, etc.
+     * [Fumadocs: stringify] Extra info for stringifying the node:
+     * - `children-only`: only stringify its children.
+     * - `{ node }`: stringify as another node.
+     * - `{ text }`: the stringified form of node.
      */
-    _string?: string | (() => string);
+    _stringify?: 'children-only' | { node: Nodes } | { text: string };
   }
 }
 
@@ -61,6 +64,9 @@ export interface StringifyOptions<Context = undefined> extends Options {
     attribute: MdxJsxAttribute | MdxJsxExpressionAttribute,
   ) => boolean;
 
+  /**
+   * run before stringifying any nodes
+   */
   onStringify?: (node: Nodes, ctx: Context) => void;
 }
 
@@ -97,22 +103,39 @@ export function defaultStringifier<Context>(
 
   function modHandler(handler: Handle, ctx: Context): Handle {
     return function (node: Nodes, parent, state, info) {
-      if (onStringify) onStringify(node, ctx);
-      if (node.data?._string)
-        return typeof node.data._string === 'function' ? node.data._string() : node.data._string;
-      const visibility = filterElement(node);
-
+      let visibility = filterElement(node);
       if (visibility === false) return '';
 
+      if (onStringify) onStringify(node, ctx);
+
+      const extraInfo = node.data?._stringify;
+      if (extraInfo) {
+        if (extraInfo === 'children-only') {
+          visibility = 'children-only';
+        } else if ('text' in extraInfo) {
+          return extraInfo.text;
+        } else {
+          node = extraInfo.node;
+        }
+      }
+
+      if (visibility === 'children-only') {
+        if (!('children' in node)) return '';
+
+        switch (node.type) {
+          case 'mdxJsxTextElement':
+          case 'paragraph':
+            return state.containerPhrasing(node, info);
+          case 'mdxJsxFlowElement':
+            return state.containerFlow(node, info);
+          default:
+            return state.containerFlow({ type: 'root', children: node.children }, info);
+        }
+      }
+
       switch (node.type) {
         case 'mdxJsxFlowElement':
         case 'mdxJsxTextElement': {
-          if (visibility === 'children-only') {
-            return node.type === 'mdxJsxTextElement'
-              ? state.containerPhrasing(node, info)
-              : state.containerFlow(node, info);
-          }
-
           const stringifiedAttributes: MdxJsxAttribute[] = [];
           for (const attr of node.attributes) {
             if (attr.type === 'mdxJsxExpressionAttribute') continue;
@@ -121,23 +144,15 @@ export function defaultStringifier<Context>(
             if (!str) continue;
 
             stringifiedAttributes.push({
-              ...attr,
+              type: 'mdxJsxAttribute',
+              name: attr.name,
               value: str,
             });
           }
 
-          const temp = node.attributes;
-          node.attributes = stringifiedAttributes;
-          const s = handler(node, parent, state, info);
-          node.attributes = temp;
-          return s;
+          return handler({ ...node, attributes: stringifiedAttributes }, parent, state, info);
         }
         default:
-          if (visibility === 'children-only')
-            return 'children' in node
-              ? state.containerFlow({ type: 'root', children: node.children }, info)
-              : '';
-
           return handler(node, parent, state, info);
       }
     };

packages/core/tsdown.config.ts

@@ -29,6 +29,6 @@ export default defineConfig({
     'src/framework/*',
   ],
   deps: {
-    onlyAllowBundle: ['remove-markdown'],
+    onlyBundle: ['remove-markdown'],
   },
 });

packages/create-app/tsdown.config.ts

@@ -7,6 +7,6 @@ export default defineConfig({
   dts: true,
   fixedExtension: false,
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
   },
 });

packages/doc-gen/tsdown.config.ts

@@ -7,6 +7,6 @@ export default defineConfig({
   format: 'esm',
   entry: ['src/index.ts', 'src/remark-ts2js.ts'],
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
   },
 });

packages/epub/tsdown.config.ts

@@ -7,6 +7,6 @@ export default defineConfig({
   fixedExtension: false,
   target: 'node20',
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
   },
 });

packages/mdx-remote/tsdown.config.ts

@@ -7,7 +7,7 @@ export default defineConfig({
   entry: ['./src/index.ts', './src/client/index.ts'],
   format: 'esm',
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
     neverBundle: ['mdx/types'],
   },
 });

packages/mdx/src/loaders/mdx/remark-postprocess.ts

@@ -1,11 +1,10 @@
 import type { Processor, Transformer } from 'unified';
-import type { Heading, Root, RootContent } from 'mdast';
+import type { Root, RootContent } from 'mdast';
 import { visit } from 'unist-util-visit';
-import { toMarkdown } from 'mdast-util-to-markdown';
 import { valueToEstree } from 'estree-util-value-to-estree';
 import { removePosition } from 'unist-util-remove-position';
 import { flattenNode } from './mdast-utils';
-import { mdxToMarkdown } from 'mdast-util-mdx';
+import { remarkLLMs, type LLMsOptions } from 'fumadocs-core/mdx-plugins/remark-llms';
 
 export interface ExtractedReference {
   href: string;
@@ -22,14 +21,7 @@ export interface PostprocessOptions {
   /**
    * stringify MDAST and export via `_markdown`.
    */
-  includeProcessedMarkdown?:
-    | boolean
-    | {
-        /**
-         * include heading IDs into the processed markdown.
-         */
-        headingIds?: boolean;
-      };
+  includeProcessedMarkdown?: boolean | LLMsOptions;
 
   /**
    * extract link references, export via `extractedReferences`.
@@ -53,7 +45,6 @@ export interface PostprocessOptions {
 export function remarkPostprocess(
   this: Processor,
   {
-    _format,
     includeProcessedMarkdown = false,
     includeMDAST = false,
     extractLinkReferences = false,
@@ -94,28 +85,11 @@ export function remarkPostprocess(
     }
 
     if (includeProcessedMarkdown) {
-      const { headingIds = true } =
-        typeof includeProcessedMarkdown === 'object' ? includeProcessedMarkdown : {};
-      const defaultExtensions = this.data('toMarkdownExtensions');
-      const extensions = defaultExtensions ? [...defaultExtensions] : [];
-      if (_format === 'md') extensions.push(mdxToMarkdown());
-
-      const markdown = toMarkdown(tree, {
-        ...this.data('settings'),
-        extensions,
-        handlers: {
-          heading(node: Heading) {
-            const id = node.data?.hProperties?.id;
-            const content = flattenNode(node);
-            return headingIds && id ? `${content} [#${id}]` : content;
-          },
-        },
-      });
-
-      file.data['mdx-export'].push({
-        name: '_markdown',
-        value: markdown,
-      });
+      const llms = remarkLLMs.call(
+        this,
+        typeof includeProcessedMarkdown === 'object' ? includeProcessedMarkdown : undefined,
+      );
+      llms(tree, file, () => undefined);
     }
 
     if (includeMDAST) {

packages/mdx/tsdown.config.ts

@@ -5,6 +5,8 @@ const external = ['next', 'typescript', 'webpack', 'bun', 'mdx/types'];
 const noExternal = [
   // TODO: remove this when the min `fumadocs-core` version is above 16.2.3
   'fumadocs-core/source/schema',
+  // TODO: remove this when the min `fumadocs-core` version is above 16.6.17
+  'fumadocs-core/mdx-plugins/remark-llms',
 ];
 
 export default defineConfig([
@@ -22,7 +24,7 @@ export default defineConfig([
     fixedExtension: false,
     target: 'node22',
     deps: {
-      onlyAllowBundle: [],
+      onlyBundle: [],
       alwaysBundle: noExternal,
       neverBundle: external,
     },
@@ -37,7 +39,7 @@ export default defineConfig([
     fixedExtension: false,
     target: 'node22',
     deps: {
-      onlyAllowBundle: [],
+      onlyBundle: [],
       alwaysBundle: noExternal,
       neverBundle: external,
     },

packages/obsidian/tsdown.config.ts

@@ -7,6 +7,6 @@ export default defineConfig({
   dts: true,
   entry: ['src/index.ts', 'src/ui/index.tsx', 'src/mdx/index.ts'],
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
   },
 });

packages/openapi/tsdown.config.ts

@@ -21,7 +21,7 @@ export default defineConfig({
     await compileInline();
   },
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
   },
   exports: {
     enabled: true,

packages/press/tsdown.config.ts

@@ -7,6 +7,6 @@ export default defineConfig({
   outDir: 'dist/lib',
   dts: true,
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
   },
 });

packages/python/tsdown.config.ts

@@ -9,6 +9,6 @@ export default defineConfig({
   fixedExtension: false,
   target: 'es2023',
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
   },
 });

packages/radix-ui/tsdown.config.ts

@@ -26,6 +26,6 @@ export default defineConfig({
     console.log('CSS import updated');
   },
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
   },
 });

packages/stf/tsdown.config.ts

@@ -9,7 +9,7 @@ export default defineConfig({
     sourcemap: false,
   },
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
   },
   exports: true,
 });

packages/story/tsdown.config.ts

@@ -14,6 +14,6 @@ export default defineConfig({
     sourcemap: false,
   },
   deps: {
-    onlyAllowBundle: ['@fastify/deepmerge', '@ungap/structured-clone'],
+    onlyBundle: ['@fastify/deepmerge', '@ungap/structured-clone'],
   },
 });

packages/tailwind/tsdown.config.ts

@@ -9,7 +9,7 @@ export default defineConfig({
     sourcemap: false,
   },
   deps: {
-    onlyAllowBundle: ['lodash.merge'],
+    onlyBundle: ['lodash.merge'],
     neverBundle: ['@tailwindcss/oxide'],
   },
 });

packages/twoslash/tsdown.config.ts

@@ -9,6 +9,6 @@ export default defineConfig({
     sourcemap: false,
   },
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
   },
 });

packages/typescript/tsdown.config.ts

@@ -9,7 +9,7 @@ export default defineConfig({
   format: 'esm',
   entry: ['src/index.ts', 'src/ui/index.ts'],
   deps: {
-    onlyAllowBundle: [],
+    onlyBundle: [],
     neverBundle: ['server-only'],
   },
 });