Core: use mdast-util-to-markdown for accurate stringification

.changeset/major-pandas-check.md

@@ -0,0 +1,5 @@
+---
+'fumadocs-core': patch
+---
+
+use `mdast-util-to-markdown` for accurate stringification

apps/docs/source.config.ts

@@ -59,9 +59,6 @@ export const docs = defineDocs({
         shiki: shikiConfig,
       };
       return applyMdxPreset({
-        remarkStructureOptions: {
-          types: [...remarkStructureDefaultOptions.types, 'code'],
-        },
         rehypeCodeOptions: isLint
           ? false
           : {

packages/core/package.json

@@ -132,6 +132,8 @@
     "hast-util-to-estree": "^3.1.3",
     "hast-util-to-jsx-runtime": "^2.3.6",
     "image-size": "^2.0.2",
+    "mdast-util-mdx": "^3.0.0",
+    "mdast-util-to-markdown": "^2.1.2",
     "negotiator": "^1.0.0",
     "npm-to-yarn": "^3.0.1",
     "path-to-regexp": "^8.3.0",
@@ -161,8 +163,6 @@
     "algoliasearch": "5.48.0",
     "eslint-config-custom": "workspace:*",
     "lucide-react": "^0.563.0",
-    "mdast-util-mdx-jsx": "^3.2.0",
-    "mdast-util-mdxjs-esm": "^2.0.1",
     "next": "16.1.6",
     "react-router": "^7.13.0",
     "remark-directive": "^4.0.0",
@@ -186,7 +186,6 @@
     "@types/react": "*",
     "algoliasearch": "5.x.x",
     "lucide-react": "*",
-    "mdast-util-mdx-jsx": "*",
     "next": "16.x.x",
     "react": "^19.2.0",
     "react-dom": "^19.2.0",
@@ -195,9 +194,6 @@
     "zod": "4.x.x"
   },
   "peerDependenciesMeta": {
-    "mdast-util-mdx-jsx": {
-      "optional": true
-    },
     "@mdx-js/mdx": {
       "optional": true
     },

packages/core/src/mdx-plugins/codeblock-utils.ts

@@ -1,4 +1,4 @@
-import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx-jsx';
+import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
 import type { BlockContent, Text } from 'mdast';
 
 export interface CodeBlockTabsOptions {

packages/core/src/mdx-plugins/rehype-code.core.ts

@@ -9,7 +9,7 @@ import {
 } from '@shikijs/transformers';
 import type { Processor, Transformer } from 'unified';
 import type { Highlighter, ShikiTransformer } from 'shiki';
-import type { MdxJsxFlowElement } from 'mdast-util-mdx-jsx';
+import type { MdxJsxFlowElement } from 'mdast-util-mdx';
 import type { CodeBlockIcon, IconOptions } from './transformer-icon';
 import { transformerIcon } from './transformer-icon';
 import { parseCodeBlockAttributes } from '@/mdx-plugins/codeblock-utils';

packages/core/src/mdx-plugins/remark-code-tab.ts

@@ -1,7 +1,7 @@
 import type { Processor, Transformer } from 'unified';
 import type { BlockContent, Code, DefinitionContent, Root } from 'mdast';
 import { visit } from 'unist-util-visit';
-import type { MdxJsxFlowElement } from 'mdast-util-mdx-jsx';
+import type { MdxJsxFlowElement } from 'mdast-util-mdx';
 import { generateCodeBlockTabs, parseCodeBlockAttributes } from '@/mdx-plugins/codeblock-utils';
 
 type TabType = keyof typeof Types;

packages/core/src/mdx-plugins/remark-directive-admonition.ts

@@ -1,5 +1,5 @@
 import type { BlockContent, DefinitionContent, PhrasingContent, Root } from 'mdast';
-import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx-jsx';
+import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
 import { Transformer } from 'unified';
 import { visit } from 'unist-util-visit';
 

packages/core/src/mdx-plugins/remark-feedback-block.ts

@@ -1,6 +1,6 @@
 import type { Transformer } from 'unified';
 import type { BlockContent, Root, RootContent } from 'mdast';
-import type { MdxJsxFlowElement } from 'mdast-util-mdx-jsx';
+import type { MdxJsxFlowElement } from 'mdast-util-mdx';
 import { visit } from 'unist-util-visit';
 import { createHash } from 'node:crypto';
 import { flattenNode } from './mdast-utils';

packages/core/src/mdx-plugins/remark-image.ts

@@ -2,9 +2,9 @@ import * as path from 'node:path';
 import type { Image, Root } from 'mdast';
 import type { Transformer } from 'unified';
 import { visit } from 'unist-util-visit';
-import type { MdxjsEsm } from 'mdast-util-mdxjs-esm';
+import type { MdxjsEsm } from 'mdast-util-mdx';
 import type { ISizeCalculationResult } from 'image-size/types/interface';
-import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx-jsx';
+import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
 import { fileURLToPath } from 'node:url';
 
 const VALID_BLUR_EXT = ['.jpeg', '.png', '.webp', '.avif', '.jpg'];

packages/core/src/mdx-plugins/remark-mdx-files.ts

@@ -1,7 +1,7 @@
 import type { Root } from 'mdast';
 import { visit } from 'unist-util-visit';
 import type { Transformer } from 'unified';
-import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx-jsx';
+import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
 import type { VFile } from 'vfile';
 import path from 'node:path';
 

packages/core/src/mdx-plugins/remark-mdx-mermaid.ts

@@ -1,7 +1,7 @@
 import { visit } from 'unist-util-visit';
 import type { Transformer } from 'unified';
 import type { Root } from 'mdast';
-import type { MdxJsxFlowElement } from 'mdast-util-mdx-jsx';
+import type { MdxJsxFlowElement } from 'mdast-util-mdx';
 
 function toMDX(code: string): MdxJsxFlowElement {
   return {

packages/core/src/mdx-plugins/remark-steps.ts

@@ -1,7 +1,7 @@
 import type { Transformer } from 'unified';
 import type { BlockContent, Heading, Root, RootContent } from 'mdast';
 import { visit } from 'unist-util-visit';
-import type { MdxJsxFlowElement } from 'mdast-util-mdx-jsx';
+import type { MdxJsxFlowElement } from 'mdast-util-mdx';
 
 export interface RemarkStepsOptions {
   /**

packages/core/src/mdx-plugins/remark-structure.ts

@@ -2,14 +2,17 @@ import Slugger from 'github-slugger';
 import type { Nodes, Root } from 'mdast';
 import { remark } from 'remark';
 import remarkGfm from 'remark-gfm';
-import type { PluggableList, Transformer } from 'unified';
+import type { PluggableList, Processor, Transformer } from 'unified';
 import { visit } from 'unist-util-visit';
 import { flattenNode, toMdxExport } from './mdast-utils';
-import type {
-  MdxJsxAttribute,
-  MdxJsxExpressionAttribute,
-  MdxJsxFlowElement,
-} from 'mdast-util-mdx-jsx';
+import {
+  mdxToMarkdown,
+  type MdxJsxAttribute,
+  type MdxJsxExpressionAttribute,
+  type MdxJsxFlowElement,
+  type MdxJsxTextElement,
+} from 'mdast-util-mdx';
+import { type Handle, toMarkdown } from 'mdast-util-to-markdown';
 
 interface Heading {
   id: string;
@@ -31,14 +34,21 @@ export interface StructuredData {
 
 export interface StructureOptions {
   /**
-   * Types to be scanned as content.
+   * MDAST **block** types to be scanned as content.
+   *
+   * If a node's type is represented in this array, it will be stringified and its children will not be scanned.
    *
    * @defaultValue ['heading', 'paragraph', 'blockquote', 'tableCell', 'mdxJsxFlowElement']
    */
   types?: string[] | ((node: Nodes) => boolean);
 
   /**
-   * A list of indexable MDX attributes, either:
+   * stringify a given node & its children, you can use something like `mdast-util-to-markdown`.
+   */
+  stringify?: (this: Processor, node: Nodes) => string;
+
+  /**
+   * By default, it will not index MDX attributes. You can define a list of MDX attributes to index, either:
    *
    * - an array of attribute names.
    * - a function that determines if attribute should be indexed.
@@ -46,12 +56,12 @@ export interface StructureOptions {
   allowedMdxAttributes?:
     | string[]
     | ((
-        node: MdxJsxFlowElement,
+        node: MdxJsxFlowElement | MdxJsxTextElement,
         attribute: MdxJsxAttribute | MdxJsxExpressionAttribute,
       ) => boolean);
 
   /**
-   * export as `structuredData` or specified variable name.
+   * export as `structuredData` (if true) or specified variable name.
    */
   exportAs?: string | boolean;
 }
@@ -76,24 +86,32 @@ declare module 'vfile' {
 
 export const remarkStructureDefaultOptions = {
   types: ['heading', 'paragraph', 'blockquote', 'tableCell', 'mdxJsxFlowElement'],
-  allowedMdxAttributes: (node) => {
-    if (!node.name) return false;
-
-    return ['TypeTable', 'Callout'].includes(node.name);
+  allowedMdxAttributes(node) {
+    switch (node.name) {
+      case 'TypeTable':
+      case 'Callout':
+        return true;
+      default:
+        return false;
+    }
   },
   exportAs: false,
-} satisfies Required<StructureOptions>;
+} satisfies StructureOptions;
 
 /**
  * Extract content into structured data.
  *
  * By default, the output is stored into VFile (`vfile.data.structuredData`), you can specify `exportAs` to export it.
  */
-export function remarkStructure({
-  types = remarkStructureDefaultOptions.types,
-  allowedMdxAttributes = remarkStructureDefaultOptions.allowedMdxAttributes,
-  exportAs = remarkStructureDefaultOptions.exportAs,
-}: StructureOptions = {}): Transformer<Root, Root> {
+export function remarkStructure(
+  this: Processor,
+  {
+    types = remarkStructureDefaultOptions.types,
+    stringify,
+    allowedMdxAttributes = remarkStructureDefaultOptions.allowedMdxAttributes,
+    exportAs = remarkStructureDefaultOptions.exportAs,
+  }: StructureOptions = {},
+): Transformer<Root, Root> {
   const slugger = new Slugger();
 
   if (Array.isArray(allowedMdxAttributes)) {
@@ -107,6 +125,29 @@ export function remarkStructure({
     types = (node) => arr.includes(node.type);
   }
 
+  stringify ??= (root) => {
+    const { mdxJsxFlowElement } = mdxToMarkdown({ tightSelfClosing: true }).extensions![1]
+      .handlers!;
+    const wrapper: Handle = (node: MdxJsxFlowElement | MdxJsxTextElement, ...rest) => {
+      const originalAttributes = node.attributes;
+      if (allowedMdxAttributes)
+        node.attributes = node.attributes.filter((attr) => allowedMdxAttributes(node, attr));
+      const s = mdxJsxFlowElement!(node, ...rest);
+      node.attributes = originalAttributes;
+      return s;
+    };
+
+    return toMarkdown(root, {
+      ...this.data('settings'),
+      // from https://github.com/remarkjs/remark/blob/main/packages/remark-stringify/lib/index.js
+      extensions: this.data('toMarkdownExtensions') ?? [],
+      handlers: {
+        mdxJsxFlowElement: wrapper,
+        mdxJsxTextElement: wrapper,
+      },
+    });
+  };
+
   return (tree, file) => {
     slugger.reset();
     const data: StructuredData = { contents: [], headings: [] };
@@ -156,32 +197,14 @@ export function remarkStructure({
         return 'skip';
       }
 
-      if (element.type === 'mdxJsxFlowElement' && element.name) {
-        data.contents.push(
-          ...element.attributes.flatMap((attribute) => {
-            const value =
-              typeof attribute.value === 'string' ? attribute.value : attribute.value?.value;
-            if (!value || value.length === 0) return [];
-            if (allowedMdxAttributes && !allowedMdxAttributes(element, attribute)) return [];
-
-            return {
-              heading: lastHeading,
-              content: attribute.type === 'mdxJsxAttribute' ? `${attribute.name}: ${value}` : value,
-            };
-          }),
-        );
-
-        return;
+      const content = stringify.call(this, element).trim();
+      if (content.length > 0) {
+        data.contents.push({
+          heading: lastHeading,
+          content,
+        });
       }
 
-      const content = flattenNode(element).trim();
-      if (content.length === 0) return;
-
-      data.contents.push({
-        heading: lastHeading,
-        content,
-      });
-
       return 'skip';
     });
 

packages/core/test/fixtures/remark-structure.mdx

@@ -17,3 +17,11 @@ function main() {
   console.log('indexed');
 }
 ```
+
+<Card title="Test">wow</Card>
+
+<Card title="Test" className="test-sm">
+
+wow
+
+</Card>

packages/core/test/fixtures/remark-structure.output.json

@@ -1,42 +1,50 @@
 {
   "contents": [
     {
-      "content": "Some text here",
       "heading": "heading-1",
+      "content": "Some text here"
     },
     {
-      "content": "Some text here",
       "heading": "heading-2",
+      "content": "Some text here"
     },
     {
-      "content": "Name",
       "heading": "heading-3",
+      "content": "Name"
     },
     {
-      "content": "Description",
       "heading": "heading-3",
+      "content": "Description"
     },
     {
-      "content": "Hello World",
       "heading": "heading-3",
+      "content": "Hello World"
     },
     {
-      "content": "Goodbye",
       "heading": "heading-3",
+      "content": "Goodbye"
     },
+    {
+      "heading": "heading-3",
+      "content": "<Card title=\"Test\">wow</Card>"
+    },
+    {
+      "heading": "heading-3",
+      "content": "<Card title=\"Test\">\n  wow\n</Card>"
+    }
   ],
   "headings": [
     {
-      "content": "Heading 1",
       "id": "heading-1",
+      "content": "Heading 1"
     },
     {
-      "content": "Heading 2",
       "id": "heading-2",
+      "content": "Heading 2"
     },
     {
-      "content": "Heading 3",
       "id": "heading-3",
-    },
-  ],
+      "content": "Heading 3"
+    }
+  ]
 }

packages/core/test/mdx-plugins.test.ts

@@ -46,10 +46,14 @@ test('Remark Mdx Files', async () => {
 });
 
 test('Remark Structure', async () => {
-  const content = await fs.readFile(path.resolve(cwd, './fixtures/remark-structure.md'));
-  const result = await remark().use(remarkGfm).use(remarkStructure).process(content);
+  const content = await fs.readFile(path.resolve(cwd, './fixtures/remark-structure.mdx'));
+  const result = await remark()
+    .use(remarkGfm)
+    .use(remarkMdx)
+    .use(remarkStructure, { allowedMdxAttributes: ['title'] })
+    .process(content);
 
-  await expect(result.data.structuredData).toMatchFileSnapshot(
+  await expect(JSON.stringify(result.data.structuredData, null, 2)).toMatchFileSnapshot(
     path.resolve(cwd, './fixtures/remark-structure.output.json'),
   );
 });