Core: Support placeholder() API in llms.txt generation

.changeset/tall-mice-study.md

@@ -0,0 +1,5 @@
+---
+'fumadocs-core': patch
+---
+
+Support `placeholder()` API in llms.txt generation

packages/core/src/mdx-plugins/remark-llms.runtime.ts

@@ -0,0 +1,42 @@
+import type { Awaitable } from '@/types';
+
+export interface PlaceholderData {
+  name: string | null;
+  attributes: Record<string, unknown>;
+  children: string;
+}
+
+const regex = /\0(.+?)\0/gs;
+
+export async function renderPlaceholder(
+  text: string,
+  renderers: Record<string, (data: PlaceholderData) => Awaitable<string>>,
+): Promise<string> {
+  let out = '';
+  let idx = 0;
+
+  for (const match of text.matchAll(regex)) {
+    out += text.slice(idx, match.index);
+    const inner = match[1];
+    try {
+      const data = JSON.parse(inner) as PlaceholderData;
+      const renderer = data.name && renderers[data.name];
+      if (data.children.trim()) {
+        data.children = await renderPlaceholder(data.children, renderers);
+      }
+
+      if (renderer) {
+        out += await renderer(data);
+      } else {
+        out += data.children;
+      }
+    } catch {
+      out += match[0];
+    }
+
+    idx = match.index + match[0].length;
+  }
+  out += text.slice(idx);
+
+  return out;
+}

packages/core/src/mdx-plugins/remark-llms.ts

@@ -1,7 +1,10 @@
 import type { Processor, Transformer } from 'unified';
 import { toMdxExport } from './mdast-utils';
-import type { Heading, Root } from 'mdast';
+import type { Heading, Parents, Root } from 'mdast';
 import { defaultStringifier, type StringifyOptions } from './stringifier';
+import type { MdxJsxFlowElement, MdxJsxTextElement } from 'mdast-util-mdx';
+import type { Info, State } from 'mdast-util-to-markdown';
+import type { PlaceholderData } from './remark-llms.runtime';
 
 export interface LLMsOptions extends StringifyOptions {
   /**
@@ -39,6 +42,11 @@ export interface LLMsOptions extends StringifyOptions {
    * ```
    */
   filterElement?: StringifyOptions['filterElement'];
+
+  /**
+   * @private output in file data
+   */
+  _data?: boolean;
 }
 
 /**
@@ -46,7 +54,7 @@ export interface LLMsOptions extends StringifyOptions {
  */
 export function remarkLLMs(
   this: Processor,
-  { as = '_markdown', headingIds = true, ...stringify }: LLMsOptions = {},
+  { as = '_markdown', headingIds = true, _data = false, ...stringify }: LLMsOptions = {},
 ): Transformer<Root, Root> {
   const stringifier = defaultStringifier({
     ...stringify,
@@ -68,7 +76,28 @@ export function remarkLLMs(
     },
   });
 
-  return (node) => {
-    node.children.unshift(toMdxExport(as, stringifier.call(this, node, undefined)));
+  return (node, file) => {
+    const value = stringifier.call(this, node, undefined);
+    node.children.unshift(toMdxExport(as, value));
+    if (_data) file.data.markdown = value;
   };
 }
+
+export function placeholder(
+  node: MdxJsxTextElement | MdxJsxFlowElement,
+  _parent: Parents | undefined,
+  state: State,
+  info: Info,
+) {
+  const attributes: Record<string, unknown> = {};
+  for (const attr of node.attributes) {
+    if (attr.type === 'mdxJsxExpressionAttribute') continue;
+    attributes[attr.name] = attr.value;
+  }
+
+  return `\0${JSON.stringify({
+    name: node.name,
+    children: state.containerPhrasing(node, info),
+    attributes,
+  } satisfies PlaceholderData)}\0`;
+}

packages/core/src/mdx-plugins/remark-structure.ts

@@ -249,9 +249,9 @@ export function defaultStringifier(config: StringifyOptions): Stringifier {
       },
       ...config.handlers,
     },
-    onStringify(node, ctx) {
+    stringify(node, parent, state, info, ctx) {
       if (node.data?.structuredData) ctx.addContent(...node.data.structuredData.contents);
-      config.onStringify?.(node, ctx);
+      return config.stringify?.(node, parent, state, info, ctx);
     },
   });
 }

packages/core/src/mdx-plugins/stringifier.ts

@@ -1,4 +1,4 @@
-import type { Nodes } from 'mdast';
+import type { Nodes, Parents } from 'mdast';
 import {
   type MdxJsxFlowElement,
   type MdxJsxTextElement,
@@ -6,7 +6,13 @@ import {
   type MdxJsxExpressionAttribute,
   mdxToMarkdown,
 } from 'mdast-util-mdx';
-import { type Handle, type Options, toMarkdown } from 'mdast-util-to-markdown';
+import {
+  type Handle,
+  type Options,
+  toMarkdown,
+  type Info,
+  type State,
+} from 'mdast-util-to-markdown';
 import type { Processor } from 'unified';
 
 declare module 'mdast' {
@@ -65,9 +71,15 @@ export interface StringifyOptions<Context = undefined> extends Options {
   ) => boolean;
 
   /**
-   * run before stringifying any nodes
+   * override default stringifier
    */
-  onStringify?: (node: Nodes, ctx: Context) => void;
+  stringify?: (
+    node: Nodes,
+    parent: Parents | undefined,
+    state: State,
+    info: Info,
+    ctx: Context,
+  ) => string | undefined;
 }
 
 interface CustomRootNode<Context> {
@@ -97,7 +109,7 @@ export function defaultStringifier<Context>(
 
       return true;
     },
-    onStringify,
+    stringify,
     ...customExtension
   } = config;
 
@@ -106,7 +118,10 @@ export function defaultStringifier<Context>(
       let visibility = filterElement(node);
       if (visibility === false) return '';
 
-      if (onStringify) onStringify(node, ctx);
+      if (stringify) {
+        const v = stringify(node, parent, state, info, ctx);
+        if (v) return v;
+      }
 
       const extraInfo = node.data?._stringify;
       if (extraInfo) {

packages/core/test/fixtures/remark-llms.mdx

@@ -0,0 +1,8 @@
+Below is a **MDX** component:
+
+<MyPage title="test">
+  Hello `world`.
+  <Leaf />
+
+  <MyPage />
+</MyPage>

packages/core/test/mdx-plugins.test.ts

@@ -18,6 +18,8 @@ import remarkGfm from 'remark-gfm';
 import { createProcessor } from '@mdx-js/mdx';
 import { remarkSteps } from '@/mdx-plugins/remark-steps';
 import remarkDirective from 'remark-directive';
+import { placeholder, remarkLLMs } from '@/mdx-plugins/remark-llms';
+import { renderPlaceholder } from '@/mdx-plugins/remark-llms.runtime';
 
 const cwd = path.dirname(fileURLToPath(import.meta.url));
 
@@ -166,3 +168,28 @@ test('parse meta strings', () => {
     }
   `);
 });
+
+test('Remark LLMs: placeholder', async () => {
+  const file = path.resolve(cwd, './fixtures/remark-llms.mdx');
+  const content = await fs.readFile(file);
+  const result = await remark()
+    .use(remarkMdx)
+    .use(remarkLLMs, {
+      _data: true,
+      stringify(node, parent, state, info) {
+        if (node.type === 'mdxJsxFlowElement' && node.name === 'MyPage')
+          return placeholder(node, parent, state, info);
+      },
+    })
+    .process(content);
+
+  const markdown = result.data.markdown as string;
+  const rendered = await renderPlaceholder(markdown, {
+    async MyPage({ attributes, children }) {
+      return `title: ${attributes.title}, children: ${children}`;
+    },
+  });
+  await expect(
+    `\`\`\`md\n${markdown}\n\`\`\`\n\n\`\`\`md\n${rendered}\n\`\`\``,
+  ).toMatchFileSnapshot(path.resolve(cwd, './fixtures/remark-llms.output.md'));
+});

packages/openapi/src/ui/base.tsx

@@ -314,3 +314,5 @@ export function createAPIPage(
     return <APIPage {...props} ctx={ctx} />;
   };
 }
+
+export { ClientCodeBlockProvider } from './components/codeblock';