MDX: Compile Meta files into inline JSON objects

.changeset/eight-pets-hide.md

@@ -0,0 +1,5 @@
+---
+'fumadocs-mdx': patch
+---
+
+Compile Meta files into inline JSON objects

apps/docs/content/docs/openapi/meta.json

@@ -1,6 +1,6 @@
 {
   "title": "OpenAPI Example",
   "description": "A demo for Fumadocs OpenAPI",
-  "root": "true",
+  "root": true,
   "icon": "Rocket"
 }

packages/mdx/src/config/define.ts

@@ -1,20 +1,13 @@
-import { type ProcessorOptions } from '@mdx-js/mdx';
 import { type MDXOptions } from '@/utils/build-mdx';
 import { type GlobalConfig } from '@/config/types';
 import { frontmatterSchema, metaSchema } from '@/utils/schema';
 import type { StandardSchemaV1 } from '@standard-schema/spec';
 
-export interface TransformContext {
-  path: string;
-  source: string;
+export type CollectionSchema<Schema extends StandardSchemaV1, Context> =
+  | Schema
+  | ((ctx: Context) => Schema);
 
-  /**
-   * Compile MDX to JavaScript
-   */
-  buildMDX: (source: string, options?: ProcessorOptions) => Promise<string>;
-}
-
-export interface BaseCollection<Schema> {
+export interface BaseCollection {
   /**
    * Directories to scan
    */
@@ -26,20 +19,20 @@ export interface BaseCollection<Schema> {
    * Include all files if not specified
    */
   files?: string[];
-
-  schema?: Schema | ((ctx: TransformContext) => Schema);
 }
 
 export interface MetaCollection<
   Schema extends StandardSchemaV1 = StandardSchemaV1,
-> extends BaseCollection<Schema> {
+> extends BaseCollection {
   type: 'meta';
+
+  schema?: CollectionSchema<Schema, { path: string; source: string }>;
 }
 
 export interface DocCollection<
   Schema extends StandardSchemaV1 = StandardSchemaV1,
   Async extends boolean = boolean,
-> extends BaseCollection<Schema> {
+> extends BaseCollection {
   type: 'doc';
 
   mdxOptions?: MDXOptions;
@@ -48,6 +41,8 @@ export interface DocCollection<
    * Load files with async
    */
   async?: Async;
+
+  schema?: CollectionSchema<Schema, { path: string; source: string }>;
 }
 
 export interface DocsCollection<

packages/mdx/src/loader-mdx.ts

@@ -4,9 +4,8 @@ import grayMatter from 'gray-matter';
 import { type LoaderContext } from 'webpack';
 import { getConfigHash, loadConfig } from '@/utils/config';
 import { buildMDX } from '@/utils/build-mdx';
-import { formatError } from '@/utils/format-error';
 import { getGitTimestamp } from './utils/git-timestamp';
-import type { StandardSchemaV1 } from '@standard-schema/spec';
+import { validate } from '@/utils/schema';
 
 export interface Options {
   /**
@@ -70,38 +69,15 @@ export default async function loader(
     collection?.mdxOptions ?? (await config.getDefaultMDXOptions());
 
   if (collection?.schema) {
-    let schema = collection.schema;
-
-    if (typeof schema === 'function' && !('~standard' in schema)) {
-      schema = schema({
-        async buildMDX(v, options = mdxOptions) {
-          const res = await buildMDX(
-            `${configHash}:${collectionId ?? 'global'}`,
-            v,
-            options,
-          );
-          return String(res.value);
-        },
+    matter.data = (await validate(
+      collection.schema,
+      matter.data,
+      {
         source,
         path: filePath,
-      });
-    }
-
-    if ('~standard' in schema) {
-      const result = await (schema as StandardSchemaV1)['~standard'].validate(
-        matter.data,
-      );
-      if (result.issues) {
-        callback(
-          new Error(
-            formatError(`invalid frontmatter in ${filePath}:`, result.issues),
-          ),
-        );
-        return;
-      }
-
-      matter.data = result.value as Record<string, unknown>;
-    }
+      },
+      `invalid frontmatter in ${filePath}:`,
+    )) as Record<string, unknown>;
   }
 
   let timestamp: number | undefined;

packages/mdx/src/map/file-cache.ts

@@ -0,0 +1,17 @@
+const map = new Map<string, unknown>();
+
+export const fileCache = {
+  read<Data>(namespace: string, path: string) {
+    return map.get(`${namespace}.${path}}`) as Data;
+  },
+  write(namespace: string, path: string, data: unknown) {
+    map.set(`${namespace}.${path}}`, data);
+  },
+  removeCache(path: string) {
+    for (const key of map.keys()) {
+      const keyPath = key.slice(key.indexOf('.') + 1);
+
+      if (keyPath === path) map.delete(key);
+    }
+  },
+};

packages/mdx/src/map/generate.ts

@@ -1,16 +1,28 @@
 import * as path from 'node:path';
+import * as fs from 'node:fs/promises';
 import fg from 'fast-glob';
 import { getTypeFromPath } from '@/utils/get-type-from-path';
 import type { FileInfo } from '@/config/types';
 import { type LoadedConfig } from '@/utils/config';
 import type { DocCollection, MetaCollection } from '@/config';
+import { validate } from '@/utils/schema';
+import { fileCache } from '@/map/file-cache';
+import { readFrontmatter } from '@/utils/read-frontmatter';
+
+async function readFrontmatterWithCache(file: string) {
+  const cached = fileCache.read('read-frontmatter', file);
+  if (cached) return cached;
+
+  const res = await readFrontmatter(file);
+  fileCache.write('read-frontmatter', file, res);
+  return res;
+}
 
 export async function generateJS(
   configPath: string,
   config: LoadedConfig,
   outputPath: string,
   configHash: string,
-  getFrontmatter: (file: string) => unknown | Promise<unknown>,
 ): Promise<string> {
   const outDir = path.dirname(outputPath);
   let asyncInit = false;
@@ -37,7 +49,32 @@ export async function generateJS(
   ) {
     const items = files.map(async (file, i) => {
       config._runtime.files.set(file.absolutePath, collectionName);
-      const importId = `${collectionName}_${collection.type}_${i}`;
+
+      if (collection.type === 'meta') {
+        const cached = fileCache.read<string>('generate-js', file.absolutePath);
+        if (cached) return cached;
+
+        const source = (await fs.readFile(file.absolutePath)).toString();
+        let data = JSON.parse(source);
+
+        if (collection?.schema) {
+          data = await validate(
+            collection.schema,
+            data,
+            {
+              source,
+              path: file.absolutePath,
+            },
+            `invalid data in ${file.absolutePath}:`,
+          );
+        }
+
+        const entry = `{ info: ${JSON.stringify(file)}, data: ${JSON.stringify(data)} }`;
+        fileCache.write('generate-js', file.absolutePath, entry);
+        return entry;
+      }
+
+      const importId = `${collectionName}_${i}`;
       lines.unshift(
         getImportCode({
           type: 'namespace',
@@ -68,11 +105,9 @@ export async function generateJS(
     }
 
     const entries = files.map(async (file) => {
-      const frontmatter = await getFrontmatter(file.absolutePath);
-
       return JSON.stringify({
         info: file,
-        data: frontmatter,
+        data: await readFrontmatterWithCache(file.absolutePath),
       });
     });
 
@@ -83,12 +118,12 @@ export async function generateJS(
     if (collection.type === 'docs') {
       const docs = await getCollectionFiles(collection.docs);
       const metas = await getCollectionFiles(collection.meta);
+      const metaEntries = (await getEntries(k, collection.meta, metas)).join(
+        ', ',
+      );
 
       if (collection.docs.async) {
         const docsEntries = (await getAsyncEntries(docs)).join(', ');
-        const metaEntries = (await getEntries(k, collection.meta, metas)).join(
-          ', ',
-        );
 
         return `export const ${k} = _runtimeAsync.docs<typeof _source.${k}>([${docsEntries}], [${metaEntries}], "${k}", _sourceConfig)`;
       }
@@ -96,9 +131,6 @@ export async function generateJS(
       const docsEntries = (await getEntries(k, collection.docs, docs)).join(
         ', ',
       );
-      const metaEntries = (await getEntries(k, collection.meta, metas)).join(
-        ', ',
-      );
 
       return `export const ${k} = _runtime.docs<typeof _source.${k}>([${docsEntries}], [${metaEntries}])`;
     }

packages/mdx/src/map/index.ts

@@ -2,7 +2,7 @@ import * as path from 'node:path';
 import * as fs from 'node:fs/promises';
 import { getConfigHash, loadConfig } from '@/utils/config';
 import { generateJS } from '@/map/generate';
-import { readFrontmatter } from '@/utils/read-frontmatter';
+import { fileCache } from '@/map/file-cache';
 
 /**
  * Start a MDX server that builds index and manifest files.
@@ -24,35 +24,18 @@ export async function start(
   let config = await loadConfig(configPath, configHash, true);
   const outPath = path.resolve(outDir, `index.ts`);
 
-  const frontmatterCache = new Map<string, unknown>();
-  let hookUpdate = false;
-
-  async function readFrontmatterWithCache(file: string) {
-    hookUpdate = true;
-    const cached = frontmatterCache.get(file);
-    if (cached) return cached;
-
-    const res = await readFrontmatter(file);
-    frontmatterCache.set(file, res);
-    return res;
-  }
-
   async function updateMapFile() {
+    console.time(`[MDX] update map file`);
+
     await fs.writeFile(
       outPath,
-      await generateJS(
-        configPath,
-        config,
-        outPath,
-        configHash,
-        readFrontmatterWithCache,
-      ),
+      await generateJS(configPath, config, outPath, configHash),
     );
+
+    console.timeEnd(`[MDX] update map file`);
   }
 
-  console.time(`[MDX] initialize map file`);
   await updateMapFile();
-  console.timeEnd(`[MDX] initialize map file`);
 
   if (dev) {
     const { watcher } = await import('@/map/watcher');
@@ -64,21 +47,19 @@ export async function start(
 
     instance.on('all', (event, file) => {
       if (typeof file !== 'string') return;
+      const absolutePath = path.resolve(file);
 
       const onUpdate = async () => {
-        const isConfigFile = path.resolve(file) === configPath;
+        const isConfigFile = absolutePath === configPath;
 
         if (isConfigFile) {
           configHash = await getConfigHash(configPath);
           config = await loadConfig(configPath, configHash, true);
         }
 
-        if (isConfigFile || event !== 'change' || hookUpdate) {
-          if (event === 'change') frontmatterCache.delete(file);
+        if (event === 'change') fileCache.removeCache(absolutePath);
 
-          await updateMapFile();
-          console.log('[MDX] Updated map file');
-        }
+        await updateMapFile();
       };
 
       void onUpdate();

packages/mdx/src/postinstall.ts

@@ -2,7 +2,6 @@ import * as path from 'node:path';
 import * as fs from 'node:fs';
 import { findConfigFile, getConfigHash, loadConfig } from '@/utils/config';
 import { generateJS } from '@/map/generate';
-import { readFrontmatter } from '@/utils/read-frontmatter';
 
 export async function postInstall(
   configPath = findConfigFile(),
@@ -19,7 +18,6 @@ export async function postInstall(
       config,
       path.resolve('.source/index.ts'),
       hash,
-      readFrontmatter,
     ),
   );
   console.log('[MDX] types generated');

packages/mdx/src/runtime/index.ts

@@ -26,7 +26,7 @@ export const _runtime: Runtime = {
   meta(files) {
     return files.map((file) => {
       return {
-        ...(file.data.default as object),
+        ...file.data,
         _file: file.info,
       };
     }) as any;

packages/mdx/src/utils/config.ts

@@ -28,13 +28,10 @@ export interface LoadedConfig {
   };
 }
 
-const cache = new Map<
-  string,
-  {
-    hash: string;
-    config: Promise<LoadedConfig>;
-  }
->();
+let cache: {
+  hash: string;
+  config: Promise<LoadedConfig>;
+} | null = null;
 
 async function compileConfig(configPath: string) {
   const { build } = await import('esbuild');
@@ -71,9 +68,8 @@ export async function loadConfig(
   hash: string,
   build = false,
 ): Promise<LoadedConfig> {
-  const cached = cache.get(configPath);
-  if (cached && cached.hash === hash) {
-    return await cached.config;
+  if (cache && cache.hash === hash) {
+    return await cache.config;
   }
 
   if (build) await compileConfig(configPath);
@@ -91,7 +87,7 @@ export async function loadConfig(
     return config;
   });
 
-  cache.set(configPath, { config, hash });
+  cache = { config, hash };
   return await config;
 }
 

packages/mdx/src/utils/format-error.ts

@@ -1,8 +0,0 @@
-import { StandardSchemaV1 } from '@standard-schema/spec';
-
-export function formatError(
-  message: string,
-  issues: readonly StandardSchemaV1.Issue[],
-): string {
-  return `${message}:\n${issues.map((issue) => `  ${issue.path}: ${issue.message}`).join('\n')}`;
-}

packages/mdx/src/utils/schema.ts

@@ -1,4 +1,6 @@
 import { z } from 'zod';
+import type { CollectionSchema } from '@/config';
+import type { StandardSchemaV1 } from '@standard-schema/spec';
 
 export const metaSchema = z.object({
   title: z.string().optional(),
@@ -18,3 +20,35 @@ export const frontmatterSchema = z.object({
   // Fumadocs OpenAPI generated
   _openapi: z.object({}).passthrough().optional(),
 });
+
+export async function validate<Schema extends StandardSchemaV1, Context>(
+  schema: CollectionSchema<Schema, Context>,
+  data: unknown,
+  context: Context,
+  errorMessage: string,
+): Promise<StandardSchemaV1.InferOutput<Schema>> {
+  if (typeof schema === 'function' && !('~standard' in schema)) {
+    schema = schema(context);
+  }
+
+  if ('~standard' in schema) {
+    const result = await (schema as StandardSchemaV1)['~standard'].validate(
+      data,
+    );
+
+    if (result.issues) {
+      throw new Error(formatError(errorMessage, result.issues));
+    }
+
+    return result.value;
+  }
+
+  return data;
+}
+
+export function formatError(
+  message: string,
+  issues: readonly StandardSchemaV1.Issue[],
+): string {
+  return `${message}:\n${issues.map((issue) => `  ${issue.path}: ${issue.message}`).join('\n')}`;
+}

packages/mdx/test/fixtures/index-sync.output.js

@@ -1,6 +1,6 @@
 // @ts-nocheck -- skip type checking
-import * as docs_doc_1 from "./folder/test.mdx?collection=docs&hash=hash"
-import * as docs_doc_0 from "./index.mdx?collection=docs&hash=hash"
+import * as docs_1 from "./folder/test.mdx?collection=docs&hash=hash"
+import * as docs_0 from "./index.mdx?collection=docs&hash=hash"
 import { _runtime } from "fumadocs-mdx"
 import * as _source from "./config"
-export const docs = _runtime.doc<typeof _source.docs>([{ info: {"path":"index.mdx","absolutePath":"$cwd/packages/mdx/test/fixtures/index.mdx"}, data: docs_doc_0 }, { info: {"path":"folder/test.mdx","absolutePath":"$cwd/packages/mdx/test/fixtures/folder/test.mdx"}, data: docs_doc_1 }]);
+export const docs = _runtime.doc<typeof _source.docs>([{ info: {"path":"index.mdx","absolutePath":"$cwd/packages/mdx/test/fixtures/index.mdx"}, data: docs_0 }, { info: {"path":"folder/test.mdx","absolutePath":"$cwd/packages/mdx/test/fixtures/folder/test.mdx"}, data: docs_1 }]);

packages/mdx/test/index.test.ts

@@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url';
 import * as path from 'node:path';
 import { expect, test } from 'vitest';
 import { z } from 'zod';
-import { formatError } from '@/utils/format-error';
+import { formatError } from '@/utils/schema';
 import { generateJS } from '@/map/generate';
 import { defineCollections } from '@/config';
 
@@ -67,7 +67,6 @@ for (const { name, collection } of cases) {
       },
       path.join(file, './fixtures/index-async.output.js'),
       'hash',
-      () => ({}),
     );
 
     await expect(out.replaceAll(process.cwd(), '$cwd')).toMatchFileSnapshot(