MDX: Always transform meta files in collection

This commit is contained in:
Fuma Nama
2025-10-31 23:24:44 +08:00
parent 27fc4ed9db
commit 61b90c8b61
20 changed files with 320 additions and 209 deletions
+5
View File
@@ -0,0 +1,5 @@
---
'fumadocs-mdx': patch
---
Always transform meta files in collection, this includes runtime loaders like Node.js and Bun.
+1 -1
View File
@@ -1,7 +1,7 @@
module.exports = function loader(code) {
const callback = this.async();
import('./dist/webpack/index.js').then((mod) =>
import('./dist/webpack/mdx.js').then((mod) =>
mod.default.call(this, code, callback),
);
};
+7
View File
@@ -0,0 +1,7 @@
module.exports = function loader(code) {
const callback = this.async();
import('./dist/webpack/meta.js').then((mod) =>
mod.default.call(this, code, callback),
);
};
+4 -1
View File
@@ -13,6 +13,7 @@
"type": "module",
"exports": {
"./loader-mdx": "./loader-mdx.cjs",
"./loader-meta": "./loader-meta.cjs",
"./config": {
"import": "./dist/config/index.js",
"types": "./dist/config/index.d.ts",
@@ -71,7 +72,8 @@
"bin": "./dist/bin.js",
"files": [
"dist/*",
"loader-mdx.cjs"
"loader-mdx.cjs",
"loader-meta.cjs"
],
"scripts": {
"build": "tsup",
@@ -108,6 +110,7 @@
"@types/node": "^24.9.2",
"@types/picomatch": "^4.0.2",
"@types/react": "^19.2.2",
"astro": "^5.6.1",
"eslint-config-custom": "workspace:*",
"fumadocs-core": "workspace:*",
"mdast-util-directive": "^3.1.0",
+15 -37
View File
@@ -1,10 +1,12 @@
import type { BunPlugin } from 'bun';
import { createMdxLoader } from '@/loaders/mdx';
import { buildConfig } from '@/config/build';
import { parse } from 'node:querystring';
import { pathToFileURL } from 'node:url';
import { type CoreOptions, createCore, findConfigFile } from '@/core';
import { createIntegratedConfigLoader } from '@/loaders/config';
import { createMetaLoader } from '@/loaders/meta';
import { toBun } from '@/loaders/adapter';
import { mdxLoaderGlob, metaLoaderGlob } from '@/loaders';
export type MdxPluginOptions = Partial<CoreOptions>;
@@ -15,45 +17,21 @@ export function createMdxPlugin(options: MdxPluginOptions = {}): BunPlugin {
configPath = findConfigFile(),
} = options;
async function getMdxLoader() {
const importPath = pathToFileURL(configPath).href;
const core = await createCore({
environment,
outDir,
configPath,
}).init({
config: buildConfig(await import(importPath)),
});
return createMdxLoader(createIntegratedConfigLoader(core));
}
return {
name: 'bun-plugin-fumadocs-mdx',
setup(build) {
const mdxLoader = getMdxLoader();
build.onLoad({ filter: /\.mdx(\?.+?)?$/ }, async (args) => {
const [filePath, query] = args.path.split('?', 2);
const content = await Bun.file(filePath).text();
const result = await (
await mdxLoader
)({
source: content,
query: parse(query),
filePath,
development: false,
compiler: {
addDependency() {},
},
});
return {
contents: result.code,
loader: 'js',
};
async setup(build) {
const importPath = pathToFileURL(configPath).href;
const core = await createCore({
environment,
outDir,
configPath,
}).init({
config: buildConfig(await import(importPath)),
});
const configLoader = createIntegratedConfigLoader(core);
toBun(createMdxLoader(configLoader), mdxLoaderGlob)(build);
toBun(createMetaLoader(configLoader), metaLoaderGlob)(build);
},
};
}
+43 -17
View File
@@ -23,19 +23,13 @@ export interface LoaderOutput {
map?: unknown;
}
export type Loader = (input: LoaderInput) => Promise<LoaderOutput>;
export type Loader = (input: LoaderInput) => Promise<LoaderOutput | null>;
export function toNode(
loader: Loader,
filterByPath: (filePath: string) => boolean,
): LoadHook {
export function toNode(loader: Loader, test: RegExp): LoadHook {
return async (url, _context, nextLoad): Promise<LoadFnOutput> => {
if (!url.startsWith('file:///')) return nextLoad(url);
const parsedUrl = new URL(url);
const filePath = fileURLToPath(parsedUrl);
if (filterByPath(filePath)) {
if (url.startsWith('file:///') && test.test(url)) {
const parsedUrl = new URL(url);
const filePath = fileURLToPath(parsedUrl);
const source = (await fs.readFile(filePath)).toString();
const result = await loader({
@@ -48,11 +42,13 @@ export function toNode(
},
});
return {
source: result.code,
format: 'module',
shortCircuit: true,
};
if (result) {
return {
source: result.code,
format: 'module',
shortCircuit: true,
};
}
}
return nextLoad(url);
@@ -80,6 +76,7 @@ export function toVite(loader: Loader): ViteLoader {
},
});
if (result === null) return null;
return {
code: result.code,
map: result.map as SourceMap,
@@ -104,7 +101,11 @@ export function toWebpack(loader: Loader): WebpackLoader {
compiler: this,
});
callback(undefined, result.code, result.map as string);
if (result === null) {
callback(undefined, source);
} else {
callback(undefined, result.code, result.map as string);
}
} catch (error) {
if (error instanceof ValidationError) {
return callback(new Error(error.toStringFormatted()));
@@ -118,3 +119,28 @@ export function toWebpack(loader: Loader): WebpackLoader {
}
};
}
export function toBun(loader: Loader, test: RegExp) {
return (build: Bun.PluginBuilder) => {
build.onLoad({ filter: test }, async (args) => {
const [filePath, query] = args.path.split('?', 2);
const content = await Bun.file(filePath).text();
const result = await loader({
source: content,
query: parse(query),
filePath,
development: false,
compiler: {
addDependency() {},
},
});
if (result === null) return;
return {
contents: result.code,
loader: 'js',
};
});
};
}
+2
View File
@@ -0,0 +1,2 @@
export const metaLoaderGlob = /\.(json|yaml)(\?.+?)?$/;
export const mdxLoaderGlob = /\.mdx?(\?.+?)?$/;
+77
View File
@@ -0,0 +1,77 @@
import type { Loader } from '@/loaders/adapter';
import type { ConfigLoader } from '@/loaders/config';
import { dump, load } from 'js-yaml';
import { validate } from '@/utils/validation';
import { z } from 'zod';
const querySchema = z
.object({
collection: z.string().optional(),
})
.loose();
/**
* load meta files, fallback to bundler's built-in plugins when ?collection is unspecified.
*/
export function createMetaLoader(
configLoader: ConfigLoader,
resolve: {
json?: 'json' | 'js';
yaml?: 'yaml' | 'js';
} = {},
): Loader {
const { json: resolveJson = 'js', yaml: resolveYaml = 'js' } = resolve;
return async ({ filePath, query, source }) => {
const isJson = filePath.endsWith('.json');
const parsed = querySchema.parse(query);
const collection = parsed.collection
? (await configLoader.getConfig()).getCollection(parsed.collection)
: undefined;
if (!collection) return null;
let data: unknown;
try {
data = isJson ? JSON.parse(source) : load(source);
} catch (e) {
throw new Error(`invalid data in ${filePath}`, { cause: e });
}
let schema;
switch (collection?.type) {
case 'meta':
schema = collection.schema;
break;
case 'docs':
schema = collection.meta.schema;
break;
}
if (schema) {
data = await validate(
schema,
data,
{ path: filePath, source },
`invalid data in ${filePath}`,
);
}
let code: string;
if (isJson) {
code =
resolveJson === 'json'
? JSON.stringify(data)
: `export default ${JSON.stringify(data)}`;
} else {
code =
resolveYaml === 'yaml'
? dump(data)
: `export default ${JSON.stringify(data)}`;
}
return {
code,
map: null,
};
};
}
+6 -5
View File
@@ -1,6 +1,6 @@
import type { NextConfig } from 'next';
import type { Configuration } from 'webpack';
import { type Options as MDXLoaderOptions } from '@/webpack';
import type { WebpackLoaderOptions } from '@/webpack';
import type {
TurbopackLoaderOptions,
TurbopackOptions,
@@ -11,6 +11,7 @@ import { removeFileCache } from '@/next/file-cache';
import { ValidationError } from '@/utils/validation';
import next from '@/plugins/next';
import { type Core, createCore, findConfigFile } from '@/core';
import { mdxLoaderGlob } from '@/loaders';
export interface CreateMDXOptions {
/**
@@ -39,7 +40,7 @@ export function createMDX(createOptions: CreateMDXOptions = {}) {
}
return (nextConfig: NextConfig = {}): NextConfig => {
const mdxLoaderOptions: MDXLoaderOptions = {
const loaderOptions: WebpackLoaderOptions = {
...options,
isDev,
};
@@ -52,7 +53,7 @@ export function createMDX(createOptions: CreateMDXOptions = {}) {
loaders: [
{
loader: 'fumadocs-mdx/loader-mdx',
options: mdxLoaderOptions as unknown as TurbopackLoaderOptions,
options: loaderOptions as unknown as TurbopackLoaderOptions,
},
],
as: '*.js',
@@ -71,12 +72,12 @@ export function createMDX(createOptions: CreateMDXOptions = {}) {
config.module.rules ||= [];
config.module.rules.push({
test: /\.mdx?$/,
test: mdxLoaderGlob,
use: [
options.defaultLoaders.babel,
{
loader: 'fumadocs-mdx/loader-mdx',
options: mdxLoaderOptions,
options: loaderOptions,
},
],
});
+17 -10
View File
@@ -2,6 +2,9 @@ import { createCore, findConfigFile } from '@/core';
import { createMdxLoader } from '@/loaders/mdx';
import { toNode } from '@/loaders/adapter';
import { createStandaloneConfigLoader } from '@/loaders/config';
import type { LoadHook } from 'node:module';
import { createMetaLoader } from '@/loaders/meta';
import { mdxLoaderGlob, metaLoaderGlob } from '@/loaders';
const core = createCore({
environment: 'node',
@@ -9,13 +12,17 @@ const core = createCore({
outDir: '.source',
});
export const load = toNode(
createMdxLoader(
createStandaloneConfigLoader({
core,
buildConfig: true,
mode: 'production',
}),
),
(filePath) => filePath.endsWith('.md') || filePath.endsWith('.mdx'),
);
const configLoader = createStandaloneConfigLoader({
core,
buildConfig: true,
mode: 'production',
});
const mdxLoader = toNode(createMdxLoader(configLoader), mdxLoaderGlob);
const metaLoader = toNode(createMetaLoader(configLoader), metaLoaderGlob);
export const load: LoadHook = (url, context, nextLoad) => {
return mdxLoader(url, context, (v, ctx) =>
metaLoader(v, { ...context, ...ctx }, nextLoad),
);
};
+19 -18
View File
@@ -5,10 +5,8 @@ import type {
LoadedConfig,
MetaCollectionItem,
} from '@/config/build';
import type { MetaCollection } from '@/config';
import { validate } from '@/utils/validation';
import { readFileWithCache } from '@/next/file-cache';
import { load } from 'js-yaml';
import { getGitTimestamp } from '@/utils/git-timestamp';
import { fumaMatter } from '@/utils/fuma-matter';
import {
@@ -18,6 +16,7 @@ import {
} from '@/utils/import-formatter';
import type { FileInfo } from '@/runtime/shared';
import type { Plugin } from '@/core';
import { load } from 'js-yaml';
export default function next(): Plugin {
let config: LoadedConfig;
@@ -32,7 +31,8 @@ export default function next(): Plugin {
shouldEmitOnChange = config.collectionList.some((collection) => {
return (
(collection.type === 'doc' && collection.async) ||
(collection.type === 'docs' && collection.docs.async)
collection.type === 'docs' ||
collection.type === 'meta'
);
});
},
@@ -64,7 +64,6 @@ export async function indexFile(
configPath: string,
config: LoadedConfig,
importPath: ImportPathConfig,
configHash: string | false = false,
): Promise<string> {
let asyncInit = false;
const lines: string[] = [
@@ -80,13 +79,10 @@ export async function indexFile(
}),
];
async function getDocEntries(collectionName: string, files: FileInfo[]) {
const items = files.map(async (file, i) => {
const importId = `${collectionName}_${i}`;
const params = [`collection=${collectionName}`];
if (configHash) {
params.push(`hash=${configHash}`);
}
function getDocEntries(collection: DocCollectionItem, files: FileInfo[]) {
return files.map((file, i) => {
const importId = `d_${collection.name}_${i}`;
const params = [`collection=${collection.name}`];
lines.unshift(
getImportCode({
@@ -98,11 +94,12 @@ export async function indexFile(
return `{ info: ${JSON.stringify(file)}, data: ${importId} }`;
});
return Promise.all(items);
}
async function getMetaEntries(collection: MetaCollection, files: FileInfo[]) {
async function getMetaEntries(
collection: MetaCollectionItem,
files: FileInfo[],
) {
const items = files.map(async (file) => {
const source = await readFileWithCache(file.fullPath).catch(() => '');
let data =
@@ -202,18 +199,22 @@ export async function indexFile(
return `export const ${k} = _runtimeAsync.docs<typeof _source.${k}>([${docsEntries}], [${metaEntries}], "${k}", _sourceConfig)`;
}
const docsEntries = (await getDocEntries(k, docs)).join(', ');
const docsEntries = getDocEntries(collection.docs, docs).join(', ');
return `export const ${k} = _runtime.docs<typeof _source.${k}>([${docsEntries}], [${metaEntries}])`;
}
const files = await globCollectionFiles(collection);
if (collection.type === 'doc' && collection.async) {
if (collection.type === 'meta') {
return `export const ${k} = _runtime.meta<typeof _source.${k}>([${(await getMetaEntries(collection, files)).join(', ')}]);`;
}
if (collection.async) {
return `export const ${k} = _runtimeAsync.doc<typeof _source.${k}>([${(await getAsyncEntries(collection, files)).join(', ')}], "${k}", _sourceConfig)`;
}
return `export const ${k} = _runtime.${collection.type}<typeof _source.${k}>([${(await getDocEntries(k, files)).join(', ')}]);`;
return `export const ${k} = _runtime.doc<typeof _source.${k}>([${getDocEntries(collection, files).join(', ')}]);`;
});
const resolvedDeclares = await Promise.all(declares);
@@ -255,7 +256,7 @@ async function globCollectionFiles(
});
for (const item of result) {
if (collection.isFileSupported(item)) continue;
if (!collection.isFileSupported(item)) continue;
const fullPath = path.join(dir, item);
files.set(fullPath, {
+19 -64
View File
@@ -1,20 +1,14 @@
import {
mergeConfig,
type Plugin,
type TransformResult,
type UserConfig,
} from 'vite';
import { mergeConfig, type Plugin, type UserConfig } from 'vite';
import { buildConfig } from '@/config/build';
import { parse } from 'node:querystring';
import { validate, ValidationError } from '@/utils/validation';
import * as path from 'node:path';
import { load } from 'js-yaml';
import { ValidationError } from '@/utils/validation';
import { createMdxLoader } from '@/loaders/mdx';
import { toVite } from '@/loaders/adapter';
import vite, { type IndexFileOptions } from '@/plugins/vite';
import type { FSWatcher } from 'chokidar';
import { createCore, findConfigFile } from '@/core';
import { createIntegratedConfigLoader } from '@/loaders/config';
import { createMetaLoader } from '@/loaders/meta';
import { mdxLoaderGlob, metaLoaderGlob } from '@/loaders';
const FumadocsDeps = ['fumadocs-core', 'fumadocs-ui', 'fumadocs-openapi'];
@@ -54,54 +48,14 @@ export default async function mdx(
const core = await createViteCore(options).init({
config: buildConfig(config),
});
const mdxLoader = toVite(createMdxLoader(createIntegratedConfigLoader(core)));
async function transformMeta(
path: string,
query: string,
value: string,
): Promise<TransformResult | null> {
const isJson = path.endsWith('.json');
const parsed = parse(query) as {
collection?: string;
};
const collection = parsed.collection
? core.getConfig().getCollection(parsed.collection)
: undefined;
if (!collection) return null;
let schema;
switch (collection.type) {
case 'meta':
schema = collection.schema;
break;
case 'docs':
schema = collection.meta.schema;
break;
}
if (!schema) return null;
let data;
try {
data = isJson ? JSON.parse(value) : load(value);
} catch {
return null;
}
const out = await validate(
schema,
data,
{ path, source: value },
`invalid data in ${path}`,
);
return {
code: isJson
? JSON.stringify(out)
: `export default ${JSON.stringify(out)}`,
map: null,
};
}
const configLoader = createIntegratedConfigLoader(core);
const mdxLoader = toVite(createMdxLoader(configLoader));
const metaLoader = toVite(
createMetaLoader(configLoader, {
// vite has built-in plugin for JSON files
json: 'json',
}),
);
return {
name: 'fumadocs-mdx',
@@ -129,15 +83,16 @@ export default async function mdx(
});
},
async transform(value, id) {
const [file, query = ''] = id.split('?');
const ext = path.extname(file);
try {
if (['.yaml', '.json'].includes(ext))
return await transformMeta(file, query, value);
if (metaLoaderGlob.test(id)) {
const [file, query = ''] = id.split('?', 2);
return await metaLoader.call(this, file, query, value);
}
if (['.md', '.mdx'].includes(ext))
if (mdxLoaderGlob.test(id)) {
const [file, query = ''] = id.split('?', 2);
return await mdxLoader.call(this, file, query, value);
}
} catch (e) {
if (e instanceof ValidationError) {
throw new Error(e.toStringFormatted());
+1 -44
View File
@@ -1,48 +1,5 @@
import { type LoaderContext } from 'webpack';
import { createMdxLoader } from '@/loaders/mdx';
import { toWebpack, type WebpackLoader } from '@/loaders/adapter';
import { createCore } from '@/core';
import { createStandaloneConfigLoader } from '@/loaders/config';
export interface Options {
export interface WebpackLoaderOptions {
configPath: string;
outDir: string;
isDev: boolean;
}
let instance: WebpackLoader | undefined;
/**
* Load MDX/markdown files
*
* it supports frontmatter by parsing and injecting the data in `vfile.data.frontmatter`
*/
export default async function loader(
this: LoaderContext<Options>,
source: string,
callback: LoaderContext<Options>['callback'],
): Promise<void> {
const { isDev, outDir, configPath } = this.getOptions();
this.cacheable(true);
this.addDependency(configPath);
if (!instance) {
const core = createCore({
environment: 'webpack',
outDir,
configPath,
});
instance = toWebpack(
createMdxLoader(
createStandaloneConfigLoader({
core,
buildConfig: false,
mode: isDev ? 'dev' : 'production',
}),
),
);
}
await instance.call(this, source, callback);
}
+38
View File
@@ -0,0 +1,38 @@
import { type LoaderContext } from 'webpack';
import { createMdxLoader } from '@/loaders/mdx';
import { toWebpack, type WebpackLoader } from '@/loaders/adapter';
import { createCore } from '@/core';
import { createStandaloneConfigLoader } from '@/loaders/config';
import type { WebpackLoaderOptions } from '@/webpack';
let instance: WebpackLoader | undefined;
export default async function loader(
this: LoaderContext<WebpackLoaderOptions>,
source: string,
callback: LoaderContext<WebpackLoaderOptions>['callback'],
): Promise<void> {
const { isDev, outDir, configPath } = this.getOptions();
this.cacheable(true);
this.addDependency(configPath);
if (!instance) {
const core = createCore({
environment: 'webpack',
outDir,
configPath,
});
instance = toWebpack(
createMdxLoader(
createStandaloneConfigLoader({
core,
buildConfig: false,
mode: isDev ? 'dev' : 'production',
}),
),
);
}
await instance.call(this, source, callback);
}
+45
View File
@@ -0,0 +1,45 @@
import { type LoaderContext } from 'webpack';
import { toWebpack, type WebpackLoader } from '@/loaders/adapter';
import { createCore } from '@/core';
import { createStandaloneConfigLoader } from '@/loaders/config';
import { createMetaLoader } from '@/loaders/meta';
import type { WebpackLoaderOptions } from '@/webpack';
let instance: WebpackLoader | undefined;
// TODO: currently unused, it's too intrusive and consuming for Turbopack
// It can be invoked for many JSON files in multiple processes, hence repeatedly loading the config file
// the cost of loading the loader itself is higher than transforming the actual content, hence parallelization isn't very useful.
export default async function loader(
this: LoaderContext<WebpackLoaderOptions>,
source: string,
callback: LoaderContext<WebpackLoaderOptions>['callback'],
): Promise<void> {
const { isDev, outDir, configPath } = this.getOptions();
this.cacheable(true);
this.addDependency(configPath);
if (!instance) {
const core = createCore({
environment: 'webpack',
outDir,
configPath,
});
instance = toWebpack(
createMetaLoader(
createStandaloneConfigLoader({
core,
buildConfig: false,
mode: isDev ? 'dev' : 'production',
}),
{
json: 'json',
yaml: 'yaml',
},
),
);
}
await instance.call(this, source, callback);
}
+1
View File
@@ -0,0 +1 @@
{}
+4
View File
@@ -0,0 +1,4 @@
// @ts-nocheck -- skip type checking
import { _runtime } from "fumadocs-mdx/runtime/next"
import * as _source from "./config"
export const docs = _runtime.meta<typeof _source.docs>([{"info":{"path":"meta.json","fullPath":"packages/mdx/test/fixtures/generate-index/meta.json"},"data":{}}]);
+3 -3
View File
@@ -1,6 +1,6 @@
// @ts-nocheck -- skip type checking
import * as docs_1 from "./generate-index/folder/test.mdx?collection=docs&hash=hash"
import * as docs_0 from "./generate-index/index.mdx?collection=docs&hash=hash"
import * as d_docs_1 from "./generate-index/folder/test.mdx?collection=docs"
import * as d_docs_0 from "./generate-index/index.mdx?collection=docs"
import { _runtime } from "fumadocs-mdx/runtime/next"
import * as _source from "./config"
export const docs = _runtime.doc<typeof _source.docs>([{ info: {"path":"index.mdx","fullPath":"packages/mdx/test/fixtures/generate-index/index.mdx"}, data: docs_0 }, { info: {"path":"folder/test.mdx","fullPath":"packages/mdx/test/fixtures/generate-index/folder/test.mdx"}, data: docs_1 }]);
export const docs = _runtime.doc<typeof _source.docs>([{ info: {"path":"index.mdx","fullPath":"packages/mdx/test/fixtures/generate-index/index.mdx"}, data: d_docs_0 }, { info: {"path":"folder/test.mdx","fullPath":"packages/mdx/test/fixtures/generate-index/folder/test.mdx"}, data: d_docs_1 }]);
+11 -8
View File
@@ -6,6 +6,7 @@ import { ValidationError } from '@/utils/validation';
import { defineCollections } from '@/config';
import { fumaMatter } from '@/utils/fuma-matter';
import { indexFile } from '@/plugins/next';
import { buildConfig } from '@/config/build';
test('format errors', async () => {
const schema = z.object({
@@ -50,6 +51,13 @@ const cases = [
dir: path.join(baseDir, './fixtures/generate-index'),
}),
},
{
name: 'sync-meta',
collection: defineCollections({
type: 'meta',
dir: path.join(baseDir, './fixtures/generate-index'),
}),
},
{
name: 'async',
collection: defineCollections({
@@ -64,17 +72,12 @@ for (const { name, collection } of cases) {
test(`generate JS index file: ${name}`, async () => {
const out = await indexFile(
path.join(baseDir, './fixtures/config.ts'),
{
// @ts-expect-error -- test file
_runtime: {
files: new Map(),
},
collections: new Map([['docs', collection]]),
},
buildConfig({
docs: collection,
}),
{
relativeTo: path.join(baseDir, './fixtures'),
},
'hash',
);
await expect(out.replaceAll(process.cwd(), '$cwd')).toMatchFileSnapshot(
+2 -1
View File
@@ -3,7 +3,8 @@ import { defineConfig } from 'tsup';
export default defineConfig({
entry: [
'./src/{index,bin}.ts',
'./src/{config,webpack,next,vite,bun}/index.ts',
'./src/{config,next,vite,bun}/index.ts',
'./src/webpack/{mdx,meta}.ts',
'./src/node/loader.ts',
'./src/runtime/next/{index,async}.ts',
'./src/runtime/vite/{browser,server}.ts',