Docs: Improve i18n section

.changeset/large-donuts-serve.md

@@ -0,0 +1,5 @@
+---
+'fumadocs-mdx': patch
+---
+
+Fix nested references for `<include />`

.changeset/tall-walls-pay.md

@@ -0,0 +1,5 @@
+---
+'fumadocs-core': patch
+---
+
+Support `source.getPageTree()` function

apps/docs/content/docs/headless/page-conventions.mdx

@@ -253,21 +253,4 @@ By default, index pages are not considered as a page "under" the folder, you mus
 
 ## Internationalization
 
-You can create a localized page for specific language by adding `.{locale}` to your file name. Pages can't be language-specific, you must create a page for default
-locale in order to have its localized version.
-
-This works for meta files too, you can add `.{locale}` to the file name like `meta.cn.json`.
-
-> If it's the default language, just leave it empty like `get-started.mdx`. Do
-> not add locale code to file name.
-
-### Example
-
-Assume your default language is `en`.
-
-| Name              |                                           |
-| ----------------- | ----------------------------------------- |
-| file.mdx          | Correct                                   |
-| file.cn.mdx       | Correct                                   |
-| file.en.mdx       | Default locale doesn't need a locale code |
-| components.cn.mdx | Pages can't be language-specific          |
+<include>../../shared/page-conventions.i18n.mdx</include>

apps/docs/content/docs/ui/internationalization.mdx

@@ -6,15 +6,15 @@ icon: Globe
 
 <Callout title='Before you get started'>
 
-    Fumadocs is not a full-powered i18n library, you can also use other libraries like [next-intl](https://github.com/amannn/next-intl).
-    I18n functionality of Fumadocs works without the built-in middleware.
+    Fumadocs is not a full-powered i18n library, it manages only its own components and utilities.
+
+    You can use other libraries like [next-intl](https://github.com/amannn/next-intl) for the rest of your app.
+    Read the [Next.js Docs](https://nextjs.org/docs/app/building-your-application/routing/internationalization) to learn more about implementing I18n in Next.js.
 
 </Callout>
 
 You can setup i18n using Fumadocs CLI or update the configurations manually.
 
-> Read the [Next.js Docs](https://nextjs.org/docs/app/building-your-application/routing/internationalization) to learn more about implementing I18n in Next.js.
-
 ## Auto Setup
 
 Initialize i18n with CLI.
@@ -65,10 +65,11 @@ Create a middleware that redirects users to appropriate locale.
 
 See [Middleware](/docs/headless/internationalization#middleware) for customisable options.
 
+> Note that this is optional, you can also use your own middleware or the one provided by i18n libraries.
+
 ### Routing
 
-Create a dynamic route `/app/[lang]`, and move all special files from `/app` to
-the folder (e.g. `page.tsx`, `layout.tsx`).
+Create a dynamic route `/app/[lang]`, and move all special files (e.g. `page.tsx`, `layout.tsx`) from `/app` to the folder.
 
 A `I18nProvider` is needed for localization.
 Wrap the root provider inside your I18n provider, and provide available languages & translations to it.
@@ -128,57 +129,60 @@ export default async function RootLayout({
 
 ### Source
 
-Update the references to your `source` object to include a locale code:
+For most cases, `source` object is the API to interact with your content source.
+For example, `source.getPage()` allows you to get a page with its slugs.
 
-```ts
+To get the page with appropriate locale, you need to pass the locale.
+
+Update the references to your `source` object in your `page.tsx` and `layout.tsx` files, and include a locale code:
+
+```ts title="layout.tsx"
 import { source } from '@/lib/source';
 
-// get page tree
-source.pageTree[params.lang];
-
-// get page
-source.getPage(params.slug, params.lang);
-
-// get pages
-source.getPages(params.lang);
-```
-
-In code editors, you can click on the variable in `lib/source.ts` to quickly find its references.
-For example, it looks like this for docs layout:
-
-```tsx title="app/[lang]/docs/layout.tsx"
-import { source } from '@/lib/source';
-import { DocsLayout } from 'fumadocs-ui/docs';
-import type { ReactNode } from 'react';
-
 export default async function Layout({
   params,
   children,
 }: {
   params: Promise<{ lang: string }>;
-  children: ReactNode;
+  children: React.ReactNode;
 }) {
-  const pageTree = source.pageTree[(await params).lang];
-
-  return <DocsLayout pageTree={pageTree}>{children}</DocsLayout>;
+  // get page tree
+  source.pageTree; // [!code --]
+  source.pageTree[(await params).lang]; // [!code ++]
 }
 ```
 
-Note that without providing a locale code, it uses your default locale instead.
-You can see [Source API](/docs/headless/source-api) for other usages.
+```ts title="page.tsx"
+import { source } from '@/lib/source';
 
-### Writing Documents
+export default async function Page({
+  params,
+}: {
+  params: Promise<{ lang: string; slug?: string[] }>;
+}) {
+  // get page
+  source.getPage((await params).slug); // [!code --]
+  source.getPage((await params).slug, (await params).lang); // [!code ++]
 
-see [Page Conventions](/docs/ui/page-conventions#internationalization) to learn how to organize your documents.
+  // get pages
+  source.getPages(); // [!code --]
+  source.getPages((await params).lang); // [!code ++]
+}
+```
+
+<Callout title="Note">
+  Without providing a locale code, it uses your default locale instead. You can
+  see [Source API](/docs/headless/source-api#output) for other usages.
+</Callout>
 
 ### Search
 
 Configure i18n on your search solution.
 
 - Built-in Search (Orama):
-  - For `createFromSource` and most languages, no further changes are needed.
+  - For most languages, no further changes are needed.
   - For special languages like Chinese & Japanese, they require additional config.
-    See [Orama I18n](/docs/headless/search/orama#special-languages) guide.
+    See [Orama Special Languages](/docs/headless/search/orama#special-languages) guide.
 - Cloud Solutions (e.g. Algolia):
   - They usually have official support for multilingual.
 
@@ -194,13 +198,17 @@ export const baseOptions: BaseLayoutProps = {
 };
 ```
 
+## Writing Documents
+
+<include>../../shared/page-conventions.i18n.mdx</include>
+
 ## Navigation
 
 Fumadocs will only handle i18n navigation for its own layouts (e.g. sidebar components),
 you have to re-create components like `<Link>` from `next/link` and `useParams` hook to auto attend the locale to `href`.
 
 In addition, the [`fumadocs-core/dynamic-link`](/docs/headless/components/link#dynamic-hrefs) component supports linking to dynamic hrefs, including the locale prefix.
-For Markdown/MDX content, you may use it instead of the default anchor (`a`) component:
+For Markdown/MDX content, you may use it instead of the default Link component:
 
 ```mdx title="content.mdx"
 import { DynamicLink } from 'fumadocs-core/dynamic-link';

apps/docs/content/shared/page-conventions.i18n.mdx

@@ -0,0 +1,18 @@
+You can create a localized page for specific language by adding `.{locale}` to your file name, like `page.fr.md` or `page.cn.md`.
+Pages can't be language-specific, you must create a page for the default locale.
+
+This works for meta files too, you can add `.{locale}` to the file name like `meta.cn.json`.
+
+> If it's the default language, just leave it empty like `get-started.mdx`. Do
+> not add locale code to file name.
+
+### Example
+
+Assume your default language is `en`.
+
+| Name              |                                           |
+| ----------------- | ----------------------------------------- |
+| file.mdx          | Correct                                   |
+| file.cn.mdx       | Correct                                   |
+| file.en.mdx       | Default locale doesn't need a locale code |
+| components.cn.mdx | Pages can't be language-specific          |

examples/i18n/app/api/search/route.ts

@@ -1,4 +1,18 @@
 import { source } from '@/lib/source';
 import { createFromSource } from 'fumadocs-core/search/server';
+// @ts-expect-error -- untyped
+import { createTokenizer } from '@orama/tokenizers/mandarin';
 
-export const { GET } = createFromSource(source);
+export const { GET, search } = createFromSource(source, undefined, {
+  localeMap: {
+    // the prop name should be its locale code in your i18n config, (e.g. `cn`)
+    cn: {
+      // options for the language
+      tokenizer: await createTokenizer(),
+      search: {
+        threshold: 0,
+        tolerance: 0,
+      },
+    },
+  },
+});

packages/core/src/source/loader.ts

@@ -82,6 +82,8 @@ export interface LoaderOutput<Config extends LoaderConfig> {
     ? Record<string, PageTree.Root>
     : PageTree.Root;
 
+  getPageTree(locale?: string): PageTree.Root;
+
   _i18n?: I18nConfig;
 
   /**
@@ -277,6 +279,15 @@ function createOutput(options: LoaderOptions): LoaderOutput<LoaderConfig> {
 
       return walker.pathToMeta.get(node.$ref.metaFile);
     },
+    getPageTree(locale) {
+      if (options.i18n) {
+        return pageTree[
+          (locale ?? options.i18n.defaultLanguage) as keyof typeof pageTree
+        ] as PageTree.Root;
+      }
+
+      return pageTree as PageTree.Root;
+    },
     getNodePage(node) {
       if (!node.$ref?.file) return;
 

packages/mdx/src/mdx-plugins/remark-include.ts

@@ -4,9 +4,17 @@ import type { Literal, Root } from 'mdast';
 import * as path from 'node:path';
 import * as fs from 'node:fs/promises';
 import matter from 'gray-matter';
+import type { CompilerOptions } from '@/utils/build-mdx';
 
 export function remarkInclude(this: Processor): Transformer<Root, Root> {
-  return async (tree, file) => {
+  const TagName = 'include';
+
+  async function update(
+    tree: Root,
+    file: string,
+    processor: Processor,
+    compiler?: CompilerOptions,
+  ) {
     const queue: Promise<void>[] = [];
 
     visit(tree, ['mdxJsxFlowElement', 'paragraph'] as const, (node) => {
@@ -18,32 +26,42 @@ export function remarkInclude(this: Processor): Transformer<Root, Root> {
 
         if (
           open.type === 'html' &&
-          open.value === '<include>' &&
+          open.value === `<${TagName}>` &&
           content.type === 'text' &&
           closure.type === 'html' &&
-          closure.value === '</include>'
+          closure.value === `</${TagName}>`
         ) {
           specifier = content.value.trim();
         }
+      } else if (node.type === 'paragraph' && node.children.length === 1) {
+        const child = node.children[0];
+
+        if (child.type === 'mdxJsxTextElement' && child.name === TagName) {
+          const text = child.children.at(0);
+
+          if (text && text.type === 'text') {
+            specifier = text.value;
+          }
+        }
       }
 
-      if (node.type === 'mdxJsxFlowElement' && node.name === 'include') {
+      if (node.type === 'mdxJsxFlowElement' && node.name === TagName) {
         const child = node.children.at(0) as Literal | undefined;
 
-        if (!child || child.type !== 'text') return;
-        specifier = child.value;
+        if (child && child.type === 'text') {
+          specifier = child.value;
+        }
       }
 
-      if (!specifier) return 'skip';
-      const targetPath = path.resolve(path.dirname(file.path), specifier);
+      if (!specifier) return;
+      const targetPath = path.resolve(path.dirname(file), specifier);
 
       queue.push(
-        fs.readFile(targetPath).then((content) => {
-          const parsed = this.parse(matter(content).content);
+        fs.readFile(targetPath).then(async (content) => {
+          const parsed = processor.parse(matter(content).content);
 
-          if (file.data._compiler) {
-            file.data._compiler.addDependency(targetPath);
-          }
+          compiler?.addDependency(targetPath);
+          await update(parsed as Root, targetPath, processor, compiler);
           Object.assign(node, parsed);
         }),
       );
@@ -52,5 +70,9 @@ export function remarkInclude(this: Processor): Transformer<Root, Root> {
     });
 
     await Promise.all(queue);
+  }
+
+  return async (tree, file) => {
+    await update(tree, file.path, this, file.data._compiler);
   };
 }

packages/mdx/src/utils/build-mdx.ts

@@ -27,7 +27,7 @@ export interface MDXOptions extends ProcessorOptions {
   _compiler?: CompilerOptions;
 }
 
-interface CompilerOptions {
+export interface CompilerOptions {
   addDependency: (file: string) => void;
 }