Core: add default chinese config for search

.changeset/forty-buckets-cheer.md

@@ -0,0 +1,5 @@
+---
+'fumadocs-core': patch
+---
+
+Add chinese i18n configuration for Orama search if not specified

examples/i18n/app/api/search/route.ts

@@ -1,21 +1,4 @@
 import { source } from '@/lib/source';
 import { createFromSource } from 'fumadocs-core/search/server';
-// @ts-expect-error -- untyped
-import { createTokenizer } from '@orama/tokenizers/mandarin';
-import { stopwords } from '@orama/stopwords/mandarin';
 
-export const { GET } = createFromSource(source, undefined, {
-  localeMap: {
-    // the prop name should be its locale code in your i18n config, (e.g. `cn`)
-    cn: {
-      // options for the language
-      tokenizer: await createTokenizer({
-        stopWords: stopwords,
-      }),
-      search: {
-        threshold: 0,
-        tolerance: 0,
-      },
-    },
-  },
-});
+export const { GET } = createFromSource(source);

packages/core/package.json

@@ -102,6 +102,7 @@
   "devDependencies": {
     "@algolia/client-search": "4.24.0",
     "@mdx-js/mdx": "^3.1.0",
+    "@orama/tokenizers": "^2.1.1",
     "@oramacloud/client": "^2.1.4",
     "@types/estree-jsx": "^1.0.5",
     "@types/hast": "^3.0.4",
@@ -124,6 +125,7 @@
     "vfile": "^6.0.3"
   },
   "peerDependencies": {
+    "@orama/tokenizers": "2.x.x",
     "@oramacloud/client": "1.x.x || 2.x.x",
     "algoliasearch": "4.24.0",
     "next": "14.x.x || 15.x.x",
@@ -131,6 +133,9 @@
     "react-dom": "18.x.x || 19.x.x"
   },
   "peerDependenciesMeta": {
+    "@orama/tokenizers": {
+      "optional": true
+    },
     "@oramacloud/client": {
       "optional": true
     },

packages/core/src/search/orama/create-i18n.ts

@@ -1,4 +1,4 @@
-import { type Language } from '@orama/orama';
+import { type Language, type Tokenizer } from '@orama/orama';
 import {
   type AdvancedIndex,
   type AdvancedOptions,
@@ -37,10 +37,28 @@ type WithLocale<T> = T & {
   locale: string;
 };
 
-function defaultLocaleMap(locale: string): string {
-  const map = STEMMERS;
+async function getTokenizer(
+  locale: string,
+): Promise<{ language: string } | { tokenizer: Tokenizer }> {
+  const mandarin = ['cn', 'zh'];
+  const language =
+    Object.keys(STEMMERS).find((lang) => STEMMERS[lang] === locale) ?? locale;
 
-  return Object.keys(map).find((lang) => map[lang] === locale) ?? locale;
+  if (
+    mandarin.some((code) => locale === code || locale.startsWith(`${code}-`))
+  ) {
+    const { createTokenizer } = await import(
+      '@orama/tokenizers/mandarin' as string
+    );
+
+    return {
+      tokenizer: await createTokenizer(),
+    };
+  }
+
+  return {
+    language,
+  };
 }
 
 async function initSimple(
@@ -59,7 +77,7 @@ async function initSimple(
   for (const locale of options.i18n.languages) {
     const localeIndexes = indexes.filter((index) => index.locale === locale);
     const searchLocale =
-      options.localeMap?.[locale] ?? defaultLocaleMap(locale);
+      options.localeMap?.[locale] ?? (await getTokenizer(locale));
 
     map.set(
       locale,
@@ -96,7 +114,7 @@ async function initAdvanced(
   for (const locale of options.i18n.languages) {
     const localeIndexes = indexes.filter((index) => index.locale === locale);
     const searchLocale =
-      options.localeMap?.[locale] ?? defaultLocaleMap(locale);
+      options.localeMap?.[locale] ?? (await getTokenizer(locale));
 
     map.set(
       locale,

pnpm-lock.yaml

@@ -565,6 +565,9 @@ importers:
       '@mdx-js/mdx':
         specifier: ^3.1.0
         version: 3.1.0([email protected])
+      '@orama/tokenizers':
+        specifier: ^2.1.1
+        version: 2.1.1
       '@oramacloud/client':
         specifier: ^2.1.4
         version: 2.1.4