Core: Migrate to Orama

.changeset/six-tips-doubt.md

@@ -0,0 +1,5 @@
+---
+'fumadocs-core': minor
+---
+
+Replace flexearch with Orama

.changeset/smart-tables-work.md

@@ -0,0 +1,5 @@
+---
+'fumadocs-core': minor
+---
+
+Support static search (without server)

apps/docs/app/(home)/page.tsx

@@ -77,7 +77,6 @@ export default function Page(): React.ReactElement {
             <UwuHero />
           </div>
           <Feedback />
-          <Why />
           <div className="container border-x border-t py-16 md:py-24">
             <Terminal className="mx-auto mb-2 size-8 text-muted-foreground" />
             <h2 className="text-center text-2xl font-semibold sm:text-3xl">
@@ -87,7 +86,7 @@ export default function Page(): React.ReactElement {
             </h2>
           </div>
           <Introduction />
-          <Highlights />
+          <Why />
           <div
             className="container relative overflow-hidden border-x border-t py-16 sm:py-24"
             style={{
@@ -102,6 +101,7 @@ export default function Page(): React.ReactElement {
             </h2>
           </div>
           <Features />
+          <Highlights />
           <Contributing />
           <End />
         </div>
@@ -160,7 +160,7 @@ function Why(): React.ReactElement {
         <p className="max-md:hidden">with Fumadocs</p>
       </h2>
       <p className="mt-4 text-foreground/50">
-        Fumadocs offers a complete toolchain to maintain your docs.
+        Fumadocs offers a complete toolchain to build and maintain your docs.
       </p>
       <Accordions
         type="single"

examples/content-collections/tsconfig.json

@@ -10,7 +10,7 @@
     "noEmit": true,
     "esModuleInterop": true,
     "module": "esnext",
-    "moduleResolution": "node",
+    "moduleResolution": "bundler",
     "resolveJsonModule": true,
     "isolatedModules": true,
     "jsx": "preserve",

examples/i18n/tsconfig.json

@@ -10,7 +10,7 @@
     "noEmit": true,
     "esModuleInterop": true,
     "module": "esnext",
-    "moduleResolution": "node",
+    "moduleResolution": "bundler",
     "resolveJsonModule": true,
     "isolatedModules": true,
     "jsx": "preserve",

examples/next-mdx/app/static.json/route.ts

@@ -0,0 +1,23 @@
+import { NextResponse } from 'next/server';
+import { type DocumentRecord } from 'fumadocs-core/search-algolia/server';
+import { source } from '@/lib/source';
+
+export const revalidate = false;
+
+export function GET(): Response {
+  const results: DocumentRecord[] = [];
+  const pages = source.getPages();
+
+  for (const page of pages) {
+    results.push({
+      _id: page.url,
+      structured: page.data.structuredData,
+      tag: page.slugs[0],
+      url: page.url,
+      title: page.data.title,
+      description: page.data.description,
+    });
+  }
+
+  return NextResponse.json(results);
+}

examples/next-mdx/tsconfig.json

@@ -10,7 +10,7 @@
     "noEmit": true,
     "esModuleInterop": true,
     "module": "esnext",
-    "moduleResolution": "node",
+    "moduleResolution": "bundler",
     "resolveJsonModule": true,
     "isolatedModules": true,
     "jsx": "preserve",

examples/openapi/tsconfig.json

@@ -10,7 +10,7 @@
     "noEmit": true,
     "esModuleInterop": true,
     "module": "esnext",
-    "moduleResolution": "node",
+    "moduleResolution": "bundler",
     "resolveJsonModule": true,
     "isolatedModules": true,
     "jsx": "preserve",

packages/core/package.json

@@ -127,26 +127,23 @@
   },
   "dependencies": {
     "@formatjs/intl-localematcher": "^0.5.4",
+    "@orama/orama": "^2.1.0",
     "@shikijs/rehype": "^1.17.5",
     "@shikijs/transformers": "^1.17.5",
-    "flexsearch": "0.7.21",
     "github-slugger": "^2.0.0",
     "image-size": "^1.1.1",
     "negotiator": "^0.6.3",
-    "npm-to-yarn": "^3.0.0",
     "react-remove-scroll": "^2.6.0",
     "remark": "^15.0.0",
     "remark-gfm": "^4.0.0",
     "scroll-into-view-if-needed": "^3.1.0",
     "shiki": "^1.17.5",
-    "swr": "^2.2.5",
     "unist-util-visit": "^5.0.0"
   },
   "devDependencies": {
     "@algolia/client-search": "^4.24.0",
     "@mdx-js/mdx": "^3.0.1",
     "@types/estree-jsx": "^1.0.5",
-    "@types/flexsearch": "0.7.6",
     "@types/hast": "^3.0.4",
     "@types/mdast": "^4.0.3",
     "@types/negotiator": "^0.6.3",

packages/core/src/search/_stemmers.ts

@@ -0,0 +1,33 @@
+/* Cloned from Orama codebase */
+
+export const STEMMERS: Record<string, string> = {
+  arabic: 'ar',
+  armenian: 'am',
+  bulgarian: 'bg',
+  danish: 'dk',
+  dutch: 'nl',
+  english: 'en',
+  finnish: 'fi',
+  french: 'fr',
+  german: 'de',
+  greek: 'gr',
+  hungarian: 'hu',
+  indian: 'in',
+  indonesian: 'id',
+  irish: 'ie',
+  italian: 'it',
+  lithuanian: 'lt',
+  nepali: 'np',
+  norwegian: 'no',
+  portuguese: 'pt',
+  romanian: 'ro',
+  russian: 'ru',
+  serbian: 'rs',
+  slovenian: 'ru',
+  spanish: 'es',
+  swedish: 'se',
+  tamil: 'ta',
+  turkish: 'tr',
+  ukrainian: 'uk',
+  sanskrit: 'sk',
+};

packages/core/src/search/client.ts

@@ -1,36 +1,20 @@
-import { useState } from 'react';
-import useSWR, { type SWRResponse } from 'swr';
+import { useRef, useState } from 'react';
 import { useDebounce } from '@/utils/use-debounce';
 import type { SortedResult } from '@/server/types';
+import { fetchDocs } from '@/search/client/fetch';
+import { useOnChange } from '@/utils/use-on-change';
 
 interface UseDocsSearch {
   search: string;
   setSearch: (v: string) => void;
-  query: SWRResponse<
-    SortedResult[] | 'empty',
-    Error,
-    { keepPreviousData: true }
-  >;
+  query: {
+    isLoading: boolean;
+    data?: SortedResult[] | 'empty';
+    error?: Error;
+  };
 }
 
-async function fetchDocs(
-  api: string,
-  query: string,
-  locale: string | undefined,
-  tag: string | undefined,
-): Promise<SortedResult[] | 'empty'> {
-  if (query.length === 0) return 'empty';
-
-  const params = new URLSearchParams();
-  params.set('query', query);
-  if (locale) params.set('locale', locale);
-  if (tag) params.set('tag', tag);
-
-  const res = await fetch(`${api}?${params.toString()}`);
-
-  if (!res.ok) throw new Error(await res.text());
-  return (await res.json()) as SortedResult[];
-}
+const cache = new Map<string, SortedResult[] | 'empty'>();
 
 /**
  * @param locale - Filter with locale
@@ -45,15 +29,43 @@ export function useDocsSearch(
   delayMs = 100,
 ): UseDocsSearch {
   const [search, setSearch] = useState('');
+  const [results, setResults] = useState<SortedResult[] | 'empty'>('empty');
+  const [error, setError] = useState<Error>();
+  const [isLoading, setIsLoading] = useState(false);
   const debouncedValue = useDebounce(search, delayMs);
+  const onStart = useRef<() => void>();
 
-  const query: UseDocsSearch['query'] = useSWR(
-    [api, debouncedValue, locale, tag],
-    (args) => fetchDocs(...args),
-    {
-      keepPreviousData: true,
-    },
-  );
+  useOnChange(debouncedValue, () => {
+    if (onStart.current) onStart.current();
 
-  return { search, setSearch, query };
+    const cached = cache.get(debouncedValue);
+    if (cached) {
+      setError(undefined);
+      setResults(cached);
+      return;
+    }
+
+    setIsLoading(true);
+    let interrupt = false;
+    onStart.current = () => {
+      interrupt = true;
+    };
+
+    void fetchDocs(api, debouncedValue, locale, tag)
+      .then((res) => {
+        cache.set(debouncedValue, res);
+        if (interrupt) return;
+
+        setError(undefined);
+        setResults(res);
+      })
+      .catch((err: unknown) => {
+        setError(err as Error);
+      })
+      .finally(() => {
+        setIsLoading(false);
+      });
+  });
+
+  return { search, setSearch, query: { isLoading, data: results, error } };
 }

packages/core/src/search/client/fetch.ts

@@ -0,0 +1,20 @@
+import type { SortedResult } from '@/server';
+
+export async function fetchDocs(
+  api: string,
+  query: string,
+  locale: string | undefined,
+  tag: string | undefined,
+): Promise<SortedResult[] | 'empty'> {
+  if (query.length === 0) return 'empty';
+
+  const params = new URLSearchParams();
+  params.set('query', query);
+  if (locale) params.set('locale', locale);
+  if (tag) params.set('tag', tag);
+
+  const res = await fetch(`${api}?${params.toString()}`);
+
+  if (!res.ok) throw new Error(await res.text());
+  return (await res.json()) as SortedResult[];
+}

packages/core/src/search/client/static.ts

@@ -0,0 +1,94 @@
+import {
+  type AnyOrama,
+  create,
+  load,
+  type Orama,
+  type RawData,
+} from '@orama/orama';
+import { type SortedResult } from '@/server';
+import { searchSimple } from '@/search/search/simple';
+import { searchAdvanced } from '@/search/search/advanced';
+import { type advancedSchema } from '@/search/create-db';
+import { type schema } from '@/search/create-db-simple';
+
+interface Client {
+  search: (
+    query: string,
+    locale: string | undefined,
+    tag: string | undefined,
+  ) => Promise<SortedResult[]>;
+}
+
+type Type = 'simple' | 'advanced';
+
+type ExportedData =
+  | (RawData & { type: Type })
+  | {
+      type: 'i18n';
+      data: Record<string, RawData & { type: Type }>;
+    };
+
+// locale -> db
+const dbs = new Map<
+  string,
+  {
+    type: 'simple' | 'advanced';
+    db: AnyOrama;
+  }
+>();
+let started = false;
+
+export function createStaticClient(from = '/api/search'): Client {
+  async function init(): Promise<void> {
+    if (started) return;
+    started = true;
+    const res = await fetch(from);
+
+    if (!res.ok)
+      throw new Error(
+        `failed to fetch exported search indexes from ${from}, make sure the search database is exported and available for client.`,
+      );
+
+    const data = (await res.json()) as ExportedData;
+
+    if (data.type === 'i18n') {
+      const loadAll = Object.entries(data.data).map(async ([k, v]) => {
+        const db = await create({ schema: { _: 'string' } });
+
+        await load(db, v);
+        dbs.set(k, {
+          type: v.type,
+          db,
+        });
+      });
+
+      await Promise.all(loadAll);
+    } else {
+      const db = await create({ schema: { _: 'string' } });
+
+      await load(db, data);
+      dbs.set('', {
+        type: data.type,
+        db,
+      });
+    }
+  }
+
+  const get = init();
+  return {
+    async search(query, locale, tag) {
+      await get;
+      const db = dbs.get(locale ?? '');
+
+      if (!db) return [];
+      if (db.type === 'simple')
+        return searchSimple(db as unknown as Orama<typeof schema>, query);
+
+      return searchAdvanced(
+        db as unknown as Orama<typeof advancedSchema>,
+        query,
+        tag,
+      );
+    },
+  };
+}

packages/core/src/search/create-db-simple.ts

@@ -0,0 +1,46 @@
+import {
+  create,
+  insertMultiple,
+  type Orama,
+  type TypedDocument,
+} from '@orama/orama';
+import { type SimpleOptions } from '@/search/server';
+
+export type SimpleDocument = TypedDocument<Orama<typeof schema>>;
+export const schema = {
+  url: 'string',
+  title: 'string',
+  description: 'string',
+  content: 'string',
+  keywords: 'string',
+} as const;
+
+export async function createDBSimple({
+  indexes,
+  language,
+}: SimpleOptions): Promise<Orama<typeof schema>> {
+  const items = typeof indexes === 'function' ? await indexes() : indexes;
+  const db = await create({
+    language,
+    schema: {
+      url: 'string',
+      title: 'string',
+      description: 'string',
+      content: 'string',
+      keywords: 'string',
+    } as const,
+  });
+
+  await insertMultiple(
+    db,
+    items.map((page) => ({
+      title: page.title,
+      description: page.description,
+      url: page.url,
+      content: page.content,
+      keywords: page.keywords,
+    })),
+  );
+
+  return db;
+}

packages/core/src/search/create-db.ts

@@ -0,0 +1,89 @@
+import {
+  create,
+  insertMultiple,
+  type Orama,
+  type PartialSchemaDeep,
+  type TypedDocument,
+} from '@orama/orama';
+import { type AdvancedOptions } from '@/search/server';
+
+export type AdvancedDocument = TypedDocument<Orama<typeof advancedSchema>>;
+export const advancedSchema = {
+  content: 'string',
+  page_id: 'string',
+  type: 'string',
+  keywords: 'string',
+  tag: 'string',
+  url: 'string',
+} as const;
+
+export async function createDB({
+  indexes,
+  tokenizer,
+  search: _,
+  ...rest
+}: AdvancedOptions): Promise<Orama<typeof advancedSchema>> {
+  const items = typeof indexes === 'function' ? await indexes() : indexes;
+
+  const db = await create({
+    ...rest,
+    schema: advancedSchema,
+    components: {
+      tokenizer: tokenizer ?? {
+        stemmerSkipProperties: ['tag', 'url', 'page_id', 'type'],
+      },
+    },
+  });
+
+  const mapTo: PartialSchemaDeep<AdvancedDocument>[] = [];
+  items.forEach((page) => {
+    const data = page.structuredData;
+    let id = 0;
+
+    mapTo.push({
+      id: page.id,
+      page_id: page.id,
+      type: 'page',
+      content: page.title,
+      keywords: page.keywords,
+      tag: page.tag,
+      url: page.url,
+    });
+
+    if (page.description) {
+      mapTo.push({
+        id: `${page.id}-${(id++).toString()}`,
+        page_id: page.id,
+        tag: page.tag,
+        type: 'text',
+        url: page.url,
+        content: page.description,
+      });
+    }
+
+    for (const heading of data.headings) {
+      mapTo.push({
+        id: `${page.id}-${(id++).toString()}`,
+        page_id: page.id,
+        type: 'heading',
+        tag: page.tag,
+        url: `${page.url}#${heading.id}`,
+        content: heading.content,
+      });
+    }
+
+    for (const content of data.contents) {
+      mapTo.push({
+        id: `${page.id}-${(id++).toString()}`,
+        page_id: page.id,
+        tag: page.tag,
+        type: 'text',
+        url: content.heading ? `${page.url}#${content.heading}` : page.url,
+        content: content.content,
+      });
+    }
+  });
+
+  await insertMultiple(db, mapTo);
+  return db;
+}

packages/core/src/search/create-endpoint.ts

@@ -4,7 +4,7 @@ export function createEndpoint(server: SearchServer): SearchAPI {
   const { search } = server;
 
   return {
-    search,
+    ...server,
     async GET(request) {
       const query = request.nextUrl.searchParams.get('query');
       if (!query) return Response.json([]);

packages/core/src/search/i18n-api.ts

@@ -1,3 +1,4 @@
+import { type Language } from '@orama/orama';
 import {
   type AdvancedIndex,
   type AdvancedOptions,
@@ -11,72 +12,137 @@ import {
 } from '@/search/server';
 import { createEndpoint } from '@/search/create-endpoint';
 import { type I18nConfig } from '@/i18n';
+import { STEMMERS } from '@/search/_stemmers';
 
-interface I18nSimpleOptions
-  extends Omit<SimpleOptions, 'language' | 'indexes'> {
-  i18n: I18nConfig;
-  indexes: WithLocale<Index>[] | Dynamic<WithLocale<Index>>;
-}
+type LocaleMap<O> = Record<string, Language | O>;
 
-interface I18nAdvancedOptions
-  extends Omit<AdvancedOptions, 'language' | 'indexes'> {
+type Options<O extends SimpleOptions | AdvancedOptions, Idx> = Omit<
+  O,
+  'language' | 'indexes'
+> & {
   i18n: I18nConfig;
-  indexes: WithLocale<AdvancedIndex>[] | Dynamic<WithLocale<AdvancedIndex>>;
-}
+
+  /**
+   * Map locale name from i18n config to Orama compatible `language` or options
+   */
+  localeMap?: LocaleMap<Partial<O>>;
+
+  indexes: WithLocale<Idx>[] | Dynamic<WithLocale<Idx>>;
+};
+
+type I18nSimpleOptions = Options<SimpleOptions, Index>;
+type I18nAdvancedOptions = Options<AdvancedOptions, AdvancedIndex>;
 
 type WithLocale<T> = T & {
   locale: string;
 };
 
+function defaultLocaleMap(locale: string): string {
+  const map = STEMMERS;
+
+  return Object.keys(map).find((lang) => map[lang] === locale) ?? locale;
+}
+
+async function initSimple(
+  options: I18nSimpleOptions,
+): Promise<Map<string, SearchServer>> {
+  const map = new Map<string, SearchServer>();
+  if (options.i18n.languages.length === 0) {
+    return map;
+  }
+
+  const indexes =
+    typeof options.indexes === 'function'
+      ? await options.indexes()
+      : options.indexes;
+
+  for (const locale of options.i18n.languages) {
+    const localeIndexes = indexes.filter((index) => index.locale === locale);
+    const searchLocale =
+      options.localeMap?.[locale] ?? defaultLocaleMap(locale);
+
+    map.set(
+      locale,
+      typeof searchLocale === 'object'
+        ? initSimpleSearch({
+            ...options,
+            ...searchLocale,
+            indexes: localeIndexes,
+          })
+        : initSimpleSearch({
+            ...options,
+            language: searchLocale,
+            indexes: localeIndexes,
+          }),
+    );
+  }
+
+  return map;
+}
+
+async function initAdvanced(
+  options: I18nAdvancedOptions,
+): Promise<Map<string, SearchServer>> {
+  const map = new Map<string, SearchServer>();
+  if (options.i18n.languages.length === 0) {
+    return map;
+  }
+
+  const indexes =
+    typeof options.indexes === 'function'
+      ? await options.indexes()
+      : options.indexes;
+
+  for (const locale of options.i18n.languages) {
+    const localeIndexes = indexes.filter((index) => index.locale === locale);
+    const searchLocale =
+      options.localeMap?.[locale] ?? defaultLocaleMap(locale);
+
+    map.set(
+      locale,
+      typeof searchLocale === 'object'
+        ? initAdvancedSearch({
+            ...options,
+            indexes: localeIndexes,
+            ...searchLocale,
+          })
+        : initAdvancedSearch({
+            ...options,
+            language: searchLocale,
+            indexes: localeIndexes,
+          }),
+    );
+  }
+
+  return map;
+}
+
 export function createI18nSearchAPI<T extends 'simple' | 'advanced'>(
   type: T,
   options: T extends 'simple' ? I18nSimpleOptions : I18nAdvancedOptions,
 ): SearchAPI {
-  const map = new Map<string, SearchServer>();
-
-  async function init(): Promise<void> {
-    if (options.i18n.languages.length === 0) {
-      return;
-    }
-
-    const indexes =
-      typeof options.indexes === 'function'
-        ? await options.indexes()
-        : options.indexes;
-
-    for (const locale of options.i18n.languages) {
-      const localeIndexes = indexes.filter((index) => index.locale === locale);
-
-      if (type === 'simple') {
-        map.set(
-          locale,
-          initSimpleSearch({
-            ...options,
-            language: locale,
-            indexes: localeIndexes as SimpleOptions['indexes'],
-          }),
-        );
-        continue;
-      }
-
-      map.set(
-        locale,
-        initAdvancedSearch({
-          ...options,
-          language: locale,
-          indexes: localeIndexes as AdvancedOptions['indexes'],
-        }),
-      );
-    }
-  }
-
+  const get =
+    type === 'simple'
+      ? initSimple(options as I18nSimpleOptions)
+      : initAdvanced(options as I18nAdvancedOptions);
   return createEndpoint({
-    search: async (query, searchOptions) => {
-      if (map.size === 0) await init();
+    async export() {
+      const map = await get;
+      const entries = Object.entries(map).map(async ([k, v]) => [
+        k,
+        await (v as SearchServer).export(),
+      ]);
 
-      const handler = map.get(
-        searchOptions?.locale ?? options.i18n.defaultLanguage,
-      );
+      return {
+        type: 'i18n',
+        data: Object.fromEntries(await Promise.all(entries)) as object,
+      };
+    },
+    search: async (query, searchOptions) => {
+      const map = await get;
+
+      const locale = searchOptions?.locale ?? options.i18n.defaultLanguage;
+      const handler = map.get(locale);
 
       if (handler) return handler.search(query, searchOptions);
       return [];

packages/core/src/search/search/advanced.ts

@@ -0,0 +1,66 @@
+import { getByID, type Orama, search, type SearchParams } from '@orama/orama';
+import { type AdvancedDocument, type advancedSchema } from '@/search/create-db';
+import { removeUndefined } from '@/utils/remove-undefined';
+import type { SortedResult } from '@/server';
+
+export async function searchAdvanced(
+  db: Orama<typeof advancedSchema>,
+  query: string,
+  tag: string | undefined,
+  extraParams: Partial<
+    SearchParams<Orama<typeof advancedSchema>, AdvancedDocument>
+  > = {},
+): Promise<SortedResult[]> {
+  let params: SearchParams<typeof db, AdvancedDocument> = {
+    where: removeUndefined({
+      tag,
+      ...extraParams.where,
+    }),
+    groupBy: {
+      properties: ['page_id'],
+      maxResult: 8,
+      ...extraParams.groupBy,
+    },
+  };
+
+  if (query.length > 0) {
+    params = {
+      ...params,
+      term: query,
+      tolerance: 1,
+      properties: ['content', 'keywords'],
+      ...extraParams,
+      where: params.where,
+      groupBy: params.groupBy,
+    };
+  }
+
+  const result = await search(db, params);
+  const list: SortedResult[] = [];
+  for (const item of result.groups ?? []) {
+    const pageId = item.values[0] as string;
+
+    const page = await getByID(db, pageId);
+    if (!page) continue;
+
+    list.push({
+      id: pageId,
+      type: 'page',
+      content: page.content,
+      url: page.url,
+    });
+
+    for (const hit of item.result) {
+      if (hit.document.type === 'page') continue;
+
+      list.push({
+        id: hit.document.id.toString(),
+        content: hit.document.content,
+        type: hit.document.type as SortedResult['type'],
+        url: hit.document.url,
+      });
+    }
+  }
+
+  return list;
+}

packages/core/src/search/search/simple.ts

@@ -0,0 +1,26 @@
+import { type Orama, search, type SearchParams } from '@orama/orama';
+import type { SortedResult } from '@/server';
+import { type schema, type SimpleDocument } from '@/search/create-db-simple';
+
+export async function searchSimple(
+  db: Orama<typeof schema>,
+  query: string,
+  params: Partial<SearchParams<Orama<typeof schema>, SimpleDocument>> = {},
+): Promise<SortedResult[]> {
+  const result = await search(db, {
+    term: query,
+    tolerance: 1,
+    ...params,
+    boost: {
+      title: 2,
+      ...('boost' in params ? params.boost : undefined),
+    },
+  });
+
+  return result.hits.map<SortedResult>((hit) => ({
+    type: 'page',
+    content: hit.document.title,
+    id: hit.document.url,
+    url: hit.document.url,
+  }));
+}

packages/core/src/search/server.ts

@@ -1,14 +1,41 @@
-import { Document } from 'flexsearch';
+import {
+  type Orama,
+  type SearchParams,
+  type DefaultTokenizerConfig,
+  type Tokenizer,
+  type OramaPlugin,
+  type SorterConfig,
+  save,
+} from '@orama/orama';
 import { type NextRequest } from 'next/server';
 import type { StructuredData } from '@/mdx-plugins/remark-structure';
 import type { SortedResult } from '@/server/types';
 import { createEndpoint } from '@/search/create-endpoint';
+import {
+  type AdvancedDocument,
+  type advancedSchema,
+  createDB,
+} from '@/search/create-db';
+import { searchSimple } from '@/search/search/simple';
+import { searchAdvanced } from '@/search/search/advanced';
+import {
+  createDBSimple,
+  type schema,
+  type SimpleDocument,
+} from './create-db-simple';
 
 export interface SearchServer {
   search: (
     query: string,
     options?: { locale?: string; tag?: string },
   ) => Promise<SortedResult[]>;
+
+  /**
+   * Export the database
+   *
+   * You can reference the exported database to implement client-side search
+   */
+  export: () => Promise<unknown>;
 }
 
 export interface SearchAPI extends SearchServer {
@@ -25,21 +52,32 @@ export interface SearchAPI extends SearchServer {
  */
 export type Dynamic<T> = () => T[] | Promise<T[]>;
 
-export interface SimpleOptions {
-  indexes: Index[] | Dynamic<Index>;
+interface SharedOptions {
   language?: string;
+
+  sort?: SorterConfig;
+  tokenizer?: Tokenizer | DefaultTokenizerConfig;
+  plugins?: OramaPlugin[];
 }
 
-export interface AdvancedOptions {
+export interface SimpleOptions extends SharedOptions {
+  indexes: Index[] | Dynamic<Index>;
+
+  /**
+   * Customise search options on server
+   */
+  search?: Partial<SearchParams<Orama<typeof schema>, SimpleDocument>>;
+}
+
+export interface AdvancedOptions extends SharedOptions {
   indexes: AdvancedIndex[] | Dynamic<AdvancedIndex>;
 
   /**
-   * Enable search tags for filtering results
-   *
-   * @defaultValue false
+   * Customise search options on server
    */
-  tag?: boolean;
-  language?: string;
+  search?: Partial<
+    SearchParams<Orama<typeof advancedSchema>, AdvancedDocument>
+  >;
 }
 
 export function createSearchAPI<T extends 'simple' | 'advanced'>(
@@ -61,80 +99,20 @@ export interface Index {
   keywords?: string;
 }
 
-export function initSimpleSearch({
-  indexes,
-  language,
-}: SimpleOptions): SearchServer {
-  const store = ['title', 'url'];
+export function initSimpleSearch(options: SimpleOptions): SearchServer {
+  const doc = createDBSimple(options);
 
-  async function getDocument(): Promise<Document<Index, string[]>> {
-    const items = typeof indexes === 'function' ? await indexes() : indexes;
-    const index = new Document<Index, typeof store>({
-      language,
-      optimize: true,
-      cache: 100,
-      document: {
-        id: 'url',
-        store,
-        index: [
-          {
-            field: 'title',
-            tokenize: 'forward',
-            resolution: 9,
-          },
-          {
-            field: 'description',
-            tokenize: 'strict',
-            context: {
-              depth: 1,
-              resolution: 9,
-            },
-          },
-          {
-            field: 'content',
-            tokenize: 'strict',
-            context: {
-              depth: 1,
-              resolution: 9,
-            },
-          },
-          {
-            field: 'keywords',
-            tokenize: 'strict',
-            resolution: 9,
-          },
-        ],
-      },
-    });
-
-    for (const page of items) {
-      index.add({
-        title: page.title,
-        description: page.description,
-        url: page.url,
-        content: page.content,
-        keywords: page.keywords,
-      });
-    }
-    return index;
-  }
-
-  const doc = getDocument();
   return {
+    async export() {
+      return {
+        type: 'simple',
+        ...(await save(await doc)),
+      };
+    },
     search: async (query) => {
-      const results = (await doc).search(query, 5, {
-        enrich: true,
-        suggest: true,
-      });
+      const db = await doc;
 
-      if (results.length === 0) return [];
-
-      return results[0].result.map<SortedResult>((page) => ({
-        type: 'page',
-        content: page.doc.title,
-        id: page.doc.url,
-        url: page.doc.url,
-      }));
+      return searchSimple(db, query, options.search);
     },
   };
 }
@@ -157,154 +135,20 @@ export interface AdvancedIndex {
   url: string;
 }
 
-interface InternalIndex {
-  id: string;
-  url: string;
-  page_id: string;
-  type: 'page' | 'heading' | 'text';
-  content: string;
-
-  tag?: string;
-  keywords?: string;
-}
-
-export function initAdvancedSearch({
-  indexes,
-  language,
-  tag = false,
-}: AdvancedOptions): SearchServer {
-  const store = ['id', 'url', 'content', 'page_id', 'type', 'keywords'];
-
-  async function getDocument(): Promise<Document<InternalIndex, string[]>> {
-    const items = typeof indexes === 'function' ? await indexes() : indexes;
-    const index = new Document<InternalIndex, typeof store>({
-      language,
-      cache: 100,
-      optimize: true,
-      document: {
-        id: 'id',
-        tag: tag ? 'tag' : undefined,
-        store,
-        index: [
-          {
-            field: 'content',
-            tokenize: 'forward',
-            context: { depth: 2, bidirectional: true, resolution: 9 },
-          },
-          {
-            field: 'keywords',
-            tokenize: 'strict',
-            resolution: 9,
-          },
-        ],
-      },
-    });
-
-    for (const page of items) {
-      const data = page.structuredData;
-      let id = 0;
-
-      index.add({
-        id: page.id,
-        page_id: page.id,
-        type: 'page',
-        content: page.title,
-        keywords: page.keywords,
-        tag: page.tag,
-        url: page.url,
-      });
-
-      if (page.description) {
-        index.add({
-          id: page.id + (id++).toString(),
-          page_id: page.id,
-          tag: page.tag,
-          type: 'text',
-          url: page.url,
-          content: page.description,
-        });
-      }
-
-      for (const heading of data.headings) {
-        index.add({
-          id: page.id + (id++).toString(),
-          page_id: page.id,
-          type: 'heading',
-          tag: page.tag,
-          url: `${page.url}#${heading.id}`,
-          content: heading.content,
-        });
-      }
-
-      for (const content of data.contents) {
-        index.add({
-          id: page.id + (id++).toString(),
-          page_id: page.id,
-          tag: page.tag,
-          type: 'text',
-          url: content.heading ? `${page.url}#${content.heading}` : page.url,
-          content: content.content,
-        });
-      }
-    }
-
-    return index;
-  }
-
-  const doc = getDocument();
+export function initAdvancedSearch(options: AdvancedOptions): SearchServer {
+  const get = createDB(options);
 
   return {
-    search: async (query, options) => {
-      const index = await doc;
-      const results = index.search(query, 5, {
-        enrich: true,
-        tag: options?.tag,
-        limit: 6,
-      });
+    async export() {
+      return {
+        type: 'advanced',
+        ...(await save(await get)),
+      };
+    },
+    search: async (query, searchOptions) => {
+      const db = await get;
 
-      const map = new Map<string, SortedResult[]>();
-
-      for (const item of results[0]?.result ?? []) {
-        if (item.doc.type === 'page') {
-          if (!map.has(item.doc.id)) {
-            map.set(item.doc.id, []);
-          }
-
-          continue;
-        }
-
-        const list = map.get(item.doc.page_id) ?? [];
-
-        list.push({
-          id: item.doc.id,
-          content: item.doc.content,
-          type: item.doc.type,
-          url: item.doc.url,
-        });
-
-        map.set(item.doc.page_id, list);
-      }
-
-      const sortedResult: SortedResult[] = [];
-      for (const [id, items] of map.entries()) {
-        const page = (
-          index as unknown as {
-            get: (id: string) => InternalIndex | null;
-          }
-        ).get(id);
-
-        if (!page) continue;
-
-        sortedResult.push({
-          id: page.id,
-          content: page.content,
-          type: 'page',
-          url: page.url,
-        });
-        sortedResult.push(...items);
-      }
-
-      return sortedResult;
+      return searchAdvanced(db, query, searchOptions?.tag, options.search);
     },
   };
 }

packages/core/src/source/page-tree-builder.ts

@@ -1,5 +1,6 @@
 import type { ReactElement } from 'react';
 import type { I18nConfig } from '@/i18n';
+import { removeUndefined } from '@/utils/remove-undefined';
 import type * as PageTree from '../server/page-tree';
 import type { File, Folder, MetaFile, PageFile, Storage } from './file-system';
 import { resolvePath } from './path';
@@ -314,13 +315,3 @@ function pathToName(name: string, resolveGroup = false): string {
 
   return result.join('');
 }
-
-function removeUndefined<T extends object>(value: T): T {
-  const obj = value as Record<string, unknown>;
-  Object.keys(obj).forEach((key) => {
-    // eslint-disable-next-line @typescript-eslint/no-dynamic-delete -- Remove undefined values
-    if (obj[key] === undefined) delete obj[key];
-  });
-
-  return value;
-}

packages/core/src/utils/remove-undefined.ts

@@ -0,0 +1,9 @@
+export function removeUndefined<T extends object>(value: T): T {
+  const obj = value as Record<string, unknown>;
+  Object.keys(obj).forEach((key) => {
+    // eslint-disable-next-line @typescript-eslint/no-dynamic-delete -- Remove undefined values
+    if (obj[key] === undefined) delete obj[key];
+  });
+
+  return value;
+}

packages/core/test/search.test.ts

@@ -24,7 +24,6 @@ test('Search API', async () => {
 
 test('Search API Advanced', async () => {
   const api = createSearchAPI('advanced', {
-    tag: true,
     indexes: [
       {
         id: '1',
@@ -46,7 +45,7 @@ something`,
 something`,
         ),
         url: '/page',
-        tag: '',
+        tag: 'test',
       },
     ],
   });
@@ -65,7 +64,7 @@ something`,
       },
       {
         "content": "Hello World",
-        "id": "10",
+        "id": "1-0",
         "type": "heading",
         "url": "/#hello-world",
       },
@@ -76,15 +75,15 @@ something`,
 test('Search API I18n', async () => {
   const api = createI18nSearchAPI('simple', {
     i18n: {
-      languages: ['cn', 'en'],
+      languages: ['italian', 'en'],
       defaultLanguage: 'en',
     },
     indexes: [
       {
-        title: 'Hello World Chinese',
-        content: 'Hello World',
+        title: 'ciao mondo amico italian',
+        content: 'ciao mondo amico',
         url: '/hello-world',
-        locale: 'cn',
+        locale: 'italian',
       },
       {
         title: 'Hello World English',
@@ -96,8 +95,6 @@ test('Search API I18n', async () => {
   });
 
   expect(await api.search('English', { locale: 'en' })).toHaveLength(1);
-  expect(
-    await api.search('Hello World Chinese', { locale: 'cn' }),
-  ).toHaveLength(1);
-  expect(await api.search('Chinese', { locale: 'en' })).toHaveLength(0);
+  expect(await api.search('amico', { locale: 'italian' })).toHaveLength(1);
+  expect(await api.search('italian', { locale: 'en' })).toHaveLength(0);
 });

pnpm-lock.yaml

@@ -453,15 +453,15 @@ importers:
       '@formatjs/intl-localematcher':
         specifier: ^0.5.4
         version: 0.5.4
+      '@orama/orama':
+        specifier: ^2.1.0
+        version: 2.1.0
       '@shikijs/rehype':
         specifier: ^1.17.5
         version: 1.17.5
       '@shikijs/transformers':
         specifier: ^1.17.5
         version: 1.17.5
-      flexsearch:
-        specifier: 0.7.21
-        version: 0.7.21
       github-slugger:
         specifier: ^2.0.0
         version: 2.0.0
@@ -471,9 +471,6 @@ importers:
       negotiator:
         specifier: ^0.6.3
         version: 0.6.3
-      npm-to-yarn:
-        specifier: ^3.0.0
-        version: 3.0.0
       react:
         specifier: '>= 18'
         version: 18.3.1
@@ -495,9 +492,6 @@ importers:
       shiki:
         specifier: ^1.17.5
         version: 1.17.5
-      swr:
-        specifier: ^2.2.5
-        version: 2.2.5([email protected])
       unist-util-visit:
         specifier: ^5.0.0
         version: 5.0.0
@@ -511,9 +505,6 @@ importers:
       '@types/estree-jsx':
         specifier: ^1.0.5
         version: 1.0.5
-      '@types/flexsearch':
-        specifier: 0.7.6
-        version: 0.7.6
       '@types/hast':
         specifier: ^3.0.4
         version: 3.0.4
@@ -1996,6 +1987,10 @@ packages:
     resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==}
     engines: {node: '>= 8'}
 
+  '@orama/[email protected]':
+    resolution: {integrity: sha512-6Z6acBHmXGhp2Dom0ZvGJsJ1tkCwvs+uqNZUnhBbUTDvX4U1EMSGiTZt3fJFXy1enLKse6lOpG+E76/MnQIdBw==}
+    engines: {node: '>= 16.0.0'}
+
   '@parcel/[email protected]':
     resolution: {integrity: sha512-LOi/WTbbh3aTn2RYddrO8pnapixAziFl6SMxHM69r3tvdSm94JtCenaKgk1GRg5FJ5wpMCpHeW+7yqPlvZv7kg==}
     engines: {node: '>= 10.0.0'}
@@ -3193,9 +3188,6 @@ packages:
   '@types/[email protected]':
     resolution: {integrity: sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==}
 
-  '@types/[email protected]':
-    resolution: {integrity: sha512-H5IXcRn96/gaDmo+rDl2aJuIJsob8dgOXDqf8K0t8rWZd1AFNaaspmRsElESiU+EWE33qfbFPgI0OC/B1g9FCA==}
-
   '@types/[email protected]':
     resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==}
 
@@ -4510,9 +4502,6 @@ packages:
   [email protected]:
     resolution: {integrity: sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==}
 
-  [email protected]:
-    resolution: {integrity: sha512-W7cHV7Hrwjid6lWmy0IhsWDFQboWSng25U3VVywpHOTJnnAZNPScog67G+cVpeX9f7yDD21ih0WDrMMT+JoaYg==}
-
   [email protected]:
     resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==}
 
@@ -8134,6 +8123,8 @@ snapshots:
       '@nodelib/fs.scandir': 2.1.5
       fastq: 1.17.1
 
+  '@orama/[email protected]': {}
+
   '@parcel/[email protected]':
     optional: true
 
@@ -9172,8 +9163,6 @@ snapshots:
 
   '@types/[email protected]': {}
 
-  '@types/[email protected]': {}
-
   '@types/[email protected]':
     dependencies:
       '@types/unist': 3.0.2
@@ -10837,8 +10826,6 @@ snapshots:
 
   [email protected]: {}
 
-  [email protected]: {}
-
   [email protected]:
     dependencies:
       is-callable: 1.2.7