mirror of
https://github.com/hanzo-docs/docs.git
synced 2026-10-25 04:20:02 +00:00
Core: Support flexsearch server & client
This commit is contained in:
Fuma Nama
@@ -0,0 +1,5 @@
|
||||
---
|
||||
'fumadocs-core': patch
|
||||
---
|
||||
|
||||
Support flexsearch server & client
|
||||
@@ -154,6 +154,7 @@
|
||||
"@oramacloud/client": "^2.1.4",
|
||||
"@tanstack/react-router": "1.166.2",
|
||||
"@types/estree-jsx": "^1.0.5",
|
||||
"@types/flexsearch": "^0.7.42",
|
||||
"@types/hast": "^3.0.4",
|
||||
"@types/mdast": "^4.0.4",
|
||||
"@types/negotiator": "^0.6.4",
|
||||
@@ -162,6 +163,7 @@
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"algoliasearch": "5.49.1",
|
||||
"eslint-config-custom": "workspace:*",
|
||||
"flexsearch": "^0.8.212",
|
||||
"lucide-react": "^0.577.0",
|
||||
"next": "16.1.6",
|
||||
"react-router": "^7.13.1",
|
||||
@@ -185,6 +187,7 @@
|
||||
"@types/mdast": "*",
|
||||
"@types/react": "*",
|
||||
"algoliasearch": "5.x.x",
|
||||
"flexsearch": "*",
|
||||
"lucide-react": "*",
|
||||
"next": "16.x.x",
|
||||
"react": "^19.2.0",
|
||||
@@ -194,6 +197,9 @@
|
||||
"zod": "4.x.x"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"flexsearch": {
|
||||
"optional": true
|
||||
},
|
||||
"@mdx-js/mdx": {
|
||||
"optional": true
|
||||
},
|
||||
|
||||
@@ -2,7 +2,7 @@ import { type DependencyList, useRef, useState } from 'react';
|
||||
import { useDebounce } from '@/utils/use-debounce';
|
||||
import { type FetchOptions } from '@/search/client/fetch';
|
||||
import { useOnChange } from '@/utils/use-on-change';
|
||||
import { type StaticOptions } from '@/search/client/static';
|
||||
import { type StaticOptions } from '@/search/client/orama-static';
|
||||
import { type AlgoliaOptions } from '@/search/client/algolia';
|
||||
import { type OramaCloudOptions } from '@/search/client/orama-cloud';
|
||||
import { type OramaCloudLegacyOptions } from '@/search/client/orama-cloud-legacy';
|
||||
@@ -132,7 +132,7 @@ export function useDocsSearch(
|
||||
return search(debouncedValue, client);
|
||||
}
|
||||
case 'static': {
|
||||
const { search } = await import('./client/static');
|
||||
const { search } = await import('./client/orama-static');
|
||||
return search(debouncedValue, client);
|
||||
}
|
||||
default:
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
import type { SearchAPI } from './server';
|
||||
import Search, { type DocumentData } from 'flexsearch';
|
||||
import { createEndpoint } from './server/endpoint';
|
||||
import { buildBreadcrumbsDefault, buildIndexDefault, type SharedIndex } from './server/build-index';
|
||||
import { buildDocuments, SharedDocument } from './server/build-doc';
|
||||
import type { SortedResult } from '.';
|
||||
import type { LoaderConfig, LoaderOutput, Page } from '@/source';
|
||||
import type { Awaitable } from '@/types';
|
||||
|
||||
export type Index = SharedIndex;
|
||||
|
||||
type Doc = SharedDocument & DocumentData;
|
||||
|
||||
export interface Options {
|
||||
indexes: Index[] | (() => Awaitable<Index[]>);
|
||||
}
|
||||
|
||||
export function flexsearch(options: Options): SearchAPI {
|
||||
function initIndex(indexes: Index[]) {
|
||||
const index = new Search.Document<Doc>({
|
||||
tokenize: 'full',
|
||||
document: {
|
||||
id: 'id',
|
||||
index: ['content'],
|
||||
store: true,
|
||||
},
|
||||
});
|
||||
|
||||
for (const doc of buildDocuments(indexes)) {
|
||||
index.add(doc.id, doc as Doc);
|
||||
}
|
||||
|
||||
return index;
|
||||
}
|
||||
|
||||
const indexPromise =
|
||||
typeof options.indexes === 'function'
|
||||
? Promise.resolve(options.indexes()).then(initIndex)
|
||||
: initIndex(options.indexes);
|
||||
|
||||
return createEndpoint({
|
||||
async export() {
|
||||
const map = new Map<string, unknown>();
|
||||
(await indexPromise).export((key, data) => {
|
||||
map.set(key, data);
|
||||
});
|
||||
return map;
|
||||
},
|
||||
async search(query) {
|
||||
const index = await indexPromise;
|
||||
const arr = await index.searchAsync(query, {
|
||||
index: 'content',
|
||||
});
|
||||
const out: SortedResult[] = [];
|
||||
if (arr.length === 0) return out;
|
||||
|
||||
const results = arr[0].result;
|
||||
// page id -> heading/content item
|
||||
const grouped = new Map<string, Doc[]>();
|
||||
|
||||
for (const id of results) {
|
||||
const doc = index.get(id);
|
||||
if (!doc) continue;
|
||||
|
||||
let list = grouped.get(doc.page_id);
|
||||
if (!list) {
|
||||
list = [];
|
||||
grouped.set(doc.page_id, list);
|
||||
}
|
||||
|
||||
if (doc.type !== 'page') {
|
||||
list.push(doc);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [page_id, items] of grouped) {
|
||||
const page = index.get(page_id);
|
||||
if (!page) continue;
|
||||
|
||||
out.push({
|
||||
id: page_id,
|
||||
type: 'page',
|
||||
content: page.content,
|
||||
breadcrumbs: page.breadcrumbs,
|
||||
url: page.url,
|
||||
});
|
||||
|
||||
for (const item of items) {
|
||||
out.push({
|
||||
id: item.id,
|
||||
content: item.content,
|
||||
breadcrumbs: item.breadcrumbs,
|
||||
type: item.type,
|
||||
url: item.url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export interface FromSourceOptions<C extends LoaderConfig> {
|
||||
buildIndex?: (page: Page<C['source']['pageData']>) => Awaitable<Index>;
|
||||
}
|
||||
|
||||
export function flexsearchFromSource<C extends LoaderConfig>(
|
||||
loader: LoaderOutput<C>,
|
||||
options: FromSourceOptions<NoInfer<C>> = {},
|
||||
) {
|
||||
const { buildIndex = buildIndexDefault } = options;
|
||||
|
||||
return flexsearch({
|
||||
indexes() {
|
||||
return Promise.all(
|
||||
loader.getPages().map(async (page) => {
|
||||
const index = await buildIndex(page);
|
||||
if (index.breadcrumbs) return index;
|
||||
return { ...index, breadcrumbs: buildBreadcrumbsDefault(loader, page) };
|
||||
}),
|
||||
);
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -3,7 +3,7 @@ import type Mixedbread from '@mixedbread/sdk';
|
||||
import type { StoreSearchParams, StoreSearchResponse } from '@mixedbread/sdk/resources/stores';
|
||||
import removeMd from 'remove-markdown';
|
||||
import Slugger from 'github-slugger';
|
||||
import { createEndpoint } from '@/search/orama/create-endpoint';
|
||||
import { createEndpoint } from '@/search/server/endpoint';
|
||||
import type { SearchAPI } from '@/search/server';
|
||||
|
||||
export interface SearchMetadata {
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
type TypedDocument,
|
||||
} from '@orama/orama';
|
||||
import { type AdvancedOptions, type SimpleOptions } from '@/search/server';
|
||||
import { buildDocuments } from '../server/build-doc';
|
||||
|
||||
export type SimpleDocument = TypedDocument<Orama<typeof simpleSchema>>;
|
||||
export const simpleSchema = {
|
||||
@@ -45,59 +46,7 @@ export async function createDB({
|
||||
},
|
||||
}) as Orama<typeof advancedSchema>;
|
||||
|
||||
const mapTo: PartialSchemaDeep<AdvancedDocument>[] = [];
|
||||
items.forEach((page) => {
|
||||
const pageTag = page.tag ?? [];
|
||||
const tags = Array.isArray(pageTag) ? pageTag : [pageTag];
|
||||
const data = page.structuredData;
|
||||
let id = 0;
|
||||
|
||||
mapTo.push({
|
||||
id: page.id,
|
||||
page_id: page.id,
|
||||
type: 'page',
|
||||
content: page.title,
|
||||
breadcrumbs: page.breadcrumbs,
|
||||
tags,
|
||||
url: page.url,
|
||||
});
|
||||
|
||||
const nextId = () => `${page.id}-${id++}`;
|
||||
|
||||
if (page.description) {
|
||||
mapTo.push({
|
||||
id: nextId(),
|
||||
page_id: page.id,
|
||||
tags,
|
||||
type: 'text',
|
||||
url: page.url,
|
||||
content: page.description,
|
||||
});
|
||||
}
|
||||
|
||||
for (const heading of data.headings) {
|
||||
mapTo.push({
|
||||
id: nextId(),
|
||||
page_id: page.id,
|
||||
type: 'heading',
|
||||
tags,
|
||||
url: `${page.url}#${heading.id}`,
|
||||
content: heading.content,
|
||||
});
|
||||
}
|
||||
|
||||
for (const content of data.contents) {
|
||||
mapTo.push({
|
||||
id: nextId(),
|
||||
page_id: page.id,
|
||||
tags,
|
||||
type: 'text',
|
||||
url: content.heading ? `${page.url}#${content.heading}` : page.url,
|
||||
content: content.content,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const mapTo: PartialSchemaDeep<AdvancedDocument>[] = buildDocuments(items);
|
||||
await insertMultiple(db, mapTo);
|
||||
return db;
|
||||
}
|
||||
|
||||
@@ -2,70 +2,15 @@ import {
|
||||
type AdvancedIndex,
|
||||
type AdvancedOptions,
|
||||
createI18nSearchAPI,
|
||||
createSearchAPI,
|
||||
type SearchAPI,
|
||||
} from '@/search/server';
|
||||
initAdvancedSearch,
|
||||
} from './create-server';
|
||||
import type { LoaderConfig, LoaderOutput, Page } from '@/source';
|
||||
import { type StructuredData } from '@/mdx-plugins';
|
||||
import { basename, extname } from '@/source/path';
|
||||
import type { I18nConfig } from '@/i18n';
|
||||
import type { Language } from '@orama/orama';
|
||||
import { findPath } from '@/page-tree/utils';
|
||||
import type { Awaitable } from '@/types';
|
||||
|
||||
async function buildIndexDefault(page: Page): Promise<AdvancedIndex> {
|
||||
let structuredData: StructuredData | undefined;
|
||||
|
||||
if ('structuredData' in page.data) {
|
||||
structuredData =
|
||||
typeof page.data.structuredData === 'function'
|
||||
? ((await page.data.structuredData()) as StructuredData)
|
||||
: (page.data.structuredData as StructuredData);
|
||||
} else if ('load' in page.data && typeof page.data.load === 'function') {
|
||||
structuredData = (await page.data.load()).structuredData;
|
||||
}
|
||||
|
||||
if (!structuredData)
|
||||
throw new Error(
|
||||
'Cannot find structured data from page, please define the page to index function.',
|
||||
);
|
||||
|
||||
return {
|
||||
title: page.data.title ?? basename(page.path, extname(page.path)),
|
||||
description: page.data.description,
|
||||
url: page.url,
|
||||
id: page.url,
|
||||
structuredData,
|
||||
};
|
||||
}
|
||||
|
||||
function isBreadcrumbItem(item: unknown): item is string {
|
||||
return typeof item === 'string' && item.length > 0;
|
||||
}
|
||||
|
||||
export function buildBreadcrumbsDefault<C extends LoaderConfig>(
|
||||
source: LoaderOutput<C>,
|
||||
page: Page,
|
||||
): string[] | undefined {
|
||||
const pageTree = source.getPageTree(page.locale);
|
||||
const path = findPath(pageTree.children, (node) => node.type === 'page' && node.url === page.url);
|
||||
|
||||
if (path) {
|
||||
const breadcrumbs: string[] = [];
|
||||
path.pop();
|
||||
|
||||
if (isBreadcrumbItem(pageTree.name)) {
|
||||
breadcrumbs.push(pageTree.name);
|
||||
}
|
||||
|
||||
for (const segment of path) {
|
||||
if (!isBreadcrumbItem(segment.name)) continue;
|
||||
breadcrumbs.push(segment.name);
|
||||
}
|
||||
|
||||
return breadcrumbs;
|
||||
}
|
||||
}
|
||||
import { createEndpoint } from '../server/endpoint';
|
||||
import type { SearchAPI } from '../server/types';
|
||||
import { buildIndexDefault, buildBreadcrumbsDefault } from '../server/build-index';
|
||||
|
||||
interface Options<C extends LoaderConfig> extends Omit<AdvancedOptions, 'indexes'> {
|
||||
localeMap?: {
|
||||
@@ -109,17 +54,21 @@ export function createFromSource<C extends LoaderConfig>(
|
||||
});
|
||||
}
|
||||
|
||||
return createSearchAPI('advanced', {
|
||||
...options,
|
||||
async indexes() {
|
||||
const indexes = source.getPages().map(async (page) => {
|
||||
const index = await buildIndex(page);
|
||||
index.breadcrumbs ??= buildBreadcrumbsDefault(source, page);
|
||||
return createEndpoint(
|
||||
initAdvancedSearch({
|
||||
...options,
|
||||
async indexes() {
|
||||
const indexes = source.getPages().map(async (page) => {
|
||||
const index = await buildIndex(page);
|
||||
index.breadcrumbs ??= buildBreadcrumbsDefault(source, page);
|
||||
|
||||
return index;
|
||||
});
|
||||
return index;
|
||||
});
|
||||
|
||||
return Promise.all(indexes);
|
||||
},
|
||||
});
|
||||
return Promise.all(indexes);
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export { buildBreadcrumbsDefault };
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
import { type Language, type Tokenizer } from '@orama/orama';
|
||||
import {
|
||||
type AdvancedIndex,
|
||||
type AdvancedOptions,
|
||||
type Dynamic,
|
||||
type Index,
|
||||
initAdvancedSearch,
|
||||
initSimpleSearch,
|
||||
type SearchAPI,
|
||||
type SearchServer,
|
||||
type SimpleOptions,
|
||||
} from '@/search/server';
|
||||
import { createEndpoint } from '@/search/orama/create-endpoint';
|
||||
import { type I18nConfig } from '@/i18n';
|
||||
import { STEMMERS } from '@/search/orama/_stemmers';
|
||||
|
||||
type I18nOptions<O extends SimpleOptions | AdvancedOptions, Idx> = Omit<
|
||||
O,
|
||||
'language' | 'indexes'
|
||||
> & {
|
||||
i18n: I18nConfig;
|
||||
|
||||
/**
|
||||
* Map locale name from i18n config to Orama compatible `language` or options
|
||||
*/
|
||||
localeMap?: Record<string, Language | Partial<O> | undefined>;
|
||||
|
||||
indexes: WithLocale<Idx>[] | Dynamic<WithLocale<Idx>>;
|
||||
};
|
||||
|
||||
type I18nSimpleOptions = I18nOptions<SimpleOptions, Index>;
|
||||
type I18nAdvancedOptions = I18nOptions<AdvancedOptions, AdvancedIndex>;
|
||||
|
||||
type WithLocale<T> = T & {
|
||||
locale: string;
|
||||
};
|
||||
|
||||
async function getTokenizer(
|
||||
locale: string,
|
||||
): Promise<{ language: string } | { tokenizer: Tokenizer }> {
|
||||
return {
|
||||
language: Object.keys(STEMMERS).find((lang) => STEMMERS[lang] === locale) ?? locale,
|
||||
};
|
||||
}
|
||||
|
||||
async function initSimple(options: I18nSimpleOptions): Promise<Map<string, SearchServer>> {
|
||||
const map = new Map<string, SearchServer>();
|
||||
if (options.i18n.languages.length === 0) {
|
||||
return map;
|
||||
}
|
||||
|
||||
const indexes = typeof options.indexes === 'function' ? await options.indexes() : options.indexes;
|
||||
|
||||
for (const locale of options.i18n.languages) {
|
||||
const localeIndexes = indexes.filter((index) => index.locale === locale);
|
||||
const mapped = options.localeMap?.[locale] ?? (await getTokenizer(locale));
|
||||
|
||||
map.set(
|
||||
locale,
|
||||
typeof mapped === 'object'
|
||||
? initSimpleSearch({
|
||||
...options,
|
||||
...mapped,
|
||||
indexes: localeIndexes,
|
||||
})
|
||||
: initSimpleSearch({
|
||||
...options,
|
||||
language: mapped,
|
||||
indexes: localeIndexes,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
async function initAdvanced(options: I18nAdvancedOptions): Promise<Map<string, SearchServer>> {
|
||||
const map = new Map<string, SearchServer>();
|
||||
if (options.i18n.languages.length === 0) {
|
||||
return map;
|
||||
}
|
||||
|
||||
const indexes = typeof options.indexes === 'function' ? await options.indexes() : options.indexes;
|
||||
|
||||
for (const locale of options.i18n.languages) {
|
||||
const localeIndexes = indexes.filter((index) => index.locale === locale);
|
||||
const mapped = options.localeMap?.[locale] ?? (await getTokenizer(locale));
|
||||
|
||||
map.set(
|
||||
locale,
|
||||
typeof mapped === 'object'
|
||||
? initAdvancedSearch({
|
||||
...options,
|
||||
indexes: localeIndexes,
|
||||
...mapped,
|
||||
})
|
||||
: initAdvancedSearch({
|
||||
...options,
|
||||
language: mapped,
|
||||
indexes: localeIndexes,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
export function createI18nSearchAPI<T extends 'simple' | 'advanced'>(
|
||||
type: T,
|
||||
options: T extends 'simple' ? I18nSimpleOptions : I18nAdvancedOptions,
|
||||
): SearchAPI {
|
||||
const get =
|
||||
type === 'simple'
|
||||
? initSimple(options as I18nSimpleOptions)
|
||||
: initAdvanced(options as I18nAdvancedOptions);
|
||||
return createEndpoint({
|
||||
async export() {
|
||||
const map = await get;
|
||||
const entries = Array.from(map.entries()).map(async ([k, v]) => [
|
||||
k,
|
||||
await (v as SearchServer).export(),
|
||||
]);
|
||||
|
||||
return {
|
||||
type: 'i18n',
|
||||
data: Object.fromEntries(await Promise.all(entries)),
|
||||
};
|
||||
},
|
||||
async search(query, searchOptions) {
|
||||
const map = await get;
|
||||
|
||||
const locale = searchOptions?.locale ?? options.i18n.defaultLanguage;
|
||||
const handler = map.get(locale);
|
||||
|
||||
if (handler) return handler.search(query, searchOptions);
|
||||
return [];
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
import {
|
||||
create,
|
||||
Orama,
|
||||
RawData,
|
||||
save,
|
||||
SearchParams,
|
||||
type Language,
|
||||
type Tokenizer,
|
||||
} from '@orama/orama';
|
||||
import type { SearchAPI, SearchServer } from '@/search/server';
|
||||
import type { I18nConfig } from '@/i18n';
|
||||
import { STEMMERS } from '@/search/orama/_stemmers';
|
||||
import { createEndpoint } from '../server/endpoint';
|
||||
import {
|
||||
AdvancedDocument,
|
||||
advancedSchema,
|
||||
createDB,
|
||||
createDBSimple,
|
||||
SimpleDocument,
|
||||
simpleSchema,
|
||||
} from './create-db';
|
||||
import { searchSimple } from './search/simple';
|
||||
import { searchAdvanced } from './search/advanced';
|
||||
import type { SharedIndex } from '../server/build-index';
|
||||
|
||||
type OramaInput = Parameters<typeof create>[0];
|
||||
|
||||
type SharedOptions = Pick<OramaInput, 'sort' | 'components' | 'plugins'> & {
|
||||
language?: string;
|
||||
tokenizer?: Required<OramaInput>['components']['tokenizer'];
|
||||
};
|
||||
|
||||
/**
|
||||
* Resolve indexes dynamically
|
||||
*/
|
||||
export type Dynamic<T> = () => T[] | Promise<T[]>;
|
||||
|
||||
export interface SimpleOptions extends SharedOptions {
|
||||
indexes: Index[] | Dynamic<Index>;
|
||||
|
||||
/**
|
||||
* Customise search options on server
|
||||
*/
|
||||
search?: Partial<SearchParams<Orama<typeof simpleSchema>, SimpleDocument>>;
|
||||
}
|
||||
|
||||
export interface AdvancedOptions extends SharedOptions {
|
||||
indexes: AdvancedIndex[] | Dynamic<AdvancedIndex>;
|
||||
|
||||
/**
|
||||
* Customise search options on server
|
||||
*/
|
||||
search?: Partial<SearchParams<Orama<typeof advancedSchema>, AdvancedDocument>>;
|
||||
}
|
||||
|
||||
export interface Index {
|
||||
title: string;
|
||||
description?: string;
|
||||
breadcrumbs?: string[];
|
||||
content: string;
|
||||
url: string;
|
||||
keywords?: string;
|
||||
}
|
||||
|
||||
export type ExportedData =
|
||||
| (RawData & { type: 'simple' | 'advanced' })
|
||||
| {
|
||||
type: 'i18n';
|
||||
data: Record<string, RawData & { type: 'simple' | 'advanced' }>;
|
||||
};
|
||||
|
||||
export function initSimpleSearch(options: SimpleOptions): SearchServer {
|
||||
const doc = createDBSimple(options);
|
||||
|
||||
return {
|
||||
async export() {
|
||||
return {
|
||||
type: 'simple',
|
||||
...save(await doc),
|
||||
};
|
||||
},
|
||||
async search(query) {
|
||||
const db = await doc;
|
||||
|
||||
return searchSimple(db, query, options.search);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type AdvancedIndex = SharedIndex;
|
||||
|
||||
export function initAdvancedSearch(options: AdvancedOptions): SearchServer {
|
||||
const get = createDB(options);
|
||||
|
||||
return {
|
||||
async export() {
|
||||
return {
|
||||
type: 'advanced',
|
||||
...save(await get),
|
||||
};
|
||||
},
|
||||
async search(query, searchOptions) {
|
||||
const db = await get;
|
||||
const mode = searchOptions?.mode;
|
||||
|
||||
return searchAdvanced(db, query, searchOptions?.tag, {
|
||||
...options.search,
|
||||
mode: mode === 'vector' ? 'vector' : 'fulltext',
|
||||
}).catch((err) => {
|
||||
if (mode === 'vector') {
|
||||
throw new Error(
|
||||
'failed to search, make sure you have installed `@orama/plugin-embeddings` according to their docs.',
|
||||
{
|
||||
cause: err,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
throw err;
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createSearchAPI(type: 'simple', options: SimpleOptions): SearchAPI;
|
||||
export function createSearchAPI(type: 'advanced', options: AdvancedOptions): SearchAPI;
|
||||
|
||||
export function createSearchAPI(
|
||||
...args: ['simple', SimpleOptions] | ['advanced', AdvancedOptions]
|
||||
): SearchAPI {
|
||||
if (args[0] === 'simple') {
|
||||
return createEndpoint(initSimpleSearch(args[1]));
|
||||
}
|
||||
|
||||
return createEndpoint(initAdvancedSearch(args[1]));
|
||||
}
|
||||
|
||||
type I18nOptions<O extends SimpleOptions | AdvancedOptions, Idx> = Omit<
|
||||
O,
|
||||
'language' | 'indexes'
|
||||
> & {
|
||||
i18n: I18nConfig;
|
||||
|
||||
/**
|
||||
* Map locale name from i18n config to Orama compatible `language` or options
|
||||
*/
|
||||
localeMap?: Record<string, Language | Partial<O> | undefined>;
|
||||
|
||||
indexes: WithLocale<Idx>[] | Dynamic<WithLocale<Idx>>;
|
||||
};
|
||||
|
||||
type I18nSimpleOptions = I18nOptions<SimpleOptions, Index>;
|
||||
type I18nAdvancedOptions = I18nOptions<AdvancedOptions, AdvancedIndex>;
|
||||
|
||||
type WithLocale<T> = T & {
|
||||
locale: string;
|
||||
};
|
||||
|
||||
function getTokenizer(locale: string): { language: string } | { tokenizer: Tokenizer } {
|
||||
return {
|
||||
language: Object.keys(STEMMERS).find((lang) => STEMMERS[lang] === locale) ?? locale,
|
||||
};
|
||||
}
|
||||
|
||||
export function createI18nSearchAPI(type: 'simple', options: I18nSimpleOptions): SearchAPI;
|
||||
export function createI18nSearchAPI(type: 'advanced', options: I18nAdvancedOptions): SearchAPI;
|
||||
|
||||
export function createI18nSearchAPI(
|
||||
...[type, options]: ['simple', I18nSimpleOptions] | ['advanced', I18nAdvancedOptions]
|
||||
): SearchAPI {
|
||||
async function initSearchServers() {
|
||||
const map = new Map<string, SearchServer>();
|
||||
if (options.i18n.languages.length === 0) {
|
||||
return map;
|
||||
}
|
||||
|
||||
const indexes =
|
||||
typeof options.indexes === 'function' ? await options.indexes() : options.indexes;
|
||||
|
||||
for (const locale of options.i18n.languages) {
|
||||
const localeIndexes = indexes.filter((index) => index.locale === locale);
|
||||
const mapped = options.localeMap?.[locale] ?? getTokenizer(locale);
|
||||
|
||||
if (type === 'simple') {
|
||||
map.set(
|
||||
locale,
|
||||
typeof mapped === 'object'
|
||||
? initSimpleSearch({
|
||||
...options,
|
||||
...mapped,
|
||||
indexes: localeIndexes,
|
||||
} as SimpleOptions)
|
||||
: initSimpleSearch({
|
||||
...options,
|
||||
language: mapped,
|
||||
indexes: localeIndexes,
|
||||
} as SimpleOptions),
|
||||
);
|
||||
} else {
|
||||
map.set(
|
||||
locale,
|
||||
typeof mapped === 'object'
|
||||
? initAdvancedSearch({
|
||||
...options,
|
||||
indexes: localeIndexes,
|
||||
...mapped,
|
||||
} as AdvancedOptions)
|
||||
: initAdvancedSearch({
|
||||
...options,
|
||||
language: mapped,
|
||||
indexes: localeIndexes,
|
||||
} as AdvancedOptions),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
const get = initSearchServers();
|
||||
return createEndpoint({
|
||||
async export() {
|
||||
const map = await get;
|
||||
const entries = Array.from(map.entries()).map(async ([k, v]) => [k, await v.export()]);
|
||||
|
||||
return {
|
||||
type: 'i18n',
|
||||
data: Object.fromEntries(await Promise.all(entries)),
|
||||
};
|
||||
},
|
||||
async search(query, searchOptions) {
|
||||
const map = await get;
|
||||
const locale = searchOptions?.locale ?? options.i18n.defaultLanguage;
|
||||
const handler = map.get(locale);
|
||||
|
||||
if (handler) return handler.search(query, searchOptions);
|
||||
return [];
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1,173 +1,3 @@
|
||||
import { create, type Orama, type RawData, save, type SearchParams } from '@orama/orama';
|
||||
import type { StructuredData } from '@/mdx-plugins/remark-structure';
|
||||
import { createEndpoint } from '@/search/orama/create-endpoint';
|
||||
import {
|
||||
type AdvancedDocument,
|
||||
type advancedSchema,
|
||||
createDB,
|
||||
createDBSimple,
|
||||
type SimpleDocument,
|
||||
simpleSchema,
|
||||
} from '@/search/orama/create-db';
|
||||
import { searchSimple } from '@/search/orama/search/simple';
|
||||
import { searchAdvanced } from '@/search/orama/search/advanced';
|
||||
import type { SortedResult } from '@/search';
|
||||
|
||||
type SearchType = 'simple' | 'advanced';
|
||||
|
||||
export type ExportedData =
|
||||
| (RawData & { type: SearchType })
|
||||
| {
|
||||
type: 'i18n';
|
||||
data: Record<string, RawData & { type: SearchType }>;
|
||||
};
|
||||
|
||||
export interface SearchServer {
|
||||
search: (
|
||||
query: string,
|
||||
options?: {
|
||||
locale?: string;
|
||||
tag?: string | string[];
|
||||
mode?: 'vector' | 'full';
|
||||
},
|
||||
) => Promise<SortedResult[]>;
|
||||
|
||||
/**
|
||||
* Export the database
|
||||
*
|
||||
* You can reference the exported database to implement client-side search
|
||||
*/
|
||||
export: () => Promise<ExportedData>;
|
||||
}
|
||||
|
||||
export interface SearchAPI extends SearchServer {
|
||||
GET: (request: Request) => Promise<Response>;
|
||||
|
||||
/**
|
||||
* `GET` route handler that exports search indexes for static search.
|
||||
*/
|
||||
staticGET: () => Promise<Response>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve indexes dynamically
|
||||
*/
|
||||
export type Dynamic<T> = () => T[] | Promise<T[]>;
|
||||
|
||||
type OramaInput = Parameters<typeof create>[0];
|
||||
|
||||
type SharedOptions = Pick<OramaInput, 'sort' | 'components' | 'plugins'> & {
|
||||
language?: string;
|
||||
tokenizer?: Required<OramaInput>['components']['tokenizer'];
|
||||
};
|
||||
|
||||
export interface SimpleOptions extends SharedOptions {
|
||||
indexes: Index[] | Dynamic<Index>;
|
||||
|
||||
/**
|
||||
* Customise search options on server
|
||||
*/
|
||||
search?: Partial<SearchParams<Orama<typeof simpleSchema>, SimpleDocument>>;
|
||||
}
|
||||
|
||||
export interface AdvancedOptions extends SharedOptions {
|
||||
indexes: AdvancedIndex[] | Dynamic<AdvancedIndex>;
|
||||
|
||||
/**
|
||||
* Customise search options on server
|
||||
*/
|
||||
search?: Partial<SearchParams<Orama<typeof advancedSchema>, AdvancedDocument>>;
|
||||
}
|
||||
|
||||
export function createSearchAPI<T extends SearchType>(
|
||||
type: T,
|
||||
options: T extends 'simple' ? SimpleOptions : AdvancedOptions,
|
||||
): SearchAPI {
|
||||
if (type === 'simple') {
|
||||
return createEndpoint(initSimpleSearch(options as SimpleOptions));
|
||||
}
|
||||
|
||||
return createEndpoint(initAdvancedSearch(options as AdvancedOptions));
|
||||
}
|
||||
|
||||
export interface Index {
|
||||
title: string;
|
||||
description?: string;
|
||||
breadcrumbs?: string[];
|
||||
content: string;
|
||||
url: string;
|
||||
keywords?: string;
|
||||
}
|
||||
|
||||
export function initSimpleSearch(options: SimpleOptions): SearchServer {
|
||||
const doc = createDBSimple(options);
|
||||
|
||||
return {
|
||||
async export() {
|
||||
return {
|
||||
type: 'simple',
|
||||
...save(await doc),
|
||||
};
|
||||
},
|
||||
async search(query) {
|
||||
const db = await doc;
|
||||
|
||||
return searchSimple(db, query, options.search);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export interface AdvancedIndex {
|
||||
id: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
breadcrumbs?: string[];
|
||||
|
||||
/**
|
||||
* Required if tag filter is enabled
|
||||
*/
|
||||
tag?: string | string[];
|
||||
|
||||
/**
|
||||
* preprocess mdx content with `structure`
|
||||
*/
|
||||
structuredData: StructuredData;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export function initAdvancedSearch(options: AdvancedOptions): SearchServer {
|
||||
const get = createDB(options);
|
||||
|
||||
return {
|
||||
async export() {
|
||||
return {
|
||||
type: 'advanced',
|
||||
...save(await get),
|
||||
};
|
||||
},
|
||||
async search(query, searchOptions) {
|
||||
const db = await get;
|
||||
const mode = searchOptions?.mode;
|
||||
|
||||
return searchAdvanced(db, query, searchOptions?.tag, {
|
||||
...options.search,
|
||||
mode: mode === 'vector' ? 'vector' : 'fulltext',
|
||||
}).catch((err) => {
|
||||
if (mode === 'vector') {
|
||||
throw new Error(
|
||||
'failed to search, make sure you have installed `@orama/plugin-embeddings` according to their docs.',
|
||||
{
|
||||
cause: err,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
throw err;
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export { createFromSource } from './orama/create-from-source';
|
||||
export { createI18nSearchAPI } from './orama/create-i18n';
|
||||
export * from './index';
|
||||
export * from './orama/create-server';
|
||||
export * from './server/types';
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import type { SharedIndex } from './build-index';
|
||||
|
||||
export interface SharedDocument {
|
||||
id: string;
|
||||
content: string;
|
||||
page_id: string;
|
||||
type: 'page' | 'heading' | 'text';
|
||||
breadcrumbs?: string[];
|
||||
tags: string[];
|
||||
url: string;
|
||||
}
|
||||
|
||||
export function buildDocuments(indexes: SharedIndex[]) {
|
||||
const docs: SharedDocument[] = [];
|
||||
|
||||
for (const page of indexes) {
|
||||
const pageTag = page.tag ?? [];
|
||||
const tags = Array.isArray(pageTag) ? pageTag : [pageTag];
|
||||
const data = page.structuredData;
|
||||
let id = 0;
|
||||
|
||||
docs.push({
|
||||
id: page.id,
|
||||
page_id: page.id,
|
||||
type: 'page',
|
||||
content: page.title,
|
||||
breadcrumbs: page.breadcrumbs,
|
||||
tags,
|
||||
url: page.url,
|
||||
});
|
||||
|
||||
const nextId = () => `${page.id}-${id++}`;
|
||||
|
||||
if (page.description) {
|
||||
docs.push({
|
||||
id: nextId(),
|
||||
page_id: page.id,
|
||||
tags,
|
||||
type: 'text',
|
||||
url: page.url,
|
||||
content: page.description,
|
||||
});
|
||||
}
|
||||
|
||||
for (const heading of data.headings) {
|
||||
docs.push({
|
||||
id: nextId(),
|
||||
page_id: page.id,
|
||||
type: 'heading',
|
||||
tags,
|
||||
url: `${page.url}#${heading.id}`,
|
||||
content: heading.content,
|
||||
});
|
||||
}
|
||||
|
||||
for (const content of data.contents) {
|
||||
docs.push({
|
||||
id: nextId(),
|
||||
page_id: page.id,
|
||||
tags,
|
||||
type: 'text',
|
||||
url: content.heading ? `${page.url}#${content.heading}` : page.url,
|
||||
content: content.content,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return docs;
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import type { LoaderConfig, LoaderOutput, Page } from '@/source';
|
||||
import type { StructuredData } from '@/mdx-plugins/remark-structure';
|
||||
import { basename, extname } from '@/source/path';
|
||||
import { findPath } from '@/page-tree/utils';
|
||||
|
||||
export interface SharedIndex {
|
||||
id: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
breadcrumbs?: string[];
|
||||
|
||||
/**
|
||||
* Required if tag filter is enabled
|
||||
*/
|
||||
tag?: string | string[];
|
||||
|
||||
/**
|
||||
* preprocess mdx content with `structure`
|
||||
*/
|
||||
structuredData: StructuredData;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export async function buildIndexDefault(page: Page): Promise<SharedIndex> {
|
||||
let structuredData: StructuredData | undefined;
|
||||
|
||||
if ('structuredData' in page.data) {
|
||||
structuredData =
|
||||
typeof page.data.structuredData === 'function'
|
||||
? ((await page.data.structuredData()) as StructuredData)
|
||||
: (page.data.structuredData as StructuredData);
|
||||
} else if ('load' in page.data && typeof page.data.load === 'function') {
|
||||
structuredData = (await page.data.load()).structuredData;
|
||||
}
|
||||
|
||||
if (!structuredData)
|
||||
throw new Error(
|
||||
'Cannot find structured data from page, please define the page to index function.',
|
||||
);
|
||||
|
||||
return {
|
||||
title: page.data.title ?? basename(page.path, extname(page.path)),
|
||||
description: page.data.description,
|
||||
url: page.url,
|
||||
id: page.url,
|
||||
structuredData,
|
||||
};
|
||||
}
|
||||
|
||||
function isBreadcrumbItem(item: unknown): item is string {
|
||||
return typeof item === 'string' && item.length > 0;
|
||||
}
|
||||
|
||||
export function buildBreadcrumbsDefault<C extends LoaderConfig>(
|
||||
source: LoaderOutput<C>,
|
||||
page: Page<C['source']['pageData']>,
|
||||
): string[] | undefined {
|
||||
const pageTree = source.getPageTree(page.locale);
|
||||
const path = findPath(pageTree.children, (node) => node.type === 'page' && node.url === page.url);
|
||||
|
||||
if (path) {
|
||||
const breadcrumbs: string[] = [];
|
||||
path.pop();
|
||||
|
||||
if (isBreadcrumbItem(pageTree.name)) {
|
||||
breadcrumbs.push(pageTree.name);
|
||||
}
|
||||
|
||||
for (const segment of path) {
|
||||
if (!isBreadcrumbItem(segment.name)) continue;
|
||||
breadcrumbs.push(segment.name);
|
||||
}
|
||||
|
||||
return breadcrumbs;
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
import { type SearchAPI, type SearchServer } from '@/search/server';
|
||||
import type { SearchAPI, SearchServer } from './types';
|
||||
|
||||
export function createEndpoint(server: SearchServer): SearchAPI {
|
||||
const { search } = server;
|
||||
@@ -0,0 +1,28 @@
|
||||
import type { SortedResult } from '..';
|
||||
|
||||
export interface SearchServer {
|
||||
search: (
|
||||
query: string,
|
||||
options?: {
|
||||
locale?: string;
|
||||
tag?: string | string[];
|
||||
mode?: 'vector' | 'full';
|
||||
},
|
||||
) => Promise<SortedResult[]>;
|
||||
|
||||
/**
|
||||
* Export the database
|
||||
*
|
||||
* You can reference the exported database to implement client-side search
|
||||
*/
|
||||
export: () => Promise<unknown>;
|
||||
}
|
||||
|
||||
export interface SearchAPI extends SearchServer {
|
||||
GET: (request: Request) => Promise<Response>;
|
||||
|
||||
/**
|
||||
* `GET` route handler that exports search indexes for static search.
|
||||
*/
|
||||
staticGET: () => Promise<Response>;
|
||||
}
|
||||
@@ -15,7 +15,7 @@ export default defineConfig({
|
||||
'src/source/{index,schema}.ts',
|
||||
'src/source/client/*.{ts,tsx}',
|
||||
'src/source/plugins/{lucide-icons,slugs,status-badges}.{ts,tsx}',
|
||||
'src/search/{index,client,server,algolia,orama-cloud,orama-cloud-legacy,mixedbread}.ts',
|
||||
'src/search/{index,client,server,algolia,orama-cloud,orama-cloud-legacy,mixedbread,flexsearch}.ts',
|
||||
'src/utils/use-on-change.ts',
|
||||
'src/utils/use-media-query.ts',
|
||||
'src/i18n/*.ts',
|
||||
|
||||
Generated
+23
-4
Reference in New Issue
Block a user