hanzoai/docs
2
0
Fork 0
mirror of https://github.com/hanzo-docs/docs.git synced 2026-10-26 05:23:40 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Core: Redesign search client adapter interface

Browse Source
This commit is contained in:
Fuma Nama
2026-03-08 21:53:17 +08:00
parent d35f30c0ff
commit 269dfb385d
19 changed files with 836 additions and 558 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/funky-sites-itch.md
+5
View File
@@ -0,0 +1,5 @@
---
'fumadocs-core': patch
---
Redesign search client adapter interface
packages/core/src/search/client.ts
+126 -91
View File
@@ -1,13 +1,15 @@
import { type DependencyList, useRef, useState } from 'react';
import { type DependencyList, use, useRef, useState } from 'react';
import { useDebounce } from '@/utils/use-debounce';
import { type FetchOptions } from '@/search/client/fetch';
import { useOnChange } from '@/utils/use-on-change';
import { type StaticOptions } from '@/search/client/orama-static';
import { type AlgoliaOptions } from '@/search/client/algolia';
import { type OramaCloudOptions } from '@/search/client/orama-cloud';
import { type OramaCloudLegacyOptions } from '@/search/client/orama-cloud-legacy';
import { type MixedbreadOptions } from '@/search/client/mixedbread';
import type { StaticOptions } from '@/search/client/orama-static';
import type { AlgoliaOptions } from '@/search/client/algolia';
import type { OramaCloudOptions } from '@/search/client/orama-cloud';
import type { OramaCloudLegacyOptions } from '@/search/client/orama-cloud-legacy';
import type { MixedbreadOptions } from '@/search/client/mixedbread';
import type { SortedResult } from '@/search';
import type { Awaitable } from '@/types';
import type { FlexsearchStaticOptions } from './client/flexsearch-static';
interface UseDocsSearch {
search: string;
@@ -19,7 +21,7 @@ interface UseDocsSearch {
};
}
export type Client =
export type ClientPreset =
| ({
type: 'fetch';
} & FetchOptions)
@@ -36,39 +38,34 @@ export type Client =
type: 'orama-cloud-legacy';
} & OramaCloudLegacyOptions)
| ({
type: 'flexsearch-static';
} & FlexsearchStaticOptions)
| ({
/**
* @deprecated Use `createMixedbreadSearchAPI` from `fumadocs-core/search/mixedbread` instead.
* This client-side approach exposes your API key in the browser.
* The server-side approach keeps the key secure and uses `type: 'fetch'` on the client.
*/
type: 'mixedbread';
} & MixedbreadOptions);
} & MixedbreadOptions)
| {
client: SearchClient;
};
function isDeepEqual(a: unknown, b: unknown): boolean {
if (a === b) return true;
if (Array.isArray(a) && Array.isArray(b)) {
return b.length === a.length && a.every((v, i) => isDeepEqual(v, b[i]));
}
if (typeof a === 'object' && a && typeof b === 'object' && b) {
const aKeys = Object.keys(a);
const bKeys = Object.keys(b);
return (
aKeys.length === bKeys.length &&
aKeys.every(
(key) =>
Object.hasOwn(b, key) && isDeepEqual(a[key as keyof object], b[key as keyof object]),
)
);
}
return false;
export interface SearchClient {
search: (query: string) => Awaitable<SortedResult[]>;
deps?: DependencyList;
}
const promiseMap: Record<string, Promise<unknown>> = {};
/**
* Provide a hook to query different official search clients.
*
* Note: it will re-query when its parameters changed, make sure to use `useMemo()` on `clientOptions` or define `deps` array.
* Note: it will re-query when its parameters changed, make sure to define `deps` array if you encounter rendering issues.
*/
export function useDocsSearch(
clientOptions: Client & {
clientOptions: ClientPreset & {
/**
* The debounced delay for performing a search (in ms).
* .
@@ -85,8 +82,7 @@ export function useDocsSearch(
},
deps?: DependencyList,
): UseDocsSearch {
const { delayMs = 100, allowEmpty = false, ...client } = clientOptions;
const { delayMs = 100, allowEmpty = false, ...clientRest } = clientOptions;
const [search, setSearch] = useState('');
const [results, setResults] = useState<SortedResult[] | 'empty'>('empty');
const [error, setError] = useState<Error>();
@@ -94,70 +90,109 @@ export function useDocsSearch(
const debouncedValue = useDebounce(search, delayMs);
const onStart = useRef<() => void>(undefined);
useOnChange(
[deps ?? clientOptions, debouncedValue],
() => {
if (onStart.current) {
onStart.current();
onStart.current = undefined;
let client: SearchClient;
if ('type' in clientRest) {
switch (clientRest.type) {
case 'fetch': {
const res = (promiseMap[clientRest.type] ??= import('./client/fetch')) as Promise<
typeof import('./client/fetch')
>;
const { fetchClient } = use(res);
client = fetchClient(clientRest);
break;
}
setIsLoading(true);
let interrupt = false;
onStart.current = () => {
interrupt = true;
};
async function run(): Promise<SortedResult[] | 'empty'> {
if (debouncedValue.length === 0 && !allowEmpty) return 'empty';
switch (client.type) {
case 'fetch': {
const { fetchDocs } = await import('./client/fetch');
return fetchDocs(debouncedValue, client);
}
case 'algolia': {
const { searchDocs } = await import('./client/algolia');
return searchDocs(debouncedValue, client);
}
case 'orama-cloud': {
const { searchDocs } = await import('./client/orama-cloud');
return searchDocs(debouncedValue, client);
}
case 'orama-cloud-legacy': {
const { searchDocs } = await import('./client/orama-cloud-legacy');
return searchDocs(debouncedValue, client);
}
case 'mixedbread': {
const { search } = await import('./client/mixedbread');
return search(debouncedValue, client);
}
case 'static': {
const { search } = await import('./client/orama-static');
return search(debouncedValue, client);
}
default:
throw new Error('unknown search client');
}
case 'algolia': {
const res = (promiseMap[clientRest.type] ??= import('./client/algolia')) as Promise<
typeof import('./client/algolia')
>;
const { algoliaClient } = use(res);
client = algoliaClient(clientRest);
break;
}
case 'orama-cloud': {
const res = (promiseMap[clientRest.type] ??= import('./client/orama-cloud')) as Promise<
typeof import('./client/orama-cloud')
>;
const { oramaCloudClient } = use(res);
client = oramaCloudClient(clientRest);
break;
}
case 'orama-cloud-legacy': {
const res = (promiseMap[clientRest.type] ??=
import('./client/orama-cloud-legacy')) as Promise<
typeof import('./client/orama-cloud-legacy')
>;
const { oramaCloudLegacyClient } = use(res);
client = oramaCloudLegacyClient(clientRest);
break;
}
case 'mixedbread': {
const res = (promiseMap[clientRest.type] ??= import('./client/mixedbread')) as Promise<
typeof import('./client/mixedbread')
>;
const { mixedbreadClient } = use(res);
client = mixedbreadClient(clientRest);
break;
}
case 'static': {
const res = (promiseMap[clientRest.type] ??= import('./client/orama-static')) as Promise<
typeof import('./client/orama-static')
>;
const { oramaStaticClient } = use(res);
client = oramaStaticClient(clientRest);
break;
}
case 'flexsearch-static': {
const res = (promiseMap[clientRest.type] ??=
import('./client/flexsearch-static')) as Promise<
typeof import('./client/flexsearch-static')
>;
const { flexsearchStaticClient } = use(res);
client = flexsearchStaticClient(clientRest);
break;
}
default:
throw new Error('unknown search client');
}
} else {
client = clientRest.client;
}
void run()
.then((res) => {
if (interrupt) return;
useOnChange([deps ?? client.deps, debouncedValue], () => {
if (onStart.current) {
onStart.current();
onStart.current = undefined;
}
setError(undefined);
setResults(res);
})
.catch((err: Error) => {
setError(err);
})
.finally(() => {
setIsLoading(false);
});
},
deps ? undefined : (a, b) => !isDeepEqual(a, b),
);
setIsLoading(true);
let interrupt = false;
onStart.current = () => {
interrupt = true;
};
async function run(): Promise<SortedResult[] | 'empty'> {
if (debouncedValue.length === 0 && !allowEmpty) return 'empty';
return client.search(debouncedValue);
}
void run()
.then((res) => {
if (interrupt) return;
setError(undefined);
setResults(res);
})
.catch((err: Error) => {
setError(err);
})
.finally(() => {
setIsLoading(false);
});
});
return { search, setSearch, query: { isLoading, data: results, error } };
}
// TODO: remove this on next major
export type { OramaCloudOptions, FetchOptions, StaticOptions, AlgoliaOptions };
packages/core/src/search/client/algolia.ts
+35 -31
View File
@@ -1,6 +1,7 @@
import type { BaseIndex } from '@/search/algolia';
import type { Hit, LiteClient, SearchResponse } from 'algoliasearch/lite';
import { createContentHighlighter, type SortedResult } from '@/search';
import type { SearchClient } from '../client';
export interface AlgoliaOptions {
indexName: string;
@@ -22,7 +23,7 @@ export interface AlgoliaOptions {
}>;
}
export function groupResults(hits: Hit<BaseIndex>[]): SortedResult[] {
function groupResults(hits: Hit<BaseIndex>[]): SortedResult[] {
const grouped: SortedResult[] = [];
const scannedUrls = new Set<string>();
@@ -50,36 +51,39 @@ export function groupResults(hits: Hit<BaseIndex>[]): SortedResult[] {
return grouped;
}
export async function searchDocs(
query: string,
{ indexName, onSearch, client, locale, tag }: AlgoliaOptions,
): Promise<SortedResult[]> {
if (query.trim().length === 0) return [];
export function algoliaClient(options: AlgoliaOptions): SearchClient {
const { indexName, onSearch, client, locale, tag } = options;
return {
deps: [indexName, client, locale, tag],
async search(query) {
if (query.trim().length === 0) return [];
const result = onSearch
? await onSearch(query, tag, locale)
: await client.searchForHits<BaseIndex>({
requests: [
{
type: 'default',
indexName,
query,
distinct: 5,
hitsPerPage: 10,
filters: tag ? `tag:${tag}` : undefined,
},
],
const result = onSearch
? await onSearch(query, tag, locale)
: await client.searchForHits<BaseIndex>({
requests: [
{
type: 'default',
indexName,
query,
distinct: 5,
hitsPerPage: 10,
filters: tag ? `tag:${tag}` : undefined,
},
],
});
const highlighter = createContentHighlighter(query);
return groupResults(result.results[0].hits).flatMap((hit) => {
if (hit.type === 'page') {
return {
...hit,
content: highlighter.highlightMarkdown(hit.content),
};
}
return [];
});
const highlighter = createContentHighlighter(query);
return groupResults(result.results[0].hits).flatMap((hit) => {
if (hit.type === 'page') {
return {
...hit,
content: highlighter.highlightMarkdown(hit.content),
};
}
return [];
});
},
};
}
packages/core/src/search/client/fetch.ts
+27 -17
View File
@@ -1,4 +1,5 @@
import type { SortedResult } from '@/search';
import type { SearchClient } from '../client';
export interface FetchOptions {
/**
@@ -17,26 +18,35 @@ export interface FetchOptions {
* Filter by locale
*/
locale?: string;
cache?: Map<string, SortedResult[]>;
}
const cache = new Map<string, SortedResult[]>();
const globalCache = new Map();
export async function fetchDocs(
query: string,
{ api = '/api/search', locale, tag }: FetchOptions,
): Promise<SortedResult[]> {
const url = new URL(api, window.location.origin);
url.searchParams.set('query', query);
if (locale) url.searchParams.set('locale', locale);
if (tag) url.searchParams.set('tag', Array.isArray(tag) ? tag.join(',') : tag);
export function fetchClient({
api = '/api/search',
locale,
tag,
cache = globalCache,
}: FetchOptions): SearchClient {
return {
deps: [api, locale, tag],
async search(query) {
const url = new URL(api, window.location.origin);
url.searchParams.set('query', query);
if (locale) url.searchParams.set('locale', locale);
if (tag) url.searchParams.set('tag', Array.isArray(tag) ? tag.join(',') : tag);
const key = url.toString();
const cached = cache.get(key);
if (cached) return cached;
const key = url.toString();
const cached = cache.get(key);
if (cached) return cached;
const res = await fetch(url);
if (!res.ok) throw new Error(await res.text());
const result = (await res.json()) as SortedResult[];
cache.set(key, result);
return result;
const res = await fetch(url);
if (!res.ok) throw new Error(await res.text());
const result = (await res.json()) as SortedResult[];
cache.set(key, result);
return result;
},
};
}
packages/core/src/search/client/flexsearch-static.ts
+64
View File
@@ -0,0 +1,64 @@
import type { SearchClient } from '../client';
import type { ExportedData } from '../flexsearch';
import type { Document } from 'flexsearch';
import { createDocument, search, type Doc } from '../flexsearch/utils';
export interface FlexsearchStaticOptions {
/**
* @defaultValue `/api/search`
*/
from?: string;
locale?: string;
}
function initDocument(data: Record<string, string>) {
const document = createDocument();
for (const [k, v] of Object.entries(data)) document.import(k, v);
return document;
}
const cacheMap = new Map<string, Promise<Map<string, Document<Doc>>>>();
export function flexsearchStaticClient(options: FlexsearchStaticOptions): SearchClient {
const { from = '/api/search', locale = '' } = options;
let dbs = cacheMap.get(from);
if (!dbs) {
dbs = init(from);
cacheMap.set(from, dbs);
}
return {
deps: [from, locale],
async search(query) {
const loaded = await dbs;
const db = loaded.get(locale);
if (!db) return [];
return search(db, query);
},
};
}
async function init(from: string) {
const res = await fetch(from);
if (!res.ok)
throw new Error(
`failed to fetch exported search indexes from ${from}, make sure the search database is exported and available for client.`,
);
const data = (await res.json()) as ExportedData;
const dbs = new Map<string, Document<Doc>>();
if (data.type === 'i18n') {
for (const [locale, map] of Object.entries(data.raw)) {
dbs.set(locale, initDocument(map));
}
return dbs;
} else {
dbs.set('', initDocument(data.raw));
}
return dbs;
}
packages/core/src/search/client/mixedbread.ts
+53 -51
View File
@@ -3,12 +3,8 @@ import type Mixedbread from '@mixedbread/sdk';
import removeMd from 'remove-markdown';
import Slugger from 'github-slugger';
import type { StoreSearchResponse } from '@mixedbread/sdk/resources/stores';
import type { SearchClient } from '../client';
/**
* @deprecated Use `createMixedbreadSearchAPI` from `fumadocs-core/search/mixedbread` instead.
* This client-side approach exposes your API key in the browser.
* The server-side approach keeps the key secure and uses `type: 'fetch'` on the client.
*/
export interface MixedbreadOptions {
/**
* The identifier of the store to search in
@@ -69,55 +65,61 @@ function extractHeadingTitle(text: string): string {
* This client-side approach exposes your API key in the browser.
* The server-side approach keeps the key secure and uses `type: 'fetch'` on the client.
*/
export async function search(query: string, options: MixedbreadOptions): Promise<SortedResult[]> {
export function mixedbreadClient(options: MixedbreadOptions): SearchClient {
const { client, storeIdentifier, tag } = options;
if (!query.trim()) {
return [];
}
return {
deps: [client, storeIdentifier, tag],
async search(query) {
if (!query.trim()) {
return [];
}
const res = await client.stores.search({
query,
store_identifiers: [storeIdentifier],
top_k: 10,
filters: {
key: 'generated_metadata.tag',
operator: 'eq',
value: tag,
},
search_options: {
return_metadata: true,
},
});
return (res.data as StoreSearchResult[]).flatMap((item) => {
const metadata = item.generated_metadata;
const url = metadata.url || '#';
const title = metadata.title || 'Untitled';
const chunkResults: SortedResult[] = [
{
id: `${item.file_id}-${item.chunk_index}-page`,
type: 'page',
content: title,
url,
},
];
const headingTitle = item.type === 'text' && item.text ? extractHeadingTitle(item.text) : '';
if (headingTitle) {
slugger.reset();
chunkResults.push({
id: `${item.file_id}-${item.chunk_index}-heading`,
type: 'heading',
content: headingTitle,
url: `${url}#${slugger.slug(headingTitle)}`,
const res = await client.stores.search({
query,
store_identifiers: [storeIdentifier],
top_k: 10,
filters: {
key: 'generated_metadata.tag',
operator: 'eq',
value: tag,
},
search_options: {
return_metadata: true,
},
});
}
return chunkResults;
});
return (res.data as StoreSearchResult[]).flatMap((item) => {
const metadata = item.generated_metadata;
const url = metadata.url || '#';
const title = metadata.title || 'Untitled';
const chunkResults: SortedResult[] = [
{
id: `${item.file_id}-${item.chunk_index}-page`,
type: 'page',
content: title,
url,
},
];
const headingTitle =
item.type === 'text' && item.text ? extractHeadingTitle(item.text) : '';
if (headingTitle) {
slugger.reset();
chunkResults.push({
id: `${item.file_id}-${item.chunk_index}-heading`,
type: 'heading',
content: headingTitle,
url: `${url}#${slugger.slug(headingTitle)}`,
});
}
return chunkResults;
});
},
};
}
packages/core/src/search/client/orama-cloud-legacy.ts
+92 -84
View File
@@ -2,6 +2,7 @@ import type { ClientSearchParams, OramaClient } from '@oramacloud/client';
import { removeUndefined } from '@/utils/remove-undefined';
import type { OramaIndex } from '@/search/orama-cloud-legacy';
import { createContentHighlighter, type SortedResult } from '@/search';
import type { SearchClient } from '../client';
interface CrawlerIndex {
path: string;
@@ -16,9 +17,13 @@ export interface OramaCloudLegacyOptions {
/**
* The type of your index.
*
* You can set it to `crawler` if you use crawler instead of the JSON index with schema provided by Fumadocs
* You can set it to `crawler` if you use crawler instead of the JSON index with schema provided by Fumadocs.
*/
index?: 'default' | 'crawler';
/**
* Note: not included into dependency list.
*/
params?: ClientSearchParams;
/**
@@ -32,94 +37,97 @@ export interface OramaCloudLegacyOptions {
locale?: string;
}
export async function searchDocs(
query: string,
options: OramaCloudLegacyOptions,
): Promise<SortedResult[]> {
const highlighter = createContentHighlighter(query);
const list: SortedResult[] = [];
export function oramaCloudLegacyClient(options: OramaCloudLegacyOptions): SearchClient {
const { index = 'default', client, params: extraParams = {}, tag } = options;
if (index === 'crawler') {
const result = await client.search({
...extraParams,
term: query,
where: {
category: tag
? {
eq: tag.slice(0, 1).toUpperCase() + tag.slice(1),
}
: undefined,
...extraParams.where,
},
limit: 10,
});
if (!result) return list;
return {
deps: [index, client, tag],
async search(query) {
const highlighter = createContentHighlighter(query);
const list: SortedResult[] = [];
for (const hit of result.hits) {
const doc = hit.document as unknown as CrawlerIndex;
list.push(
{
id: hit.id,
type: 'page',
content: highlighter.highlightMarkdown(doc.title),
url: doc.path,
},
{
id: 'page' + hit.id,
type: 'text',
content: highlighter.highlightMarkdown(doc.content),
url: doc.path,
},
);
}
return list;
}
const params: ClientSearchParams = {
...extraParams,
term: query,
where: removeUndefined({
tag,
...extraParams.where,
}),
groupBy: {
properties: ['page_id'],
maxResult: 7,
...extraParams.groupBy,
},
};
const result = await client.search(params);
if (!result || !result.groups) return list;
for (const item of result.groups) {
let addedHead = false;
for (const hit of item.result) {
const doc = hit.document as unknown as OramaIndex;
if (!addedHead) {
list.push({
id: doc.page_id,
type: 'page',
content: highlighter.highlightMarkdown(doc.title),
breadcrumbs: doc.breadcrumbs,
url: doc.url,
if (index === 'crawler') {
const result = await client.search({
...extraParams,
term: query,
where: {
category: tag
? {
eq: tag.slice(0, 1).toUpperCase() + tag.slice(1),
}
: undefined,
...extraParams.where,
},
limit: 10,
});
addedHead = true;
if (!result) return list;
for (const hit of result.hits) {
const doc = hit.document as unknown as CrawlerIndex;
list.push(
{
id: hit.id,
type: 'page',
content: highlighter.highlightMarkdown(doc.title),
url: doc.path,
},
{
id: 'page' + hit.id,
type: 'text',
content: highlighter.highlightMarkdown(doc.content),
url: doc.path,
},
);
}
return list;
}
list.push({
id: doc.id,
content: highlighter.highlightMarkdown(doc.content),
type: doc.content === doc.section ? 'heading' : 'text',
url: doc.section_id ? `${doc.url}#${doc.section_id}` : doc.url,
});
}
}
const params: ClientSearchParams = {
...extraParams,
term: query,
where: removeUndefined({
tag,
...extraParams.where,
}),
groupBy: {
properties: ['page_id'],
maxResult: 7,
...extraParams.groupBy,
},
};
return list;
const result = await client.search(params);
if (!result || !result.groups) return list;
for (const item of result.groups) {
let addedHead = false;
for (const hit of item.result) {
const doc = hit.document as unknown as OramaIndex;
if (!addedHead) {
list.push({
id: doc.page_id,
type: 'page',
content: highlighter.highlightMarkdown(doc.title),
breadcrumbs: doc.breadcrumbs,
url: doc.url,
});
addedHead = true;
}
list.push({
id: doc.id,
content: highlighter.highlightMarkdown(doc.content),
type: doc.content === doc.section ? 'heading' : 'text',
url: doc.section_id ? `${doc.url}#${doc.section_id}` : doc.url,
});
}
}
return list;
},
};
}
packages/core/src/search/client/orama-cloud.ts
+91 -83
View File
@@ -2,6 +2,7 @@ import type { OramaCloud, OramaCloudSearchParams } from '@orama/core';
import { removeUndefined } from '@/utils/remove-undefined';
import type { OramaIndex } from '@/search/orama-cloud';
import { createContentHighlighter, type SortedResult } from '@/search';
import type { SearchClient } from '../client';
interface CrawlerIndex {
path: string;
@@ -19,6 +20,10 @@ export interface OramaCloudOptions {
* You can set it to `crawler` if you use crawler instead of the JSON index with schema provided by Fumadocs
*/
index?: 'default' | 'crawler';
/**
* Note: not included in dependency list.
*/
params?: Partial<OramaCloudSearchParams>;
/**
@@ -32,95 +37,98 @@ export interface OramaCloudOptions {
locale?: string;
}
export async function searchDocs(
query: string,
options: OramaCloudOptions,
): Promise<SortedResult[]> {
const highlighter = createContentHighlighter(query);
const list: SortedResult[] = [];
export function oramaCloudClient(options: OramaCloudOptions): SearchClient {
const { index = 'default', client, params: extraParams, tag } = options;
if (index === 'crawler') {
const result = await client.search({
datasources: [],
...extraParams,
term: query,
where: {
category: tag
? {
eq: tag.slice(0, 1).toUpperCase() + tag.slice(1),
}
: undefined,
...extraParams?.where,
},
limit: 10,
});
if (!result) return list;
return {
deps: [index, client, tag],
async search(query) {
const highlighter = createContentHighlighter(query);
const list: SortedResult[] = [];
for (const hit of result.hits) {
const doc = hit.document as unknown as CrawlerIndex;
list.push(
{
id: hit.id,
type: 'page',
content: highlighter.highlightMarkdown(doc.title),
url: doc.path,
},
{
id: 'page' + hit.id,
type: 'text',
content: highlighter.highlightMarkdown(doc.content),
url: doc.path,
},
);
}
return list;
}
const result = await client.search({
datasources: [],
...extraParams,
term: query,
limit: 10,
where: removeUndefined({
tag,
...extraParams?.where,
}),
groupBy: {
properties: ['page_id'],
max_results: 7,
...extraParams?.groupBy,
},
});
if (!result || !result.groups) return list;
for (const item of result.groups) {
let addedHead = false;
for (const hit of item.result) {
const doc = hit.document as unknown as OramaIndex;
if (!addedHead) {
list.push({
id: doc.page_id,
type: 'page',
content: highlighter.highlightMarkdown(doc.title),
breadcrumbs: doc.breadcrumbs,
url: doc.url,
if (index === 'crawler') {
const result = await client.search({
datasources: [],
...extraParams,
term: query,
where: {
category: tag
? {
eq: tag.slice(0, 1).toUpperCase() + tag.slice(1),
}
: undefined,
...extraParams?.where,
},
limit: 10,
});
addedHead = true;
if (!result) return list;
for (const hit of result.hits) {
const doc = hit.document as unknown as CrawlerIndex;
list.push(
{
id: hit.id,
type: 'page',
content: highlighter.highlightMarkdown(doc.title),
url: doc.path,
},
{
id: 'page' + hit.id,
type: 'text',
content: highlighter.highlightMarkdown(doc.content),
url: doc.path,
},
);
}
return list;
}
list.push({
id: doc.id,
content: highlighter.highlightMarkdown(doc.content),
type: doc.content === doc.section ? 'heading' : 'text',
url: doc.section_id ? `${doc.url}#${doc.section_id}` : doc.url,
const result = await client.search({
datasources: [],
...extraParams,
term: query,
limit: 20,
where: removeUndefined({
tag,
...extraParams?.where,
}),
groupBy: {
properties: ['page_id'],
max_results: 7,
...extraParams?.groupBy,
},
});
}
}
if (!result || !result.groups) return list;
return list.length > 80 ? list.slice(0, 80) : list;
for (const item of result.groups) {
let addedHead = false;
for (const hit of item.result) {
const doc = hit.document as unknown as OramaIndex;
if (!addedHead) {
list.push({
id: doc.page_id,
type: 'page',
content: highlighter.highlightMarkdown(doc.title),
breadcrumbs: doc.breadcrumbs,
url: doc.url,
});
addedHead = true;
}
list.push({
id: doc.id,
content: highlighter.highlightMarkdown(doc.content),
type: doc.content === doc.section ? 'heading' : 'text',
url: doc.section_id ? `${doc.url}#${doc.section_id}` : doc.url,
});
}
}
return list.length > 80 ? list.slice(0, 80) : list;
},
};
}
packages/core/src/search/client/orama-static.ts
+12 -5
View File
@@ -3,6 +3,7 @@ import { searchSimple } from '@/search/orama/search/simple';
import { searchAdvanced } from '@/search/orama/search/advanced';
import { type advancedSchema, type simpleSchema } from '@/search/orama/create-db';
import type { ExportedData } from '@/search/server';
import type { SearchClient } from '../client';
export interface StaticOptions {
/**
@@ -85,13 +86,19 @@ async function loadDB({
return result;
}
export async function search(query: string, options: StaticOptions) {
export function oramaStaticClient(options: StaticOptions): SearchClient {
const { tag, locale } = options;
const db = (await loadDB(options)).get(locale ?? '');
return {
deps: [tag, locale],
async search(query) {
const db = (await loadDB(options)).get(locale ?? '');
if (!db) return [];
if (db.type === 'simple') return searchSimple(db as unknown as Orama<typeof simpleSchema>, query);
if (!db) return [];
if (db.type === 'simple')
return searchSimple(db as unknown as Orama<typeof simpleSchema>, query);
return searchAdvanced(db.db as Orama<typeof advancedSchema>, query, tag);
return searchAdvanced(db.db as Orama<typeof advancedSchema>, query, tag);
},
};
}
packages/core/src/search/flexsearch.ts
+127 -80
View File
@@ -1,30 +1,46 @@
import type { SearchAPI } from './server';
import Search, { type DocumentData } from 'flexsearch';
import type { SearchAPI, SearchServer } from './server';
import Search, { type DocumentOptions } from 'flexsearch';
import { createEndpoint } from './server/endpoint';
import { buildBreadcrumbsDefault, buildIndexDefault, type SharedIndex } from './server/build-index';
import { buildDocuments, type SharedDocument } from './server/build-doc';
import { createContentHighlighter, type SortedResult } from '.';
import { buildBreadcrumbs, buildIndexDefault, type SharedIndex } from './server/build-index';
import { buildDocuments } from './server/build-doc';
import type { LoaderConfig, LoaderOutput, Page } from '@/source';
import type { Awaitable } from '@/types';
import type { I18nConfig } from '@/i18n';
import { createDocument, search, type Doc } from './flexsearch/utils';
export type Index = SharedIndex;
type Doc = SharedDocument & DocumentData;
export interface IndexWithLocale extends Index {
locale: string;
}
export interface Options {
indexes: Index[] | (() => Awaitable<Index[]>);
document?: DocumentOptions<Doc>;
}
export function flexsearch(options: Options): SearchAPI {
export type ExportedData =
| {
type: 'default';
raw: Record<string, string>;
}
| {
type: 'i18n';
raw: Record<string, Record<string, string>>;
};
export interface I18nOptions extends Omit<Options, 'indexes'> {
i18n: I18nConfig;
indexes: IndexWithLocale[] | (() => Awaitable<IndexWithLocale[]>);
/**
* options for each locale, see https://github.com/nextapps-de/flexsearch/blob/master/doc/encoder.md.
*/
localeMap?: Record<string, Partial<DocumentOptions<Doc>> | 'cjk'>;
}
function server(options: Options): SearchServer {
function initIndex(indexes: Index[]) {
const index = new Search.Document<Doc>({
tokenize: 'full',
document: {
id: 'id',
index: ['content'],
store: true,
},
});
const index = createDocument(options.document);
for (const doc of buildDocuments(indexes)) {
index.add(doc.id, doc as Doc);
@@ -38,70 +54,88 @@ export function flexsearch(options: Options): SearchAPI {
? Promise.resolve(options.indexes()).then(initIndex)
: initIndex(options.indexes);
return createEndpoint({
async export() {
const map = new Map<string, unknown>();
(await indexPromise).export((key, data) => {
map.set(key, data);
return {
async export(): Promise<ExportedData> {
const index = await indexPromise;
const raw: Record<string, string> = {};
index.export((key, data) => {
raw[key] = data;
});
return map;
return { type: 'default', raw };
},
async search(query) {
const index = await indexPromise;
const arr = await index.searchAsync(query, {
index: 'content',
return search(await indexPromise, query);
},
};
}
export function flexsearch(options: Options): SearchAPI {
return createEndpoint(server(options));
}
export function flexsearchI18n(options: I18nOptions): SearchAPI {
const { indexes: inputIndexes, localeMap } = options;
async function initSearchServers() {
const map = new Map<string, SearchServer>();
const indexMap = new Map<string, IndexWithLocale[]>();
const indexes = typeof inputIndexes === 'function' ? await inputIndexes() : inputIndexes;
for (const index of indexes) {
let list = indexMap.get(index.locale);
if (!list) {
list = [];
indexMap.set(index.locale, list);
}
list.push(index);
}
for (const [locale, list] of indexMap) {
const override = localeMap?.[locale];
map.set(
locale,
server({
indexes: list,
document:
override === 'cjk'
? { ...options.document, encoder: Search.Charset.CJK }
: { ...options.document, ...override },
}),
);
}
return map;
}
const get = initSearchServers();
return createEndpoint({
async export(): Promise<ExportedData> {
const map = await get;
const entries = Array.from(map.entries()).map(async ([k, v]) => {
const data = (await v.export()) as Extract<ExportedData, { type: 'default' }>;
return [k, data.raw];
});
const out: SortedResult[] = [];
if (arr.length === 0) return out;
const results = arr[0].result;
const highlighter = createContentHighlighter(query);
// page id -> heading/content item
const grouped = new Map<string, Doc[]>();
return {
type: 'i18n',
raw: Object.fromEntries(await Promise.all(entries)),
};
},
async search(query, searchOptions) {
const map = await get;
const handler = map.get(searchOptions?.locale ?? options.i18n.defaultLanguage);
for (const id of results) {
const doc = index.get(id);
if (!doc) continue;
let list = grouped.get(doc.page_id);
if (!list) {
list = [];
grouped.set(doc.page_id, list);
}
if (doc.type !== 'page') {
list.push(doc);
}
}
for (const [page_id, items] of grouped) {
const page = index.get(page_id);
if (!page) continue;
out.push({
id: page_id,
type: 'page',
content: highlighter.highlightMarkdown(page.content),
breadcrumbs: page.breadcrumbs,
url: page.url,
});
for (const item of items) {
out.push({
id: item.id,
content: highlighter.highlightMarkdown(item.content),
breadcrumbs: item.breadcrumbs,
type: item.type,
url: item.url,
});
}
}
return out;
if (handler) return handler.search(query, searchOptions);
return [];
},
});
}
export interface FromSourceOptions<C extends LoaderConfig> {
export interface FromSourceOptions<C extends LoaderConfig> extends Pick<
I18nOptions,
'localeMap' | 'document'
> {
buildIndex?: (page: Page<C['source']['pageData']>) => Awaitable<Index>;
}
@@ -109,17 +143,30 @@ export function flexsearchFromSource<C extends LoaderConfig>(
loader: LoaderOutput<C>,
options: FromSourceOptions<NoInfer<C>> = {},
) {
const { buildIndex = buildIndexDefault } = options;
const { buildIndex = buildIndexDefault, ...rest } = options;
function indexes(): Promise<IndexWithLocale[]> {
return Promise.all(
loader.getPages().map(async (page) => {
const index = await buildIndex(page);
return {
...index,
locale: page.locale!,
breadcrumbs: index.breadcrumbs ?? buildBreadcrumbs(loader, page),
};
}),
);
}
if (loader._i18n) {
return flexsearchI18n({
indexes,
i18n: loader._i18n,
...rest,
});
}
return flexsearch({
indexes() {
return Promise.all(
loader.getPages().map(async (page) => {
const index = await buildIndex(page);
if (index.breadcrumbs) return index;
return { ...index, breadcrumbs: buildBreadcrumbsDefault(loader, page) };
}),
);
},
indexes,
...rest,
});
}
packages/core/src/search/flexsearch/utils.ts
+71
View File
@@ -0,0 +1,71 @@
import Search, { type DocumentOptions, type Document, type DocumentData } from 'flexsearch';
import { createContentHighlighter, type SortedResult } from '..';
import type { SharedDocument } from '../server/build-doc';
export type Doc = SharedDocument & DocumentData;
export async function search(index: Document<Doc>, query: string) {
const arr = await index.searchAsync(query, {
index: 'content',
limit: 60,
});
const out: SortedResult[] = [];
if (arr.length === 0) return out;
const results = arr[0].result;
const highlighter = createContentHighlighter(query);
// page id -> heading/content item
const grouped = new Map<string, Doc[]>();
for (const id of results) {
const doc = index.get(id);
if (!doc) continue;
let list = grouped.get(doc.page_id);
if (!list) {
list = [];
grouped.set(doc.page_id, list);
}
if (doc.type !== 'page') {
list.push(doc);
}
}
for (const [page_id, items] of grouped) {
const page = index.get(page_id);
if (!page) continue;
out.push({
id: page_id,
type: 'page',
content: highlighter.highlightMarkdown(page.content),
breadcrumbs: page.breadcrumbs,
url: page.url,
});
for (const item of items) {
out.push({
id: item.id,
content: highlighter.highlightMarkdown(item.content),
breadcrumbs: item.breadcrumbs,
type: item.type,
url: item.url,
});
}
}
return out;
}
export function createDocument(options?: DocumentOptions<Doc>) {
return new Search.Document<Doc>({
tokenize: 'full',
...options,
document: {
id: 'id',
index: ['content'],
store: true,
...options?.document,
},
});
}
packages/core/src/search/orama/create-from-source.ts
-74
View File
@@ -1,74 +0,0 @@
import {
type AdvancedIndex,
type AdvancedOptions,
createI18nSearchAPI,
initAdvancedSearch,
} from './create-server';
import type { LoaderConfig, LoaderOutput, Page } from '@/source';
import type { I18nConfig } from '@/i18n';
import type { Language } from '@orama/orama';
import type { Awaitable } from '@/types';
import { createEndpoint } from '../server/endpoint';
import type { SearchAPI } from '../server/types';
import { buildIndexDefault, buildBreadcrumbsDefault } from '../server/build-index';
interface Options<C extends LoaderConfig> extends Omit<AdvancedOptions, 'indexes'> {
localeMap?: {
[K in C['i18n'] extends I18nConfig<infer Languages> ? Languages : string]?:
| Partial<AdvancedOptions>
| Language;
};
buildIndex?: (page: Page<C['source']['pageData']>) => Awaitable<AdvancedIndex>;
}
export function createFromSource<C extends LoaderConfig>(
source: LoaderOutput<C>,
options?: Options<C>,
): SearchAPI;
export function createFromSource<C extends LoaderConfig>(
source: LoaderOutput<C>,
options: Options<C> = {},
): SearchAPI {
const { buildIndex = buildIndexDefault } = options;
if (source._i18n) {
return createI18nSearchAPI('advanced', {
...options,
i18n: source._i18n,
async indexes() {
const indexes = source.getLanguages().flatMap((entry) => {
return entry.pages.map(async (page) => {
const index = await buildIndex(page);
index.breadcrumbs ??= buildBreadcrumbsDefault(source, page);
return {
...index,
locale: entry.language,
};
});
});
return Promise.all(indexes);
},
});
}
return createEndpoint(
initAdvancedSearch({
...options,
async indexes() {
const indexes = source.getPages().map(async (page) => {
const index = await buildIndex(page);
index.breadcrumbs ??= buildBreadcrumbsDefault(source, page);
return index;
});
return Promise.all(indexes);
},
}),
);
}
export { buildBreadcrumbsDefault };
packages/core/src/search/orama/create-server.ts
+103 -19
View File
@@ -7,7 +7,7 @@ import {
type Language,
type Tokenizer,
} from '@orama/orama';
import type { SearchAPI, SearchServer } from '@/search/server';
import type { QueryOptions, SearchAPI, SearchServer } from '@/search/server';
import type { I18nConfig } from '@/i18n';
import { STEMMERS } from '@/search/orama/_stemmers';
import { createEndpoint } from '../server/endpoint';
@@ -21,19 +21,25 @@ import {
} from './create-db';
import { searchSimple } from './search/simple';
import { searchAdvanced } from './search/advanced';
import type { SharedIndex } from '../server/build-index';
import { buildBreadcrumbs, buildIndexDefault, type SharedIndex } from '../server/build-index';
import type { LoaderConfig, LoaderOutput, Page } from '@/source/loader';
import type { Awaitable } from '@/types';
type OramaInput = Parameters<typeof create>[0];
type SharedOptions = Pick<OramaInput, 'sort' | 'components' | 'plugins'> & {
interface SharedOptions extends Pick<OramaInput, 'sort' | 'components' | 'plugins'> {
language?: string;
tokenizer?: Required<OramaInput>['components']['tokenizer'];
};
}
interface OramaQueryOptions extends QueryOptions {
mode?: 'full' | 'vector';
}
/**
* Resolve indexes dynamically
*/
export type Dynamic<T> = () => T[] | Promise<T[]>;
type Dynamic<T> = () => Awaitable<T[]>;
export interface SimpleOptions extends SharedOptions {
indexes: Index[] | Dynamic<Index>;
@@ -69,7 +75,7 @@ export type ExportedData =
data: Record<string, RawData & { type: 'simple' | 'advanced' }>;
};
export function initSimpleSearch(options: SimpleOptions): SearchServer {
export function initSimpleSearch(options: SimpleOptions): SearchServer<OramaQueryOptions> {
const doc = createDBSimple(options);
return {
@@ -89,7 +95,7 @@ export function initSimpleSearch(options: SimpleOptions): SearchServer {
export type AdvancedIndex = SharedIndex;
export function initAdvancedSearch(options: AdvancedOptions): SearchServer {
export function initAdvancedSearch(options: AdvancedOptions): SearchServer<OramaQueryOptions> {
const get = createDB(options);
return {
@@ -122,17 +128,19 @@ export function initAdvancedSearch(options: AdvancedOptions): SearchServer {
};
}
export function createSearchAPI(type: 'simple', options: SimpleOptions): SearchAPI;
export function createSearchAPI(type: 'advanced', options: AdvancedOptions): SearchAPI;
export function createSearchAPI(
type: 'simple',
options: SimpleOptions,
): SearchAPI<OramaQueryOptions>;
export function createSearchAPI(
type: 'advanced',
options: AdvancedOptions,
): SearchAPI<OramaQueryOptions>;
export function createSearchAPI(
...args: ['simple', SimpleOptions] | ['advanced', AdvancedOptions]
): SearchAPI {
if (args[0] === 'simple') {
return createEndpoint(initSimpleSearch(args[1]));
}
return createEndpoint(initAdvancedSearch(args[1]));
): SearchAPI<OramaQueryOptions> {
return toAPI(args[0] === 'simple' ? initSimpleSearch(args[1]) : initAdvancedSearch(args[1]));
}
type I18nOptions<O extends SimpleOptions | AdvancedOptions, Idx> = Omit<
@@ -162,12 +170,18 @@ function getTokenizer(locale: string): { language: string } | { tokenizer: Token
};
}
export function createI18nSearchAPI(type: 'simple', options: I18nSimpleOptions): SearchAPI;
export function createI18nSearchAPI(type: 'advanced', options: I18nAdvancedOptions): SearchAPI;
export function createI18nSearchAPI(
type: 'simple',
options: I18nSimpleOptions,
): SearchAPI<OramaQueryOptions>;
export function createI18nSearchAPI(
type: 'advanced',
options: I18nAdvancedOptions,
): SearchAPI<OramaQueryOptions>;
export function createI18nSearchAPI(
...[type, options]: ['simple', I18nSimpleOptions] | ['advanced', I18nAdvancedOptions]
): SearchAPI {
): SearchAPI<OramaQueryOptions> {
async function initSearchServers() {
const map = new Map<string, SearchServer>();
if (options.i18n.languages.length === 0) {
@@ -216,8 +230,9 @@ export function createI18nSearchAPI(
return map;
}
const get = initSearchServers();
return createEndpoint({
return toAPI({
async export() {
const map = await get;
const entries = Array.from(map.entries()).map(async ([k, v]) => [k, await v.export()]);
@@ -237,3 +252,72 @@ export function createI18nSearchAPI(
},
});
}
interface Options<C extends LoaderConfig> extends Omit<AdvancedOptions, 'indexes'> {
localeMap?: {
[K in C['i18n'] extends I18nConfig<infer Languages> ? Languages : string]?:
| Partial<AdvancedOptions>
| Language;
};
buildIndex?: (page: Page<C['source']['pageData']>) => Awaitable<AdvancedIndex>;
}
export function createFromSource<C extends LoaderConfig>(
source: LoaderOutput<C>,
options?: Options<C>,
): SearchAPI<OramaQueryOptions>;
export function createFromSource<C extends LoaderConfig>(
source: LoaderOutput<C>,
options: Options<C> = {},
): SearchAPI<OramaQueryOptions> {
const { buildIndex = buildIndexDefault } = options;
if (source._i18n) {
return createI18nSearchAPI('advanced', {
...options,
i18n: source._i18n,
async indexes() {
const indexes = source.getLanguages().flatMap((entry) => {
return entry.pages.map(async (page) => {
const index = await buildIndex(page);
return {
...index,
breadcrumbs: index.breadcrumbs ?? buildBreadcrumbs(source, page),
locale: entry.language,
};
});
});
return Promise.all(indexes);
},
});
}
return toAPI(
initAdvancedSearch({
...options,
async indexes() {
const indexes = source.getPages().map(async (page) => {
const index = await buildIndex(page);
if (index.breadcrumbs) return index;
return { ...index, breadcrumbs: buildBreadcrumbs(source, page) };
});
return Promise.all(indexes);
},
}),
);
}
function toAPI(server: SearchServer<OramaQueryOptions>): SearchAPI<OramaQueryOptions> {
return createEndpoint(server, {
readOptions(url) {
return {
tag: url.searchParams.get('tag')?.split(','),
locale: url.searchParams.get('locale'),
mode: url.searchParams.get('mode') === 'vector' ? 'vector' : 'full',
};
},
});
}
packages/core/src/search/orama/search/advanced.ts
+2 -2
View File
@@ -15,8 +15,9 @@ export async function searchAdvanced(
if (typeof tag === 'string') tag = [tag];
const params = {
...override,
limit: 60,
mode,
...override,
where: removeUndefined({
tags:
tag.length > 0
@@ -26,7 +27,6 @@ export async function searchAdvanced(
: undefined,
...override.where,
}),
limit: 10,
groupBy: {
properties: ['page_id'],
maxResult: 8,
packages/core/src/search/server.ts
-1
View File
@@ -1,3 +1,2 @@
export { createFromSource } from './orama/create-from-source';
export * from './orama/create-server';
export * from './server/types';
packages/core/src/search/server/build-index.ts
+1 -1
View File
@@ -51,7 +51,7 @@ function isBreadcrumbItem(item: unknown): item is string {
return typeof item === 'string' && item.length > 0;
}
export function buildBreadcrumbsDefault<C extends LoaderConfig>(
export function buildBreadcrumbs<C extends LoaderConfig>(
source: LoaderOutput<C>,
page: Page<C['source']['pageData']>,
): string[] | undefined {
packages/core/src/search/server/endpoint.ts
+18 -9
View File
@@ -1,7 +1,22 @@
import type { SearchAPI, SearchServer } from './types';
import type { QueryOptions, SearchAPI, SearchServer } from './types';
export function createEndpoint(server: SearchServer): SearchAPI {
export interface EndpointOptions<Q extends QueryOptions> {
readOptions?: (url: URL, request: Request) => Q;
}
export function createEndpoint<Q extends QueryOptions>(
server: SearchServer<Q>,
options: EndpointOptions<NoInfer<Q>> = {},
): SearchAPI<Q> {
const { search } = server;
const {
readOptions = (url) => {
return {
tag: url.searchParams.get('tag')?.split(','),
locale: url.searchParams.get('locale'),
} as Q;
},
} = options;
return {
...server,
@@ -13,13 +28,7 @@ export function createEndpoint(server: SearchServer): SearchAPI {
const query = url.searchParams.get('query');
if (!query) return Response.json([]);
return Response.json(
await search(query, {
tag: url.searchParams.get('tag')?.split(',') ?? undefined,
locale: url.searchParams.get('locale') ?? undefined,
mode: url.searchParams.get('mode') === 'vector' ? 'vector' : 'full',
}),
);
return Response.json(await search(query, readOptions(url, request)));
},
};
}
packages/core/src/search/server/types.ts
+8 -10
View File
@@ -1,14 +1,12 @@
import type { SortedResult } from '..';
export interface SearchServer {
search: (
query: string,
options?: {
locale?: string;
tag?: string | string[];
mode?: 'vector' | 'full';
},
) => Promise<SortedResult[]>;
export interface QueryOptions {
locale?: string | null;
tag?: string | string[];
}
export interface SearchServer<Q extends QueryOptions = QueryOptions> {
search: (query: string, options?: Q) => Promise<SortedResult[]>;
/**
* Export the database
@@ -18,7 +16,7 @@ export interface SearchServer {
export: () => Promise<unknown>;
}
export interface SearchAPI extends SearchServer {
export interface SearchAPI<Q extends QueryOptions = QueryOptions> extends SearchServer<Q> {
GET: (request: Request) => Promise<Response>;
/**
packages/core/tsdown.config.ts
+1
View File
@@ -16,6 +16,7 @@ export default defineConfig({
'src/source/client/*.{ts,tsx}',
'src/source/plugins/{lucide-icons,slugs,status-badges}.{ts,tsx}',
'src/search/{index,client,server,algolia,orama-cloud,orama-cloud-legacy,mixedbread,flexsearch}.ts',
'src/search/client/*.ts',
'src/utils/use-on-change.ts',
'src/utils/use-media-query.ts',
'src/i18n/*.ts',