mirror of
https://github.com/hanzoai/team.git
synced 2026-09-30 01:57:35 +00:00
Ben SchumacherandGitHub
54438407cf
Add LIKE-based CJK (Chinese, Japanese, Korean) search support for PostgreSQL, gated behind a `CJKSearch` feature flag. PostgreSQL's built-in full-text search (`to_tsvector`/`to_tsquery`) does not support CJK languages because it relies on whitespace-based tokenization, which doesn't work for logographic/syllabic scripts that don't use spaces between words. This PR adds a `LIKE`-based fallback for search terms containing CJK characters. **How it works:** When the `CJKSearch` feature flag is enabled and a search term contains CJK characters (Han, Hiragana, Katakana, or Hangul), the query builder generates `LIKE '%term%'` clauses instead of `to_tsvector @@ to_tsquery` expressions. Case-sensitive `LIKE` is used rather than `ILIKE` since CJK scripts have no letter casing, which also allows potential use of standard B-tree indexes.
37 lines
1.0 KiB
Go
37 lines
1.0 KiB
Go
package model
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestContainsCJK(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
s string
|
|
want bool
|
|
}{
|
|
{name: "empty string", s: "", want: false},
|
|
{name: "latin only", s: "hello world", want: false},
|
|
{name: "chinese characters", s: "你好", want: true},
|
|
{name: "japanese hiragana", s: "こんにちは", want: true},
|
|
{name: "japanese katakana", s: "カタカナ", want: true},
|
|
{name: "korean hangul", s: "안녕하세요", want: true},
|
|
{name: "mixed latin and chinese", s: "hello 你好 world", want: true},
|
|
{name: "mixed latin and japanese", s: "test こんにちは", want: true},
|
|
{name: "cyrillic not CJK", s: "слово", want: false},
|
|
{name: "special characters only", s: "!@#$%^&*()", want: false},
|
|
{name: "numbers only", s: "12345", want: false},
|
|
{name: "single CJK char", s: "中", want: true},
|
|
}
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got := ContainsCJK(tc.s)
|
|
require.Equal(t, tc.want, got)
|
|
})
|
|
}
|
|
}
|