mirror of
https://github.com/hanzokv/go.git
synced 2026-09-24 00:31:51 +00:00
feat(vector): validate FT.HYBRID vector input type. Add proper vector types (#3756)
* fix FT.HYBIRD vector validation * Add missing vector types for FT.HYBRID --------- Co-authored-by: Nedyalko Dyakov <[email protected]>
This commit is contained in:
Yi Deng
co-authored by
Nedyalko Dyakov
Nedyalko Dyakov
parent
5a3f74cc81
commit
5e36de8b1d
+41
-10
@@ -3006,6 +3006,42 @@ func (c cmdable) FTHybrid(ctx context.Context, index string, searchExpr string,
|
||||
return c.FTHybridWithArgs(ctx, index, options)
|
||||
}
|
||||
|
||||
func hybridVectorBlob(v Vector) (interface{}, error) {
|
||||
if v == nil {
|
||||
return nil, fmt.Errorf("FT.HYBRID: vector data is required")
|
||||
}
|
||||
|
||||
switch vector := v.(type) {
|
||||
case *VectorFP32:
|
||||
return hybridVectorBytes(vector.Val)
|
||||
case *VectorFloat16:
|
||||
return hybridVectorBytes(vector.Val)
|
||||
case *VectorBFloat16:
|
||||
return hybridVectorBytes(vector.Val)
|
||||
case *VectorFloat64:
|
||||
return hybridVectorBytes(vector.Val)
|
||||
case *VectorInt8:
|
||||
return hybridVectorBytes(vector.Val)
|
||||
case *VectorUint8:
|
||||
return hybridVectorBytes(vector.Val)
|
||||
case *VectorValues, *VectorRef:
|
||||
return nil, fmt.Errorf("FT.HYBRID: unsupported vector type %T", v)
|
||||
default:
|
||||
values := v.Value()
|
||||
if len(values) < 2 {
|
||||
return nil, fmt.Errorf("FT.HYBRID: vector Value must contain a blob at index 1")
|
||||
}
|
||||
return values[1], nil
|
||||
}
|
||||
}
|
||||
|
||||
func hybridVectorBytes(blob []byte) ([]byte, error) {
|
||||
if len(blob) == 0 {
|
||||
return nil, fmt.Errorf("FT.HYBRID: vector blob is required")
|
||||
}
|
||||
return blob, nil
|
||||
}
|
||||
|
||||
// FTHybridWithArgs - Executes a hybrid search with advanced options
|
||||
// FTHybridWithArgs is still experimental, the command behaviour and signature may change
|
||||
func (c cmdable) FTHybridWithArgs(ctx context.Context, index string, options *FTHybridOptions) *FTHybridCmd {
|
||||
@@ -3032,16 +3068,11 @@ func (c cmdable) FTHybridWithArgs(ctx context.Context, index string, options *FT
|
||||
for _, vectorExpr := range options.VectorExpressions {
|
||||
args = append(args, "VSIM", "@"+vectorExpr.VectorField)
|
||||
|
||||
// For FT.HYBRID, we need to send just the raw vector bytes, not the Value() format
|
||||
// Value() returns [format, data] but FT.HYBRID expects just the blob
|
||||
vectorValue := vectorExpr.VectorData.Value()
|
||||
var vectorBlob interface{}
|
||||
if len(vectorValue) >= 2 {
|
||||
// vectorValue is [format, data, ...] - we only want the data part
|
||||
vectorBlob = vectorValue[1]
|
||||
} else {
|
||||
// Fallback for unexpected format
|
||||
vectorBlob = vectorValue
|
||||
vectorBlob, err := hybridVectorBlob(vectorExpr.VectorData)
|
||||
if err != nil {
|
||||
cmd := newFTHybridCmd(ctx, options, args...)
|
||||
cmd.SetErr(err)
|
||||
return cmd
|
||||
}
|
||||
|
||||
// If VectorParamName is provided, use PARAMS mechanism (required for Redis 8.6+)
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
package redis
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type legacyHybridVector struct {
|
||||
value []any
|
||||
}
|
||||
|
||||
func (v legacyHybridVector) Value() []any {
|
||||
return v.value
|
||||
}
|
||||
|
||||
func TestHybridVectorBlob(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
vector Vector
|
||||
want interface{}
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "fp32",
|
||||
vector: &VectorFP32{Val: []byte{1, 2, 3, 4}},
|
||||
want: []byte{1, 2, 3, 4},
|
||||
},
|
||||
{
|
||||
name: "float16",
|
||||
vector: &VectorFloat16{Val: []byte{1, 2, 3, 4}},
|
||||
want: []byte{1, 2, 3, 4},
|
||||
},
|
||||
{
|
||||
name: "bfloat16",
|
||||
vector: &VectorBFloat16{Val: []byte{1, 2, 3, 4}},
|
||||
want: []byte{1, 2, 3, 4},
|
||||
},
|
||||
{
|
||||
name: "float64",
|
||||
vector: &VectorFloat64{Val: []byte{1, 2, 3, 4}},
|
||||
want: []byte{1, 2, 3, 4},
|
||||
},
|
||||
{
|
||||
name: "int8",
|
||||
vector: &VectorInt8{Val: []byte{1, 2, 3, 4}},
|
||||
want: []byte{1, 2, 3, 4},
|
||||
},
|
||||
{
|
||||
name: "uint8",
|
||||
vector: &VectorUint8{Val: []byte{1, 2, 3, 4}},
|
||||
want: []byte{1, 2, 3, 4},
|
||||
},
|
||||
{
|
||||
name: "custom legacy vector",
|
||||
vector: legacyHybridVector{value: []any{"CUSTOM", []byte{1, 2, 3, 4}}},
|
||||
want: []byte{1, 2, 3, 4},
|
||||
},
|
||||
{
|
||||
name: "nil",
|
||||
vector: nil,
|
||||
wantErr: "vector data is required",
|
||||
},
|
||||
{
|
||||
name: "empty fp32",
|
||||
vector: &VectorFP32{},
|
||||
wantErr: "vector blob is required",
|
||||
},
|
||||
{
|
||||
name: "values",
|
||||
vector: &VectorValues{Val: []float64{1, 2}},
|
||||
wantErr: "unsupported vector type *redis.VectorValues",
|
||||
},
|
||||
{
|
||||
name: "ref",
|
||||
vector: &VectorRef{Name: "vec1"},
|
||||
wantErr: "unsupported vector type *redis.VectorRef",
|
||||
},
|
||||
{
|
||||
name: "invalid custom legacy vector",
|
||||
vector: legacyHybridVector{value: []any{"CUSTOM"}},
|
||||
wantErr: "vector Value must contain a blob at index 1",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := hybridVectorBlob(tt.vector)
|
||||
if tt.wantErr != "" {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error %q, got nil", tt.wantErr)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("expected error containing %q, got %q", tt.wantErr, err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Fatalf("got %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFTHybridWithArgsRejectsUnsupportedVectors(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
vector Vector
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "values",
|
||||
vector: &VectorValues{Val: []float64{1, 2}},
|
||||
wantErr: "unsupported vector type *redis.VectorValues",
|
||||
},
|
||||
{
|
||||
name: "ref",
|
||||
vector: &VectorRef{Name: "vec1"},
|
||||
wantErr: "unsupported vector type *redis.VectorRef",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
m := &mockCmdable{}
|
||||
c := m.asCmdable()
|
||||
cmd := c.FTHybridWithArgs(context.Background(), "idx", &FTHybridOptions{
|
||||
SearchExpressions: []FTHybridSearchExpression{{Query: "*"}},
|
||||
VectorExpressions: []FTHybridVectorExpression{{
|
||||
VectorField: "embedding",
|
||||
VectorData: tt.vector,
|
||||
}},
|
||||
})
|
||||
if cmd.Err() == nil {
|
||||
t.Fatalf("expected error containing %q, got nil", tt.wantErr)
|
||||
}
|
||||
if !strings.Contains(cmd.Err().Error(), tt.wantErr) {
|
||||
t.Fatalf("expected error containing %q, got %q", tt.wantErr, cmd.Err().Error())
|
||||
}
|
||||
if m.lastCmd != nil {
|
||||
t.Fatalf("expected command not to be executed")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFTHybridWithArgsAcceptsVectorFP32(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
vector Vector
|
||||
}{
|
||||
{name: "fp32", vector: &VectorFP32{Val: []byte{1, 2, 3, 4}}},
|
||||
{name: "float16", vector: &VectorFloat16{Val: []byte{1, 2, 3, 4}}},
|
||||
{name: "bfloat16", vector: &VectorBFloat16{Val: []byte{1, 2, 3, 4}}},
|
||||
{name: "float64", vector: &VectorFloat64{Val: []byte{1, 2, 3, 4}}},
|
||||
{name: "int8", vector: &VectorInt8{Val: []byte{1, 2, 3, 4}}},
|
||||
{name: "uint8", vector: &VectorUint8{Val: []byte{1, 2, 3, 4}}},
|
||||
{name: "custom legacy vector", vector: legacyHybridVector{value: []any{"CUSTOM", []byte{1, 2, 3, 4}}}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
m := &mockCmdable{}
|
||||
c := m.asCmdable()
|
||||
cmd := c.FTHybridWithArgs(context.Background(), "idx", &FTHybridOptions{
|
||||
SearchExpressions: []FTHybridSearchExpression{{Query: "*"}},
|
||||
VectorExpressions: []FTHybridVectorExpression{{
|
||||
VectorField: "embedding",
|
||||
VectorData: tt.vector,
|
||||
}},
|
||||
})
|
||||
if cmd.Err() != nil {
|
||||
t.Fatalf("unexpected error: %v", cmd.Err())
|
||||
}
|
||||
if m.lastCmd == nil {
|
||||
t.Fatalf("expected command to be executed")
|
||||
}
|
||||
gotCmd, ok := m.lastCmd.(*FTHybridCmd)
|
||||
if !ok {
|
||||
t.Fatalf("expected FTHybridCmd, got %T", m.lastCmd)
|
||||
}
|
||||
foundBlob := false
|
||||
for _, arg := range gotCmd.args {
|
||||
if blob, ok := arg.([]byte); ok && string(blob) == string([]byte{1, 2, 3, 4}) {
|
||||
foundBlob = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !foundBlob {
|
||||
t.Fatalf("expected raw vector blob in args, got %v", gotCmd.args)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,11 @@ type Vector interface {
|
||||
const (
|
||||
vectorFormatFP32 string = "FP32"
|
||||
vectorFormatValues string = "Values"
|
||||
vectorFormatF16 string = "FLOAT16"
|
||||
vectorFormatBF16 string = "BFLOAT16"
|
||||
vectorFormatF64 string = "FLOAT64"
|
||||
vectorFormatI8 string = "INT8"
|
||||
vectorFormatU8 string = "UINT8"
|
||||
)
|
||||
|
||||
type VectorFP32 struct {
|
||||
@@ -51,6 +56,66 @@ func (v *VectorFP32) Value() []any {
|
||||
|
||||
var _ Vector = (*VectorFP32)(nil)
|
||||
|
||||
// VectorFloat16 represents a FLOAT16-encoded vector blob.
|
||||
// note: intended for search/index query commands such as FT.HYBRID.
|
||||
type VectorFloat16 struct {
|
||||
Val []byte
|
||||
}
|
||||
|
||||
func (v *VectorFloat16) Value() []any {
|
||||
return []any{vectorFormatF16, v.Val}
|
||||
}
|
||||
|
||||
var _ Vector = (*VectorFloat16)(nil)
|
||||
|
||||
// VectorBFloat16 represents a BFLOAT16-encoded vector blob.
|
||||
// note: intended for search/index query commands such as FT.HYBRID.
|
||||
type VectorBFloat16 struct {
|
||||
Val []byte
|
||||
}
|
||||
|
||||
func (v *VectorBFloat16) Value() []any {
|
||||
return []any{vectorFormatBF16, v.Val}
|
||||
}
|
||||
|
||||
var _ Vector = (*VectorBFloat16)(nil)
|
||||
|
||||
// VectorFloat64 represents a FLOAT64-encoded vector blob.
|
||||
// note: intended for search/index query commands such as FT.HYBRID.
|
||||
type VectorFloat64 struct {
|
||||
Val []byte
|
||||
}
|
||||
|
||||
func (v *VectorFloat64) Value() []any {
|
||||
return []any{vectorFormatF64, v.Val}
|
||||
}
|
||||
|
||||
var _ Vector = (*VectorFloat64)(nil)
|
||||
|
||||
// VectorInt8 represents an INT8-encoded vector blob.
|
||||
// note: intended for search/index query commands such as FT.HYBRID.
|
||||
type VectorInt8 struct {
|
||||
Val []byte
|
||||
}
|
||||
|
||||
func (v *VectorInt8) Value() []any {
|
||||
return []any{vectorFormatI8, v.Val}
|
||||
}
|
||||
|
||||
var _ Vector = (*VectorInt8)(nil)
|
||||
|
||||
// VectorUint8 represents a UINT8-encoded vector blob.
|
||||
// note: intended for search/index query commands such as FT.HYBRID.
|
||||
type VectorUint8 struct {
|
||||
Val []byte
|
||||
}
|
||||
|
||||
func (v *VectorUint8) Value() []any {
|
||||
return []any{vectorFormatU8, v.Val}
|
||||
}
|
||||
|
||||
var _ Vector = (*VectorUint8)(nil)
|
||||
|
||||
type VectorValues struct {
|
||||
Val []float64
|
||||
}
|
||||
|
||||
@@ -20,6 +20,51 @@ func TestVectorFP32_Value(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorFloat16_Value(t *testing.T) {
|
||||
v := &VectorFloat16{Val: []byte{1, 2, 3, 4}}
|
||||
got := v.Value()
|
||||
want := []any{"FLOAT16", []byte{1, 2, 3, 4}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("VectorFloat16.Value() = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorBFloat16_Value(t *testing.T) {
|
||||
v := &VectorBFloat16{Val: []byte{1, 2, 3, 4}}
|
||||
got := v.Value()
|
||||
want := []any{"BFLOAT16", []byte{1, 2, 3, 4}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("VectorBFloat16.Value() = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorFloat64_Value(t *testing.T) {
|
||||
v := &VectorFloat64{Val: []byte{1, 2, 3, 4}}
|
||||
got := v.Value()
|
||||
want := []any{"FLOAT64", []byte{1, 2, 3, 4}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("VectorFloat64.Value() = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorInt8_Value(t *testing.T) {
|
||||
v := &VectorInt8{Val: []byte{1, 2, 3, 4}}
|
||||
got := v.Value()
|
||||
want := []any{"INT8", []byte{1, 2, 3, 4}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("VectorInt8.Value() = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorUint8_Value(t *testing.T) {
|
||||
v := &VectorUint8{Val: []byte{1, 2, 3, 4}}
|
||||
got := v.Value()
|
||||
want := []any{"UINT8", []byte{1, 2, 3, 4}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("VectorUint8.Value() = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorValues_Value(t *testing.T) {
|
||||
v := &VectorValues{Val: []float64{1.1, 2.2}}
|
||||
got := v.Value()
|
||||
|
||||
Reference in New Issue
Block a user