Skip to content

Commit

Permalink
Fix filtered query and _id validation
Browse files Browse the repository at this point in the history 
The query filters are now also correctly recursively validated.
  • Loading branch information
@nuric
nuric committed Sep 17, 2024
1 parent 29bfcfa commit 954c50d
Show file tree
Hide file tree
Showing 3 changed files with 247 additions and 48 deletions.
97 changes: 50 additions & 47 deletions models/index_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,55 +48,58 @@ func TestIndexSchema_Validate_Haversine(t *testing.T) {
}
}

func TestIndexSchema_CheckCompatibleMap(t *testing.T) {
// Check if the schema is compatible with a map
// ---------------------------
// Here is a kitchen sink schema
schema := models.IndexSchema{
"propVectorFlat": models.IndexSchemaValue{
Type: models.IndexTypeVectorFlat,
VectorFlat: &models.IndexVectorFlatParameters{
DistanceMetric: models.DistanceEuclidean,
VectorSize: 2,
},
},
"propVectorVamana": models.IndexSchemaValue{
Type: models.IndexTypeVectorVamana,
VectorVamana: &models.IndexVectorVamanaParameters{
DistanceMetric: models.DistanceEuclidean,
VectorSize: 2,
},
},
"propText": models.IndexSchemaValue{
Type: models.IndexTypeText,
Text: &models.IndexTextParameters{
Analyser: "standard",
},
},
"propString": models.IndexSchemaValue{
Type: models.IndexTypeString,
String: &models.IndexStringParameters{
// ---------------------------
// Here is a kitchen sink schema
var sampleSchema models.IndexSchema = models.IndexSchema{
"propVectorFlat": models.IndexSchemaValue{
Type: models.IndexTypeVectorFlat,
VectorFlat: &models.IndexVectorFlatParameters{
DistanceMetric: models.DistanceEuclidean,
VectorSize: 2,
},
},
"propVectorVamana": models.IndexSchemaValue{
Type: models.IndexTypeVectorVamana,
VectorVamana: &models.IndexVectorVamanaParameters{
DistanceMetric: models.DistanceEuclidean,
VectorSize: 2,
},
},
"propText": models.IndexSchemaValue{
Type: models.IndexTypeText,
Text: &models.IndexTextParameters{
Analyser: "standard",
},
},
"propString": models.IndexSchemaValue{
Type: models.IndexTypeString,
String: &models.IndexStringParameters{
CaseSensitive: false,
},
},
"propInteger": models.IndexSchemaValue{
Type: models.IndexTypeInteger,
},
"propFloat": models.IndexSchemaValue{
Type: models.IndexTypeFloat,
},
"propStringArray": models.IndexSchemaValue{
Type: models.IndexTypeStringArray,
StringArray: &models.IndexStringArrayParameters{
IndexStringParameters: models.IndexStringParameters{
CaseSensitive: false,
},
},
"propInteger": models.IndexSchemaValue{
Type: models.IndexTypeInteger,
},
"propFloat": models.IndexSchemaValue{
Type: models.IndexTypeFloat,
},
"propStringArray": models.IndexSchemaValue{
Type: models.IndexTypeStringArray,
StringArray: &models.IndexStringArrayParameters{
IndexStringParameters: models.IndexStringParameters{
CaseSensitive: false,
},
},
},
"nested.propInteger": models.IndexSchemaValue{
Type: models.IndexTypeInteger,
},
}
},
"nested.propInteger": models.IndexSchemaValue{
Type: models.IndexTypeInteger,
},
}

// ---------------------------

func TestIndexSchema_CheckCompatibleMap(t *testing.T) {
// Check if the schema is compatible with a map
// ---------------------------
tests := []struct {
name string
Expand Down Expand Up @@ -205,7 +208,7 @@ func TestIndexSchema_CheckCompatibleMap(t *testing.T) {
var m models.PointAsMap
err := json.Unmarshal([]byte(tt.jsonString), &m)
require.NoError(t, err)
err = schema.CheckCompatibleMap(m)
err = sampleSchema.CheckCompatibleMap(m)
if tt.fail {
require.Error(t, err)
} else {
Expand Down
29 changes: 28 additions & 1 deletion models/search.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
package models

import "fmt"
import (
"fmt"

"github.com/google/uuid"
)

/* The search query design is based on the following key steps:
*
Expand Down Expand Up @@ -59,10 +63,18 @@ func (q Query) Validate(schema IndexSchema) error {
if q.String.Operator != OperatorEquals {
return fmt.Errorf("invalid operator %s for %s, expected %s", q.String.Operator, q.Property, OperatorEquals)
}
if _, err := uuid.Parse(q.String.Value); err != nil {
return fmt.Errorf("invalid UUID %v for %s, %v", q.String.Value, q.Property, err)
}
case q.StringArray != nil:
if q.StringArray.Operator != OperatorContainsAny {
return fmt.Errorf("invalid operator %s for %s, expected %s", q.StringArray.Operator, q.Property, OperatorContainsAny)
}
for _, v := range q.StringArray.Value {
if _, err := uuid.Parse(v); err != nil {
return fmt.Errorf("invalid UUID %s for %s, %v", v, q.Property, err)
}
}
default:
return fmt.Errorf("invalid query for _id, expected string or stringArray")
}
Expand All @@ -82,6 +94,11 @@ func (q Query) Validate(schema IndexSchema) error {
if len(q.VectorFlat.Vector) != int(value.VectorFlat.VectorSize) {
return fmt.Errorf("vectorFlat query vector length mismatch for property %s, expected %d got %d", q.Property, value.VectorFlat.VectorSize, len(q.VectorFlat.Vector))
}
if q.VectorFlat.Filter != nil {
if err := q.VectorFlat.Filter.Validate(schema); err != nil {
return err
}
}
case IndexTypeVectorVamana:
if q.VectorVamana == nil {
return fmt.Errorf("vectorVamana query options not provided for property %s", q.Property)
Expand All @@ -92,10 +109,20 @@ func (q Query) Validate(schema IndexSchema) error {
if q.VectorVamana.SearchSize < q.VectorVamana.Limit {
return fmt.Errorf("searchSize must be greater than or equal to limit for property %s", q.Property)
}
if q.VectorVamana.Filter != nil {
if err := q.VectorVamana.Filter.Validate(schema); err != nil {
return err
}
}
case IndexTypeText:
if q.Text == nil {
return fmt.Errorf("text query options not provided for property %s", q.Property)
}
if q.Text.Filter != nil {
if err := q.Text.Filter.Validate(schema); err != nil {
return err
}
}
case IndexTypeString:
if q.String == nil {
return fmt.Errorf("string query options not provided for property %s", q.Property)
Expand Down
169 changes: 169 additions & 0 deletions models/search_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
package models_test

import (
"testing"

"github.com/semafind/semadb/models"
"github.com/stretchr/testify/require"
)

func TestSearch_QueryValidate(t *testing.T) {
// ---------------------------
tests := []struct {
name string
query models.Query
fail bool
}{
{
name: "_id String Non-UUID",
query: models.Query{
Property: "_id",
String: &models.SearchStringOptions{
Operator: models.OperatorEquals,
},
},
fail: true,
},
{
name: "_id String Valid",
query: models.Query{
Property: "_id",
String: &models.SearchStringOptions{
Operator: models.OperatorEquals,
Value: "123e4567-e89b-12d3-a456-426614174000",
},
},
fail: false,
},
{
name: "_id StringArray Non-UUID",
query: models.Query{
Property: "_id",
StringArray: &models.SearchStringArrayOptions{
Operator: models.OperatorContainsAny,
Value: []string{"123e4567-e89b-12d3-a456-426614174000", "gandalf"},
},
},
fail: true,
},
{
name: "_id StringArray Non-UUID",
query: models.Query{
Property: "_id",
StringArray: &models.SearchStringArrayOptions{
Operator: models.OperatorContainsAny,
Value: []string{"123e4567-e89b-12d3-a456-426614174000"},
},
},
fail: false,
},
{
name: "Invalid vector vamana length",
query: models.Query{
Property: "propVectorVamana",
VectorVamana: &models.SearchVectorVamanaOptions{
Vector: []float32{1.0},
},
},
fail: true,
},
{
name: "Invalid Vector Vamana filter",
query: models.Query{
Property: "propVectorVamana",
VectorVamana: &models.SearchVectorVamanaOptions{
Vector: []float32{1.0, 2.0},
SearchSize: 10,
Limit: 10,
Filter: &models.Query{
Property: "propInteger",
Float: &models.SearchFloatOptions{
Operator: models.OperatorEquals,
Value: 1.0,
},
},
},
},
fail: true,
},
{
name: "Invalid flat vector filter",
query: models.Query{
Property: "propVectorFlat",
VectorFlat: &models.SearchVectorFlatOptions{
Vector: []float32{1.0, 2.0},
Filter: &models.Query{
Property: "propString",
Float: &models.SearchFloatOptions{
Operator: models.OperatorEquals,
Value: 1.0,
},
},
},
},
fail: true,
},
{
name: "Invalid text filter",
query: models.Query{
Property: "propText",
Text: &models.SearchTextOptions{
Value: "text",
Filter: &models.Query{
Property: "propString",
Float: &models.SearchFloatOptions{
Operator: models.OperatorEquals,
Value: 1.0,
},
},
},
},
fail: true,
},
{
name: "Valid composite query",
query: models.Query{
Property: "_and",
And: []models.Query{
{
Property: "propString",
String: &models.SearchStringOptions{
Operator: models.OperatorEquals,
Value: "string",
},
},
{
Property: "_or",
Or: []models.Query{
{
Property: "propFloat",
Float: &models.SearchFloatOptions{
Operator: models.OperatorEquals,
Value: 1.0,
},
},
},
},
},
},
},
{
name: "Non-existent property",
query: models.Query{
Property: "nonExistent",
},
fail: true,
},
}
// ---------------------------
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.query.Validate(sampleSchema)
if tt.fail {
require.Error(t, err)
} else {
require.NoError(t, err)
}
})
}
}

0 comments on commit 954c50d

Please sign in to comment.