Skip to content

Commit

Permalink
Add count and query test cases
Browse files Browse the repository at this point in the history 
Signed-off-by: ThreadDao <yufen.zong@zilliz.com>
  • Loading branch information
@ThreadDao
ThreadDao committed Oct 9, 2023
1 parent b1c9ccb commit c7f581c
Show file tree
Hide file tree
Showing 5 changed files with 524 additions and 103 deletions.
11 changes: 9 additions & 2 deletions test/base/milvus_client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,12 +384,19 @@ func (mc *MilvusClient) DeleteByPks(ctx context.Context, collName string, partit

// Delete deletes entries match expression
func (mc *MilvusClient) Delete(ctx context.Context, collName string, partitionName string, expr string) error {
preRequest("DeleteByPks", ctx, collName, partitionName, expr)
preRequest("Delete", ctx, collName, partitionName, expr)
err := mc.mClient.Delete(ctx, collName, partitionName, expr)
postResponse("DeleteByPks", err)
postResponse("Delete", err)
return err
}

func (mc *MilvusClient) Upsert(ctx context.Context, collName string, partitionName string, columns ...entity.Column) (entity.Column, error) {
preRequest("Upsert", ctx, collName, partitionName, columns)
ids, err := mc.mClient.Upsert(ctx, collName, partitionName, columns...)
postResponse("Upsert", err, ids)
return ids, err
}

// Search search from collection
func (mc *MilvusClient) Search(ctx context.Context, collName string, partitions []string, expr string,
outputFields []string, vectors []entity.Vector, vectorField string, metricType entity.MetricType, topK int, sp entity.SearchParam, opts ...client.SearchQueryOptionFunc,
Expand Down
25 changes: 25 additions & 0 deletions test/common/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ const (
const (
MaxPartitionNum = 4096
DefaultDynamicFieldName = "$meta"
QueryCountFieldName = "count(*)"
DefaultPartition = "_default"
DefaultIndexName = "_default_idx_102"
DefaultIndexNameBinary = "_default_idx_100"
Expand Down Expand Up @@ -744,4 +745,28 @@ func GenSearchVectors(nq int, dim int64, dataType entity.FieldType) []entity.Vec
return vectors
}

// invalid expr
type InvalidExprStruct struct {
Expr string
ErrNil bool
ErrMsg string
}

var InvalidExpressions = []InvalidExprStruct{
{Expr: "id in [0]", ErrNil: true, ErrMsg: "fieldName(id) not found"}, // not exist field but no error
{Expr: "int64 in not [0]", ErrNil: false, ErrMsg: "cannot parse expression"}, // wrong term expr keyword
{Expr: "int64 < floatVec", ErrNil: false, ErrMsg: "not supported"}, // unsupported compare field
{Expr: "floatVec in [0]", ErrNil: false, ErrMsg: "cannot be casted to FloatVector"}, // value and field type mismatch
{Expr: fmt.Sprintf("%s == 1", DefaultJSONFieldName), ErrNil: false, ErrMsg: "can not comparisons jsonField directly"},
{Expr: fmt.Sprintf("%s == 1", DefaultDynamicFieldName), ErrNil: false, ErrMsg: "can not comparisons jsonField directly"},
{Expr: fmt.Sprintf("%s[\"dynamicList\"] == [2, 3]", DefaultDynamicFieldName), ErrNil: true, ErrMsg: ""},
{Expr: fmt.Sprintf("%s['a'] == [2, 3]", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // json field not exist
{Expr: fmt.Sprintf("%s['number'] == [2, 3]", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // field exist but type not match
{Expr: fmt.Sprintf("json_contains (%s['number'], 2)", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""},
{Expr: fmt.Sprintf("json_contains (%s['list'], [2])", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""},
{Expr: fmt.Sprintf("json_contains_all (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "contains_all operation element must be an array"},
{Expr: fmt.Sprintf("JSON_CONTAINS_ANY (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "contains_any operation element must be an array"},
{Expr: fmt.Sprintf("json_contains_aby (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "invalid expression: json_contains_aby"},
}

// --- search utils ---
127 changes: 127 additions & 0 deletions test/testcases/delete_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,3 +265,130 @@ func TestDeleteDuplicatedPks(t *testing.T) {
common.CheckErr(t, errQuery, true)
require.Zero(t, queryRes[0].Len())
}

func TestDeleteExpressions(t *testing.T) {
ctx := createContext(t, time.Second*common.DefaultTimeout)
// connect
mc := createMilvusClient(ctx, t)

// create collection
cp := CollectionParams{
CollectionFieldsType: Int64FloatVecJSON,
AutoID: false,
EnableDynamicField: true,
ShardsNum: common.DefaultShards,
Dim: common.DefaultDim,
}
collName := createCollection(ctx, t, mc, cp, client.WithConsistencyLevel(entity.ClStrong))

// insert
dp := DataParams{
CollectionName: collName,
PartitionName: "",
CollectionFieldsType: Int64FloatVecJSON,
start: 0,
nb: common.DefaultNb,
dim: common.DefaultDim,
EnableDynamicField: true,
}
_, _ = insertData(ctx, t, mc, dp)

idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96)
_ = mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false)

// Load collection
errLoad := mc.LoadCollection(ctx, collName, false)
common.CheckErr(t, errLoad, true)

// query with different expr and count
type exprCount struct {
expr string
count int64
}
exprCounts := []exprCount{
// pk int64 field expr: < in && ||
{expr: fmt.Sprintf("%s < 1000", common.DefaultIntFieldName), count: 1000},
{expr: fmt.Sprintf("%s in [0, 1, 2]", common.DefaultIntFieldName), count: 3},
{expr: fmt.Sprintf("%s >= 1000 && %s < 2000", common.DefaultIntFieldName, common.DefaultIntFieldName), count: 1000},
{expr: fmt.Sprintf("%s >= 1000 || %s > 2000", common.DefaultIntFieldName, common.DefaultIntFieldName), count: 2000},
{expr: fmt.Sprintf("%s < 1000", common.DefaultFloatFieldName), count: 1000},

// json and dynamic field filter expr: == < in bool/ list/ int
{expr: fmt.Sprintf("%s['number'] == 0", common.DefaultJSONFieldName), count: 1500},
{expr: fmt.Sprintf("%s['number'] < 100 and %s['number'] != 0", common.DefaultJSONFieldName, common.DefaultJSONFieldName), count: 50},
{expr: fmt.Sprintf("%s < 100", common.DefaultDynamicNumberField), count: 100},
{expr: "dynamicNumber % 2 == 0", count: 1500},
{expr: fmt.Sprintf("%s['bool'] == true", common.DefaultJSONFieldName), count: 1500},
{expr: fmt.Sprintf("%s == false", common.DefaultDynamicBoolField), count: 2000},
{expr: fmt.Sprintf("%s in ['1', '2'] ", common.DefaultDynamicStringField), count: 2},
{expr: fmt.Sprintf("%s['string'] in ['1', '2', '5'] ", common.DefaultJSONFieldName), count: 3},
{expr: fmt.Sprintf("%s['list'] == [1, 2] ", common.DefaultJSONFieldName), count: 1},
{expr: fmt.Sprintf("%s['list'] == [0, 1] ", common.DefaultJSONFieldName), count: 0},
{expr: fmt.Sprintf("%s['list'][0] < 10 ", common.DefaultJSONFieldName), count: 5},
{expr: fmt.Sprintf("%s[\"dynamicList\"] != [2, 3]", common.DefaultDynamicFieldName), count: 0},

// json contains
{expr: fmt.Sprintf("json_contains (%s['list'], 2)", common.DefaultJSONFieldName), count: 1},
{expr: fmt.Sprintf("json_contains (%s['number'], 0)", common.DefaultJSONFieldName), count: 0},
{expr: fmt.Sprintf("json_contains_all (%s['list'], [1, 2])", common.DefaultJSONFieldName), count: 1},
{expr: fmt.Sprintf("JSON_CONTAINS_ANY (%s['list'], [1, 3])", common.DefaultJSONFieldName), count: 2},
// string like
{expr: "dynamicString like '1%' ", count: 1111},

// key exist
{expr: fmt.Sprintf("exists %s['list']", common.DefaultJSONFieldName), count: common.DefaultNb},
{expr: fmt.Sprintf("exists a "), count: 0},
{expr: fmt.Sprintf("exists %s ", common.DefaultDynamicListField), count: 0},
{expr: fmt.Sprintf("exists %s ", common.DefaultDynamicStringField), count: common.DefaultNb},
// data type not match and no error
{expr: fmt.Sprintf("%s['number'] == '0' ", common.DefaultJSONFieldName), count: 0},
}

for _, _exprCount := range exprCounts {
err := mc.Delete(ctx, collName, "", _exprCount.expr)
common.CheckErr(t, err, true)

// query
countRes, _ := mc.Query(ctx, collName, []string{common.DefaultPartition}, _exprCount.expr, []string{common.QueryCountFieldName})
require.Equal(t, int64(0), countRes.GetColumn(common.QueryCountFieldName).(*entity.ColumnInt64).Data()[0])
}
}

func TestDeleteInvalidExpr(t *testing.T) {
t.Skip("invalid error message like failed to create expr plan or collection not loaded")
t.Parallel()
ctx := createContext(t, time.Second*common.DefaultTimeout)
// connect
mc := createMilvusClient(ctx, t)

// create collection
cp := CollectionParams{
CollectionFieldsType: Int64FloatVecJSON,
AutoID: false,
EnableDynamicField: true,
ShardsNum: common.DefaultShards,
Dim: common.DefaultDim,
}
collName := createCollection(ctx, t, mc, cp)

// insert
dp := DataParams{
CollectionName: collName,
PartitionName: "",
CollectionFieldsType: Int64FloatVecJSON,
start: 0,
nb: common.DefaultNb,
dim: common.DefaultDim,
EnableDynamicField: true,
}
_, _ = insertData(ctx, t, mc, dp)

err := mc.Delete(ctx, collName, "", "")
common.CheckErr(t, err, false, "invalid expression: expected=valid expr, actual=empty expr: invalid parameter")

for _, _invalidExprs := range common.InvalidExpressions {
err := mc.Delete(ctx, collName, "", _invalidExprs.Expr)
common.CheckErr(t, err, _invalidExprs.ErrNil, _invalidExprs.ErrMsg)
}
}

Loading

0 comments on commit c7f581c

Please sign in to comment.