diff --git a/entity/schema.go b/entity/schema.go index 6b31bc08..03086bfa 100644 --- a/entity/schema.go +++ b/entity/schema.go @@ -419,6 +419,8 @@ func (t FieldType) String() string { return "[]byte" case FieldTypeBFloat16Vector: return "[]byte" + case FieldTypeSparseVector: + return "[]SparseEmbedding" default: return "undefined" } diff --git a/test/base/milvus_client.go b/test/base/milvus_client.go index 4505c29f..e3a41e89 100644 --- a/test/base/milvus_client.go +++ b/test/base/milvus_client.go @@ -487,6 +487,15 @@ func (mc *MilvusClient) Get(ctx context.Context, collName string, ids entity.Col return queryResults, err } +// QueryIterator QueryIterator from collection +func (mc *MilvusClient) QueryIterator(ctx context.Context, opt *client.QueryIteratorOption) (*client.QueryIterator, error) { + funcName := "QueryIterator" + preRequest(funcName, ctx, opt) + itr, err := mc.mClient.QueryIterator(ctx, opt) + postResponse(funcName, err, itr) + return itr, err +} + // -- row based apis -- // CreateCollectionByRow Create Collection By Row diff --git a/test/common/response_check.go b/test/common/response_check.go index 869d6109..aa118e99 100644 --- a/test/common/response_check.go +++ b/test/common/response_check.go @@ -1,7 +1,9 @@ package common import ( + "context" "fmt" + "io" "log" "strings" "testing" @@ -160,14 +162,48 @@ func EqualColumn(t *testing.T, columnA entity.Column, columnB entity.Column) { require.ElementsMatch(t, columnA.(*entity.ColumnFloatVector).Data(), columnB.(*entity.ColumnFloatVector).Data()) case entity.FieldTypeBinaryVector: require.ElementsMatch(t, columnA.(*entity.ColumnBinaryVector).Data(), columnB.(*entity.ColumnBinaryVector).Data()) + case entity.FieldTypeFloat16Vector: + require.ElementsMatch(t, columnA.(*entity.ColumnFloat16Vector).Data(), columnB.(*entity.ColumnFloat16Vector).Data()) + case entity.FieldTypeBFloat16Vector: + require.ElementsMatch(t, columnA.(*entity.ColumnBFloat16Vector).Data(), columnB.(*entity.ColumnBFloat16Vector).Data()) + case entity.FieldTypeSparseVector: + require.ElementsMatch(t, columnA.(*entity.ColumnSparseFloatVector).Data(), columnB.(*entity.ColumnSparseFloatVector).Data()) case entity.FieldTypeArray: - log.Println("TODO support column element type") + EqualArrayColumn(t, columnA, columnB) default: log.Printf("The column type not in: [%v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v]", entity.FieldTypeBool, entity.FieldTypeInt8, entity.FieldTypeInt16, entity.FieldTypeInt32, entity.FieldTypeInt64, entity.FieldTypeFloat, entity.FieldTypeDouble, entity.FieldTypeString, entity.FieldTypeVarChar, entity.FieldTypeArray, entity.FieldTypeFloatVector, entity.FieldTypeBinaryVector) + } +} +// EqualColumn assert field data is equal of two columns +func EqualArrayColumn(t *testing.T, columnA entity.Column, columnB entity.Column) { + require.Equal(t, columnA.Name(), columnB.Name()) + require.IsType(t, columnA.Type(), entity.FieldTypeArray) + require.IsType(t, columnB.Type(), entity.FieldTypeArray) + switch columnA.(type) { + case *entity.ColumnBoolArray: + require.ElementsMatch(t, columnA.(*entity.ColumnBoolArray).Data(), columnB.(*entity.ColumnBoolArray).Data()) + case *entity.ColumnInt8Array: + require.ElementsMatch(t, columnA.(*entity.ColumnInt8Array).Data(), columnB.(*entity.ColumnInt8Array).Data()) + case *entity.ColumnInt16Array: + require.ElementsMatch(t, columnA.(*entity.ColumnInt16Array).Data(), columnB.(*entity.ColumnInt16Array).Data()) + case *entity.ColumnInt32Array: + require.ElementsMatch(t, columnA.(*entity.ColumnInt32Array).Data(), columnB.(*entity.ColumnInt32Array).Data()) + case *entity.ColumnInt64Array: + require.ElementsMatch(t, columnA.(*entity.ColumnInt64Array).Data(), columnB.(*entity.ColumnInt64Array).Data()) + case *entity.ColumnFloatArray: + require.ElementsMatch(t, columnA.(*entity.ColumnFloatArray).Data(), columnB.(*entity.ColumnFloatArray).Data()) + case *entity.ColumnDoubleArray: + require.ElementsMatch(t, columnA.(*entity.ColumnDoubleArray).Data(), columnB.(*entity.ColumnDoubleArray).Data()) + case *entity.ColumnVarCharArray: + require.ElementsMatch(t, columnA.(*entity.ColumnVarCharArray).Data(), columnB.(*entity.ColumnVarCharArray).Data()) + default: + log.Printf("Now support array type: [%v, %v, %v, %v, %v, %v, %v, %v]", + entity.FieldTypeBool, entity.FieldTypeInt8, entity.FieldTypeInt16, entity.FieldTypeInt32, + entity.FieldTypeInt64, entity.FieldTypeFloat, entity.FieldTypeDouble, entity.FieldTypeVarChar) } } @@ -203,6 +239,75 @@ func CheckSearchResult(t *testing.T, actualSearchResults []client.SearchResult, } +func EqualIntSlice(a []int, b []int) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +type CheckIteratorOption func(opt *checkIteratorOpt) + +type checkIteratorOpt struct { + expBatchSize []int + expOutputFields []string +} + +func WithExpBatchSize(expBatchSize []int) CheckIteratorOption { + return func(opt *checkIteratorOpt) { + opt.expBatchSize = expBatchSize + } +} + +func WithExpOutputFields(expOutputFields []string) CheckIteratorOption { + return func(opt *checkIteratorOpt) { + opt.expOutputFields = expOutputFields + } +} + +// check queryIterator: result limit, each batch size, output fields +func CheckQueryIteratorResult(ctx context.Context, t *testing.T, itr *client.QueryIterator, expLimit int, opts ...CheckIteratorOption) { + opt := &checkIteratorOpt{} + for _, o := range opts { + o(opt) + } + actualLimit := 0 + var actualBatchSize []int + for { + rs, err := itr.Next(ctx) + if err != nil { + if err == io.EOF { + break + } + log.Fatalf("QueryIterator next gets error: %v", err) + } + //log.Printf("QueryIterator result len: %d", rs.Len()) + //log.Printf("QueryIterator result data: %d", rs.GetColumn("int64")) + + if opt.expBatchSize != nil { + actualBatchSize = append(actualBatchSize, rs.Len()) + } + var actualOutputFields []string + if opt.expOutputFields != nil { + for _, column := range rs { + actualOutputFields = append(actualOutputFields, column.Name()) + } + require.ElementsMatch(t, opt.expOutputFields, actualOutputFields) + } + actualLimit = actualLimit + rs.Len() + } + require.Equal(t, expLimit, actualLimit) + if opt.expBatchSize != nil { + log.Printf("QueryIterator result len: %v", actualBatchSize) + require.True(t, EqualIntSlice(opt.expBatchSize, actualBatchSize)) + } +} + // CheckPersistentSegments check persistent segments func CheckPersistentSegments(t *testing.T, actualSegments []*entity.Segment, expNb int64) { actualNb := int64(0) diff --git a/test/common/utils.go b/test/common/utils.go index c0fc80eb..6df077ae 100644 --- a/test/common/utils.go +++ b/test/common/utils.go @@ -78,6 +78,7 @@ const ( DefaultPartitionNum = 16 // default num_partitions MaxTopK = 16384 MaxVectorFieldNum = 4 + DefaultBatchSize = 1000 ) var IndexStateValue = map[string]int32{ @@ -842,7 +843,7 @@ func GenDefaultJSONRows(start int, nb int, dim int64, enableDynamicField bool) [ } for i := start; i < start+nb; i++ { - // jsonStruct row and dynamic row + //jsonStruct row and dynamic row var jsonStruct JSONStruct if i%2 == 0 { jsonStruct = JSONStruct{ @@ -1386,6 +1387,8 @@ type InvalidExprStruct struct { var InvalidExpressions = []InvalidExprStruct{ {Expr: "id in [0]", ErrNil: true, ErrMsg: "fieldName(id) not found"}, // not exist field but no error {Expr: "int64 in not [0]", ErrNil: false, ErrMsg: "cannot parse expression"}, // wrong term expr keyword + {Expr: "int64 > 10 AND int64 < 100", ErrNil: false, ErrMsg: "cannot parse expression"}, // AND isn't supported + {Expr: "int64 < 10 OR int64 > 100", ErrNil: false, ErrMsg: "cannot parse expression"}, // OR isn't supported {Expr: "int64 < floatVec", ErrNil: false, ErrMsg: "not supported"}, // unsupported compare field {Expr: "floatVec in [0]", ErrNil: false, ErrMsg: "cannot be casted to FloatVector"}, // value and field type mismatch {Expr: fmt.Sprintf("%s == 1", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // hist empty @@ -1406,4 +1409,23 @@ var InvalidExpressions = []InvalidExprStruct{ {Expr: fmt.Sprintf(fmt.Sprintf("%s[-1] > 1", DefaultJSONFieldName)), ErrNil: false, ErrMsg: "invalid expression"}, // json[-1] > } +func GenBatchSizes(limit int, batch int) []int { + if batch == 0 { + log.Fatal("Batch should be larger than 0") + } + if limit == 0 { + return []int{} + } + _loop := limit / batch + _last := limit % batch + batchSizes := make([]int, 0, _loop+1) + for i := 0; i < _loop; i++ { + batchSizes = append(batchSizes, batch) + } + if _last > 0 { + batchSizes = append(batchSizes, _last) + } + return batchSizes +} + // --- search utils --- diff --git a/test/testcases/configure_test.go b/test/testcases/configure_test.go index 115f76d6..79ace802 100644 --- a/test/testcases/configure_test.go +++ b/test/testcases/configure_test.go @@ -74,7 +74,7 @@ func TestCompactAfterDelete(t *testing.T) { common.CheckErr(t, err, true) // delete half ids - deleteIds := entity.NewColumnInt64(common.DefaultIntFieldName, ids.(*entity.ColumnInt64).Data()[:common.DefaultNb/2]) + deleteIds := ids.Slice(0, common.DefaultNb/2) errDelete := mc.DeleteByPks(ctx, collName, "", deleteIds) common.CheckErr(t, errDelete, true) diff --git a/test/testcases/delete_test.go b/test/testcases/delete_test.go index 6ba19f7b..0ae6b24e 100644 --- a/test/testcases/delete_test.go +++ b/test/testcases/delete_test.go @@ -28,7 +28,7 @@ func TestDelete(t *testing.T) { common.CheckErr(t, errLoad, true) // delete - deleteIds := entity.NewColumnInt64(common.DefaultIntFieldName, ids.(*entity.ColumnInt64).Data()[:10]) + deleteIds := ids.Slice(0, 10) errDelete := mc.DeleteByPks(ctx, collName, common.DefaultPartition, deleteIds) common.CheckErr(t, errDelete, true) @@ -48,7 +48,7 @@ func TestDeleteStringPks(t *testing.T) { collName, ids := createVarcharCollectionWithDataIndex(ctx, t, mc, true, client.WithConsistencyLevel(entity.ClStrong)) // delete - deleteIds := entity.NewColumnVarChar(common.DefaultVarcharFieldName, ids.(*entity.ColumnVarChar).Data()[:10]) + deleteIds := ids.Slice(0, 10) errDelete := mc.DeleteByPks(ctx, collName, common.DefaultPartition, deleteIds) common.CheckErr(t, errDelete, true) @@ -103,7 +103,7 @@ func TestDeleteNotExistPartition(t *testing.T) { common.CheckErr(t, errLoad, true) // delete - deleteIds := entity.NewColumnInt64(common.DefaultIntFieldName, ids.(*entity.ColumnInt64).Data()[:10]) + deleteIds := ids.Slice(0, 10) errDelete := mc.DeleteByPks(ctx, collName, "p1", deleteIds) common.CheckErr(t, errDelete, false, fmt.Sprintf("partition p1 of collection %s does not exist", collName)) } @@ -125,7 +125,7 @@ func TestDeleteEmptyPartitionNames(t *testing.T) { mc.Flush(ctx, collName, false) // delete - deleteIds := entity.NewColumnInt64(common.DefaultIntFieldName, intColumn.(*entity.ColumnInt64).Data()[:10]) + deleteIds := intColumn.Slice(0, 10) errDelete := mc.DeleteByPks(ctx, collName, emptyPartitionName, deleteIds) common.CheckErr(t, errDelete, true) @@ -160,7 +160,7 @@ func TestDeleteEmptyPartition(t *testing.T) { common.CheckErr(t, errLoad, true) // delete from empty partition p1 - deleteIds := entity.NewColumnInt64(common.DefaultIntFieldName, ids.(*entity.ColumnInt64).Data()[:10]) + deleteIds := ids.Slice(0, 10) errDelete := mc.DeleteByPks(ctx, collName, "p1", deleteIds) common.CheckErr(t, errDelete, true) @@ -186,7 +186,7 @@ func TestDeletePartitionIdsNotMatch(t *testing.T) { partitionName, vecColumnDefault, _ := createInsertTwoPartitions(ctx, t, mc, collName, common.DefaultNb) // delete [0:10) from new partition -> delete nothing - deleteIds := entity.NewColumnInt64(common.DefaultIntFieldName, vecColumnDefault.IdsColumn.(*entity.ColumnInt64).Data()[:10]) + deleteIds := vecColumnDefault.IdsColumn.Slice(0, 10) errDelete := mc.DeleteByPks(ctx, collName, partitionName, deleteIds) common.CheckErr(t, errDelete, true) diff --git a/test/testcases/highlevel_test.go b/test/testcases/highlevel_test.go index 0cf02b50..03b6e8c6 100644 --- a/test/testcases/highlevel_test.go +++ b/test/testcases/highlevel_test.go @@ -65,13 +65,13 @@ func TestNewCollection(t *testing.T) { queryResult, err := mc.Get( ctx, collName, - entity.NewColumnInt64(DefaultPkFieldName, pkColumn.(*entity.ColumnInt64).Data()[:10]), + pkColumn.Slice(0, 10), ) common.CheckErr(t, err, true) common.CheckOutputFields(t, queryResult, []string{DefaultPkFieldName, DefaultVectorFieldName}) common.CheckQueryResult(t, queryResult, []entity.Column{ - entity.NewColumnInt64(DefaultPkFieldName, pkColumn.(*entity.ColumnInt64).Data()[:10]), - entity.NewColumnFloatVector(DefaultVectorFieldName, int(common.DefaultDim), vecColumn.(*entity.ColumnFloatVector).Data()[:10]), + pkColumn.Slice(0, 10), + vecColumn.Slice(0, 10), }) // search @@ -142,13 +142,13 @@ func TestNewCollectionCustomize(t *testing.T) { queryResult, err := mc.Get( ctx, collName, - entity.NewColumnVarChar(pkFieldName, pkColumn.(*entity.ColumnVarChar).Data()[:10]), + pkColumn.Slice(0, 10), ) common.CheckErr(t, err, true) common.CheckOutputFields(t, queryResult, []string{pkFieldName, vectorFieldName}) common.CheckQueryResult(t, queryResult, []entity.Column{ - entity.NewColumnVarChar(pkFieldName, pkColumn.(*entity.ColumnVarChar).Data()[:10]), - entity.NewColumnFloatVector(vectorFieldName, int(common.DefaultDim), vecColumn.(*entity.ColumnFloatVector).Data()[:10]), + pkColumn.Slice(0, 10), + vecColumn.Slice(0, 10), }) // search diff --git a/test/testcases/main_test.go b/test/testcases/main_test.go index db7e16fc..a75d3532 100644 --- a/test/testcases/main_test.go +++ b/test/testcases/main_test.go @@ -248,7 +248,7 @@ const ( Int64FloatVecJSON CollectionFieldsType = "PkInt64FloatVecJson" // int64 + float + floatVec + json Int64FloatVecArray CollectionFieldsType = "Int64FloatVecArray" // int64 + float + floatVec + all array Int64VarcharSparseVec CollectionFieldsType = "Int64VarcharSparseVec" // int64 + varchar + float32Vec + sparseVec - AllVectors CollectionFieldsType = "AllVectors" // int64 + fp32Vec + fp16Vec + binaryVec + AllVectors CollectionFieldsType = "AllVectors" // int64 + fp32Vec + fp16Vec + bf16Vec + binaryVec AllFields CollectionFieldsType = "AllFields" // all scalar fields + floatVec ) diff --git a/test/testcases/query_test.go b/test/testcases/query_test.go index 1aa41fdc..55bd6ec1 100644 --- a/test/testcases/query_test.go +++ b/test/testcases/query_test.go @@ -5,6 +5,7 @@ package testcases import ( "encoding/json" "fmt" + "io" "log" "strconv" "testing" @@ -32,16 +33,15 @@ func TestQueryDefaultPartition(t *testing.T) { common.CheckErr(t, errLoad, true) //query - pks := ids.(*entity.ColumnInt64).Data() + pks := ids.Slice(0, 10) var queryResult, _ = mc.QueryByPks( ctx, collName, []string{common.DefaultPartition}, - entity.NewColumnInt64(common.DefaultIntFieldName, pks[:10]), + pks, []string{common.DefaultIntFieldName}, ) - expColumn := entity.NewColumnInt64(common.DefaultIntFieldName, pks[:10]) - common.CheckQueryResult(t, queryResult, []entity.Column{expColumn}) + common.CheckQueryResult(t, queryResult, []entity.Column{pks}) } // test query with varchar field filter @@ -58,16 +58,15 @@ func TestQueryVarcharField(t *testing.T) { common.CheckErr(t, errLoad, true) //query - pks := ids.(*entity.ColumnVarChar).Data() + pks := ids.Slice(0, 10) queryResult, _ := mc.QueryByPks( ctx, collName, []string{common.DefaultPartition}, - entity.NewColumnVarChar(common.DefaultVarcharFieldName, pks[:10]), + pks, []string{common.DefaultVarcharFieldName}, ) - expColumn := entity.NewColumnVarChar(common.DefaultVarcharFieldName, pks[:10]) - common.CheckQueryResult(t, queryResult, []entity.Column{expColumn}) + common.CheckQueryResult(t, queryResult, []entity.Column{pks}) } // query from not existed collection @@ -84,12 +83,12 @@ func TestQueryNotExistCollection(t *testing.T) { common.CheckErr(t, errLoad, true) //query - pks := ids.(*entity.ColumnInt64).Data() + pks := ids.Slice(0, 10) _, errQuery := mc.QueryByPks( ctx, "collName", []string{common.DefaultPartition}, - entity.NewColumnInt64(common.DefaultIntFieldName, pks[:10]), + pks, []string{common.DefaultIntFieldName}, ) common.CheckErr(t, errQuery, false, "can't find collection") @@ -109,12 +108,12 @@ func TestQueryNotExistPartition(t *testing.T) { common.CheckErr(t, errLoad, true) //query - pks := ids.(*entity.ColumnInt64).Data() + pks := ids.Slice(0, 10) _, errQuery := mc.QueryByPks( ctx, collName, []string{"aaa"}, - entity.NewColumnInt64(common.DefaultIntFieldName, pks[:10]), + pks, []string{common.DefaultIntFieldName}, ) common.CheckErr(t, errQuery, false, "partition name aaa not found") @@ -150,7 +149,7 @@ func TestQueryEmptyPartitionName(t *testing.T) { ctx, collName, []string{emptyPartitionName}, - entity.NewColumnInt64(common.DefaultIntFieldName, intColumn.(*entity.ColumnInt64).Data()[:10]), + intColumn.Slice(0, 10), []string{common.DefaultIntFieldName}, ) common.CheckErr(t, errQuery, false, "Partition name should not be empty") @@ -286,7 +285,7 @@ func TestQueryEmptyOutputFields(t *testing.T) { //query with empty output fields []string{}-> output "int64" queryEmptyOutputs, _ := mc.QueryByPks( ctx, collName, []string{common.DefaultPartition}, - entity.NewColumnInt64(common.DefaultIntFieldName, ids.(*entity.ColumnInt64).Data()[:10]), + ids.Slice(0, 10), []string{}, ) common.CheckOutputFields(t, queryEmptyOutputs, []string{common.DefaultIntFieldName}) @@ -294,7 +293,7 @@ func TestQueryEmptyOutputFields(t *testing.T) { //query with empty output fields []string{""}-> output "int64" and dynamic field queryEmptyOutputs, err := mc.QueryByPks( ctx, collName, []string{common.DefaultPartition}, - entity.NewColumnInt64(common.DefaultIntFieldName, ids.(*entity.ColumnInt64).Data()[:10]), + ids.Slice(0, 10), []string{""}, ) if enableDynamic { @@ -306,7 +305,7 @@ func TestQueryEmptyOutputFields(t *testing.T) { // query with "float" output fields -> output "int64, float" queryFloatOutputs, _ := mc.QueryByPks( ctx, collName, []string{common.DefaultPartition}, - entity.NewColumnInt64(common.DefaultIntFieldName, ids.(*entity.ColumnInt64).Data()[:10]), + ids.Slice(0, 10), []string{common.DefaultFloatFieldName}, ) common.CheckOutputFields(t, queryFloatOutputs, []string{common.DefaultIntFieldName, common.DefaultFloatFieldName}) @@ -341,14 +340,13 @@ func TestQueryOutputFields(t *testing.T) { queryResult, _ := mc.QueryByPks( ctx, collName, []string{common.DefaultPartition}, - entity.NewColumnInt64(common.DefaultIntFieldName, intColumn.(*entity.ColumnInt64).Data()[:pos]), + intColumn.Slice(0, pos), []string{common.DefaultIntFieldName, common.DefaultFloatFieldName, common.DefaultFloatVecFieldName}, ) common.CheckQueryResult(t, queryResult, []entity.Column{ - entity.NewColumnInt64(common.DefaultIntFieldName, intColumn.(*entity.ColumnInt64).Data()[:pos]), - entity.NewColumnFloat(common.DefaultFloatFieldName, floatColumn.(*entity.ColumnFloat).Data()[:pos]), - entity.NewColumnFloatVector(common.DefaultFloatVecFieldName, int(common.DefaultDim), vecColumn.(*entity.ColumnFloatVector).Data()[:pos]), - }) + intColumn.Slice(0, pos), + floatColumn.Slice(0, pos), + vecColumn.Slice(0, pos)}) common.CheckOutputFields(t, queryResult, []string{common.DefaultIntFieldName, common.DefaultFloatFieldName, common.DefaultFloatVecFieldName}) } @@ -382,33 +380,78 @@ func TestQueryOutputBinaryAndVarchar(t *testing.T) { ctx, collName, []string{common.DefaultPartition}, - entity.NewColumnVarChar(common.DefaultVarcharFieldName, varcharColumn.(*entity.ColumnVarChar).Data()[:pos]), + varcharColumn.Slice(0, pos), []string{common.DefaultBinaryVecFieldName}, ) common.CheckQueryResult(t, queryResult, []entity.Column{ - entity.NewColumnVarChar(common.DefaultVarcharFieldName, varcharColumn.(*entity.ColumnVarChar).Data()[:pos]), - entity.NewColumnBinaryVector(common.DefaultBinaryVecFieldName, int(common.DefaultDim), vecColumn.(*entity.ColumnBinaryVector).Data()[:pos]), - }) + varcharColumn.Slice(0, pos), + vecColumn.Slice(0, pos)}) common.CheckOutputFields(t, queryResult, []string{common.DefaultBinaryVecFieldName, common.DefaultVarcharFieldName}) } // test query output all fields -func TestOutputAllFields(t *testing.T) { +func TestOutputAllFieldsRows(t *testing.T) { ctx := createContext(t, time.Second*common.DefaultTimeout) // connect mc := createMilvusClient(ctx, t) - for _, withRows := range []bool{true, false} { - // create collection - var capacity int64 = common.TestCapacity - cp := CollectionParams{CollectionFieldsType: AllFields, AutoID: false, EnableDynamicField: true, + // create collection + var capacity int64 = common.TestCapacity + cp := CollectionParams{CollectionFieldsType: AllFields, AutoID: false, EnableDynamicField: true, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim, MaxCapacity: capacity} + collName := createCollection(ctx, t, mc, cp) + + // prepare and insert data + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: AllFields, + start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: true} + _, _ = insertData(ctx, t, mc, dp, common.WithArrayCapacity(capacity)) + + // flush and check row count + errFlush := mc.Flush(ctx, collName, false) + common.CheckErr(t, errFlush, true) + + idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + for _, fieldName := range []string{"floatVec", "fp16Vec", "bf16Vec"} { + _ = mc.CreateIndex(ctx, collName, fieldName, idx, false) + } + binIdx, _ := entity.NewIndexBinFlat(entity.JACCARD, 16) + _ = mc.CreateIndex(ctx, collName, "binaryVec", binIdx, false) + + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + // query output all fields -> output all fields, includes vector and $meta field + allFieldsName := append(common.AllArrayFieldsName, "int64", "bool", "int8", "int16", "int32", "float", + "double", "varchar", "json", "floatVec", "fp16Vec", "bf16Vec", "binaryVec", common.DefaultDynamicFieldName) + queryResultAll, errQuery := mc.Query(ctx, collName, []string{}, + fmt.Sprintf("%s == 0", common.DefaultIntFieldName), []string{"*"}) + common.CheckErr(t, errQuery, true) + common.CheckOutputFields(t, queryResultAll, allFieldsName) +} + +// test query output all fields and verify data +func TestOutputAllFieldsColumn(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + + // create collection + var capacity int64 = common.TestCapacity + for _, isDynamic := range [2]bool{true, false} { + cp := CollectionParams{CollectionFieldsType: AllFields, AutoID: false, EnableDynamicField: isDynamic, ShardsNum: common.DefaultShards, Dim: common.DefaultDim, MaxCapacity: capacity} collName := createCollection(ctx, t, mc, cp) // prepare and insert data - dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: AllFields, - start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: withRows} - _, _ = insertData(ctx, t, mc, dp, common.WithArrayCapacity(capacity)) + data := common.GenAllFieldsData(0, common.DefaultNb, common.DefaultDim, common.WithArrayCapacity(10)) + _data := data + if isDynamic { + _data = append(_data, common.GenDynamicFieldData(0, common.DefaultNb)...) + } + ids, err := mc.Insert(ctx, collName, "", _data...) + common.CheckErr(t, err, true) + require.Equal(t, common.DefaultNb, ids.Len()) // flush and check row count errFlush := mc.Flush(ctx, collName, false) @@ -426,12 +469,24 @@ func TestOutputAllFields(t *testing.T) { common.CheckErr(t, errLoad, true) // query output all fields -> output all fields, includes vector and $meta field + pos := 10 allFieldsName := append(common.AllArrayFieldsName, "int64", "bool", "int8", "int16", "int32", "float", - "double", "varchar", "json", "floatVec", "fp16Vec", "bf16Vec", "binaryVec", common.DefaultDynamicFieldName) - queryResultAll, errQuery := mc.Query(ctx, collName, []string{}, - fmt.Sprintf("%s == 0", common.DefaultIntFieldName), []string{"*"}) + "double", "varchar", "json", "floatVec", "fp16Vec", "bf16Vec", "binaryVec") + if isDynamic { + allFieldsName = append(allFieldsName, common.DefaultDynamicFieldName) + } + queryResultAll, errQuery := mc.Query(ctx, collName, []string{}, fmt.Sprintf("%s < %d", common.DefaultIntFieldName, pos), []string{"*"}) common.CheckErr(t, errQuery, true) common.CheckOutputFields(t, queryResultAll, allFieldsName) + + expColumns := make([]entity.Column, 0, len(data)+1) + for _, column := range data { + expColumns = append(expColumns, column.Slice(0, pos)) + } + if isDynamic { + expColumns = append(expColumns, common.MergeColumnsToDynamic(pos, common.GenDynamicFieldData(0, pos))) + } + common.CheckQueryResult(t, queryResultAll, expColumns) } } @@ -453,7 +508,7 @@ func TestQueryOutputNotExistField(t *testing.T) { ctx, collName, []string{common.DefaultPartition}, - entity.NewColumnInt64(common.DefaultIntFieldName, ids.(*entity.ColumnInt64).Data()[:10]), + ids.Slice(0, 10), []string{common.DefaultIntFieldName, "varchar"}, ) common.CheckErr(t, errQuery, false, "field varchar not exist") @@ -708,6 +763,7 @@ func TestQueryArrayFieldExpr(t *testing.T) { log.Println(_exprCount.expr) countRes, err := mc.Query(ctx, collName, []string{}, _exprCount.expr, []string{common.QueryCountFieldName}) + log.Println(countRes.GetColumn(common.QueryCountFieldName).FieldData()) common.CheckErr(t, err, true) require.Equal(t, _exprCount.count, countRes.GetColumn(common.QueryCountFieldName).(*entity.ColumnInt64).Data()[0]) } @@ -875,14 +931,13 @@ func TestQueryJsonDynamicFieldRows(t *testing.T) { []string{common.DefaultIntFieldName, common.DefaultJSONFieldName, common.DefaultDynamicFieldName}, ) common.CheckErr(t, err, true) - //jsonColumn := common.GenDefaultJSONData(common.DefaultJSONFieldName, 0, 2) m0 := common.JSONStruct{String: strconv.Itoa(0), Bool: true} j0, _ := json.Marshal(&m0) m1 := common.JSONStruct{Number: int32(1), String: strconv.Itoa(1), Bool: false, List: []int64{int64(1), int64(2)}} j1, _ := json.Marshal(&m1) jsonValues := [][]byte{j0, j1} jsonColumn := entity.NewColumnJSONBytes(common.DefaultJSONFieldName, jsonValues) - dynamicColumn := common.MergeColumnsToDynamic(2, common.GenDynamicFieldData(0, 2)) + dynamicColumn := common.MergeColumnsToDynamic(10, common.GenDynamicFieldData(0, 10)) // gen dynamic json column for _, column := range queryResult { @@ -899,24 +954,17 @@ func TestQueryJsonDynamicFieldRows(t *testing.T) { log.Println(jsonData) } } - common.CheckQueryResult(t, queryResult, []entity.Column{pkColumn, jsonColumn, dynamicColumn}) + common.CheckQueryResult(t, queryResult, []entity.Column{pkColumn, jsonColumn, dynamicColumn.Slice(0, 2)}) // query with different expr and count expr := fmt.Sprintf("%s['number'] < 10 && %s < 10", common.DefaultJSONFieldName, common.DefaultDynamicNumberField) queryRes, _ := mc.Query(ctx, collName, - []string{common.DefaultPartition}, - expr, []string{common.DefaultJSONFieldName, common.DefaultDynamicNumberField}) + []string{common.DefaultPartition}, expr, []string{common.DefaultDynamicNumberField}) // verify output fields and count, dynamicNumber value - common.CheckOutputFields(t, queryRes, []string{common.DefaultIntFieldName, common.DefaultJSONFieldName, common.DefaultDynamicNumberField}) - require.Equal(t, 10, queryRes.GetColumn(common.DefaultJSONFieldName).Len()) - dynamicNumColumn := queryRes.GetColumn(common.DefaultDynamicNumberField) - var numberData []int64 - for i := 0; i < dynamicNumColumn.Len(); i++ { - line, _ := dynamicNumColumn.GetAsInt64(i) - numberData = append(numberData, line) - } - require.Equal(t, numberData, []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + common.CheckOutputFields(t, queryRes, []string{common.DefaultIntFieldName, common.DefaultDynamicNumberField}) + pkColumn2 := common.GenColumnData(0, 10, entity.FieldTypeInt64, common.DefaultIntFieldName) + common.CheckQueryResult(t, queryRes, []entity.Column{pkColumn2, dynamicColumn}) } // test query with invalid expr @@ -1097,7 +1145,7 @@ func TestQuerySparseVector(t *testing.T) { // insert intColumn, _, floatColumn := common.GenDefaultColumnData(0, common.DefaultNb, common.DefaultDim) varColumn := common.GenColumnData(0, common.DefaultNb, entity.FieldTypeVarChar, common.DefaultVarcharFieldName) - sparseColumn := common.GenColumnData(0, common.DefaultNb, entity.FieldTypeSparseVector, common.DefaultSparseVecFieldName) + sparseColumn := common.GenColumnData(0, common.DefaultNb, entity.FieldTypeSparseVector, common.DefaultSparseVecFieldName, common.WithSparseVectorLen(20)) mc.Insert(ctx, collName, "", intColumn, varColumn, floatColumn, sparseColumn) mc.Flush(ctx, collName, false) mc.LoadCollection(ctx, collName, false) @@ -1107,14 +1155,440 @@ func TestQuerySparseVector(t *testing.T) { require.Equal(t, int64(common.DefaultNb), countRes.GetColumn(common.QueryCountFieldName).(*entity.ColumnInt64).Data()[0]) // query - queryResult, err := mc.Query(ctx, collName, []string{}, fmt.Sprintf("%s == 0", common.DefaultIntFieldName), []string{"*"}) + queryResult, err := mc.Query(ctx, collName, []string{}, fmt.Sprintf("%s in [0, 1]", common.DefaultIntFieldName), []string{"*"}) common.CheckErr(t, err, true) - expIntColumn := entity.NewColumnInt64(common.DefaultIntFieldName, intColumn.(*entity.ColumnInt64).Data()[:1]) - expVarcharColumn := entity.NewColumnVarChar(common.DefaultVarcharFieldName, varColumn.(*entity.ColumnVarChar).Data()[:1]) - expVecColumn := entity.NewColumnFloatVector(common.DefaultFloatVecFieldName, int(common.DefaultDim), floatColumn.(*entity.ColumnFloatVector).Data()[:1]) - expSparseColumn := entity.NewColumnSparseVectors(common.DefaultSparseVecFieldName, sparseColumn.(*entity.ColumnSparseFloatVector).Data()[:1]) common.CheckOutputFields(t, queryResult, []string{common.DefaultIntFieldName, common.DefaultVarcharFieldName, common.DefaultFloatVecFieldName, common.DefaultSparseVecFieldName}) - common.CheckQueryResult(t, queryResult, []entity.Column{expIntColumn, expVarcharColumn, expVecColumn, expSparseColumn}) + t.Log("https://github.com/milvus-io/milvus-sdk-go/issues/769") + //common.CheckQueryResult(t, queryResult, []entity.Column{intColumn.Slice(0, 2), varColumn.Slice(0, 2), floatColumn.Slice(0, 2), sparseColumn.Slice(0, 2)}) + } +} + +// test query iterator default +func TestQueryIteratorDefault(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + + // create collection + cp := CollectionParams{CollectionFieldsType: Int64FloatVec, AutoID: false, EnableDynamicField: true, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim} + collName := createCollection(ctx, t, mc, cp) + + // insert + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVec, + start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: false} + _, _ = insertData(ctx, t, mc, dp) + + dp2 := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVec, + start: common.DefaultNb, nb: common.DefaultNb * 2, dim: common.DefaultDim, EnableDynamicField: true, WithRows: true} + _, _ = insertData(ctx, t, mc, dp2) + + idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + _ = mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false) + + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + // query iterator with default batch + itr, err := mc.QueryIterator(ctx, client.NewQueryIteratorOption(collName)) + common.CheckErr(t, err, true) + common.CheckQueryIteratorResult(ctx, t, itr, common.DefaultNb*3, common.WithExpBatchSize(common.GenBatchSizes(common.DefaultNb*3, common.DefaultBatchSize))) +} + +// test query iterator default +func TestQueryIteratorHitEmpty(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + + // create collection + cp := CollectionParams{CollectionFieldsType: Int64FloatVec, AutoID: false, EnableDynamicField: true, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim} + collName := createCollection(ctx, t, mc, cp) + + idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + _ = mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false) + + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + // query iterator with default batch + itr, err := mc.QueryIterator(ctx, client.NewQueryIteratorOption(collName)) + common.CheckErr(t, err, true) + rs, err := itr.Next(ctx) + require.Empty(t, rs) + require.Error(t, err, io.EOF) + common.CheckQueryIteratorResult(ctx, t, itr, 0, common.WithExpBatchSize(common.GenBatchSizes(0, common.DefaultBatchSize))) +} + +func TestQueryIteratorBatchSize(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + // create collection + cp := CollectionParams{CollectionFieldsType: Int64FloatVec, AutoID: false, EnableDynamicField: true, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim} + collName := createCollection(ctx, t, mc, cp) + + // insert + nb := 201 + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVec, + start: 0, nb: nb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: false} + _, _ = insertData(ctx, t, mc, dp) + + idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + _ = mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false) + + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + type batchStruct struct { + batch int + expBatchSize []int + } + batchStructs := []batchStruct{ + {batch: nb / 2, expBatchSize: common.GenBatchSizes(nb, nb/2)}, + {batch: nb, expBatchSize: common.GenBatchSizes(nb, nb)}, + {batch: nb + 1, expBatchSize: common.GenBatchSizes(nb, nb+1)}, + } + + for _, _batchStruct := range batchStructs { + // query iterator with default batch + itr, err := mc.QueryIterator(ctx, client.NewQueryIteratorOption(collName).WithBatchSize(_batchStruct.batch)) + common.CheckErr(t, err, true) + common.CheckQueryIteratorResult(ctx, t, itr, nb, common.WithExpBatchSize(_batchStruct.expBatchSize)) + } +} + +func TestQueryIteratorOutputAllFields(t *testing.T) { + t.Parallel() + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + for _, dynamic := range [2]bool{false, true} { + // create collection + cp := CollectionParams{CollectionFieldsType: AllFields, AutoID: false, EnableDynamicField: dynamic, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim} + collName := createCollection(ctx, t, mc, cp, client.WithConsistencyLevel(entity.ClStrong)) + + // insert + nb := 2501 + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: AllFields, + start: 0, nb: nb, dim: common.DefaultDim, EnableDynamicField: dynamic, WithRows: false} + insertData(ctx, t, mc, dp) + + indexHnsw, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + indexBinary, _ := entity.NewIndexBinIvfFlat(entity.JACCARD, 64) + for _, fieldName := range common.AllVectorsFieldsName { + if fieldName == common.DefaultBinaryVecFieldName { + mc.CreateIndex(ctx, collName, fieldName, indexBinary, false) + } else { + mc.CreateIndex(ctx, collName, fieldName, indexHnsw, false) + } + } + + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + // output * fields + nbFilter := 1001 + batch := 500 + expr := fmt.Sprintf("%s < %d", common.DefaultIntFieldName, nbFilter) + + itr, err := mc.QueryIterator(ctx, client.NewQueryIteratorOption(collName).WithBatchSize(batch).WithOutputFields("*").WithExpr(expr)) + common.CheckErr(t, err, true) + allFields := common.GetAllFieldsName(dynamic, false) + common.CheckQueryIteratorResult(ctx, t, itr, nbFilter, common.WithExpBatchSize(common.GenBatchSizes(nbFilter, batch)), common.WithExpOutputFields(allFields)) + } +} + +func TestQueryIteratorOutputSparseFieldsRows(t *testing.T) { + t.Parallel() + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + for _, withRows := range [2]bool{true, false} { + // create collection + cp := CollectionParams{CollectionFieldsType: Int64VarcharSparseVec, AutoID: false, EnableDynamicField: true, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim, MaxLength: common.TestMaxLen} + collName := createCollection(ctx, t, mc, cp) + + // insert + nb := 2501 + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64VarcharSparseVec, + start: 0, nb: nb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: withRows, maxLenSparse: 1000} + _, _ = insertData(ctx, t, mc, dp) + + indexHnsw, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + indexSparse, _ := entity.NewIndexSparseInverted(entity.IP, 0.1) + mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, indexHnsw, false) + mc.CreateIndex(ctx, collName, common.DefaultSparseVecFieldName, indexSparse, false) + + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + // output * fields + itr, err := mc.QueryIterator(ctx, client.NewQueryIteratorOption(collName).WithBatchSize(400).WithOutputFields("*")) + common.CheckErr(t, err, true) + fields := []string{common.DefaultIntFieldName, common.DefaultVarcharFieldName, common.DefaultFloatVecFieldName, common.DefaultSparseVecFieldName, common.DefaultDynamicFieldName} + common.CheckQueryIteratorResult(ctx, t, itr, nb, common.WithExpBatchSize(common.GenBatchSizes(nb, 400)), common.WithExpOutputFields(fields)) + } +} + +// test query iterator with non-existed collection/partition name, invalid batch size +func TestQueryIteratorInvalid(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + // create collection + cp := CollectionParams{CollectionFieldsType: Int64FloatVec, AutoID: false, EnableDynamicField: false, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim} + collName := createCollection(ctx, t, mc, cp) + + // insert + nb := 201 + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVec, + start: 0, nb: nb, dim: common.DefaultDim, EnableDynamicField: false, WithRows: false} + _, _ = insertData(ctx, t, mc, dp) + + idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + _ = mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false) + + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + // query iterator with not existed collection name + _, err := mc.QueryIterator(ctx, client.NewQueryIteratorOption("aaa")) + common.CheckErr(t, err, false, "can't find collection") + + // query iterator with not existed partition name + _, errPar := mc.QueryIterator(ctx, client.NewQueryIteratorOption(collName).WithPartitions("aaa")) + common.CheckErr(t, errPar, false, "partition name aaa not found") + + // query iterator with not existed partition name + _, errPar = mc.QueryIterator(ctx, client.NewQueryIteratorOption(collName).WithPartitions("aaa", common.DefaultPartition)) + common.CheckErr(t, errPar, false, "partition name aaa not found") + + _, errOutput := mc.QueryIterator(ctx, client.NewQueryIteratorOption(collName).WithOutputFields(common.QueryCountFieldName)) + common.CheckErr(t, errOutput, false, "count entities with pagination is not allowed") + + // query iterator with invalid batch size + for _, batch := range []int{-1, 0} { + // query iterator with default batch + _, err := mc.QueryIterator(ctx, client.NewQueryIteratorOption(collName).WithBatchSize(batch)) + common.CheckErr(t, err, false, "batch size cannot less than 1") + } +} + +// query iterator with invalid expr +func TestQueryIteratorInvalidExpr(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + + // create collection + cp := CollectionParams{CollectionFieldsType: Int64FloatVecJSON, AutoID: false, EnableDynamicField: true, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim, + } + collName := createCollection(ctx, t, mc, cp) + + // insert + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVecJSON, + start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, + } + _, _ = insertData(ctx, t, mc, dp) + + idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + _ = mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false) + + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + for _, _invalidExprs := range common.InvalidExpressions { + _, err := mc.QueryIterator(ctx, client.NewQueryIteratorOption(collName).WithExpr(_invalidExprs.Expr)) + common.CheckErr(t, err, _invalidExprs.ErrNil, _invalidExprs.ErrMsg) + } +} + +// test query iterator with non-existed field when dynamic or not +func TestQueryIteratorOutputFieldDynamic(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + for _, dynamic := range [2]bool{true, false} { + // create collection + cp := CollectionParams{CollectionFieldsType: Int64FloatVec, AutoID: false, EnableDynamicField: dynamic, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim} + collName := createCollection(ctx, t, mc, cp) + // insert + nb := 201 + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVec, + start: 0, nb: nb, dim: common.DefaultDim, EnableDynamicField: dynamic, WithRows: false} + _, _ = insertData(ctx, t, mc, dp) + + idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + _ = mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false) + + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + // query iterator with not existed output fields: if dynamic, non-existent field are equivalent to dynamic field + itr, errOutput := mc.QueryIterator(ctx, client.NewQueryIteratorOption(collName).WithOutputFields("aaa")) + if dynamic { + common.CheckErr(t, errOutput, true) + expFields := []string{common.DefaultIntFieldName, common.DefaultDynamicFieldName} + common.CheckQueryIteratorResult(ctx, t, itr, nb, common.WithExpBatchSize(common.GenBatchSizes(nb, common.DefaultBatchSize)), common.WithExpOutputFields(expFields)) + } else { + common.CheckErr(t, errOutput, false, "field aaa not exist") + } + } +} + +func TestQueryIteratorExpr(t *testing.T) { + //t.Log("https://github.com/milvus-io/milvus-sdk-go/issues/756") + type exprCount struct { + expr string + count int + } + capacity := common.TestCapacity + exprLimits := []exprCount{ + {expr: fmt.Sprintf("%s in [0, 1, 2]", common.DefaultIntFieldName), count: 3}, + {expr: fmt.Sprintf("%s >= 1000 || %s > 2000", common.DefaultIntFieldName, common.DefaultIntFieldName), count: 2000}, + {expr: fmt.Sprintf("%s >= 1000 and %s < 2000", common.DefaultIntFieldName, common.DefaultIntFieldName), count: 1000}, + + //json and dynamic field filter expr: == < in bool/ list/ int + {expr: fmt.Sprintf("%s['number'] == 0", common.DefaultJSONFieldName), count: 1500 / 2}, + {expr: fmt.Sprintf("%s['number'] < 100 and %s['number'] != 0", common.DefaultJSONFieldName, common.DefaultJSONFieldName), count: 50}, + {expr: fmt.Sprintf("%s < 100", common.DefaultDynamicNumberField), count: 100}, + {expr: "dynamicNumber % 2 == 0", count: 1500}, + {expr: fmt.Sprintf("%s == false", common.DefaultDynamicBoolField), count: 2000}, + {expr: fmt.Sprintf("%s in ['1', '2'] ", common.DefaultDynamicStringField), count: 2}, + {expr: fmt.Sprintf("%s['string'] in ['1', '2', '5'] ", common.DefaultJSONFieldName), count: 3}, + {expr: fmt.Sprintf("%s['list'] == [1, 2] ", common.DefaultJSONFieldName), count: 1}, + {expr: fmt.Sprintf("%s['list'][0] < 10 ", common.DefaultJSONFieldName), count: 5}, + {expr: fmt.Sprintf("%s[\"dynamicList\"] != [2, 3]", common.DefaultDynamicFieldName), count: 0}, + + // json contains + {expr: fmt.Sprintf("json_contains (%s['list'], 2)", common.DefaultJSONFieldName), count: 1}, + {expr: fmt.Sprintf("json_contains (%s['number'], 0)", common.DefaultJSONFieldName), count: 0}, + {expr: fmt.Sprintf("JSON_CONTAINS_ANY (%s['list'], [1, 3])", common.DefaultJSONFieldName), count: 2}, + // string like + {expr: "dynamicString like '1%' ", count: 1111}, + + // key exist + {expr: fmt.Sprintf("exists %s['list']", common.DefaultJSONFieldName), count: common.DefaultNb / 2}, + {expr: fmt.Sprintf("exists a "), count: 0}, + {expr: fmt.Sprintf("exists %s ", common.DefaultDynamicStringField), count: common.DefaultNb}, + + // data type not match and no error + {expr: fmt.Sprintf("%s['number'] == '0' ", common.DefaultJSONFieldName), count: 0}, + + // json field + {expr: fmt.Sprintf("%s >= 1500", common.DefaultJSONFieldName), count: 1500 / 2}, // json >= 1500 + {expr: fmt.Sprintf("%s > 1499.5", common.DefaultJSONFieldName), count: 1500 / 2}, // json >= 1500.0 + {expr: fmt.Sprintf("%s like '21%%'", common.DefaultJSONFieldName), count: 100 / 4}, // json like '21%' + {expr: fmt.Sprintf("%s == [1503, 1504]", common.DefaultJSONFieldName), count: 1}, // json == [1,2] + {expr: fmt.Sprintf("%s[0] > 1", common.DefaultJSONFieldName), count: 1500 / 4}, // json[0] > 1 + {expr: fmt.Sprintf("%s[0][0] > 1", common.DefaultJSONFieldName), count: 0}, // json == [1,2] + {expr: fmt.Sprintf("%s[0] == false", common.DefaultBoolArrayField), count: common.DefaultNb / 2}, // array[0] == + {expr: fmt.Sprintf("%s[0] > 0", common.DefaultInt64ArrayField), count: common.DefaultNb - 1}, // array[0] > + {expr: fmt.Sprintf("%s[0] > 0", common.DefaultInt8ArrayField), count: 1524}, // array[0] > int8 range: [-128, 127] + {expr: fmt.Sprintf("array_contains (%s, %d)", common.DefaultInt16ArrayField, capacity), count: capacity}, // array_contains(array, 1) + {expr: fmt.Sprintf("json_contains (%s, 1)", common.DefaultInt32ArrayField), count: 2}, // json_contains(array, 1) + {expr: fmt.Sprintf("array_contains (%s, 1000000)", common.DefaultInt32ArrayField), count: 0}, // array_contains(array, 1) + {expr: fmt.Sprintf("json_contains_all (%s, [90, 91])", common.DefaultInt64ArrayField), count: 91}, // json_contains_all(array, [x]) + {expr: fmt.Sprintf("json_contains_any (%s, [0, 100, 10])", common.DefaultFloatArrayField), count: 101}, // json_contains_any (array, [x]) + {expr: fmt.Sprintf("%s == [0, 1]", common.DefaultDoubleArrayField), count: 0}, // array == + {expr: fmt.Sprintf("array_length(%s) == %d", common.DefaultDoubleArrayField, capacity), count: common.DefaultNb}, // array_length + } + + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + + // create collection + cp := CollectionParams{CollectionFieldsType: AllFields, AutoID: false, EnableDynamicField: true, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim, MaxCapacity: common.TestCapacity} + collName := createCollection(ctx, t, mc, cp, client.WithConsistencyLevel(entity.ClStrong)) + + // insert + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: AllFields, + start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: false} + _, err := insertData(ctx, t, mc, dp, common.WithArrayCapacity(common.TestCapacity)) + common.CheckErr(t, err, true) + mc.Flush(ctx, collName, false) + + indexHnsw, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + indexBinary, _ := entity.NewIndexBinIvfFlat(entity.JACCARD, 64) + for _, fieldName := range common.AllVectorsFieldsName { + if fieldName == common.DefaultBinaryVecFieldName { + mc.CreateIndex(ctx, collName, fieldName, indexBinary, false) + } else { + mc.CreateIndex(ctx, collName, fieldName, indexHnsw, false) + } + } + + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + batch := 500 + + for _, exprLimit := range exprLimits { + log.Printf("case expr is: %s, limit=%d", exprLimit.expr, exprLimit.count) + itr, err := mc.QueryIterator(ctx, client.NewQueryIteratorOption(collName).WithBatchSize(batch).WithExpr(exprLimit.expr)) + common.CheckErr(t, err, true) + common.CheckQueryIteratorResult(ctx, t, itr, exprLimit.count, common.WithExpBatchSize(common.GenBatchSizes(exprLimit.count, batch))) + } +} + +// test query iterator with partition +func TestQueryIteratorPartitions(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + // create collection + cp := CollectionParams{CollectionFieldsType: Int64FloatVec, AutoID: false, EnableDynamicField: false, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim} + collName := createCollection(ctx, t, mc, cp) + pName := "p1" + err := mc.CreatePartition(ctx, collName, pName) + common.CheckErr(t, err, true) + + // insert [0, nb) into partition: _default + nb := 1500 + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVec, + start: 0, nb: nb, dim: common.DefaultDim, EnableDynamicField: false, WithRows: false} + _, _ = insertData(ctx, t, mc, dp) + // insert [nb, nb*2) into partition: p1 + dp1 := DataParams{CollectionName: collName, PartitionName: pName, CollectionFieldsType: Int64FloatVec, + start: nb, nb: nb, dim: common.DefaultDim, EnableDynamicField: false, WithRows: false} + _, _ = insertData(ctx, t, mc, dp1) + + idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + _ = mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false) + + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + // query iterator with partition + expr := fmt.Sprintf("%s < %d", common.DefaultIntFieldName, nb) + mParLimit := map[string]int{ + common.DefaultPartition: nb, + pName: 0, + } + for par, limit := range mParLimit { + itr, err := mc.QueryIterator(ctx, client.NewQueryIteratorOption(collName).WithExpr(expr).WithPartitions(par)) + common.CheckErr(t, err, true) + common.CheckQueryIteratorResult(ctx, t, itr, limit, common.WithExpBatchSize(common.GenBatchSizes(limit, common.DefaultBatchSize))) } } diff --git a/test/testcases/search_test.go b/test/testcases/search_test.go index fe5827c8..7cca2adb 100644 --- a/test/testcases/search_test.go +++ b/test/testcases/search_test.go @@ -1,4 +1,4 @@ -///go:build L0 +//go:build L0 package testcases @@ -667,7 +667,7 @@ func TestSearchInvalidVectors(t *testing.T) { // dim not match {vectors: common.GenSearchVectors(common.DefaultNq, 64, entity.FieldTypeFloatVector), errMsg: "vector dimension mismatch"}, - // vector type not match + //vector type not match {vectors: common.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeBinaryVector), errMsg: "vector type must be the same"}, // empty vectors @@ -693,6 +693,53 @@ func TestSearchInvalidVectors(t *testing.T) { } } +// test search with invalid vectors +func TestSearchInvalidVectorsEmptyCollection(t *testing.T) { + t.Skip("https://github.com/milvus-io/milvus/issues/33639") + t.Skip("https://github.com/milvus-io/milvus/issues/33637") + t.Parallel() + ctx := createContext(t, time.Second*common.DefaultTimeout*2) + // connect + mc := createMilvusClient(ctx, t) + + // create collection with data + collName := createDefaultCollection(ctx, t, mc, false, common.DefaultShards) + + // index + idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + err := mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false, client.WithIndexName("")) + common.CheckErr(t, err, true) + + // load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + type invalidVectorsStruct struct { + vectors []entity.Vector + errMsg string + } + + invalidVectors := []invalidVectorsStruct{ + // dim not match + {vectors: common.GenSearchVectors(common.DefaultNq, 64, entity.FieldTypeFloatVector), errMsg: "vector dimension mismatch"}, + + //vector type not match + {vectors: common.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeBinaryVector), errMsg: "vector type must be the same"}, + + // empty vectors + {vectors: []entity.Vector{}, errMsg: "nq [0] is invalid"}, + {vectors: []entity.Vector{entity.FloatVector{}}, errMsg: "vector dimension mismatch"}, + } + + sp, _ := entity.NewIndexHNSWSearchParam(74) + for _, invalidVector := range invalidVectors { + // search vectors empty slice + _, errSearchEmpty := mc.Search(ctx, collName, []string{}, "", []string{"*"}, invalidVector.vectors, + common.DefaultFloatVecFieldName, entity.L2, common.DefaultTopK, sp) + common.CheckErr(t, errSearchEmpty, false, invalidVector.errMsg) + } +} + // test search metric type isn't the same with index metric type func TestSearchNotMatchMetricType(t *testing.T) { ctx := createContext(t, time.Second*common.DefaultTimeout*2)