Fix dynamic field $meta not in result set if using wildcard

Signed-off-by: Congqi Xia <[email protected]>
milvus-io · Nov 8, 2023 · 50102fc · 50102fc
1 parent baea5b9
commit 50102fc
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 20 deletions.
diff --git a/client/data.go b/client/data.go
@@ -93,7 +93,8 @@ func (c *GrpcClient) Search(ctx context.Context, collName string, partitions []s
 }
 
 func (c *GrpcClient) parseSearchResult(sch *entity.Schema, outputFields []string, fieldDataList []*schemapb.FieldData, _, from, to int) ([]entity.Column, error) {
-	outputFields = expandWildcard(sch, outputFields)
+	var wildcard bool
+	outputFields, wildcard = expandWildcard(sch, outputFields)
 	// duplicated name will have only one column now
 	outputSet := make(map[string]struct{})
 	for _, output := range outputFields {
@@ -114,8 +115,8 @@ func (c *GrpcClient) parseSearchResult(sch *entity.Schema, outputFields []string
 				return nil, errors.New("dynamic field not json")
 			}
 
-			// return json column only explicitly specified in output fields
-			if _, ok := outputSet[fieldData.GetFieldName()]; !ok {
+			// return json column only explicitly specified in output fields and not in wildcard mode
+			if _, ok := outputSet[fieldData.GetFieldName()]; !ok && !wildcard {
 				continue
 			}
 		}
@@ -142,15 +143,15 @@ func (c *GrpcClient) parseSearchResult(sch *entity.Schema, outputFields []string
 	return columns, nil
 }
 
-func expandWildcard(schema *entity.Schema, outputFields []string) []string {
+func expandWildcard(schema *entity.Schema, outputFields []string) ([]string, bool) {
 	wildcard := false
 	for _, outputField := range outputFields {
 		if outputField == "*" {
 			wildcard = true
 		}
 	}
 	if !wildcard {
-		return outputFields
+		return outputFields, false
 	}
 
 	set := make(map[string]struct{})
@@ -170,7 +171,7 @@ func expandWildcard(schema *entity.Schema, outputFields []string) []string {
 			result = append(result, output)
 		}
 	}
-	return result
+	return result, true
 }
 
 func PKs2Expr(backName string, ids entity.Column) string {

diff --git a/client/data_test.go b/client/data_test.go
@@ -1195,21 +1195,23 @@ func (s *WildcardSuite) SetupTest() {
 
 func (s *WildcardSuite) TestExpandWildcard() {
 	type testCase struct {
-		tag    string
-		input  []string
-		expect []string
+		tag            string
+		input          []string
+		expect         []string
+		expectWildCard bool
 	}
 
 	cases := []testCase{
 		{tag: "normal", input: []string{"pk", "attr"}, expect: []string{"pk", "attr"}},
-		{tag: "with_wildcard", input: []string{"*"}, expect: []string{"pk", "attr", "$meta", "vector"}},
-		{tag: "wildcard_dynamic", input: []string{"*", "a"}, expect: []string{"pk", "attr", "$meta", "vector", "a"}},
+		{tag: "with_wildcard", input: []string{"*"}, expect: []string{"pk", "attr", "$meta", "vector"}, expectWildCard: true},
+		{tag: "wildcard_dynamic", input: []string{"*", "a"}, expect: []string{"pk", "attr", "$meta", "vector", "a"}, expectWildCard: true},
 	}
 
 	for _, tc := range cases {
 		s.Run(tc.tag, func() {
-			output := expandWildcard(s.schema, tc.input)
+			output, wildCard := expandWildcard(s.schema, tc.input)
 			s.ElementsMatch(tc.expect, output)
+			s.Equal(tc.expectWildCard, wildCard)
 		})
 	}
 }

diff --git a/test/testcases/search_test.go b/test/testcases/search_test.go
@@ -1061,13 +1061,13 @@ func TestSearchScannAllMetricsWithRawData(t *testing.T) {
 			mc := createMilvusClient(ctx, t)
 
 			// create collection
-			cp := CollectionParams{CollectionFieldsType: Int64FloatVecJSON, AutoID: false, EnableDynamicField: true,
+			cp := CollectionParams{CollectionFieldsType: Int64FloatVecJSON, AutoID: false, EnableDynamicField: false,
 				ShardsNum: common.DefaultShards, Dim: common.DefaultDim}
 			collName := createCollection(ctx, t, mc, cp)
 
 			// insert
 			dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVecJSON,
-				start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: false}
+				start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: false, WithRows: false}
 			_, _ = insertData(ctx, t, mc, dp)
 			mc.Flush(ctx, collName, false)
 
@@ -1113,13 +1113,13 @@ func TestRangeSearchScannL2(t *testing.T) {
 	mc := createMilvusClient(ctx, t)
 
 	// create collection
-	cp := CollectionParams{CollectionFieldsType: Int64FloatVecJSON, AutoID: false, EnableDynamicField: true,
+	cp := CollectionParams{CollectionFieldsType: Int64FloatVecJSON, AutoID: false, EnableDynamicField: false,
 		ShardsNum: common.DefaultShards, Dim: common.DefaultDim}
 	collName := createCollection(ctx, t, mc, cp)
 
 	// insert
 	dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVecJSON,
-		start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: false}
+		start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: false, WithRows: false}
 	_, _ = insertData(ctx, t, mc, dp)
 
 	// create scann index
@@ -1171,13 +1171,13 @@ func TestRangeSearchScannIPCosine(t *testing.T) {
 		mc := createMilvusClient(ctx, t)
 
 		// create collection
-		cp := CollectionParams{CollectionFieldsType: Int64FloatVecJSON, AutoID: false, EnableDynamicField: true,
+		cp := CollectionParams{CollectionFieldsType: Int64FloatVecJSON, AutoID: false, EnableDynamicField: false,
 			ShardsNum: common.DefaultShards, Dim: common.DefaultDim}
 		collName := createCollection(ctx, t, mc, cp)
 
 		// insert
 		dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVecJSON,
-			start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: false}
+			start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: false, WithRows: false}
 		_, _ = insertData(ctx, t, mc, dp)
 		mc.Flush(ctx, collName, false)
 
@@ -1232,13 +1232,13 @@ func TestRangeSearchScannBinary(t *testing.T) {
 		mc := createMilvusClient(ctx, t)
 
 		// create collection
-		cp := CollectionParams{CollectionFieldsType: Int64BinaryVec, AutoID: false, EnableDynamicField: true,
+		cp := CollectionParams{CollectionFieldsType: Int64BinaryVec, AutoID: false, EnableDynamicField: false,
 			ShardsNum: common.DefaultShards, Dim: common.DefaultDim}
 		collName := createCollection(ctx, t, mc, cp)
 
 		// insert
 		dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64BinaryVec,
-			start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: false}
+			start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: false, WithRows: false}
 		_, _ = insertData(ctx, t, mc, dp)
 		mc.Flush(ctx, collName, false)