diff --git a/configs/milvus.yaml b/configs/milvus.yaml index b2510a3f2c51c..ecfb91acc88f3 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -179,6 +179,14 @@ queryCoord: overloadedMemoryThresholdPercentage: 90 # The threshold percentage that memory overload balanceIntervalSeconds: 60 memoryUsageMaxDifferencePercentage: 30 + checkInterval: 1000 + channelTaskTimeout: 60000 # 1 minute + segmentTaskTimeout: 15000 # 15 seconds + distPullInterval: 500 + loadTimeoutSeconds: 600 + checkHandoffInterval: 5000 + + # Related configuration of queryNode, used to run hybrid search between vector and scalar data. queryNode: diff --git a/internal/core/src/pb/common.pb.cc b/internal/core/src/pb/common.pb.cc index 05fb81fc29dec..7c202c81775ba 100644 --- a/internal/core/src/pb/common.pb.cc +++ b/internal/core/src/pb/common.pb.cc @@ -397,7 +397,7 @@ const char descriptor_table_protodef_common_2eproto[] PROTOBUF_SECTION_VARIABLE( "\003\022\013\n\007Flushed\020\004\022\014\n\010Flushing\020\005\022\013\n\007Dropped\020" "\006\022\r\n\tImporting\020\007*>\n\017PlaceholderType\022\010\n\004N" "one\020\000\022\020\n\014BinaryVector\020d\022\017\n\013FloatVector\020e" - "*\266\014\n\007MsgType\022\r\n\tUndefined\020\000\022\024\n\020CreateCol" + "*\370\014\n\007MsgType\022\r\n\tUndefined\020\000\022\024\n\020CreateCol" "lection\020d\022\022\n\016DropCollection\020e\022\021\n\rHasColl" "ection\020f\022\026\n\022DescribeCollection\020g\022\023\n\017Show" "Collections\020h\022\024\n\020GetSystemConfigs\020i\022\022\n\016L" @@ -422,53 +422,55 @@ const char descriptor_table_protodef_common_2eproto[] PROTOBUF_SECTION_VARIABLE( "nels\020\375\003\022\027\n\022WatchQueryChannels\020\376\003\022\030\n\023Remo" "veQueryChannels\020\377\003\022\035\n\030SealedSegmentsChan" "geInfo\020\200\004\022\027\n\022WatchDeltaChannels\020\201\004\022\024\n\017Ge" - "tShardLeaders\020\202\004\022\020\n\013GetReplicas\020\203\004\022\020\n\013Se" - "gmentInfo\020\330\004\022\017\n\nSystemInfo\020\331\004\022\024\n\017GetReco" - "veryInfo\020\332\004\022\024\n\017GetSegmentState\020\333\004\022\r\n\010Tim" - "eTick\020\260\t\022\023\n\016QueryNodeStats\020\261\t\022\016\n\tLoadInd" - "ex\020\262\t\022\016\n\tRequestID\020\263\t\022\017\n\nRequestTSO\020\264\t\022\024" - "\n\017AllocateSegment\020\265\t\022\026\n\021SegmentStatistic" - "s\020\266\t\022\025\n\020SegmentFlushDone\020\267\t\022\017\n\nDataNodeT" - "t\020\270\t\022\025\n\020CreateCredential\020\334\013\022\022\n\rGetCreden" - "tial\020\335\013\022\025\n\020DeleteCredential\020\336\013\022\025\n\020Update" - "Credential\020\337\013\022\026\n\021ListCredUsernames\020\340\013\022\017\n" - "\nCreateRole\020\300\014\022\r\n\010DropRole\020\301\014\022\024\n\017Operate" - "UserRole\020\302\014\022\017\n\nSelectRole\020\303\014\022\017\n\nSelectUs" - "er\020\304\014\022\023\n\016SelectResource\020\305\014\022\025\n\020OperatePri" - "vilege\020\306\014\022\020\n\013SelectGrant\020\307\014\022\033\n\026RefreshPo" - "licyInfoCache\020\310\014\022\017\n\nListPolicy\020\311\014*\"\n\007Dsl" - "Type\022\007\n\003Dsl\020\000\022\016\n\nBoolExprV1\020\001*B\n\017Compact" - "ionState\022\021\n\rUndefiedState\020\000\022\r\n\tExecuting" - "\020\001\022\r\n\tCompleted\020\002*X\n\020ConsistencyLevel\022\n\n" - "\006Strong\020\000\022\013\n\007Session\020\001\022\013\n\007Bounded\020\002\022\016\n\nE" - "ventually\020\003\022\016\n\nCustomized\020\004*\257\001\n\013ImportSt" - "ate\022\021\n\rImportPending\020\000\022\020\n\014ImportFailed\020\001" - "\022\021\n\rImportStarted\020\002\022\024\n\020ImportDownloaded\020" - "\003\022\020\n\014ImportParsed\020\004\022\023\n\017ImportPersisted\020\005" - "\022\023\n\017ImportCompleted\020\006\022\026\n\022ImportAllocSegm" - "ent\020\n*2\n\nObjectType\022\016\n\nCollection\020\000\022\n\n\006G" - "lobal\020\001\022\010\n\004User\020\002*\206\005\n\017ObjectPrivilege\022\020\n" - "\014PrivilegeAll\020\000\022\035\n\031PrivilegeCreateCollec" - "tion\020\001\022\033\n\027PrivilegeDropCollection\020\002\022\037\n\033P" - "rivilegeDescribeCollection\020\003\022\034\n\030Privileg" - "eShowCollections\020\004\022\021\n\rPrivilegeLoad\020\005\022\024\n" - "\020PrivilegeRelease\020\006\022\027\n\023PrivilegeCompacti" - "on\020\007\022\023\n\017PrivilegeInsert\020\010\022\023\n\017PrivilegeDe" - "lete\020\t\022\032\n\026PrivilegeGetStatistics\020\n\022\030\n\024Pr" - "ivilegeCreateIndex\020\013\022\030\n\024PrivilegeIndexDe" - "tail\020\014\022\026\n\022PrivilegeDropIndex\020\r\022\023\n\017Privil" - "egeSearch\020\016\022\022\n\016PrivilegeFlush\020\017\022\022\n\016Privi" - "legeQuery\020\020\022\030\n\024PrivilegeLoadBalance\020\021\022\023\n" - "\017PrivilegeImport\020\022\022\034\n\030PrivilegeCreateOwn" - "ership\020\023\022\027\n\023PrivilegeUpdateUser\020\024\022\032\n\026Pri" - "vilegeDropOwnership\020\025\022\034\n\030PrivilegeSelect" - "Ownership\020\026\022\034\n\030PrivilegeManageOwnership\020" - "\027\022\027\n\023PrivilegeSelectUser\020\030:^\n\021privilege_" - "ext_obj\022\037.google.protobuf.MessageOptions" - "\030\351\007 \001(\0132!.milvus.proto.common.PrivilegeE" - "xtBW\n\016io.milvus.grpcB\013CommonProtoP\001Z3git" - "hub.com/milvus-io/milvus/internal/proto/" - "commonpb\240\001\001b\006proto3" + "tShardLeaders\020\202\004\022\020\n\013GetReplicas\020\203\004\022\023\n\016Un" + "subDmChannel\020\204\004\022\024\n\017GetDistribution\020\205\004\022\025\n" + "\020SyncDistribution\020\206\004\022\020\n\013SegmentInfo\020\330\004\022\017" + "\n\nSystemInfo\020\331\004\022\024\n\017GetRecoveryInfo\020\332\004\022\024\n" + "\017GetSegmentState\020\333\004\022\r\n\010TimeTick\020\260\t\022\023\n\016Qu" + "eryNodeStats\020\261\t\022\016\n\tLoadIndex\020\262\t\022\016\n\tReque" + "stID\020\263\t\022\017\n\nRequestTSO\020\264\t\022\024\n\017AllocateSegm" + "ent\020\265\t\022\026\n\021SegmentStatistics\020\266\t\022\025\n\020Segmen" + "tFlushDone\020\267\t\022\017\n\nDataNodeTt\020\270\t\022\025\n\020Create" + "Credential\020\334\013\022\022\n\rGetCredential\020\335\013\022\025\n\020Del" + "eteCredential\020\336\013\022\025\n\020UpdateCredential\020\337\013\022" + "\026\n\021ListCredUsernames\020\340\013\022\017\n\nCreateRole\020\300\014" + "\022\r\n\010DropRole\020\301\014\022\024\n\017OperateUserRole\020\302\014\022\017\n" + "\nSelectRole\020\303\014\022\017\n\nSelectUser\020\304\014\022\023\n\016Selec" + "tResource\020\305\014\022\025\n\020OperatePrivilege\020\306\014\022\020\n\013S" + "electGrant\020\307\014\022\033\n\026RefreshPolicyInfoCache\020" + "\310\014\022\017\n\nListPolicy\020\311\014*\"\n\007DslType\022\007\n\003Dsl\020\000\022" + "\016\n\nBoolExprV1\020\001*B\n\017CompactionState\022\021\n\rUn" + "defiedState\020\000\022\r\n\tExecuting\020\001\022\r\n\tComplete" + "d\020\002*X\n\020ConsistencyLevel\022\n\n\006Strong\020\000\022\013\n\007S" + "ession\020\001\022\013\n\007Bounded\020\002\022\016\n\nEventually\020\003\022\016\n" + "\nCustomized\020\004*\257\001\n\013ImportState\022\021\n\rImportP" + "ending\020\000\022\020\n\014ImportFailed\020\001\022\021\n\rImportStar" + "ted\020\002\022\024\n\020ImportDownloaded\020\003\022\020\n\014ImportPar" + "sed\020\004\022\023\n\017ImportPersisted\020\005\022\023\n\017ImportComp" + "leted\020\006\022\026\n\022ImportAllocSegment\020\n*2\n\nObjec" + "tType\022\016\n\nCollection\020\000\022\n\n\006Global\020\001\022\010\n\004Use" + "r\020\002*\206\005\n\017ObjectPrivilege\022\020\n\014PrivilegeAll\020" + "\000\022\035\n\031PrivilegeCreateCollection\020\001\022\033\n\027Priv" + "ilegeDropCollection\020\002\022\037\n\033PrivilegeDescri" + "beCollection\020\003\022\034\n\030PrivilegeShowCollectio" + "ns\020\004\022\021\n\rPrivilegeLoad\020\005\022\024\n\020PrivilegeRele" + "ase\020\006\022\027\n\023PrivilegeCompaction\020\007\022\023\n\017Privil" + "egeInsert\020\010\022\023\n\017PrivilegeDelete\020\t\022\032\n\026Priv" + "ilegeGetStatistics\020\n\022\030\n\024PrivilegeCreateI" + "ndex\020\013\022\030\n\024PrivilegeIndexDetail\020\014\022\026\n\022Priv" + "ilegeDropIndex\020\r\022\023\n\017PrivilegeSearch\020\016\022\022\n" + "\016PrivilegeFlush\020\017\022\022\n\016PrivilegeQuery\020\020\022\030\n" + "\024PrivilegeLoadBalance\020\021\022\023\n\017PrivilegeImpo" + "rt\020\022\022\034\n\030PrivilegeCreateOwnership\020\023\022\027\n\023Pr" + "ivilegeUpdateUser\020\024\022\032\n\026PrivilegeDropOwne" + "rship\020\025\022\034\n\030PrivilegeSelectOwnership\020\026\022\034\n" + "\030PrivilegeManageOwnership\020\027\022\027\n\023Privilege" + "SelectUser\020\030:^\n\021privilege_ext_obj\022\037.goog" + "le.protobuf.MessageOptions\030\351\007 \001(\0132!.milv" + "us.proto.common.PrivilegeExtBW\n\016io.milvu" + "s.grpcB\013CommonProtoP\001Z3github.com/milvus" + "-io/milvus/internal/proto/commonpb\240\001\001b\006p" + "roto3" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_common_2eproto_deps[1] = { &::descriptor_table_google_2fprotobuf_2fdescriptor_2eproto, @@ -489,7 +491,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_com static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_common_2eproto_once; static bool descriptor_table_common_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_common_2eproto = { - &descriptor_table_common_2eproto_initialized, descriptor_table_protodef_common_2eproto, "common.proto", 5179, + &descriptor_table_common_2eproto_initialized, descriptor_table_protodef_common_2eproto, "common.proto", 5245, &descriptor_table_common_2eproto_once, descriptor_table_common_2eproto_sccs, descriptor_table_common_2eproto_deps, 11, 1, schemas, file_default_instances, TableStruct_common_2eproto::offsets, file_level_metadata_common_2eproto, 11, file_level_enum_descriptors_common_2eproto, file_level_service_descriptors_common_2eproto, @@ -668,6 +670,9 @@ bool MsgType_IsValid(int value) { case 513: case 514: case 515: + case 516: + case 517: + case 518: case 600: case 601: case 602: diff --git a/internal/core/src/pb/common.pb.h b/internal/core/src/pb/common.pb.h index 828ce02b210df..11ccdd5c9fcc3 100644 --- a/internal/core/src/pb/common.pb.h +++ b/internal/core/src/pb/common.pb.h @@ -319,6 +319,9 @@ enum MsgType : int { WatchDeltaChannels = 513, GetShardLeaders = 514, GetReplicas = 515, + UnsubDmChannel = 516, + GetDistribution = 517, + SyncDistribution = 518, SegmentInfo = 600, SystemInfo = 601, GetRecoveryInfo = 602, diff --git a/internal/distributed/querycoord/service.go b/internal/distributed/querycoord/service.go index 2174a2b74c94d..507625ef15e03 100644 --- a/internal/distributed/querycoord/service.go +++ b/internal/distributed/querycoord/service.go @@ -39,7 +39,7 @@ import ( "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/milvuspb" "github.com/milvus-io/milvus/internal/proto/querypb" - qc "github.com/milvus-io/milvus/internal/querycoord" + qc "github.com/milvus-io/milvus/internal/querycoordv2" "github.com/milvus-io/milvus/internal/types" "github.com/milvus-io/milvus/internal/util/dependency" "github.com/milvus-io/milvus/internal/util/etcd" diff --git a/internal/distributed/querynode/client/client.go b/internal/distributed/querynode/client/client.go index b2059e0b4df7b..f4d8132f42451 100644 --- a/internal/distributed/querynode/client/client.go +++ b/internal/distributed/querynode/client/client.go @@ -152,6 +152,20 @@ func (c *Client) WatchDmChannels(ctx context.Context, req *querypb.WatchDmChanne return ret.(*commonpb.Status), err } +// UnsubDmChannel unsubscribes the channels about data manipulation. +func (c *Client) UnsubDmChannel(ctx context.Context, req *querypb.UnsubDmChannelRequest) (*commonpb.Status, error) { + ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) { + if !funcutil.CheckCtxValid(ctx) { + return nil, ctx.Err() + } + return client.(querypb.QueryNodeClient).UnsubDmChannel(ctx, req) + }) + if err != nil || ret == nil { + return nil, err + } + return ret.(*commonpb.Status), err +} + // LoadSegments loads the segments to search. func (c *Client) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) { @@ -305,3 +319,29 @@ func (c *Client) GetStatistics(ctx context.Context, request *querypb.GetStatisti } return ret.(*internalpb.GetStatisticsResponse), err } + +func (c *Client) GetDataDistribution(ctx context.Context, req *querypb.GetDataDistributionRequest) (*querypb.GetDataDistributionResponse, error) { + ret, err := c.grpcClient.Call(ctx, func(client interface{}) (interface{}, error) { + if !funcutil.CheckCtxValid(ctx) { + return nil, ctx.Err() + } + return client.(querypb.QueryNodeClient).GetDataDistribution(ctx, req) + }) + if err != nil || ret == nil { + return nil, err + } + return ret.(*querypb.GetDataDistributionResponse), err +} + +func (c *Client) SyncDistribution(ctx context.Context, req *querypb.SyncDistributionRequest) (*commonpb.Status, error) { + ret, err := c.grpcClient.Call(ctx, func(client interface{}) (interface{}, error) { + if !funcutil.CheckCtxValid(ctx) { + return nil, ctx.Err() + } + return client.(querypb.QueryNodeClient).SyncDistribution(ctx, req) + }) + if err != nil || ret == nil { + return nil, err + } + return ret.(*commonpb.Status), err +} diff --git a/internal/distributed/querynode/service.go b/internal/distributed/querynode/service.go index 696b4d767f1e8..d1797ed31acf2 100644 --- a/internal/distributed/querynode/service.go +++ b/internal/distributed/querynode/service.go @@ -268,6 +268,10 @@ func (s *Server) WatchDmChannels(ctx context.Context, req *querypb.WatchDmChanne return s.querynode.WatchDmChannels(ctx, req) } +func (s *Server) UnsubDmChannel(ctx context.Context, req *querypb.UnsubDmChannelRequest) (*commonpb.Status, error) { + return s.querynode.UnsubDmChannel(ctx, req) +} + // LoadSegments loads the segments to search. func (s *Server) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { // ignore ctx @@ -321,3 +325,12 @@ func (s *Server) ShowConfigurations(ctx context.Context, req *internalpb.ShowCon func (s *Server) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) { return s.querynode.GetMetrics(ctx, req) } + +// GetDataDistribution gets the distribution information of QueryNode. +func (s *Server) GetDataDistribution(ctx context.Context, req *querypb.GetDataDistributionRequest) (*querypb.GetDataDistributionResponse, error) { + return s.querynode.GetDataDistribution(ctx, req) +} + +func (s *Server) SyncDistribution(ctx context.Context, req *querypb.SyncDistributionRequest) (*commonpb.Status, error) { + return s.querynode.SyncDistribution(ctx, req) +} diff --git a/internal/distributed/querynode/service_test.go b/internal/distributed/querynode/service_test.go index c4cc94fb07002..57bfa71e29bfc 100644 --- a/internal/distributed/querynode/service_test.go +++ b/internal/distributed/querynode/service_test.go @@ -48,6 +48,7 @@ type MockQueryNode struct { StatsResp *internalpb.GetStatisticsResponse searchResp *internalpb.SearchResults queryResp *internalpb.RetrieveResults + distResp *querypb.GetDataDistributionResponse } func (m *MockQueryNode) Init() error { @@ -140,7 +141,16 @@ func (m *MockQueryNode) ShowConfigurations(ctx context.Context, req *internalpb. return m.configResp, m.err } -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +func (m *MockQueryNode) UnsubDmChannel(ctx context.Context, req *querypb.UnsubDmChannelRequest) (*commonpb.Status, error) { + return m.status, m.err +} +func (m *MockQueryNode) GetDataDistribution(context.Context, *querypb.GetDataDistributionRequest) (*querypb.GetDataDistributionResponse, error) { + return m.distResp, m.err +} +func (m *MockQueryNode) SyncDistribution(context.Context, *querypb.SyncDistributionRequest) (*commonpb.Status, error) { + return m.status, m.err +} + type MockRootCoord struct { types.RootCoord initErr error diff --git a/internal/kv/etcd/etcd_kv.go b/internal/kv/etcd/etcd_kv.go index 11dc55a9cc7a7..23c9e2d3e2c44 100644 --- a/internal/kv/etcd/etcd_kv.go +++ b/internal/kv/etcd/etcd_kv.go @@ -27,9 +27,10 @@ import ( "github.com/milvus-io/milvus/internal/common" - "github.com/milvus-io/milvus/internal/log" clientv3 "go.etcd.io/etcd/client/v3" + "github.com/milvus-io/milvus/internal/log" + "go.uber.org/zap" ) @@ -284,7 +285,7 @@ func (kv *EtcdKV) LoadWithRevision(key string) ([]string, []string, int64, error ctx, cancel := context.WithTimeout(context.TODO(), RequestTimeout) defer cancel() resp, err := kv.client.Get(ctx, key, clientv3.WithPrefix(), - clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) + clientv3.WithSort(clientv3.SortByCreateRevision, clientv3.SortAscend)) if err != nil { return nil, nil, 0, err } diff --git a/internal/log/global.go b/internal/log/global.go index 576bb6043f4fc..f9e7580c05e2b 100644 --- a/internal/log/global.go +++ b/internal/log/global.go @@ -103,7 +103,7 @@ func RatedWarn(cost float64, msg string, fields ...zap.Field) bool { // With creates a child logger and adds structured context to it. // Fields added to the child don't affect the parent, and vice versa. func With(fields ...zap.Field) *zap.Logger { - return L().With(fields...) + return L().With(fields...).WithOptions(zap.AddCallerSkip(-1)) } // SetLevel alters the logging level. diff --git a/internal/metastore/catalog.go b/internal/metastore/catalog.go index c56110e723f64..ec3c384d3baaf 100644 --- a/internal/metastore/catalog.go +++ b/internal/metastore/catalog.go @@ -6,6 +6,7 @@ import ( "github.com/milvus-io/milvus/internal/metastore/model" "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/util/typeutil" ) @@ -93,3 +94,17 @@ type IndexCoordCatalog interface { AlterSegmentIndexes(ctx context.Context, newSegIdxes []*model.SegmentIndex) error DropSegmentIndex(ctx context.Context, collID, partID, segID, buildID typeutil.UniqueID) error } + +type QueryCoordCatalog interface { + SaveCollection(info *querypb.CollectionLoadInfo) error + SavePartition(info ...*querypb.PartitionLoadInfo) error + SaveReplica(replica *querypb.Replica) error + GetCollections() ([]*querypb.CollectionLoadInfo, error) + GetPartitions() (map[int64][]*querypb.PartitionLoadInfo, error) + GetReplicas() ([]*querypb.Replica, error) + ReleaseCollection(id int64) error + ReleasePartition(collection int64, partitions ...int64) error + ReleaseReplicas(collectionID int64) error + ReleaseReplica(collection, replica int64) error + RemoveHandoffEvent(segmentInfo *querypb.SegmentInfo) error +} diff --git a/internal/metastore/kv/querycoord/kv_catalog.go b/internal/metastore/kv/querycoord/kv_catalog.go new file mode 100644 index 0000000000000..d2c3977baa792 --- /dev/null +++ b/internal/metastore/kv/querycoord/kv_catalog.go @@ -0,0 +1,266 @@ +package querycoord + +import ( + "errors" + "fmt" + + "github.com/golang/protobuf/proto" + "github.com/samber/lo" + clientv3 "go.etcd.io/etcd/client/v3" + + "github.com/milvus-io/milvus/internal/kv" + "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/util" +) + +var ( + ErrInvalidKey = errors.New("invalid load info key") +) + +const ( + CollectionLoadInfoPrefix = "querycoord-collection-loadinfo" + PartitionLoadInfoPrefix = "querycoord-partition-loadinfo" + ReplicaPrefix = "querycoord-replica" + CollectionMetaPrefixV1 = "queryCoord-collectionMeta" + ReplicaMetaPrefixV1 = "queryCoord-ReplicaMeta" +) + +type WatchStoreChan = clientv3.WatchChan + +type Catalog struct { + cli kv.MetaKv +} + +func NewCatalog(cli kv.MetaKv) Catalog { + return Catalog{ + cli: cli, + } +} + +func (s Catalog) SaveCollection(info *querypb.CollectionLoadInfo) error { + k := encodeCollectionLoadInfoKey(info.GetCollectionID()) + v, err := proto.Marshal(info) + if err != nil { + return err + } + return s.cli.Save(k, string(v)) +} + +func (s Catalog) SavePartition(info ...*querypb.PartitionLoadInfo) error { + kvs := make(map[string]string) + for _, partition := range info { + key := encodePartitionLoadInfoKey(partition.GetCollectionID(), partition.GetPartitionID()) + value, err := proto.Marshal(partition) + if err != nil { + return err + } + kvs[key] = string(value) + } + return s.cli.MultiSave(kvs) +} + +func (s Catalog) SaveReplica(replica *querypb.Replica) error { + key := encodeReplicaKey(replica.GetCollectionID(), replica.GetID()) + value, err := proto.Marshal(replica) + if err != nil { + return err + } + return s.cli.Save(key, string(value)) +} + +func (s Catalog) GetCollections() ([]*querypb.CollectionLoadInfo, error) { + _, values, err := s.cli.LoadWithPrefix(CollectionLoadInfoPrefix) + if err != nil { + return nil, err + } + ret := make([]*querypb.CollectionLoadInfo, 0, len(values)) + for _, v := range values { + info := querypb.CollectionLoadInfo{} + if err := proto.Unmarshal([]byte(v), &info); err != nil { + return nil, err + } + ret = append(ret, &info) + } + + collectionsV1, err := s.getCollectionsFromV1() + if err != nil { + return nil, err + } + ret = append(ret, collectionsV1...) + + return ret, nil +} + +// getCollectionsFromV1 recovers collections from 2.1 meta store +func (s Catalog) getCollectionsFromV1() ([]*querypb.CollectionLoadInfo, error) { + _, collectionValues, err := s.cli.LoadWithPrefix(CollectionMetaPrefixV1) + if err != nil { + return nil, err + } + ret := make([]*querypb.CollectionLoadInfo, 0, len(collectionValues)) + for _, value := range collectionValues { + collectionInfo := querypb.CollectionInfo{} + err = proto.Unmarshal([]byte(value), &collectionInfo) + if err != nil { + return nil, err + } + if collectionInfo.LoadType != querypb.LoadType_LoadCollection { + continue + } + ret = append(ret, &querypb.CollectionLoadInfo{ + CollectionID: collectionInfo.GetCollectionID(), + ReleasedPartitions: collectionInfo.GetReleasedPartitionIDs(), + ReplicaNumber: collectionInfo.GetReplicaNumber(), + Status: querypb.LoadStatus_Loaded, + }) + } + return ret, nil +} + +func (s Catalog) GetPartitions() (map[int64][]*querypb.PartitionLoadInfo, error) { + _, values, err := s.cli.LoadWithPrefix(PartitionLoadInfoPrefix) + if err != nil { + return nil, err + } + ret := make(map[int64][]*querypb.PartitionLoadInfo) + for _, v := range values { + info := querypb.PartitionLoadInfo{} + if err := proto.Unmarshal([]byte(v), &info); err != nil { + return nil, err + } + ret[info.GetCollectionID()] = append(ret[info.GetCollectionID()], &info) + } + + partitionsV1, err := s.getPartitionsFromV1() + if err != nil { + return nil, err + } + for _, partition := range partitionsV1 { + ret[partition.GetCollectionID()] = append(ret[partition.GetCollectionID()], partition) + } + + return ret, nil +} + +// getCollectionsFromV1 recovers collections from 2.1 meta store +func (s Catalog) getPartitionsFromV1() ([]*querypb.PartitionLoadInfo, error) { + _, collectionValues, err := s.cli.LoadWithPrefix(CollectionMetaPrefixV1) + if err != nil { + return nil, err + } + ret := make([]*querypb.PartitionLoadInfo, 0, len(collectionValues)) + for _, value := range collectionValues { + collectionInfo := querypb.CollectionInfo{} + err = proto.Unmarshal([]byte(value), &collectionInfo) + if err != nil { + return nil, err + } + if collectionInfo.LoadType != querypb.LoadType_LoadPartition { + continue + } + + for _, partition := range collectionInfo.GetPartitionIDs() { + ret = append(ret, &querypb.PartitionLoadInfo{ + CollectionID: collectionInfo.GetCollectionID(), + PartitionID: partition, + ReplicaNumber: collectionInfo.GetReplicaNumber(), + Status: querypb.LoadStatus_Loaded, + }) + } + } + return ret, nil +} + +func (s Catalog) GetReplicas() ([]*querypb.Replica, error) { + _, values, err := s.cli.LoadWithPrefix(ReplicaPrefix) + if err != nil { + return nil, err + } + ret := make([]*querypb.Replica, 0, len(values)) + for _, v := range values { + info := querypb.Replica{} + if err := proto.Unmarshal([]byte(v), &info); err != nil { + return nil, err + } + ret = append(ret, &info) + } + + replicasV1, err := s.getReplicasFromV1() + if err != nil { + return nil, err + } + ret = append(ret, replicasV1...) + + return ret, nil +} + +func (s Catalog) getReplicasFromV1() ([]*querypb.Replica, error) { + _, replicaValues, err := s.cli.LoadWithPrefix(ReplicaMetaPrefixV1) + if err != nil { + return nil, err + } + + ret := make([]*querypb.Replica, 0, len(replicaValues)) + for _, value := range replicaValues { + replicaInfo := milvuspb.ReplicaInfo{} + err = proto.Unmarshal([]byte(value), &replicaInfo) + if err != nil { + return nil, err + } + + ret = append(ret, &querypb.Replica{ + ID: replicaInfo.GetReplicaID(), + CollectionID: replicaInfo.GetCollectionID(), + Nodes: replicaInfo.GetNodeIds(), + }) + } + return ret, nil +} + +func (s Catalog) ReleaseCollection(id int64) error { + k := encodeCollectionLoadInfoKey(id) + return s.cli.Remove(k) +} + +func (s Catalog) ReleasePartition(collection int64, partitions ...int64) error { + keys := lo.Map(partitions, func(partition int64, _ int) string { + return encodePartitionLoadInfoKey(collection, partition) + }) + return s.cli.MultiRemove(keys) +} + +func (s Catalog) ReleaseReplicas(collectionID int64) error { + key := encodeCollectionReplicaKey(collectionID) + return s.cli.RemoveWithPrefix(key) +} + +func (s Catalog) ReleaseReplica(collection, replica int64) error { + key := encodeReplicaKey(collection, replica) + return s.cli.Remove(key) +} + +func (s Catalog) RemoveHandoffEvent(info *querypb.SegmentInfo) error { + key := encodeHandoffEventKey(info.CollectionID, info.PartitionID, info.SegmentID) + return s.cli.Remove(key) +} + +func encodeCollectionLoadInfoKey(collection int64) string { + return fmt.Sprintf("%s/%d", CollectionLoadInfoPrefix, collection) +} + +func encodePartitionLoadInfoKey(collection, partition int64) string { + return fmt.Sprintf("%s/%d/%d", PartitionLoadInfoPrefix, collection, partition) +} + +func encodeReplicaKey(collection, replica int64) string { + return fmt.Sprintf("%s/%d/%d", ReplicaPrefix, collection, replica) +} + +func encodeCollectionReplicaKey(collection int64) string { + return fmt.Sprintf("%s/%d", ReplicaPrefix, collection) +} + +func encodeHandoffEventKey(collection, partition, segment int64) string { + return fmt.Sprintf("%s/%d/%d/%d", util.HandoffSegmentPrefix, collection, partition, segment) +} diff --git a/internal/metastore/kv/querycoord/kv_catalog_test.go b/internal/metastore/kv/querycoord/kv_catalog_test.go new file mode 100644 index 0000000000000..f953dc29d057f --- /dev/null +++ b/internal/metastore/kv/querycoord/kv_catalog_test.go @@ -0,0 +1,27 @@ +package querycoord + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type StoreTestSuite struct { + suite.Suite + store Catalog +} + +func (suite *StoreTestSuite) SetupTest() { + //kv := memkv.NewMemoryKV() + //suite.store = NewMetaStore(kv) +} + +func (suite *StoreTestSuite) TearDownTest() {} + +func (suite *StoreTestSuite) TestLoadRelease() { + // TODO(sunby): add ut +} + +func TestStoreSuite(t *testing.T) { + suite.Run(t, new(StoreTestSuite)) +} diff --git a/internal/proto/common.proto b/internal/proto/common.proto index 78dfea1defec1..c595289fc8843 100644 --- a/internal/proto/common.proto +++ b/internal/proto/common.proto @@ -187,6 +187,9 @@ enum MsgType { WatchDeltaChannels = 513; GetShardLeaders = 514; GetReplicas = 515; + UnsubDmChannel = 516; + GetDistribution = 517; + SyncDistribution = 518; /* DATA SERVICE */ SegmentInfo = 600; diff --git a/internal/proto/commonpb/common.pb.go b/internal/proto/commonpb/common.pb.go index 536d3b3674cf5..623d14655f426 100644 --- a/internal/proto/commonpb/common.pb.go +++ b/internal/proto/commonpb/common.pb.go @@ -351,6 +351,9 @@ const ( MsgType_WatchDeltaChannels MsgType = 513 MsgType_GetShardLeaders MsgType = 514 MsgType_GetReplicas MsgType = 515 + MsgType_UnsubDmChannel MsgType = 516 + MsgType_GetDistribution MsgType = 517 + MsgType_SyncDistribution MsgType = 518 // DATA SERVICE MsgType_SegmentInfo MsgType = 600 MsgType_SystemInfo MsgType = 601 @@ -435,6 +438,9 @@ var MsgType_name = map[int32]string{ 513: "WatchDeltaChannels", 514: "GetShardLeaders", 515: "GetReplicas", + 516: "UnsubDmChannel", + 517: "GetDistribution", + 518: "SyncDistribution", 600: "SegmentInfo", 601: "SystemInfo", 602: "GetRecoveryInfo", @@ -515,6 +521,9 @@ var MsgType_value = map[string]int32{ "WatchDeltaChannels": 513, "GetShardLeaders": 514, "GetReplicas": 515, + "UnsubDmChannel": 516, + "GetDistribution": 517, + "SyncDistribution": 518, "SegmentInfo": 600, "SystemInfo": 601, "GetRecoveryInfo": 602, @@ -1379,160 +1388,163 @@ func init() { func init() { proto.RegisterFile("common.proto", fileDescriptor_555bd8c177793206) } var fileDescriptor_555bd8c177793206 = []byte{ - // 2480 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x58, 0x49, 0x73, 0x24, 0x47, - 0x15, 0x56, 0x75, 0xb7, 0x96, 0xce, 0x6e, 0x49, 0x4f, 0x29, 0x8d, 0xa6, 0x3d, 0x8b, 0x47, 0x16, - 0x36, 0x0c, 0x8d, 0xad, 0x81, 0x71, 0x04, 0x10, 0x44, 0x98, 0x40, 0xea, 0x96, 0x34, 0x0a, 0x8f, - 0x16, 0x4a, 0x1a, 0x9b, 0x20, 0x02, 0x26, 0x52, 0x55, 0x4f, 0xad, 0x9a, 0xa9, 0xae, 0x2c, 0x2a, - 0xb3, 0x35, 0x6a, 0x4e, 0xc6, 0x44, 0x70, 0x06, 0xf3, 0x07, 0xf8, 0x01, 0xec, 0x8b, 0xe1, 0xc8, - 0x8e, 0xcd, 0x76, 0xe1, 0xc2, 0x0e, 0x47, 0xb8, 0xb3, 0x7a, 0x25, 0x5e, 0x66, 0x6d, 0xad, 0x19, - 0xc3, 0x81, 0x5b, 0xe7, 0xf7, 0x5e, 0xbe, 0x7d, 0xc9, 0x6a, 0xd6, 0xf4, 0x64, 0xbf, 0x2f, 0xa3, - 0x95, 0x38, 0x91, 0x5a, 0xf2, 0xf9, 0x7e, 0x10, 0x9e, 0x0c, 0x94, 0x3d, 0xad, 0x58, 0xd2, 0x85, - 0xa5, 0x9e, 0x94, 0xbd, 0x10, 0xaf, 0x19, 0xf0, 0x70, 0x70, 0x74, 0xcd, 0x47, 0xe5, 0x25, 0x41, - 0xac, 0x65, 0x62, 0x19, 0x97, 0x6f, 0xb3, 0x89, 0x7d, 0x2d, 0xf4, 0x40, 0xf1, 0xa7, 0x18, 0xc3, - 0x24, 0x91, 0xc9, 0x6d, 0x4f, 0xfa, 0xd8, 0x72, 0x96, 0x9c, 0xab, 0x33, 0xd7, 0x1f, 0x5e, 0x79, - 0x80, 0xd4, 0x95, 0x75, 0x62, 0xeb, 0x48, 0x1f, 0xdd, 0x3a, 0x66, 0x3f, 0xf9, 0x22, 0x9b, 0x48, - 0x50, 0x28, 0x19, 0xb5, 0x2a, 0x4b, 0xce, 0xd5, 0xba, 0x9b, 0x9e, 0x96, 0xdf, 0xcb, 0x9a, 0x4f, - 0xe3, 0xf0, 0x19, 0x11, 0x0e, 0x70, 0x4f, 0x04, 0x09, 0x07, 0x56, 0xbd, 0x8b, 0x43, 0x23, 0xbf, - 0xee, 0xd2, 0x4f, 0xbe, 0xc0, 0xc6, 0x4f, 0x88, 0x9c, 0x5e, 0xb4, 0x87, 0xe5, 0x27, 0x59, 0xe3, - 0x69, 0x1c, 0x76, 0x85, 0x16, 0x6f, 0x71, 0x8d, 0xb3, 0x9a, 0x2f, 0xb4, 0x30, 0xb7, 0x9a, 0xae, - 0xf9, 0xbd, 0x7c, 0x89, 0xd5, 0xd6, 0x42, 0x79, 0x58, 0x88, 0x74, 0x0c, 0x31, 0x15, 0x79, 0xc2, - 0x60, 0x2f, 0x14, 0x1e, 0x1e, 0xcb, 0xd0, 0xc7, 0xc4, 0x98, 0x44, 0x72, 0xb5, 0xe8, 0x65, 0x72, - 0xb5, 0xe8, 0xf1, 0xf7, 0xb3, 0x9a, 0x1e, 0xc6, 0xd6, 0x9a, 0x99, 0xeb, 0x8f, 0x3e, 0x30, 0x02, - 0x25, 0x31, 0x07, 0xc3, 0x18, 0x5d, 0x73, 0x83, 0x42, 0x60, 0x14, 0xa9, 0x56, 0x75, 0xa9, 0x7a, - 0xb5, 0xe9, 0xa6, 0xa7, 0xe5, 0x8f, 0x8d, 0xe8, 0xdd, 0x4c, 0xe4, 0x20, 0xe6, 0x5b, 0xac, 0x19, - 0x17, 0x98, 0x6a, 0x39, 0x4b, 0xd5, 0xab, 0x8d, 0xeb, 0x8f, 0xfd, 0x2f, 0x6d, 0xc6, 0x68, 0x77, - 0xe4, 0xea, 0xf2, 0x13, 0x6c, 0x72, 0xd5, 0xf7, 0x13, 0x54, 0x8a, 0xcf, 0xb0, 0x4a, 0x10, 0xa7, - 0xce, 0x54, 0x82, 0x98, 0x62, 0x14, 0xcb, 0x44, 0x1b, 0x5f, 0xaa, 0xae, 0xf9, 0xbd, 0xfc, 0x82, - 0xc3, 0x26, 0xb7, 0x55, 0x6f, 0x4d, 0x28, 0xe4, 0xef, 0x63, 0x53, 0x7d, 0xd5, 0xbb, 0x6d, 0xfc, - 0xb5, 0x19, 0xbf, 0xf4, 0x40, 0x0b, 0xb6, 0x55, 0xcf, 0xf8, 0x39, 0xd9, 0xb7, 0x3f, 0x28, 0xc0, - 0x7d, 0xd5, 0xdb, 0xea, 0xa6, 0x92, 0xed, 0x81, 0x5f, 0x62, 0x75, 0x1d, 0xf4, 0x51, 0x69, 0xd1, - 0x8f, 0x5b, 0xd5, 0x25, 0xe7, 0x6a, 0xcd, 0x2d, 0x00, 0x7e, 0x81, 0x4d, 0x29, 0x39, 0x48, 0x3c, - 0xdc, 0xea, 0xb6, 0x6a, 0xe6, 0x5a, 0x7e, 0x5e, 0x7e, 0x8a, 0xd5, 0xb7, 0x55, 0xef, 0x06, 0x0a, - 0x1f, 0x13, 0xfe, 0x6e, 0x56, 0x3b, 0x14, 0xca, 0x5a, 0xd4, 0x78, 0x6b, 0x8b, 0xc8, 0x03, 0xd7, - 0x70, 0x2e, 0x7f, 0x9c, 0x35, 0xbb, 0xdb, 0x37, 0xff, 0x0f, 0x09, 0x64, 0xba, 0x3a, 0x16, 0x89, - 0xbf, 0x23, 0xfa, 0x59, 0x21, 0x16, 0xc0, 0xf2, 0xab, 0x0e, 0x6b, 0xee, 0x25, 0xc1, 0x49, 0x10, - 0x62, 0x0f, 0xd7, 0x4f, 0x35, 0xff, 0x10, 0x6b, 0xc8, 0xc3, 0x3b, 0xe8, 0xe9, 0x72, 0xec, 0xae, - 0x3c, 0x50, 0xcf, 0xae, 0xe1, 0x33, 0xe1, 0x63, 0x32, 0xff, 0xcd, 0x77, 0x19, 0xa4, 0x12, 0xe2, - 0x4c, 0xf0, 0x7f, 0x2d, 0x39, 0x2b, 0x26, 0x37, 0xc2, 0x9d, 0x95, 0xa3, 0x00, 0x6f, 0xb3, 0xb9, - 0x54, 0x60, 0x24, 0xfa, 0x78, 0x3b, 0x88, 0x7c, 0x3c, 0x35, 0x49, 0x18, 0xcf, 0x78, 0xc9, 0x95, - 0x2d, 0x82, 0xf9, 0xe3, 0x8c, 0xdf, 0xc7, 0xab, 0x4c, 0x52, 0xc6, 0x5d, 0x38, 0xc3, 0xac, 0xda, - 0xbf, 0x9a, 0x62, 0xf5, 0xbc, 0xe7, 0x79, 0x83, 0x4d, 0xee, 0x0f, 0x3c, 0x0f, 0x95, 0x82, 0x31, - 0x3e, 0xcf, 0x66, 0x6f, 0x45, 0x78, 0x1a, 0xa3, 0xa7, 0xd1, 0x37, 0x3c, 0xe0, 0xf0, 0x39, 0x36, - 0xdd, 0x91, 0x51, 0x84, 0x9e, 0xde, 0x10, 0x41, 0x88, 0x3e, 0x54, 0xf8, 0x02, 0x83, 0x3d, 0x4c, - 0xfa, 0x81, 0x52, 0x81, 0x8c, 0xba, 0x18, 0x05, 0xe8, 0x43, 0x95, 0x9f, 0x67, 0xf3, 0x1d, 0x19, - 0x86, 0xe8, 0xe9, 0x40, 0x46, 0x3b, 0x52, 0xaf, 0x9f, 0x06, 0x4a, 0x2b, 0xa8, 0x91, 0xd8, 0xad, - 0x30, 0xc4, 0x9e, 0x08, 0x57, 0x93, 0xde, 0xa0, 0x8f, 0x91, 0x86, 0x71, 0x92, 0x91, 0x82, 0xdd, - 0xa0, 0x8f, 0x11, 0x49, 0x82, 0xc9, 0x12, 0x6a, 0xac, 0xa5, 0xd8, 0xc2, 0x14, 0x7f, 0x88, 0x9d, - 0x4b, 0xd1, 0x92, 0x02, 0xd1, 0x47, 0xa8, 0xf3, 0x59, 0xd6, 0x48, 0x49, 0x07, 0xbb, 0x7b, 0x4f, - 0x03, 0x2b, 0x49, 0x70, 0xe5, 0x3d, 0x17, 0x3d, 0x99, 0xf8, 0xd0, 0x28, 0x99, 0xf0, 0x0c, 0x7a, - 0x5a, 0x26, 0x5b, 0x5d, 0x68, 0x92, 0xc1, 0x29, 0xb8, 0x8f, 0x22, 0xf1, 0x8e, 0x5d, 0x54, 0x83, - 0x50, 0xc3, 0x34, 0x07, 0xd6, 0xdc, 0x08, 0x42, 0xdc, 0x91, 0x7a, 0x43, 0x0e, 0x22, 0x1f, 0x66, - 0xf8, 0x0c, 0x63, 0xdb, 0xa8, 0x45, 0x1a, 0x81, 0x59, 0x52, 0xdb, 0x11, 0xde, 0x31, 0xa6, 0x00, - 0xf0, 0x45, 0xc6, 0x3b, 0x22, 0x8a, 0xa4, 0xee, 0x24, 0x28, 0x34, 0x6e, 0x98, 0x6e, 0x86, 0x39, - 0x32, 0x67, 0x04, 0x0f, 0x42, 0x04, 0x5e, 0x70, 0x77, 0x31, 0xc4, 0x9c, 0x7b, 0xbe, 0xe0, 0x4e, - 0x71, 0xe2, 0x5e, 0x20, 0xe3, 0xd7, 0x06, 0x41, 0xe8, 0x9b, 0x90, 0xd8, 0xb4, 0x9c, 0x23, 0x1b, - 0x53, 0xe3, 0x77, 0x6e, 0x6e, 0xed, 0x1f, 0xc0, 0x22, 0x3f, 0xc7, 0xe6, 0x52, 0x64, 0x1b, 0x75, - 0x12, 0x78, 0x26, 0x78, 0xe7, 0xc9, 0xd4, 0xdd, 0x81, 0xde, 0x3d, 0xda, 0xc6, 0xbe, 0x4c, 0x86, - 0xd0, 0xa2, 0x84, 0x1a, 0x49, 0x59, 0x8a, 0xe0, 0x21, 0xd2, 0xb0, 0xde, 0x8f, 0xf5, 0xb0, 0x08, - 0x2f, 0x5c, 0xe0, 0x17, 0xd9, 0xf9, 0x5b, 0xb1, 0x2f, 0x34, 0x6e, 0xf5, 0x69, 0xd4, 0x1c, 0x08, - 0x75, 0x97, 0xdc, 0x1d, 0x24, 0x08, 0x17, 0xf9, 0x05, 0xb6, 0x38, 0x9a, 0x8b, 0x3c, 0x58, 0x97, - 0xe8, 0xa2, 0xf5, 0xb6, 0x93, 0xa0, 0x8f, 0x91, 0x0e, 0x44, 0x98, 0x5d, 0xbc, 0x5c, 0x48, 0xbd, - 0x9f, 0xf8, 0x30, 0x11, 0xad, 0xe7, 0xf7, 0x13, 0xaf, 0xf0, 0x16, 0x5b, 0xd8, 0x44, 0x7d, 0x3f, - 0x65, 0x89, 0x28, 0x37, 0x03, 0x65, 0x48, 0xb7, 0x14, 0x26, 0x2a, 0xa3, 0x3c, 0xc2, 0x39, 0x9b, - 0xd9, 0x44, 0x4d, 0x60, 0x86, 0x2d, 0x53, 0x9c, 0xac, 0x79, 0xae, 0x0c, 0x31, 0x83, 0xdf, 0x46, - 0x31, 0xe8, 0x26, 0x32, 0x2e, 0x83, 0x8f, 0x92, 0x9b, 0xbb, 0x31, 0x26, 0x42, 0x23, 0xc9, 0x28, - 0xd3, 0x1e, 0x23, 0x39, 0xfb, 0x48, 0x11, 0x28, 0xc3, 0x6f, 0x2f, 0xe0, 0xb2, 0xd6, 0x77, 0x50, - 0x0d, 0xa7, 0xdc, 0x68, 0xe7, 0x64, 0x46, 0xba, 0x4a, 0x5e, 0xa7, 0x4a, 0xf2, 0xfe, 0xcf, 0x88, - 0xef, 0xa4, 0x52, 0xb1, 0xf7, 0x36, 0x13, 0x11, 0xe9, 0x0c, 0x6f, 0xf3, 0x47, 0xd8, 0x65, 0x17, - 0x8f, 0x12, 0x54, 0xc7, 0x7b, 0x32, 0x0c, 0xbc, 0xe1, 0x56, 0x74, 0x24, 0xf3, 0x92, 0x24, 0x96, - 0x77, 0x91, 0x25, 0x14, 0x16, 0x4b, 0xcf, 0xe0, 0xc7, 0x29, 0x26, 0x3b, 0x52, 0xef, 0xd3, 0x38, - 0xbc, 0x69, 0x06, 0x2c, 0x3c, 0x41, 0x5a, 0x76, 0xa4, 0x8b, 0x71, 0x18, 0x78, 0x62, 0xf5, 0x44, - 0x04, 0xa1, 0x38, 0x0c, 0x11, 0x56, 0x28, 0x28, 0xfb, 0xd8, 0xa3, 0x96, 0xcd, 0xf3, 0x7b, 0x8d, - 0x73, 0x36, 0xdd, 0xed, 0xba, 0xf8, 0x89, 0x01, 0x2a, 0xed, 0x0a, 0x0f, 0xe1, 0x2f, 0x93, 0x6d, - 0x8f, 0x31, 0x53, 0x54, 0xf4, 0xfc, 0x40, 0x52, 0x51, 0x9c, 0x76, 0x64, 0x84, 0x30, 0xc6, 0x9b, - 0x6c, 0xea, 0x56, 0x14, 0x28, 0x35, 0x40, 0x1f, 0x1c, 0x6a, 0xa8, 0xad, 0x68, 0x2f, 0x91, 0x3d, - 0xda, 0x74, 0x50, 0x21, 0xea, 0x46, 0x10, 0x05, 0xea, 0xd8, 0x8c, 0x12, 0xc6, 0x26, 0xd2, 0xce, - 0xaa, 0xf1, 0x3a, 0x1b, 0x77, 0x51, 0x27, 0x43, 0x18, 0x6f, 0x3f, 0xef, 0xb0, 0x66, 0x6a, 0x8e, - 0xd5, 0xb3, 0xc0, 0xa0, 0x7c, 0x2e, 0x34, 0xe5, 0xb5, 0xed, 0xd0, 0x84, 0xdb, 0x4c, 0xe4, 0xbd, - 0x20, 0xea, 0x41, 0x85, 0x04, 0xef, 0xa3, 0x08, 0x8d, 0x92, 0x06, 0x9b, 0xdc, 0x08, 0x07, 0x46, - 0x63, 0xcd, 0xe8, 0xa7, 0x03, 0xb1, 0x8d, 0x13, 0x89, 0x6a, 0x21, 0x46, 0x1f, 0x26, 0xf8, 0x34, - 0xab, 0xdb, 0x0e, 0x20, 0xda, 0x64, 0xfb, 0x83, 0x6c, 0xf6, 0xcc, 0x83, 0x81, 0x4f, 0xb1, 0x5a, - 0xaa, 0x1a, 0x58, 0x73, 0x2d, 0x88, 0x44, 0x32, 0xb4, 0x63, 0x06, 0x7c, 0x6a, 0xbf, 0x8d, 0x50, - 0x0a, 0x9d, 0x02, 0xd8, 0x7e, 0xb1, 0x69, 0x36, 0xb6, 0xb9, 0x38, 0xcd, 0xea, 0xb7, 0x22, 0x1f, - 0x8f, 0x82, 0x08, 0x7d, 0x18, 0x33, 0xed, 0x6f, 0x1b, 0xa7, 0xe8, 0x43, 0x9f, 0x82, 0x49, 0xc6, - 0x94, 0x30, 0xa4, 0x1e, 0xbe, 0x21, 0x54, 0x09, 0x3a, 0xa2, 0x14, 0x76, 0xcd, 0x7b, 0xf0, 0xb0, - 0x7c, 0xbd, 0x67, 0x52, 0x78, 0x2c, 0xef, 0x15, 0x98, 0x82, 0x63, 0xd2, 0xb4, 0x89, 0x7a, 0x7f, - 0xa8, 0x34, 0xf6, 0x3b, 0x32, 0x3a, 0x0a, 0x7a, 0x0a, 0x02, 0xd2, 0x74, 0x53, 0x0a, 0xbf, 0x74, - 0xfd, 0x0e, 0x15, 0x91, 0x8b, 0x21, 0x0a, 0x55, 0x96, 0x7a, 0xd7, 0x0c, 0x40, 0x63, 0xea, 0x6a, - 0x18, 0x08, 0x05, 0x21, 0xb9, 0x42, 0x56, 0xda, 0x63, 0x9f, 0xf2, 0xbb, 0x1a, 0x6a, 0x4c, 0xec, - 0x39, 0xe2, 0x0b, 0x6c, 0xd6, 0xf2, 0xef, 0x89, 0x44, 0x07, 0x46, 0xc8, 0x4b, 0x8e, 0xa9, 0xa4, - 0x44, 0xc6, 0x05, 0xf6, 0x32, 0xed, 0x9b, 0xe6, 0x0d, 0xa1, 0x0a, 0xe8, 0xa7, 0x0e, 0x5f, 0x64, - 0x73, 0x99, 0x6b, 0x05, 0xfe, 0x33, 0x87, 0xcf, 0xb3, 0x19, 0x72, 0x2d, 0xc7, 0x14, 0xfc, 0xdc, - 0x80, 0xe4, 0x44, 0x09, 0xfc, 0x85, 0x91, 0x90, 0x7a, 0x51, 0xc2, 0x7f, 0x69, 0x94, 0x91, 0x84, - 0xb4, 0x88, 0x14, 0xbc, 0xe2, 0x90, 0xa5, 0x99, 0xb2, 0x14, 0x86, 0x57, 0x0d, 0x23, 0x49, 0xcd, - 0x19, 0x5f, 0x33, 0x8c, 0xa9, 0xcc, 0x1c, 0x7d, 0xdd, 0xa0, 0x37, 0x44, 0xe4, 0xcb, 0xa3, 0xa3, - 0x1c, 0x7d, 0xc3, 0xe1, 0x2d, 0x36, 0x4f, 0xd7, 0xd7, 0x44, 0x28, 0x22, 0xaf, 0xe0, 0x7f, 0xd3, - 0xe1, 0xe7, 0x18, 0x9c, 0x51, 0xa7, 0xe0, 0xb9, 0x0a, 0x87, 0x2c, 0xbe, 0xa6, 0x8f, 0xe0, 0x8b, - 0x15, 0x13, 0xab, 0x94, 0xd1, 0x62, 0x5f, 0xaa, 0xf0, 0x19, 0x1b, 0x74, 0x7b, 0xfe, 0x72, 0x85, - 0x37, 0xd8, 0xc4, 0x56, 0xa4, 0x30, 0xd1, 0xf0, 0x59, 0xaa, 0xef, 0x09, 0x3b, 0x4c, 0xe1, 0x73, - 0xd4, 0x51, 0xe3, 0xa6, 0xbe, 0xe1, 0x05, 0x5a, 0xd4, 0xdc, 0x45, 0x85, 0x91, 0x5f, 0xea, 0x1d, - 0x05, 0x9f, 0x37, 0x37, 0xec, 0x26, 0x84, 0xbf, 0x55, 0x4d, 0x68, 0xca, 0x6b, 0xf1, 0xef, 0x55, - 0x32, 0x61, 0x13, 0x75, 0xd1, 0xd9, 0xf0, 0x8f, 0x2a, 0xbf, 0xc0, 0xce, 0x65, 0x98, 0x59, 0x52, - 0x79, 0x4f, 0xff, 0xb3, 0xca, 0x2f, 0xb1, 0xf3, 0x34, 0xb1, 0xf3, 0xba, 0xa1, 0x4b, 0x81, 0xd2, - 0x81, 0xa7, 0xe0, 0x5f, 0x55, 0x7e, 0x91, 0x2d, 0x6e, 0xa2, 0xce, 0xf3, 0x51, 0x22, 0xfe, 0xbb, - 0xca, 0xa7, 0xd9, 0x14, 0x75, 0x7d, 0x80, 0x27, 0x08, 0xaf, 0x54, 0x29, 0xa9, 0xd9, 0x31, 0x35, - 0xe7, 0xd5, 0x2a, 0x85, 0xfa, 0x59, 0xa1, 0xbd, 0xe3, 0x6e, 0xbf, 0x73, 0x2c, 0xa2, 0x08, 0x43, - 0x05, 0xaf, 0x55, 0x29, 0xa0, 0x2e, 0xf6, 0xe5, 0x09, 0x96, 0xe0, 0xd7, 0x8d, 0xd3, 0x86, 0xf9, - 0xc3, 0x03, 0x4c, 0x86, 0x39, 0xe1, 0x8d, 0x2a, 0xa5, 0xc6, 0xf2, 0x8f, 0x52, 0xde, 0xac, 0xf2, - 0xcb, 0xac, 0x65, 0x87, 0x45, 0x96, 0x18, 0x22, 0xf6, 0x90, 0x26, 0x2d, 0x3c, 0x57, 0xcb, 0x25, - 0x76, 0x31, 0xd4, 0x22, 0xbf, 0xf7, 0xa9, 0x1a, 0xd9, 0x45, 0xcd, 0x55, 0x0c, 0x58, 0x05, 0xcf, - 0xd7, 0x28, 0xa3, 0x9b, 0xa8, 0xd3, 0x19, 0xab, 0xe0, 0xd3, 0x06, 0x49, 0x25, 0x1b, 0x91, 0xbf, - 0xae, 0xf1, 0x59, 0xc6, 0x6c, 0x4f, 0x1a, 0xe0, 0x37, 0x99, 0x28, 0x7a, 0xc6, 0x9c, 0x60, 0x62, - 0x66, 0x3c, 0xfc, 0x36, 0x57, 0x50, 0x9a, 0x7c, 0xf0, 0xbb, 0x1a, 0x85, 0xec, 0x20, 0xe8, 0xe3, - 0x41, 0xe0, 0xdd, 0x85, 0xaf, 0xd6, 0x29, 0x64, 0xc6, 0xa3, 0x1d, 0xe9, 0xa3, 0xcd, 0xf0, 0xd7, - 0xea, 0x54, 0x30, 0x54, 0x87, 0xb6, 0x60, 0xbe, 0x6e, 0xce, 0xe9, 0x20, 0xdf, 0xea, 0xc2, 0x37, - 0xe8, 0x39, 0xc5, 0xd2, 0xf3, 0xc1, 0xfe, 0x2e, 0x7c, 0xb3, 0x4e, 0xaa, 0x56, 0xc3, 0x50, 0x7a, - 0x42, 0xe7, 0xdd, 0xf0, 0xad, 0x3a, 0xb5, 0x53, 0x49, 0x7b, 0x9a, 0xb5, 0x17, 0xeb, 0x14, 0xfb, - 0x14, 0x37, 0xc5, 0xd6, 0xa5, 0xa1, 0xf8, 0x6d, 0x23, 0x95, 0x3e, 0xfd, 0xc8, 0x92, 0x03, 0x0d, - 0xdf, 0x31, 0x7c, 0x67, 0x5f, 0x08, 0xf0, 0xfb, 0x46, 0x5a, 0x5f, 0x25, 0xec, 0x0f, 0x0d, 0xdb, - 0x1f, 0xa3, 0x4f, 0x02, 0xf8, 0xa3, 0x81, 0xcf, 0x3e, 0x23, 0xe0, 0x4f, 0x0d, 0x32, 0xac, 0xfc, - 0x12, 0xa0, 0xf7, 0xb0, 0x82, 0x3f, 0x37, 0xc8, 0x82, 0x62, 0xe7, 0xc3, 0x77, 0x9b, 0x14, 0xac, - 0x6c, 0xdb, 0xc3, 0xf7, 0x9a, 0xe4, 0xe6, 0x99, 0x3d, 0x0f, 0xdf, 0x6f, 0x9a, 0x74, 0xe4, 0x1b, - 0x1e, 0x7e, 0x50, 0x02, 0x88, 0x0b, 0x7e, 0xd8, 0x34, 0x13, 0x68, 0x64, 0xab, 0xc3, 0x8f, 0x9a, - 0x64, 0xdb, 0xd9, 0x7d, 0x0e, 0x3f, 0x6e, 0xda, 0x74, 0xe7, 0x9b, 0x1c, 0x7e, 0xd2, 0xa4, 0x0e, - 0x78, 0xf0, 0x0e, 0x87, 0x97, 0x8c, 0xae, 0x62, 0x7b, 0xc3, 0xcb, 0xcd, 0xf6, 0x32, 0x9b, 0xec, - 0xaa, 0xd0, 0xec, 0x8d, 0x49, 0x56, 0xed, 0xaa, 0x10, 0xc6, 0x68, 0xcc, 0xae, 0x49, 0x19, 0xae, - 0x9f, 0xc6, 0xc9, 0x33, 0xef, 0x01, 0xa7, 0xbd, 0xc6, 0x66, 0x3b, 0xb2, 0x1f, 0x8b, 0xbc, 0xdd, - 0xcc, 0xaa, 0xb0, 0x3b, 0x06, 0x7d, 0x5b, 0x2a, 0x63, 0x34, 0xab, 0xd7, 0x4f, 0xd1, 0x1b, 0x98, - 0x8d, 0xe6, 0xd0, 0x91, 0x2e, 0x51, 0x90, 0x7d, 0xa8, 0xb4, 0x3f, 0xc2, 0xa0, 0x23, 0x23, 0x15, - 0x28, 0x8d, 0x91, 0x37, 0xbc, 0x89, 0x27, 0x18, 0x9a, 0xbd, 0xa9, 0x13, 0x19, 0xf5, 0x60, 0xcc, - 0x7c, 0x32, 0xa0, 0x79, 0xfa, 0xdb, 0xed, 0xba, 0x46, 0xcf, 0x02, 0xf3, 0x5d, 0x30, 0xc3, 0xd8, - 0xfa, 0x09, 0x46, 0x7a, 0x20, 0xc2, 0x70, 0x08, 0x55, 0x3a, 0x77, 0x06, 0x4a, 0xcb, 0x7e, 0xf0, - 0x49, 0x5a, 0xb2, 0xed, 0xaf, 0x38, 0xac, 0x61, 0x57, 0x69, 0x6e, 0x9a, 0x3d, 0xee, 0x61, 0xe4, - 0x07, 0x46, 0x38, 0x3d, 0x6b, 0x0d, 0x94, 0xee, 0x7f, 0xa7, 0x60, 0xda, 0xd7, 0x22, 0xd1, 0xd9, - 0xf7, 0x87, 0x85, 0xba, 0xf2, 0x5e, 0x14, 0x4a, 0xe1, 0x9b, 0x7d, 0x9e, 0x5f, 0xdd, 0x13, 0x89, - 0x32, 0x4b, 0x9d, 0x5e, 0xfd, 0xa9, 0xfc, 0xc4, 0xf8, 0xe3, 0xc3, 0x78, 0x01, 0x16, 0x3e, 0x4f, - 0xd0, 0xf2, 0xb4, 0xa0, 0x29, 0xf6, 0xac, 0xd2, 0x59, 0xfb, 0x3a, 0x63, 0xc5, 0x17, 0x9f, 0xf1, - 0xa7, 0x58, 0x82, 0x63, 0x14, 0x95, 0xcd, 0x50, 0x1e, 0x8a, 0x10, 0x1c, 0x7a, 0x03, 0x98, 0xa2, - 0xa8, 0xb4, 0x3f, 0x33, 0xce, 0x66, 0xcf, 0x7c, 0xdf, 0x91, 0x6d, 0xf9, 0x61, 0x35, 0xa4, 0xcc, - 0x5d, 0x66, 0x0f, 0xe5, 0xc8, 0x7d, 0x4b, 0xdf, 0xa1, 0x37, 0x61, 0x4e, 0x3e, 0xb3, 0xfd, 0x2b, - 0xfc, 0x0a, 0xbb, 0x58, 0x10, 0xef, 0xdf, 0xf9, 0x34, 0x78, 0x5b, 0x39, 0xc3, 0xd9, 0xe5, 0x5f, - 0xa3, 0x88, 0xe6, 0x54, 0x9a, 0x06, 0xf6, 0x6b, 0xac, 0xf8, 0x18, 0xb5, 0x4b, 0x0d, 0x26, 0xe8, - 0x03, 0xa9, 0xb0, 0x31, 0x2f, 0x2b, 0x98, 0xa4, 0x18, 0xe6, 0x84, 0x74, 0xe1, 0x4c, 0x8d, 0x80, - 0xe9, 0xe2, 0xa9, 0xd3, 0x03, 0x3a, 0x07, 0x69, 0x66, 0x15, 0xe3, 0x82, 0xd1, 0xb3, 0xfd, 0x4c, - 0x08, 0xec, 0x5c, 0x6a, 0x8c, 0x50, 0x0c, 0xd6, 0x45, 0x2d, 0x82, 0x10, 0x9a, 0x94, 0xa8, 0x91, - 0xb8, 0xd8, 0x1b, 0xd3, 0x23, 0xca, 0xd3, 0x1d, 0x36, 0x43, 0xef, 0x99, 0xe2, 0x45, 0x6d, 0xb6, - 0xdf, 0xec, 0x08, 0x66, 0xe6, 0x23, 0xc0, 0x88, 0xba, 0xd2, 0x9a, 0x86, 0xb9, 0x51, 0x47, 0x4d, - 0x81, 0x00, 0x1f, 0x89, 0xae, 0xb5, 0x7b, 0xf7, 0x5e, 0x84, 0x89, 0x3a, 0x0e, 0x62, 0x98, 0x1f, - 0x09, 0x9a, 0x1d, 0x51, 0xa6, 0x2e, 0x16, 0x46, 0x42, 0x41, 0xa6, 0x17, 0x97, 0xce, 0x8d, 0x26, - 0xcc, 0x0c, 0x89, 0x82, 0xba, 0x38, 0x42, 0xdd, 0x16, 0x91, 0xe8, 0x95, 0x14, 0x9e, 0x1f, 0x51, - 0x58, 0x9a, 0x4e, 0xad, 0x0f, 0x48, 0x36, 0x97, 0xff, 0x1b, 0x71, 0x1b, 0x4f, 0xf5, 0x6d, 0x79, - 0x78, 0x87, 0x5f, 0x59, 0xb1, 0xff, 0x22, 0xae, 0x64, 0xff, 0x22, 0xae, 0x6c, 0xa3, 0x52, 0x24, - 0x32, 0x36, 0xf5, 0xd1, 0xfa, 0xeb, 0xa4, 0xf9, 0x9b, 0xe5, 0x91, 0x07, 0xff, 0x79, 0x55, 0xfa, - 0xdb, 0xc4, 0x9d, 0x8d, 0x4b, 0xa7, 0xdd, 0xc3, 0x3b, 0x6b, 0xcf, 0xb2, 0x99, 0x40, 0x66, 0xf7, - 0x7a, 0x49, 0xec, 0xad, 0x35, 0x3a, 0xe6, 0xde, 0x1e, 0xc9, 0xd8, 0x73, 0x3e, 0xfa, 0x64, 0x2f, - 0xd0, 0xc7, 0x83, 0x43, 0x92, 0x76, 0xcd, 0xb2, 0x3d, 0x11, 0xc8, 0xf4, 0xd7, 0xb5, 0x20, 0xd2, - 0x34, 0xb1, 0x43, 0xfb, 0xff, 0xe6, 0x35, 0xab, 0x31, 0x3e, 0xfc, 0x82, 0xe3, 0x1c, 0x4e, 0x18, - 0xe8, 0xc9, 0xff, 0x04, 0x00, 0x00, 0xff, 0xff, 0x79, 0x7a, 0x2a, 0x7d, 0x25, 0x15, 0x00, 0x00, + // 2520 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x58, 0x59, 0x73, 0x24, 0x47, + 0xf1, 0x57, 0xcf, 0x8c, 0x8e, 0xa9, 0x19, 0x49, 0xa9, 0x92, 0x56, 0x3b, 0xde, 0xc3, 0x2b, 0xeb, + 0x6f, 0xff, 0x59, 0x06, 0x5b, 0x0b, 0xeb, 0x08, 0x20, 0x88, 0x30, 0x81, 0x34, 0x23, 0x69, 0x15, + 0x5e, 0x1d, 0x8c, 0xb4, 0x36, 0x41, 0x04, 0x6c, 0xd4, 0x74, 0xa7, 0x46, 0xbd, 0xdb, 0xd3, 0xd5, + 0x74, 0xd5, 0x68, 0x35, 0x3c, 0x19, 0x03, 0x7e, 0x06, 0xf3, 0x05, 0xf8, 0x00, 0xdc, 0xf7, 0x23, + 0x37, 0x36, 0xd7, 0x0b, 0x2f, 0xdc, 0xf0, 0x08, 0xef, 0x1c, 0xc6, 0xeb, 0x83, 0xc8, 0xaa, 0xbe, + 0x46, 0x2b, 0xc3, 0x03, 0x6f, 0x5d, 0xbf, 0xcc, 0xca, 0xcc, 0xca, 0xca, 0xab, 0x9a, 0xd5, 0x5d, + 0xd9, 0xef, 0xcb, 0x70, 0x25, 0x8a, 0xa5, 0x96, 0x7c, 0xbe, 0xef, 0x07, 0xc7, 0x03, 0x65, 0x57, + 0x2b, 0x96, 0x74, 0x61, 0xa9, 0x27, 0x65, 0x2f, 0xc0, 0x6b, 0x06, 0xec, 0x0e, 0x0e, 0xaf, 0x79, + 0xa8, 0xdc, 0xd8, 0x8f, 0xb4, 0x8c, 0x2d, 0xe3, 0xf2, 0x6d, 0x36, 0xb1, 0xaf, 0x85, 0x1e, 0x28, + 0xfe, 0x14, 0x63, 0x18, 0xc7, 0x32, 0xbe, 0xed, 0x4a, 0x0f, 0x1b, 0xce, 0x92, 0x73, 0x75, 0xe6, + 0xfa, 0xc3, 0x2b, 0x67, 0x48, 0x5d, 0x59, 0x27, 0xb6, 0x96, 0xf4, 0xb0, 0x53, 0xc5, 0xf4, 0x93, + 0x2f, 0xb2, 0x89, 0x18, 0x85, 0x92, 0x61, 0xa3, 0xb4, 0xe4, 0x5c, 0xad, 0x76, 0x92, 0xd5, 0xf2, + 0xbb, 0x59, 0xfd, 0x69, 0x1c, 0x3e, 0x23, 0x82, 0x01, 0xee, 0x09, 0x3f, 0xe6, 0xc0, 0xca, 0x77, + 0x71, 0x68, 0xe4, 0x57, 0x3b, 0xf4, 0xc9, 0x17, 0xd8, 0xf8, 0x31, 0x91, 0x93, 0x8d, 0x76, 0xb1, + 0xfc, 0x24, 0xab, 0x3d, 0x8d, 0xc3, 0xb6, 0xd0, 0xe2, 0x2d, 0xb6, 0x71, 0x56, 0xf1, 0x84, 0x16, + 0x66, 0x57, 0xbd, 0x63, 0xbe, 0x97, 0x2f, 0xb1, 0xca, 0x5a, 0x20, 0xbb, 0xb9, 0x48, 0xc7, 0x10, + 0x13, 0x91, 0xc7, 0x0c, 0xf6, 0x02, 0xe1, 0xe2, 0x91, 0x0c, 0x3c, 0x8c, 0x8d, 0x49, 0x24, 0x57, + 0x8b, 0x5e, 0x2a, 0x57, 0x8b, 0x1e, 0x7f, 0x2f, 0xab, 0xe8, 0x61, 0x64, 0xad, 0x99, 0xb9, 0xfe, + 0xe8, 0x99, 0x1e, 0x28, 0x88, 0x39, 0x18, 0x46, 0xd8, 0x31, 0x3b, 0xc8, 0x05, 0x46, 0x91, 0x6a, + 0x94, 0x97, 0xca, 0x57, 0xeb, 0x9d, 0x64, 0xb5, 0xfc, 0x91, 0x11, 0xbd, 0x9b, 0xb1, 0x1c, 0x44, + 0x7c, 0x8b, 0xd5, 0xa3, 0x1c, 0x53, 0x0d, 0x67, 0xa9, 0x7c, 0xb5, 0x76, 0xfd, 0xb1, 0xff, 0xa6, + 0xcd, 0x18, 0xdd, 0x19, 0xd9, 0xba, 0xfc, 0x04, 0x9b, 0x5c, 0xf5, 0xbc, 0x18, 0x95, 0xe2, 0x33, + 0xac, 0xe4, 0x47, 0xc9, 0x61, 0x4a, 0x7e, 0x44, 0x3e, 0x8a, 0x64, 0xac, 0xcd, 0x59, 0xca, 0x1d, + 0xf3, 0xbd, 0xfc, 0xa2, 0xc3, 0x26, 0xb7, 0x55, 0x6f, 0x4d, 0x28, 0xe4, 0xef, 0x61, 0x53, 0x7d, + 0xd5, 0xbb, 0x6d, 0xce, 0x6b, 0x6f, 0xfc, 0xd2, 0x99, 0x16, 0x6c, 0xab, 0x9e, 0x39, 0xe7, 0x64, + 0xdf, 0x7e, 0x90, 0x83, 0xfb, 0xaa, 0xb7, 0xd5, 0x4e, 0x24, 0xdb, 0x05, 0xbf, 0xc4, 0xaa, 0xda, + 0xef, 0xa3, 0xd2, 0xa2, 0x1f, 0x35, 0xca, 0x4b, 0xce, 0xd5, 0x4a, 0x27, 0x07, 0xf8, 0x05, 0x36, + 0xa5, 0xe4, 0x20, 0x76, 0x71, 0xab, 0xdd, 0xa8, 0x98, 0x6d, 0xd9, 0x7a, 0xf9, 0x29, 0x56, 0xdd, + 0x56, 0xbd, 0x1b, 0x28, 0x3c, 0x8c, 0xf9, 0x3b, 0x59, 0xa5, 0x2b, 0x94, 0xb5, 0xa8, 0xf6, 0xd6, + 0x16, 0xd1, 0x09, 0x3a, 0x86, 0x73, 0xf9, 0xa3, 0xac, 0xde, 0xde, 0xbe, 0xf9, 0x3f, 0x48, 0x20, + 0xd3, 0xd5, 0x91, 0x88, 0xbd, 0x1d, 0xd1, 0x4f, 0x03, 0x31, 0x07, 0x96, 0xef, 0x3b, 0xac, 0xbe, + 0x17, 0xfb, 0xc7, 0x7e, 0x80, 0x3d, 0x5c, 0x3f, 0xd1, 0xfc, 0x03, 0xac, 0x26, 0xbb, 0x77, 0xd0, + 0xd5, 0x45, 0xdf, 0x5d, 0x39, 0x53, 0xcf, 0xae, 0xe1, 0x33, 0xee, 0x63, 0x32, 0xfb, 0xe6, 0xbb, + 0x0c, 0x12, 0x09, 0x51, 0x2a, 0xf8, 0x3f, 0x86, 0x9c, 0x15, 0x93, 0x19, 0xd1, 0x99, 0x95, 0xa3, + 0x00, 0x6f, 0xb2, 0xb9, 0x44, 0x60, 0x28, 0xfa, 0x78, 0xdb, 0x0f, 0x3d, 0x3c, 0x31, 0x97, 0x30, + 0x9e, 0xf2, 0xd2, 0x51, 0xb6, 0x08, 0xe6, 0x8f, 0x33, 0xfe, 0x00, 0xaf, 0x32, 0x97, 0x32, 0xde, + 0x81, 0x53, 0xcc, 0xaa, 0xf9, 0xab, 0x29, 0x56, 0xcd, 0x72, 0x9e, 0xd7, 0xd8, 0xe4, 0xfe, 0xc0, + 0x75, 0x51, 0x29, 0x18, 0xe3, 0xf3, 0x6c, 0xf6, 0x56, 0x88, 0x27, 0x11, 0xba, 0x1a, 0x3d, 0xc3, + 0x03, 0x0e, 0x9f, 0x63, 0xd3, 0x2d, 0x19, 0x86, 0xe8, 0xea, 0x0d, 0xe1, 0x07, 0xe8, 0x41, 0x89, + 0x2f, 0x30, 0xd8, 0xc3, 0xb8, 0xef, 0x2b, 0xe5, 0xcb, 0xb0, 0x8d, 0xa1, 0x8f, 0x1e, 0x94, 0xf9, + 0x79, 0x36, 0xdf, 0x92, 0x41, 0x80, 0xae, 0xf6, 0x65, 0xb8, 0x23, 0xf5, 0xfa, 0x89, 0xaf, 0xb4, + 0x82, 0x0a, 0x89, 0xdd, 0x0a, 0x02, 0xec, 0x89, 0x60, 0x35, 0xee, 0x0d, 0xfa, 0x18, 0x6a, 0x18, + 0x27, 0x19, 0x09, 0xd8, 0xf6, 0xfb, 0x18, 0x92, 0x24, 0x98, 0x2c, 0xa0, 0xc6, 0x5a, 0xf2, 0x2d, + 0x4c, 0xf1, 0x87, 0xd8, 0xb9, 0x04, 0x2d, 0x28, 0x10, 0x7d, 0x84, 0x2a, 0x9f, 0x65, 0xb5, 0x84, + 0x74, 0xb0, 0xbb, 0xf7, 0x34, 0xb0, 0x82, 0x84, 0x8e, 0xbc, 0xd7, 0x41, 0x57, 0xc6, 0x1e, 0xd4, + 0x0a, 0x26, 0x3c, 0x83, 0xae, 0x96, 0xf1, 0x56, 0x1b, 0xea, 0x64, 0x70, 0x02, 0xee, 0xa3, 0x88, + 0xdd, 0xa3, 0x0e, 0xaa, 0x41, 0xa0, 0x61, 0x9a, 0x03, 0xab, 0x6f, 0xf8, 0x01, 0xee, 0x48, 0xbd, + 0x21, 0x07, 0xa1, 0x07, 0x33, 0x7c, 0x86, 0xb1, 0x6d, 0xd4, 0x22, 0xf1, 0xc0, 0x2c, 0xa9, 0x6d, + 0x09, 0xf7, 0x08, 0x13, 0x00, 0xf8, 0x22, 0xe3, 0x2d, 0x11, 0x86, 0x52, 0xb7, 0x62, 0x14, 0x1a, + 0x37, 0x4c, 0x36, 0xc3, 0x1c, 0x99, 0x33, 0x82, 0xfb, 0x01, 0x02, 0xcf, 0xb9, 0xdb, 0x18, 0x60, + 0xc6, 0x3d, 0x9f, 0x73, 0x27, 0x38, 0x71, 0x2f, 0x90, 0xf1, 0x6b, 0x03, 0x3f, 0xf0, 0x8c, 0x4b, + 0xec, 0xb5, 0x9c, 0x23, 0x1b, 0x13, 0xe3, 0x77, 0x6e, 0x6e, 0xed, 0x1f, 0xc0, 0x22, 0x3f, 0xc7, + 0xe6, 0x12, 0x64, 0x1b, 0x75, 0xec, 0xbb, 0xc6, 0x79, 0xe7, 0xc9, 0xd4, 0xdd, 0x81, 0xde, 0x3d, + 0xdc, 0xc6, 0xbe, 0x8c, 0x87, 0xd0, 0xa0, 0x0b, 0x35, 0x92, 0xd2, 0x2b, 0x82, 0x87, 0x48, 0xc3, + 0x7a, 0x3f, 0xd2, 0xc3, 0xdc, 0xbd, 0x70, 0x81, 0x5f, 0x64, 0xe7, 0x6f, 0x45, 0x9e, 0xd0, 0xb8, + 0xd5, 0xa7, 0x52, 0x73, 0x20, 0xd4, 0x5d, 0x3a, 0xee, 0x20, 0x46, 0xb8, 0xc8, 0x2f, 0xb0, 0xc5, + 0xd1, 0xbb, 0xc8, 0x9c, 0x75, 0x89, 0x36, 0xda, 0xd3, 0xb6, 0x62, 0xf4, 0x30, 0xd4, 0xbe, 0x08, + 0xd2, 0x8d, 0x97, 0x73, 0xa9, 0x0f, 0x12, 0x1f, 0x26, 0xa2, 0x3d, 0xf9, 0x83, 0xc4, 0x2b, 0xbc, + 0xc1, 0x16, 0x36, 0x51, 0x3f, 0x48, 0x59, 0x22, 0xca, 0x4d, 0x5f, 0x19, 0xd2, 0x2d, 0x85, 0xb1, + 0x4a, 0x29, 0x8f, 0x70, 0xce, 0x66, 0x36, 0x51, 0x13, 0x98, 0x62, 0xcb, 0xe4, 0x27, 0x6b, 0x5e, + 0x47, 0x06, 0x98, 0xc2, 0xff, 0x47, 0x3e, 0x68, 0xc7, 0x32, 0x2a, 0x82, 0x8f, 0xd2, 0x31, 0x77, + 0x23, 0x8c, 0x85, 0x46, 0x92, 0x51, 0xa4, 0x3d, 0x46, 0x72, 0xf6, 0x91, 0x3c, 0x50, 0x84, 0xff, + 0x3f, 0x87, 0x8b, 0x5a, 0xdf, 0x46, 0x31, 0x9c, 0x70, 0xa3, 0xad, 0x93, 0x29, 0xe9, 0x2a, 0x9d, + 0x3a, 0x51, 0x92, 0xe5, 0x7f, 0x4a, 0x7c, 0x3b, 0x85, 0x8a, 0xdd, 0xb7, 0x19, 0x8b, 0x50, 0xa7, + 0x78, 0x93, 0x3f, 0xc2, 0x2e, 0x77, 0xf0, 0x30, 0x46, 0x75, 0xb4, 0x27, 0x03, 0xdf, 0x1d, 0x6e, + 0x85, 0x87, 0x32, 0x0b, 0x49, 0x62, 0x79, 0x07, 0x59, 0x42, 0x6e, 0xb1, 0xf4, 0x14, 0x7e, 0x9c, + 0x7c, 0xb2, 0x23, 0xf5, 0x3e, 0x95, 0xc3, 0x9b, 0xa6, 0xc0, 0xc2, 0x13, 0xa4, 0x65, 0x47, 0x76, + 0x30, 0x0a, 0x7c, 0x57, 0xac, 0x1e, 0x0b, 0x3f, 0x10, 0xdd, 0x00, 0x61, 0x85, 0x9c, 0xb2, 0x8f, + 0x3d, 0x4a, 0xd9, 0xec, 0x7e, 0xaf, 0x71, 0xce, 0xa6, 0xdb, 0xed, 0x0e, 0x7e, 0x6c, 0x80, 0x4a, + 0x77, 0x84, 0x8b, 0xf0, 0x97, 0xc9, 0xa6, 0xcb, 0x98, 0x09, 0x2a, 0x1a, 0x3f, 0x90, 0x54, 0xe4, + 0xab, 0x1d, 0x19, 0x22, 0x8c, 0xf1, 0x3a, 0x9b, 0xba, 0x15, 0xfa, 0x4a, 0x0d, 0xd0, 0x03, 0x87, + 0x12, 0x6a, 0x2b, 0xdc, 0x8b, 0x65, 0x8f, 0x3a, 0x1d, 0x94, 0x88, 0xba, 0xe1, 0x87, 0xbe, 0x3a, + 0x32, 0xa5, 0x84, 0xb1, 0x89, 0x24, 0xb3, 0x2a, 0xbc, 0xca, 0xc6, 0x3b, 0xa8, 0xe3, 0x21, 0x8c, + 0x37, 0x9f, 0x77, 0x58, 0x3d, 0x31, 0xc7, 0xea, 0x59, 0x60, 0x50, 0x5c, 0xe7, 0x9a, 0xb2, 0xd8, + 0x76, 0xa8, 0xc2, 0x6d, 0xc6, 0xf2, 0x9e, 0x1f, 0xf6, 0xa0, 0x44, 0x82, 0xf7, 0x51, 0x04, 0x46, + 0x49, 0x8d, 0x4d, 0x6e, 0x04, 0x03, 0xa3, 0xb1, 0x62, 0xf4, 0xd3, 0x82, 0xd8, 0xc6, 0x89, 0x44, + 0xb1, 0x10, 0xa1, 0x07, 0x13, 0x7c, 0x9a, 0x55, 0x6d, 0x06, 0x10, 0x6d, 0xb2, 0xf9, 0x7e, 0x36, + 0x7b, 0x6a, 0x60, 0xe0, 0x53, 0xac, 0x92, 0xa8, 0x06, 0x56, 0x5f, 0xf3, 0x43, 0x11, 0x0f, 0x6d, + 0x99, 0x01, 0x8f, 0xd2, 0x6f, 0x23, 0x90, 0x42, 0x27, 0x00, 0x36, 0x5f, 0xa9, 0x9b, 0x8e, 0x6d, + 0x36, 0x4e, 0xb3, 0xea, 0xad, 0xd0, 0xc3, 0x43, 0x3f, 0x44, 0x0f, 0xc6, 0x4c, 0xfa, 0xdb, 0xc4, + 0xc9, 0xf3, 0xd0, 0x23, 0x67, 0x92, 0x31, 0x05, 0x0c, 0x29, 0x87, 0x6f, 0x08, 0x55, 0x80, 0x0e, + 0xe9, 0x0a, 0xdb, 0x66, 0x1e, 0xec, 0x16, 0xb7, 0xf7, 0xcc, 0x15, 0x1e, 0xc9, 0x7b, 0x39, 0xa6, + 0xe0, 0x88, 0x34, 0x6d, 0xa2, 0xde, 0x1f, 0x2a, 0x8d, 0xfd, 0x96, 0x0c, 0x0f, 0xfd, 0x9e, 0x02, + 0x9f, 0x34, 0xdd, 0x94, 0xc2, 0x2b, 0x6c, 0xbf, 0x43, 0x41, 0xd4, 0xc1, 0x00, 0x85, 0x2a, 0x4a, + 0xbd, 0x6b, 0x0a, 0xa0, 0x31, 0x75, 0x35, 0xf0, 0x85, 0x82, 0x80, 0x8e, 0x42, 0x56, 0xda, 0x65, + 0x9f, 0xee, 0x77, 0x35, 0xd0, 0x18, 0xdb, 0x75, 0xc8, 0x17, 0xd8, 0xac, 0xe5, 0xdf, 0x13, 0xb1, + 0xf6, 0x8d, 0x90, 0x97, 0x1c, 0x13, 0x49, 0xb1, 0x8c, 0x72, 0xec, 0x65, 0xea, 0x37, 0xf5, 0x1b, + 0x42, 0xe5, 0xd0, 0x4f, 0x1d, 0xbe, 0xc8, 0xe6, 0xd2, 0xa3, 0xe5, 0xf8, 0xcf, 0x1c, 0x3e, 0xcf, + 0x66, 0xe8, 0x68, 0x19, 0xa6, 0xe0, 0xe7, 0x06, 0xa4, 0x43, 0x14, 0xc0, 0x5f, 0x18, 0x09, 0xc9, + 0x29, 0x0a, 0xf8, 0x2f, 0x8d, 0x32, 0x92, 0x90, 0x04, 0x91, 0x82, 0x57, 0x1d, 0xb2, 0x34, 0x55, + 0x96, 0xc0, 0x70, 0xdf, 0x30, 0x92, 0xd4, 0x8c, 0xf1, 0x35, 0xc3, 0x98, 0xc8, 0xcc, 0xd0, 0xd7, + 0x0d, 0x7a, 0x43, 0x84, 0x9e, 0x3c, 0x3c, 0xcc, 0xd0, 0x37, 0x1c, 0xde, 0x60, 0xf3, 0xb4, 0x7d, + 0x4d, 0x04, 0x22, 0x74, 0x73, 0xfe, 0x37, 0x1d, 0x7e, 0x8e, 0xc1, 0x29, 0x75, 0x0a, 0x9e, 0x2b, + 0x71, 0x48, 0xfd, 0x6b, 0xf2, 0x08, 0xbe, 0x50, 0x32, 0xbe, 0x4a, 0x18, 0x2d, 0xf6, 0xc5, 0x12, + 0x9f, 0xb1, 0x4e, 0xb7, 0xeb, 0x2f, 0x95, 0x78, 0x8d, 0x4d, 0x6c, 0x85, 0x0a, 0x63, 0x0d, 0x9f, + 0xa1, 0xf8, 0x9e, 0xb0, 0xc5, 0x14, 0x3e, 0x4b, 0x19, 0x35, 0x6e, 0xe2, 0x1b, 0x5e, 0xa4, 0x46, + 0xcd, 0x3b, 0xa8, 0x30, 0xf4, 0x0a, 0xb9, 0xa3, 0xe0, 0x73, 0x66, 0x87, 0xed, 0x84, 0xf0, 0xb7, + 0xb2, 0x71, 0x4d, 0xb1, 0x2d, 0xfe, 0xbd, 0x4c, 0x26, 0x6c, 0xa2, 0xce, 0x33, 0x1b, 0xfe, 0x51, + 0xe6, 0x17, 0xd8, 0xb9, 0x14, 0x33, 0x4d, 0x2a, 0xcb, 0xe9, 0x7f, 0x96, 0xf9, 0x25, 0x76, 0x9e, + 0x2a, 0x76, 0x16, 0x37, 0xb4, 0xc9, 0x57, 0xda, 0x77, 0x15, 0xbc, 0x52, 0xe6, 0x17, 0xd9, 0xe2, + 0x26, 0xea, 0xec, 0x3e, 0x0a, 0xc4, 0x7f, 0x95, 0xf9, 0x34, 0x9b, 0xa2, 0xac, 0xf7, 0xf1, 0x18, + 0xe1, 0xd5, 0x32, 0x5d, 0x6a, 0xba, 0x4c, 0xcc, 0xb9, 0x5f, 0x26, 0x57, 0x3f, 0x2b, 0xb4, 0x7b, + 0xd4, 0xee, 0xb7, 0x8e, 0x44, 0x18, 0x62, 0xa0, 0xe0, 0xb5, 0x32, 0x39, 0xb4, 0x83, 0x7d, 0x79, + 0x8c, 0x05, 0xf8, 0x75, 0x73, 0x68, 0xc3, 0xfc, 0xc1, 0x01, 0xc6, 0xc3, 0x8c, 0xf0, 0x46, 0x99, + 0xae, 0xc6, 0xf2, 0x8f, 0x52, 0xde, 0x2c, 0xf3, 0xcb, 0xac, 0x61, 0x8b, 0x45, 0x7a, 0x31, 0x44, + 0xec, 0x21, 0x55, 0x5a, 0x78, 0xae, 0x92, 0x49, 0x6c, 0x63, 0xa0, 0x45, 0xb6, 0xef, 0x13, 0x15, + 0xb2, 0x8b, 0x92, 0x2b, 0x2f, 0xb0, 0x0a, 0x9e, 0xaf, 0xd0, 0x8d, 0x6e, 0xa2, 0x4e, 0x6a, 0xac, + 0x82, 0x4f, 0xd2, 0x5c, 0x34, 0x73, 0x2b, 0x54, 0x83, 0x6e, 0x66, 0x28, 0x7c, 0x2a, 0xdd, 0xdc, + 0xf6, 0x95, 0x8e, 0xfd, 0xee, 0xc0, 0x44, 0xfa, 0xa7, 0x2b, 0x74, 0xa8, 0xfd, 0x61, 0xe8, 0x8e, + 0xc0, 0x2f, 0x18, 0x99, 0x89, 0x6d, 0xc6, 0xa8, 0x5f, 0x57, 0xf8, 0x2c, 0x63, 0x36, 0xab, 0x0d, + 0xf0, 0x9b, 0x54, 0x1e, 0x0d, 0x42, 0xc7, 0x18, 0x9b, 0x2e, 0x01, 0xbf, 0xcd, 0x4c, 0x2c, 0xd4, + 0x4e, 0xf8, 0x5d, 0x85, 0x9c, 0x7e, 0xe0, 0xf7, 0xf1, 0xc0, 0x77, 0xef, 0xc2, 0x57, 0xaa, 0x64, + 0x9f, 0xf1, 0xc9, 0x8e, 0xf4, 0xd0, 0xc6, 0xc8, 0x57, 0xab, 0x14, 0x72, 0x14, 0xc9, 0x36, 0xe4, + 0xbe, 0x66, 0xd6, 0x49, 0x2b, 0xd8, 0x6a, 0xc3, 0xd7, 0x69, 0x20, 0x63, 0xc9, 0xfa, 0x60, 0x7f, + 0x17, 0xbe, 0x51, 0x25, 0x55, 0xab, 0x41, 0x20, 0x5d, 0xa1, 0xb3, 0x7c, 0xfa, 0x66, 0x95, 0x12, + 0xb2, 0xa0, 0x3d, 0xb9, 0xf7, 0x6f, 0x55, 0xcd, 0x41, 0x2d, 0x6e, 0xc2, 0xb5, 0x4d, 0x65, 0xf5, + 0xdb, 0x46, 0x2a, 0x3d, 0x1e, 0xc9, 0x92, 0x03, 0x0d, 0xdf, 0x31, 0x7c, 0xa7, 0x67, 0x0c, 0xf8, + 0x7d, 0x2d, 0x89, 0xd0, 0x02, 0xf6, 0x87, 0x9a, 0xcd, 0xb0, 0xd1, 0xa1, 0x02, 0xfe, 0x68, 0xe0, + 0xd3, 0x83, 0x08, 0xfc, 0xa9, 0x46, 0x86, 0x15, 0x67, 0x09, 0x9a, 0xa8, 0x15, 0xfc, 0xb9, 0x46, + 0x16, 0xe4, 0x53, 0x03, 0x7c, 0xb7, 0x4e, 0xce, 0x4a, 0xe7, 0x05, 0xf8, 0x5e, 0x9d, 0x8e, 0x79, + 0x6a, 0x52, 0x80, 0xef, 0xd7, 0xcd, 0x75, 0x64, 0x33, 0x02, 0xfc, 0xa0, 0x00, 0x10, 0x17, 0xfc, + 0xb0, 0x6e, 0x6a, 0xd8, 0xc8, 0x5c, 0x00, 0x3f, 0xaa, 0x93, 0x6d, 0xa7, 0x27, 0x02, 0xf8, 0x71, + 0xdd, 0x5e, 0x77, 0x36, 0x0b, 0xc0, 0x4f, 0xea, 0x94, 0x43, 0x67, 0x4f, 0x01, 0xf0, 0x92, 0xd1, + 0x95, 0xf7, 0x7f, 0x78, 0xb9, 0xde, 0x5c, 0x66, 0x93, 0x6d, 0x15, 0x98, 0xce, 0x33, 0xc9, 0xca, + 0x6d, 0x15, 0xc0, 0x18, 0x15, 0xea, 0x35, 0x29, 0x83, 0xf5, 0x93, 0x28, 0x7e, 0xe6, 0x5d, 0xe0, + 0x34, 0xd7, 0xd8, 0x6c, 0x4b, 0xf6, 0x23, 0x91, 0x25, 0xac, 0x69, 0x36, 0xb6, 0x4b, 0xa1, 0x67, + 0x43, 0x65, 0x8c, 0xaa, 0xfd, 0xfa, 0x09, 0xba, 0x03, 0xd3, 0x13, 0x1d, 0x5a, 0xd2, 0x26, 0x72, + 0xb2, 0x07, 0xa5, 0xe6, 0x87, 0x18, 0xb4, 0x64, 0xa8, 0x7c, 0xa5, 0x31, 0x74, 0x87, 0x37, 0xf1, + 0x18, 0x03, 0xd3, 0x79, 0x75, 0x2c, 0xc3, 0x1e, 0x8c, 0x99, 0x47, 0x07, 0x9a, 0xc7, 0x83, 0xed, + 0xcf, 0x6b, 0x34, 0x58, 0x98, 0x97, 0xc5, 0x0c, 0x63, 0xeb, 0xc7, 0x18, 0xea, 0x81, 0x08, 0x82, + 0x21, 0x94, 0x69, 0xdd, 0x1a, 0x28, 0x2d, 0xfb, 0xfe, 0xc7, 0xa9, 0x4d, 0x37, 0xbf, 0xec, 0xb0, + 0x9a, 0x6d, 0xc6, 0x99, 0x69, 0x76, 0xb9, 0x87, 0xa1, 0xe7, 0x1b, 0xe1, 0x34, 0x18, 0x1b, 0x28, + 0x99, 0x20, 0x9c, 0x9c, 0x69, 0x5f, 0x8b, 0x58, 0xa7, 0x2f, 0x18, 0x0b, 0xb5, 0xe5, 0xbd, 0x30, + 0x90, 0xc2, 0x33, 0x13, 0x41, 0xb6, 0x75, 0x4f, 0xc4, 0xca, 0x8c, 0x05, 0xf4, 0x6e, 0x48, 0xe4, + 0xc7, 0xe6, 0x3c, 0x1e, 0x8c, 0xe7, 0x60, 0x7e, 0xe6, 0x09, 0x6a, 0xbf, 0x16, 0x34, 0xc1, 0x9e, + 0x46, 0x3a, 0x6b, 0x5e, 0x67, 0x2c, 0x7f, 0x33, 0x9a, 0xf3, 0xe4, 0x6d, 0x74, 0x8c, 0xbc, 0xb2, + 0x19, 0xc8, 0xae, 0x08, 0xc0, 0xa1, 0x29, 0xc2, 0x04, 0x45, 0xa9, 0xf9, 0xc2, 0x38, 0x9b, 0x3d, + 0xf5, 0x42, 0x24, 0xdb, 0xb2, 0xc5, 0x6a, 0x40, 0x37, 0x77, 0x99, 0x3d, 0x94, 0x21, 0x0f, 0x8c, + 0x0d, 0x0e, 0x4d, 0x95, 0x19, 0xf9, 0xd4, 0xfc, 0x50, 0xe2, 0x57, 0xd8, 0xc5, 0x9c, 0xf8, 0xe0, + 0xd4, 0x40, 0xa5, 0xbb, 0x91, 0x31, 0x9c, 0x1e, 0x1f, 0x2a, 0xe4, 0xd1, 0x8c, 0x4a, 0xd5, 0xc0, + 0xbe, 0xe7, 0xf2, 0xe7, 0xac, 0x6d, 0x8b, 0x30, 0x41, 0x4f, 0xac, 0xdc, 0xc6, 0x2c, 0xac, 0x60, + 0x92, 0x7c, 0x98, 0x11, 0x92, 0x96, 0x35, 0x35, 0x02, 0x26, 0xad, 0xab, 0x4a, 0x23, 0x78, 0x06, + 0x52, 0xcd, 0xca, 0xcb, 0x05, 0xa3, 0xc1, 0xff, 0x94, 0x0b, 0x6c, 0x5d, 0xaa, 0x8d, 0x50, 0x0c, + 0xd6, 0x46, 0x2d, 0xfc, 0x00, 0xea, 0x74, 0x51, 0x23, 0x7e, 0xb1, 0x3b, 0xa6, 0x47, 0x94, 0x27, + 0x5d, 0x70, 0x86, 0x26, 0xa2, 0x7c, 0x26, 0x37, 0xfd, 0x73, 0x76, 0x04, 0x33, 0xf5, 0x11, 0x60, + 0x44, 0x5d, 0xa1, 0xd1, 0xc3, 0xdc, 0xe8, 0x41, 0x4d, 0x80, 0x00, 0x1f, 0xf1, 0xae, 0xb5, 0x7b, + 0xf7, 0x5e, 0x88, 0xb1, 0x3a, 0xf2, 0x23, 0x98, 0x1f, 0x71, 0x9a, 0x2d, 0x51, 0x26, 0x2e, 0x16, + 0x46, 0x5c, 0x41, 0xa6, 0xe7, 0x9b, 0xce, 0x8d, 0x5e, 0x98, 0x29, 0x12, 0x39, 0x75, 0x71, 0x84, + 0xba, 0x2d, 0x42, 0xd1, 0x2b, 0x28, 0x3c, 0x3f, 0xa2, 0xb0, 0x50, 0x9d, 0x1a, 0xef, 0x93, 0x6c, + 0x2e, 0xfb, 0x9f, 0x71, 0x1b, 0x4f, 0xf4, 0x6d, 0xd9, 0xbd, 0xc3, 0xaf, 0xac, 0xd8, 0xff, 0x90, + 0x2b, 0xe9, 0x7f, 0xc8, 0x95, 0x6d, 0x54, 0x8a, 0x44, 0x46, 0x26, 0x3e, 0x1a, 0x7f, 0x9d, 0x34, + 0x3f, 0x6a, 0x1e, 0x39, 0xfb, 0xf7, 0x57, 0xe1, 0xc7, 0x4b, 0x67, 0x36, 0x2a, 0xac, 0x76, 0xbb, + 0x77, 0xd6, 0x9e, 0x65, 0x33, 0xbe, 0x4c, 0xf7, 0xf5, 0xe2, 0xc8, 0x5d, 0xab, 0xb5, 0xcc, 0xbe, + 0x3d, 0x92, 0xb1, 0xe7, 0x7c, 0xf8, 0xc9, 0x9e, 0xaf, 0x8f, 0x06, 0x5d, 0x92, 0x76, 0xcd, 0xb2, + 0x3d, 0xe1, 0xcb, 0xe4, 0xeb, 0x9a, 0x1f, 0x6a, 0xaa, 0xd8, 0x81, 0xfd, 0x43, 0x7a, 0xcd, 0x6a, + 0x8c, 0xba, 0x9f, 0x77, 0x9c, 0xee, 0x84, 0x81, 0x9e, 0xfc, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, + 0x54, 0x7c, 0xcd, 0x1c, 0x67, 0x15, 0x00, 0x00, } diff --git a/internal/proto/query_coord.proto b/internal/proto/query_coord.proto index e864b42ac118f..b884782601f62 100644 --- a/internal/proto/query_coord.proto +++ b/internal/proto/query_coord.proto @@ -42,6 +42,7 @@ service QueryNode { rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest) returns(milvus.StringResponse){} rpc WatchDmChannels(WatchDmChannelsRequest) returns (common.Status) {} + rpc UnsubDmChannel(UnsubDmChannelRequest) returns (common.Status) {} rpc LoadSegments(LoadSegmentsRequest) returns (common.Status) {} rpc ReleaseCollection(ReleaseCollectionRequest) returns (common.Status) {} rpc ReleasePartitions(ReleasePartitionsRequest) returns (common.Status) {} @@ -56,6 +57,9 @@ service QueryNode { rpc ShowConfigurations(internal.ShowConfigurationsRequest) returns (internal.ShowConfigurationsResponse){} // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy rpc GetMetrics(milvus.GetMetricsRequest) returns (milvus.GetMetricsResponse) {} + + rpc GetDataDistribution(GetDataDistributionRequest) returns (GetDataDistributionResponse) {} + rpc SyncDistribution(SyncDistributionRequest) returns (common.Status) {} } //--------------------QueryCoord grpc request and response proto------------------ @@ -187,6 +191,13 @@ message WatchDmChannelsRequest { int64 offlineNodeID = 11; } +message UnsubDmChannelRequest { + common.MsgBase base = 1; + int64 nodeID = 2; + int64 collectionID = 3; + string channel_name = 4; +} + message SegmentLoadInfo { int64 segmentID = 1; int64 partitionID = 2; @@ -225,6 +236,8 @@ message LoadSegmentsRequest { LoadMetaInfo load_meta = 7; int64 replicaID = 8; repeated internal.MsgPosition delta_positions = 9; + int64 version = 10; + bool need_transfer = 11; } message ReleaseSegmentsRequest { @@ -236,6 +249,8 @@ message ReleaseSegmentsRequest { repeated int64 partitionIDs = 5; repeated int64 segmentIDs = 6; DataScope scope = 7; // All, Streaming, Historical + string shard = 8; + bool need_transfer = 11; } message SearchRequest { @@ -391,3 +406,83 @@ message SealedSegmentsChangeInfo { common.MsgBase base = 1; repeated SegmentChangeInfo infos = 2; } + +message GetDataDistributionRequest { + common.MsgBase base = 1; +} + +message GetDataDistributionResponse { + common.Status status = 1; + int64 nodeID = 2; + repeated int64 growing_segmentIDs = 3; + repeated SegmentVersionInfo segments = 4; + repeated ChannelVersionInfo channels = 5; + repeated LeaderView leaderViews = 6; +} + +message LeaderView { + int64 collection = 1; + string channel = 2; + map segmentNodePairs = 3; +} + + +message SegmentVersionInfo { + int64 ID = 1; + int64 collection = 2; + int64 partition = 3; + string channel = 4; + int64 version = 5; +} + +message ChannelVersionInfo { + string channel = 1; + int64 collection = 2; + int64 version = 3; +} + +enum LoadStatus { + Invalid = 0; + Loading = 1; + Loaded = 2; +} + +message CollectionLoadInfo { + int64 collectionID = 1; + repeated int64 released_partitions = 2; + int32 replica_number = 3; + LoadStatus status = 4; +} + +message PartitionLoadInfo { + int64 collectionID = 1; + int64 partitionID = 2; + int32 replica_number = 3; + LoadStatus status = 4; +} + +message Replica { + int64 ID = 1; + int64 collectionID = 2; + repeated int64 nodes = 3; +} + +enum SyncType { + Remove = 0; + Set = 1; +} + +message SyncAction { + SyncType type = 1; + int64 partitionID = 2; + int64 segmentID = 3; + int64 nodeID = 4; +} + +message SyncDistributionRequest { + common.MsgBase base = 1; + int64 collectionID = 2; + string channel = 3; + repeated SyncAction actions = 4; +} + diff --git a/internal/proto/query_coordv2.proto b/internal/proto/query_coordv2.proto deleted file mode 100644 index 0f11a0d0e8db2..0000000000000 --- a/internal/proto/query_coordv2.proto +++ /dev/null @@ -1,7 +0,0 @@ -syntax = "proto3"; - -package milvus.proto.queryv2; - -option go_package = "github.com/milvus-io/milvus/internal/proto/querypbv2"; - -service QueryCoordV2{} diff --git a/internal/proto/querypb/query_coord.pb.go b/internal/proto/querypb/query_coord.pb.go index 7ad7d94f4d63f..260cc5fb41ca2 100644 --- a/internal/proto/querypb/query_coord.pb.go +++ b/internal/proto/querypb/query_coord.pb.go @@ -162,6 +162,59 @@ func (LoadType) EnumDescriptor() ([]byte, []int) { return fileDescriptor_aab7cc9a69ed26e8, []int{3} } +type LoadStatus int32 + +const ( + LoadStatus_Invalid LoadStatus = 0 + LoadStatus_Loading LoadStatus = 1 + LoadStatus_Loaded LoadStatus = 2 +) + +var LoadStatus_name = map[int32]string{ + 0: "Invalid", + 1: "Loading", + 2: "Loaded", +} + +var LoadStatus_value = map[string]int32{ + "Invalid": 0, + "Loading": 1, + "Loaded": 2, +} + +func (x LoadStatus) String() string { + return proto.EnumName(LoadStatus_name, int32(x)) +} + +func (LoadStatus) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_aab7cc9a69ed26e8, []int{4} +} + +type SyncType int32 + +const ( + SyncType_Remove SyncType = 0 + SyncType_Set SyncType = 1 +) + +var SyncType_name = map[int32]string{ + 0: "Remove", + 1: "Set", +} + +var SyncType_value = map[string]int32{ + "Remove": 0, + "Set": 1, +} + +func (x SyncType) String() string { + return proto.EnumName(SyncType_name, int32(x)) +} + +func (SyncType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_aab7cc9a69ed26e8, []int{5} +} + //--------------------QueryCoord grpc request and response proto------------------ type ShowCollectionsRequest struct { Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` @@ -1292,6 +1345,69 @@ func (m *WatchDmChannelsRequest) GetOfflineNodeID() int64 { return 0 } +type UnsubDmChannelRequest struct { + Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` + NodeID int64 `protobuf:"varint,2,opt,name=nodeID,proto3" json:"nodeID,omitempty"` + CollectionID int64 `protobuf:"varint,3,opt,name=collectionID,proto3" json:"collectionID,omitempty"` + ChannelName string `protobuf:"bytes,4,opt,name=channel_name,json=channelName,proto3" json:"channel_name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *UnsubDmChannelRequest) Reset() { *m = UnsubDmChannelRequest{} } +func (m *UnsubDmChannelRequest) String() string { return proto.CompactTextString(m) } +func (*UnsubDmChannelRequest) ProtoMessage() {} +func (*UnsubDmChannelRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_aab7cc9a69ed26e8, []int{18} +} + +func (m *UnsubDmChannelRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_UnsubDmChannelRequest.Unmarshal(m, b) +} +func (m *UnsubDmChannelRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_UnsubDmChannelRequest.Marshal(b, m, deterministic) +} +func (m *UnsubDmChannelRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UnsubDmChannelRequest.Merge(m, src) +} +func (m *UnsubDmChannelRequest) XXX_Size() int { + return xxx_messageInfo_UnsubDmChannelRequest.Size(m) +} +func (m *UnsubDmChannelRequest) XXX_DiscardUnknown() { + xxx_messageInfo_UnsubDmChannelRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_UnsubDmChannelRequest proto.InternalMessageInfo + +func (m *UnsubDmChannelRequest) GetBase() *commonpb.MsgBase { + if m != nil { + return m.Base + } + return nil +} + +func (m *UnsubDmChannelRequest) GetNodeID() int64 { + if m != nil { + return m.NodeID + } + return 0 +} + +func (m *UnsubDmChannelRequest) GetCollectionID() int64 { + if m != nil { + return m.CollectionID + } + return 0 +} + +func (m *UnsubDmChannelRequest) GetChannelName() string { + if m != nil { + return m.ChannelName + } + return "" +} + type SegmentLoadInfo struct { SegmentID int64 `protobuf:"varint,1,opt,name=segmentID,proto3" json:"segmentID,omitempty"` PartitionID int64 `protobuf:"varint,2,opt,name=partitionID,proto3" json:"partitionID,omitempty"` @@ -1315,7 +1431,7 @@ func (m *SegmentLoadInfo) Reset() { *m = SegmentLoadInfo{} } func (m *SegmentLoadInfo) String() string { return proto.CompactTextString(m) } func (*SegmentLoadInfo) ProtoMessage() {} func (*SegmentLoadInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{18} + return fileDescriptor_aab7cc9a69ed26e8, []int{19} } func (m *SegmentLoadInfo) XXX_Unmarshal(b []byte) error { @@ -1446,7 +1562,7 @@ func (m *FieldIndexInfo) Reset() { *m = FieldIndexInfo{} } func (m *FieldIndexInfo) String() string { return proto.CompactTextString(m) } func (*FieldIndexInfo) ProtoMessage() {} func (*FieldIndexInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{19} + return fileDescriptor_aab7cc9a69ed26e8, []int{20} } func (m *FieldIndexInfo) XXX_Unmarshal(b []byte) error { @@ -1533,6 +1649,8 @@ type LoadSegmentsRequest struct { LoadMeta *LoadMetaInfo `protobuf:"bytes,7,opt,name=load_meta,json=loadMeta,proto3" json:"load_meta,omitempty"` ReplicaID int64 `protobuf:"varint,8,opt,name=replicaID,proto3" json:"replicaID,omitempty"` DeltaPositions []*internalpb.MsgPosition `protobuf:"bytes,9,rep,name=delta_positions,json=deltaPositions,proto3" json:"delta_positions,omitempty"` + Version int64 `protobuf:"varint,10,opt,name=version,proto3" json:"version,omitempty"` + NeedTransfer bool `protobuf:"varint,11,opt,name=need_transfer,json=needTransfer,proto3" json:"need_transfer,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1542,7 +1660,7 @@ func (m *LoadSegmentsRequest) Reset() { *m = LoadSegmentsRequest{} } func (m *LoadSegmentsRequest) String() string { return proto.CompactTextString(m) } func (*LoadSegmentsRequest) ProtoMessage() {} func (*LoadSegmentsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{20} + return fileDescriptor_aab7cc9a69ed26e8, []int{21} } func (m *LoadSegmentsRequest) XXX_Unmarshal(b []byte) error { @@ -1626,6 +1744,20 @@ func (m *LoadSegmentsRequest) GetDeltaPositions() []*internalpb.MsgPosition { return nil } +func (m *LoadSegmentsRequest) GetVersion() int64 { + if m != nil { + return m.Version + } + return 0 +} + +func (m *LoadSegmentsRequest) GetNeedTransfer() bool { + if m != nil { + return m.NeedTransfer + } + return false +} + type ReleaseSegmentsRequest struct { Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` NodeID int64 `protobuf:"varint,2,opt,name=nodeID,proto3" json:"nodeID,omitempty"` @@ -1635,6 +1767,8 @@ type ReleaseSegmentsRequest struct { PartitionIDs []int64 `protobuf:"varint,5,rep,packed,name=partitionIDs,proto3" json:"partitionIDs,omitempty"` SegmentIDs []int64 `protobuf:"varint,6,rep,packed,name=segmentIDs,proto3" json:"segmentIDs,omitempty"` Scope DataScope `protobuf:"varint,7,opt,name=scope,proto3,enum=milvus.proto.query.DataScope" json:"scope,omitempty"` + Shard string `protobuf:"bytes,8,opt,name=shard,proto3" json:"shard,omitempty"` + NeedTransfer bool `protobuf:"varint,11,opt,name=need_transfer,json=needTransfer,proto3" json:"need_transfer,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1644,7 +1778,7 @@ func (m *ReleaseSegmentsRequest) Reset() { *m = ReleaseSegmentsRequest{} func (m *ReleaseSegmentsRequest) String() string { return proto.CompactTextString(m) } func (*ReleaseSegmentsRequest) ProtoMessage() {} func (*ReleaseSegmentsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{21} + return fileDescriptor_aab7cc9a69ed26e8, []int{22} } func (m *ReleaseSegmentsRequest) XXX_Unmarshal(b []byte) error { @@ -1714,6 +1848,20 @@ func (m *ReleaseSegmentsRequest) GetScope() DataScope { return DataScope_UnKnown } +func (m *ReleaseSegmentsRequest) GetShard() string { + if m != nil { + return m.Shard + } + return "" +} + +func (m *ReleaseSegmentsRequest) GetNeedTransfer() bool { + if m != nil { + return m.NeedTransfer + } + return false +} + type SearchRequest struct { Req *internalpb.SearchRequest `protobuf:"bytes,1,opt,name=req,proto3" json:"req,omitempty"` DmlChannels []string `protobuf:"bytes,2,rep,name=dml_channels,json=dmlChannels,proto3" json:"dml_channels,omitempty"` @@ -1729,7 +1877,7 @@ func (m *SearchRequest) Reset() { *m = SearchRequest{} } func (m *SearchRequest) String() string { return proto.CompactTextString(m) } func (*SearchRequest) ProtoMessage() {} func (*SearchRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{22} + return fileDescriptor_aab7cc9a69ed26e8, []int{23} } func (m *SearchRequest) XXX_Unmarshal(b []byte) error { @@ -1800,7 +1948,7 @@ func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{23} + return fileDescriptor_aab7cc9a69ed26e8, []int{24} } func (m *QueryRequest) XXX_Unmarshal(b []byte) error { @@ -1869,7 +2017,7 @@ func (m *SyncReplicaSegmentsRequest) Reset() { *m = SyncReplicaSegmentsR func (m *SyncReplicaSegmentsRequest) String() string { return proto.CompactTextString(m) } func (*SyncReplicaSegmentsRequest) ProtoMessage() {} func (*SyncReplicaSegmentsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{24} + return fileDescriptor_aab7cc9a69ed26e8, []int{25} } func (m *SyncReplicaSegmentsRequest) XXX_Unmarshal(b []byte) error { @@ -1924,7 +2072,7 @@ func (m *ReplicaSegmentsInfo) Reset() { *m = ReplicaSegmentsInfo{} } func (m *ReplicaSegmentsInfo) String() string { return proto.CompactTextString(m) } func (*ReplicaSegmentsInfo) ProtoMessage() {} func (*ReplicaSegmentsInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{25} + return fileDescriptor_aab7cc9a69ed26e8, []int{26} } func (m *ReplicaSegmentsInfo) XXX_Unmarshal(b []byte) error { @@ -1980,7 +2128,7 @@ func (m *HandoffSegmentsRequest) Reset() { *m = HandoffSegmentsRequest{} func (m *HandoffSegmentsRequest) String() string { return proto.CompactTextString(m) } func (*HandoffSegmentsRequest) ProtoMessage() {} func (*HandoffSegmentsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{26} + return fileDescriptor_aab7cc9a69ed26e8, []int{27} } func (m *HandoffSegmentsRequest) XXX_Unmarshal(b []byte) error { @@ -2038,7 +2186,7 @@ func (m *LoadBalanceRequest) Reset() { *m = LoadBalanceRequest{} } func (m *LoadBalanceRequest) String() string { return proto.CompactTextString(m) } func (*LoadBalanceRequest) ProtoMessage() {} func (*LoadBalanceRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{27} + return fileDescriptor_aab7cc9a69ed26e8, []int{28} } func (m *LoadBalanceRequest) XXX_Unmarshal(b []byte) error { @@ -2116,7 +2264,7 @@ func (m *DmChannelWatchInfo) Reset() { *m = DmChannelWatchInfo{} } func (m *DmChannelWatchInfo) String() string { return proto.CompactTextString(m) } func (*DmChannelWatchInfo) ProtoMessage() {} func (*DmChannelWatchInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{28} + return fileDescriptor_aab7cc9a69ed26e8, []int{29} } func (m *DmChannelWatchInfo) XXX_Unmarshal(b []byte) error { @@ -2187,7 +2335,7 @@ func (m *QueryChannelInfo) Reset() { *m = QueryChannelInfo{} } func (m *QueryChannelInfo) String() string { return proto.CompactTextString(m) } func (*QueryChannelInfo) ProtoMessage() {} func (*QueryChannelInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{29} + return fileDescriptor_aab7cc9a69ed26e8, []int{30} } func (m *QueryChannelInfo) XXX_Unmarshal(b []byte) error { @@ -2256,7 +2404,7 @@ func (m *PartitionStates) Reset() { *m = PartitionStates{} } func (m *PartitionStates) String() string { return proto.CompactTextString(m) } func (*PartitionStates) ProtoMessage() {} func (*PartitionStates) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{30} + return fileDescriptor_aab7cc9a69ed26e8, []int{31} } func (m *PartitionStates) XXX_Unmarshal(b []byte) error { @@ -2325,7 +2473,7 @@ func (m *SegmentInfo) Reset() { *m = SegmentInfo{} } func (m *SegmentInfo) String() string { return proto.CompactTextString(m) } func (*SegmentInfo) ProtoMessage() {} func (*SegmentInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{31} + return fileDescriptor_aab7cc9a69ed26e8, []int{32} } func (m *SegmentInfo) XXX_Unmarshal(b []byte) error { @@ -2477,7 +2625,7 @@ func (m *CollectionInfo) Reset() { *m = CollectionInfo{} } func (m *CollectionInfo) String() string { return proto.CompactTextString(m) } func (*CollectionInfo) ProtoMessage() {} func (*CollectionInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{32} + return fileDescriptor_aab7cc9a69ed26e8, []int{33} } func (m *CollectionInfo) XXX_Unmarshal(b []byte) error { @@ -2573,7 +2721,7 @@ func (m *UnsubscribeChannels) Reset() { *m = UnsubscribeChannels{} } func (m *UnsubscribeChannels) String() string { return proto.CompactTextString(m) } func (*UnsubscribeChannels) ProtoMessage() {} func (*UnsubscribeChannels) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{33} + return fileDescriptor_aab7cc9a69ed26e8, []int{34} } func (m *UnsubscribeChannels) XXX_Unmarshal(b []byte) error { @@ -2620,7 +2768,7 @@ func (m *UnsubscribeChannelInfo) Reset() { *m = UnsubscribeChannelInfo{} func (m *UnsubscribeChannelInfo) String() string { return proto.CompactTextString(m) } func (*UnsubscribeChannelInfo) ProtoMessage() {} func (*UnsubscribeChannelInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{34} + return fileDescriptor_aab7cc9a69ed26e8, []int{35} } func (m *UnsubscribeChannelInfo) XXX_Unmarshal(b []byte) error { @@ -2670,7 +2818,7 @@ func (m *SegmentChangeInfo) Reset() { *m = SegmentChangeInfo{} } func (m *SegmentChangeInfo) String() string { return proto.CompactTextString(m) } func (*SegmentChangeInfo) ProtoMessage() {} func (*SegmentChangeInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{35} + return fileDescriptor_aab7cc9a69ed26e8, []int{36} } func (m *SegmentChangeInfo) XXX_Unmarshal(b []byte) error { @@ -2731,7 +2879,7 @@ func (m *SealedSegmentsChangeInfo) Reset() { *m = SealedSegmentsChangeIn func (m *SealedSegmentsChangeInfo) String() string { return proto.CompactTextString(m) } func (*SealedSegmentsChangeInfo) ProtoMessage() {} func (*SealedSegmentsChangeInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{36} + return fileDescriptor_aab7cc9a69ed26e8, []int{37} } func (m *SealedSegmentsChangeInfo) XXX_Unmarshal(b []byte) error { @@ -2766,317 +2914,969 @@ func (m *SealedSegmentsChangeInfo) GetInfos() []*SegmentChangeInfo { return nil } -func init() { - proto.RegisterEnum("milvus.proto.query.DataScope", DataScope_name, DataScope_value) - proto.RegisterEnum("milvus.proto.query.PartitionState", PartitionState_name, PartitionState_value) - proto.RegisterEnum("milvus.proto.query.TriggerCondition", TriggerCondition_name, TriggerCondition_value) - proto.RegisterEnum("milvus.proto.query.LoadType", LoadType_name, LoadType_value) - proto.RegisterType((*ShowCollectionsRequest)(nil), "milvus.proto.query.ShowCollectionsRequest") - proto.RegisterType((*ShowCollectionsResponse)(nil), "milvus.proto.query.ShowCollectionsResponse") - proto.RegisterType((*ShowPartitionsRequest)(nil), "milvus.proto.query.ShowPartitionsRequest") - proto.RegisterType((*ShowPartitionsResponse)(nil), "milvus.proto.query.ShowPartitionsResponse") - proto.RegisterType((*LoadCollectionRequest)(nil), "milvus.proto.query.LoadCollectionRequest") - proto.RegisterType((*ReleaseCollectionRequest)(nil), "milvus.proto.query.ReleaseCollectionRequest") - proto.RegisterType((*GetStatisticsRequest)(nil), "milvus.proto.query.GetStatisticsRequest") - proto.RegisterType((*LoadPartitionsRequest)(nil), "milvus.proto.query.LoadPartitionsRequest") - proto.RegisterType((*ReleasePartitionsRequest)(nil), "milvus.proto.query.ReleasePartitionsRequest") - proto.RegisterType((*GetPartitionStatesRequest)(nil), "milvus.proto.query.GetPartitionStatesRequest") - proto.RegisterType((*GetPartitionStatesResponse)(nil), "milvus.proto.query.GetPartitionStatesResponse") - proto.RegisterType((*GetSegmentInfoRequest)(nil), "milvus.proto.query.GetSegmentInfoRequest") - proto.RegisterType((*GetSegmentInfoResponse)(nil), "milvus.proto.query.GetSegmentInfoResponse") - proto.RegisterType((*GetShardLeadersRequest)(nil), "milvus.proto.query.GetShardLeadersRequest") - proto.RegisterType((*GetShardLeadersResponse)(nil), "milvus.proto.query.GetShardLeadersResponse") - proto.RegisterType((*ShardLeadersList)(nil), "milvus.proto.query.ShardLeadersList") - proto.RegisterType((*LoadMetaInfo)(nil), "milvus.proto.query.LoadMetaInfo") - proto.RegisterType((*WatchDmChannelsRequest)(nil), "milvus.proto.query.WatchDmChannelsRequest") - proto.RegisterMapType((map[int64]*datapb.SegmentInfo)(nil), "milvus.proto.query.WatchDmChannelsRequest.SegmentInfosEntry") - proto.RegisterType((*SegmentLoadInfo)(nil), "milvus.proto.query.SegmentLoadInfo") - proto.RegisterType((*FieldIndexInfo)(nil), "milvus.proto.query.FieldIndexInfo") - proto.RegisterType((*LoadSegmentsRequest)(nil), "milvus.proto.query.LoadSegmentsRequest") - proto.RegisterType((*ReleaseSegmentsRequest)(nil), "milvus.proto.query.ReleaseSegmentsRequest") - proto.RegisterType((*SearchRequest)(nil), "milvus.proto.query.SearchRequest") - proto.RegisterType((*QueryRequest)(nil), "milvus.proto.query.QueryRequest") - proto.RegisterType((*SyncReplicaSegmentsRequest)(nil), "milvus.proto.query.SyncReplicaSegmentsRequest") - proto.RegisterType((*ReplicaSegmentsInfo)(nil), "milvus.proto.query.ReplicaSegmentsInfo") - proto.RegisterType((*HandoffSegmentsRequest)(nil), "milvus.proto.query.HandoffSegmentsRequest") - proto.RegisterType((*LoadBalanceRequest)(nil), "milvus.proto.query.LoadBalanceRequest") - proto.RegisterType((*DmChannelWatchInfo)(nil), "milvus.proto.query.DmChannelWatchInfo") - proto.RegisterType((*QueryChannelInfo)(nil), "milvus.proto.query.QueryChannelInfo") - proto.RegisterType((*PartitionStates)(nil), "milvus.proto.query.PartitionStates") - proto.RegisterType((*SegmentInfo)(nil), "milvus.proto.query.SegmentInfo") - proto.RegisterType((*CollectionInfo)(nil), "milvus.proto.query.CollectionInfo") - proto.RegisterType((*UnsubscribeChannels)(nil), "milvus.proto.query.UnsubscribeChannels") - proto.RegisterType((*UnsubscribeChannelInfo)(nil), "milvus.proto.query.UnsubscribeChannelInfo") - proto.RegisterType((*SegmentChangeInfo)(nil), "milvus.proto.query.SegmentChangeInfo") - proto.RegisterType((*SealedSegmentsChangeInfo)(nil), "milvus.proto.query.SealedSegmentsChangeInfo") +type GetDataDistributionRequest struct { + Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func init() { proto.RegisterFile("query_coord.proto", fileDescriptor_aab7cc9a69ed26e8) } - -var fileDescriptor_aab7cc9a69ed26e8 = []byte{ - // 2965 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3a, 0x49, 0x8c, 0x1c, 0x57, - 0xd9, 0x53, 0xbd, 0xf7, 0xd7, 0x5b, 0xf9, 0x8d, 0x3d, 0xee, 0xf4, 0x6f, 0x27, 0x93, 0x72, 0x9c, - 0xcc, 0x3f, 0x49, 0xc6, 0xce, 0x38, 0x44, 0x09, 0x24, 0x12, 0xf6, 0x4c, 0x3c, 0x19, 0x6c, 0x4f, - 0x86, 0x6a, 0x3b, 0x20, 0x2b, 0x52, 0x53, 0xdd, 0xf5, 0xba, 0xa7, 0xe4, 0x5a, 0xda, 0xf5, 0xaa, - 0xc7, 0x99, 0x70, 0xe5, 0xc2, 0x76, 0x40, 0x5c, 0x81, 0x13, 0x1c, 0x90, 0x12, 0x71, 0xe1, 0x88, - 0x10, 0x37, 0xae, 0x48, 0x5c, 0x91, 0x38, 0x72, 0x80, 0x2b, 0x07, 0xc4, 0x05, 0xbd, 0xad, 0xb6, - 0xae, 0xf2, 0xb4, 0x67, 0xb2, 0x22, 0x6e, 0x55, 0xdf, 0x5b, 0xbe, 0x7d, 0x7b, 0xef, 0xc1, 0x99, - 0x87, 0x33, 0xec, 0x1f, 0x0d, 0x46, 0x9e, 0xe7, 0x9b, 0x1b, 0x53, 0xdf, 0x0b, 0x3c, 0x84, 0x1c, - 0xcb, 0x3e, 0x9c, 0x11, 0xfe, 0xb7, 0xc1, 0xc6, 0x7b, 0xcd, 0x91, 0xe7, 0x38, 0x9e, 0xcb, 0x61, - 0xbd, 0x66, 0x7c, 0x46, 0xaf, 0x6d, 0xb9, 0x01, 0xf6, 0x5d, 0xc3, 0x96, 0xa3, 0x64, 0x74, 0x80, - 0x1d, 0x43, 0xfc, 0xa9, 0xa6, 0x11, 0x18, 0xf1, 0xfd, 0xb5, 0xef, 0x29, 0xb0, 0xd2, 0x3f, 0xf0, - 0x1e, 0x6d, 0x79, 0xb6, 0x8d, 0x47, 0x81, 0xe5, 0xb9, 0x44, 0xc7, 0x0f, 0x67, 0x98, 0x04, 0xe8, - 0x2a, 0x94, 0x86, 0x06, 0xc1, 0x5d, 0x65, 0x55, 0x59, 0x6b, 0x6c, 0x5e, 0xd8, 0x48, 0x50, 0x22, - 0x48, 0xb8, 0x43, 0x26, 0x37, 0x0c, 0x82, 0x75, 0x36, 0x13, 0x21, 0x28, 0x99, 0xc3, 0xdd, 0xed, - 0x6e, 0x61, 0x55, 0x59, 0x2b, 0xea, 0xec, 0x1b, 0x3d, 0x07, 0xad, 0x51, 0xb8, 0xf7, 0xee, 0x36, - 0xe9, 0x16, 0x57, 0x8b, 0x6b, 0x45, 0x3d, 0x09, 0xd4, 0xfe, 0xaa, 0xc0, 0xf9, 0x39, 0x32, 0xc8, - 0xd4, 0x73, 0x09, 0x46, 0xd7, 0xa0, 0x42, 0x02, 0x23, 0x98, 0x11, 0x41, 0xc9, 0xff, 0x65, 0x52, - 0xd2, 0x67, 0x53, 0x74, 0x31, 0x75, 0x1e, 0x6d, 0x21, 0x03, 0x2d, 0x7a, 0x05, 0xce, 0x5a, 0xee, - 0x1d, 0xec, 0x78, 0xfe, 0xd1, 0x60, 0x8a, 0xfd, 0x11, 0x76, 0x03, 0x63, 0x82, 0x25, 0x8d, 0xcb, - 0x72, 0x6c, 0x3f, 0x1a, 0x42, 0xaf, 0xc1, 0x79, 0xae, 0x25, 0x82, 0xfd, 0x43, 0x6b, 0x84, 0x07, - 0xc6, 0xa1, 0x61, 0xd9, 0xc6, 0xd0, 0xc6, 0xdd, 0xd2, 0x6a, 0x71, 0xad, 0xa6, 0x9f, 0x63, 0xc3, - 0x7d, 0x3e, 0x7a, 0x5d, 0x0e, 0x6a, 0xbf, 0x52, 0xe0, 0x1c, 0xe5, 0x70, 0xdf, 0xf0, 0x03, 0xeb, - 0x53, 0x90, 0xb3, 0x06, 0xcd, 0x38, 0x6f, 0xdd, 0x22, 0x1b, 0x4b, 0xc0, 0xe8, 0x9c, 0xa9, 0x44, - 0x4f, 0x65, 0x52, 0x62, 0x6c, 0x26, 0x60, 0xda, 0x2f, 0x85, 0x41, 0xc4, 0xe9, 0x3c, 0x8d, 0x22, - 0xd2, 0x38, 0x0b, 0xf3, 0x38, 0x4f, 0xa0, 0x06, 0xed, 0x6f, 0x0a, 0x9c, 0xbb, 0xed, 0x19, 0x66, - 0x64, 0x30, 0x9f, 0xbd, 0x38, 0xdf, 0x82, 0x0a, 0xf7, 0xae, 0x6e, 0x89, 0xe1, 0xba, 0x9c, 0xc4, - 0x25, 0x3c, 0x2f, 0xa2, 0xb0, 0xcf, 0x00, 0xba, 0x58, 0x84, 0x2e, 0x43, 0xdb, 0xc7, 0x53, 0xdb, - 0x1a, 0x19, 0x03, 0x77, 0xe6, 0x0c, 0xb1, 0xdf, 0x2d, 0xaf, 0x2a, 0x6b, 0x65, 0xbd, 0x25, 0xa0, - 0x7b, 0x0c, 0xa8, 0xfd, 0x4c, 0x81, 0xae, 0x8e, 0x6d, 0x6c, 0x10, 0xfc, 0x79, 0x32, 0xbb, 0x02, - 0x15, 0xd7, 0x33, 0xf1, 0xee, 0x36, 0x63, 0xb6, 0xa8, 0x8b, 0x3f, 0xed, 0x5f, 0x0a, 0x9c, 0xdd, - 0xc1, 0x01, 0xd5, 0xba, 0x45, 0x02, 0x6b, 0x14, 0x9a, 0xf5, 0x5b, 0x50, 0xf4, 0xf1, 0x43, 0x41, - 0xd9, 0x8b, 0x49, 0xca, 0xc2, 0x20, 0x95, 0xb5, 0x52, 0xa7, 0xeb, 0xd0, 0xb3, 0xd0, 0x34, 0x1d, - 0x7b, 0x30, 0x3a, 0x30, 0x5c, 0x17, 0xdb, 0xdc, 0x6e, 0xea, 0x7a, 0xc3, 0x74, 0xec, 0x2d, 0x01, - 0x42, 0x4f, 0x03, 0x10, 0x3c, 0x71, 0xb0, 0x1b, 0x44, 0x71, 0x25, 0x06, 0x41, 0xeb, 0x70, 0x66, - 0xec, 0x7b, 0xce, 0x80, 0x1c, 0x18, 0xbe, 0x39, 0xb0, 0xb1, 0x61, 0x62, 0x9f, 0x51, 0x5f, 0xd3, - 0x3b, 0x74, 0xa0, 0x4f, 0xe1, 0xb7, 0x19, 0x18, 0x5d, 0x83, 0x32, 0x19, 0x79, 0x53, 0xcc, 0x74, - 0xd0, 0xde, 0xbc, 0xb8, 0x31, 0x1f, 0x77, 0x37, 0xb6, 0x8d, 0xc0, 0xe8, 0xd3, 0x49, 0x3a, 0x9f, - 0xab, 0xfd, 0xb0, 0xc0, 0x8d, 0xf0, 0x0b, 0xee, 0xd3, 0x31, 0x43, 0x2d, 0x7f, 0x32, 0x86, 0x5a, - 0xc9, 0x32, 0xd4, 0x3f, 0x44, 0x86, 0xfa, 0x45, 0x17, 0x48, 0x64, 0xcc, 0xe5, 0x84, 0x31, 0xff, - 0x5a, 0x81, 0xa7, 0x76, 0x70, 0x10, 0x92, 0x4f, 0x6d, 0x13, 0x7f, 0x41, 0x03, 0xf5, 0xc7, 0x0a, - 0xf4, 0xb2, 0x68, 0x3d, 0x4d, 0xb0, 0xbe, 0x0f, 0x2b, 0x21, 0x8e, 0x81, 0x89, 0xc9, 0xc8, 0xb7, - 0xa6, 0x4c, 0x8d, 0xcc, 0xfd, 0x1a, 0x9b, 0x97, 0xb2, 0xdc, 0x22, 0x4d, 0xc1, 0xb9, 0x70, 0x8b, - 0xed, 0xd8, 0x0e, 0xda, 0x8f, 0x15, 0x38, 0x47, 0xdd, 0x5d, 0xf8, 0xa7, 0x3b, 0xf6, 0x4e, 0x2e, - 0xd7, 0xa4, 0xe7, 0x17, 0xe6, 0x3c, 0x7f, 0x01, 0x19, 0xb3, 0xca, 0x27, 0x4d, 0xcf, 0x69, 0x64, - 0xf7, 0x15, 0x28, 0x5b, 0xee, 0xd8, 0x93, 0xa2, 0x7a, 0x26, 0x4b, 0x54, 0x71, 0x64, 0x7c, 0xb6, - 0xe6, 0x72, 0x2a, 0xa2, 0x50, 0x74, 0x0a, 0x73, 0x4b, 0xb3, 0x5d, 0xc8, 0x60, 0xfb, 0x47, 0x0a, - 0x9c, 0x9f, 0x43, 0x78, 0x1a, 0xbe, 0xdf, 0x84, 0x0a, 0x0b, 0xb0, 0x92, 0xf1, 0xe7, 0x32, 0x19, - 0x8f, 0xa1, 0xbb, 0x6d, 0x91, 0x40, 0x17, 0x6b, 0x34, 0x0f, 0xd4, 0xf4, 0x18, 0x0d, 0xfd, 0x22, - 0xec, 0x0f, 0x5c, 0xc3, 0xe1, 0x02, 0xa8, 0xeb, 0x0d, 0x01, 0xdb, 0x33, 0x1c, 0x8c, 0x9e, 0x82, - 0x1a, 0x75, 0xd9, 0x81, 0x65, 0x4a, 0xf5, 0x57, 0x99, 0x0b, 0x9b, 0x04, 0x5d, 0x04, 0x60, 0x43, - 0x86, 0x69, 0xfa, 0x3c, 0x2b, 0xd4, 0xf5, 0x3a, 0x85, 0x5c, 0xa7, 0x00, 0xed, 0x27, 0x0a, 0x34, - 0x69, 0xcc, 0xbe, 0x83, 0x03, 0x83, 0xea, 0x01, 0xbd, 0x01, 0x75, 0xdb, 0x33, 0xcc, 0x41, 0x70, - 0x34, 0xe5, 0xa8, 0xda, 0x69, 0x59, 0x73, 0x16, 0xe8, 0xa2, 0xbb, 0x47, 0x53, 0xac, 0xd7, 0x6c, - 0xf1, 0xb5, 0x88, 0xbc, 0xe7, 0x5c, 0xb9, 0x98, 0xe1, 0xca, 0x1f, 0x95, 0x61, 0xe5, 0x5b, 0x46, - 0x30, 0x3a, 0xd8, 0x76, 0x64, 0x72, 0x3b, 0xb9, 0x11, 0x44, 0xb1, 0xad, 0x10, 0x8f, 0x6d, 0x9f, - 0x58, 0xec, 0x0c, 0xed, 0xbc, 0x9c, 0x65, 0xe7, 0xb4, 0xc1, 0xd8, 0x78, 0x4f, 0xa8, 0x2a, 0x66, - 0xe7, 0xb1, 0x1c, 0x54, 0x39, 0x49, 0x0e, 0xda, 0x82, 0x16, 0xfe, 0x60, 0x64, 0xcf, 0xa8, 0xce, - 0x19, 0xf6, 0x2a, 0xc3, 0xfe, 0x74, 0x06, 0xf6, 0xb8, 0x93, 0x35, 0xc5, 0xa2, 0x5d, 0x41, 0x03, - 0x57, 0xb5, 0x83, 0x03, 0xa3, 0x5b, 0x63, 0x64, 0xac, 0xe6, 0xa9, 0x5a, 0xda, 0x07, 0x57, 0x37, - 0xfd, 0x43, 0x17, 0xa0, 0x2e, 0x32, 0xde, 0xee, 0x76, 0xb7, 0xce, 0xc4, 0x17, 0x01, 0x90, 0x01, - 0x2d, 0x11, 0x81, 0x04, 0x85, 0xc0, 0x28, 0x7c, 0x33, 0x0b, 0x41, 0xb6, 0xb2, 0xe3, 0x94, 0x93, - 0xb7, 0xdd, 0xc0, 0x3f, 0xd2, 0x9b, 0x24, 0x06, 0xa2, 0x4d, 0x8d, 0x37, 0x1e, 0xdb, 0x96, 0x8b, - 0xf7, 0xb8, 0x86, 0x1b, 0x8c, 0x88, 0x24, 0xb0, 0x37, 0x80, 0x33, 0x73, 0x1b, 0x21, 0x15, 0x8a, - 0x0f, 0xf0, 0x11, 0x33, 0xa3, 0xa2, 0x4e, 0x3f, 0xd1, 0xab, 0x50, 0x3e, 0x34, 0xec, 0x19, 0x66, - 0x66, 0x72, 0xbc, 0x24, 0xf9, 0xe4, 0xaf, 0x16, 0x5e, 0x57, 0xb4, 0xdf, 0x97, 0xa0, 0x23, 0x86, - 0xa8, 0xa4, 0x98, 0x17, 0x5d, 0x80, 0x7a, 0x18, 0x7f, 0x05, 0x96, 0x08, 0x80, 0x56, 0xa1, 0x11, - 0xb3, 0x21, 0x61, 0x98, 0x71, 0xd0, 0x42, 0xd6, 0x29, 0xb3, 0x69, 0x29, 0x96, 0x4d, 0x2f, 0x02, - 0x8c, 0xed, 0x19, 0x39, 0x18, 0x04, 0x96, 0x83, 0x45, 0x36, 0xaf, 0x33, 0xc8, 0x5d, 0xcb, 0xc1, - 0xe8, 0x3a, 0x34, 0x87, 0x96, 0x6b, 0x7b, 0x93, 0xc1, 0xd4, 0x08, 0x0e, 0x48, 0xb7, 0x92, 0x6b, - 0x35, 0x37, 0x2d, 0x6c, 0x9b, 0x37, 0xd8, 0x5c, 0xbd, 0xc1, 0xd7, 0xec, 0xd3, 0x25, 0xe8, 0x69, - 0x68, 0xb8, 0x33, 0x67, 0xe0, 0x8d, 0x07, 0xbe, 0xf7, 0x88, 0xda, 0x1d, 0x43, 0xe1, 0xce, 0x9c, - 0x77, 0xc7, 0xba, 0xf7, 0x88, 0xc6, 0xbf, 0x3a, 0x8d, 0x84, 0xc4, 0xf6, 0x26, 0xa4, 0x5b, 0x5b, - 0x68, 0xff, 0x68, 0x01, 0x5d, 0x6d, 0x62, 0x3b, 0x30, 0xd8, 0xea, 0xfa, 0x62, 0xab, 0xc3, 0x05, - 0xe8, 0x79, 0x68, 0x8f, 0x3c, 0x67, 0x6a, 0x30, 0x09, 0xdd, 0xf4, 0x3d, 0x87, 0x19, 0x5d, 0x51, - 0x4f, 0x41, 0xd1, 0x16, 0x34, 0x2c, 0xd7, 0xc4, 0x1f, 0x08, 0xcb, 0x6c, 0x30, 0x3c, 0x5a, 0x96, - 0x65, 0x32, 0x44, 0xbb, 0x74, 0x2e, 0xd3, 0x3a, 0x58, 0xf2, 0x93, 0xd0, 0xb0, 0x2c, 0x0d, 0x9c, - 0x58, 0x1f, 0xe2, 0x6e, 0x93, 0x6b, 0x51, 0xc0, 0xfa, 0xd6, 0x87, 0x98, 0x56, 0x8a, 0x96, 0x4b, - 0xb0, 0x1f, 0xc8, 0xba, 0xbd, 0xdb, 0x62, 0xb1, 0xbb, 0xc5, 0xa1, 0xc2, 0xde, 0xb5, 0xdf, 0x14, - 0xa0, 0x9d, 0x44, 0x84, 0xba, 0x50, 0x1d, 0x33, 0x88, 0xb4, 0x1e, 0xf9, 0x4b, 0xd1, 0x62, 0x97, - 0xb6, 0xd0, 0x03, 0x46, 0x0b, 0x33, 0x9e, 0x9a, 0xde, 0xe0, 0x30, 0xb6, 0x01, 0x35, 0x02, 0xce, - 0x1e, 0x4b, 0x17, 0x45, 0x86, 0xb2, 0xce, 0x20, 0x2c, 0x59, 0x74, 0xa1, 0xca, 0xd9, 0x90, 0xa6, - 0x23, 0x7f, 0xe9, 0xc8, 0x70, 0x66, 0x31, 0xac, 0xdc, 0x74, 0xe4, 0x2f, 0xda, 0x86, 0x26, 0xdf, - 0x72, 0x6a, 0xf8, 0x86, 0x23, 0x0d, 0xe7, 0xd9, 0xcc, 0xf8, 0x7b, 0x0b, 0x1f, 0xbd, 0x47, 0x9d, - 0x63, 0xdf, 0xb0, 0x7c, 0x9d, 0x0b, 0x7a, 0x9f, 0xad, 0x42, 0x6b, 0xa0, 0xf2, 0x5d, 0xc6, 0x96, - 0x8d, 0x85, 0x09, 0x56, 0x59, 0x46, 0x6a, 0x33, 0xf8, 0x4d, 0xcb, 0xc6, 0xdc, 0xca, 0x42, 0x16, - 0x98, 0x68, 0x6b, 0xdc, 0xc8, 0x18, 0x84, 0x0a, 0x56, 0xfb, 0x73, 0x11, 0x96, 0xa9, 0xaf, 0x09, - 0xb7, 0x3b, 0x45, 0x7a, 0xb8, 0x08, 0x60, 0x92, 0x60, 0x90, 0x48, 0x11, 0x75, 0x93, 0x04, 0x3c, - 0x78, 0xa0, 0x37, 0x64, 0x74, 0x2f, 0xe6, 0x17, 0x7c, 0x29, 0xdf, 0x9f, 0x8f, 0xf0, 0x27, 0x6a, - 0x87, 0x2f, 0x41, 0x8b, 0x78, 0x33, 0x7f, 0x84, 0x07, 0x89, 0xd2, 0xbc, 0xc9, 0x81, 0x7b, 0xd9, - 0x49, 0xac, 0x92, 0xd9, 0x96, 0xc7, 0xa2, 0x7c, 0xf5, 0x74, 0x51, 0xbe, 0x96, 0x8e, 0xf2, 0xb7, - 0xa0, 0xc3, 0xdc, 0x6f, 0x30, 0xf5, 0x08, 0xef, 0x70, 0x84, 0xd7, 0x6a, 0x39, 0x1d, 0xee, 0x1d, - 0x32, 0xd9, 0x17, 0x53, 0xf5, 0x36, 0x5b, 0x2a, 0x7f, 0x89, 0xf6, 0xd3, 0x02, 0xac, 0x88, 0x8e, - 0xe9, 0xf4, 0x8a, 0xcd, 0xcb, 0xfb, 0x32, 0x6a, 0x16, 0x1f, 0xd3, 0x83, 0x94, 0x16, 0xa8, 0x05, - 0xca, 0x19, 0xb5, 0x40, 0xb2, 0x0e, 0xaf, 0xcc, 0xd5, 0xe1, 0x61, 0x57, 0x5d, 0x7d, 0x82, 0xae, - 0xfa, 0xef, 0x0a, 0xb4, 0xfa, 0xd8, 0xf0, 0x47, 0x07, 0x52, 0x18, 0xaf, 0xc5, 0x8f, 0x12, 0x9e, - 0xcb, 0x11, 0x74, 0x62, 0xc9, 0x97, 0xe7, 0x0c, 0xe1, 0x1f, 0x0a, 0x34, 0xbf, 0x49, 0x87, 0x24, - 0xb3, 0xaf, 0xc7, 0x99, 0x7d, 0x3e, 0x87, 0x59, 0x1d, 0x07, 0xbe, 0x85, 0x0f, 0xf1, 0x97, 0x8e, - 0xdd, 0x3f, 0x2a, 0xd0, 0xeb, 0x1f, 0xb9, 0x23, 0x9d, 0x7b, 0xd4, 0xe9, 0xcd, 0xfe, 0x12, 0xb4, - 0x0e, 0x13, 0xdd, 0x42, 0x81, 0x85, 0xff, 0xe6, 0x61, 0xbc, 0x5d, 0xd0, 0x41, 0x95, 0x27, 0x18, - 0x82, 0x59, 0x19, 0xe0, 0x5e, 0xc8, 0xa2, 0x3a, 0x45, 0x1c, 0x0b, 0x10, 0x1d, 0x3f, 0x09, 0xd4, - 0x7c, 0x58, 0xce, 0x98, 0x87, 0xce, 0x43, 0x55, 0x74, 0x26, 0x22, 0x91, 0x71, 0x3f, 0x34, 0xa9, - 0x76, 0xa2, 0xde, 0xda, 0x32, 0xe7, 0x8b, 0x20, 0x13, 0x3d, 0x03, 0x8d, 0xb0, 0x84, 0x34, 0xe7, - 0xd4, 0x63, 0x12, 0xed, 0x77, 0x0a, 0xac, 0xbc, 0x63, 0xb8, 0xa6, 0x37, 0x1e, 0x9f, 0x5e, 0x72, - 0x5b, 0x90, 0xa8, 0x2e, 0x17, 0xed, 0x5b, 0x93, 0x25, 0xe9, 0x8b, 0x70, 0xc6, 0xe7, 0x11, 0xcc, - 0x4c, 0x8a, 0xb6, 0xa8, 0xab, 0x72, 0x20, 0x14, 0xd9, 0x47, 0x05, 0x40, 0x34, 0xea, 0xde, 0x30, - 0x6c, 0xc3, 0x1d, 0xe1, 0x93, 0x93, 0x7e, 0x19, 0xda, 0x89, 0x5c, 0x11, 0x1e, 0xef, 0xc7, 0x93, - 0x05, 0x41, 0xb7, 0xa0, 0x3d, 0xe4, 0xa8, 0x06, 0x3e, 0x36, 0x88, 0xe7, 0xb2, 0x20, 0xd8, 0xce, - 0x6e, 0x51, 0xef, 0xfa, 0xd6, 0x64, 0x82, 0xfd, 0x2d, 0xcf, 0x35, 0x79, 0xb4, 0x6e, 0x0d, 0x25, - 0x99, 0x74, 0x29, 0x55, 0x4e, 0x94, 0x38, 0x65, 0x6b, 0x04, 0x61, 0xe6, 0x64, 0xa2, 0x20, 0xd8, - 0xb0, 0x23, 0x41, 0x44, 0x51, 0x53, 0xe5, 0x03, 0xfd, 0xfc, 0x13, 0x8a, 0x8c, 0x44, 0xa6, 0xfd, - 0x56, 0x01, 0x14, 0x36, 0x09, 0xac, 0x65, 0x60, 0x16, 0x96, 0x5e, 0xaa, 0x64, 0x04, 0xef, 0x0b, - 0x50, 0x37, 0xe5, 0x4a, 0xe1, 0x11, 0x11, 0x80, 0xfa, 0x0c, 0x67, 0x63, 0x40, 0xb3, 0x1e, 0x36, - 0x65, 0xb5, 0xcd, 0x81, 0xb7, 0x19, 0x2c, 0x99, 0x07, 0x4b, 0xe9, 0x3c, 0x18, 0x6f, 0xc0, 0xcb, - 0x89, 0x06, 0x5c, 0xfb, 0xb8, 0x00, 0x2a, 0x8b, 0x68, 0x5b, 0x51, 0x17, 0xb8, 0x10, 0xd1, 0x97, - 0xa0, 0x25, 0x2e, 0xc0, 0x12, 0x84, 0x37, 0x1f, 0xc6, 0x36, 0x43, 0x57, 0xe1, 0x2c, 0x9f, 0xe4, - 0x63, 0x32, 0xb3, 0xa3, 0x42, 0x93, 0x57, 0x7d, 0xe8, 0x21, 0x0f, 0xa5, 0x74, 0x48, 0xae, 0xb8, - 0x07, 0x2b, 0x13, 0xdb, 0x1b, 0x1a, 0xf6, 0x20, 0xa9, 0x1e, 0xae, 0xc3, 0x05, 0x2c, 0xfe, 0x2c, - 0x5f, 0xde, 0x8f, 0xeb, 0x90, 0xa0, 0x1d, 0xda, 0xef, 0xe1, 0x07, 0x61, 0x21, 0x20, 0xce, 0x56, - 0x17, 0xa9, 0x03, 0x9a, 0x74, 0xa1, 0xfc, 0xd3, 0x7e, 0xa1, 0x40, 0x27, 0x75, 0x86, 0x96, 0x6e, - 0x98, 0x94, 0xf9, 0x86, 0xe9, 0x75, 0x28, 0xd3, 0x2e, 0x82, 0xc7, 0xbb, 0x76, 0x76, 0x31, 0x9f, - 0xdc, 0x55, 0xe7, 0x0b, 0xd0, 0x15, 0x58, 0xce, 0xb8, 0x6d, 0x11, 0x36, 0x80, 0xe6, 0x2f, 0x5b, - 0xb4, 0xbf, 0x94, 0xa0, 0x11, 0x93, 0xc7, 0x31, 0xbd, 0xde, 0x22, 0x87, 0x22, 0x29, 0xf6, 0x8a, - 0xf3, 0xec, 0xe5, 0x5c, 0x37, 0x50, 0xbb, 0x73, 0xb0, 0xc3, 0xab, 0x64, 0x51, 0xb2, 0x3b, 0xd8, - 0x61, 0xcd, 0x07, 0x35, 0xc9, 0x99, 0xc3, 0xbb, 0x34, 0xee, 0x4e, 0x55, 0x77, 0xe6, 0xb0, 0x1e, - 0x2d, 0xd9, 0x20, 0x54, 0x1f, 0xd3, 0x20, 0xd4, 0x92, 0x0d, 0x42, 0xc2, 0x8f, 0xea, 0x69, 0x3f, - 0x5a, 0xb4, 0xfd, 0xba, 0x0a, 0xcb, 0x23, 0x1f, 0x1b, 0x01, 0x36, 0x6f, 0x1c, 0x6d, 0x85, 0x43, - 0xac, 0x7b, 0xaf, 0xe9, 0x59, 0x43, 0xe8, 0x66, 0x74, 0x98, 0xc0, 0xb5, 0xdc, 0x64, 0x5a, 0xce, - 0xee, 0x3f, 0x84, 0x6e, 0xb8, 0x92, 0x65, 0x78, 0x66, 0x7f, 0xe9, 0xc6, 0xaf, 0x75, 0xa2, 0xc6, - 0xef, 0x19, 0x68, 0xc8, 0xec, 0x49, 0xdd, 0xbd, 0xcd, 0x23, 0x9f, 0x8c, 0x05, 0x26, 0x49, 0x04, - 0x83, 0x4e, 0xf2, 0x34, 0x2e, 0xdd, 0xbd, 0xa9, 0x73, 0xdd, 0x9b, 0xf6, 0xa7, 0x22, 0xb4, 0xa3, - 0xae, 0x60, 0xe1, 0x68, 0xb1, 0xc8, 0xc5, 0xe2, 0x1e, 0xa8, 0x51, 0xce, 0x65, 0x82, 0x7c, 0x6c, - 0x63, 0x93, 0x3e, 0xc9, 0xee, 0x4c, 0x53, 0x6e, 0x99, 0x38, 0x2b, 0x2c, 0x3d, 0xd1, 0x59, 0xe1, - 0x29, 0xef, 0x60, 0xae, 0xc1, 0xb9, 0x30, 0xcf, 0x26, 0xd8, 0xe6, 0x45, 0xf7, 0x59, 0x39, 0xb8, - 0x1f, 0x67, 0x3f, 0xc7, 0xd3, 0xab, 0x79, 0x9e, 0x9e, 0xd6, 0x74, 0x6d, 0x4e, 0xd3, 0xf3, 0x57, - 0x41, 0xf5, 0xac, 0xab, 0xa0, 0x7b, 0xb0, 0x7c, 0xcf, 0x25, 0xb3, 0x21, 0x19, 0xf9, 0xd6, 0x10, - 0x87, 0xd5, 0xe7, 0x22, 0x6a, 0xed, 0x41, 0x2d, 0x55, 0xc0, 0x86, 0xff, 0xda, 0x0f, 0x14, 0x58, - 0x99, 0xdf, 0x97, 0x59, 0x4c, 0x14, 0x2f, 0x94, 0x44, 0xbc, 0xf8, 0x36, 0x2c, 0x47, 0xdb, 0x27, - 0x4b, 0xe3, 0x9c, 0xe2, 0x2f, 0x83, 0x70, 0x1d, 0x45, 0x7b, 0x48, 0x98, 0xf6, 0x4f, 0x25, 0x3c, - 0x67, 0xa3, 0xb0, 0x09, 0x3b, 0x63, 0xa4, 0x39, 0xcc, 0x73, 0x6d, 0xcb, 0x0d, 0xbb, 0x58, 0xc1, - 0x23, 0x07, 0x8a, 0x2e, 0xf6, 0x1d, 0xe8, 0x88, 0x49, 0x61, 0x2a, 0x5a, 0xb0, 0xf8, 0x6a, 0xf3, - 0x75, 0x61, 0x12, 0xba, 0x0c, 0x6d, 0x71, 0xf8, 0x27, 0xf1, 0x15, 0x33, 0x8e, 0x04, 0xd1, 0x37, - 0x40, 0x95, 0xd3, 0x9e, 0x34, 0xf9, 0x75, 0xc4, 0xc2, 0xb0, 0x88, 0xfb, 0xbe, 0x02, 0xdd, 0x64, - 0x2a, 0x8c, 0xb1, 0xff, 0xe4, 0xa5, 0xdc, 0xd7, 0x92, 0xd7, 0x26, 0x97, 0x1f, 0x43, 0x4f, 0x84, - 0x47, 0x1c, 0x39, 0xac, 0x7f, 0x1d, 0xea, 0x61, 0x87, 0x81, 0x1a, 0x50, 0xbd, 0xe7, 0xde, 0x72, - 0xbd, 0x47, 0xae, 0xba, 0x84, 0xaa, 0x50, 0xbc, 0x6e, 0xdb, 0xaa, 0x82, 0x5a, 0x50, 0xef, 0x07, - 0x3e, 0x36, 0x1c, 0xcb, 0x9d, 0xa8, 0x05, 0xd4, 0x06, 0x78, 0xc7, 0x22, 0x81, 0xe7, 0x5b, 0x23, - 0xc3, 0x56, 0x8b, 0xeb, 0x1f, 0x42, 0x3b, 0xe9, 0xf5, 0xa8, 0x09, 0xb5, 0x3d, 0x2f, 0x78, 0xfb, - 0x03, 0x8b, 0x04, 0xea, 0x12, 0x9d, 0xbf, 0xe7, 0x05, 0xfb, 0x3e, 0x26, 0xd8, 0x0d, 0x54, 0x05, - 0x01, 0x54, 0xde, 0x75, 0xb7, 0x2d, 0xf2, 0x40, 0x2d, 0xa0, 0x65, 0x91, 0xb7, 0x0d, 0x7b, 0x57, - 0xb8, 0x92, 0x5a, 0xa4, 0xcb, 0xc3, 0xbf, 0x12, 0x52, 0xa1, 0x19, 0x4e, 0xd9, 0xd9, 0xbf, 0xa7, - 0x96, 0x51, 0x1d, 0xca, 0xfc, 0xb3, 0xb2, 0x6e, 0x82, 0x9a, 0x2e, 0x3a, 0xe9, 0x9e, 0x9c, 0x89, - 0x10, 0xa4, 0x2e, 0x51, 0xce, 0x44, 0xd5, 0xaf, 0x2a, 0xa8, 0x03, 0x8d, 0x58, 0x0d, 0xad, 0x16, - 0x28, 0x60, 0xc7, 0x9f, 0x8e, 0x44, 0x35, 0xcd, 0x49, 0xa0, 0x7a, 0xdf, 0xa6, 0x92, 0x28, 0xad, - 0xdf, 0x80, 0x9a, 0x0c, 0x47, 0x74, 0xaa, 0x10, 0x11, 0xfd, 0x55, 0x97, 0xd0, 0x19, 0x68, 0x25, - 0x2e, 0xb0, 0x55, 0x05, 0x21, 0x68, 0x27, 0x1f, 0x56, 0xa8, 0x85, 0xcd, 0x9f, 0xb7, 0x00, 0x78, - 0x49, 0xe7, 0x79, 0xbe, 0x89, 0xa6, 0x80, 0x76, 0x70, 0x40, 0xd3, 0x95, 0xe7, 0xca, 0x54, 0x43, - 0xd0, 0xd5, 0xfc, 0x3b, 0xfe, 0xd4, 0x54, 0x41, 0x6a, 0x2f, 0xaf, 0xbb, 0x4d, 0x4d, 0xd7, 0x96, - 0x90, 0xc3, 0x30, 0xde, 0xb5, 0x1c, 0x7c, 0xd7, 0x1a, 0x3d, 0x08, 0x6b, 0xc1, 0x7c, 0x8c, 0xa9, - 0xa9, 0x12, 0x63, 0x2a, 0xec, 0x8b, 0x9f, 0x7e, 0xe0, 0x5b, 0xee, 0x44, 0x5e, 0x83, 0x69, 0x4b, - 0xe8, 0x61, 0xea, 0x4d, 0x83, 0x44, 0xb8, 0xb9, 0xc8, 0x33, 0x86, 0x93, 0xa1, 0xb4, 0xa1, 0x93, - 0x7a, 0x00, 0x85, 0xd6, 0xb3, 0x6f, 0xd2, 0xb2, 0x1e, 0x6b, 0xf5, 0x5e, 0x5c, 0x68, 0x6e, 0x88, - 0xcd, 0x82, 0x76, 0xf2, 0x91, 0x0f, 0xfa, 0xff, 0xbc, 0x0d, 0xe6, 0xee, 0xf2, 0x7b, 0xeb, 0x8b, - 0x4c, 0x0d, 0x51, 0xdd, 0xe7, 0xf6, 0x74, 0x1c, 0xaa, 0xcc, 0x77, 0x14, 0xbd, 0xc7, 0xdd, 0x40, - 0x6a, 0x4b, 0xe8, 0x3b, 0x70, 0x66, 0xee, 0xc5, 0x01, 0x7a, 0x29, 0xbb, 0xa5, 0xcf, 0x7e, 0x98, - 0x70, 0x1c, 0x86, 0xfb, 0x69, 0x6f, 0xc8, 0xa7, 0x7e, 0xee, 0x75, 0xce, 0xe2, 0xd4, 0xc7, 0xb6, - 0x7f, 0x1c, 0xf5, 0x4f, 0x8c, 0x61, 0xc6, 0xdc, 0x26, 0xdd, 0x5c, 0xbc, 0x9c, 0x85, 0x22, 0xf7, - 0xd9, 0x43, 0x6f, 0x63, 0xd1, 0xe9, 0x71, 0xeb, 0x4a, 0xde, 0xac, 0x67, 0x0b, 0x2d, 0xf3, 0x35, - 0x40, 0xb6, 0x75, 0x65, 0x5f, 0xd4, 0x6b, 0x4b, 0xe8, 0x6e, 0x22, 0x1a, 0xa2, 0xe7, 0xf3, 0x94, - 0x93, 0x3c, 0x72, 0x38, 0x4e, 0x6e, 0xdf, 0x05, 0xc4, 0x7d, 0xc7, 0x1d, 0x5b, 0x93, 0x99, 0x6f, - 0x70, 0xc3, 0xca, 0x0b, 0x37, 0xf3, 0x53, 0x25, 0x9a, 0x57, 0x9e, 0x60, 0x45, 0xc8, 0xd2, 0x00, - 0x60, 0x07, 0x07, 0x77, 0x70, 0xe0, 0x5b, 0x23, 0x92, 0xe6, 0x48, 0xfc, 0x44, 0x13, 0x24, 0xaa, - 0x17, 0x8e, 0x9d, 0x17, 0x22, 0x18, 0x42, 0x63, 0x07, 0x07, 0xe2, 0xf0, 0x8a, 0xa0, 0xdc, 0x95, - 0x72, 0x86, 0x44, 0xb1, 0x76, 0xfc, 0xc4, 0x78, 0x38, 0x4b, 0xbd, 0x32, 0x40, 0xb9, 0x8a, 0x9d, - 0x7f, 0xfb, 0x90, 0x1d, 0xce, 0x72, 0x9e, 0x2d, 0x68, 0x4b, 0x9b, 0xff, 0x6e, 0x40, 0x9d, 0xe5, - 0x27, 0x9a, 0xf7, 0xfe, 0x97, 0x9e, 0x3e, 0x85, 0xf4, 0xf4, 0x3e, 0x74, 0x52, 0x97, 0xd6, 0xd9, - 0xfa, 0xcc, 0xbe, 0xd9, 0x3e, 0xce, 0xdf, 0xde, 0xe3, 0x6f, 0x32, 0xc2, 0xb2, 0xf6, 0x85, 0x3c, - 0x37, 0x4e, 0x9d, 0x7a, 0x7e, 0xfe, 0x11, 0xf6, 0xd3, 0xcf, 0x40, 0xef, 0x43, 0x27, 0x75, 0x47, - 0x94, 0x2d, 0xf9, 0xec, 0x8b, 0xa4, 0xe3, 0x76, 0xff, 0x0c, 0x43, 0xb5, 0x09, 0xcb, 0x19, 0x27, - 0xff, 0x28, 0x33, 0xbd, 0xe4, 0x5f, 0x11, 0x1c, 0xcf, 0x50, 0x2b, 0x61, 0xee, 0x68, 0x2d, 0x8f, - 0xc8, 0xf4, 0xbb, 0xd3, 0xde, 0x4b, 0x8b, 0x3d, 0x52, 0x0d, 0x19, 0xea, 0x43, 0x85, 0x5f, 0x3a, - 0xa1, 0x67, 0xb3, 0xbb, 0x96, 0xd8, 0x85, 0x54, 0xef, 0xb8, 0x6b, 0x2b, 0x32, 0xb3, 0x03, 0xc2, - 0x36, 0x2d, 0xb3, 0x48, 0x86, 0x32, 0xef, 0x2c, 0xe3, 0x37, 0x45, 0xbd, 0xe3, 0x2f, 0x87, 0xe4, - 0xa6, 0xff, 0xd5, 0xf9, 0xec, 0xc6, 0xab, 0xf7, 0x37, 0x27, 0x56, 0x70, 0x30, 0x1b, 0x52, 0x63, - 0xb8, 0xc2, 0x67, 0xbe, 0x6c, 0x79, 0xe2, 0xeb, 0x8a, 0xa4, 0xf2, 0x0a, 0xdb, 0xe9, 0x0a, 0x13, - 0xe4, 0x74, 0x38, 0xac, 0xb0, 0xdf, 0x6b, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0xf4, 0x19, 0x46, - 0xe1, 0x75, 0x31, 0x00, 0x00, +func (m *GetDataDistributionRequest) Reset() { *m = GetDataDistributionRequest{} } +func (m *GetDataDistributionRequest) String() string { return proto.CompactTextString(m) } +func (*GetDataDistributionRequest) ProtoMessage() {} +func (*GetDataDistributionRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_aab7cc9a69ed26e8, []int{38} } -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn +func (m *GetDataDistributionRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetDataDistributionRequest.Unmarshal(m, b) +} +func (m *GetDataDistributionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetDataDistributionRequest.Marshal(b, m, deterministic) +} +func (m *GetDataDistributionRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetDataDistributionRequest.Merge(m, src) +} +func (m *GetDataDistributionRequest) XXX_Size() int { + return xxx_messageInfo_GetDataDistributionRequest.Size(m) +} +func (m *GetDataDistributionRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetDataDistributionRequest.DiscardUnknown(m) +} -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 +var xxx_messageInfo_GetDataDistributionRequest proto.InternalMessageInfo -// QueryCoordClient is the client API for QueryCoord service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type QueryCoordClient interface { - GetComponentStates(ctx context.Context, in *internalpb.GetComponentStatesRequest, opts ...grpc.CallOption) (*internalpb.ComponentStates, error) - GetTimeTickChannel(ctx context.Context, in *internalpb.GetTimeTickChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) - GetStatisticsChannel(ctx context.Context, in *internalpb.GetStatisticsChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) - ShowCollections(ctx context.Context, in *ShowCollectionsRequest, opts ...grpc.CallOption) (*ShowCollectionsResponse, error) - ShowPartitions(ctx context.Context, in *ShowPartitionsRequest, opts ...grpc.CallOption) (*ShowPartitionsResponse, error) - LoadPartitions(ctx context.Context, in *LoadPartitionsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) - ReleasePartitions(ctx context.Context, in *ReleasePartitionsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) - LoadCollection(ctx context.Context, in *LoadCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) - ReleaseCollection(ctx context.Context, in *ReleaseCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) - GetPartitionStates(ctx context.Context, in *GetPartitionStatesRequest, opts ...grpc.CallOption) (*GetPartitionStatesResponse, error) - GetSegmentInfo(ctx context.Context, in *GetSegmentInfoRequest, opts ...grpc.CallOption) (*GetSegmentInfoResponse, error) - LoadBalance(ctx context.Context, in *LoadBalanceRequest, opts ...grpc.CallOption) (*commonpb.Status, error) - ShowConfigurations(ctx context.Context, in *internalpb.ShowConfigurationsRequest, opts ...grpc.CallOption) (*internalpb.ShowConfigurationsResponse, error) - // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy - GetMetrics(ctx context.Context, in *milvuspb.GetMetricsRequest, opts ...grpc.CallOption) (*milvuspb.GetMetricsResponse, error) - // https://wiki.lfaidata.foundation/display/MIL/MEP+23+--+Multiple+memory+replication+design - GetReplicas(ctx context.Context, in *milvuspb.GetReplicasRequest, opts ...grpc.CallOption) (*milvuspb.GetReplicasResponse, error) - GetShardLeaders(ctx context.Context, in *GetShardLeadersRequest, opts ...grpc.CallOption) (*GetShardLeadersResponse, error) +func (m *GetDataDistributionRequest) GetBase() *commonpb.MsgBase { + if m != nil { + return m.Base + } + return nil } -type queryCoordClient struct { - cc *grpc.ClientConn +type GetDataDistributionResponse struct { + Status *commonpb.Status `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` + NodeID int64 `protobuf:"varint,2,opt,name=nodeID,proto3" json:"nodeID,omitempty"` + GrowingSegmentIDs []int64 `protobuf:"varint,3,rep,packed,name=growing_segmentIDs,json=growingSegmentIDs,proto3" json:"growing_segmentIDs,omitempty"` + Segments []*SegmentVersionInfo `protobuf:"bytes,4,rep,name=segments,proto3" json:"segments,omitempty"` + Channels []*ChannelVersionInfo `protobuf:"bytes,5,rep,name=channels,proto3" json:"channels,omitempty"` + LeaderViews []*LeaderView `protobuf:"bytes,6,rep,name=leaderViews,proto3" json:"leaderViews,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func NewQueryCoordClient(cc *grpc.ClientConn) QueryCoordClient { - return &queryCoordClient{cc} +func (m *GetDataDistributionResponse) Reset() { *m = GetDataDistributionResponse{} } +func (m *GetDataDistributionResponse) String() string { return proto.CompactTextString(m) } +func (*GetDataDistributionResponse) ProtoMessage() {} +func (*GetDataDistributionResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_aab7cc9a69ed26e8, []int{39} } -func (c *queryCoordClient) GetComponentStates(ctx context.Context, in *internalpb.GetComponentStatesRequest, opts ...grpc.CallOption) (*internalpb.ComponentStates, error) { - out := new(internalpb.ComponentStates) - err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryCoord/GetComponentStates", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil +func (m *GetDataDistributionResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetDataDistributionResponse.Unmarshal(m, b) +} +func (m *GetDataDistributionResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetDataDistributionResponse.Marshal(b, m, deterministic) +} +func (m *GetDataDistributionResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetDataDistributionResponse.Merge(m, src) +} +func (m *GetDataDistributionResponse) XXX_Size() int { + return xxx_messageInfo_GetDataDistributionResponse.Size(m) +} +func (m *GetDataDistributionResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetDataDistributionResponse.DiscardUnknown(m) } -func (c *queryCoordClient) GetTimeTickChannel(ctx context.Context, in *internalpb.GetTimeTickChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) { - out := new(milvuspb.StringResponse) - err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryCoord/GetTimeTickChannel", in, out, opts...) - if err != nil { - return nil, err +var xxx_messageInfo_GetDataDistributionResponse proto.InternalMessageInfo + +func (m *GetDataDistributionResponse) GetStatus() *commonpb.Status { + if m != nil { + return m.Status } - return out, nil + return nil } -func (c *queryCoordClient) GetStatisticsChannel(ctx context.Context, in *internalpb.GetStatisticsChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) { - out := new(milvuspb.StringResponse) - err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryCoord/GetStatisticsChannel", in, out, opts...) - if err != nil { - return nil, err +func (m *GetDataDistributionResponse) GetNodeID() int64 { + if m != nil { + return m.NodeID } - return out, nil + return 0 } -func (c *queryCoordClient) ShowCollections(ctx context.Context, in *ShowCollectionsRequest, opts ...grpc.CallOption) (*ShowCollectionsResponse, error) { - out := new(ShowCollectionsResponse) - err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryCoord/ShowCollections", in, out, opts...) - if err != nil { - return nil, err +func (m *GetDataDistributionResponse) GetGrowingSegmentIDs() []int64 { + if m != nil { + return m.GrowingSegmentIDs } - return out, nil + return nil +} + +func (m *GetDataDistributionResponse) GetSegments() []*SegmentVersionInfo { + if m != nil { + return m.Segments + } + return nil +} + +func (m *GetDataDistributionResponse) GetChannels() []*ChannelVersionInfo { + if m != nil { + return m.Channels + } + return nil +} + +func (m *GetDataDistributionResponse) GetLeaderViews() []*LeaderView { + if m != nil { + return m.LeaderViews + } + return nil +} + +type LeaderView struct { + Collection int64 `protobuf:"varint,1,opt,name=collection,proto3" json:"collection,omitempty"` + Channel string `protobuf:"bytes,2,opt,name=channel,proto3" json:"channel,omitempty"` + SegmentNodePairs map[int64]int64 `protobuf:"bytes,3,rep,name=segmentNodePairs,proto3" json:"segmentNodePairs,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *LeaderView) Reset() { *m = LeaderView{} } +func (m *LeaderView) String() string { return proto.CompactTextString(m) } +func (*LeaderView) ProtoMessage() {} +func (*LeaderView) Descriptor() ([]byte, []int) { + return fileDescriptor_aab7cc9a69ed26e8, []int{40} +} + +func (m *LeaderView) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_LeaderView.Unmarshal(m, b) +} +func (m *LeaderView) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_LeaderView.Marshal(b, m, deterministic) +} +func (m *LeaderView) XXX_Merge(src proto.Message) { + xxx_messageInfo_LeaderView.Merge(m, src) +} +func (m *LeaderView) XXX_Size() int { + return xxx_messageInfo_LeaderView.Size(m) +} +func (m *LeaderView) XXX_DiscardUnknown() { + xxx_messageInfo_LeaderView.DiscardUnknown(m) +} + +var xxx_messageInfo_LeaderView proto.InternalMessageInfo + +func (m *LeaderView) GetCollection() int64 { + if m != nil { + return m.Collection + } + return 0 +} + +func (m *LeaderView) GetChannel() string { + if m != nil { + return m.Channel + } + return "" +} + +func (m *LeaderView) GetSegmentNodePairs() map[int64]int64 { + if m != nil { + return m.SegmentNodePairs + } + return nil +} + +type SegmentVersionInfo struct { + ID int64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + Collection int64 `protobuf:"varint,2,opt,name=collection,proto3" json:"collection,omitempty"` + Partition int64 `protobuf:"varint,3,opt,name=partition,proto3" json:"partition,omitempty"` + Channel string `protobuf:"bytes,4,opt,name=channel,proto3" json:"channel,omitempty"` + Version int64 `protobuf:"varint,5,opt,name=version,proto3" json:"version,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SegmentVersionInfo) Reset() { *m = SegmentVersionInfo{} } +func (m *SegmentVersionInfo) String() string { return proto.CompactTextString(m) } +func (*SegmentVersionInfo) ProtoMessage() {} +func (*SegmentVersionInfo) Descriptor() ([]byte, []int) { + return fileDescriptor_aab7cc9a69ed26e8, []int{41} +} + +func (m *SegmentVersionInfo) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SegmentVersionInfo.Unmarshal(m, b) +} +func (m *SegmentVersionInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SegmentVersionInfo.Marshal(b, m, deterministic) +} +func (m *SegmentVersionInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_SegmentVersionInfo.Merge(m, src) +} +func (m *SegmentVersionInfo) XXX_Size() int { + return xxx_messageInfo_SegmentVersionInfo.Size(m) +} +func (m *SegmentVersionInfo) XXX_DiscardUnknown() { + xxx_messageInfo_SegmentVersionInfo.DiscardUnknown(m) +} + +var xxx_messageInfo_SegmentVersionInfo proto.InternalMessageInfo + +func (m *SegmentVersionInfo) GetID() int64 { + if m != nil { + return m.ID + } + return 0 +} + +func (m *SegmentVersionInfo) GetCollection() int64 { + if m != nil { + return m.Collection + } + return 0 +} + +func (m *SegmentVersionInfo) GetPartition() int64 { + if m != nil { + return m.Partition + } + return 0 +} + +func (m *SegmentVersionInfo) GetChannel() string { + if m != nil { + return m.Channel + } + return "" +} + +func (m *SegmentVersionInfo) GetVersion() int64 { + if m != nil { + return m.Version + } + return 0 +} + +type ChannelVersionInfo struct { + Channel string `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"` + Collection int64 `protobuf:"varint,2,opt,name=collection,proto3" json:"collection,omitempty"` + Version int64 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ChannelVersionInfo) Reset() { *m = ChannelVersionInfo{} } +func (m *ChannelVersionInfo) String() string { return proto.CompactTextString(m) } +func (*ChannelVersionInfo) ProtoMessage() {} +func (*ChannelVersionInfo) Descriptor() ([]byte, []int) { + return fileDescriptor_aab7cc9a69ed26e8, []int{42} +} + +func (m *ChannelVersionInfo) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ChannelVersionInfo.Unmarshal(m, b) +} +func (m *ChannelVersionInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ChannelVersionInfo.Marshal(b, m, deterministic) +} +func (m *ChannelVersionInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_ChannelVersionInfo.Merge(m, src) +} +func (m *ChannelVersionInfo) XXX_Size() int { + return xxx_messageInfo_ChannelVersionInfo.Size(m) +} +func (m *ChannelVersionInfo) XXX_DiscardUnknown() { + xxx_messageInfo_ChannelVersionInfo.DiscardUnknown(m) +} + +var xxx_messageInfo_ChannelVersionInfo proto.InternalMessageInfo + +func (m *ChannelVersionInfo) GetChannel() string { + if m != nil { + return m.Channel + } + return "" +} + +func (m *ChannelVersionInfo) GetCollection() int64 { + if m != nil { + return m.Collection + } + return 0 +} + +func (m *ChannelVersionInfo) GetVersion() int64 { + if m != nil { + return m.Version + } + return 0 +} + +type CollectionLoadInfo struct { + CollectionID int64 `protobuf:"varint,1,opt,name=collectionID,proto3" json:"collectionID,omitempty"` + ReleasedPartitions []int64 `protobuf:"varint,2,rep,packed,name=released_partitions,json=releasedPartitions,proto3" json:"released_partitions,omitempty"` + ReplicaNumber int32 `protobuf:"varint,3,opt,name=replica_number,json=replicaNumber,proto3" json:"replica_number,omitempty"` + Status LoadStatus `protobuf:"varint,4,opt,name=status,proto3,enum=milvus.proto.query.LoadStatus" json:"status,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CollectionLoadInfo) Reset() { *m = CollectionLoadInfo{} } +func (m *CollectionLoadInfo) String() string { return proto.CompactTextString(m) } +func (*CollectionLoadInfo) ProtoMessage() {} +func (*CollectionLoadInfo) Descriptor() ([]byte, []int) { + return fileDescriptor_aab7cc9a69ed26e8, []int{43} +} + +func (m *CollectionLoadInfo) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CollectionLoadInfo.Unmarshal(m, b) +} +func (m *CollectionLoadInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CollectionLoadInfo.Marshal(b, m, deterministic) +} +func (m *CollectionLoadInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_CollectionLoadInfo.Merge(m, src) +} +func (m *CollectionLoadInfo) XXX_Size() int { + return xxx_messageInfo_CollectionLoadInfo.Size(m) +} +func (m *CollectionLoadInfo) XXX_DiscardUnknown() { + xxx_messageInfo_CollectionLoadInfo.DiscardUnknown(m) +} + +var xxx_messageInfo_CollectionLoadInfo proto.InternalMessageInfo + +func (m *CollectionLoadInfo) GetCollectionID() int64 { + if m != nil { + return m.CollectionID + } + return 0 +} + +func (m *CollectionLoadInfo) GetReleasedPartitions() []int64 { + if m != nil { + return m.ReleasedPartitions + } + return nil +} + +func (m *CollectionLoadInfo) GetReplicaNumber() int32 { + if m != nil { + return m.ReplicaNumber + } + return 0 +} + +func (m *CollectionLoadInfo) GetStatus() LoadStatus { + if m != nil { + return m.Status + } + return LoadStatus_Invalid +} + +type PartitionLoadInfo struct { + CollectionID int64 `protobuf:"varint,1,opt,name=collectionID,proto3" json:"collectionID,omitempty"` + PartitionID int64 `protobuf:"varint,2,opt,name=partitionID,proto3" json:"partitionID,omitempty"` + ReplicaNumber int32 `protobuf:"varint,3,opt,name=replica_number,json=replicaNumber,proto3" json:"replica_number,omitempty"` + Status LoadStatus `protobuf:"varint,4,opt,name=status,proto3,enum=milvus.proto.query.LoadStatus" json:"status,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PartitionLoadInfo) Reset() { *m = PartitionLoadInfo{} } +func (m *PartitionLoadInfo) String() string { return proto.CompactTextString(m) } +func (*PartitionLoadInfo) ProtoMessage() {} +func (*PartitionLoadInfo) Descriptor() ([]byte, []int) { + return fileDescriptor_aab7cc9a69ed26e8, []int{44} +} + +func (m *PartitionLoadInfo) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PartitionLoadInfo.Unmarshal(m, b) +} +func (m *PartitionLoadInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PartitionLoadInfo.Marshal(b, m, deterministic) +} +func (m *PartitionLoadInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_PartitionLoadInfo.Merge(m, src) +} +func (m *PartitionLoadInfo) XXX_Size() int { + return xxx_messageInfo_PartitionLoadInfo.Size(m) +} +func (m *PartitionLoadInfo) XXX_DiscardUnknown() { + xxx_messageInfo_PartitionLoadInfo.DiscardUnknown(m) +} + +var xxx_messageInfo_PartitionLoadInfo proto.InternalMessageInfo + +func (m *PartitionLoadInfo) GetCollectionID() int64 { + if m != nil { + return m.CollectionID + } + return 0 +} + +func (m *PartitionLoadInfo) GetPartitionID() int64 { + if m != nil { + return m.PartitionID + } + return 0 +} + +func (m *PartitionLoadInfo) GetReplicaNumber() int32 { + if m != nil { + return m.ReplicaNumber + } + return 0 +} + +func (m *PartitionLoadInfo) GetStatus() LoadStatus { + if m != nil { + return m.Status + } + return LoadStatus_Invalid +} + +type Replica struct { + ID int64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + CollectionID int64 `protobuf:"varint,2,opt,name=collectionID,proto3" json:"collectionID,omitempty"` + Nodes []int64 `protobuf:"varint,3,rep,packed,name=nodes,proto3" json:"nodes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Replica) Reset() { *m = Replica{} } +func (m *Replica) String() string { return proto.CompactTextString(m) } +func (*Replica) ProtoMessage() {} +func (*Replica) Descriptor() ([]byte, []int) { + return fileDescriptor_aab7cc9a69ed26e8, []int{45} +} + +func (m *Replica) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Replica.Unmarshal(m, b) +} +func (m *Replica) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Replica.Marshal(b, m, deterministic) +} +func (m *Replica) XXX_Merge(src proto.Message) { + xxx_messageInfo_Replica.Merge(m, src) +} +func (m *Replica) XXX_Size() int { + return xxx_messageInfo_Replica.Size(m) +} +func (m *Replica) XXX_DiscardUnknown() { + xxx_messageInfo_Replica.DiscardUnknown(m) +} + +var xxx_messageInfo_Replica proto.InternalMessageInfo + +func (m *Replica) GetID() int64 { + if m != nil { + return m.ID + } + return 0 +} + +func (m *Replica) GetCollectionID() int64 { + if m != nil { + return m.CollectionID + } + return 0 +} + +func (m *Replica) GetNodes() []int64 { + if m != nil { + return m.Nodes + } + return nil +} + +type SyncAction struct { + Type SyncType `protobuf:"varint,1,opt,name=type,proto3,enum=milvus.proto.query.SyncType" json:"type,omitempty"` + PartitionID int64 `protobuf:"varint,2,opt,name=partitionID,proto3" json:"partitionID,omitempty"` + SegmentID int64 `protobuf:"varint,3,opt,name=segmentID,proto3" json:"segmentID,omitempty"` + NodeID int64 `protobuf:"varint,4,opt,name=nodeID,proto3" json:"nodeID,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SyncAction) Reset() { *m = SyncAction{} } +func (m *SyncAction) String() string { return proto.CompactTextString(m) } +func (*SyncAction) ProtoMessage() {} +func (*SyncAction) Descriptor() ([]byte, []int) { + return fileDescriptor_aab7cc9a69ed26e8, []int{46} +} + +func (m *SyncAction) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SyncAction.Unmarshal(m, b) +} +func (m *SyncAction) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SyncAction.Marshal(b, m, deterministic) +} +func (m *SyncAction) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncAction.Merge(m, src) +} +func (m *SyncAction) XXX_Size() int { + return xxx_messageInfo_SyncAction.Size(m) +} +func (m *SyncAction) XXX_DiscardUnknown() { + xxx_messageInfo_SyncAction.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncAction proto.InternalMessageInfo + +func (m *SyncAction) GetType() SyncType { + if m != nil { + return m.Type + } + return SyncType_Remove +} + +func (m *SyncAction) GetPartitionID() int64 { + if m != nil { + return m.PartitionID + } + return 0 +} + +func (m *SyncAction) GetSegmentID() int64 { + if m != nil { + return m.SegmentID + } + return 0 +} + +func (m *SyncAction) GetNodeID() int64 { + if m != nil { + return m.NodeID + } + return 0 +} + +type SyncDistributionRequest struct { + Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` + CollectionID int64 `protobuf:"varint,2,opt,name=collectionID,proto3" json:"collectionID,omitempty"` + Channel string `protobuf:"bytes,3,opt,name=channel,proto3" json:"channel,omitempty"` + Actions []*SyncAction `protobuf:"bytes,4,rep,name=actions,proto3" json:"actions,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SyncDistributionRequest) Reset() { *m = SyncDistributionRequest{} } +func (m *SyncDistributionRequest) String() string { return proto.CompactTextString(m) } +func (*SyncDistributionRequest) ProtoMessage() {} +func (*SyncDistributionRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_aab7cc9a69ed26e8, []int{47} +} + +func (m *SyncDistributionRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SyncDistributionRequest.Unmarshal(m, b) +} +func (m *SyncDistributionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SyncDistributionRequest.Marshal(b, m, deterministic) +} +func (m *SyncDistributionRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncDistributionRequest.Merge(m, src) +} +func (m *SyncDistributionRequest) XXX_Size() int { + return xxx_messageInfo_SyncDistributionRequest.Size(m) +} +func (m *SyncDistributionRequest) XXX_DiscardUnknown() { + xxx_messageInfo_SyncDistributionRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncDistributionRequest proto.InternalMessageInfo + +func (m *SyncDistributionRequest) GetBase() *commonpb.MsgBase { + if m != nil { + return m.Base + } + return nil +} + +func (m *SyncDistributionRequest) GetCollectionID() int64 { + if m != nil { + return m.CollectionID + } + return 0 +} + +func (m *SyncDistributionRequest) GetChannel() string { + if m != nil { + return m.Channel + } + return "" +} + +func (m *SyncDistributionRequest) GetActions() []*SyncAction { + if m != nil { + return m.Actions + } + return nil +} + +func init() { + proto.RegisterEnum("milvus.proto.query.DataScope", DataScope_name, DataScope_value) + proto.RegisterEnum("milvus.proto.query.PartitionState", PartitionState_name, PartitionState_value) + proto.RegisterEnum("milvus.proto.query.TriggerCondition", TriggerCondition_name, TriggerCondition_value) + proto.RegisterEnum("milvus.proto.query.LoadType", LoadType_name, LoadType_value) + proto.RegisterEnum("milvus.proto.query.LoadStatus", LoadStatus_name, LoadStatus_value) + proto.RegisterEnum("milvus.proto.query.SyncType", SyncType_name, SyncType_value) + proto.RegisterType((*ShowCollectionsRequest)(nil), "milvus.proto.query.ShowCollectionsRequest") + proto.RegisterType((*ShowCollectionsResponse)(nil), "milvus.proto.query.ShowCollectionsResponse") + proto.RegisterType((*ShowPartitionsRequest)(nil), "milvus.proto.query.ShowPartitionsRequest") + proto.RegisterType((*ShowPartitionsResponse)(nil), "milvus.proto.query.ShowPartitionsResponse") + proto.RegisterType((*LoadCollectionRequest)(nil), "milvus.proto.query.LoadCollectionRequest") + proto.RegisterType((*ReleaseCollectionRequest)(nil), "milvus.proto.query.ReleaseCollectionRequest") + proto.RegisterType((*GetStatisticsRequest)(nil), "milvus.proto.query.GetStatisticsRequest") + proto.RegisterType((*LoadPartitionsRequest)(nil), "milvus.proto.query.LoadPartitionsRequest") + proto.RegisterType((*ReleasePartitionsRequest)(nil), "milvus.proto.query.ReleasePartitionsRequest") + proto.RegisterType((*GetPartitionStatesRequest)(nil), "milvus.proto.query.GetPartitionStatesRequest") + proto.RegisterType((*GetPartitionStatesResponse)(nil), "milvus.proto.query.GetPartitionStatesResponse") + proto.RegisterType((*GetSegmentInfoRequest)(nil), "milvus.proto.query.GetSegmentInfoRequest") + proto.RegisterType((*GetSegmentInfoResponse)(nil), "milvus.proto.query.GetSegmentInfoResponse") + proto.RegisterType((*GetShardLeadersRequest)(nil), "milvus.proto.query.GetShardLeadersRequest") + proto.RegisterType((*GetShardLeadersResponse)(nil), "milvus.proto.query.GetShardLeadersResponse") + proto.RegisterType((*ShardLeadersList)(nil), "milvus.proto.query.ShardLeadersList") + proto.RegisterType((*LoadMetaInfo)(nil), "milvus.proto.query.LoadMetaInfo") + proto.RegisterType((*WatchDmChannelsRequest)(nil), "milvus.proto.query.WatchDmChannelsRequest") + proto.RegisterMapType((map[int64]*datapb.SegmentInfo)(nil), "milvus.proto.query.WatchDmChannelsRequest.SegmentInfosEntry") + proto.RegisterType((*UnsubDmChannelRequest)(nil), "milvus.proto.query.UnsubDmChannelRequest") + proto.RegisterType((*SegmentLoadInfo)(nil), "milvus.proto.query.SegmentLoadInfo") + proto.RegisterType((*FieldIndexInfo)(nil), "milvus.proto.query.FieldIndexInfo") + proto.RegisterType((*LoadSegmentsRequest)(nil), "milvus.proto.query.LoadSegmentsRequest") + proto.RegisterType((*ReleaseSegmentsRequest)(nil), "milvus.proto.query.ReleaseSegmentsRequest") + proto.RegisterType((*SearchRequest)(nil), "milvus.proto.query.SearchRequest") + proto.RegisterType((*QueryRequest)(nil), "milvus.proto.query.QueryRequest") + proto.RegisterType((*SyncReplicaSegmentsRequest)(nil), "milvus.proto.query.SyncReplicaSegmentsRequest") + proto.RegisterType((*ReplicaSegmentsInfo)(nil), "milvus.proto.query.ReplicaSegmentsInfo") + proto.RegisterType((*HandoffSegmentsRequest)(nil), "milvus.proto.query.HandoffSegmentsRequest") + proto.RegisterType((*LoadBalanceRequest)(nil), "milvus.proto.query.LoadBalanceRequest") + proto.RegisterType((*DmChannelWatchInfo)(nil), "milvus.proto.query.DmChannelWatchInfo") + proto.RegisterType((*QueryChannelInfo)(nil), "milvus.proto.query.QueryChannelInfo") + proto.RegisterType((*PartitionStates)(nil), "milvus.proto.query.PartitionStates") + proto.RegisterType((*SegmentInfo)(nil), "milvus.proto.query.SegmentInfo") + proto.RegisterType((*CollectionInfo)(nil), "milvus.proto.query.CollectionInfo") + proto.RegisterType((*UnsubscribeChannels)(nil), "milvus.proto.query.UnsubscribeChannels") + proto.RegisterType((*UnsubscribeChannelInfo)(nil), "milvus.proto.query.UnsubscribeChannelInfo") + proto.RegisterType((*SegmentChangeInfo)(nil), "milvus.proto.query.SegmentChangeInfo") + proto.RegisterType((*SealedSegmentsChangeInfo)(nil), "milvus.proto.query.SealedSegmentsChangeInfo") + proto.RegisterType((*GetDataDistributionRequest)(nil), "milvus.proto.query.GetDataDistributionRequest") + proto.RegisterType((*GetDataDistributionResponse)(nil), "milvus.proto.query.GetDataDistributionResponse") + proto.RegisterType((*LeaderView)(nil), "milvus.proto.query.LeaderView") + proto.RegisterMapType((map[int64]int64)(nil), "milvus.proto.query.LeaderView.SegmentNodePairsEntry") + proto.RegisterType((*SegmentVersionInfo)(nil), "milvus.proto.query.SegmentVersionInfo") + proto.RegisterType((*ChannelVersionInfo)(nil), "milvus.proto.query.ChannelVersionInfo") + proto.RegisterType((*CollectionLoadInfo)(nil), "milvus.proto.query.CollectionLoadInfo") + proto.RegisterType((*PartitionLoadInfo)(nil), "milvus.proto.query.PartitionLoadInfo") + proto.RegisterType((*Replica)(nil), "milvus.proto.query.Replica") + proto.RegisterType((*SyncAction)(nil), "milvus.proto.query.SyncAction") + proto.RegisterType((*SyncDistributionRequest)(nil), "milvus.proto.query.SyncDistributionRequest") +} + +func init() { proto.RegisterFile("query_coord.proto", fileDescriptor_aab7cc9a69ed26e8) } + +var fileDescriptor_aab7cc9a69ed26e8 = []byte{ + // 3485 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x1b, 0x49, 0x8f, 0x1c, 0x57, + 0x79, 0xaa, 0x97, 0x99, 0xee, 0xaf, 0x97, 0xa9, 0x79, 0xb3, 0xb8, 0xd3, 0xf1, 0x96, 0x72, 0xec, + 0x0c, 0xe3, 0x64, 0xec, 0x8c, 0x43, 0xe4, 0x40, 0x22, 0xc5, 0x9e, 0x89, 0x27, 0x83, 0xed, 0xc9, + 0x50, 0x6d, 0x1b, 0x64, 0x45, 0x74, 0xaa, 0xbb, 0xde, 0xf4, 0x94, 0x5c, 0x4b, 0xbb, 0xaa, 0x7a, + 0xec, 0x09, 0x57, 0x2e, 0x6c, 0x07, 0x38, 0x70, 0x02, 0x4e, 0x20, 0x81, 0x94, 0x08, 0x21, 0x71, + 0xe0, 0x80, 0x10, 0x12, 0x07, 0x38, 0x21, 0x7e, 0x00, 0x12, 0x07, 0x0e, 0x1c, 0x80, 0x23, 0x07, + 0x6e, 0xe8, 0x6d, 0xb5, 0xd7, 0x74, 0x7b, 0x26, 0xce, 0x82, 0xb8, 0x75, 0x7d, 0x6f, 0xf9, 0xbe, + 0xf7, 0xed, 0xdf, 0xf7, 0x5e, 0xc3, 0xdc, 0xc3, 0x11, 0x76, 0x0f, 0xba, 0x7d, 0xc7, 0x71, 0xf5, + 0xd5, 0xa1, 0xeb, 0xf8, 0x0e, 0x42, 0x96, 0x61, 0xee, 0x8f, 0x3c, 0xf6, 0xb5, 0x4a, 0xc7, 0xdb, + 0xf5, 0xbe, 0x63, 0x59, 0x8e, 0xcd, 0x60, 0xed, 0x7a, 0x74, 0x46, 0xbb, 0x69, 0xd8, 0x3e, 0x76, + 0x6d, 0xcd, 0x14, 0xa3, 0x5e, 0x7f, 0x0f, 0x5b, 0x1a, 0xff, 0x92, 0x75, 0xcd, 0xd7, 0xa2, 0xfb, + 0x2b, 0xdf, 0x90, 0x60, 0xa9, 0xb3, 0xe7, 0x3c, 0x5a, 0x77, 0x4c, 0x13, 0xf7, 0x7d, 0xc3, 0xb1, + 0x3d, 0x15, 0x3f, 0x1c, 0x61, 0xcf, 0x47, 0x97, 0xa1, 0xd4, 0xd3, 0x3c, 0xdc, 0x92, 0xce, 0x4a, + 0xcb, 0xb5, 0xb5, 0x93, 0xab, 0x31, 0x4a, 0x38, 0x09, 0xb7, 0xbd, 0xc1, 0x75, 0xcd, 0xc3, 0x2a, + 0x9d, 0x89, 0x10, 0x94, 0xf4, 0xde, 0xd6, 0x46, 0xab, 0x70, 0x56, 0x5a, 0x2e, 0xaa, 0xf4, 0x37, + 0x7a, 0x1e, 0x1a, 0xfd, 0x60, 0xef, 0xad, 0x0d, 0xaf, 0x55, 0x3c, 0x5b, 0x5c, 0x2e, 0xaa, 0x71, + 0xa0, 0xf2, 0x57, 0x09, 0x4e, 0xa4, 0xc8, 0xf0, 0x86, 0x8e, 0xed, 0x61, 0x74, 0x05, 0xa6, 0x3d, + 0x5f, 0xf3, 0x47, 0x1e, 0xa7, 0xe4, 0xd9, 0x4c, 0x4a, 0x3a, 0x74, 0x8a, 0xca, 0xa7, 0xa6, 0xd1, + 0x16, 0x32, 0xd0, 0xa2, 0x97, 0x61, 0xc1, 0xb0, 0x6f, 0x63, 0xcb, 0x71, 0x0f, 0xba, 0x43, 0xec, + 0xf6, 0xb1, 0xed, 0x6b, 0x03, 0x2c, 0x68, 0x9c, 0x17, 0x63, 0x3b, 0xe1, 0x10, 0x7a, 0x15, 0x4e, + 0x30, 0x29, 0x79, 0xd8, 0xdd, 0x37, 0xfa, 0xb8, 0xab, 0xed, 0x6b, 0x86, 0xa9, 0xf5, 0x4c, 0xdc, + 0x2a, 0x9d, 0x2d, 0x2e, 0x57, 0xd4, 0x45, 0x3a, 0xdc, 0x61, 0xa3, 0xd7, 0xc4, 0xa0, 0xf2, 0x53, + 0x09, 0x16, 0xc9, 0x09, 0x77, 0x34, 0xd7, 0x37, 0x9e, 0x02, 0x9f, 0x15, 0xa8, 0x47, 0xcf, 0xd6, + 0x2a, 0xd2, 0xb1, 0x18, 0x8c, 0xcc, 0x19, 0x0a, 0xf4, 0x84, 0x27, 0x25, 0x7a, 0xcc, 0x18, 0x4c, + 0xf9, 0x09, 0x57, 0x88, 0x28, 0x9d, 0xc7, 0x11, 0x44, 0x12, 0x67, 0x21, 0x8d, 0xf3, 0x08, 0x62, + 0x50, 0xfe, 0x2e, 0xc1, 0xe2, 0x2d, 0x47, 0xd3, 0x43, 0x85, 0xf9, 0xf8, 0xd9, 0xf9, 0x06, 0x4c, + 0x33, 0xeb, 0x6a, 0x95, 0x28, 0xae, 0xf3, 0x71, 0x5c, 0xdc, 0xf2, 0x42, 0x0a, 0x3b, 0x14, 0xa0, + 0xf2, 0x45, 0xe8, 0x3c, 0x34, 0x5d, 0x3c, 0x34, 0x8d, 0xbe, 0xd6, 0xb5, 0x47, 0x56, 0x0f, 0xbb, + 0xad, 0xf2, 0x59, 0x69, 0xb9, 0xac, 0x36, 0x38, 0x74, 0x9b, 0x02, 0x95, 0x1f, 0x4a, 0xd0, 0x52, + 0xb1, 0x89, 0x35, 0x0f, 0x7f, 0x92, 0x87, 0x5d, 0x82, 0x69, 0xdb, 0xd1, 0xf1, 0xd6, 0x06, 0x3d, + 0x6c, 0x51, 0xe5, 0x5f, 0xca, 0x7f, 0x24, 0x58, 0xd8, 0xc4, 0x3e, 0x91, 0xba, 0xe1, 0xf9, 0x46, + 0x3f, 0x50, 0xeb, 0x37, 0xa0, 0xe8, 0xe2, 0x87, 0x9c, 0xb2, 0x8b, 0x71, 0xca, 0x02, 0x27, 0x95, + 0xb5, 0x52, 0x25, 0xeb, 0xd0, 0x73, 0x50, 0xd7, 0x2d, 0xb3, 0xdb, 0xdf, 0xd3, 0x6c, 0x1b, 0x9b, + 0x4c, 0x6f, 0xaa, 0x6a, 0x4d, 0xb7, 0xcc, 0x75, 0x0e, 0x42, 0xa7, 0x01, 0x3c, 0x3c, 0xb0, 0xb0, + 0xed, 0x87, 0x7e, 0x25, 0x02, 0x41, 0x2b, 0x30, 0xb7, 0xeb, 0x3a, 0x56, 0xd7, 0xdb, 0xd3, 0x5c, + 0xbd, 0x6b, 0x62, 0x4d, 0xc7, 0x2e, 0xa5, 0xbe, 0xa2, 0xce, 0x92, 0x81, 0x0e, 0x81, 0xdf, 0xa2, + 0x60, 0x74, 0x05, 0xca, 0x5e, 0xdf, 0x19, 0x62, 0x2a, 0x83, 0xe6, 0xda, 0xa9, 0xd5, 0xb4, 0xdf, + 0x5d, 0xdd, 0xd0, 0x7c, 0xad, 0x43, 0x26, 0xa9, 0x6c, 0xae, 0xf2, 0xed, 0x02, 0x53, 0xc2, 0x4f, + 0xb9, 0x4d, 0x47, 0x14, 0xb5, 0xfc, 0xd1, 0x28, 0xea, 0x74, 0x96, 0xa2, 0xfe, 0x2e, 0x54, 0xd4, + 0x4f, 0x3b, 0x43, 0x42, 0x65, 0x2e, 0xc7, 0x94, 0xf9, 0xe7, 0x12, 0x3c, 0xb3, 0x89, 0xfd, 0x80, + 0x7c, 0xa2, 0x9b, 0xf8, 0x53, 0xea, 0xa8, 0x3f, 0x94, 0xa0, 0x9d, 0x45, 0xeb, 0x71, 0x9c, 0xf5, + 0x7d, 0x58, 0x0a, 0x70, 0x74, 0x75, 0xec, 0xf5, 0x5d, 0x63, 0x48, 0xc5, 0x48, 0xcd, 0xaf, 0xb6, + 0x76, 0x2e, 0xcb, 0x2c, 0x92, 0x14, 0x2c, 0x06, 0x5b, 0x6c, 0x44, 0x76, 0x50, 0xbe, 0x2b, 0xc1, + 0x22, 0x31, 0x77, 0x6e, 0x9f, 0xf6, 0xae, 0x73, 0x74, 0xbe, 0xc6, 0x2d, 0xbf, 0x90, 0xb2, 0xfc, + 0x09, 0x78, 0x4c, 0x33, 0x9f, 0x24, 0x3d, 0xc7, 0xe1, 0xdd, 0xe7, 0xa1, 0x6c, 0xd8, 0xbb, 0x8e, + 0x60, 0xd5, 0x99, 0x2c, 0x56, 0x45, 0x91, 0xb1, 0xd9, 0x8a, 0xcd, 0xa8, 0x08, 0x5d, 0xd1, 0x31, + 0xd4, 0x2d, 0x79, 0xec, 0x42, 0xc6, 0xb1, 0xbf, 0x23, 0xc1, 0x89, 0x14, 0xc2, 0xe3, 0x9c, 0xfb, + 0x75, 0x98, 0xa6, 0x0e, 0x56, 0x1c, 0xfc, 0xf9, 0xcc, 0x83, 0x47, 0xd0, 0xdd, 0x32, 0x3c, 0x5f, + 0xe5, 0x6b, 0x14, 0x07, 0xe4, 0xe4, 0x18, 0x71, 0xfd, 0xdc, 0xed, 0x77, 0x6d, 0xcd, 0x62, 0x0c, + 0xa8, 0xaa, 0x35, 0x0e, 0xdb, 0xd6, 0x2c, 0x8c, 0x9e, 0x81, 0x0a, 0x31, 0xd9, 0xae, 0xa1, 0x0b, + 0xf1, 0xcf, 0x50, 0x13, 0xd6, 0x3d, 0x74, 0x0a, 0x80, 0x0e, 0x69, 0xba, 0xee, 0xb2, 0xa8, 0x50, + 0x55, 0xab, 0x04, 0x72, 0x8d, 0x00, 0x94, 0xef, 0x49, 0x50, 0x27, 0x3e, 0xfb, 0x36, 0xf6, 0x35, + 0x22, 0x07, 0xf4, 0x1a, 0x54, 0x4d, 0x47, 0xd3, 0xbb, 0xfe, 0xc1, 0x90, 0xa1, 0x6a, 0x26, 0x79, + 0xcd, 0x8e, 0x40, 0x16, 0xdd, 0x39, 0x18, 0x62, 0xb5, 0x62, 0xf2, 0x5f, 0x93, 0xf0, 0x3b, 0x65, + 0xca, 0xc5, 0x0c, 0x53, 0xfe, 0xa0, 0x0c, 0x4b, 0x5f, 0xd1, 0xfc, 0xfe, 0xde, 0x86, 0x25, 0x82, + 0xdb, 0xd1, 0x95, 0x20, 0xf4, 0x6d, 0x85, 0xa8, 0x6f, 0xfb, 0xc8, 0x7c, 0x67, 0xa0, 0xe7, 0xe5, + 0x2c, 0x3d, 0x27, 0x05, 0xc6, 0xea, 0x3d, 0x2e, 0xaa, 0x88, 0x9e, 0x47, 0x62, 0xd0, 0xf4, 0x51, + 0x62, 0xd0, 0x3a, 0x34, 0xf0, 0xe3, 0xbe, 0x39, 0x22, 0x32, 0xa7, 0xd8, 0x67, 0x28, 0xf6, 0xd3, + 0x19, 0xd8, 0xa3, 0x46, 0x56, 0xe7, 0x8b, 0xb6, 0x38, 0x0d, 0x4c, 0xd4, 0x16, 0xf6, 0xb5, 0x56, + 0x85, 0x92, 0x71, 0x36, 0x4f, 0xd4, 0x42, 0x3f, 0x98, 0xb8, 0xc9, 0x17, 0x3a, 0x09, 0x55, 0x1e, + 0xf1, 0xb6, 0x36, 0x5a, 0x55, 0xca, 0xbe, 0x10, 0x80, 0x34, 0x68, 0x70, 0x0f, 0xc4, 0x29, 0x04, + 0x4a, 0xe1, 0xeb, 0x59, 0x08, 0xb2, 0x85, 0x1d, 0xa5, 0xdc, 0x7b, 0xcb, 0xf6, 0xdd, 0x03, 0xb5, + 0xee, 0x45, 0x40, 0xa4, 0xa8, 0x71, 0x76, 0x77, 0x4d, 0xc3, 0xc6, 0xdb, 0x4c, 0xc2, 0x35, 0x4a, + 0x44, 0x1c, 0xd8, 0xee, 0xc2, 0x5c, 0x6a, 0x23, 0x24, 0x43, 0xf1, 0x01, 0x3e, 0xa0, 0x6a, 0x54, + 0x54, 0xc9, 0x4f, 0xf4, 0x0a, 0x94, 0xf7, 0x35, 0x73, 0x84, 0xa9, 0x9a, 0x8c, 0xe7, 0x24, 0x9b, + 0xfc, 0x85, 0xc2, 0x55, 0x49, 0xf9, 0x99, 0x04, 0x8b, 0x77, 0x6d, 0x6f, 0xd4, 0x0b, 0x4e, 0xf0, + 0xc9, 0x68, 0x6b, 0xd2, 0x4f, 0x94, 0x52, 0x7e, 0x42, 0xf9, 0x6d, 0x09, 0x66, 0xf9, 0x29, 0x88, + 0x50, 0xa9, 0xc1, 0x9f, 0x84, 0x6a, 0x10, 0x2a, 0x38, 0x43, 0x42, 0x00, 0x3a, 0x0b, 0xb5, 0x88, + 0xba, 0x73, 0xaa, 0xa2, 0xa0, 0x89, 0x48, 0x13, 0x81, 0xbf, 0x14, 0x09, 0xfc, 0xa7, 0x00, 0x76, + 0xcd, 0x91, 0xb7, 0xd7, 0xf5, 0x0d, 0x0b, 0xf3, 0xc4, 0xa3, 0x4a, 0x21, 0x77, 0x0c, 0x0b, 0xa3, + 0x6b, 0x50, 0xef, 0x19, 0xb6, 0xe9, 0x0c, 0xba, 0x43, 0xcd, 0xdf, 0xf3, 0x5a, 0xd3, 0xb9, 0x0a, + 0x7e, 0xc3, 0xc0, 0xa6, 0x7e, 0x9d, 0xce, 0x55, 0x6b, 0x6c, 0xcd, 0x0e, 0x59, 0x82, 0x4e, 0x43, + 0xcd, 0x1e, 0x59, 0x5d, 0x67, 0xb7, 0xeb, 0x3a, 0x8f, 0x88, 0x89, 0x50, 0x14, 0xf6, 0xc8, 0x7a, + 0x67, 0x57, 0x75, 0x1e, 0x11, 0x57, 0x5d, 0x25, 0x4e, 0xdb, 0x33, 0x9d, 0x81, 0xd7, 0xaa, 0x4c, + 0xb4, 0x7f, 0xb8, 0x80, 0xac, 0xd6, 0xb1, 0xe9, 0x6b, 0x74, 0x75, 0x75, 0xb2, 0xd5, 0xc1, 0x02, + 0x74, 0x01, 0x9a, 0x7d, 0xc7, 0x1a, 0x6a, 0x94, 0x43, 0x37, 0x5c, 0xc7, 0xa2, 0xf6, 0x51, 0x54, + 0x13, 0x50, 0xb4, 0x0e, 0x35, 0xc3, 0xd6, 0xf1, 0x63, 0x6e, 0x44, 0x35, 0x8a, 0x47, 0xc9, 0x32, + 0x22, 0x8a, 0x68, 0x8b, 0xcc, 0xa5, 0x0a, 0x0a, 0x86, 0xf8, 0xe9, 0x11, 0xcd, 0x10, 0xb6, 0xe8, + 0x19, 0xef, 0xe3, 0x56, 0x9d, 0x49, 0x91, 0xc3, 0x3a, 0xc6, 0xfb, 0x98, 0x24, 0xb5, 0x86, 0xed, + 0x61, 0xd7, 0x17, 0x25, 0x46, 0xab, 0x41, 0xd5, 0xa7, 0xc1, 0xa0, 0x5c, 0xb1, 0x95, 0x5f, 0x14, + 0xa0, 0x19, 0x47, 0x84, 0x5a, 0x30, 0xb3, 0x4b, 0x21, 0x42, 0x7b, 0xc4, 0x27, 0x41, 0x8b, 0x6d, + 0x52, 0xed, 0x77, 0x29, 0x2d, 0x54, 0x79, 0x2a, 0x6a, 0x8d, 0xc1, 0xe8, 0x06, 0x44, 0x09, 0xd8, + 0xf1, 0xa8, 0xc6, 0x16, 0x29, 0xca, 0x2a, 0x85, 0xd0, 0xb8, 0xd6, 0x82, 0x19, 0x76, 0x0c, 0xa1, + 0x3a, 0xe2, 0x93, 0x8c, 0xf4, 0x46, 0x06, 0xc5, 0xca, 0x54, 0x47, 0x7c, 0xa2, 0x0d, 0xa8, 0xb3, + 0x2d, 0x87, 0x9a, 0xab, 0x59, 0x42, 0x71, 0x9e, 0xcb, 0x34, 0xbe, 0x9b, 0xf8, 0xe0, 0x1e, 0xb1, + 0xe3, 0x1d, 0xcd, 0x70, 0x55, 0xc6, 0xe8, 0x1d, 0xba, 0x0a, 0x2d, 0x83, 0xcc, 0x76, 0xd9, 0x35, + 0x4c, 0xcc, 0x55, 0x70, 0x86, 0x06, 0xcf, 0x26, 0x85, 0xdf, 0x30, 0x4c, 0xcc, 0xb4, 0x2c, 0x38, + 0x02, 0x65, 0x6d, 0x85, 0x29, 0x19, 0x85, 0x10, 0xc6, 0x2a, 0xdf, 0x2f, 0xc1, 0x3c, 0xb1, 0x35, + 0x6e, 0x76, 0xc7, 0x88, 0x64, 0xa7, 0x00, 0x74, 0xcf, 0xef, 0xc6, 0xfc, 0x43, 0x55, 0xf7, 0x7c, + 0xe6, 0xe7, 0xd0, 0x6b, 0x22, 0x10, 0x15, 0xf3, 0x73, 0xd3, 0x84, 0xed, 0xa7, 0x83, 0xd1, 0x91, + 0x2a, 0xf7, 0x73, 0xd0, 0xf0, 0x9c, 0x91, 0xdb, 0xc7, 0xdd, 0x58, 0x15, 0x51, 0x67, 0xc0, 0xed, + 0x6c, 0x0f, 0x36, 0x9d, 0xd9, 0x41, 0x88, 0x04, 0xa4, 0x99, 0xe3, 0x05, 0xa4, 0x4a, 0x32, 0x20, + 0xdd, 0x84, 0x59, 0x6a, 0x7e, 0xdd, 0xa1, 0xe3, 0xb1, 0x62, 0x8c, 0x5b, 0xad, 0x92, 0x53, 0x8c, + 0xdf, 0xf6, 0x06, 0x3b, 0x7c, 0xaa, 0xda, 0xa4, 0x4b, 0xc5, 0xa7, 0x47, 0xd4, 0x6f, 0x1f, 0xbb, + 0x9e, 0xe1, 0xd8, 0x2d, 0x60, 0xea, 0xc7, 0x3f, 0x09, 0x33, 0x6c, 0x8c, 0xf5, 0xae, 0xef, 0x6a, + 0xb6, 0xb7, 0x8b, 0x5d, 0x1a, 0x94, 0x2a, 0x6a, 0x9d, 0x00, 0xef, 0x70, 0x98, 0xf2, 0xa7, 0x02, + 0x2c, 0xf1, 0xda, 0xf0, 0xf8, 0x7a, 0x91, 0x17, 0x33, 0x84, 0xd3, 0x2d, 0x1e, 0x52, 0x6d, 0x95, + 0x26, 0xc8, 0x7a, 0xca, 0x19, 0x59, 0x4f, 0xbc, 0xe2, 0x98, 0x4e, 0x55, 0x1c, 0x41, 0xff, 0x60, + 0x66, 0xf2, 0xfe, 0x01, 0x5a, 0x80, 0x32, 0x4d, 0x83, 0xa9, 0xec, 0xaa, 0x2a, 0xfb, 0x98, 0x8c, + 0xa1, 0xff, 0x90, 0xa0, 0xd1, 0xc1, 0x9a, 0xdb, 0xdf, 0x13, 0x7c, 0x7c, 0x35, 0xda, 0x6f, 0x79, + 0x3e, 0x47, 0xc4, 0xb1, 0x25, 0x9f, 0x9d, 0x46, 0xcb, 0x3f, 0x25, 0xa8, 0x7f, 0x99, 0x0c, 0x89, + 0xc3, 0x5e, 0x8d, 0x1e, 0xf6, 0x42, 0xce, 0x61, 0x55, 0xec, 0xbb, 0x06, 0xde, 0xc7, 0x9f, 0xb9, + 0xe3, 0xfe, 0x41, 0x82, 0x76, 0xe7, 0xc0, 0xee, 0xab, 0xcc, 0x96, 0x8f, 0x6f, 0x31, 0xe7, 0xa0, + 0xb1, 0x1f, 0x4b, 0x95, 0x0a, 0x54, 0xe1, 0xea, 0xfb, 0xd1, 0x9a, 0x4a, 0x05, 0x59, 0xb4, 0x79, + 0xf8, 0x61, 0x85, 0x6b, 0x7d, 0x21, 0x8b, 0xea, 0x04, 0x71, 0xd4, 0x35, 0xcd, 0xba, 0x71, 0xa0, + 0xe2, 0xc2, 0x7c, 0xc6, 0x3c, 0x74, 0x02, 0x66, 0x78, 0xf9, 0xc6, 0x43, 0x28, 0x33, 0x61, 0x9d, + 0x48, 0x27, 0x6c, 0x40, 0x18, 0x7a, 0x3a, 0xfd, 0xd2, 0xd1, 0x19, 0xa8, 0x05, 0x79, 0xb6, 0x9e, + 0x12, 0x8f, 0xee, 0x29, 0xbf, 0x91, 0x60, 0xe9, 0x6d, 0xcd, 0xd6, 0x9d, 0xdd, 0xdd, 0xe3, 0x73, + 0x6e, 0x1d, 0x62, 0x29, 0xf8, 0xa4, 0xc5, 0x7d, 0x3c, 0x6f, 0xbf, 0x08, 0x73, 0x2e, 0x73, 0x7e, + 0x7a, 0x9c, 0xb5, 0x45, 0x55, 0x16, 0x03, 0x01, 0xcb, 0x3e, 0x28, 0x00, 0x22, 0xfe, 0xfe, 0xba, + 0x66, 0x6a, 0x76, 0x1f, 0x1f, 0x9d, 0xf4, 0xf3, 0xd0, 0x8c, 0x45, 0xa9, 0xe0, 0x0e, 0x24, 0x1a, + 0xa6, 0x3c, 0x74, 0x13, 0x9a, 0x3d, 0x86, 0xaa, 0xeb, 0x62, 0xcd, 0x73, 0x6c, 0xea, 0x3f, 0x9b, + 0xd9, 0x75, 0xfc, 0x1d, 0xd7, 0x18, 0x0c, 0xb0, 0xbb, 0xee, 0xd8, 0x3a, 0x8b, 0x13, 0x8d, 0x9e, + 0x20, 0x93, 0x2c, 0x25, 0xc2, 0x09, 0x43, 0xb6, 0xa8, 0x1f, 0x21, 0x88, 0xd9, 0x94, 0x15, 0x1e, + 0xd6, 0xcc, 0x90, 0x11, 0xa1, 0xc3, 0x95, 0xd9, 0x40, 0x27, 0xbf, 0x8d, 0x93, 0x11, 0x42, 0x95, + 0x5f, 0x49, 0x80, 0x82, 0x3a, 0x84, 0xd6, 0x55, 0x54, 0xc3, 0x92, 0x4b, 0xa5, 0x0c, 0xbf, 0x7f, + 0x12, 0xaa, 0xba, 0x58, 0xc9, 0x2d, 0x22, 0x04, 0x50, 0x37, 0x4c, 0x89, 0xee, 0x92, 0x78, 0x8b, + 0x75, 0x91, 0xe7, 0x33, 0xe0, 0x2d, 0x0a, 0x8b, 0x47, 0xe0, 0x52, 0x32, 0x02, 0x47, 0xbb, 0x14, + 0xe5, 0x58, 0x97, 0x42, 0xf9, 0xb0, 0x00, 0x32, 0xf5, 0x68, 0xeb, 0x61, 0xa9, 0x3c, 0x11, 0xd1, + 0xe7, 0xa0, 0xc1, 0x6f, 0x09, 0x63, 0x84, 0xd7, 0x1f, 0x46, 0x36, 0x43, 0x97, 0x61, 0x81, 0x4d, + 0x72, 0xb1, 0x37, 0x32, 0xc3, 0x14, 0x97, 0xe5, 0x9b, 0xe8, 0x21, 0x73, 0xa5, 0x64, 0x48, 0xac, + 0xb8, 0x0b, 0x4b, 0x03, 0xd3, 0xe9, 0x69, 0x66, 0x37, 0x2e, 0x1e, 0x26, 0xc3, 0x09, 0x34, 0x7e, + 0x81, 0x2d, 0xef, 0x44, 0x65, 0xe8, 0xa1, 0x4d, 0x52, 0x14, 0xe3, 0x07, 0x41, 0x0a, 0xc2, 0x1b, + 0xd0, 0x93, 0x64, 0x20, 0x75, 0xb2, 0x50, 0x7c, 0x29, 0x3f, 0x96, 0x60, 0x36, 0xd1, 0x68, 0x4c, + 0x96, 0x6a, 0x52, 0xba, 0x54, 0xbb, 0x0a, 0x65, 0x52, 0xbf, 0x30, 0x7f, 0xd7, 0xcc, 0x2e, 0x23, + 0xe2, 0xbb, 0xaa, 0x6c, 0x01, 0xba, 0x04, 0xf3, 0x19, 0x57, 0x52, 0x5c, 0x07, 0x50, 0xfa, 0x46, + 0x4a, 0xf9, 0x4b, 0x09, 0x6a, 0x11, 0x7e, 0x8c, 0xa9, 0x32, 0x27, 0xe9, 0x1c, 0x25, 0x8e, 0x57, + 0x4c, 0x1f, 0x2f, 0xe7, 0x4e, 0x86, 0xe8, 0x9d, 0x85, 0x2d, 0x96, 0x9f, 0xf3, 0x62, 0xc1, 0xc2, + 0x16, 0x2d, 0x7b, 0x88, 0x4a, 0x8e, 0x2c, 0x56, 0x1f, 0x32, 0x73, 0x9a, 0xb1, 0x47, 0x16, 0xad, + 0x0e, 0xe3, 0xa5, 0xc9, 0xcc, 0x21, 0xa5, 0x49, 0x25, 0x5e, 0x9a, 0xc4, 0xec, 0xa8, 0x9a, 0xb4, + 0xa3, 0x49, 0x0b, 0xbf, 0xcb, 0x30, 0xdf, 0x77, 0xb1, 0xe6, 0x63, 0xfd, 0xfa, 0xc1, 0x7a, 0x30, + 0xc4, 0x93, 0x9f, 0xac, 0x21, 0x74, 0x23, 0xec, 0xb8, 0x30, 0x29, 0xd7, 0xa9, 0x94, 0xb3, 0x2b, + 0x1f, 0x2e, 0x1b, 0x26, 0x64, 0xe1, 0x9e, 0xe9, 0x57, 0xb2, 0xe4, 0x6c, 0x1c, 0xa9, 0xe4, 0x3c, + 0x03, 0x35, 0x11, 0x3d, 0x89, 0xb9, 0x37, 0x99, 0xe7, 0x13, 0xbe, 0x40, 0xf7, 0x62, 0xce, 0x60, + 0x36, 0xde, 0xb2, 0x4c, 0xd6, 0x8d, 0x72, 0xaa, 0x6e, 0x54, 0xfe, 0x5c, 0x84, 0x66, 0x58, 0x8f, + 0x4c, 0xec, 0x2d, 0x26, 0xb9, 0x7d, 0xdd, 0x06, 0x39, 0x8c, 0xb9, 0x94, 0x91, 0x87, 0x96, 0x54, + 0xc9, 0x76, 0xff, 0xec, 0x30, 0x61, 0x96, 0xb1, 0x86, 0x6a, 0xe9, 0x89, 0x1a, 0xaa, 0xc7, 0xbc, + 0xa8, 0xba, 0x02, 0x8b, 0x41, 0x9c, 0x8d, 0x1d, 0x9b, 0xe5, 0xeb, 0x0b, 0x62, 0x70, 0x27, 0x7a, + 0xfc, 0x1c, 0x4b, 0x9f, 0xc9, 0xb3, 0xf4, 0xa4, 0xa4, 0x2b, 0x29, 0x49, 0xa7, 0xef, 0xcb, 0xaa, + 0x59, 0xf7, 0x65, 0x77, 0x61, 0x9e, 0x76, 0xd1, 0xbc, 0xbe, 0x6b, 0xf4, 0x70, 0x90, 0x7d, 0x4e, + 0x22, 0xd6, 0x36, 0x54, 0x12, 0x09, 0x6c, 0xf0, 0xad, 0x7c, 0x4b, 0x82, 0xa5, 0xf4, 0xbe, 0x54, + 0x63, 0x42, 0x7f, 0x21, 0xc5, 0xfc, 0xc5, 0x57, 0x61, 0x3e, 0xdc, 0x3e, 0x9e, 0x1a, 0xe7, 0x24, + 0x7f, 0x19, 0x84, 0xab, 0x28, 0xdc, 0x43, 0xc0, 0x94, 0x7f, 0x4b, 0x41, 0x33, 0x92, 0xc0, 0x06, + 0xb4, 0x11, 0x4b, 0x62, 0x98, 0x63, 0x9b, 0x86, 0x1d, 0xd4, 0xcf, 0xfc, 0x8c, 0x0c, 0xc8, 0xeb, + 0xe7, 0xb7, 0x61, 0x96, 0x4f, 0x0a, 0x42, 0xd1, 0x84, 0xc9, 0x57, 0x93, 0xad, 0x0b, 0x82, 0xd0, + 0x79, 0x68, 0xf2, 0x0e, 0xa9, 0xc0, 0x57, 0xcc, 0xe8, 0x9b, 0xa2, 0x2f, 0x81, 0x2c, 0xa6, 0x3d, + 0x69, 0xf0, 0x9b, 0xe5, 0x0b, 0x83, 0x24, 0xee, 0x9b, 0x12, 0xb4, 0xe2, 0xa1, 0x30, 0x72, 0xfc, + 0x27, 0x4f, 0xe5, 0xbe, 0x18, 0xbf, 0x5b, 0x3a, 0x7f, 0x08, 0x3d, 0x21, 0x1e, 0x71, 0xc3, 0xb4, + 0x4d, 0xef, 0x09, 0x49, 0x91, 0xb1, 0x61, 0x78, 0xbe, 0x6b, 0xf4, 0x46, 0xc7, 0x7a, 0x41, 0xa0, + 0xfc, 0xad, 0x00, 0xcf, 0x66, 0x6e, 0x78, 0x9c, 0x5b, 0xa4, 0xbc, 0x9a, 0xfe, 0x25, 0x40, 0x03, + 0xd7, 0x79, 0x64, 0xd8, 0x83, 0x6e, 0xaa, 0x26, 0x9b, 0xe3, 0x23, 0x91, 0x8c, 0xf1, 0x3a, 0x54, + 0x12, 0xb2, 0xbb, 0x70, 0x08, 0xaf, 0xee, 0xb1, 0x16, 0x06, 0xeb, 0xaa, 0x88, 0x75, 0x64, 0x8f, + 0xc0, 0x04, 0xca, 0xf9, 0x7b, 0x70, 0x1d, 0x8f, 0xed, 0x21, 0xd6, 0xa1, 0x37, 0xa1, 0xc6, 0xea, + 0xc2, 0x7b, 0x06, 0x7e, 0x94, 0xd3, 0xcb, 0xe5, 0x5e, 0x30, 0x98, 0xa6, 0x46, 0x97, 0x28, 0xff, + 0x92, 0x00, 0xc2, 0x31, 0x52, 0x93, 0x86, 0xe6, 0xc5, 0xed, 0x25, 0x02, 0x21, 0xd1, 0x39, 0x9e, + 0x10, 0x8a, 0x4f, 0xf4, 0x1e, 0xc8, 0xfc, 0x68, 0x44, 0xcf, 0x77, 0x34, 0xc3, 0x15, 0xee, 0xfd, + 0x95, 0xc3, 0xe9, 0x11, 0x5c, 0x0a, 0x96, 0xb1, 0x2b, 0x89, 0xd4, 0x6e, 0xed, 0x75, 0x58, 0xcc, + 0x9c, 0x9a, 0x71, 0xe9, 0xb0, 0x10, 0xbd, 0x74, 0x28, 0x46, 0x2f, 0x15, 0x7e, 0x20, 0x01, 0x4a, + 0x8b, 0x05, 0x35, 0xa1, 0x10, 0xf8, 0x87, 0xc2, 0xd6, 0x46, 0x82, 0x0f, 0x85, 0x14, 0x1f, 0x4e, + 0x42, 0x35, 0xf0, 0xfc, 0xdc, 0xcc, 0x43, 0x40, 0x94, 0x4b, 0xa5, 0x38, 0x97, 0x22, 0xfd, 0xad, + 0x72, 0xac, 0xbf, 0xa5, 0xec, 0x01, 0x4a, 0x8b, 0x3a, 0xba, 0x93, 0x14, 0xdf, 0x69, 0x1c, 0x85, + 0x11, 0x4c, 0xc5, 0x38, 0xa6, 0x3f, 0x4a, 0x80, 0xc2, 0xd8, 0x16, 0xdc, 0x57, 0x4c, 0x12, 0x10, + 0x2e, 0xc1, 0x7c, 0x3a, 0xf2, 0x89, 0x70, 0x8f, 0x52, 0x71, 0x2f, 0x2b, 0x46, 0x15, 0x33, 0x62, + 0x14, 0x7a, 0x35, 0xb0, 0x65, 0x16, 0xc8, 0x4f, 0xe7, 0x05, 0xf2, 0xb8, 0x39, 0x2b, 0xbf, 0x96, + 0x60, 0x2e, 0xc0, 0xf6, 0x44, 0x27, 0x19, 0x7f, 0xff, 0xf2, 0x94, 0x49, 0xef, 0xc0, 0x0c, 0x6f, + 0x59, 0xa4, 0x94, 0x6f, 0x92, 0xac, 0x7d, 0x01, 0xca, 0xc4, 0x75, 0x09, 0x1f, 0xc5, 0x3e, 0x88, + 0x76, 0x43, 0xe7, 0xc0, 0xee, 0x5f, 0x63, 0x3a, 0x70, 0x19, 0x4a, 0xe3, 0xae, 0x9b, 0xc9, 0x6c, + 0x9a, 0x1d, 0xd1, 0x99, 0x13, 0xb0, 0x25, 0x56, 0x70, 0x14, 0x93, 0x05, 0x47, 0xde, 0xf3, 0xad, + 0xdf, 0x4b, 0x70, 0x82, 0xa0, 0xfa, 0x48, 0x42, 0xc3, 0x44, 0x0c, 0x8a, 0x58, 0x4e, 0x31, 0x6e, + 0x39, 0x57, 0x61, 0x86, 0x65, 0xf6, 0xc2, 0x77, 0x9f, 0xce, 0x63, 0x0c, 0x63, 0xa3, 0x2a, 0xa6, + 0xaf, 0xbc, 0x09, 0xd5, 0xa0, 0x89, 0x86, 0x6a, 0x30, 0x73, 0xd7, 0xbe, 0x69, 0x3b, 0x8f, 0x6c, + 0x79, 0x0a, 0xcd, 0x40, 0xf1, 0x9a, 0x69, 0xca, 0x12, 0x6a, 0x40, 0xb5, 0xe3, 0xbb, 0x58, 0xb3, + 0x0c, 0x7b, 0x20, 0x17, 0x50, 0x13, 0xe0, 0x6d, 0xc3, 0xf3, 0x1d, 0xd7, 0xe8, 0x6b, 0xa6, 0x5c, + 0x5c, 0x79, 0x1f, 0x9a, 0xf1, 0xc4, 0x16, 0xd5, 0xa1, 0xb2, 0xed, 0xf8, 0x6f, 0x3d, 0x36, 0x3c, + 0x5f, 0x9e, 0x22, 0xf3, 0xb7, 0x1d, 0x7f, 0xc7, 0xc5, 0x1e, 0xb6, 0x7d, 0x59, 0x42, 0x00, 0xd3, + 0xef, 0xd8, 0x1b, 0x86, 0xf7, 0x40, 0x2e, 0xa0, 0x79, 0x5e, 0x9a, 0x6a, 0xe6, 0x16, 0xcf, 0x16, + 0xe5, 0x22, 0x59, 0x1e, 0x7c, 0x95, 0x90, 0x0c, 0xf5, 0x60, 0xca, 0xe6, 0xce, 0x5d, 0xb9, 0x8c, + 0xaa, 0x50, 0x66, 0x3f, 0xa7, 0x57, 0x74, 0x90, 0x93, 0x7d, 0x15, 0xb2, 0x27, 0x3b, 0x44, 0x00, + 0x92, 0xa7, 0xc8, 0xc9, 0x78, 0x63, 0x4b, 0x96, 0xd0, 0x2c, 0xd4, 0x22, 0x6d, 0x22, 0xb9, 0x40, + 0x00, 0x9b, 0xee, 0xb0, 0xcf, 0xa5, 0xc7, 0x48, 0x20, 0x0e, 0x79, 0x83, 0x70, 0xa2, 0xb4, 0x72, + 0x1d, 0x2a, 0x22, 0xe3, 0x26, 0x53, 0x39, 0x8b, 0xc8, 0xa7, 0x3c, 0x85, 0xe6, 0xa0, 0x11, 0x7b, + 0xc8, 0x26, 0x4b, 0x08, 0x41, 0x33, 0xfe, 0xc0, 0x52, 0x2e, 0xac, 0xac, 0x01, 0x84, 0x16, 0x43, + 0xc8, 0xd9, 0xb2, 0xf7, 0x35, 0xd3, 0xd0, 0x19, 0x6d, 0x64, 0x88, 0x70, 0x97, 0x72, 0x87, 0x35, + 0x48, 0xe4, 0xc2, 0xca, 0x19, 0xa8, 0x08, 0x5d, 0x26, 0x70, 0x15, 0x5b, 0xce, 0x3e, 0x66, 0x92, + 0xe9, 0x60, 0x5f, 0x96, 0xd6, 0x7e, 0xd4, 0x00, 0x60, 0xad, 0x10, 0xc7, 0x71, 0x75, 0x34, 0x04, + 0xb4, 0x89, 0x7d, 0x52, 0xe6, 0x39, 0xb6, 0x28, 0xd1, 0x3c, 0x74, 0x39, 0xff, 0x01, 0x61, 0x62, + 0x2a, 0x3f, 0x7f, 0x3b, 0xaf, 0x2b, 0x9c, 0x98, 0xae, 0x4c, 0x21, 0x8b, 0x62, 0xbc, 0x63, 0x58, + 0xf8, 0x8e, 0xd1, 0x7f, 0x10, 0xf4, 0x50, 0xf2, 0x31, 0x26, 0xa6, 0x0a, 0x8c, 0x89, 0x72, 0x89, + 0x7f, 0x74, 0x7c, 0xd7, 0xb0, 0x07, 0x22, 0x3b, 0x52, 0xa6, 0xd0, 0xc3, 0xc4, 0x83, 0x49, 0x81, + 0x70, 0x6d, 0x92, 0x37, 0x92, 0x47, 0x43, 0x69, 0xc2, 0x6c, 0xe2, 0x75, 0x35, 0x5a, 0xc9, 0x7e, + 0xa6, 0x93, 0xf5, 0x12, 0xbc, 0x7d, 0x71, 0xa2, 0xb9, 0x01, 0x36, 0x03, 0x9a, 0xf1, 0x17, 0xc4, + 0xe8, 0x73, 0x79, 0x1b, 0xa4, 0x1e, 0x0a, 0xb6, 0x57, 0x26, 0x99, 0x1a, 0xa0, 0xba, 0xcf, 0x94, + 0x74, 0x1c, 0xaa, 0xcc, 0x47, 0x9a, 0xed, 0xc3, 0x12, 0x53, 0x65, 0x0a, 0xbd, 0x07, 0x73, 0xa9, + 0xe7, 0x8c, 0xe8, 0xc5, 0xec, 0x56, 0x78, 0xf6, 0xab, 0xc7, 0x71, 0x18, 0xee, 0x27, 0x4d, 0x2c, + 0x9f, 0xfa, 0xd4, 0xd3, 0xdf, 0xc9, 0xa9, 0x8f, 0x6c, 0x7f, 0x18, 0xf5, 0x4f, 0x8c, 0x61, 0x44, + 0xcd, 0x26, 0xd9, 0x94, 0x7b, 0x29, 0x0b, 0x45, 0xee, 0x9b, 0xca, 0xf6, 0xea, 0xa4, 0xd3, 0xa3, + 0xda, 0x15, 0x7f, 0xb6, 0x97, 0xcd, 0xb4, 0xcc, 0xa7, 0x86, 0xd9, 0xda, 0x95, 0xfd, 0x0a, 0x50, + 0x99, 0x42, 0x77, 0x62, 0x2e, 0x16, 0x5d, 0xc8, 0x13, 0x4e, 0xbc, 0x55, 0x3f, 0x8e, 0x6f, 0x5f, + 0x07, 0xc4, 0x6c, 0xc7, 0xde, 0x35, 0x06, 0x23, 0x57, 0x63, 0x8a, 0x95, 0xe7, 0x6e, 0xd2, 0x53, + 0x05, 0x9a, 0x97, 0x9f, 0x60, 0x45, 0x70, 0xa4, 0x2e, 0xc0, 0x26, 0xf6, 0x6f, 0x63, 0xdf, 0x35, + 0xfa, 0x5e, 0xf2, 0x44, 0xfc, 0x23, 0x9c, 0x20, 0x50, 0xbd, 0x30, 0x76, 0x5e, 0x80, 0xa0, 0x07, + 0xb5, 0x4d, 0xec, 0xf3, 0x0c, 0xca, 0x43, 0xb9, 0x2b, 0xc5, 0x0c, 0x81, 0x62, 0x79, 0xfc, 0xc4, + 0xa8, 0x3b, 0x4b, 0x3c, 0x61, 0x44, 0xb9, 0x82, 0x4d, 0x3f, 0xac, 0xcc, 0x76, 0x67, 0x39, 0x6f, + 0x22, 0x95, 0xa9, 0xb5, 0x5f, 0x36, 0xa1, 0x4a, 0xe3, 0x13, 0x09, 0xa6, 0xff, 0x0f, 0x4f, 0x4f, + 0x21, 0x3c, 0xbd, 0x0b, 0xb3, 0x89, 0x17, 0x71, 0xd9, 0xf2, 0xcc, 0x7e, 0x36, 0x37, 0x81, 0x97, + 0x8d, 0xbf, 0x56, 0xcb, 0x76, 0x18, 0x99, 0x2f, 0xda, 0xc6, 0xed, 0x7d, 0x8f, 0x3d, 0x26, 0x0d, + 0x5a, 0x4d, 0x2f, 0xe4, 0x16, 0x19, 0xf1, 0x9b, 0xc8, 0x4f, 0xde, 0x7b, 0x3f, 0xfd, 0xe8, 0xf6, + 0x2e, 0xcc, 0x26, 0x9e, 0x7c, 0x64, 0x4b, 0x35, 0xfb, 0x5d, 0xc8, 0xb8, 0xdd, 0x3f, 0xc6, 0x30, + 0xa0, 0xc3, 0x7c, 0xc6, 0x6d, 0x3c, 0x5a, 0xcd, 0xab, 0x4e, 0xb2, 0xaf, 0xed, 0xc7, 0x1f, 0xa8, + 0x11, 0x33, 0x25, 0xb4, 0x9c, 0x47, 0x64, 0xf2, 0x0f, 0x33, 0xed, 0x17, 0x27, 0xfb, 0x77, 0x4d, + 0x70, 0xa0, 0x0e, 0x4c, 0xb3, 0x87, 0x20, 0xe8, 0xb9, 0xec, 0xee, 0x58, 0xe4, 0x91, 0x48, 0x7b, + 0xdc, 0x53, 0x12, 0x6f, 0x64, 0xfa, 0x1e, 0xdd, 0xb4, 0x4c, 0xbd, 0x24, 0xca, 0x7c, 0xc1, 0x14, + 0x7d, 0xbd, 0xd1, 0x1e, 0xff, 0x60, 0x43, 0x6c, 0xfa, 0xbf, 0x1d, 0x2b, 0x1f, 0xc3, 0x7c, 0x46, + 0x23, 0x15, 0xe5, 0xe5, 0x44, 0x39, 0x2d, 0xdc, 0xf6, 0xa5, 0x89, 0xe7, 0x07, 0x98, 0xbf, 0x06, + 0x72, 0xb2, 0xea, 0x47, 0x17, 0xf3, 0xf4, 0x39, 0x0b, 0xe7, 0xe1, 0xca, 0x7c, 0xfd, 0x95, 0xfb, + 0x6b, 0x03, 0xc3, 0xdf, 0x1b, 0xf5, 0xc8, 0xc8, 0x25, 0x36, 0xf5, 0x25, 0xc3, 0xe1, 0xbf, 0x2e, + 0x09, 0xfe, 0x5f, 0xa2, 0xab, 0x2f, 0x51, 0x54, 0xc3, 0x5e, 0x6f, 0x9a, 0x7e, 0x5e, 0xf9, 0x6f, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x4d, 0x43, 0x08, 0x3d, 0x08, 0x3b, 0x00, 0x00, +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// QueryCoordClient is the client API for QueryCoord service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type QueryCoordClient interface { + GetComponentStates(ctx context.Context, in *internalpb.GetComponentStatesRequest, opts ...grpc.CallOption) (*internalpb.ComponentStates, error) + GetTimeTickChannel(ctx context.Context, in *internalpb.GetTimeTickChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) + GetStatisticsChannel(ctx context.Context, in *internalpb.GetStatisticsChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) + ShowCollections(ctx context.Context, in *ShowCollectionsRequest, opts ...grpc.CallOption) (*ShowCollectionsResponse, error) + ShowPartitions(ctx context.Context, in *ShowPartitionsRequest, opts ...grpc.CallOption) (*ShowPartitionsResponse, error) + LoadPartitions(ctx context.Context, in *LoadPartitionsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) + ReleasePartitions(ctx context.Context, in *ReleasePartitionsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) + LoadCollection(ctx context.Context, in *LoadCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) + ReleaseCollection(ctx context.Context, in *ReleaseCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) + GetPartitionStates(ctx context.Context, in *GetPartitionStatesRequest, opts ...grpc.CallOption) (*GetPartitionStatesResponse, error) + GetSegmentInfo(ctx context.Context, in *GetSegmentInfoRequest, opts ...grpc.CallOption) (*GetSegmentInfoResponse, error) + LoadBalance(ctx context.Context, in *LoadBalanceRequest, opts ...grpc.CallOption) (*commonpb.Status, error) + ShowConfigurations(ctx context.Context, in *internalpb.ShowConfigurationsRequest, opts ...grpc.CallOption) (*internalpb.ShowConfigurationsResponse, error) + // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy + GetMetrics(ctx context.Context, in *milvuspb.GetMetricsRequest, opts ...grpc.CallOption) (*milvuspb.GetMetricsResponse, error) + // https://wiki.lfaidata.foundation/display/MIL/MEP+23+--+Multiple+memory+replication+design + GetReplicas(ctx context.Context, in *milvuspb.GetReplicasRequest, opts ...grpc.CallOption) (*milvuspb.GetReplicasResponse, error) + GetShardLeaders(ctx context.Context, in *GetShardLeadersRequest, opts ...grpc.CallOption) (*GetShardLeadersResponse, error) +} + +type queryCoordClient struct { + cc *grpc.ClientConn +} + +func NewQueryCoordClient(cc *grpc.ClientConn) QueryCoordClient { + return &queryCoordClient{cc} +} + +func (c *queryCoordClient) GetComponentStates(ctx context.Context, in *internalpb.GetComponentStatesRequest, opts ...grpc.CallOption) (*internalpb.ComponentStates, error) { + out := new(internalpb.ComponentStates) + err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryCoord/GetComponentStates", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *queryCoordClient) GetTimeTickChannel(ctx context.Context, in *internalpb.GetTimeTickChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) { + out := new(milvuspb.StringResponse) + err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryCoord/GetTimeTickChannel", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *queryCoordClient) GetStatisticsChannel(ctx context.Context, in *internalpb.GetStatisticsChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) { + out := new(milvuspb.StringResponse) + err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryCoord/GetStatisticsChannel", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *queryCoordClient) ShowCollections(ctx context.Context, in *ShowCollectionsRequest, opts ...grpc.CallOption) (*ShowCollectionsResponse, error) { + out := new(ShowCollectionsResponse) + err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryCoord/ShowCollections", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil } func (c *queryCoordClient) ShowPartitions(ctx context.Context, in *ShowPartitionsRequest, opts ...grpc.CallOption) (*ShowPartitionsResponse, error) { @@ -3635,6 +4435,7 @@ type QueryNodeClient interface { GetTimeTickChannel(ctx context.Context, in *internalpb.GetTimeTickChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) GetStatisticsChannel(ctx context.Context, in *internalpb.GetStatisticsChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) WatchDmChannels(ctx context.Context, in *WatchDmChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) + UnsubDmChannel(ctx context.Context, in *UnsubDmChannelRequest, opts ...grpc.CallOption) (*commonpb.Status, error) LoadSegments(ctx context.Context, in *LoadSegmentsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) ReleaseCollection(ctx context.Context, in *ReleaseCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) ReleasePartitions(ctx context.Context, in *ReleasePartitionsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) @@ -3647,6 +4448,8 @@ type QueryNodeClient interface { ShowConfigurations(ctx context.Context, in *internalpb.ShowConfigurationsRequest, opts ...grpc.CallOption) (*internalpb.ShowConfigurationsResponse, error) // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy GetMetrics(ctx context.Context, in *milvuspb.GetMetricsRequest, opts ...grpc.CallOption) (*milvuspb.GetMetricsResponse, error) + GetDataDistribution(ctx context.Context, in *GetDataDistributionRequest, opts ...grpc.CallOption) (*GetDataDistributionResponse, error) + SyncDistribution(ctx context.Context, in *SyncDistributionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) } type queryNodeClient struct { @@ -3693,6 +4496,15 @@ func (c *queryNodeClient) WatchDmChannels(ctx context.Context, in *WatchDmChanne return out, nil } +func (c *queryNodeClient) UnsubDmChannel(ctx context.Context, in *UnsubDmChannelRequest, opts ...grpc.CallOption) (*commonpb.Status, error) { + out := new(commonpb.Status) + err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryNode/UnsubDmChannel", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *queryNodeClient) LoadSegments(ctx context.Context, in *LoadSegmentsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) { out := new(commonpb.Status) err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryNode/LoadSegments", in, out, opts...) @@ -3792,12 +4604,31 @@ func (c *queryNodeClient) GetMetrics(ctx context.Context, in *milvuspb.GetMetric return out, nil } +func (c *queryNodeClient) GetDataDistribution(ctx context.Context, in *GetDataDistributionRequest, opts ...grpc.CallOption) (*GetDataDistributionResponse, error) { + out := new(GetDataDistributionResponse) + err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryNode/GetDataDistribution", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *queryNodeClient) SyncDistribution(ctx context.Context, in *SyncDistributionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) { + out := new(commonpb.Status) + err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryNode/SyncDistribution", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // QueryNodeServer is the server API for QueryNode service. type QueryNodeServer interface { GetComponentStates(context.Context, *internalpb.GetComponentStatesRequest) (*internalpb.ComponentStates, error) GetTimeTickChannel(context.Context, *internalpb.GetTimeTickChannelRequest) (*milvuspb.StringResponse, error) GetStatisticsChannel(context.Context, *internalpb.GetStatisticsChannelRequest) (*milvuspb.StringResponse, error) WatchDmChannels(context.Context, *WatchDmChannelsRequest) (*commonpb.Status, error) + UnsubDmChannel(context.Context, *UnsubDmChannelRequest) (*commonpb.Status, error) LoadSegments(context.Context, *LoadSegmentsRequest) (*commonpb.Status, error) ReleaseCollection(context.Context, *ReleaseCollectionRequest) (*commonpb.Status, error) ReleasePartitions(context.Context, *ReleasePartitionsRequest) (*commonpb.Status, error) @@ -3810,6 +4641,8 @@ type QueryNodeServer interface { ShowConfigurations(context.Context, *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy GetMetrics(context.Context, *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) + GetDataDistribution(context.Context, *GetDataDistributionRequest) (*GetDataDistributionResponse, error) + SyncDistribution(context.Context, *SyncDistributionRequest) (*commonpb.Status, error) } // UnimplementedQueryNodeServer can be embedded to have forward compatible implementations. @@ -3828,6 +4661,9 @@ func (*UnimplementedQueryNodeServer) GetStatisticsChannel(ctx context.Context, r func (*UnimplementedQueryNodeServer) WatchDmChannels(ctx context.Context, req *WatchDmChannelsRequest) (*commonpb.Status, error) { return nil, status.Errorf(codes.Unimplemented, "method WatchDmChannels not implemented") } +func (*UnimplementedQueryNodeServer) UnsubDmChannel(ctx context.Context, req *UnsubDmChannelRequest) (*commonpb.Status, error) { + return nil, status.Errorf(codes.Unimplemented, "method UnsubDmChannel not implemented") +} func (*UnimplementedQueryNodeServer) LoadSegments(ctx context.Context, req *LoadSegmentsRequest) (*commonpb.Status, error) { return nil, status.Errorf(codes.Unimplemented, "method LoadSegments not implemented") } @@ -3861,6 +4697,12 @@ func (*UnimplementedQueryNodeServer) ShowConfigurations(ctx context.Context, req func (*UnimplementedQueryNodeServer) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetMetrics not implemented") } +func (*UnimplementedQueryNodeServer) GetDataDistribution(ctx context.Context, req *GetDataDistributionRequest) (*GetDataDistributionResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetDataDistribution not implemented") +} +func (*UnimplementedQueryNodeServer) SyncDistribution(ctx context.Context, req *SyncDistributionRequest) (*commonpb.Status, error) { + return nil, status.Errorf(codes.Unimplemented, "method SyncDistribution not implemented") +} func RegisterQueryNodeServer(s *grpc.Server, srv QueryNodeServer) { s.RegisterService(&_QueryNode_serviceDesc, srv) @@ -3938,6 +4780,24 @@ func _QueryNode_WatchDmChannels_Handler(srv interface{}, ctx context.Context, de return interceptor(ctx, in, info, handler) } +func _QueryNode_UnsubDmChannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UnsubDmChannelRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(QueryNodeServer).UnsubDmChannel(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/milvus.proto.query.QueryNode/UnsubDmChannel", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(QueryNodeServer).UnsubDmChannel(ctx, req.(*UnsubDmChannelRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _QueryNode_LoadSegments_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(LoadSegmentsRequest) if err := dec(in); err != nil { @@ -4136,6 +4996,42 @@ func _QueryNode_GetMetrics_Handler(srv interface{}, ctx context.Context, dec fun return interceptor(ctx, in, info, handler) } +func _QueryNode_GetDataDistribution_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetDataDistributionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(QueryNodeServer).GetDataDistribution(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/milvus.proto.query.QueryNode/GetDataDistribution", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(QueryNodeServer).GetDataDistribution(ctx, req.(*GetDataDistributionRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _QueryNode_SyncDistribution_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SyncDistributionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(QueryNodeServer).SyncDistribution(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/milvus.proto.query.QueryNode/SyncDistribution", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(QueryNodeServer).SyncDistribution(ctx, req.(*SyncDistributionRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _QueryNode_serviceDesc = grpc.ServiceDesc{ ServiceName: "milvus.proto.query.QueryNode", HandlerType: (*QueryNodeServer)(nil), @@ -4156,6 +5052,10 @@ var _QueryNode_serviceDesc = grpc.ServiceDesc{ MethodName: "WatchDmChannels", Handler: _QueryNode_WatchDmChannels_Handler, }, + { + MethodName: "UnsubDmChannel", + Handler: _QueryNode_UnsubDmChannel_Handler, + }, { MethodName: "LoadSegments", Handler: _QueryNode_LoadSegments_Handler, @@ -4200,6 +5100,14 @@ var _QueryNode_serviceDesc = grpc.ServiceDesc{ MethodName: "GetMetrics", Handler: _QueryNode_GetMetrics_Handler, }, + { + MethodName: "GetDataDistribution", + Handler: _QueryNode_GetDataDistribution_Handler, + }, + { + MethodName: "SyncDistribution", + Handler: _QueryNode_SyncDistribution_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "query_coord.proto", diff --git a/internal/proto/querypbv2/query_coordv2.pb.go b/internal/proto/querypbv2/query_coordv2.pb.go deleted file mode 100644 index 22cb6e946a9f4..0000000000000 --- a/internal/proto/querypbv2/query_coordv2.pb.go +++ /dev/null @@ -1,79 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: query_coordv2.proto - -package querypbv2 - -import ( - context "context" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -func init() { proto.RegisterFile("query_coordv2.proto", fileDescriptor_06e29c474dd8c71e) } - -var fileDescriptor_06e29c474dd8c71e = []byte{ - // 117 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x2e, 0x2c, 0x4d, 0x2d, - 0xaa, 0x8c, 0x4f, 0xce, 0xcf, 0x2f, 0x4a, 0x29, 0x33, 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, - 0x12, 0xc9, 0xcd, 0xcc, 0x29, 0x2b, 0x2d, 0x86, 0xf0, 0xf4, 0xc0, 0x2a, 0xca, 0x8c, 0x8c, 0xf8, - 0xb8, 0x78, 0x02, 0x41, 0x4c, 0x67, 0x90, 0xda, 0x30, 0x23, 0x27, 0xb3, 0x28, 0x93, 0xf4, 0xcc, - 0x92, 0x8c, 0xd2, 0x24, 0xbd, 0xe4, 0xfc, 0x5c, 0x7d, 0x88, 0x16, 0xdd, 0xcc, 0x7c, 0x28, 0x4b, - 0x3f, 0x33, 0xaf, 0x24, 0xb5, 0x28, 0x2f, 0x31, 0x47, 0x1f, 0x6c, 0x8a, 0x3e, 0xd8, 0x94, 0x82, - 0xa4, 0x32, 0xa3, 0x24, 0x36, 0xb0, 0x80, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x0f, 0x12, 0x44, - 0x18, 0x7b, 0x00, 0x00, 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 - -// QueryCoordV2Client is the client API for QueryCoordV2 service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type QueryCoordV2Client interface { -} - -type queryCoordV2Client struct { - cc *grpc.ClientConn -} - -func NewQueryCoordV2Client(cc *grpc.ClientConn) QueryCoordV2Client { - return &queryCoordV2Client{cc} -} - -// QueryCoordV2Server is the server API for QueryCoordV2 service. -type QueryCoordV2Server interface { -} - -// UnimplementedQueryCoordV2Server can be embedded to have forward compatible implementations. -type UnimplementedQueryCoordV2Server struct { -} - -func RegisterQueryCoordV2Server(s *grpc.Server, srv QueryCoordV2Server) { - s.RegisterService(&_QueryCoordV2_serviceDesc, srv) -} - -var _QueryCoordV2_serviceDesc = grpc.ServiceDesc{ - ServiceName: "milvus.proto.queryv2.QueryCoordV2", - HandlerType: (*QueryCoordV2Server)(nil), - Methods: []grpc.MethodDesc{}, - Streams: []grpc.StreamDesc{}, - Metadata: "query_coordv2.proto", -} diff --git a/internal/proxy/query_node_mock_test.go b/internal/proxy/query_node_mock_test.go index edb0cd2b6c091..1c65a08d19a50 100644 --- a/internal/proxy/query_node_mock_test.go +++ b/internal/proxy/query_node_mock_test.go @@ -121,3 +121,13 @@ func (m *QueryNodeMock) GetTimeTickChannel(ctx context.Context) (*milvuspb.Strin func (m *QueryNodeMock) ShowConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) { return nil, nil } + +func (m *QueryNodeMock) UnsubDmChannel(ctx context.Context, req *querypb.UnsubDmChannelRequest) (*commonpb.Status, error) { + return nil, nil +} +func (m *QueryNodeMock) GetDataDistribution(context.Context, *querypb.GetDataDistributionRequest) (*querypb.GetDataDistributionResponse, error) { + return nil, nil +} +func (m *QueryNodeMock) SyncDistribution(context.Context, *querypb.SyncDistributionRequest) (*commonpb.Status, error) { + return nil, nil +} diff --git a/internal/querycoord/mock_querynode_client_test.go b/internal/querycoord/mock_querynode_client_test.go index d0aa97c7b5573..b0592b43405cb 100644 --- a/internal/querycoord/mock_querynode_client_test.go +++ b/internal/querycoord/mock_querynode_client_test.go @@ -160,3 +160,14 @@ func (client *queryNodeClientMock) SyncReplicaSegments(ctx context.Context, req func (client *queryNodeClientMock) ShowConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) { return client.grpcClient.ShowConfigurations(ctx, req) } + +func (client *queryNodeClientMock) UnsubDmChannel(ctx context.Context, req *querypb.UnsubDmChannelRequest) (*commonpb.Status, error) { + return client.grpcClient.UnsubDmChannel(ctx, req) +} + +func (client *queryNodeClientMock) GetDataDistribution(ctx context.Context, req *querypb.GetDataDistributionRequest) (*querypb.GetDataDistributionResponse, error) { + return client.grpcClient.GetDataDistribution(ctx, req) +} +func (client *queryNodeClientMock) SyncDistribution(ctx context.Context, req *querypb.SyncDistributionRequest) (*commonpb.Status, error) { + return client.grpcClient.SyncDistribution(ctx, req) +} diff --git a/internal/querycoordv2/balance/balance.go b/internal/querycoordv2/balance/balance.go new file mode 100644 index 0000000000000..e037ec111133c --- /dev/null +++ b/internal/querycoordv2/balance/balance.go @@ -0,0 +1,103 @@ +package balance + +import ( + "sort" + + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/task" +) + +type SegmentAssignPlan struct { + Segment *meta.Segment + ReplicaID int64 + From int64 // -1 if empty + To int64 +} + +type ChannelAssignPlan struct { + Channel *meta.DmChannel + ReplicaID int64 + From int64 + To int64 +} + +type Balance interface { + AssignSegment(segments []*meta.Segment, nodes []int64) []SegmentAssignPlan + AssignChannel(channels []*meta.DmChannel, nodes []int64) []ChannelAssignPlan + Balance() ([]SegmentAssignPlan, []ChannelAssignPlan) +} + +type RoundRobinBalancer struct { + scheduler task.Scheduler + nodeManager *session.NodeManager +} + +func (b *RoundRobinBalancer) AssignSegment(segments []*meta.Segment, nodes []int64) []SegmentAssignPlan { + nodesInfo := b.getNodes(nodes) + if len(nodesInfo) == 0 { + return nil + } + sort.Slice(nodesInfo, func(i, j int) bool { + cnt1, cnt2 := nodesInfo[i].SegmentCnt(), nodesInfo[j].SegmentCnt() + id1, id2 := nodesInfo[i].ID(), nodesInfo[j].ID() + delta1, delta2 := b.scheduler.GetNodeSegmentDelta(id1), b.scheduler.GetNodeSegmentDelta(id2) + return cnt1+delta1 < cnt2+delta2 + }) + ret := make([]SegmentAssignPlan, 0, len(segments)) + for i, s := range segments { + plan := SegmentAssignPlan{ + Segment: s, + From: -1, + To: nodesInfo[i%len(nodesInfo)].ID(), + } + ret = append(ret, plan) + } + return ret +} + +func (b *RoundRobinBalancer) AssignChannel(channels []*meta.DmChannel, nodes []int64) []ChannelAssignPlan { + nodesInfo := b.getNodes(nodes) + if len(nodesInfo) == 0 { + return nil + } + sort.Slice(nodesInfo, func(i, j int) bool { + cnt1, cnt2 := nodesInfo[i].ChannelCnt(), nodesInfo[j].ChannelCnt() + id1, id2 := nodesInfo[i].ID(), nodesInfo[j].ID() + delta1, delta2 := b.scheduler.GetNodeChannelDelta(id1), b.scheduler.GetNodeChannelDelta(id2) + return cnt1+delta1 < cnt2+delta2 + }) + ret := make([]ChannelAssignPlan, 0, len(channels)) + for i, c := range channels { + plan := ChannelAssignPlan{ + Channel: c, + From: -1, + To: nodesInfo[i%len(nodesInfo)].ID(), + } + ret = append(ret, plan) + } + return ret +} + +func (b *RoundRobinBalancer) getNodes(nodes []int64) []*session.NodeInfo { + ret := make([]*session.NodeInfo, 0, len(nodes)) + for _, n := range nodes { + node := b.nodeManager.Get(n) + if node != nil { + ret = append(ret, node) + } + } + return ret +} + +func (b *RoundRobinBalancer) Balance() ([]SegmentAssignPlan, []ChannelAssignPlan) { + // TODO(sunby) + return nil, nil +} + +func NewRoundRobinBalancer(scheduler task.Scheduler, nodeManager *session.NodeManager) *RoundRobinBalancer { + return &RoundRobinBalancer{ + scheduler: scheduler, + nodeManager: nodeManager, + } +} diff --git a/internal/querycoordv2/balance/balance_test.go b/internal/querycoordv2/balance/balance_test.go new file mode 100644 index 0000000000000..7639cd6499544 --- /dev/null +++ b/internal/querycoordv2/balance/balance_test.go @@ -0,0 +1,135 @@ +package balance + +import ( + "testing" + + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/task" + "github.com/stretchr/testify/suite" +) + +type BalanceTestSuite struct { + suite.Suite + mockScheduler *task.MockScheduler + roundRobinBalancer *RoundRobinBalancer +} + +func (suite *BalanceTestSuite) SetupTest() { + nodeManager := session.NewNodeManager() + suite.mockScheduler = task.NewMockScheduler(suite.T()) + suite.roundRobinBalancer = NewRoundRobinBalancer(suite.mockScheduler, nodeManager) +} + +func (suite *BalanceTestSuite) TestAssignBalance() { + cases := []struct { + name string + nodeIDs []int64 + segmentCnts []int + deltaCnts []int + assignments []*meta.Segment + expectPlans []SegmentAssignPlan + }{ + { + name: "normal assignment", + nodeIDs: []int64{1, 2}, + segmentCnts: []int{100, 200}, + deltaCnts: []int{0, -200}, + assignments: []*meta.Segment{ + {SegmentInfo: &datapb.SegmentInfo{ID: 1}}, + {SegmentInfo: &datapb.SegmentInfo{ID: 2}}, + {SegmentInfo: &datapb.SegmentInfo{ID: 3}}, + }, + expectPlans: []SegmentAssignPlan{ + {Segment: &meta.Segment{SegmentInfo: &datapb.SegmentInfo{ID: 1}}, From: -1, To: 2}, + {Segment: &meta.Segment{SegmentInfo: &datapb.SegmentInfo{ID: 2}}, From: -1, To: 1}, + {Segment: &meta.Segment{SegmentInfo: &datapb.SegmentInfo{ID: 3}}, From: -1, To: 2}, + }, + }, + { + name: "empty assignment", + nodeIDs: []int64{}, + segmentCnts: []int{}, + deltaCnts: []int{}, + assignments: []*meta.Segment{ + {SegmentInfo: &datapb.SegmentInfo{ID: 1}}, + {SegmentInfo: &datapb.SegmentInfo{ID: 2}}, + {SegmentInfo: &datapb.SegmentInfo{ID: 3}}, + }, + expectPlans: []SegmentAssignPlan{}, + }, + } + + for _, c := range cases { + suite.Run(c.name, func() { + suite.SetupTest() + for i := range c.nodeIDs { + nodeInfo := session.NewNodeInfo(c.nodeIDs[i], "127.0.0.1:0") + nodeInfo.UpdateStats(session.WithSegmentCnt(c.segmentCnts[i])) + suite.roundRobinBalancer.nodeManager.Add(nodeInfo) + suite.mockScheduler.EXPECT().GetNodeSegmentDelta(c.nodeIDs[i]).Return(c.deltaCnts[i]) + } + plans := suite.roundRobinBalancer.AssignSegment(c.assignments, c.nodeIDs) + suite.ElementsMatch(c.expectPlans, plans) + }) + } +} + +func (suite *BalanceTestSuite) TestAssignChannel() { + cases := []struct { + name string + nodeIDs []int64 + channelCnts []int + deltaCnts []int + assignments []*meta.DmChannel + expectPlans []ChannelAssignPlan + }{ + { + name: "normal assignment", + nodeIDs: []int64{1, 2}, + channelCnts: []int{100, 200}, + deltaCnts: []int{0, -200}, + assignments: []*meta.DmChannel{ + {VchannelInfo: &datapb.VchannelInfo{ChannelName: "channel-1"}}, + {VchannelInfo: &datapb.VchannelInfo{ChannelName: "channel-2"}}, + {VchannelInfo: &datapb.VchannelInfo{ChannelName: "channel-3"}}, + }, + expectPlans: []ChannelAssignPlan{ + {Channel: &meta.DmChannel{VchannelInfo: &datapb.VchannelInfo{ChannelName: "channel-1"}}, From: -1, To: 2}, + {Channel: &meta.DmChannel{VchannelInfo: &datapb.VchannelInfo{ChannelName: "channel-2"}}, From: -1, To: 1}, + {Channel: &meta.DmChannel{VchannelInfo: &datapb.VchannelInfo{ChannelName: "channel-3"}}, From: -1, To: 2}, + }, + }, + { + name: "empty assignment", + nodeIDs: []int64{}, + channelCnts: []int{}, + deltaCnts: []int{}, + assignments: []*meta.DmChannel{ + {VchannelInfo: &datapb.VchannelInfo{ChannelName: "channel-1"}}, + {VchannelInfo: &datapb.VchannelInfo{ChannelName: "channel-2"}}, + {VchannelInfo: &datapb.VchannelInfo{ChannelName: "channel-3"}}, + }, + expectPlans: []ChannelAssignPlan{}, + }, + } + + for _, c := range cases { + suite.Run(c.name, func() { + suite.SetupTest() + for i := range c.nodeIDs { + nodeInfo := session.NewNodeInfo(c.nodeIDs[i], "127.0.0.1:0") + nodeInfo.UpdateStats(session.WithChannelCnt(c.channelCnts[i])) + suite.roundRobinBalancer.nodeManager.Add(nodeInfo) + suite.mockScheduler.EXPECT().GetNodeChannelDelta(c.nodeIDs[i]).Return(c.deltaCnts[i]) + } + plans := suite.roundRobinBalancer.AssignChannel(c.assignments, c.nodeIDs) + suite.ElementsMatch(c.expectPlans, plans) + }) + } +} + +func TestBalanceSuite(t *testing.T) { + suite.Run(t, new(BalanceTestSuite)) +} diff --git a/internal/querycoordv2/balance/mock_balancer.go b/internal/querycoordv2/balance/mock_balancer.go new file mode 100644 index 0000000000000..e78a3a0f8971b --- /dev/null +++ b/internal/querycoordv2/balance/mock_balancer.go @@ -0,0 +1,163 @@ +// Code generated by mockery v2.14.0. DO NOT EDIT. + +package balance + +import ( + meta "github.com/milvus-io/milvus/internal/querycoordv2/meta" + mock "github.com/stretchr/testify/mock" +) + +// MockBalancer is an autogenerated mock type for the Balance type +type MockBalancer struct { + mock.Mock +} + +type MockBalancer_Expecter struct { + mock *mock.Mock +} + +func (_m *MockBalancer) EXPECT() *MockBalancer_Expecter { + return &MockBalancer_Expecter{mock: &_m.Mock} +} + +// AssignChannel provides a mock function with given fields: channels, nodes +func (_m *MockBalancer) AssignChannel(channels []*meta.DmChannel, nodes []int64) []ChannelAssignPlan { + ret := _m.Called(channels, nodes) + + var r0 []ChannelAssignPlan + if rf, ok := ret.Get(0).(func([]*meta.DmChannel, []int64) []ChannelAssignPlan); ok { + r0 = rf(channels, nodes) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]ChannelAssignPlan) + } + } + + return r0 +} + +// MockBalancer_AssignChannel_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AssignChannel' +type MockBalancer_AssignChannel_Call struct { + *mock.Call +} + +// AssignChannel is a helper method to define mock.On call +// - channels []*meta.DmChannel +// - nodes []int64 +func (_e *MockBalancer_Expecter) AssignChannel(channels interface{}, nodes interface{}) *MockBalancer_AssignChannel_Call { + return &MockBalancer_AssignChannel_Call{Call: _e.mock.On("AssignChannel", channels, nodes)} +} + +func (_c *MockBalancer_AssignChannel_Call) Run(run func(channels []*meta.DmChannel, nodes []int64)) *MockBalancer_AssignChannel_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].([]*meta.DmChannel), args[1].([]int64)) + }) + return _c +} + +func (_c *MockBalancer_AssignChannel_Call) Return(_a0 []ChannelAssignPlan) *MockBalancer_AssignChannel_Call { + _c.Call.Return(_a0) + return _c +} + +// AssignSegment provides a mock function with given fields: segments, nodes +func (_m *MockBalancer) AssignSegment(segments []*meta.Segment, nodes []int64) []SegmentAssignPlan { + ret := _m.Called(segments, nodes) + + var r0 []SegmentAssignPlan + if rf, ok := ret.Get(0).(func([]*meta.Segment, []int64) []SegmentAssignPlan); ok { + r0 = rf(segments, nodes) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]SegmentAssignPlan) + } + } + + return r0 +} + +// MockBalancer_AssignSegment_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AssignSegment' +type MockBalancer_AssignSegment_Call struct { + *mock.Call +} + +// AssignSegment is a helper method to define mock.On call +// - segments []*meta.Segment +// - nodes []int64 +func (_e *MockBalancer_Expecter) AssignSegment(segments interface{}, nodes interface{}) *MockBalancer_AssignSegment_Call { + return &MockBalancer_AssignSegment_Call{Call: _e.mock.On("AssignSegment", segments, nodes)} +} + +func (_c *MockBalancer_AssignSegment_Call) Run(run func(segments []*meta.Segment, nodes []int64)) *MockBalancer_AssignSegment_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].([]*meta.Segment), args[1].([]int64)) + }) + return _c +} + +func (_c *MockBalancer_AssignSegment_Call) Return(_a0 []SegmentAssignPlan) *MockBalancer_AssignSegment_Call { + _c.Call.Return(_a0) + return _c +} + +// Balance provides a mock function with given fields: +func (_m *MockBalancer) Balance() ([]SegmentAssignPlan, []ChannelAssignPlan) { + ret := _m.Called() + + var r0 []SegmentAssignPlan + if rf, ok := ret.Get(0).(func() []SegmentAssignPlan); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]SegmentAssignPlan) + } + } + + var r1 []ChannelAssignPlan + if rf, ok := ret.Get(1).(func() []ChannelAssignPlan); ok { + r1 = rf() + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).([]ChannelAssignPlan) + } + } + + return r0, r1 +} + +// MockBalancer_Balance_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Balance' +type MockBalancer_Balance_Call struct { + *mock.Call +} + +// Balance is a helper method to define mock.On call +func (_e *MockBalancer_Expecter) Balance() *MockBalancer_Balance_Call { + return &MockBalancer_Balance_Call{Call: _e.mock.On("Balance")} +} + +func (_c *MockBalancer_Balance_Call) Run(run func()) *MockBalancer_Balance_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockBalancer_Balance_Call) Return(_a0 []SegmentAssignPlan, _a1 []ChannelAssignPlan) *MockBalancer_Balance_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +type mockConstructorTestingTNewMockBalancer interface { + mock.TestingT + Cleanup(func()) +} + +// NewMockBalancer creates a new instance of MockBalancer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMockBalancer(t mockConstructorTestingTNewMockBalancer) *MockBalancer { + mock := &MockBalancer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/querycoordv2/balance/priority_queue.go b/internal/querycoordv2/balance/priority_queue.go new file mode 100644 index 0000000000000..b8e61aa4083be --- /dev/null +++ b/internal/querycoordv2/balance/priority_queue.go @@ -0,0 +1,69 @@ +package balance + +import ( + "container/heap" +) + +type item interface { + getPriority() int + setPriority(priority int) +} + +type baseItem struct { + priority int +} + +func (b *baseItem) getPriority() int { + return b.priority +} + +func (b *baseItem) setPriority(priority int) { + b.priority = priority +} + +type heapQueue []item + +func (hq heapQueue) Len() int { + return len(hq) +} + +func (hq heapQueue) Less(i, j int) bool { + return hq[i].getPriority() < hq[j].getPriority() +} + +func (hq heapQueue) Swap(i, j int) { + hq[i], hq[j] = hq[j], hq[i] +} + +func (hq *heapQueue) Push(x any) { + i := x.(item) + *hq = append(*hq, i) +} + +func (hq *heapQueue) Pop() any { + arr := *hq + l := len(arr) + ret := arr[l-1] + *hq = arr[0 : l-1] + return ret +} + +type priorityQueue struct { + heapQueue +} + +func newPriorityQueue() priorityQueue { + hq := make(heapQueue, 0) + heap.Init(&hq) + return priorityQueue{ + heapQueue: hq, + } +} + +func (pq *priorityQueue) push(item item) { + heap.Push(&pq.heapQueue, item) +} + +func (pq *priorityQueue) pop() item { + return heap.Pop(&pq.heapQueue).(item) +} diff --git a/internal/querycoordv2/balance/rowcount_based_balancer.go b/internal/querycoordv2/balance/rowcount_based_balancer.go new file mode 100644 index 0000000000000..1c3bdee716760 --- /dev/null +++ b/internal/querycoordv2/balance/rowcount_based_balancer.go @@ -0,0 +1,193 @@ +package balance + +import ( + "sort" + + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/task" +) + +type RowCountBasedBalancer struct { + RoundRobinBalancer + nodeManager *session.NodeManager + dist *meta.DistributionManager + meta *meta.Meta +} + +func (b *RowCountBasedBalancer) AssignSegment(segments []*meta.Segment, nodes []int64) []SegmentAssignPlan { + if len(nodes) == 0 { + return nil + } + nodeItems := b.convertToNodeItems(nodes) + queue := newPriorityQueue() + for _, item := range nodeItems { + queue.push(item) + } + + sort.Slice(segments, func(i, j int) bool { + return segments[i].GetNumOfRows() > segments[j].GetNumOfRows() + }) + + plans := make([]SegmentAssignPlan, 0, len(segments)) + for _, s := range segments { + // pick the node with the least row count and allocate to it. + ni := queue.pop().(*nodeItem) + plan := SegmentAssignPlan{ + From: -1, + To: ni.nodeID, + Segment: s, + } + plans = append(plans, plan) + // change node's priority and push back + p := ni.getPriority() + ni.setPriority(p + int(s.GetNumOfRows())) + queue.push(ni) + } + return plans +} + +func (b *RowCountBasedBalancer) convertToNodeItems(nodeIDs []int64) []*nodeItem { + ret := make([]*nodeItem, 0, len(nodeIDs)) + for _, node := range nodeIDs { + segments := b.dist.SegmentDistManager.GetByNode(node) + rowcnt := 0 + for _, s := range segments { + rowcnt += int(s.GetNumOfRows()) + } + // more row count, less priority + nodeItem := newNodeItem(rowcnt, node) + ret = append(ret, &nodeItem) + } + return ret +} + +func (b *RowCountBasedBalancer) Balance() ([]SegmentAssignPlan, []ChannelAssignPlan) { + ids := b.meta.CollectionManager.GetAll() + + segmentPlans, channelPlans := make([]SegmentAssignPlan, 0), make([]ChannelAssignPlan, 0) + for _, cid := range ids { + replicas := b.meta.ReplicaManager.GetByCollection(cid) + for _, replica := range replicas { + splans, cplans := b.balanceReplica(replica) + segmentPlans = append(segmentPlans, splans...) + channelPlans = append(channelPlans, cplans...) + } + } + return segmentPlans, channelPlans +} + +func (b *RowCountBasedBalancer) balanceReplica(replica *meta.Replica) ([]SegmentAssignPlan, []ChannelAssignPlan) { + nodes := replica.Nodes.Collect() + if len(nodes) == 0 { + return nil, nil + } + nodesRowCnt := make(map[int64]int) + nodesSegments := make(map[int64][]*meta.Segment) + totalCnt := 0 + for _, nid := range nodes { + segments := b.dist.SegmentDistManager.GetByCollectionAndNode(replica.GetCollectionID(), nid) + cnt := 0 + for _, s := range segments { + cnt += int(s.GetNumOfRows()) + } + nodesRowCnt[nid] = cnt + nodesSegments[nid] = segments + totalCnt += cnt + } + + average := totalCnt / len(nodes) + neededRowCnt := 0 + for _, rowcnt := range nodesRowCnt { + if rowcnt < average { + neededRowCnt += average - rowcnt + } + } + + if neededRowCnt == 0 { + return nil, nil + } + + segmentsToMove := make([]*meta.Segment, 0) + + // select segments to be moved +outer: + for nodeID, rowcnt := range nodesRowCnt { + if rowcnt <= average { + continue + } + segments := nodesSegments[nodeID] + sort.Slice(segments, func(i, j int) bool { + return segments[i].GetNumOfRows() > segments[j].GetNumOfRows() + }) + + for _, s := range segments { + if rowcnt-int(s.GetNumOfRows()) < average { + continue + } + rowcnt -= int(s.GetNumOfRows()) + segmentsToMove = append(segmentsToMove, s) + neededRowCnt -= int(s.GetNumOfRows()) + if neededRowCnt <= 0 { + break outer + } + } + } + + sort.Slice(segmentsToMove, func(i, j int) bool { + return segmentsToMove[i].GetNumOfRows() < segmentsToMove[j].GetNumOfRows() + }) + + // allocate segments to those nodes with row cnt less than average + queue := newPriorityQueue() + for nodeID, rowcnt := range nodesRowCnt { + if rowcnt >= average { + continue + } + item := newNodeItem(rowcnt, nodeID) + queue.push(&item) + } + + plans := make([]SegmentAssignPlan, 0) + for _, s := range segmentsToMove { + node := queue.pop().(*nodeItem) + plan := SegmentAssignPlan{ + ReplicaID: replica.GetID(), + From: s.Node, + To: node.nodeID, + Segment: s, + } + plans = append(plans, plan) + node.setPriority(node.getPriority() + int(s.GetNumOfRows())) + queue.push(node) + } + return plans, nil +} + +func NewRowCountBasedBalancer( + scheduler task.Scheduler, + nodeManager *session.NodeManager, + dist *meta.DistributionManager, + meta *meta.Meta, +) *RowCountBasedBalancer { + return &RowCountBasedBalancer{ + RoundRobinBalancer: *NewRoundRobinBalancer(scheduler, nodeManager), + nodeManager: nodeManager, + dist: dist, + meta: meta, + } +} + +type nodeItem struct { + baseItem + nodeID int64 +} + +func newNodeItem(priority int, nodeID int64) nodeItem { + return nodeItem{ + baseItem: baseItem{ + priority: priority, + }, + nodeID: nodeID, + } +} diff --git a/internal/querycoordv2/balance/rowcount_based_balancer_test.go b/internal/querycoordv2/balance/rowcount_based_balancer_test.go new file mode 100644 index 0000000000000..127ff0e46df66 --- /dev/null +++ b/internal/querycoordv2/balance/rowcount_based_balancer_test.go @@ -0,0 +1,146 @@ +package balance + +import ( + "testing" + + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/internal/util/etcd" + "github.com/stretchr/testify/suite" +) + +type RowCountBasedBalancerTestSuite struct { + suite.Suite + balancer *RowCountBasedBalancer + kv *etcdkv.EtcdKV +} + +func (suite *RowCountBasedBalancerTestSuite) SetupSuite() { + Params.Init() +} + +func (suite *RowCountBasedBalancerTestSuite) SetupTest() { + var err error + config := GenerateEtcdConfig() + cli, err := etcd.GetEtcdClient(&config) + suite.Require().NoError(err) + suite.kv = etcdkv.NewEtcdKV(cli, config.MetaRootPath) + + store := meta.NewMetaStore(suite.kv) + idAllocator := RandomIncrementIDAllocator() + testMeta := meta.NewMeta(idAllocator, store) + + distManager := meta.NewDistributionManager() + nodeManager := session.NewNodeManager() + suite.balancer = NewRowCountBasedBalancer(nil, nodeManager, distManager, testMeta) +} + +func (suite *RowCountBasedBalancerTestSuite) TearDownTest() { + suite.kv.Close() +} + +func (suite *RowCountBasedBalancerTestSuite) TestAssignSegment() { + cases := []struct { + name string + distributions map[int64][]*meta.Segment + assignments []*meta.Segment + nodes []int64 + expectPlans []SegmentAssignPlan + }{ + { + name: "test normal assignment", + distributions: map[int64][]*meta.Segment{ + 2: {{SegmentInfo: &datapb.SegmentInfo{ID: 1, NumOfRows: 20}, Node: 2}}, + 3: {{SegmentInfo: &datapb.SegmentInfo{ID: 2, NumOfRows: 30}, Node: 3}}, + }, + assignments: []*meta.Segment{ + {SegmentInfo: &datapb.SegmentInfo{ID: 3, NumOfRows: 5}}, + {SegmentInfo: &datapb.SegmentInfo{ID: 4, NumOfRows: 10}}, + {SegmentInfo: &datapb.SegmentInfo{ID: 5, NumOfRows: 15}}, + }, + nodes: []int64{1, 2, 3}, + expectPlans: []SegmentAssignPlan{ + {Segment: &meta.Segment{SegmentInfo: &datapb.SegmentInfo{ID: 3, NumOfRows: 5}}, From: -1, To: 2}, + {Segment: &meta.Segment{SegmentInfo: &datapb.SegmentInfo{ID: 4, NumOfRows: 10}}, From: -1, To: 1}, + {Segment: &meta.Segment{SegmentInfo: &datapb.SegmentInfo{ID: 5, NumOfRows: 15}}, From: -1, To: 1}, + }, + }, + // TODO: add more cases + } + + for _, c := range cases { + suite.Run(c.name, func() { + // I do not find a better way to do the setup and teardown work for subtests yet. + // If you do, please replace with it. + suite.SetupSuite() + defer suite.TearDownTest() + balancer := suite.balancer + for node, s := range c.distributions { + balancer.dist.SegmentDistManager.Update(node, s...) + } + plans := balancer.AssignSegment(c.assignments, c.nodes) + suite.ElementsMatch(c.expectPlans, plans) + }) + } +} + +func (suite *RowCountBasedBalancerTestSuite) TestBalance() { + cases := []struct { + name string + nodes []int64 + distributions map[int64][]*meta.Segment + expectPlans []SegmentAssignPlan + }{ + { + name: "normal balance", + nodes: []int64{1, 2}, + distributions: map[int64][]*meta.Segment{ + 1: {{SegmentInfo: &datapb.SegmentInfo{ID: 1, CollectionID: 1, NumOfRows: 10}, Node: 1}}, + 2: { + {SegmentInfo: &datapb.SegmentInfo{ID: 2, CollectionID: 1, NumOfRows: 20}, Node: 2}, + {SegmentInfo: &datapb.SegmentInfo{ID: 3, CollectionID: 1, NumOfRows: 30}, Node: 2}, + }, + }, + expectPlans: []SegmentAssignPlan{ + {Segment: &meta.Segment{SegmentInfo: &datapb.SegmentInfo{ID: 2, CollectionID: 1, NumOfRows: 20}, Node: 2}, From: 2, To: 1, ReplicaID: 1}, + }, + }, + { + name: "already balanced", + nodes: []int64{1, 2}, + distributions: map[int64][]*meta.Segment{ + 1: {{SegmentInfo: &datapb.SegmentInfo{ID: 1, CollectionID: 1, NumOfRows: 30}, Node: 1}}, + 2: { + {SegmentInfo: &datapb.SegmentInfo{ID: 2, CollectionID: 1, NumOfRows: 20}, Node: 2}, + {SegmentInfo: &datapb.SegmentInfo{ID: 3, CollectionID: 1, NumOfRows: 30}, Node: 2}, + }, + }, + expectPlans: []SegmentAssignPlan{}, + }, + } + + for _, c := range cases { + suite.Run(c.name, func() { + suite.SetupSuite() + defer suite.TearDownTest() + balancer := suite.balancer + balancer.meta.CollectionManager.PutCollection(utils.CreateTestCollection(1, 1)) + balancer.meta.ReplicaManager.Put(utils.CreateTestReplica(1, 1, c.nodes)) + for node, s := range c.distributions { + balancer.dist.SegmentDistManager.Update(node, s...) + } + segmentPlans, channelPlans := balancer.Balance() + suite.Empty(channelPlans) + suite.ElementsMatch(c.expectPlans, segmentPlans) + }) + } + +} + +func TestRowCountBasedBalancerSuite(t *testing.T) { + suite.Run(t, new(RowCountBasedBalancerTestSuite)) +} diff --git a/internal/querycoordv2/balance/utils.go b/internal/querycoordv2/balance/utils.go new file mode 100644 index 0000000000000..5f20cf3aaf967 --- /dev/null +++ b/internal/querycoordv2/balance/utils.go @@ -0,0 +1,51 @@ +package balance + +import ( + "context" + "time" + + "github.com/milvus-io/milvus/internal/querycoordv2/task" +) + +func CreateSegmentTasksFromPlans(ctx context.Context, checkerID int64, timeout time.Duration, plans []SegmentAssignPlan) []task.Task { + ret := make([]task.Task, 0) + for _, p := range plans { + actions := make([]task.Action, 0) + if p.To != -1 { + action := task.NewSegmentAction(p.To, task.ActionTypeGrow, p.Segment.GetID()) + actions = append(actions, action) + } + if p.From != -1 { + action := task.NewSegmentAction(p.From, task.ActionTypeReduce, p.Segment.GetID()) + actions = append(actions, action) + } + task := task.NewSegmentTask( + ctx, + timeout, + checkerID, + p.Segment.GetCollectionID(), + p.ReplicaID, + actions..., + ) + ret = append(ret, task) + } + return ret +} + +func CreateChannelTasksFromPlans(ctx context.Context, checkerID int64, timeout time.Duration, plans []ChannelAssignPlan) []task.Task { + ret := make([]task.Task, 0, len(plans)) + for _, p := range plans { + actions := make([]task.Action, 0) + if p.To != -1 { + action := task.NewChannelAction(p.To, task.ActionTypeGrow, p.Channel.GetChannelName()) + actions = append(actions, action) + } + if p.From != -1 { + action := task.NewChannelAction(p.From, task.ActionTypeReduce, p.Channel.GetChannelName()) + actions = append(actions, action) + } + task := task.NewChannelTask(ctx, timeout, checkerID, p.Channel.GetCollectionID(), p.ReplicaID, actions...) + ret = append(ret, task) + } + return ret +} diff --git a/internal/querycoordv2/checkers/balance_checker.go b/internal/querycoordv2/checkers/balance_checker.go new file mode 100644 index 0000000000000..02f82f4ad6411 --- /dev/null +++ b/internal/querycoordv2/checkers/balance_checker.go @@ -0,0 +1,35 @@ +package checkers + +import ( + "context" + + "github.com/milvus-io/milvus/internal/querycoordv2/balance" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/task" +) + +// BalanceChecker checks the cluster distribution and generates balance tasks. +type BalanceChecker struct { + baseChecker + balance.Balance +} + +func NewBalanceChecker(balancer balance.Balance) *BalanceChecker { + return &BalanceChecker{ + Balance: balancer, + } +} + +func (b *BalanceChecker) Description() string { + return "BalanceChecker checks the cluster distribution and generates balance tasks" +} + +func (b *BalanceChecker) Check(ctx context.Context) []task.Task { + ret := make([]task.Task, 0) + segmentPlans, channelPlans := b.Balance.Balance() + tasks := balance.CreateSegmentTasksFromPlans(ctx, b.ID(), Params.QueryCoordCfg.SegmentTaskTimeout, segmentPlans) + ret = append(ret, tasks...) + tasks = balance.CreateChannelTasksFromPlans(ctx, b.ID(), Params.QueryCoordCfg.ChannelTaskTimeout, channelPlans) + ret = append(ret, tasks...) + return ret +} diff --git a/internal/querycoordv2/checkers/channel_checker.go b/internal/querycoordv2/checkers/channel_checker.go new file mode 100644 index 0000000000000..983513ea496e6 --- /dev/null +++ b/internal/querycoordv2/checkers/channel_checker.go @@ -0,0 +1,139 @@ +package checkers + +import ( + "context" + + "github.com/milvus-io/milvus/internal/querycoordv2/balance" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/task" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" +) + +// TODO(sunby): have too much similar codes with SegmentChecker +type ChannelChecker struct { + baseChecker + meta *meta.Meta + dist *meta.DistributionManager + targetMgr *meta.TargetManager + balancer balance.Balance +} + +func NewChannelChecker( + meta *meta.Meta, + dist *meta.DistributionManager, + targetMgr *meta.TargetManager, + balancer balance.Balance, +) *ChannelChecker { + return &ChannelChecker{ + meta: meta, + dist: dist, + targetMgr: targetMgr, + balancer: balancer, + } +} + +func (c *ChannelChecker) Description() string { + return "DmChannelChecker checks the lack of DmChannels, or some DmChannels are redundant" +} + +func (c *ChannelChecker) Check(ctx context.Context) []task.Task { + collectionIDs := c.meta.CollectionManager.GetAll() + tasks := make([]task.Task, 0) + for _, cid := range collectionIDs { + replicas := c.meta.ReplicaManager.GetByCollection(cid) + for _, r := range replicas { + tasks = append(tasks, c.checkReplica(ctx, r)...) + } + } + + channels := c.dist.ChannelDistManager.GetAll() + released := utils.FilterReleased(channels, collectionIDs) + tasks = append(tasks, c.createChannelReduceTasks(ctx, released, -1)...) + return tasks +} + +func (c *ChannelChecker) checkReplica(ctx context.Context, replica *meta.Replica) []task.Task { + ret := make([]task.Task, 0) + targets := c.targetMgr.GetDmChannelsByCollection(replica.GetCollectionID()) + dists := c.getChannelDist(replica) + + lacks, redundancies := diffChannels(targets, dists) + tasks := c.createChannelLoadTask(ctx, lacks, replica) + ret = append(ret, tasks...) + tasks = c.createChannelReduceTasks(ctx, redundancies, replica.GetID()) + ret = append(ret, tasks...) + + repeated := findRepeatedChannels(dists) + tasks = c.createChannelReduceTasks(ctx, repeated, replica.GetID()) + ret = append(ret, tasks...) + return ret +} + +func (c *ChannelChecker) getChannelDist(replica *meta.Replica) []*meta.DmChannel { + dists := make([]*meta.DmChannel, 0) + for _, nodeID := range replica.Nodes.Collect() { + dists = append(dists, c.dist.ChannelDistManager.GetByCollectionAndNode(replica.GetCollectionID(), nodeID)...) + } + return dists +} + +func diffChannels(targets, dists []*meta.DmChannel) (lacks, redundancies []*meta.DmChannel) { + distMap := make(map[string]struct{}) + targetMap := make(map[string]struct{}) + for _, ch := range targets { + targetMap[ch.GetChannelName()] = struct{}{} + } + for _, ch := range dists { + distMap[ch.GetChannelName()] = struct{}{} + if _, ok := targetMap[ch.GetChannelName()]; !ok { + redundancies = append(redundancies, ch) + } + } + for _, ch := range targets { + if _, ok := distMap[ch.GetChannelName()]; !ok { + lacks = append(lacks, ch) + } + } + return +} + +func findRepeatedChannels(dists []*meta.DmChannel) []*meta.DmChannel { + ret := make([]*meta.DmChannel, 0) + versionsMap := make(map[string]*meta.DmChannel) + for _, ch := range dists { + maxVer, ok := versionsMap[ch.GetChannelName()] + if !ok { + versionsMap[ch.GetChannelName()] = ch + continue + } + if maxVer.Version <= ch.Version { + ret = append(ret, maxVer) + versionsMap[ch.GetChannelName()] = ch + } else { + ret = append(ret, ch) + } + } + return ret +} + +func (c *ChannelChecker) createChannelLoadTask(ctx context.Context, channels []*meta.DmChannel, replica *meta.Replica) []task.Task { + plans := c.balancer.AssignChannel(channels, replica.Replica.GetNodes()) + for i := range plans { + plans[i].ReplicaID = replica.GetID() + } + // log.Debug("try to subscribe channels", + // zap.Any("channels", channels), + // zap.Any("plans", plans)) + return balance.CreateChannelTasksFromPlans(ctx, c.ID(), Params.QueryCoordCfg.ChannelTaskTimeout, plans) +} + +func (c *ChannelChecker) createChannelReduceTasks(ctx context.Context, channels []*meta.DmChannel, replicaID int64) []task.Task { + ret := make([]task.Task, 0, len(channels)) + for _, ch := range channels { + action := task.NewChannelAction(ch.Node, task.ActionTypeReduce, ch.GetChannelName()) + task := task.NewChannelTask(ctx, Params.QueryCoordCfg.ChannelTaskTimeout, c.ID(), ch.GetCollectionID(), replicaID, action) + ret = append(ret, task) + } + return ret +} diff --git a/internal/querycoordv2/checkers/channel_checker_test.go b/internal/querycoordv2/checkers/channel_checker_test.go new file mode 100644 index 0000000000000..869ae11097894 --- /dev/null +++ b/internal/querycoordv2/checkers/channel_checker_test.go @@ -0,0 +1,125 @@ +package checkers + +import ( + "context" + "testing" + + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/internal/querycoordv2/balance" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/task" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/internal/util/etcd" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type ChannelCheckerTestSuite struct { + suite.Suite + kv *etcdkv.EtcdKV + checker *ChannelChecker +} + +func (suite *ChannelCheckerTestSuite) SetupSuite() { + Params.Init() +} + +func (suite *ChannelCheckerTestSuite) SetupTest() { + var err error + config := GenerateEtcdConfig() + cli, err := etcd.GetEtcdClient(&config) + suite.Require().NoError(err) + suite.kv = etcdkv.NewEtcdKV(cli, config.MetaRootPath) + + // meta + store := meta.NewMetaStore(suite.kv) + idAllocator := RandomIncrementIDAllocator() + testMeta := meta.NewMeta(idAllocator, store) + + distManager := meta.NewDistributionManager() + targetManager := meta.NewTargetManager() + + balancer := suite.createMockBalancer() + suite.checker = NewChannelChecker(testMeta, distManager, targetManager, balancer) +} + +func (suite *ChannelCheckerTestSuite) TearDownTest() { + suite.kv.Close() +} + +func (suite *ChannelCheckerTestSuite) createMockBalancer() balance.Balance { + balancer := balance.NewMockBalancer(suite.T()) + balancer.EXPECT().AssignChannel(mock.Anything, mock.Anything).Maybe().Return(func(channels []*meta.DmChannel, nodes []int64) []balance.ChannelAssignPlan { + plans := make([]balance.ChannelAssignPlan, 0, len(channels)) + for i, c := range channels { + plan := balance.ChannelAssignPlan{ + Channel: c, + From: -1, + To: nodes[i%len(nodes)], + ReplicaID: -1, + } + plans = append(plans, plan) + } + return plans + }) + return balancer +} + +func (suite *ChannelCheckerTestSuite) TestLoadChannel() { + checker := suite.checker + checker.meta.CollectionManager.PutCollection(utils.CreateTestCollection(1, 1)) + checker.meta.ReplicaManager.Put(utils.CreateTestReplica(1, 1, []int64{1})) + + checker.targetMgr.AddDmChannel(utils.CreateTestChannel(1, 1, 1, "test-insert-channel")) + + tasks := checker.Check(context.TODO()) + suite.Len(tasks, 1) + suite.EqualValues(1, tasks[0].ReplicaID()) + suite.Len(tasks[0].Actions(), 1) + suite.IsType((*task.ChannelAction)(nil), tasks[0].Actions()[0]) + action := tasks[0].Actions()[0].(*task.ChannelAction) + suite.Equal(task.ActionTypeGrow, action.Type()) + suite.EqualValues(1, action.Node()) + suite.EqualValues("test-insert-channel", action.ChannelName()) +} + +func (suite *ChannelCheckerTestSuite) TestReduceChannel() { + checker := suite.checker + checker.meta.CollectionManager.PutCollection(utils.CreateTestCollection(1, 1)) + checker.meta.ReplicaManager.Put(utils.CreateTestReplica(1, 1, []int64{1})) + + checker.dist.ChannelDistManager.Update(1, utils.CreateTestChannel(1, 1, 1, "test-insert-channel")) + tasks := checker.Check(context.TODO()) + suite.Len(tasks, 1) + suite.EqualValues(1, tasks[0].ReplicaID()) + suite.Len(tasks[0].Actions(), 1) + suite.IsType((*task.ChannelAction)(nil), tasks[0].Actions()[0]) + action := tasks[0].Actions()[0].(*task.ChannelAction) + suite.Equal(task.ActionTypeReduce, action.Type()) + suite.EqualValues(1, action.Node()) + suite.EqualValues("test-insert-channel", action.ChannelName()) +} + +func (suite *ChannelCheckerTestSuite) TestRepeatedChannels() { + checker := suite.checker + checker.meta.CollectionManager.PutCollection(utils.CreateTestCollection(1, 1)) + checker.meta.ReplicaManager.Put(utils.CreateTestReplica(1, 1, []int64{1, 2})) + checker.targetMgr.AddDmChannel(utils.CreateTestChannel(1, 1, 1, "test-insert-channel")) + checker.dist.ChannelDistManager.Update(1, utils.CreateTestChannel(1, 1, 1, "test-insert-channel")) + checker.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 2, "test-insert-channel")) + + tasks := checker.Check(context.TODO()) + suite.Len(tasks, 1) + suite.EqualValues(1, tasks[0].ReplicaID()) + suite.Len(tasks[0].Actions(), 1) + suite.IsType((*task.ChannelAction)(nil), tasks[0].Actions()[0]) + action := tasks[0].Actions()[0].(*task.ChannelAction) + suite.Equal(task.ActionTypeReduce, action.Type()) + suite.EqualValues(1, action.Node()) + suite.EqualValues("test-insert-channel", action.ChannelName()) +} + +func TestChannelCheckerSuite(t *testing.T) { + suite.Run(t, new(ChannelCheckerTestSuite)) +} diff --git a/internal/querycoordv2/checkers/checker.go b/internal/querycoordv2/checkers/checker.go new file mode 100644 index 0000000000000..59becddde8084 --- /dev/null +++ b/internal/querycoordv2/checkers/checker.go @@ -0,0 +1,26 @@ +package checkers + +import ( + "context" + + "github.com/milvus-io/milvus/internal/querycoordv2/task" +) + +type Checker interface { + ID() int64 + SetID(id int64) + Description() string + Check(ctx context.Context) []task.Task +} + +type baseChecker struct { + id int64 +} + +func (checker *baseChecker) ID() int64 { + return checker.id +} + +func (checker *baseChecker) SetID(id int64) { + checker.id = id +} diff --git a/internal/querycoordv2/checkers/controller.go b/internal/querycoordv2/checkers/controller.go new file mode 100644 index 0000000000000..591bb74eb6b0d --- /dev/null +++ b/internal/querycoordv2/checkers/controller.go @@ -0,0 +1,105 @@ +package checkers + +import ( + "context" + "time" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/querycoordv2/balance" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/task" + "go.uber.org/zap" +) + +var ( + checkRoundTaskNumLimit = 128 +) + +type CheckerController struct { + stopCh chan struct{} + meta *meta.Meta + dist *meta.DistributionManager + targetMgr *meta.TargetManager + broker *meta.CoordinatorBroker + nodeMgr *session.NodeManager + balancer balance.Balance + + scheduler task.Scheduler + checkers []Checker +} + +func NewCheckerController( + meta *meta.Meta, + dist *meta.DistributionManager, + targetMgr *meta.TargetManager, + balancer balance.Balance, + scheduler task.Scheduler) *CheckerController { + + // CheckerController runs checkers with the order, + // the former checker has higher priority + checkers := []Checker{ + NewChannelChecker(meta, dist, targetMgr, balancer), + NewSegmentChecker(meta, dist, targetMgr, balancer), + NewBalanceChecker(balancer), + } + for i, checker := range checkers { + checker.SetID(int64(i + 1)) + } + + return &CheckerController{ + stopCh: make(chan struct{}), + meta: meta, + dist: dist, + targetMgr: targetMgr, + scheduler: scheduler, + checkers: checkers, + } +} + +func (controller *CheckerController) Start(ctx context.Context) { + go func() { + ticker := time.NewTicker(Params.QueryCoordCfg.CheckInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + log.Info("CheckerController stopped due to context canceled") + return + + case <-controller.stopCh: + log.Info("CheckerController stopped") + return + + case <-ticker.C: + controller.check(ctx) + } + } + }() +} + +func (controller *CheckerController) Stop() { + close(controller.stopCh) +} + +// check is the real implementation of Check +func (controller *CheckerController) check(ctx context.Context) { + tasks := make([]task.Task, 0) + for id, checker := range controller.checkers { + log := log.With(zap.Int("checkerID", id)) + + tasks = append(tasks, checker.Check(ctx)...) + if len(tasks) >= checkRoundTaskNumLimit { + log.Info("checkers have spawn too many tasks, won't run subsequent checkers, and truncate the spawned tasks", + zap.Int("taskNum", len(tasks)), + zap.Int("taskNumLimit", checkRoundTaskNumLimit)) + tasks = tasks[:checkRoundTaskNumLimit] + break + } + } + + for _, task := range tasks { + controller.scheduler.Add(task) + } +} diff --git a/internal/querycoordv2/checkers/segment_checker.go b/internal/querycoordv2/checkers/segment_checker.go new file mode 100644 index 0000000000000..4f36fd9d31942 --- /dev/null +++ b/internal/querycoordv2/checkers/segment_checker.go @@ -0,0 +1,233 @@ +package checkers + +import ( + "context" + + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/balance" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/task" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/internal/util/typeutil" +) + +type SegmentChecker struct { + baseChecker + meta *meta.Meta + dist *meta.DistributionManager + targetMgr *meta.TargetManager + balancer balance.Balance +} + +func NewSegmentChecker( + meta *meta.Meta, + dist *meta.DistributionManager, + targetMgr *meta.TargetManager, + balancer balance.Balance, +) *SegmentChecker { + return &SegmentChecker{ + meta: meta, + dist: dist, + targetMgr: targetMgr, + balancer: balancer, + } +} + +func (c *SegmentChecker) Description() string { + return "SegmentChecker checks the lack of segments, or some segments are redundant" +} + +func (c *SegmentChecker) Check(ctx context.Context) []task.Task { + collectionIDs := c.meta.CollectionManager.GetAll() + tasks := make([]task.Task, 0) + for _, cid := range collectionIDs { + replicas := c.meta.ReplicaManager.GetByCollection(cid) + for _, r := range replicas { + tasks = append(tasks, c.checkReplica(ctx, r)...) + } + } + + // find already released segments which are not contained in target + segments := c.dist.SegmentDistManager.GetAll() + released := utils.FilterReleased(segments, collectionIDs) + tasks = append(tasks, c.createSegmentReduceTasks(ctx, released, -1, querypb.DataScope_All)...) + return tasks +} + +func (c *SegmentChecker) checkReplica(ctx context.Context, replica *meta.Replica) []task.Task { + ret := make([]task.Task, 0) + targets := c.targetMgr.GetSegmentsByCollection(replica.CollectionID) + dists := c.getSegmentsDist(replica) + + // compare with targets to find the lack and redundancy of segments + lacks, redundancies := diffSegments(targets, dists) + tasks := c.createSegmentLoadTasks(ctx, lacks, replica) + ret = append(ret, tasks...) + + tasks = c.createSegmentReduceTasks(ctx, redundancies, replica.GetID(), querypb.DataScope_All) + ret = append(ret, tasks...) + + // compare inner dists to find repeated loaded segments + redundancies = findRepeatedSegments(dists) + redundancies = c.filterExistedOnLeader(replica, redundancies) + tasks = c.createSegmentReduceTasks(ctx, redundancies, replica.GetID(), querypb.DataScope_All) + ret = append(ret, tasks...) + + // release redundant growing segments + leaderReduancies := c.findNeedReleasedGrowingSegments(replica) + redundancies = make([]*meta.Segment, 0) + for leaderID, segmentIDs := range leaderReduancies { + segments := packSegments(segmentIDs, leaderID, replica.GetCollectionID()) + redundancies = append(redundancies, segments...) + } + tasks = c.createSegmentReduceTasks(ctx, redundancies, replica.GetID(), querypb.DataScope_Streaming) + ret = append(ret, tasks...) + + return ret +} + +func (c *SegmentChecker) getSegmentsDist(replica *meta.Replica) []*meta.Segment { + ret := make([]*meta.Segment, 0) + for _, node := range replica.Nodes.Collect() { + ret = append(ret, c.dist.SegmentDistManager.GetByCollectionAndNode(replica.CollectionID, node)...) + } + return ret +} + +func diffSegments(targets []*datapb.SegmentInfo, dists []*meta.Segment) (lacks []*datapb.SegmentInfo, redundancies []*meta.Segment) { + distMap := typeutil.NewUniqueSet() + targetMap := typeutil.NewUniqueSet() + for _, s := range targets { + targetMap.Insert(s.GetID()) + } + for _, s := range dists { + distMap.Insert(s.GetID()) + if !targetMap.Contain(s.GetID()) { + redundancies = append(redundancies, s) + } + } + for _, s := range targets { + if !distMap.Contain(s.GetID()) { + lacks = append(lacks, s) + } + } + return +} + +func findRepeatedSegments(dists []*meta.Segment) []*meta.Segment { + ret := make([]*meta.Segment, 0) + versions := make(map[int64]*meta.Segment) + for _, s := range dists { + maxVer, ok := versions[s.GetID()] + if !ok { + versions[s.GetID()] = s + continue + } + if maxVer.Version <= s.Version { + ret = append(ret, maxVer) + versions[s.GetID()] = s + } else { + ret = append(ret, s) + } + } + return ret +} + +func (c *SegmentChecker) filterExistedOnLeader(replica *meta.Replica, segments []*meta.Segment) []*meta.Segment { + filtered := make([]*meta.Segment, 0, len(segments)) + for _, s := range segments { + leaderID, ok := c.dist.ChannelDistManager.GetShardLeader(replica, s.GetInsertChannel()) + if !ok { + continue + } + onLeader := false + leaderViews := c.dist.LeaderViewManager.GetLeaderView(leaderID) + for _, view := range leaderViews { + node, ok := view.Segments[s.GetID()] + if ok && node == s.Node { + onLeader = true + break + } + } + if onLeader { + // if this segment is serving on leader, do not remove it for search available + continue + } + filtered = append(filtered, s) + } + return filtered +} + +func (c *SegmentChecker) findNeedReleasedGrowingSegments(replica *meta.Replica) map[int64][]int64 { + ret := make(map[int64][]int64, 0) // leaderID -> segment ids + leaders := c.dist.ChannelDistManager.GetShardLeadersByReplica(replica) + for shard, leaderID := range leaders { + lview := c.dist.LeaderViewManager.GetLeaderShardView(leaderID, shard) + // find growing segments from leaderview's sealed segments + // because growing segments should be released only after loading the compaction created segment successfully. + for sid := range lview.Segments { + segment := c.targetMgr.GetSegment(sid) + if segment == nil { + continue + } + sources := append(segment.GetCompactionFrom(), segment.GetID()) + for _, source := range sources { + if lview.GrowingSegments.Contain(source) { + ret[lview.ID] = append(ret[lview.ID], source) + } + } + } + } + return ret +} + +func packSegments(segmentIDs []int64, nodeID int64, collectionID int64) []*meta.Segment { + ret := make([]*meta.Segment, 0, len(segmentIDs)) + for _, id := range segmentIDs { + segment := &meta.Segment{ + SegmentInfo: &datapb.SegmentInfo{ + ID: id, + CollectionID: collectionID, + }, + Node: nodeID, + } + ret = append(ret, segment) + } + return ret +} + +func (c *SegmentChecker) createSegmentLoadTasks(ctx context.Context, segments []*datapb.SegmentInfo, replica *meta.Replica) []task.Task { + if len(segments) == 0 { + return nil + } + packedSegments := make([]*meta.Segment, 0, len(segments)) + for _, s := range segments { + if len(c.dist.LeaderViewManager.GetLeadersByShard(s.GetInsertChannel())) == 0 { + continue + } + packedSegments = append(packedSegments, &meta.Segment{SegmentInfo: s}) + } + plans := c.balancer.AssignSegment(packedSegments, replica.Replica.GetNodes()) + for i := range plans { + plans[i].ReplicaID = replica.GetID() + } + return balance.CreateSegmentTasksFromPlans(ctx, c.ID(), Params.QueryCoordCfg.SegmentTaskTimeout, plans) +} + +func (c *SegmentChecker) createSegmentReduceTasks(ctx context.Context, segments []*meta.Segment, replicaID int64, scope querypb.DataScope) []task.Task { + ret := make([]task.Task, 0, len(segments)) + for _, s := range segments { + action := task.NewSegmentActionWithScope(s.Node, task.ActionTypeReduce, s.GetID(), scope) + ret = append(ret, task.NewSegmentTask( + ctx, + Params.QueryCoordCfg.SegmentTaskTimeout, + c.ID(), + s.GetCollectionID(), + replicaID, + action, + )) + } + return ret +} diff --git a/internal/querycoordv2/checkers/segment_checker_test.go b/internal/querycoordv2/checkers/segment_checker_test.go new file mode 100644 index 0000000000000..3575c3a3b9d1f --- /dev/null +++ b/internal/querycoordv2/checkers/segment_checker_test.go @@ -0,0 +1,195 @@ +package checkers + +import ( + "context" + "testing" + + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/internal/querycoordv2/balance" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/task" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/internal/util/etcd" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type SegmentCheckerTestSuite struct { + suite.Suite + kv *etcdkv.EtcdKV + checker *SegmentChecker +} + +func (suite *SegmentCheckerTestSuite) SetupSuite() { + Params.Init() +} + +func (suite *SegmentCheckerTestSuite) SetupTest() { + var err error + config := GenerateEtcdConfig() + cli, err := etcd.GetEtcdClient(&config) + suite.Require().NoError(err) + suite.kv = etcdkv.NewEtcdKV(cli, config.MetaRootPath) + + // meta + store := meta.NewMetaStore(suite.kv) + idAllocator := RandomIncrementIDAllocator() + testMeta := meta.NewMeta(idAllocator, store) + + distManager := meta.NewDistributionManager() + targetManager := meta.NewTargetManager() + + balancer := suite.createMockBalancer() + suite.checker = NewSegmentChecker(testMeta, distManager, targetManager, balancer) +} + +func (suite *SegmentCheckerTestSuite) TearDownTest() { + suite.kv.Close() +} + +func (suite *SegmentCheckerTestSuite) createMockBalancer() balance.Balance { + balancer := balance.NewMockBalancer(suite.T()) + balancer.EXPECT().AssignSegment(mock.Anything, mock.Anything).Maybe().Return(func(segments []*meta.Segment, nodes []int64) []balance.SegmentAssignPlan { + plans := make([]balance.SegmentAssignPlan, 0, len(segments)) + for i, s := range segments { + plan := balance.SegmentAssignPlan{ + Segment: s, + From: -1, + To: nodes[i%len(nodes)], + ReplicaID: -1, + } + plans = append(plans, plan) + } + return plans + }) + return balancer +} + +func (suite *SegmentCheckerTestSuite) TestLoadSegments() { + checker := suite.checker + // set meta + checker.meta.CollectionManager.PutCollection(utils.CreateTestCollection(1, 1)) + checker.meta.ReplicaManager.Put(utils.CreateTestReplica(1, 1, []int64{1, 2})) + + // set target + checker.targetMgr.AddSegment(utils.CreateTestSegmentInfo(1, 1, 1, "test-insert-channel")) + + // set dist + checker.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel")) + checker.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, []int64{})) + + tasks := checker.Check(context.TODO()) + suite.Len(tasks, 1) + suite.Len(tasks[0].Actions(), 1) + action, ok := tasks[0].Actions()[0].(*task.SegmentAction) + suite.True(ok) + suite.EqualValues(1, tasks[0].ReplicaID()) + suite.Equal(task.ActionTypeGrow, action.Type()) + suite.EqualValues(1, action.SegmentID()) + +} + +func (suite *SegmentCheckerTestSuite) TestReleaseSegments() { + checker := suite.checker + // set meta + checker.meta.CollectionManager.PutCollection(utils.CreateTestCollection(1, 1)) + checker.meta.ReplicaManager.Put(utils.CreateTestReplica(1, 1, []int64{1, 2})) + + // set dist + checker.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel")) + checker.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, []int64{})) + checker.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 2, 1, 1, "test-insert-channel")) + + tasks := checker.Check(context.TODO()) + suite.Len(tasks, 1) + suite.Len(tasks[0].Actions(), 1) + action, ok := tasks[0].Actions()[0].(*task.SegmentAction) + suite.True(ok) + suite.EqualValues(1, tasks[0].ReplicaID()) + suite.Equal(task.ActionTypeReduce, action.Type()) + suite.EqualValues(2, action.SegmentID()) +} + +func (suite *SegmentCheckerTestSuite) TestReleaseRepeatedSegments() { + checker := suite.checker + // set meta + checker.meta.CollectionManager.PutCollection(utils.CreateTestCollection(1, 1)) + checker.meta.ReplicaManager.Put(utils.CreateTestReplica(1, 1, []int64{1, 2})) + + // set target + checker.targetMgr.AddSegment(utils.CreateTestSegmentInfo(1, 1, 1, "test-insert-channel")) + + // set dist + checker.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel")) + checker.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{1: 2}, []int64{})) + checker.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 1, 1, "test-insert-channel")) + checker.dist.SegmentDistManager.Update(2, utils.CreateTestSegment(1, 1, 1, 1, 2, "test-insert-channel")) + + tasks := checker.Check(context.TODO()) + suite.Len(tasks, 1) + suite.Len(tasks[0].Actions(), 1) + action, ok := tasks[0].Actions()[0].(*task.SegmentAction) + suite.True(ok) + suite.EqualValues(1, tasks[0].ReplicaID()) + suite.Equal(task.ActionTypeReduce, action.Type()) + suite.EqualValues(1, action.SegmentID()) + suite.EqualValues(1, action.Node()) + + // test less version exist on leader + checker.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{1: 1}, []int64{})) + tasks = checker.Check(context.TODO()) + suite.Len(tasks, 0) +} + +func (suite *SegmentCheckerTestSuite) TestReleaseGrowingSegments() { + checker := suite.checker + // segment3 is compacted from segment2, and node2 has growing segments 2 and 3. checker should generate + // 2 tasks to reduce segment 2 and 3. + checker.meta.CollectionManager.PutCollection(utils.CreateTestCollection(1, 1)) + checker.meta.ReplicaManager.Put(utils.CreateTestReplica(1, 1, []int64{1, 2})) + + segment := utils.CreateTestSegmentInfo(1, 1, 3, "test-insert-channel") + segment.CompactionFrom = append(segment.CompactionFrom, 2) + checker.targetMgr.AddSegment(segment) + + checker.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel")) + checker.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{3: 2}, []int64{2, 3})) + checker.dist.SegmentDistManager.Update(2, utils.CreateTestSegment(1, 1, 3, 2, 1, "test-insert-channel")) + + tasks := checker.Check(context.TODO()) + suite.Len(tasks, 2) + suite.Len(tasks[0].Actions(), 1) + action, ok := tasks[0].Actions()[0].(*task.SegmentAction) + suite.True(ok) + suite.EqualValues(1, tasks[0].ReplicaID()) + suite.Equal(task.ActionTypeReduce, action.Type()) + suite.EqualValues(2, action.SegmentID()) + suite.EqualValues(2, action.Node()) + + suite.Len(tasks[1].Actions(), 1) + action, ok = tasks[1].Actions()[0].(*task.SegmentAction) + suite.True(ok) + suite.EqualValues(1, tasks[1].ReplicaID()) + suite.Equal(task.ActionTypeReduce, action.Type()) + suite.EqualValues(3, action.SegmentID()) + suite.EqualValues(2, action.Node()) +} + +func (suite *SegmentCheckerTestSuite) TestReleaseDroppedSegments() { + checker := suite.checker + checker.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 1, 1, "test-insert-channel")) + tasks := checker.Check(context.TODO()) + suite.Len(tasks, 1) + suite.Len(tasks[0].Actions(), 1) + action, ok := tasks[0].Actions()[0].(*task.SegmentAction) + suite.True(ok) + suite.EqualValues(-1, tasks[0].ReplicaID()) + suite.Equal(task.ActionTypeReduce, action.Type()) + suite.EqualValues(1, action.SegmentID()) + suite.EqualValues(1, action.Node()) +} + +func TestSegmentCheckerSuite(t *testing.T) { + suite.Run(t, new(SegmentCheckerTestSuite)) +} diff --git a/internal/querycoordv2/dist/dist_controller.go b/internal/querycoordv2/dist/dist_controller.go new file mode 100644 index 0000000000000..e6be9d1090611 --- /dev/null +++ b/internal/querycoordv2/dist/dist_controller.go @@ -0,0 +1,83 @@ +package dist + +import ( + "context" + "sync" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/task" + "go.uber.org/zap" +) + +type Controller struct { + mu sync.RWMutex + handlers map[int64]*distHandler + client session.Cluster + nodeManager *session.NodeManager + dist *meta.DistributionManager + targetMgr *meta.TargetManager + scheduler task.Scheduler +} + +func (dc *Controller) StartDistInstance(ctx context.Context, nodeID int64) { + dc.mu.Lock() + defer dc.mu.Unlock() + if _, ok := dc.handlers[nodeID]; ok { + log.Info("node has started", zap.Int64("nodeID", nodeID)) + return + } + h := newDistHandler(ctx, nodeID, dc.client, dc.nodeManager, dc.scheduler, dc.dist, dc.targetMgr) + dc.handlers[nodeID] = h +} + +func (dc *Controller) Remove(nodeID int64) { + dc.mu.Lock() + defer dc.mu.Unlock() + if h, ok := dc.handlers[nodeID]; ok { + h.stop() + delete(dc.handlers, nodeID) + } +} + +func (dc *Controller) SyncAll(ctx context.Context) { + dc.mu.RLock() + defer dc.mu.RUnlock() + + wg := sync.WaitGroup{} + for _, h := range dc.handlers { + handler := h + wg.Add(1) + go func() { + defer wg.Done() + handler.getDistribution(ctx) + }() + } + wg.Wait() +} + +func (dc *Controller) Stop() { + dc.mu.Lock() + defer dc.mu.Unlock() + for _, h := range dc.handlers { + h.stop() + } +} + +func NewDistController( + client session.Cluster, + nodeManager *session.NodeManager, + dist *meta.DistributionManager, + targetMgr *meta.TargetManager, + scheduler task.Scheduler, +) *Controller { + return &Controller{ + handlers: make(map[int64]*distHandler), + client: client, + nodeManager: nodeManager, + dist: dist, + targetMgr: targetMgr, + scheduler: scheduler, + } +} diff --git a/internal/querycoordv2/dist/dist_controller_test.go b/internal/querycoordv2/dist/dist_controller_test.go new file mode 100644 index 0000000000000..ec53aed2fa651 --- /dev/null +++ b/internal/querycoordv2/dist/dist_controller_test.go @@ -0,0 +1,121 @@ +package dist + +import ( + "context" + "testing" + "time" + + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/task" + "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "go.uber.org/atomic" +) + +type DistControllerTestSuite struct { + suite.Suite + controller *Controller + mockCluster *session.MockCluster + mockScheduler *task.MockScheduler +} + +func (suite *DistControllerTestSuite) SetupTest() { + Params.Init() + + suite.mockCluster = session.NewMockCluster(suite.T()) + nodeManager := session.NewNodeManager() + distManager := meta.NewDistributionManager() + targetManager := meta.NewTargetManager() + suite.mockScheduler = task.NewMockScheduler(suite.T()) + suite.controller = NewDistController(suite.mockCluster, nodeManager, distManager, targetManager, suite.mockScheduler) +} + +func (suite *DistControllerTestSuite) TestStart() { + dispatchCalled := atomic.NewBool(false) + suite.mockCluster.EXPECT().GetDataDistribution(mock.Anything, mock.Anything, mock.Anything).Return( + &querypb.GetDataDistributionResponse{NodeID: 1}, + nil, + ) + suite.mockScheduler.EXPECT().Dispatch(int64(1)).Run(func(node int64) { dispatchCalled.Store(true) }) + suite.controller.StartDistInstance(context.TODO(), 1) + suite.Eventually( + func() bool { + return dispatchCalled.Load() + }, + 10*time.Second, + 500*time.Millisecond, + ) + + suite.controller.Remove(1) + dispatchCalled.Store(false) + suite.Never( + func() bool { + return dispatchCalled.Load() + }, + 3*time.Second, + 500*time.Millisecond, + ) +} + +func (suite *DistControllerTestSuite) TestStop() { + suite.controller.StartDistInstance(context.TODO(), 1) + called := atomic.NewBool(false) + suite.mockCluster.EXPECT().GetDataDistribution(mock.Anything, mock.Anything, mock.Anything).Maybe().Return( + &querypb.GetDataDistributionResponse{NodeID: 1}, + nil, + ).Run(func(args mock.Arguments) { + called.Store(true) + }) + suite.mockScheduler.EXPECT().Dispatch(mock.Anything).Maybe() + suite.controller.Stop() + called.Store(false) + suite.Never( + func() bool { + return called.Load() + }, + 3*time.Second, + 500*time.Millisecond, + ) +} + +func (suite *DistControllerTestSuite) TestSyncAll() { + suite.controller.StartDistInstance(context.TODO(), 1) + suite.controller.StartDistInstance(context.TODO(), 2) + + calledSet := typeutil.NewConcurrentSet[int64]() + suite.mockCluster.EXPECT().GetDataDistribution(mock.Anything, mock.Anything, mock.Anything).Call.Return( + func(ctx context.Context, nodeID int64, req *querypb.GetDataDistributionRequest) *querypb.GetDataDistributionResponse { + return &querypb.GetDataDistributionResponse{ + NodeID: nodeID, + } + }, + nil, + ).Run(func(args mock.Arguments) { + calledSet.Insert(args[1].(int64)) + }) + suite.mockScheduler.EXPECT().Dispatch(mock.Anything) + + // stop inner loop + suite.controller.handlers[1].stop() + suite.controller.handlers[2].stop() + + calledSet.Remove(1) + calledSet.Remove(2) + + suite.controller.SyncAll(context.TODO()) + suite.Eventually( + func() bool { + return calledSet.Contain(1) && calledSet.Contain(2) + }, + 5*time.Second, + 500*time.Millisecond, + ) +} + +func TestDistControllerSuite(t *testing.T) { + suite.Run(t, new(DistControllerTestSuite)) +} diff --git a/internal/querycoordv2/dist/dist_handler.go b/internal/querycoordv2/dist/dist_handler.go new file mode 100644 index 0000000000000..e3925c6664c94 --- /dev/null +++ b/internal/querycoordv2/dist/dist_handler.go @@ -0,0 +1,214 @@ +package dist + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/golang/protobuf/proto" + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/task" + "github.com/milvus-io/milvus/internal/util/typeutil" + "go.uber.org/zap" +) + +const ( + distReqTimeout = 3 * time.Second + maxFailureTimes = 3 +) + +type distHandler struct { + nodeID int64 + c chan struct{} + wg sync.WaitGroup + client session.Cluster + nodeManager *session.NodeManager + scheduler task.Scheduler + dist *meta.DistributionManager + target *meta.TargetManager + mu sync.Mutex +} + +func (dh *distHandler) start(ctx context.Context) { + defer dh.wg.Done() + logger := log.With(zap.Int64("nodeID", dh.nodeID)) + logger.Info("start dist handler") + ticker := time.NewTicker(Params.QueryCoordCfg.DistPullInterval) + id := int64(1) + failures := 0 + for { + select { + case <-ctx.Done(): + logger.Info("close dist handler due to context done") + return + case <-dh.c: + logger.Info("close dist handelr") + return + case <-ticker.C: + dh.mu.Lock() + cctx, cancel := context.WithTimeout(ctx, distReqTimeout) + resp, err := dh.client.GetDataDistribution(cctx, dh.nodeID, &querypb.GetDataDistributionRequest{}) + cancel() + + if err != nil || resp.GetStatus().GetErrorCode() != commonpb.ErrorCode_Success { + failures++ + dh.logFailureInfo(resp, err) + } else { + failures = 0 + dh.handleDistResp(resp) + } + + if failures >= maxFailureTimes { + log.RatedInfo(30.0, fmt.Sprintf("can not get data distribution from node %d for %d times", dh.nodeID, failures)) + // TODO: kill the querynode server and stop the loop? + } + id++ + dh.mu.Unlock() + } + } +} + +func (dh *distHandler) logFailureInfo(resp *querypb.GetDataDistributionResponse, err error) { + log.With(zap.Int64("nodeID", dh.nodeID)) + if err != nil { + log.Warn("failed to get data distribution", + zap.Error(err)) + } else if resp.GetStatus().GetErrorCode() != commonpb.ErrorCode_Success { + log.Warn("failed to get data distribution", + zap.Any("errorCode", resp.GetStatus().GetErrorCode()), + zap.Any("reason", resp.GetStatus().GetReason())) + } +} + +func (dh *distHandler) handleDistResp(resp *querypb.GetDataDistributionResponse) { + node := dh.nodeManager.Get(resp.GetNodeID()) + if node != nil { + node.UpdateStats( + session.WithSegmentCnt(len(resp.GetSegments())), + session.WithChannelCnt(len(resp.GetChannels())), + ) + } + + dh.updateSegmentsDistribution(resp) + dh.updateChannelsDistribution(resp) + dh.updateLeaderView(resp) + + dh.scheduler.Dispatch(dh.nodeID) +} + +func (dh *distHandler) updateSegmentsDistribution(resp *querypb.GetDataDistributionResponse) { + updates := make([]*meta.Segment, 0, len(resp.GetSegments())) + for _, s := range resp.GetSegments() { + segmentInfo := dh.target.GetSegment(s.GetID()) + var segment *meta.Segment + if segmentInfo == nil { + segment = &meta.Segment{ + SegmentInfo: &datapb.SegmentInfo{ + ID: s.GetID(), + CollectionID: s.GetCollection(), + PartitionID: s.GetPartition(), + InsertChannel: s.GetChannel(), + }, + Node: resp.GetNodeID(), + Version: s.GetVersion(), + } + } else { + segment = &meta.Segment{ + SegmentInfo: proto.Clone(segmentInfo).(*datapb.SegmentInfo), + Node: resp.GetNodeID(), + Version: s.GetVersion(), + } + } + updates = append(updates, segment) + } + + dh.dist.SegmentDistManager.Update(resp.GetNodeID(), updates...) +} + +func (dh *distHandler) updateChannelsDistribution(resp *querypb.GetDataDistributionResponse) { + updates := make([]*meta.DmChannel, 0, len(resp.GetChannels())) + for _, ch := range resp.GetChannels() { + channelInfo := dh.target.GetDmChannel(ch.GetChannel()) + var channel *meta.DmChannel + if channelInfo == nil { + channel = &meta.DmChannel{ + VchannelInfo: &datapb.VchannelInfo{ + ChannelName: ch.GetChannel(), + CollectionID: ch.GetCollection(), + }, + Node: resp.GetNodeID(), + Version: ch.GetVersion(), + } + } else { + channel = channelInfo.Clone() + } + updates = append(updates, channel) + } + + dh.dist.ChannelDistManager.Update(resp.GetNodeID(), updates...) +} + +func (dh *distHandler) updateLeaderView(resp *querypb.GetDataDistributionResponse) { + updates := make([]*meta.LeaderView, 0, len(resp.GetLeaderViews())) + for _, lview := range resp.GetLeaderViews() { + view := &meta.LeaderView{ + ID: resp.GetNodeID(), + CollectionID: lview.GetCollection(), + Channel: lview.GetChannel(), + Segments: lview.GetSegmentNodePairs(), + GrowingSegments: typeutil.NewUniqueSet(resp.GetGrowingSegmentIDs()...), + } + updates = append(updates, view) + } + + dh.dist.LeaderViewManager.Update(resp.GetNodeID(), updates...) +} + +func (dh *distHandler) getDistribution(ctx context.Context) { + dh.mu.Lock() + defer dh.mu.Unlock() + cctx, cancel := context.WithTimeout(ctx, distReqTimeout) + resp, err := dh.client.GetDataDistribution(cctx, dh.nodeID, &querypb.GetDataDistributionRequest{}) + cancel() + + if err != nil || resp.GetStatus().GetErrorCode() != commonpb.ErrorCode_Success { + dh.logFailureInfo(resp, err) + } else { + dh.handleDistResp(resp) + } +} + +func (dh *distHandler) stop() { + close(dh.c) + dh.wg.Wait() +} + +func newDistHandler( + ctx context.Context, + nodeID int64, + client session.Cluster, + nodeManager *session.NodeManager, + scheduler task.Scheduler, + dist *meta.DistributionManager, + targetMgr *meta.TargetManager, +) *distHandler { + h := &distHandler{ + nodeID: nodeID, + c: make(chan struct{}), + client: client, + nodeManager: nodeManager, + scheduler: scheduler, + dist: dist, + target: targetMgr, + } + h.wg.Add(1) + go h.start(ctx) + return h +} diff --git a/internal/querycoordv2/errors.go b/internal/querycoordv2/errors.go new file mode 100644 index 0000000000000..8deee5bb3066f --- /dev/null +++ b/internal/querycoordv2/errors.go @@ -0,0 +1,7 @@ +package querycoordv2 + +import "errors" + +var ( + ErrNotHealthy = errors.New("NotHealthy") +) diff --git a/internal/querycoordv2/handlers.go b/internal/querycoordv2/handlers.go new file mode 100644 index 0000000000000..56d45f2b63e4b --- /dev/null +++ b/internal/querycoordv2/handlers.go @@ -0,0 +1,288 @@ +package querycoordv2 + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/task" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/internal/util/metricsinfo" + "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/milvus-io/milvus/internal/util/uniquegenerator" + "github.com/samber/lo" + "go.uber.org/zap" +) + +// checkAnyReplicaAvailable checks if the collection has enough distinct available shards. These shards +// may come from different replica group. We only need these shards to form a replica that serves query +// requests. +func (s *Server) checkAnyReplicaAvailable(collectionID int64) bool { + for _, replica := range s.meta.ReplicaManager.GetByCollection(collectionID) { + isAvailable := true + for node := range replica.Nodes { + if s.nodeMgr.Get(node) == nil { + isAvailable = false + break + } + } + if isAvailable { + return true + } + } + return false +} + +func (s *Server) getCollectionSegmentInfo(collection int64) []*querypb.SegmentInfo { + segments := s.dist.SegmentDistManager.GetByCollection(collection) + infos := make(map[int64]*querypb.SegmentInfo) + for _, segment := range segments { + info, ok := infos[segment.GetID()] + if !ok { + info = &querypb.SegmentInfo{} + infos[segment.GetID()] = info + } + utils.MergeMetaSegmentIntoSegmentInfo(info, segment) + } + + return lo.Values(infos) +} + +// parseBalanceRequest parses the load balance request, +// returns the collection, replica, and segments +func (s *Server) balanceSegments(ctx context.Context, req *querypb.LoadBalanceRequest, replica *meta.Replica) error { + const ( + manualBalanceTimeout = 10 * time.Second + ) + + srcNode := req.GetSourceNodeIDs()[0] + dstNodeSet := typeutil.NewUniqueSet(req.GetDstNodeIDs()...) + if dstNodeSet.Len() == 0 { + dstNodeSet.Insert(replica.GetNodes()...) + } + dstNodeSet.Remove(srcNode) + + sealedSegmentSet := typeutil.NewUniqueSet(req.GetSealedSegmentIDs()...) + toBalance := typeutil.NewSet[*meta.Segment]() + segments := s.dist.SegmentDistManager.GetByNode(srcNode) + if len(req.GetSealedSegmentIDs()) == 0 { + toBalance.Insert(segments...) + } else { + for _, segment := range segments { + if !sealedSegmentSet.Contain(segment.GetID()) { + return fmt.Errorf("segment %d not found in source node %d", segment.GetID(), srcNode) + } + toBalance.Insert(segment) + } + } + + plans := s.balancer.AssignSegment(toBalance.Collect(), dstNodeSet.Collect()) + tasks := make([]task.Task, 0, len(plans)) + for _, plan := range plans { + task := task.NewSegmentTask(ctx, + manualBalanceTimeout, + req.Base.GetMsgID(), + req.GetCollectionID(), + replica.GetID(), + task.NewSegmentAction(plan.To, task.ActionTypeGrow, plan.Segment.GetID()), + task.NewSegmentAction(plan.From, task.ActionTypeReduce, plan.Segment.GetID()), + ) + err := s.taskScheduler.Add(task) + if err != nil { + return err + } + tasks = append(tasks, task) + } + return task.Wait(ctx, manualBalanceTimeout, tasks...) +} + +// TODO(dragondriver): add more detail metrics +func (s *Server) getSystemInfoMetrics( + ctx context.Context, + req *milvuspb.GetMetricsRequest) (string, error) { + + clusterTopology := metricsinfo.QueryClusterTopology{ + Self: metricsinfo.QueryCoordInfos{ + BaseComponentInfos: metricsinfo.BaseComponentInfos{ + Name: metricsinfo.ConstructComponentName(typeutil.QueryCoordRole, Params.QueryCoordCfg.GetNodeID()), + HardwareInfos: metricsinfo.HardwareMetrics{ + IP: s.session.Address, + CPUCoreCount: metricsinfo.GetCPUCoreCount(false), + CPUCoreUsage: metricsinfo.GetCPUUsage(), + Memory: metricsinfo.GetMemoryCount(), + MemoryUsage: metricsinfo.GetUsedMemoryCount(), + Disk: metricsinfo.GetDiskCount(), + DiskUsage: metricsinfo.GetDiskUsage(), + }, + SystemInfo: metricsinfo.DeployMetrics{}, + CreatedTime: Params.QueryCoordCfg.CreatedTime.String(), + UpdatedTime: Params.QueryCoordCfg.UpdatedTime.String(), + Type: typeutil.QueryCoordRole, + ID: s.session.ServerID, + }, + SystemConfigurations: metricsinfo.QueryCoordConfiguration{ + SearchChannelPrefix: Params.CommonCfg.QueryCoordSearch, + SearchResultChannelPrefix: Params.CommonCfg.QueryCoordSearchResult, + }, + }, + ConnectedNodes: make([]metricsinfo.QueryNodeInfos, 0), + } + metricsinfo.FillDeployMetricsWithEnv(&clusterTopology.Self.SystemInfo) + nodesMetrics := s.tryGetNodesMetrics(ctx, req, s.nodeMgr.GetAll()...) + s.fillMetricsWithNodes(&clusterTopology, nodesMetrics) + + coordTopology := metricsinfo.QueryCoordTopology{ + Cluster: clusterTopology, + Connections: metricsinfo.ConnTopology{ + Name: metricsinfo.ConstructComponentName(typeutil.QueryCoordRole, Params.QueryCoordCfg.GetNodeID()), + // TODO(dragondriver): fill ConnectedComponents if necessary + ConnectedComponents: []metricsinfo.ConnectionInfo{}, + }, + } + + resp, err := metricsinfo.MarshalTopology(coordTopology) + if err != nil { + return "", err + } + + return resp, nil +} + +func (s *Server) fillMetricsWithNodes(topo *metricsinfo.QueryClusterTopology, nodeMetrics []*metricResp) { + for _, metric := range nodeMetrics { + if metric.err != nil { + log.Warn("invalid metrics of query node was found", + zap.Error(metric.err)) + topo.ConnectedNodes = append(topo.ConnectedNodes, metricsinfo.QueryNodeInfos{ + BaseComponentInfos: metricsinfo.BaseComponentInfos{ + HasError: true, + ErrorReason: metric.err.Error(), + // Name doesn't matter here because we can't get it when error occurs, using address as the Name? + Name: "", + ID: int64(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()), + }, + }) + continue + } + + if metric.resp.Status.ErrorCode != commonpb.ErrorCode_Success { + log.Warn("invalid metrics of query node was found", + zap.Any("error_code", metric.resp.Status.ErrorCode), + zap.Any("error_reason", metric.resp.Status.Reason)) + topo.ConnectedNodes = append(topo.ConnectedNodes, metricsinfo.QueryNodeInfos{ + BaseComponentInfos: metricsinfo.BaseComponentInfos{ + HasError: true, + ErrorReason: metric.resp.Status.Reason, + Name: metric.resp.ComponentName, + ID: int64(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()), + }, + }) + continue + } + + infos := metricsinfo.QueryNodeInfos{} + err := metricsinfo.UnmarshalComponentInfos(metric.resp.Response, &infos) + if err != nil { + log.Warn("invalid metrics of query node was found", + zap.Error(err)) + topo.ConnectedNodes = append(topo.ConnectedNodes, metricsinfo.QueryNodeInfos{ + BaseComponentInfos: metricsinfo.BaseComponentInfos{ + HasError: true, + ErrorReason: err.Error(), + Name: metric.resp.ComponentName, + ID: int64(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()), + }, + }) + continue + } + topo.ConnectedNodes = append(topo.ConnectedNodes, infos) + } +} + +type metricResp struct { + resp *milvuspb.GetMetricsResponse + err error +} + +func (s *Server) tryGetNodesMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest, nodes ...*session.NodeInfo) []*metricResp { + wg := sync.WaitGroup{} + ret := make([]*metricResp, 0, len(nodes)) + retCh := make(chan *metricResp, len(nodes)) + for _, node := range nodes { + node := node + wg.Add(1) + go func() { + defer wg.Done() + + resp, err := s.cluster.GetMetrics(ctx, node.ID(), req) + if err != nil { + log.Warn("failed to get metric from QueryNode", + zap.Int64("nodeID", node.ID())) + return + } + retCh <- &metricResp{ + resp: resp, + err: err, + } + }() + } + wg.Wait() + close(retCh) + for resp := range retCh { + ret = append(ret, resp) + } + return ret +} + +func (s *Server) fillReplicaInfo(replica *meta.Replica, withShardNodes bool) (*milvuspb.ReplicaInfo, error) { + info := utils.Replica2ReplicaInfo(replica.Replica) + + channels := s.targetMgr.GetDmChannelsByCollection(replica.GetCollectionID()) + if len(channels) == 0 { + msg := "failed to get channels, collection not loaded" + log.Warn(msg) + return nil, utils.WrapError(msg, meta.ErrCollectionNotFound) + } + var segments []*meta.Segment + if withShardNodes { + segments = s.dist.SegmentDistManager.GetByCollection(replica.GetCollectionID()) + } + + for _, channel := range channels { + leader, ok := s.dist.ChannelDistManager.GetShardLeader(replica, channel.GetChannelName()) + var leaderInfo *session.NodeInfo + if ok { + leaderInfo = s.nodeMgr.Get(leader) + } + if leaderInfo == nil { + msg := fmt.Sprintf("failed to get shard leader for shard %s, the collection not loaded or leader is offline", channel) + log.Warn(msg) + return nil, utils.WrapError(msg, session.WrapErrNodeNotFound(leader)) + } + + shard := &milvuspb.ShardReplica{ + LeaderID: leader, + LeaderAddr: leaderInfo.Addr(), + DmChannelName: channel.GetChannelName(), + NodeIds: []int64{leader}, + } + if withShardNodes { + shardNodes := lo.FilterMap(segments, func(segment *meta.Segment, _ int) (int64, bool) { + if replica.Nodes.Contain(segment.Node) { + return segment.Node, true + } + return 0, false + }) + shard.NodeIds = append(shard.NodeIds, shardNodes...) + } + info.ShardReplicas = append(info.ShardReplicas, shard) + } + return info, nil +} diff --git a/internal/querycoordv2/job/errors.go b/internal/querycoordv2/job/errors.go new file mode 100644 index 0000000000000..803618ff7a40c --- /dev/null +++ b/internal/querycoordv2/job/errors.go @@ -0,0 +1,12 @@ +package job + +import "errors" + +var ( + // Common errors + ErrInvalidRequest = errors.New("InvalidRequest") + + // Load errors + ErrCollectionLoaded = errors.New("CollectionLoaded") + ErrLoadParameterMismatched = errors.New("LoadParameterMismatched") +) diff --git a/internal/querycoordv2/job/job.go b/internal/querycoordv2/job/job.go new file mode 100644 index 0000000000000..8be5b5aa1e385 --- /dev/null +++ b/internal/querycoordv2/job/job.go @@ -0,0 +1,508 @@ +package job + +import ( + "context" + "fmt" + "time" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/observers" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/samber/lo" + "go.uber.org/zap" +) + +// Job is request of loading/releasing collection/partitions, +// the execution flow is: +// 1. PreExecute() +// 2. Execute(), skip this step if PreExecute() failed +// 3. PostExecute() +type Job interface { + MsgID() int64 + CollectionID() int64 + // PreExecute does checks, DO NOT persists any thing within this stage, + PreExecute() error + // Execute processes the request + Execute() error + // PostExecute clears resources, it will be always processed + PostExecute() + Error() error + SetError(err error) + Done() + Wait() error +} + +type BaseJob struct { + ctx context.Context + msgID int64 + collectionID int64 + err error + doneCh chan struct{} +} + +func NewBaseJob(ctx context.Context, msgID, collectionID int64) *BaseJob { + return &BaseJob{ + ctx: ctx, + msgID: msgID, + collectionID: collectionID, + doneCh: make(chan struct{}), + } +} + +func (job *BaseJob) MsgID() int64 { + return job.msgID +} + +func (job *BaseJob) CollectionID() int64 { + return job.collectionID +} + +func (job *BaseJob) Error() error { + return job.err +} + +func (job *BaseJob) SetError(err error) { + job.err = err +} + +func (job *BaseJob) Done() { + close(job.doneCh) +} + +func (job *BaseJob) Wait() error { + <-job.doneCh + return job.err +} + +func (job *BaseJob) PreExecute() error { + return nil +} + +func (job *BaseJob) PostExecute() {} + +type LoadCollectionJob struct { + *BaseJob + req *querypb.LoadCollectionRequest + + dist *meta.DistributionManager + meta *meta.Meta + targetMgr *meta.TargetManager + broker meta.Broker + nodeMgr *session.NodeManager + handoffObserver *observers.HandoffObserver +} + +func NewLoadCollectionJob( + ctx context.Context, + req *querypb.LoadCollectionRequest, + dist *meta.DistributionManager, + meta *meta.Meta, + targetMgr *meta.TargetManager, + broker meta.Broker, + nodeMgr *session.NodeManager, + handoffObserver *observers.HandoffObserver, +) *LoadCollectionJob { + return &LoadCollectionJob{ + BaseJob: NewBaseJob(ctx, req.Base.GetMsgID(), req.GetCollectionID()), + req: req, + dist: dist, + meta: meta, + targetMgr: targetMgr, + broker: broker, + nodeMgr: nodeMgr, + handoffObserver: handoffObserver, + } +} + +func (job *LoadCollectionJob) PreExecute() error { + req := job.req + log := log.With( + zap.Int64("msgID", req.Base.GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + if req.GetReplicaNumber() <= 0 { + log.Info("request doesn't indicate the number of replicas, set it to 1", + zap.Int32("replicaNumber", req.GetReplicaNumber())) + req.ReplicaNumber = 1 + } + + if job.meta.Exist(req.GetCollectionID()) { + old := job.meta.GetCollection(req.GetCollectionID()) + if old == nil { + msg := "collection with different load type existed, please release it first" + log.Warn(msg) + return utils.WrapError(msg, ErrLoadParameterMismatched) + } + if old.GetReplicaNumber() != req.GetReplicaNumber() { + msg := fmt.Sprintf("collection with different replica number %d existed, release this collection first before changing its replica number", + job.meta.GetReplicaNumber(req.GetCollectionID()), + ) + log.Warn(msg) + return utils.WrapError(msg, ErrLoadParameterMismatched) + } + return ErrCollectionLoaded + } + + return nil +} + +func (job *LoadCollectionJob) Execute() error { + req := job.req + log := log.With( + zap.Int64("msgID", req.GetBase().GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + // Create replicas + // TODO(yah01): store replicas and collection atomically + replicas, err := utils.SpawnReplicas(job.meta.ReplicaManager, + job.nodeMgr, + req.GetCollectionID(), + req.GetReplicaNumber()) + if err != nil { + msg := "failed to spawn replica for collection" + log.Error(msg, zap.Error(err)) + return utils.WrapError(msg, err) + } + for _, replica := range replicas { + log.Info("replica created", + zap.Int64("replicaID", replica.GetID()), + zap.Int64s("nodes", replica.GetNodes())) + } + + // Fetch channels and segments from DataCoord + partitions, err := job.broker.GetPartitions(job.ctx, req.GetCollectionID()) + if err != nil { + msg := "failed to get partitions from RootCoord" + log.Error(msg, zap.Error(err)) + return utils.WrapError(msg, err) + } + + job.handoffObserver.Register(job.CollectionID()) + err = utils.RegisterTargets(job.ctx, + job.targetMgr, + job.broker, + req.GetCollectionID(), + partitions) + if err != nil { + msg := "failed to register channels and segments" + log.Error(msg, zap.Error(err)) + return utils.WrapError(msg, err) + } + job.handoffObserver.StartHandoff(job.CollectionID()) + + err = job.meta.CollectionManager.PutCollection(&meta.Collection{ + CollectionLoadInfo: &querypb.CollectionLoadInfo{ + CollectionID: req.GetCollectionID(), + ReplicaNumber: req.GetReplicaNumber(), + Status: querypb.LoadStatus_Loading, + }, + CreatedAt: time.Now(), + }) + if err != nil { + msg := "failed to store collection" + log.Error(msg, zap.Error(err)) + return utils.WrapError(msg, err) + } + + return nil +} + +func (job *LoadCollectionJob) PostExecute() { + if job.Error() != nil && !job.meta.Exist(job.CollectionID()) { + job.meta.ReplicaManager.RemoveCollection(job.CollectionID()) + job.handoffObserver.Unregister(job.ctx) + job.targetMgr.RemoveCollection(job.req.GetCollectionID()) + } +} + +type ReleaseCollectionJob struct { + *BaseJob + req *querypb.ReleaseCollectionRequest + dist *meta.DistributionManager + meta *meta.Meta + targetMgr *meta.TargetManager + handoffObserver *observers.HandoffObserver +} + +func NewReleaseCollectionJob(ctx context.Context, + req *querypb.ReleaseCollectionRequest, + dist *meta.DistributionManager, + meta *meta.Meta, + targetMgr *meta.TargetManager, + handoffObserver *observers.HandoffObserver, +) *ReleaseCollectionJob { + return &ReleaseCollectionJob{ + BaseJob: NewBaseJob(ctx, req.Base.GetMsgID(), req.GetCollectionID()), + req: req, + dist: dist, + meta: meta, + targetMgr: targetMgr, + handoffObserver: handoffObserver, + } +} + +func (job *ReleaseCollectionJob) Execute() error { + req := job.req + log := log.With( + zap.Int64("msgID", req.GetBase().GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + if !job.meta.CollectionManager.Exist(req.GetCollectionID()) { + log.Info("release collection end, the collection has not been loaded into QueryNode") + return nil + } + + err := job.meta.CollectionManager.RemoveCollection(req.GetCollectionID()) + if err != nil { + msg := "failed to remove collection" + log.Warn(msg, zap.Error(err)) + return utils.WrapError(msg, err) + } + + job.handoffObserver.Unregister(job.ctx, job.CollectionID()) + + err = job.meta.ReplicaManager.RemoveCollection(req.GetCollectionID()) + if err != nil { + msg := "failed to remove replicas" + log.Warn(msg, zap.Error(err)) + } + + job.targetMgr.RemoveCollection(req.GetCollectionID()) + waitCollectionReleased(job.dist, req.GetCollectionID()) + return nil +} + +type LoadPartitionJob struct { + *BaseJob + req *querypb.LoadPartitionsRequest + + dist *meta.DistributionManager + meta *meta.Meta + targetMgr *meta.TargetManager + broker meta.Broker + nodeMgr *session.NodeManager + handoffObserver *observers.HandoffObserver +} + +func NewLoadPartitionJob( + ctx context.Context, + req *querypb.LoadPartitionsRequest, + dist *meta.DistributionManager, + meta *meta.Meta, + targetMgr *meta.TargetManager, + broker meta.Broker, + nodeMgr *session.NodeManager, + handoffObserver *observers.HandoffObserver, +) *LoadPartitionJob { + return &LoadPartitionJob{ + BaseJob: NewBaseJob(ctx, req.Base.GetMsgID(), req.GetCollectionID()), + req: req, + dist: dist, + meta: meta, + targetMgr: targetMgr, + broker: broker, + nodeMgr: nodeMgr, + handoffObserver: handoffObserver, + } +} + +func (job *LoadPartitionJob) PreExecute() error { + req := job.req + log := log.With( + zap.Int64("msgID", req.Base.GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + if req.GetReplicaNumber() <= 0 { + log.Info("request doesn't indicate the number of replicas, set it to 1", + zap.Int32("replicaNumber", req.GetReplicaNumber())) + req.ReplicaNumber = 1 + } + + if job.meta.Exist(req.GetCollectionID()) { + old := job.meta.GetCollection(req.GetCollectionID()) + if old != nil { + msg := "collection with different load type existed, please release it first" + log.Warn(msg) + return utils.WrapError(msg, ErrLoadParameterMismatched) + } + if job.meta.GetReplicaNumber(req.GetCollectionID()) != req.GetReplicaNumber() { + msg := "collection with different replica number existed, release this collection first before changing its replica number" + log.Warn(msg) + return utils.WrapError(msg, ErrLoadParameterMismatched) + } + + // Check whether one of the given partitions not loaded + for _, partitionID := range req.GetPartitionIDs() { + partition := job.meta.GetPartition(partitionID) + if partition == nil { + msg := fmt.Sprintf("some partitions %v of collection %v has been loaded into QueryNode, please release partitions firstly", + req.GetPartitionIDs(), + req.GetCollectionID()) + log.Warn(msg) + return utils.WrapError(msg, ErrLoadParameterMismatched) + } + } + return ErrCollectionLoaded + } + + return nil +} + +func (job *LoadPartitionJob) Execute() error { + req := job.req + log := log.With( + zap.Int64("msgID", req.GetBase().GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + // Create replicas + // TODO(yah01): store replicas and collection atomically + replicas, err := utils.SpawnReplicas(job.meta.ReplicaManager, + job.nodeMgr, + req.GetCollectionID(), + req.GetReplicaNumber()) + if err != nil { + msg := "failed to spawn replica for collection" + log.Error(msg, zap.Error(err)) + return utils.WrapError(msg, err) + } + for _, replica := range replicas { + log.Info("replica created", + zap.Int64("replicaID", replica.GetID()), + zap.Int64s("nodes", replica.GetNodes())) + } + + job.handoffObserver.Register(job.CollectionID()) + err = utils.RegisterTargets(job.ctx, + job.targetMgr, + job.broker, + req.GetCollectionID(), + req.GetPartitionIDs()) + if err != nil { + msg := "failed to register channels and segments" + log.Error(msg, zap.Error(err)) + return utils.WrapError(msg, err) + } + job.handoffObserver.StartHandoff(job.CollectionID()) + + partitions := lo.Map(req.GetPartitionIDs(), func(partition int64, _ int) *meta.Partition { + return &meta.Partition{ + PartitionLoadInfo: &querypb.PartitionLoadInfo{ + CollectionID: req.GetCollectionID(), + PartitionID: partition, + ReplicaNumber: req.GetReplicaNumber(), + Status: querypb.LoadStatus_Loading, + }, + CreatedAt: time.Now(), + } + }) + err = job.meta.CollectionManager.PutPartition(partitions...) + if err != nil { + msg := "failed to store partitions" + log.Error(msg, zap.Error(err)) + return utils.WrapError(msg, err) + } + + return nil +} + +func (job *LoadPartitionJob) PostExecute() { + if job.Error() != nil && !job.meta.Exist(job.CollectionID()) { + job.meta.ReplicaManager.RemoveCollection(job.CollectionID()) + job.handoffObserver.Unregister(job.ctx, job.CollectionID()) + job.targetMgr.RemoveCollection(job.req.GetCollectionID()) + } +} + +type ReleasePartitionJob struct { + *BaseJob + req *querypb.ReleasePartitionsRequest + dist *meta.DistributionManager + meta *meta.Meta + targetMgr *meta.TargetManager + handoffObserver *observers.HandoffObserver +} + +func NewReleasePartitionJob(ctx context.Context, + req *querypb.ReleasePartitionsRequest, + dist *meta.DistributionManager, + meta *meta.Meta, + targetMgr *meta.TargetManager, + handoffObserver *observers.HandoffObserver, +) *ReleasePartitionJob { + return &ReleasePartitionJob{ + BaseJob: NewBaseJob(ctx, req.Base.GetMsgID(), req.GetCollectionID()), + req: req, + dist: dist, + meta: meta, + targetMgr: targetMgr, + handoffObserver: handoffObserver, + } +} + +func (job *ReleasePartitionJob) PreExecute() error { + if job.meta.CollectionManager.GetLoadType(job.req.GetCollectionID()) == querypb.LoadType_LoadCollection { + msg := "releasing some partitions after load collection is not supported" + log.Warn(msg) + return utils.WrapError(msg, ErrLoadParameterMismatched) + } + return nil +} + +func (job *ReleasePartitionJob) Execute() error { + req := job.req + log := log.With( + zap.Int64("msgID", req.GetBase().GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + if !job.meta.CollectionManager.Exist(req.GetCollectionID()) { + log.Info("release collection end, the collection has not been loaded into QueryNode") + return nil + } + + loadedPartitions := job.meta.CollectionManager.GetPartitionsByCollection(req.GetCollectionID()) + partitionIDs := typeutil.NewUniqueSet(req.GetPartitionIDs()...) + toRelease := make([]int64, 0) + for _, partition := range loadedPartitions { + if partitionIDs.Contain(partition.GetPartitionID()) { + toRelease = append(toRelease, partition.GetPartitionID()) + } + } + + if len(toRelease) == len(loadedPartitions) { // All partitions are released, clear all + log.Debug("release partitions covers all partitions, will remove the whole collection") + err := job.meta.CollectionManager.RemoveCollection(req.GetCollectionID()) + if err != nil { + msg := "failed to release partitions from store" + log.Warn(msg, zap.Error(err)) + return utils.WrapError(msg, err) + } + job.handoffObserver.Unregister(job.ctx, job.CollectionID()) + err = job.meta.ReplicaManager.RemoveCollection(req.GetCollectionID()) + if err != nil { + log.Warn("failed to remove replicas", zap.Error(err)) + } + job.targetMgr.RemoveCollection(req.GetCollectionID()) + waitCollectionReleased(job.dist, req.GetCollectionID()) + } else { + err := job.meta.CollectionManager.RemovePartition(toRelease...) + if err != nil { + msg := "failed to release partitions from store" + log.Warn(msg, zap.Error(err)) + return utils.WrapError(msg, err) + } + for _, partition := range toRelease { + job.targetMgr.RemovePartition(partition) + } + waitCollectionReleased(job.dist, req.GetCollectionID(), toRelease...) + } + return nil +} diff --git a/internal/querycoordv2/job/job_test.go b/internal/querycoordv2/job/job_test.go new file mode 100644 index 0000000000000..72e2a13409b1d --- /dev/null +++ b/internal/querycoordv2/job/job_test.go @@ -0,0 +1,697 @@ +package job + +import ( + "context" + "errors" + "testing" + + "github.com/milvus-io/milvus/internal/kv" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/observers" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/util/etcd" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type JobSuite struct { + suite.Suite + + // Data + collections []int64 + partitions map[int64][]int64 + channels map[int64][]string + segments map[int64]map[int64][]int64 // CollectionID, PartitionID -> Segments + loadTypes map[int64]querypb.LoadType + + // Dependencies + kv kv.MetaKv + store meta.Store + dist *meta.DistributionManager + meta *meta.Meta + targetMgr *meta.TargetManager + broker *meta.MockBroker + nodeMgr *session.NodeManager + handoffObserver *observers.HandoffObserver + + // Test objects + scheduler *Scheduler +} + +func (suite *JobSuite) SetupSuite() { + Params.Init() + + suite.collections = []int64{1000, 1001} + suite.partitions = map[int64][]int64{ + 1000: {100, 101}, + 1001: {102, 103}, + } + suite.channels = map[int64][]string{ + 1000: {"1000-dmc0", "1000-dmc1"}, + 1001: {"1001-dmc0", "1001-dmc1"}, + } + suite.segments = map[int64]map[int64][]int64{ + 1000: { + 100: {1, 2}, + 101: {3, 4}, + }, + 1001: { + 102: {5, 6}, + 103: {7, 8}, + }, + } + suite.loadTypes = map[int64]querypb.LoadType{ + 1000: querypb.LoadType_LoadCollection, + 1001: querypb.LoadType_LoadPartition, + } + + suite.broker = meta.NewMockBroker(suite.T()) + for collection, partitions := range suite.segments { + vChannels := []*datapb.VchannelInfo{} + for _, channel := range suite.channels[collection] { + vChannels = append(vChannels, &datapb.VchannelInfo{ + CollectionID: collection, + ChannelName: channel, + }) + } + for partition, segments := range partitions { + segmentBinlogs := []*datapb.SegmentBinlogs{} + for _, segment := range segments { + segmentBinlogs = append(segmentBinlogs, &datapb.SegmentBinlogs{ + SegmentID: segment, + InsertChannel: suite.channels[collection][segment%2], + }) + } + + suite.broker.EXPECT(). + GetRecoveryInfo(mock.Anything, collection, partition). + Return(vChannels, segmentBinlogs, nil) + } + } +} + +func (suite *JobSuite) SetupTest() { + config := GenerateEtcdConfig() + cli, err := etcd.GetEtcdClient(&config) + suite.Require().NoError(err) + suite.kv = etcdkv.NewEtcdKV(cli, config.MetaRootPath) + + suite.store = meta.NewMetaStore(suite.kv) + suite.dist = meta.NewDistributionManager() + suite.meta = meta.NewMeta(RandomIncrementIDAllocator(), suite.store) + suite.targetMgr = meta.NewTargetManager() + suite.nodeMgr = session.NewNodeManager() + suite.handoffObserver = observers.NewHandoffObserver( + suite.store, + suite.meta, + suite.dist, + suite.targetMgr, + ) + suite.scheduler = NewScheduler() + + suite.scheduler.Start(context.Background()) +} + +func (suite *JobSuite) TearDownTest() { + suite.kv.Close() + suite.scheduler.Stop() +} + +func (suite *JobSuite) BeforeTest(suiteName, testName string) { + switch testName { + case "TestLoadCollection": + for collection, partitions := range suite.partitions { + if suite.loadTypes[collection] != querypb.LoadType_LoadCollection { + continue + } + suite.broker.EXPECT(). + GetPartitions(mock.Anything, collection). + Return(partitions, nil) + } + } +} + +func (suite *JobSuite) TestLoadCollection() { + ctx := context.Background() + + // Test load collection + for _, collection := range suite.collections { + if suite.loadTypes[collection] != querypb.LoadType_LoadCollection { + continue + } + // Load with 1 replica + req := &querypb.LoadCollectionRequest{ + CollectionID: collection, + // It will be set to 1 + // ReplicaNumber: 1, + } + job := NewLoadCollectionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.NoError(err) + suite.EqualValues(1, suite.meta.GetReplicaNumber(collection)) + suite.assertLoaded(collection) + } + + // Test load again + for _, collection := range suite.collections { + if suite.loadTypes[collection] != querypb.LoadType_LoadCollection { + continue + } + req := &querypb.LoadCollectionRequest{ + CollectionID: collection, + } + job := NewLoadCollectionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.ErrorIs(err, ErrCollectionLoaded) + } + + // Test load existed collection with different replica number + for _, collection := range suite.collections { + if suite.loadTypes[collection] != querypb.LoadType_LoadCollection { + continue + } + req := &querypb.LoadCollectionRequest{ + CollectionID: collection, + ReplicaNumber: 3, + } + job := NewLoadCollectionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.ErrorIs(err, ErrLoadParameterMismatched) + } + + // Test load partition while collection exists + for _, collection := range suite.collections { + if suite.loadTypes[collection] != querypb.LoadType_LoadCollection { + continue + } + // Load with 1 replica + req := &querypb.LoadPartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + ReplicaNumber: 1, + } + job := NewLoadPartitionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.ErrorIs(err, ErrLoadParameterMismatched) + } +} + +func (suite *JobSuite) TestLoadPartition() { + ctx := context.Background() + + // Test load partition + for _, collection := range suite.collections { + if suite.loadTypes[collection] != querypb.LoadType_LoadPartition { + continue + } + // Load with 1 replica + req := &querypb.LoadPartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + // ReplicaNumber: 1, + } + job := NewLoadPartitionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.NoError(err) + suite.EqualValues(1, suite.meta.GetReplicaNumber(collection)) + suite.assertLoaded(collection) + } + + // Test load partition again + for _, collection := range suite.collections { + if suite.loadTypes[collection] != querypb.LoadType_LoadPartition { + continue + } + // Load with 1 replica + req := &querypb.LoadPartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + // ReplicaNumber: 1, + } + job := NewLoadPartitionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.ErrorIs(err, ErrCollectionLoaded) + } + + // Test load partition with different replica number + for _, collection := range suite.collections { + if suite.loadTypes[collection] != querypb.LoadType_LoadPartition { + continue + } + + req := &querypb.LoadPartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + ReplicaNumber: 3, + } + job := NewLoadPartitionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.ErrorIs(err, ErrLoadParameterMismatched) + } + + // Test load partition with more partition + for _, collection := range suite.collections { + if suite.loadTypes[collection] != querypb.LoadType_LoadPartition { + continue + } + + req := &querypb.LoadPartitionsRequest{ + CollectionID: collection, + PartitionIDs: append(suite.partitions[collection], 200), + ReplicaNumber: 3, + } + job := NewLoadPartitionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.ErrorIs(err, ErrLoadParameterMismatched) + } + + // Test load collection while partitions exists + for _, collection := range suite.collections { + if suite.loadTypes[collection] != querypb.LoadType_LoadPartition { + continue + } + + req := &querypb.LoadCollectionRequest{ + CollectionID: collection, + ReplicaNumber: 1, + } + job := NewLoadCollectionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.ErrorIs(err, ErrLoadParameterMismatched) + } +} + +func (suite *JobSuite) TestReleaseCollection() { + ctx := context.Background() + + suite.loadAll() + + // Test release collection and partition + for _, collection := range suite.collections { + req := &querypb.ReleaseCollectionRequest{ + CollectionID: collection, + } + job := NewReleaseCollectionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.NoError(err) + suite.assertReleased(collection) + } + + // Test release again + for _, collection := range suite.collections { + req := &querypb.ReleaseCollectionRequest{ + CollectionID: collection, + } + job := NewReleaseCollectionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.NoError(err) + suite.assertReleased(collection) + } +} + +func (suite *JobSuite) TestReleasePartition() { + ctx := context.Background() + + suite.loadAll() + + // Test release partition + for _, collection := range suite.collections { + req := &querypb.ReleasePartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + } + job := NewReleasePartitionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + if suite.loadTypes[collection] == querypb.LoadType_LoadCollection { + suite.ErrorIs(err, ErrLoadParameterMismatched) + suite.assertLoaded(collection) + } else { + suite.NoError(err) + suite.assertReleased(collection) + } + } + + // Test release again + for _, collection := range suite.collections { + req := &querypb.ReleasePartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + } + job := NewReleasePartitionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + if suite.loadTypes[collection] == querypb.LoadType_LoadCollection { + suite.ErrorIs(err, ErrLoadParameterMismatched) + suite.assertLoaded(collection) + } else { + suite.NoError(err) + suite.assertReleased(collection) + } + } + + // Test release partial partitions + suite.releaseAll() + suite.loadAll() + for _, collection := range suite.collections { + req := &querypb.ReleasePartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection][1:], + } + job := NewReleasePartitionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + if suite.loadTypes[collection] == querypb.LoadType_LoadCollection { + suite.ErrorIs(err, ErrLoadParameterMismatched) + suite.assertLoaded(collection) + } else { + suite.NoError(err) + suite.True(suite.meta.Exist(collection)) + partitions := suite.meta.GetPartitionsByCollection(collection) + suite.Len(partitions, 1) + suite.Equal(suite.partitions[collection][0], partitions[0].GetPartitionID()) + } + } +} + +func (suite *JobSuite) TestLoadCollectionStoreFailed() { + // Store collection failed + store := meta.NewMockStore(suite.T()) + suite.meta = meta.NewMeta(RandomIncrementIDAllocator(), store) + for _, collection := range suite.collections { + if suite.loadTypes[collection] != querypb.LoadType_LoadCollection { + continue + } + + err := errors.New("failed to store collection") + store.EXPECT().SaveReplica(mock.Anything).Return(nil) + store.EXPECT().SaveCollection(&querypb.CollectionLoadInfo{ + CollectionID: collection, + ReplicaNumber: 1, + Status: querypb.LoadStatus_Loading, + }).Return(err) + store.EXPECT().ReleaseReplicas(collection).Return(nil) + + req := &querypb.LoadCollectionRequest{ + CollectionID: collection, + } + job := NewLoadCollectionJob( + context.Background(), + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + loadErr := job.Wait() + suite.ErrorIs(loadErr, err) + } +} + +func (suite *JobSuite) TestLoadPartitionStoreFailed() { + // Store partition failed + store := meta.NewMockStore(suite.T()) + suite.meta = meta.NewMeta(RandomIncrementIDAllocator(), store) + err := errors.New("failed to store collection") + for _, collection := range suite.collections { + if suite.loadTypes[collection] != querypb.LoadType_LoadPartition { + continue + } + + store.EXPECT().SaveReplica(mock.Anything).Return(nil) + store.EXPECT().SavePartition(mock.Anything, mock.Anything).Return(err) + store.EXPECT().ReleaseReplicas(collection).Return(nil) + + req := &querypb.LoadPartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + } + job := NewLoadPartitionJob( + context.Background(), + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + loadErr := job.Wait() + suite.ErrorIs(loadErr, err) + } +} + +func (suite *JobSuite) TestLoadCreateReplicaFailed() { + // Store replica failed + suite.meta = meta.NewMeta(ErrorIDAllocator(), suite.store) + for _, collection := range suite.collections { + req := &querypb.LoadCollectionRequest{ + CollectionID: collection, + } + job := NewLoadCollectionJob( + context.Background(), + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.ErrorIs(err, ErrFailedAllocateID) + } +} + +func (suite *JobSuite) loadAll() { + ctx := context.Background() + for _, collection := range suite.collections { + if suite.loadTypes[collection] == querypb.LoadType_LoadCollection { + req := &querypb.LoadCollectionRequest{ + CollectionID: collection, + } + job := NewLoadCollectionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.NoError(err) + suite.EqualValues(1, suite.meta.GetReplicaNumber(collection)) + suite.True(suite.meta.Exist(collection)) + suite.NotNil(suite.meta.GetCollection(collection)) + } else { + req := &querypb.LoadPartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + } + job := NewLoadPartitionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.NoError(err) + suite.EqualValues(1, suite.meta.GetReplicaNumber(collection)) + suite.True(suite.meta.Exist(collection)) + suite.NotNil(suite.meta.GetPartitionsByCollection(collection)) + } + } +} + +func (suite *JobSuite) releaseAll() { + ctx := context.Background() + for _, collection := range suite.collections { + req := &querypb.ReleaseCollectionRequest{ + CollectionID: collection, + } + job := NewReleaseCollectionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.handoffObserver, + ) + suite.scheduler.Add(job) + err := job.Wait() + suite.NoError(err) + suite.assertReleased(collection) + } +} + +func (suite *JobSuite) assertLoaded(collection int64) { + suite.True(suite.meta.Exist(collection)) + for _, channel := range suite.channels[collection] { + suite.NotNil(suite.targetMgr.GetDmChannel(channel)) + } + for _, partitions := range suite.segments[collection] { + for _, segment := range partitions { + suite.NotNil(suite.targetMgr.GetSegment(segment)) + } + } +} + +func (suite *JobSuite) assertReleased(collection int64) { + suite.False(suite.meta.Exist(collection)) + for _, channel := range suite.channels[collection] { + suite.Nil(suite.targetMgr.GetDmChannel(channel)) + } + for _, partitions := range suite.segments[collection] { + for _, segment := range partitions { + suite.Nil(suite.targetMgr.GetSegment(segment)) + } + } +} + +func TestJob(t *testing.T) { + suite.Run(t, new(JobSuite)) +} diff --git a/internal/querycoordv2/job/scheduler.go b/internal/querycoordv2/job/scheduler.go new file mode 100644 index 0000000000000..6b86808c55802 --- /dev/null +++ b/internal/querycoordv2/job/scheduler.go @@ -0,0 +1,152 @@ +package job + +import ( + "context" + "sync" + "time" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/util/typeutil" + "go.uber.org/zap" +) + +// JobScheduler schedules jobs, +// all jobs within the same collection will run sequentially +const ( + collectionQueueCap = 64 + waitQueueCap = 512 +) + +type jobQueue chan Job + +type Scheduler struct { + stopCh chan struct{} + wg sync.WaitGroup + + processors *typeutil.ConcurrentSet[int64] // Collections of having processor + queues map[int64]jobQueue // CollectionID -> Queue + waitQueue jobQueue +} + +func NewScheduler() *Scheduler { + return &Scheduler{ + stopCh: make(chan struct{}), + processors: typeutil.NewConcurrentSet[int64](), + queues: make(map[int64]jobQueue), + waitQueue: make(jobQueue, waitQueueCap), + } +} + +func (scheduler *Scheduler) Start(ctx context.Context) { + scheduler.wg.Add(1) + go scheduler.schedule(ctx) +} + +func (scheduler *Scheduler) Stop() { + close(scheduler.stopCh) + scheduler.wg.Wait() +} + +func (scheduler *Scheduler) schedule(ctx context.Context) { + defer scheduler.wg.Done() + + ticker := time.NewTicker(500 * time.Millisecond) + go func() { + for { + select { + case <-ctx.Done(): + log.Info("JobManager stopped due to context canceled") + return + + case <-scheduler.stopCh: + log.Info("JobManager stopped") + return + + case job := <-scheduler.waitQueue: + queue, ok := scheduler.queues[job.CollectionID()] + if !ok { + queue = make(jobQueue, collectionQueueCap) + scheduler.queues[job.CollectionID()] = queue + } + queue <- job + scheduler.startProcessor(job.CollectionID(), queue) + + case <-ticker.C: + for collection, queue := range scheduler.queues { + if len(queue) > 0 { + scheduler.startProcessor(collection, queue) + } else { + // Release resource if no job for the collection + delete(scheduler.queues, collection) + } + } + } + } + }() +} + +func (scheduler *Scheduler) isStopped() bool { + select { + case <-scheduler.stopCh: + return true + default: + return false + } +} + +func (scheduler *Scheduler) Add(job Job) { + scheduler.waitQueue <- job +} + +func (scheduler *Scheduler) startProcessor(collection int64, queue jobQueue) { + if scheduler.isStopped() { + return + } + if !scheduler.processors.Insert(collection) { + return + } + + scheduler.wg.Add(1) + go scheduler.processQueue(collection, queue) +} + +// processQueue processes jobs in the given queue, +// it only processes jobs with the number of the length of queue at the time, +// to avoid leaking goroutines +func (scheduler *Scheduler) processQueue(collection int64, queue jobQueue) { + defer scheduler.wg.Done() + defer scheduler.processors.Remove(collection) + + len := len(queue) + for i := 0; i < len; i++ { + scheduler.process(<-queue) + } +} + +func (scheduler *Scheduler) process(job Job) { + log := log.With( + zap.Int64("msgID", job.MsgID()), + zap.Int64("collectionID", job.CollectionID())) + + defer func() { + log.Info("start to post-execute job") + job.PostExecute() + log.Info("job finished") + job.Done() + }() + + log.Info("start to pre-execute job") + err := job.PreExecute() + if err != nil { + log.Warn("failed to pre-execute job", zap.Error(err)) + job.SetError(err) + return + } + + log.Info("start to execute job") + err = job.Execute() + if err != nil { + log.Warn("failed to execute job", zap.Error(err)) + job.SetError(err) + } +} diff --git a/internal/querycoordv2/job/utils.go b/internal/querycoordv2/job/utils.go new file mode 100644 index 0000000000000..9aa02dacb396e --- /dev/null +++ b/internal/querycoordv2/job/utils.go @@ -0,0 +1,35 @@ +package job + +import ( + "time" + + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/samber/lo" +) + +// waitCollectionReleased blocks until +// all channels and segments of given collection(partitions) are released, +// empty partition list means wait for collection released +func waitCollectionReleased(dist *meta.DistributionManager, collection int64, partitions ...int64) { + partitionSet := typeutil.NewUniqueSet(partitions...) + for { + var ( + channels []*meta.DmChannel + segments []*meta.Segment = dist.SegmentDistManager.GetByCollection(collection) + ) + if partitionSet.Len() > 0 { + segments = lo.Filter(segments, func(segment *meta.Segment, _ int) bool { + return partitionSet.Contain(segment.GetPartitionID()) + }) + } else { + channels = dist.ChannelDistManager.GetByCollection(collection) + } + + if len(channels)+len(segments) == 0 { + break + } + + time.Sleep(200 * time.Millisecond) + } +} diff --git a/internal/querycoordv2/meta/channel_dist_manager.go b/internal/querycoordv2/meta/channel_dist_manager.go new file mode 100644 index 0000000000000..4acb0bbca3c38 --- /dev/null +++ b/internal/querycoordv2/meta/channel_dist_manager.go @@ -0,0 +1,141 @@ +package meta + +import ( + "sync" + + "github.com/golang/protobuf/proto" + "github.com/milvus-io/milvus/internal/proto/datapb" + . "github.com/milvus-io/milvus/internal/util/typeutil" +) + +type DmChannel struct { + *datapb.VchannelInfo + Node int64 + Version int64 +} + +func DmChannelFromVChannel(channel *datapb.VchannelInfo) *DmChannel { + return &DmChannel{ + VchannelInfo: channel, + } +} + +func (channel *DmChannel) Clone() *DmChannel { + return &DmChannel{ + VchannelInfo: proto.Clone(channel.VchannelInfo).(*datapb.VchannelInfo), + Node: channel.Node, + Version: channel.Version, + } +} + +type ChannelDistManager struct { + rwmutex sync.RWMutex + + // NodeID -> Channels + channels map[UniqueID][]*DmChannel +} + +func NewChannelDistManager() *ChannelDistManager { + return &ChannelDistManager{ + channels: make(map[UniqueID][]*DmChannel), + } +} + +func (m *ChannelDistManager) GetByNode(nodeID UniqueID) []*DmChannel { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + return m.getByNode(nodeID) +} + +func (m *ChannelDistManager) getByNode(nodeID UniqueID) []*DmChannel { + channels, ok := m.channels[nodeID] + if !ok { + return nil + } + + return channels +} + +func (m *ChannelDistManager) GetAll() []*DmChannel { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + result := make([]*DmChannel, 0) + for _, channels := range m.channels { + result = append(result, channels...) + } + return result +} + +// GetShardLeader returns the node whthin the given replicaNodes and subscribing the given shard, +// returns (0, false) if not found. +func (m *ChannelDistManager) GetShardLeader(replica *Replica, shard string) (int64, bool) { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + for node := range replica.Nodes { + channels := m.channels[node] + for _, dmc := range channels { + if dmc.ChannelName == shard { + return node, true + } + } + } + + return 0, false +} + +func (m *ChannelDistManager) GetShardLeadersByReplica(replica *Replica) map[string]int64 { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + ret := make(map[string]int64) + for node := range replica.Nodes { + channels := m.channels[node] + for _, dmc := range channels { + ret[dmc.GetChannelName()] = node + } + } + return ret +} + +func (m *ChannelDistManager) GetByCollection(collectionID UniqueID) []*DmChannel { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + ret := make([]*DmChannel, 0) + for _, channels := range m.channels { + for _, channel := range channels { + if channel.CollectionID == collectionID { + ret = append(ret, channel) + } + } + } + return ret +} + +func (m *ChannelDistManager) GetByCollectionAndNode(collectionID, nodeID UniqueID) []*DmChannel { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + channels := make([]*DmChannel, 0) + for _, channel := range m.getByNode(nodeID) { + if channel.CollectionID == collectionID { + channels = append(channels, channel) + } + } + + return channels +} + +func (m *ChannelDistManager) Update(nodeID UniqueID, channels ...*DmChannel) { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + for _, channel := range channels { + channel.Node = nodeID + } + + m.channels[nodeID] = channels +} diff --git a/internal/querycoordv2/meta/channel_dist_manager_test.go b/internal/querycoordv2/meta/channel_dist_manager_test.go new file mode 100644 index 0000000000000..62359febca4d6 --- /dev/null +++ b/internal/querycoordv2/meta/channel_dist_manager_test.go @@ -0,0 +1,159 @@ +package meta + +import ( + "testing" + + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/stretchr/testify/suite" +) + +type ChannelDistManagerSuite struct { + suite.Suite + dist *ChannelDistManager + collection int64 + nodes []int64 + channels map[string]*DmChannel +} + +func (suite *ChannelDistManagerSuite) SetupSuite() { + // Replica 0: 0, 2 + // Replica 1: 1 + suite.collection = 10 + suite.nodes = []int64{0, 1, 2} + suite.channels = map[string]*DmChannel{ + "dmc0": DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: suite.collection, + ChannelName: "dmc0", + }), + "dmc1": DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: suite.collection, + ChannelName: "dmc1", + }), + } +} + +func (suite *ChannelDistManagerSuite) SetupTest() { + suite.dist = NewChannelDistManager() + // Distribution: + // node 0 contains channel dmc0 + // node 1 contains channel dmc0, dmc1 + // node 2 contains channel dmc1 + suite.dist.Update(suite.nodes[0], suite.channels["dmc0"].Clone()) + suite.dist.Update(suite.nodes[1], suite.channels["dmc0"].Clone(), suite.channels["dmc1"].Clone()) + suite.dist.Update(suite.nodes[2], suite.channels["dmc1"].Clone()) +} + +func (suite *ChannelDistManagerSuite) TestGetBy() { + dist := suite.dist + + // Test GetAll + channels := dist.GetAll() + suite.Len(channels, 4) + + // Test GetByNode + for _, node := range suite.nodes { + channels := dist.GetByNode(node) + suite.AssertNode(channels, node) + } + + // Test GetByCollection + channels = dist.GetByCollection(suite.collection) + suite.Len(channels, 4) + suite.AssertCollection(channels, suite.collection) + channels = dist.GetByCollection(-1) + suite.Len(channels, 0) + + // Test GetByNodeAndCollection + // 1. Valid node and valid collection + for _, node := range suite.nodes { + channels := dist.GetByCollectionAndNode(suite.collection, node) + suite.AssertNode(channels, node) + suite.AssertCollection(channels, suite.collection) + } + + // 2. Valid node and invalid collection + channels = dist.GetByCollectionAndNode(-1, suite.nodes[1]) + suite.Len(channels, 0) + + // 3. Invalid node and valid collection + channels = dist.GetByCollectionAndNode(suite.collection, -1) + suite.Len(channels, 0) +} + +func (suite *ChannelDistManagerSuite) TestGetShardLeader() { + replicas := []*Replica{ + {Nodes: typeutil.NewUniqueSet(suite.nodes[0], suite.nodes[2])}, + {Nodes: typeutil.NewUniqueSet(suite.nodes[1])}, + } + + // Test on replica 0 + leader0, ok := suite.dist.GetShardLeader(replicas[0], "dmc0") + suite.True(ok) + suite.Equal(suite.nodes[0], leader0) + leader1, ok := suite.dist.GetShardLeader(replicas[0], "dmc1") + suite.True(ok) + suite.Equal(suite.nodes[2], leader1) + + // Test on replica 1 + leader0, ok = suite.dist.GetShardLeader(replicas[1], "dmc0") + suite.True(ok) + suite.Equal(suite.nodes[1], leader0) + leader1, ok = suite.dist.GetShardLeader(replicas[1], "dmc1") + suite.True(ok) + suite.Equal(suite.nodes[1], leader1) + + // Test no shard leader for given channel + _, ok = suite.dist.GetShardLeader(replicas[0], "invalid-shard") + suite.False(ok) + + // Test on replica 0 + leaders := suite.dist.GetShardLeadersByReplica(replicas[0]) + suite.Len(leaders, 2) + suite.Equal(leaders["dmc0"], suite.nodes[0]) + suite.Equal(leaders["dmc1"], suite.nodes[2]) + + // Test on replica 1 + leaders = suite.dist.GetShardLeadersByReplica(replicas[1]) + suite.Len(leaders, 2) + suite.Equal(leaders["dmc0"], suite.nodes[1]) + suite.Equal(leaders["dmc1"], suite.nodes[1]) +} + +func (suite *ChannelDistManagerSuite) AssertNames(channels []*DmChannel, names ...string) bool { + for _, channel := range channels { + hasChannel := false + for _, name := range names { + if channel.ChannelName == name { + hasChannel = true + break + } + } + if !suite.True(hasChannel, "channel %v not in the given expected list %+v", channel.ChannelName, names) { + return false + } + } + return true +} + +func (suite *ChannelDistManagerSuite) AssertNode(channels []*DmChannel, node int64) bool { + for _, channel := range channels { + if !suite.Equal(node, channel.Node) { + return false + } + } + return true +} + +func (suite *ChannelDistManagerSuite) AssertCollection(channels []*DmChannel, collection int64) bool { + for _, channel := range channels { + if !suite.Equal(collection, channel.GetCollectionID()) { + return false + } + } + return true +} + +func TestChannelDistManager(t *testing.T) { + suite.Run(t, new(ChannelDistManagerSuite)) +} diff --git a/internal/querycoordv2/meta/collection_manager.go b/internal/querycoordv2/meta/collection_manager.go new file mode 100644 index 0000000000000..dd45f1210a412 --- /dev/null +++ b/internal/querycoordv2/meta/collection_manager.go @@ -0,0 +1,397 @@ +package meta + +import ( + "sync" + "time" + + "github.com/golang/protobuf/proto" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/util/typeutil" + . "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/samber/lo" +) + +type Collection struct { + *querypb.CollectionLoadInfo + LoadPercentage int32 + CreatedAt time.Time +} + +func (collection *Collection) Clone() *Collection { + new := *collection + new.CollectionLoadInfo = proto.Clone(collection.CollectionLoadInfo).(*querypb.CollectionLoadInfo) + return &new +} + +type Partition struct { + *querypb.PartitionLoadInfo + LoadPercentage int32 + CreatedAt time.Time +} + +func (partition *Partition) Clone() *Partition { + new := *partition + new.PartitionLoadInfo = proto.Clone(partition.PartitionLoadInfo).(*querypb.PartitionLoadInfo) + return &new +} + +type CollectionManager struct { + rwmutex sync.RWMutex + + collections map[UniqueID]*Collection + partitions map[UniqueID]*Partition + store Store +} + +func NewCollectionManager(store Store) *CollectionManager { + return &CollectionManager{ + collections: make(map[int64]*Collection), + partitions: make(map[int64]*Partition), + store: store, + } +} + +// Recover recovers collections from kv store, +// panics if failed +func (m *CollectionManager) Recover() error { + collections, err := m.store.GetCollections() + if err != nil { + return err + } + partitions, err := m.store.GetPartitions() + if err != nil { + return err + } + + for _, collection := range collections { + // Collections not loaded done should be deprecated + if collection.GetStatus() != querypb.LoadStatus_Loaded { + m.store.ReleaseCollection(collection.GetCollectionID()) + continue + } + + m.collections[collection.CollectionID] = &Collection{ + CollectionLoadInfo: collection, + } + } + + for collection, partitions := range partitions { + for _, partition := range partitions { + // Partitions not loaded done should be deprecated + if partition.GetStatus() != querypb.LoadStatus_Loaded { + partitionIDs := lo.Map(partitions, func(partition *querypb.PartitionLoadInfo, _ int) int64 { + return partition.GetPartitionID() + }) + m.store.ReleasePartition(collection, partitionIDs...) + break + } + + m.partitions[partition.PartitionID] = &Partition{ + PartitionLoadInfo: partition, + } + } + } + + return nil +} + +func (m *CollectionManager) GetCollection(id UniqueID) *Collection { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + return m.collections[id] +} + +func (m *CollectionManager) GetPartition(id UniqueID) *Partition { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + return m.partitions[id] +} + +func (m *CollectionManager) GetLoadType(id UniqueID) querypb.LoadType { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + _, ok := m.collections[id] + if ok { + return querypb.LoadType_LoadCollection + } + if len(m.getPartitionsByCollection(id)) > 0 { + return querypb.LoadType_LoadPartition + } + return querypb.LoadType_UnKnownType +} + +func (m *CollectionManager) GetReplicaNumber(id UniqueID) int32 { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + collection, ok := m.collections[id] + if ok { + return collection.GetReplicaNumber() + } + partitions := m.getPartitionsByCollection(id) + if len(partitions) > 0 { + return partitions[0].GetReplicaNumber() + } + return -1 +} + +func (m *CollectionManager) GetLoadPercentage(id UniqueID) int32 { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + collection, ok := m.collections[id] + if ok { + return collection.LoadPercentage + } + partitions := m.getPartitionsByCollection(id) + if len(partitions) > 0 { + return lo.SumBy(partitions, func(partition *Partition) int32 { + return partition.LoadPercentage + }) / int32(len(partitions)) + } + return -1 +} + +func (m *CollectionManager) GetStatus(id UniqueID) querypb.LoadStatus { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + collection, ok := m.collections[id] + if ok { + return collection.GetStatus() + } + partitions := m.getPartitionsByCollection(id) + if len(partitions) == 0 { + return querypb.LoadStatus_Invalid + } + for _, partition := range partitions { + if partition.GetStatus() == querypb.LoadStatus_Loading { + return querypb.LoadStatus_Loading + } + } + return querypb.LoadStatus_Loaded +} + +func (m *CollectionManager) Exist(id UniqueID) bool { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + _, ok := m.collections[id] + if ok { + return true + } + partitions := m.getPartitionsByCollection(id) + return len(partitions) > 0 +} + +// GetAll returns the collection ID of all loaded collections and partitions +func (m *CollectionManager) GetAll() []int64 { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + ids := typeutil.NewUniqueSet() + for _, collection := range m.collections { + ids.Insert(collection.GetCollectionID()) + } + for _, partition := range m.partitions { + ids.Insert(partition.GetCollectionID()) + } + return ids.Collect() +} + +func (m *CollectionManager) GetAllCollections() []*Collection { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + return lo.Values(m.collections) +} + +func (m *CollectionManager) GetAllPartitions() []*Partition { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + return lo.Values(m.partitions) +} + +func (m *CollectionManager) GetPartitionsByCollection(collectionID UniqueID) []*Partition { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + return m.getPartitionsByCollection(collectionID) +} + +func (m *CollectionManager) getPartitionsByCollection(collectionID UniqueID) []*Partition { + partitions := make([]*Partition, 0) + for _, partition := range m.partitions { + if partition.CollectionID == collectionID { + partitions = append(partitions, partition) + } + } + return partitions +} + +func (m *CollectionManager) PutCollection(collection *Collection) error { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + return m.putCollection(collection, true) +} + +func (m *CollectionManager) PutCollectionWithoutSave(collection *Collection) { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + m.putCollection(collection, false) +} + +func (m *CollectionManager) UpdateCollection(collection *Collection) error { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + _, ok := m.collections[collection.GetCollectionID()] + if !ok { + return ErrCollectionNotFound + } + + return m.putCollection(collection, true) +} + +func (m *CollectionManager) UpdateCollectionInMemory(collection *Collection) bool { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + _, ok := m.collections[collection.GetCollectionID()] + if !ok { + return false + } + + m.putCollection(collection, false) + return true +} + +func (m *CollectionManager) putCollection(collection *Collection, withSave bool) error { + if withSave { + err := m.store.SaveCollection(collection.CollectionLoadInfo) + if err != nil { + return err + } + } + m.collections[collection.CollectionID] = collection + + return nil +} + +func (m *CollectionManager) PutPartition(partitions ...*Partition) error { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + return m.putPartition(partitions, true) +} + +func (m *CollectionManager) PutPartitionWithoutSave(partitions ...*Partition) { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + m.putPartition(partitions, false) +} + +func (m *CollectionManager) UpdatePartition(partition *Partition) error { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + _, ok := m.partitions[partition.GetPartitionID()] + if !ok { + return ErrPartitionNotFound + } + + return m.putPartition([]*Partition{partition}, true) +} + +func (m *CollectionManager) UpdatePartitionInMemory(partition *Partition) bool { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + _, ok := m.partitions[partition.GetPartitionID()] + if !ok { + return false + } + + m.putPartition([]*Partition{partition}, false) + return true +} + +func (m *CollectionManager) putPartition(partitions []*Partition, withSave bool) error { + if withSave { + loadInfos := lo.Map(partitions, func(partition *Partition, _ int) *querypb.PartitionLoadInfo { + return partition.PartitionLoadInfo + }) + err := m.store.SavePartition(loadInfos...) + if err != nil { + return err + } + } + for _, partition := range partitions { + m.partitions[partition.GetPartitionID()] = partition + } + return nil +} + +func (m *CollectionManager) RemoveCollection(id UniqueID) error { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + _, ok := m.collections[id] + if ok { + err := m.store.ReleaseCollection(id) + if err != nil { + return err + } + delete(m.collections, id) + return nil + } + + partitions := lo.Map(m.getPartitionsByCollection(id), + func(partition *Partition, _ int) int64 { + return partition.GetPartitionID() + }) + return m.removePartition(partitions...) +} + +func (m *CollectionManager) RemovePartition(ids ...UniqueID) error { + if len(ids) == 0 { + return nil + } + + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + return m.removePartition(ids...) +} + +func (m *CollectionManager) RemoveCollectionInMemory(id UniqueID) { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + delete(m.collections, id) + + partitions := m.getPartitionsByCollection(id) + for _, partition := range partitions { + delete(m.partitions, partition.GetPartitionID()) + } +} + +func (m *CollectionManager) removePartition(ids ...UniqueID) error { + partition := m.partitions[ids[0]] + err := m.store.ReleasePartition(partition.CollectionID, ids...) + if err != nil { + return err + } + for _, id := range ids { + delete(m.partitions, id) + } + + return nil +} diff --git a/internal/querycoordv2/meta/collection_manager_test.go b/internal/querycoordv2/meta/collection_manager_test.go new file mode 100644 index 0000000000000..01451b7de0f1c --- /dev/null +++ b/internal/querycoordv2/meta/collection_manager_test.go @@ -0,0 +1,315 @@ +package meta + +import ( + "sort" + "testing" + "time" + + "github.com/golang/protobuf/proto" + "github.com/milvus-io/milvus/internal/kv" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/internal/proto/querypb" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/util/etcd" + "github.com/stretchr/testify/suite" +) + +type CollectionManagerSuite struct { + suite.Suite + + // Data + collections []int64 + partitions map[int64][]int64 // CollectionID -> PartitionIDs + loadTypes []querypb.LoadType + replicaNumber []int32 + loadPercentage []int32 + + // Mocks + kv kv.MetaKv + store Store + + // Test object + mgr *CollectionManager +} + +func (suite *CollectionManagerSuite) SetupSuite() { + Params.Init() + + suite.collections = []int64{100, 101, 102} + suite.partitions = map[int64][]int64{ + 100: {10}, + 101: {11, 12}, + 102: {13, 14, 15}, + } + suite.loadTypes = []querypb.LoadType{ + querypb.LoadType_LoadCollection, + querypb.LoadType_LoadPartition, + querypb.LoadType_LoadCollection, + } + suite.replicaNumber = []int32{1, 2, 3} + suite.loadPercentage = []int32{0, 50, 100} +} + +func (suite *CollectionManagerSuite) SetupTest() { + var err error + config := GenerateEtcdConfig() + cli, err := etcd.GetEtcdClient(&config) + suite.Require().NoError(err) + suite.kv = etcdkv.NewEtcdKV(cli, config.MetaRootPath) + suite.store = NewMetaStore(suite.kv) + + suite.mgr = NewCollectionManager(suite.store) + suite.loadAll() +} + +func (suite *CollectionManagerSuite) TearDownTest() { + suite.kv.Close() +} + +func (suite *CollectionManagerSuite) TestGetProperty() { + mgr := suite.mgr + + for i, collection := range suite.collections { + loadType := mgr.GetLoadType(collection) + replicaNumber := mgr.GetReplicaNumber(collection) + percentage := mgr.GetLoadPercentage(collection) + exist := mgr.Exist(collection) + suite.Equal(suite.loadTypes[i], loadType) + suite.Equal(suite.replicaNumber[i], replicaNumber) + suite.Equal(suite.loadPercentage[i], percentage) + suite.True(exist) + } + + invalidCollection := -1 + loadType := mgr.GetLoadType(int64(invalidCollection)) + replicaNumber := mgr.GetReplicaNumber(int64(invalidCollection)) + percentage := mgr.GetLoadPercentage(int64(invalidCollection)) + exist := mgr.Exist(int64(invalidCollection)) + suite.Equal(querypb.LoadType_UnKnownType, loadType) + suite.EqualValues(-1, replicaNumber) + suite.EqualValues(-1, percentage) + suite.False(exist) +} + +func (suite *CollectionManagerSuite) TestGet() { + mgr := suite.mgr + + allCollections := mgr.GetAllCollections() + allPartitions := mgr.GetAllPartitions() + for i, collectionID := range suite.collections { + if suite.loadTypes[i] == querypb.LoadType_LoadCollection { + collection := mgr.GetCollection(collectionID) + suite.Equal(collectionID, collection.GetCollectionID()) + suite.Contains(allCollections, collection) + } else { + partitions := mgr.GetPartitionsByCollection(collectionID) + suite.Len(partitions, len(suite.partitions[collectionID])) + + for _, partitionID := range suite.partitions[collectionID] { + partition := mgr.GetPartition(partitionID) + suite.Equal(collectionID, partition.GetCollectionID()) + suite.Equal(partitionID, partition.GetPartitionID()) + suite.Contains(partitions, partition) + suite.Contains(allPartitions, partition) + } + } + } + + all := mgr.GetAll() + sort.Slice(all, func(i, j int) bool { return all[i] < all[j] }) + suite.Equal(suite.collections, all) +} + +func (suite *CollectionManagerSuite) TestUpdate() { + mgr := suite.mgr + + collections := mgr.GetAllCollections() + partitions := mgr.GetAllPartitions() + for _, collection := range collections { + collection := collection.Clone() + collection.LoadPercentage = 100 + ok := mgr.UpdateCollectionInMemory(collection) + suite.True(ok) + + modified := mgr.GetCollection(collection.GetCollectionID()) + suite.Equal(collection, modified) + suite.EqualValues(100, modified.LoadPercentage) + + collection.Status = querypb.LoadStatus_Loaded + err := mgr.UpdateCollection(collection) + suite.NoError(err) + } + for _, partition := range partitions { + partition := partition.Clone() + partition.LoadPercentage = 100 + ok := mgr.UpdatePartitionInMemory(partition) + suite.True(ok) + + modified := mgr.GetPartition(partition.GetPartitionID()) + suite.Equal(partition, modified) + suite.EqualValues(100, modified.LoadPercentage) + + partition.Status = querypb.LoadStatus_Loaded + err := mgr.UpdatePartition(partition) + suite.NoError(err) + } + + suite.clearMemory() + err := mgr.Recover() + suite.NoError(err) + collections = mgr.GetAllCollections() + partitions = mgr.GetAllPartitions() + for _, collection := range collections { + suite.Equal(querypb.LoadStatus_Loaded, collection.GetStatus()) + } + for _, partition := range partitions { + suite.Equal(querypb.LoadStatus_Loaded, partition.GetStatus()) + } +} + +func (suite *CollectionManagerSuite) TestRemove() { + mgr := suite.mgr + + // Remove collections/partitions + for i, collectionID := range suite.collections { + if suite.loadTypes[i] == querypb.LoadType_LoadCollection { + err := mgr.RemoveCollection(collectionID) + suite.NoError(err) + } else { + err := mgr.RemovePartition(suite.partitions[collectionID]...) + suite.NoError(err) + } + } + + // Try to get the removed items + for i, collectionID := range suite.collections { + if suite.loadTypes[i] == querypb.LoadType_LoadCollection { + collection := mgr.GetCollection(collectionID) + suite.Nil(collection) + } else { + partitions := mgr.GetPartitionsByCollection(collectionID) + suite.Empty(partitions) + } + } + + // Make sure the removes applied to meta store + err := mgr.Recover() + suite.NoError(err) + for i, collectionID := range suite.collections { + if suite.loadTypes[i] == querypb.LoadType_LoadCollection { + collection := mgr.GetCollection(collectionID) + suite.Nil(collection) + } else { + partitions := mgr.GetPartitionsByCollection(collectionID) + suite.Empty(partitions) + } + } + + // RemoveCollection also works for partitions + suite.loadAll() + for i, collectionID := range suite.collections { + if suite.loadTypes[i] == querypb.LoadType_LoadPartition { + err := mgr.RemoveCollection(collectionID) + suite.NoError(err) + partitions := mgr.GetPartitionsByCollection(collectionID) + suite.Empty(partitions) + } + } +} + +func (suite *CollectionManagerSuite) TestRecover() { + mgr := suite.mgr + + suite.clearMemory() + err := mgr.Recover() + suite.NoError(err) + for i, collection := range suite.collections { + exist := suite.loadPercentage[i] == 100 + suite.Equal(exist, mgr.Exist(collection)) + } + + // Test recover from 2.1 meta store + collectionInfo := querypb.CollectionInfo{ + CollectionID: 1000, + PartitionIDs: []int64{100, 101}, + LoadType: querypb.LoadType_LoadCollection, + ReplicaNumber: 3, + } + value, err := proto.Marshal(&collectionInfo) + suite.NoError(err) + err = suite.kv.Save(CollectionMetaPrefixV1+"/1000", string(value)) + suite.NoError(err) + + collectionInfo = querypb.CollectionInfo{ + CollectionID: 1001, + PartitionIDs: []int64{102, 103}, + LoadType: querypb.LoadType_LoadPartition, + ReplicaNumber: 1, + } + value, err = proto.Marshal(&collectionInfo) + suite.NoError(err) + err = suite.kv.Save(CollectionMetaPrefixV1+"/1001", string(value)) + suite.NoError(err) + + suite.clearMemory() + err = mgr.Recover() + suite.NoError(err) + + // Verify collection + suite.True(mgr.Exist(1000)) + suite.Equal(querypb.LoadType_LoadCollection, mgr.GetLoadType(1000)) + suite.EqualValues(3, mgr.GetReplicaNumber(1000)) + + // Verify partitions + suite.True(mgr.Exist(1001)) + suite.Equal(querypb.LoadType_LoadPartition, mgr.GetLoadType(1001)) + suite.EqualValues(1, mgr.GetReplicaNumber(1001)) + suite.NotNil(mgr.GetPartition(102)) + suite.NotNil(mgr.GetPartition(103)) + suite.Len(mgr.getPartitionsByCollection(1001), 2) +} + +func (suite *CollectionManagerSuite) loadAll() { + mgr := suite.mgr + + for i, collection := range suite.collections { + status := querypb.LoadStatus_Loaded + if suite.loadPercentage[i] < 100 { + status = querypb.LoadStatus_Loading + } + + if suite.loadTypes[i] == querypb.LoadType_LoadCollection { + mgr.PutCollection(&Collection{ + CollectionLoadInfo: &querypb.CollectionLoadInfo{ + CollectionID: collection, + ReplicaNumber: suite.replicaNumber[i], + Status: status, + }, + LoadPercentage: suite.loadPercentage[i], + CreatedAt: time.Now(), + }) + } else { + for _, partition := range suite.partitions[collection] { + mgr.PutPartition(&Partition{ + PartitionLoadInfo: &querypb.PartitionLoadInfo{ + CollectionID: collection, + PartitionID: partition, + ReplicaNumber: suite.replicaNumber[i], + Status: status, + }, + LoadPercentage: suite.loadPercentage[i], + CreatedAt: time.Now(), + }) + } + } + } +} + +func (suite *CollectionManagerSuite) clearMemory() { + suite.mgr.collections = make(map[int64]*Collection) + suite.mgr.partitions = make(map[int64]*Partition) +} + +func TestCollectionManager(t *testing.T) { + suite.Run(t, new(CollectionManagerSuite)) +} diff --git a/internal/querycoordv2/meta/coordinator_broker.go b/internal/querycoordv2/meta/coordinator_broker.go new file mode 100644 index 0000000000000..52956a1ffdb85 --- /dev/null +++ b/internal/querycoordv2/meta/coordinator_broker.go @@ -0,0 +1,191 @@ +package meta + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/indexpb" + "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/proto/schemapb" + "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/internal/types" + + "go.uber.org/zap" + + . "github.com/milvus-io/milvus/internal/util/typeutil" +) + +const ( + brokerRPCTimeout = 5 * time.Second +) + +type Broker interface { + GetCollectionSchema(ctx context.Context, collectionID UniqueID) (*schemapb.CollectionSchema, error) + GetPartitions(ctx context.Context, collectionID UniqueID) ([]UniqueID, error) + GetRecoveryInfo(ctx context.Context, collectionID UniqueID, partitionID UniqueID) ([]*datapb.VchannelInfo, []*datapb.SegmentBinlogs, error) + GetSegmentInfo(ctx context.Context, segmentID ...UniqueID) ([]*datapb.SegmentInfo, error) + GetIndexInfo(ctx context.Context, collectionID UniqueID, segmentID UniqueID) ([]*querypb.FieldIndexInfo, error) +} + +type CoordinatorBroker struct { + dataCoord types.DataCoord + rootCoord types.RootCoord + indexCoord types.IndexCoord + + cm storage.ChunkManager +} + +func NewCoordinatorBroker( + dataCoord types.DataCoord, + rootCoord types.RootCoord, + indexCoord types.IndexCoord, + cm storage.ChunkManager) *CoordinatorBroker { + return &CoordinatorBroker{ + dataCoord, + rootCoord, + indexCoord, + cm, + } +} + +func (broker *CoordinatorBroker) GetCollectionSchema(ctx context.Context, collectionID UniqueID) (*schemapb.CollectionSchema, error) { + ctx, cancel := context.WithTimeout(ctx, brokerRPCTimeout) + defer cancel() + + req := &milvuspb.DescribeCollectionRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_DescribeCollection, + }, + CollectionID: collectionID, + } + resp, err := broker.rootCoord.DescribeCollection(ctx, req) + return resp.GetSchema(), err +} + +func (broker *CoordinatorBroker) GetPartitions(ctx context.Context, collectionID UniqueID) ([]UniqueID, error) { + ctx, cancel := context.WithTimeout(ctx, brokerRPCTimeout) + defer cancel() + req := &milvuspb.ShowPartitionsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_ShowPartitions, + }, + CollectionID: collectionID, + } + resp, err := broker.rootCoord.ShowPartitions(ctx, req) + if err != nil { + log.Error("showPartition failed", zap.Int64("collectionID", collectionID), zap.Error(err)) + return nil, err + } + + if resp.Status.ErrorCode != commonpb.ErrorCode_Success { + err = errors.New(resp.Status.Reason) + log.Error("showPartition failed", zap.Int64("collectionID", collectionID), zap.Error(err)) + return nil, err + } + + return resp.PartitionIDs, nil +} + +func (broker *CoordinatorBroker) GetRecoveryInfo(ctx context.Context, collectionID UniqueID, partitionID UniqueID) ([]*datapb.VchannelInfo, []*datapb.SegmentBinlogs, error) { + ctx, cancel := context.WithTimeout(ctx, brokerRPCTimeout) + defer cancel() + + getRecoveryInfoRequest := &datapb.GetRecoveryInfoRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_GetRecoveryInfo, + }, + CollectionID: collectionID, + PartitionID: partitionID, + } + recoveryInfo, err := broker.dataCoord.GetRecoveryInfo(ctx, getRecoveryInfoRequest) + if err != nil { + log.Error("get recovery info failed", zap.Int64("collectionID", collectionID), zap.Int64("partitionID", partitionID), zap.Error(err)) + return nil, nil, err + } + + if recoveryInfo.Status.ErrorCode != commonpb.ErrorCode_Success { + err = errors.New(recoveryInfo.Status.Reason) + log.Error("get recovery info failed", zap.Int64("collectionID", collectionID), zap.Int64("partitionID", partitionID), zap.Error(err)) + return nil, nil, err + } + + return recoveryInfo.Channels, recoveryInfo.Binlogs, nil +} + +func (broker *CoordinatorBroker) GetSegmentInfo(ctx context.Context, ids ...UniqueID) ([]*datapb.SegmentInfo, error) { + ctx, cancel := context.WithTimeout(ctx, brokerRPCTimeout) + defer cancel() + + req := &datapb.GetSegmentInfoRequest{ + SegmentIDs: ids, + IncludeUnHealthy: true, + } + resp, err := broker.dataCoord.GetSegmentInfo(ctx, req) + if err != nil { + log.Error("failed to get segment info from DataCoord", + zap.Int64s("segments", ids), + zap.Error(err)) + return nil, err + } + + if len(resp.Infos) == 0 { + log.Warn("No such segment in DataCoord", + zap.Int64s("segments", ids)) + return nil, fmt.Errorf("no such segment in DataCoord") + } + + return resp.GetInfos(), nil +} + +func (broker *CoordinatorBroker) GetIndexInfo(ctx context.Context, collectionID UniqueID, segmentID UniqueID) ([]*querypb.FieldIndexInfo, error) { + ctx, cancel := context.WithTimeout(ctx, brokerRPCTimeout) + defer cancel() + + resp, err := broker.indexCoord.GetIndexInfos(ctx, &indexpb.GetIndexInfoRequest{ + SegmentIDs: []int64{segmentID}, + }) + if err != nil || resp.GetStatus().GetErrorCode() != commonpb.ErrorCode_Success { + log.Error("failed to get segment index info", + zap.Int64("collection", collectionID), + zap.Int64("segment", segmentID), + zap.Error(err)) + return nil, err + } + + segmentInfo := resp.SegmentInfo[segmentID] + + indexes := make([]*querypb.FieldIndexInfo, 0) + indexInfo := &querypb.FieldIndexInfo{ + EnableIndex: segmentInfo.EnableIndex, + } + if !segmentInfo.EnableIndex { + indexes = append(indexes, indexInfo) + return indexes, nil + } + for _, info := range segmentInfo.GetIndexInfos() { + indexInfo = &querypb.FieldIndexInfo{ + FieldID: info.GetFieldID(), + EnableIndex: segmentInfo.EnableIndex, + IndexName: info.GetIndexName(), + IndexID: info.GetIndexID(), + BuildID: info.GetBuildID(), + IndexParams: info.GetIndexParams(), + IndexFilePaths: info.GetIndexFilePaths(), + IndexSize: int64(info.GetSerializedSize()), + } + + if len(info.GetIndexFilePaths()) == 0 { + return nil, fmt.Errorf("index not ready") + } + + indexes = append(indexes, indexInfo) + } + + return indexes, nil +} diff --git a/internal/querycoordv2/meta/dist_manager.go b/internal/querycoordv2/meta/dist_manager.go new file mode 100644 index 0000000000000..58cb621294c75 --- /dev/null +++ b/internal/querycoordv2/meta/dist_manager.go @@ -0,0 +1,15 @@ +package meta + +type DistributionManager struct { + *SegmentDistManager + *ChannelDistManager + *LeaderViewManager +} + +func NewDistributionManager() *DistributionManager { + return &DistributionManager{ + SegmentDistManager: NewSegmentDistManager(), + ChannelDistManager: NewChannelDistManager(), + LeaderViewManager: NewLeaderViewManager(), + } +} diff --git a/internal/querycoordv2/meta/errors.go b/internal/querycoordv2/meta/errors.go new file mode 100644 index 0000000000000..feb6aabcbbb69 --- /dev/null +++ b/internal/querycoordv2/meta/errors.go @@ -0,0 +1,14 @@ +package meta + +import "errors" + +var ( + // Read errors + ErrCollectionNotFound = errors.New("CollectionNotFound") + ErrPartitionNotFound = errors.New("PartitionNotFound") + ErrReplicaNotFound = errors.New("ReplicaNotFound") + + // Store errors + ErrStoreCollectionFailed = errors.New("StoreCollectionFailed") + ErrStoreReplicaFailed = errors.New("StoreReplicaFailed") +) diff --git a/internal/querycoordv2/meta/leader_view_manager.go b/internal/querycoordv2/meta/leader_view_manager.go new file mode 100644 index 0000000000000..db18bb4e00235 --- /dev/null +++ b/internal/querycoordv2/meta/leader_view_manager.go @@ -0,0 +1,171 @@ +package meta + +import ( + "sync" + + "github.com/milvus-io/milvus/internal/util/typeutil" +) + +type LeaderView struct { + ID int64 + CollectionID int64 + Channel string + Segments map[int64]int64 // SegmentID -> NodeID + GrowingSegments typeutil.UniqueSet +} + +func (view *LeaderView) Clone() *LeaderView { + segments := make(map[int64]int64) + for k, v := range view.Segments { + segments[k] = v + } + growings := typeutil.NewUniqueSet(view.GrowingSegments.Collect()...) + + return &LeaderView{ + ID: view.ID, + CollectionID: view.CollectionID, + Channel: view.Channel, + Segments: segments, + GrowingSegments: growings, + } +} + +type channelViews map[string]*LeaderView + +type LeaderViewManager struct { + rwmutex sync.RWMutex + views map[int64]channelViews // LeaderID -> Views (one per shard) +} + +func NewLeaderViewManager() *LeaderViewManager { + return &LeaderViewManager{ + views: make(map[int64]channelViews), + } +} + +// GetSegmentByNode returns all segments that the given node contains, +// include growing segments +func (mgr *LeaderViewManager) GetSegmentByNode(nodeID int64) []int64 { + mgr.rwmutex.RLock() + defer mgr.rwmutex.RUnlock() + + segments := make([]int64, 0) + for leaderID, views := range mgr.views { + for _, view := range views { + for segment, node := range view.Segments { + if node == nodeID { + segments = append(segments, segment) + } + } + if leaderID == nodeID { + segments = append(segments, view.GrowingSegments.Collect()...) + } + } + } + return segments +} + +// Update updates the leader's views, all views have to be with the same leader ID +func (mgr *LeaderViewManager) Update(leaderID int64, views ...*LeaderView) { + mgr.rwmutex.Lock() + defer mgr.rwmutex.Unlock() + mgr.views[leaderID] = make(channelViews, len(views)) + for _, view := range views { + mgr.views[leaderID][view.Channel] = view + } +} + +// GetSegmentDist returns the list of nodes the given segment on +func (mgr *LeaderViewManager) GetSegmentDist(segmentID int64) []int64 { + mgr.rwmutex.RLock() + defer mgr.rwmutex.RUnlock() + + nodes := make([]int64, 0) + for leaderID, views := range mgr.views { + for _, view := range views { + node, ok := view.Segments[segmentID] + if ok { + nodes = append(nodes, node) + } + if view.GrowingSegments.Contain(segmentID) { + nodes = append(nodes, leaderID) + } + } + } + return nodes +} + +func (mgr *LeaderViewManager) GetSealedSegmentDist(segmentID int64) []int64 { + mgr.rwmutex.RLock() + defer mgr.rwmutex.RUnlock() + + nodes := make([]int64, 0) + for _, views := range mgr.views { + for _, view := range views { + node, ok := view.Segments[segmentID] + if ok { + nodes = append(nodes, node) + } + } + } + return nodes +} + +func (mgr *LeaderViewManager) GetGrowingSegmentDist(segmentID int64) []int64 { + mgr.rwmutex.RLock() + defer mgr.rwmutex.RUnlock() + + nodes := make([]int64, 0) + for leaderID, views := range mgr.views { + for _, view := range views { + if view.GrowingSegments.Contain(segmentID) { + nodes = append(nodes, leaderID) + break + } + } + } + return nodes +} + +// GetSegmentDist returns the list of nodes the given segment on +func (mgr *LeaderViewManager) GetChannelDist(channel string) []int64 { + mgr.rwmutex.RLock() + defer mgr.rwmutex.RUnlock() + + nodes := make([]int64, 0) + for leaderID, views := range mgr.views { + _, ok := views[channel] + if ok { + nodes = append(nodes, leaderID) + } + } + return nodes +} + +func (mgr *LeaderViewManager) GetLeaderView(id int64) map[string]*LeaderView { + mgr.rwmutex.RLock() + defer mgr.rwmutex.RUnlock() + + return mgr.views[id] +} + +func (mgr *LeaderViewManager) GetLeaderShardView(id int64, shard string) *LeaderView { + mgr.rwmutex.RLock() + defer mgr.rwmutex.RUnlock() + + return mgr.views[id][shard] +} + +func (mgr *LeaderViewManager) GetLeadersByShard(shard string) map[int64]*LeaderView { + mgr.rwmutex.RLock() + defer mgr.rwmutex.RUnlock() + + ret := make(map[int64]*LeaderView, 0) + for _, views := range mgr.views { + view, ok := views[shard] + if ok { + ret[view.ID] = view + } + } + return ret +} diff --git a/internal/querycoordv2/meta/leader_view_manager_test.go b/internal/querycoordv2/meta/leader_view_manager_test.go new file mode 100644 index 0000000000000..a75546d00049f --- /dev/null +++ b/internal/querycoordv2/meta/leader_view_manager_test.go @@ -0,0 +1,171 @@ +package meta + +import ( + "testing" + + "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/samber/lo" + "github.com/stretchr/testify/suite" +) + +type LeaderViewManagerSuite struct { + suite.Suite + // Data + collections []int64 + channels map[int64][]string + growingSegments map[int64]map[string]int64 + segments map[int64][]int64 + nodes []int64 + leaders map[int64]map[string]*LeaderView + + // Test object + mgr *LeaderViewManager +} + +func (suite *LeaderViewManagerSuite) SetupSuite() { + suite.collections = []int64{100, 101} + suite.channels = map[int64][]string{ + 100: {"100-dmc0", "100-dmc1"}, + 101: {"101-dmc0", "101-dmc1"}, + } + suite.growingSegments = map[int64]map[string]int64{ + 100: { + "100-dmc0": 10, + "100-dmc1": 11, + }, + 101: { + "101-dmc0": 12, + "101-dmc1": 13, + }, + } + suite.segments = map[int64][]int64{ + 100: {1, 2, 3, 4}, + 101: {5, 6, 7, 8}, + } + suite.nodes = []int64{1, 2, 3, 4} + + // Leaders: 1, 2 + suite.leaders = make(map[int64]map[string]*LeaderView) + for _, collection := range suite.collections { + for j := 1; j <= 2; j++ { + channel := suite.channels[collection][j-1] + view := &LeaderView{ + ID: int64(j), + CollectionID: collection, + Channel: channel, + GrowingSegments: typeutil.NewUniqueSet(suite.growingSegments[collection][channel]), + Segments: make(map[int64]int64), + } + for k, segment := range suite.segments[collection] { + view.Segments[segment] = suite.nodes[k] + } + suite.leaders[int64(j)] = map[string]*LeaderView{ + suite.channels[collection][j-1]: view, + } + } + } +} + +func (suite *LeaderViewManagerSuite) SetupTest() { + suite.mgr = NewLeaderViewManager() + for id, views := range suite.leaders { + suite.mgr.Update(id, lo.Values(views)...) + } +} + +func (suite *LeaderViewManagerSuite) TestGetDist() { + mgr := suite.mgr + + // Test GetSegmentDist + for segmentID := int64(1); segmentID <= 13; segmentID++ { + nodes := mgr.GetSegmentDist(segmentID) + suite.AssertSegmentDist(segmentID, nodes) + + for _, node := range nodes { + segments := mgr.GetSegmentByNode(node) + suite.Contains(segments, segmentID) + } + } + + // Test GetSealedSegmentDist + for segmentID := int64(1); segmentID <= 13; segmentID++ { + nodes := mgr.GetSealedSegmentDist(segmentID) + suite.AssertSegmentDist(segmentID, nodes) + + for _, node := range nodes { + segments := mgr.GetSegmentByNode(node) + suite.Contains(segments, segmentID) + } + } + + // Test GetGrowingSegmentDist + for segmentID := int64(1); segmentID <= 13; segmentID++ { + nodes := mgr.GetGrowingSegmentDist(segmentID) + + for _, node := range nodes { + segments := mgr.GetSegmentByNode(node) + suite.Contains(segments, segmentID) + suite.Contains(suite.leaders, node) + } + } + + // Test GetChannelDist + for _, shards := range suite.channels { + for _, shard := range shards { + nodes := mgr.GetChannelDist(shard) + suite.AssertChannelDist(shard, nodes) + } + } +} + +func (suite *LeaderViewManagerSuite) TestGetLeader() { + mgr := suite.mgr + + // Test GetLeaderView + for leader, view := range suite.leaders { + leaderView := mgr.GetLeaderView(leader) + suite.Equal(view, leaderView) + } + + // Test GetLeadersByShard + for leader, leaderViews := range suite.leaders { + for shard, view := range leaderViews { + views := mgr.GetLeadersByShard(shard) + suite.Len(views, 1) + suite.Equal(view, views[leader]) + } + } +} + +func (suite *LeaderViewManagerSuite) AssertSegmentDist(segment int64, nodes []int64) bool { + nodeSet := typeutil.NewUniqueSet(nodes...) + for leader, views := range suite.leaders { + for _, view := range views { + node, ok := view.Segments[segment] + if ok { + if !suite.True(nodeSet.Contain(node) || + node == leader && view.GrowingSegments.Contain(node)) { + return false + } + } + } + } + return true +} + +func (suite *LeaderViewManagerSuite) AssertChannelDist(channel string, nodes []int64) bool { + nodeSet := typeutil.NewUniqueSet(nodes...) + for leader, views := range suite.leaders { + _, ok := views[channel] + if ok { + if !suite.True(nodeSet.Contain(leader)) { + return false + } + } + } + return true +} + +func TestLeaderViewManager(t *testing.T) { + suite.Run(t, new(LeaderViewManagerSuite)) +} diff --git a/internal/querycoordv2/meta/meta.go b/internal/querycoordv2/meta/meta.go new file mode 100644 index 0000000000000..60746982c77a5 --- /dev/null +++ b/internal/querycoordv2/meta/meta.go @@ -0,0 +1,16 @@ +package meta + +type Meta struct { + *CollectionManager + *ReplicaManager +} + +func NewMeta( + idAllocator func() (int64, error), + store Store, +) *Meta { + return &Meta{ + NewCollectionManager(store), + NewReplicaManager(idAllocator, store), + } +} diff --git a/internal/querycoordv2/meta/mock_broker.go b/internal/querycoordv2/meta/mock_broker.go new file mode 100644 index 0000000000000..560ea9e03912f --- /dev/null +++ b/internal/querycoordv2/meta/mock_broker.go @@ -0,0 +1,302 @@ +// Code generated by mockery v2.14.0. DO NOT EDIT. + +package meta + +import ( + context "context" + + datapb "github.com/milvus-io/milvus/internal/proto/datapb" + mock "github.com/stretchr/testify/mock" + + querypb "github.com/milvus-io/milvus/internal/proto/querypb" + + schemapb "github.com/milvus-io/milvus/internal/proto/schemapb" +) + +// MockBroker is an autogenerated mock type for the Broker type +type MockBroker struct { + mock.Mock +} + +type MockBroker_Expecter struct { + mock *mock.Mock +} + +func (_m *MockBroker) EXPECT() *MockBroker_Expecter { + return &MockBroker_Expecter{mock: &_m.Mock} +} + +// GetCollectionSchema provides a mock function with given fields: ctx, collectionID +func (_m *MockBroker) GetCollectionSchema(ctx context.Context, collectionID int64) (*schemapb.CollectionSchema, error) { + ret := _m.Called(ctx, collectionID) + + var r0 *schemapb.CollectionSchema + if rf, ok := ret.Get(0).(func(context.Context, int64) *schemapb.CollectionSchema); ok { + r0 = rf(ctx, collectionID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*schemapb.CollectionSchema) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, collectionID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockBroker_GetCollectionSchema_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetCollectionSchema' +type MockBroker_GetCollectionSchema_Call struct { + *mock.Call +} + +// GetCollectionSchema is a helper method to define mock.On call +// - ctx context.Context +// - collectionID int64 +func (_e *MockBroker_Expecter) GetCollectionSchema(ctx interface{}, collectionID interface{}) *MockBroker_GetCollectionSchema_Call { + return &MockBroker_GetCollectionSchema_Call{Call: _e.mock.On("GetCollectionSchema", ctx, collectionID)} +} + +func (_c *MockBroker_GetCollectionSchema_Call) Run(run func(ctx context.Context, collectionID int64)) *MockBroker_GetCollectionSchema_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64)) + }) + return _c +} + +func (_c *MockBroker_GetCollectionSchema_Call) Return(_a0 *schemapb.CollectionSchema, _a1 error) *MockBroker_GetCollectionSchema_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// GetIndexInfo provides a mock function with given fields: ctx, collectionID, segmentID +func (_m *MockBroker) GetIndexInfo(ctx context.Context, collectionID int64, segmentID int64) ([]*querypb.FieldIndexInfo, error) { + ret := _m.Called(ctx, collectionID, segmentID) + + var r0 []*querypb.FieldIndexInfo + if rf, ok := ret.Get(0).(func(context.Context, int64, int64) []*querypb.FieldIndexInfo); ok { + r0 = rf(ctx, collectionID, segmentID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*querypb.FieldIndexInfo) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64, int64) error); ok { + r1 = rf(ctx, collectionID, segmentID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockBroker_GetIndexInfo_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetIndexInfo' +type MockBroker_GetIndexInfo_Call struct { + *mock.Call +} + +// GetIndexInfo is a helper method to define mock.On call +// - ctx context.Context +// - collectionID int64 +// - segmentID int64 +func (_e *MockBroker_Expecter) GetIndexInfo(ctx interface{}, collectionID interface{}, segmentID interface{}) *MockBroker_GetIndexInfo_Call { + return &MockBroker_GetIndexInfo_Call{Call: _e.mock.On("GetIndexInfo", ctx, collectionID, segmentID)} +} + +func (_c *MockBroker_GetIndexInfo_Call) Run(run func(ctx context.Context, collectionID int64, segmentID int64)) *MockBroker_GetIndexInfo_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64), args[2].(int64)) + }) + return _c +} + +func (_c *MockBroker_GetIndexInfo_Call) Return(_a0 []*querypb.FieldIndexInfo, _a1 error) *MockBroker_GetIndexInfo_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// GetPartitions provides a mock function with given fields: ctx, collectionID +func (_m *MockBroker) GetPartitions(ctx context.Context, collectionID int64) ([]int64, error) { + ret := _m.Called(ctx, collectionID) + + var r0 []int64 + if rf, ok := ret.Get(0).(func(context.Context, int64) []int64); ok { + r0 = rf(ctx, collectionID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]int64) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, collectionID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockBroker_GetPartitions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetPartitions' +type MockBroker_GetPartitions_Call struct { + *mock.Call +} + +// GetPartitions is a helper method to define mock.On call +// - ctx context.Context +// - collectionID int64 +func (_e *MockBroker_Expecter) GetPartitions(ctx interface{}, collectionID interface{}) *MockBroker_GetPartitions_Call { + return &MockBroker_GetPartitions_Call{Call: _e.mock.On("GetPartitions", ctx, collectionID)} +} + +func (_c *MockBroker_GetPartitions_Call) Run(run func(ctx context.Context, collectionID int64)) *MockBroker_GetPartitions_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64)) + }) + return _c +} + +func (_c *MockBroker_GetPartitions_Call) Return(_a0 []int64, _a1 error) *MockBroker_GetPartitions_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// GetRecoveryInfo provides a mock function with given fields: ctx, collectionID, partitionID +func (_m *MockBroker) GetRecoveryInfo(ctx context.Context, collectionID int64, partitionID int64) ([]*datapb.VchannelInfo, []*datapb.SegmentBinlogs, error) { + ret := _m.Called(ctx, collectionID, partitionID) + + var r0 []*datapb.VchannelInfo + if rf, ok := ret.Get(0).(func(context.Context, int64, int64) []*datapb.VchannelInfo); ok { + r0 = rf(ctx, collectionID, partitionID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*datapb.VchannelInfo) + } + } + + var r1 []*datapb.SegmentBinlogs + if rf, ok := ret.Get(1).(func(context.Context, int64, int64) []*datapb.SegmentBinlogs); ok { + r1 = rf(ctx, collectionID, partitionID) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).([]*datapb.SegmentBinlogs) + } + } + + var r2 error + if rf, ok := ret.Get(2).(func(context.Context, int64, int64) error); ok { + r2 = rf(ctx, collectionID, partitionID) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// MockBroker_GetRecoveryInfo_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetRecoveryInfo' +type MockBroker_GetRecoveryInfo_Call struct { + *mock.Call +} + +// GetRecoveryInfo is a helper method to define mock.On call +// - ctx context.Context +// - collectionID int64 +// - partitionID int64 +func (_e *MockBroker_Expecter) GetRecoveryInfo(ctx interface{}, collectionID interface{}, partitionID interface{}) *MockBroker_GetRecoveryInfo_Call { + return &MockBroker_GetRecoveryInfo_Call{Call: _e.mock.On("GetRecoveryInfo", ctx, collectionID, partitionID)} +} + +func (_c *MockBroker_GetRecoveryInfo_Call) Run(run func(ctx context.Context, collectionID int64, partitionID int64)) *MockBroker_GetRecoveryInfo_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64), args[2].(int64)) + }) + return _c +} + +func (_c *MockBroker_GetRecoveryInfo_Call) Return(_a0 []*datapb.VchannelInfo, _a1 []*datapb.SegmentBinlogs, _a2 error) *MockBroker_GetRecoveryInfo_Call { + _c.Call.Return(_a0, _a1, _a2) + return _c +} + +// GetSegmentInfo provides a mock function with given fields: ctx, segmentID +func (_m *MockBroker) GetSegmentInfo(ctx context.Context, segmentID ...int64) ([]*datapb.SegmentInfo, error) { + _va := make([]interface{}, len(segmentID)) + for _i := range segmentID { + _va[_i] = segmentID[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 []*datapb.SegmentInfo + if rf, ok := ret.Get(0).(func(context.Context, ...int64) []*datapb.SegmentInfo); ok { + r0 = rf(ctx, segmentID...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*datapb.SegmentInfo) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, ...int64) error); ok { + r1 = rf(ctx, segmentID...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockBroker_GetSegmentInfo_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSegmentInfo' +type MockBroker_GetSegmentInfo_Call struct { + *mock.Call +} + +// GetSegmentInfo is a helper method to define mock.On call +// - ctx context.Context +// - segmentID ...int64 +func (_e *MockBroker_Expecter) GetSegmentInfo(ctx interface{}, segmentID ...interface{}) *MockBroker_GetSegmentInfo_Call { + return &MockBroker_GetSegmentInfo_Call{Call: _e.mock.On("GetSegmentInfo", + append([]interface{}{ctx}, segmentID...)...)} +} + +func (_c *MockBroker_GetSegmentInfo_Call) Run(run func(ctx context.Context, segmentID ...int64)) *MockBroker_GetSegmentInfo_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]int64, len(args)-1) + for i, a := range args[1:] { + if a != nil { + variadicArgs[i] = a.(int64) + } + } + run(args[0].(context.Context), variadicArgs...) + }) + return _c +} + +func (_c *MockBroker_GetSegmentInfo_Call) Return(_a0 []*datapb.SegmentInfo, _a1 error) *MockBroker_GetSegmentInfo_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +type mockConstructorTestingTNewMockBroker interface { + mock.TestingT + Cleanup(func()) +} + +// NewMockBroker creates a new instance of MockBroker. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMockBroker(t mockConstructorTestingTNewMockBroker) *MockBroker { + mock := &MockBroker{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/querycoordv2/meta/mock_store.go b/internal/querycoordv2/meta/mock_store.go new file mode 100644 index 0000000000000..75e8b337ff679 --- /dev/null +++ b/internal/querycoordv2/meta/mock_store.go @@ -0,0 +1,598 @@ +// Code generated by mockery v2.14.0. DO NOT EDIT. + +package meta + +import ( + mock "github.com/stretchr/testify/mock" + clientv3 "go.etcd.io/etcd/client/v3" + + querypb "github.com/milvus-io/milvus/internal/proto/querypb" +) + +// MockStore is an autogenerated mock type for the Store type +type MockStore struct { + mock.Mock +} + +type MockStore_Expecter struct { + mock *mock.Mock +} + +func (_m *MockStore) EXPECT() *MockStore_Expecter { + return &MockStore_Expecter{mock: &_m.Mock} +} + +// GetCollections provides a mock function with given fields: +func (_m *MockStore) GetCollections() ([]*querypb.CollectionLoadInfo, error) { + ret := _m.Called() + + var r0 []*querypb.CollectionLoadInfo + if rf, ok := ret.Get(0).(func() []*querypb.CollectionLoadInfo); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*querypb.CollectionLoadInfo) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockStore_GetCollections_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetCollections' +type MockStore_GetCollections_Call struct { + *mock.Call +} + +// GetCollections is a helper method to define mock.On call +func (_e *MockStore_Expecter) GetCollections() *MockStore_GetCollections_Call { + return &MockStore_GetCollections_Call{Call: _e.mock.On("GetCollections")} +} + +func (_c *MockStore_GetCollections_Call) Run(run func()) *MockStore_GetCollections_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockStore_GetCollections_Call) Return(_a0 []*querypb.CollectionLoadInfo, _a1 error) *MockStore_GetCollections_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// GetPartitions provides a mock function with given fields: +func (_m *MockStore) GetPartitions() (map[int64][]*querypb.PartitionLoadInfo, error) { + ret := _m.Called() + + var r0 map[int64][]*querypb.PartitionLoadInfo + if rf, ok := ret.Get(0).(func() map[int64][]*querypb.PartitionLoadInfo); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[int64][]*querypb.PartitionLoadInfo) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockStore_GetPartitions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetPartitions' +type MockStore_GetPartitions_Call struct { + *mock.Call +} + +// GetPartitions is a helper method to define mock.On call +func (_e *MockStore_Expecter) GetPartitions() *MockStore_GetPartitions_Call { + return &MockStore_GetPartitions_Call{Call: _e.mock.On("GetPartitions")} +} + +func (_c *MockStore_GetPartitions_Call) Run(run func()) *MockStore_GetPartitions_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockStore_GetPartitions_Call) Return(_a0 map[int64][]*querypb.PartitionLoadInfo, _a1 error) *MockStore_GetPartitions_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// GetReplicas provides a mock function with given fields: +func (_m *MockStore) GetReplicas() ([]*querypb.Replica, error) { + ret := _m.Called() + + var r0 []*querypb.Replica + if rf, ok := ret.Get(0).(func() []*querypb.Replica); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*querypb.Replica) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockStore_GetReplicas_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetReplicas' +type MockStore_GetReplicas_Call struct { + *mock.Call +} + +// GetReplicas is a helper method to define mock.On call +func (_e *MockStore_Expecter) GetReplicas() *MockStore_GetReplicas_Call { + return &MockStore_GetReplicas_Call{Call: _e.mock.On("GetReplicas")} +} + +func (_c *MockStore_GetReplicas_Call) Run(run func()) *MockStore_GetReplicas_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockStore_GetReplicas_Call) Return(_a0 []*querypb.Replica, _a1 error) *MockStore_GetReplicas_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// LoadHandoffWithRevision provides a mock function with given fields: +func (_m *MockStore) LoadHandoffWithRevision() ([]string, []string, int64, error) { + ret := _m.Called() + + var r0 []string + if rf, ok := ret.Get(0).(func() []string); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 []string + if rf, ok := ret.Get(1).(func() []string); ok { + r1 = rf() + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).([]string) + } + } + + var r2 int64 + if rf, ok := ret.Get(2).(func() int64); ok { + r2 = rf() + } else { + r2 = ret.Get(2).(int64) + } + + var r3 error + if rf, ok := ret.Get(3).(func() error); ok { + r3 = rf() + } else { + r3 = ret.Error(3) + } + + return r0, r1, r2, r3 +} + +// MockStore_LoadHandoffWithRevision_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LoadHandoffWithRevision' +type MockStore_LoadHandoffWithRevision_Call struct { + *mock.Call +} + +// LoadHandoffWithRevision is a helper method to define mock.On call +func (_e *MockStore_Expecter) LoadHandoffWithRevision() *MockStore_LoadHandoffWithRevision_Call { + return &MockStore_LoadHandoffWithRevision_Call{Call: _e.mock.On("LoadHandoffWithRevision")} +} + +func (_c *MockStore_LoadHandoffWithRevision_Call) Run(run func()) *MockStore_LoadHandoffWithRevision_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockStore_LoadHandoffWithRevision_Call) Return(_a0 []string, _a1 []string, _a2 int64, _a3 error) *MockStore_LoadHandoffWithRevision_Call { + _c.Call.Return(_a0, _a1, _a2, _a3) + return _c +} + +// ReleaseCollection provides a mock function with given fields: id +func (_m *MockStore) ReleaseCollection(id int64) error { + ret := _m.Called(id) + + var r0 error + if rf, ok := ret.Get(0).(func(int64) error); ok { + r0 = rf(id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockStore_ReleaseCollection_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReleaseCollection' +type MockStore_ReleaseCollection_Call struct { + *mock.Call +} + +// ReleaseCollection is a helper method to define mock.On call +// - id int64 +func (_e *MockStore_Expecter) ReleaseCollection(id interface{}) *MockStore_ReleaseCollection_Call { + return &MockStore_ReleaseCollection_Call{Call: _e.mock.On("ReleaseCollection", id)} +} + +func (_c *MockStore_ReleaseCollection_Call) Run(run func(id int64)) *MockStore_ReleaseCollection_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(int64)) + }) + return _c +} + +func (_c *MockStore_ReleaseCollection_Call) Return(_a0 error) *MockStore_ReleaseCollection_Call { + _c.Call.Return(_a0) + return _c +} + +// ReleasePartition provides a mock function with given fields: collection, partitions +func (_m *MockStore) ReleasePartition(collection int64, partitions ...int64) error { + _va := make([]interface{}, len(partitions)) + for _i := range partitions { + _va[_i] = partitions[_i] + } + var _ca []interface{} + _ca = append(_ca, collection) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 error + if rf, ok := ret.Get(0).(func(int64, ...int64) error); ok { + r0 = rf(collection, partitions...) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockStore_ReleasePartition_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReleasePartition' +type MockStore_ReleasePartition_Call struct { + *mock.Call +} + +// ReleasePartition is a helper method to define mock.On call +// - collection int64 +// - partitions ...int64 +func (_e *MockStore_Expecter) ReleasePartition(collection interface{}, partitions ...interface{}) *MockStore_ReleasePartition_Call { + return &MockStore_ReleasePartition_Call{Call: _e.mock.On("ReleasePartition", + append([]interface{}{collection}, partitions...)...)} +} + +func (_c *MockStore_ReleasePartition_Call) Run(run func(collection int64, partitions ...int64)) *MockStore_ReleasePartition_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]int64, len(args)-1) + for i, a := range args[1:] { + if a != nil { + variadicArgs[i] = a.(int64) + } + } + run(args[0].(int64), variadicArgs...) + }) + return _c +} + +func (_c *MockStore_ReleasePartition_Call) Return(_a0 error) *MockStore_ReleasePartition_Call { + _c.Call.Return(_a0) + return _c +} + +// ReleaseReplica provides a mock function with given fields: collection, replica +func (_m *MockStore) ReleaseReplica(collection int64, replica int64) error { + ret := _m.Called(collection, replica) + + var r0 error + if rf, ok := ret.Get(0).(func(int64, int64) error); ok { + r0 = rf(collection, replica) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockStore_ReleaseReplica_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReleaseReplica' +type MockStore_ReleaseReplica_Call struct { + *mock.Call +} + +// ReleaseReplica is a helper method to define mock.On call +// - collection int64 +// - replica int64 +func (_e *MockStore_Expecter) ReleaseReplica(collection interface{}, replica interface{}) *MockStore_ReleaseReplica_Call { + return &MockStore_ReleaseReplica_Call{Call: _e.mock.On("ReleaseReplica", collection, replica)} +} + +func (_c *MockStore_ReleaseReplica_Call) Run(run func(collection int64, replica int64)) *MockStore_ReleaseReplica_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(int64), args[1].(int64)) + }) + return _c +} + +func (_c *MockStore_ReleaseReplica_Call) Return(_a0 error) *MockStore_ReleaseReplica_Call { + _c.Call.Return(_a0) + return _c +} + +// ReleaseReplicas provides a mock function with given fields: collectionID +func (_m *MockStore) ReleaseReplicas(collectionID int64) error { + ret := _m.Called(collectionID) + + var r0 error + if rf, ok := ret.Get(0).(func(int64) error); ok { + r0 = rf(collectionID) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockStore_ReleaseReplicas_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReleaseReplicas' +type MockStore_ReleaseReplicas_Call struct { + *mock.Call +} + +// ReleaseReplicas is a helper method to define mock.On call +// - collectionID int64 +func (_e *MockStore_Expecter) ReleaseReplicas(collectionID interface{}) *MockStore_ReleaseReplicas_Call { + return &MockStore_ReleaseReplicas_Call{Call: _e.mock.On("ReleaseReplicas", collectionID)} +} + +func (_c *MockStore_ReleaseReplicas_Call) Run(run func(collectionID int64)) *MockStore_ReleaseReplicas_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(int64)) + }) + return _c +} + +func (_c *MockStore_ReleaseReplicas_Call) Return(_a0 error) *MockStore_ReleaseReplicas_Call { + _c.Call.Return(_a0) + return _c +} + +// RemoveHandoffEvent provides a mock function with given fields: segmentInfo +func (_m *MockStore) RemoveHandoffEvent(segmentInfo *querypb.SegmentInfo) error { + ret := _m.Called(segmentInfo) + + var r0 error + if rf, ok := ret.Get(0).(func(*querypb.SegmentInfo) error); ok { + r0 = rf(segmentInfo) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockStore_RemoveHandoffEvent_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RemoveHandoffEvent' +type MockStore_RemoveHandoffEvent_Call struct { + *mock.Call +} + +// RemoveHandoffEvent is a helper method to define mock.On call +// - segmentInfo *querypb.SegmentInfo +func (_e *MockStore_Expecter) RemoveHandoffEvent(segmentInfo interface{}) *MockStore_RemoveHandoffEvent_Call { + return &MockStore_RemoveHandoffEvent_Call{Call: _e.mock.On("RemoveHandoffEvent", segmentInfo)} +} + +func (_c *MockStore_RemoveHandoffEvent_Call) Run(run func(segmentInfo *querypb.SegmentInfo)) *MockStore_RemoveHandoffEvent_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*querypb.SegmentInfo)) + }) + return _c +} + +func (_c *MockStore_RemoveHandoffEvent_Call) Return(_a0 error) *MockStore_RemoveHandoffEvent_Call { + _c.Call.Return(_a0) + return _c +} + +// SaveCollection provides a mock function with given fields: info +func (_m *MockStore) SaveCollection(info *querypb.CollectionLoadInfo) error { + ret := _m.Called(info) + + var r0 error + if rf, ok := ret.Get(0).(func(*querypb.CollectionLoadInfo) error); ok { + r0 = rf(info) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockStore_SaveCollection_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SaveCollection' +type MockStore_SaveCollection_Call struct { + *mock.Call +} + +// SaveCollection is a helper method to define mock.On call +// - info *querypb.CollectionLoadInfo +func (_e *MockStore_Expecter) SaveCollection(info interface{}) *MockStore_SaveCollection_Call { + return &MockStore_SaveCollection_Call{Call: _e.mock.On("SaveCollection", info)} +} + +func (_c *MockStore_SaveCollection_Call) Run(run func(info *querypb.CollectionLoadInfo)) *MockStore_SaveCollection_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*querypb.CollectionLoadInfo)) + }) + return _c +} + +func (_c *MockStore_SaveCollection_Call) Return(_a0 error) *MockStore_SaveCollection_Call { + _c.Call.Return(_a0) + return _c +} + +// SavePartition provides a mock function with given fields: info +func (_m *MockStore) SavePartition(info ...*querypb.PartitionLoadInfo) error { + _va := make([]interface{}, len(info)) + for _i := range info { + _va[_i] = info[_i] + } + var _ca []interface{} + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 error + if rf, ok := ret.Get(0).(func(...*querypb.PartitionLoadInfo) error); ok { + r0 = rf(info...) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockStore_SavePartition_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SavePartition' +type MockStore_SavePartition_Call struct { + *mock.Call +} + +// SavePartition is a helper method to define mock.On call +// - info ...*querypb.PartitionLoadInfo +func (_e *MockStore_Expecter) SavePartition(info ...interface{}) *MockStore_SavePartition_Call { + return &MockStore_SavePartition_Call{Call: _e.mock.On("SavePartition", + append([]interface{}{}, info...)...)} +} + +func (_c *MockStore_SavePartition_Call) Run(run func(info ...*querypb.PartitionLoadInfo)) *MockStore_SavePartition_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]*querypb.PartitionLoadInfo, len(args)-0) + for i, a := range args[0:] { + if a != nil { + variadicArgs[i] = a.(*querypb.PartitionLoadInfo) + } + } + run(variadicArgs...) + }) + return _c +} + +func (_c *MockStore_SavePartition_Call) Return(_a0 error) *MockStore_SavePartition_Call { + _c.Call.Return(_a0) + return _c +} + +// SaveReplica provides a mock function with given fields: replica +func (_m *MockStore) SaveReplica(replica *querypb.Replica) error { + ret := _m.Called(replica) + + var r0 error + if rf, ok := ret.Get(0).(func(*querypb.Replica) error); ok { + r0 = rf(replica) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockStore_SaveReplica_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SaveReplica' +type MockStore_SaveReplica_Call struct { + *mock.Call +} + +// SaveReplica is a helper method to define mock.On call +// - replica *querypb.Replica +func (_e *MockStore_Expecter) SaveReplica(replica interface{}) *MockStore_SaveReplica_Call { + return &MockStore_SaveReplica_Call{Call: _e.mock.On("SaveReplica", replica)} +} + +func (_c *MockStore_SaveReplica_Call) Run(run func(replica *querypb.Replica)) *MockStore_SaveReplica_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*querypb.Replica)) + }) + return _c +} + +func (_c *MockStore_SaveReplica_Call) Return(_a0 error) *MockStore_SaveReplica_Call { + _c.Call.Return(_a0) + return _c +} + +// WatchHandoffEvent provides a mock function with given fields: revision +func (_m *MockStore) WatchHandoffEvent(revision int64) clientv3.WatchChan { + ret := _m.Called(revision) + + var r0 clientv3.WatchChan + if rf, ok := ret.Get(0).(func(int64) clientv3.WatchChan); ok { + r0 = rf(revision) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(clientv3.WatchChan) + } + } + + return r0 +} + +// MockStore_WatchHandoffEvent_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'WatchHandoffEvent' +type MockStore_WatchHandoffEvent_Call struct { + *mock.Call +} + +// WatchHandoffEvent is a helper method to define mock.On call +// - revision int64 +func (_e *MockStore_Expecter) WatchHandoffEvent(revision interface{}) *MockStore_WatchHandoffEvent_Call { + return &MockStore_WatchHandoffEvent_Call{Call: _e.mock.On("WatchHandoffEvent", revision)} +} + +func (_c *MockStore_WatchHandoffEvent_Call) Run(run func(revision int64)) *MockStore_WatchHandoffEvent_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(int64)) + }) + return _c +} + +func (_c *MockStore_WatchHandoffEvent_Call) Return(_a0 clientv3.WatchChan) *MockStore_WatchHandoffEvent_Call { + _c.Call.Return(_a0) + return _c +} + +type mockConstructorTestingTNewMockStore interface { + mock.TestingT + Cleanup(func()) +} + +// NewMockStore creates a new instance of MockStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMockStore(t mockConstructorTestingTNewMockStore) *MockStore { + mock := &MockStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/querycoordv2/meta/replica_manager.go b/internal/querycoordv2/meta/replica_manager.go new file mode 100644 index 0000000000000..9d9a4841d317f --- /dev/null +++ b/internal/querycoordv2/meta/replica_manager.go @@ -0,0 +1,191 @@ +package meta + +import ( + "fmt" + "sync" + + "github.com/golang/protobuf/proto" + "github.com/milvus-io/milvus/internal/proto/querypb" + . "github.com/milvus-io/milvus/internal/util/typeutil" +) + +type Replica struct { + *querypb.Replica + Nodes UniqueSet // a helper field for manipulating replica's Nodes slice field +} + +func (replica *Replica) AddNode(nodes ...int64) { + replica.Nodes.Insert(nodes...) + replica.Replica.Nodes = replica.Nodes.Collect() +} + +func (replica *Replica) RemoveNode(nodes ...int64) { + replica.Nodes.Remove(nodes...) + replica.Replica.Nodes = replica.Nodes.Collect() +} + +func (replica *Replica) Clone() *Replica { + return &Replica{ + Replica: proto.Clone(replica.Replica).(*querypb.Replica), + Nodes: NewUniqueSet(replica.Replica.Nodes...), + } +} + +type ReplicaManager struct { + rwmutex sync.RWMutex + + idAllocator func() (int64, error) + replicas map[UniqueID]*Replica + store Store +} + +func NewReplicaManager(idAllocator func() (int64, error), store Store) *ReplicaManager { + return &ReplicaManager{ + idAllocator: idAllocator, + replicas: make(map[int64]*Replica), + store: store, + } +} + +// Recover recovers the replicas for given collections from meta store +func (m *ReplicaManager) Recover() error { + replicas, err := m.store.GetReplicas() + if err != nil { + return fmt.Errorf("failed to recover replicas, err=%w", err) + } + for _, replica := range replicas { + m.replicas[replica.GetID()] = &Replica{ + Replica: replica, + Nodes: NewUniqueSet(replica.GetNodes()...), + } + } + return nil +} + +func (m *ReplicaManager) Get(id UniqueID) *Replica { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + return m.replicas[id] +} + +// Spawn spawns replicas of the given number, for given collection, +// this doesn't store these replicas and assign nodes to them. +func (m *ReplicaManager) Spawn(collection int64, replicaNumber int32) ([]*Replica, error) { + var ( + replicas = make([]*Replica, replicaNumber) + err error + ) + for i := range replicas { + replicas[i], err = m.spawn(collection) + if err != nil { + return nil, err + } + } + return replicas, err +} + +func (m *ReplicaManager) Put(replicas ...*Replica) error { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + return m.put(replicas...) +} + +func (m *ReplicaManager) spawn(collectionID UniqueID) (*Replica, error) { + id, err := m.idAllocator() + if err != nil { + return nil, err + } + return &Replica{ + Replica: &querypb.Replica{ + ID: id, + CollectionID: collectionID, + }, + Nodes: make(UniqueSet), + }, nil +} + +func (m *ReplicaManager) put(replicas ...*Replica) error { + for _, replica := range replicas { + err := m.store.SaveReplica(replica.Replica) + if err != nil { + return err + } + m.replicas[replica.ID] = replica + } + return nil +} + +// RemoveCollection removes replicas of given collection, +// returns error if failed to remove replica from KV +func (m *ReplicaManager) RemoveCollection(collectionID UniqueID) error { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + err := m.store.ReleaseReplicas(collectionID) + if err != nil { + return err + } + for id, replica := range m.replicas { + if replica.CollectionID == collectionID { + delete(m.replicas, id) + } + } + return nil +} + +func (m *ReplicaManager) GetByCollection(collectionID UniqueID) []*Replica { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + replicas := make([]*Replica, 0, 3) + for _, replica := range m.replicas { + if replica.CollectionID == collectionID { + replicas = append(replicas, replica) + } + } + + return replicas +} + +func (m *ReplicaManager) GetByCollectionAndNode(collectionID, nodeID UniqueID) *Replica { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + for _, replica := range m.replicas { + if replica.CollectionID == collectionID && replica.Nodes.Contain(nodeID) { + return replica + } + } + + return nil +} + +func (m *ReplicaManager) AddNode(replicaID UniqueID, nodes ...UniqueID) error { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + replica, ok := m.replicas[replicaID] + if !ok { + return ErrReplicaNotFound + } + + replica = replica.Clone() + replica.AddNode(nodes...) + return m.put(replica) +} + +func (m *ReplicaManager) RemoveNode(replicaID UniqueID, nodes ...UniqueID) error { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + replica, ok := m.replicas[replicaID] + if !ok { + return ErrReplicaNotFound + } + + replica = replica.Clone() + replica.RemoveNode(nodes...) + return m.put(replica) +} diff --git a/internal/querycoordv2/meta/replica_manager_test.go b/internal/querycoordv2/meta/replica_manager_test.go new file mode 100644 index 0000000000000..b7adc1f370b05 --- /dev/null +++ b/internal/querycoordv2/meta/replica_manager_test.go @@ -0,0 +1,198 @@ +package meta + +import ( + "testing" + + "github.com/golang/protobuf/proto" + "github.com/milvus-io/milvus/internal/kv" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/internal/proto/milvuspb" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/util/etcd" + "github.com/stretchr/testify/suite" +) + +type ReplicaManagerSuite struct { + suite.Suite + + nodes []int64 + collections []int64 + replicaNumbers []int32 + idAllocator func() (int64, error) + kv kv.MetaKv + store Store + mgr *ReplicaManager +} + +func (suite *ReplicaManagerSuite) SetupSuite() { + Params.Init() + + suite.nodes = []int64{1, 2, 3} + suite.collections = []int64{100, 101, 102} + suite.replicaNumbers = []int32{1, 2, 3} +} + +func (suite *ReplicaManagerSuite) SetupTest() { + var err error + config := GenerateEtcdConfig() + cli, err := etcd.GetEtcdClient(&config) + suite.Require().NoError(err) + suite.kv = etcdkv.NewEtcdKV(cli, config.MetaRootPath) + suite.store = NewMetaStore(suite.kv) + + suite.idAllocator = RandomIncrementIDAllocator() + suite.mgr = NewReplicaManager(suite.idAllocator, suite.store) + suite.spawnAndPutAll() +} + +func (suite *ReplicaManagerSuite) TearDownTest() { + suite.kv.Close() +} + +func (suite *ReplicaManagerSuite) TestSpawn() { + mgr := suite.mgr + + for i, collection := range suite.collections { + replicas, err := mgr.Spawn(collection, suite.replicaNumbers[i]) + suite.NoError(err) + suite.Len(replicas, int(suite.replicaNumbers[i])) + } + + mgr.idAllocator = ErrorIDAllocator() + for i, collection := range suite.collections { + _, err := mgr.Spawn(collection, suite.replicaNumbers[i]) + suite.Error(err) + } +} + +func (suite *ReplicaManagerSuite) TestGet() { + mgr := suite.mgr + + for i, collection := range suite.collections { + replicas := mgr.GetByCollection(collection) + replicaNodes := make(map[int64][]int64) + nodes := make([]int64, 0) + for _, replica := range replicas { + suite.Equal(collection, replica.GetCollectionID()) + suite.Equal(replica, mgr.Get(replica.GetID())) + suite.Equal(replica.Replica.Nodes, replica.Nodes.Collect()) + replicaNodes[replica.GetID()] = replica.Replica.Nodes + nodes = append(nodes, replica.Replica.Nodes...) + } + suite.Len(nodes, int(suite.replicaNumbers[i])) + + for replicaID, nodes := range replicaNodes { + for _, node := range nodes { + replica := mgr.GetByCollectionAndNode(collection, node) + suite.Equal(replicaID, replica.GetID()) + } + } + } +} + +func (suite *ReplicaManagerSuite) TestRecover() { + mgr := suite.mgr + + // Clear data in memory, and then recover from meta store + suite.clearMemory() + mgr.Recover() + suite.TestGet() + + // Test recover from 2.1 meta store + replicaInfo := milvuspb.ReplicaInfo{ + ReplicaID: 2100, + CollectionID: 1000, + NodeIds: []int64{1, 2, 3}, + } + value, err := proto.Marshal(&replicaInfo) + suite.NoError(err) + suite.kv.Save(ReplicaMetaPrefixV1+"/2100", string(value)) + + suite.clearMemory() + mgr.Recover() + replica := mgr.Get(2100) + suite.NotNil(replica) + suite.EqualValues(1000, replica.CollectionID) + suite.EqualValues([]int64{1, 2, 3}, replica.Replica.Nodes) + suite.Len(replica.Nodes, len(replica.Replica.GetNodes())) + for _, node := range replica.Replica.GetNodes() { + suite.True(replica.Nodes.Contain(node)) + } +} + +func (suite *ReplicaManagerSuite) TestRemove() { + mgr := suite.mgr + + for _, collection := range suite.collections { + err := mgr.RemoveCollection(collection) + suite.NoError(err) + + replicas := mgr.GetByCollection(collection) + suite.Empty(replicas) + } + + // Check whether the replicas are also removed from meta store + mgr.Recover() + for _, collection := range suite.collections { + replicas := mgr.GetByCollection(collection) + suite.Empty(replicas) + } +} + +func (suite *ReplicaManagerSuite) TestNodeManipulate() { + mgr := suite.mgr + + firstNode := suite.nodes[0] + newNode := suite.nodes[len(suite.nodes)-1] + 1 + // Add a new node for the replica with node 1 of all collections, + // then remove the node 1 + for _, collection := range suite.collections { + replica := mgr.GetByCollectionAndNode(collection, firstNode) + err := mgr.AddNode(replica.GetID(), newNode) + suite.NoError(err) + + replica = mgr.GetByCollectionAndNode(collection, newNode) + suite.Contains(replica.Nodes, newNode) + suite.Contains(replica.Replica.GetNodes(), newNode) + + err = mgr.RemoveNode(replica.GetID(), firstNode) + suite.NoError(err) + replica = mgr.GetByCollectionAndNode(collection, firstNode) + suite.Nil(replica) + } + + // Check these modifications are applied to meta store + suite.clearMemory() + mgr.Recover() + for _, collection := range suite.collections { + replica := mgr.GetByCollectionAndNode(collection, firstNode) + suite.Nil(replica) + + replica = mgr.GetByCollectionAndNode(collection, newNode) + suite.Contains(replica.Nodes, newNode) + suite.Contains(replica.Replica.GetNodes(), newNode) + } +} + +func (suite *ReplicaManagerSuite) spawnAndPutAll() { + mgr := suite.mgr + + for i, collection := range suite.collections { + replicas, err := mgr.Spawn(collection, suite.replicaNumbers[i]) + suite.NoError(err) + suite.Len(replicas, int(suite.replicaNumbers[i])) + for j, replica := range replicas { + replica.AddNode(suite.nodes[j]) + } + err = mgr.Put(replicas...) + suite.NoError(err) + } +} + +func (suite *ReplicaManagerSuite) clearMemory() { + suite.mgr.replicas = make(map[int64]*Replica) +} + +func TestReplicaManager(t *testing.T) { + suite.Run(t, new(ReplicaManagerSuite)) +} diff --git a/internal/querycoordv2/meta/segment_dist_manager.go b/internal/querycoordv2/meta/segment_dist_manager.go new file mode 100644 index 0000000000000..9b717d1c0138d --- /dev/null +++ b/internal/querycoordv2/meta/segment_dist_manager.go @@ -0,0 +1,142 @@ +package meta + +import ( + "sync" + + "github.com/golang/protobuf/proto" + "github.com/milvus-io/milvus/internal/proto/datapb" + . "github.com/milvus-io/milvus/internal/util/typeutil" +) + +type Segment struct { + *datapb.SegmentInfo + Node int64 // Node the segment is in + Version int64 // Version is the timestamp of loading segment +} + +func SegmentFromInfo(info *datapb.SegmentInfo) *Segment { + return &Segment{ + SegmentInfo: info, + } +} + +func (segment *Segment) Clone() *Segment { + return &Segment{ + SegmentInfo: proto.Clone(segment.SegmentInfo).(*datapb.SegmentInfo), + Node: segment.Node, + Version: segment.Version, + } +} + +type SegmentDistManager struct { + rwmutex sync.RWMutex + + // nodeID -> []*Segment + segments map[UniqueID][]*Segment +} + +func NewSegmentDistManager() *SegmentDistManager { + return &SegmentDistManager{ + segments: make(map[UniqueID][]*Segment), + } +} + +func (m *SegmentDistManager) Update(nodeID UniqueID, segments ...*Segment) { + m.rwmutex.Lock() + defer m.rwmutex.Unlock() + + for _, segment := range segments { + segment.Node = nodeID + } + m.segments[nodeID] = segments +} + +func (m *SegmentDistManager) Get(id UniqueID) []*Segment { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + ret := make([]*Segment, 0) + for _, segments := range m.segments { + for _, segment := range segments { + if segment.GetID() == id { + ret = append(ret, segment) + } + } + } + return ret +} + +// GetAll returns all segments +func (m *SegmentDistManager) GetAll() []*Segment { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + ret := make([]*Segment, 0) + for _, segments := range m.segments { + ret = append(ret, segments...) + } + return ret +} + +// func (m *SegmentDistManager) Remove(ids ...UniqueID) { +// m.rwmutex.Lock() +// defer m.rwmutex.Unlock() + +// for _, id := range ids { +// delete(m.segments, id) +// } +// } + +// GetByNode returns all segments of the given node. +func (m *SegmentDistManager) GetByNode(nodeID UniqueID) []*Segment { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + return m.segments[nodeID] +} + +// GetByCollection returns all segments of the given collection. +func (m *SegmentDistManager) GetByCollection(collectionID UniqueID) []*Segment { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + ret := make([]*Segment, 0) + for _, segments := range m.segments { + for _, segment := range segments { + if segment.CollectionID == collectionID { + ret = append(ret, segment) + } + } + } + return ret +} + +// GetByShard returns all segments of the given collection. +func (m *SegmentDistManager) GetByShard(shard string) []*Segment { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + ret := make([]*Segment, 0) + for _, segments := range m.segments { + for _, segment := range segments { + if segment.GetInsertChannel() == shard { + ret = append(ret, segment) + } + } + } + return ret +} + +// GetByCollectionAndNode returns all segments of the given collection and node. +func (m *SegmentDistManager) GetByCollectionAndNode(collectionID, nodeID UniqueID) []*Segment { + m.rwmutex.RLock() + defer m.rwmutex.RUnlock() + + ret := make([]*Segment, 0) + for _, segment := range m.segments[nodeID] { + if segment.CollectionID == collectionID { + ret = append(ret, segment) + } + } + return ret +} diff --git a/internal/querycoordv2/meta/segment_dist_manager_test.go b/internal/querycoordv2/meta/segment_dist_manager_test.go new file mode 100644 index 0000000000000..fc20144c849e7 --- /dev/null +++ b/internal/querycoordv2/meta/segment_dist_manager_test.go @@ -0,0 +1,154 @@ +package meta + +import ( + "testing" + + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/stretchr/testify/suite" +) + +type SegmentDistManagerSuite struct { + suite.Suite + dist *SegmentDistManager + collection int64 + partitions []int64 + nodes []int64 + segments map[int64]*Segment +} + +func (suite *SegmentDistManagerSuite) SetupSuite() { + const ( + shardNum = 2 + ) + // Replica 0: 0, 2 + // Replica 1: 1 + suite.collection = 10 + suite.partitions = []int64{1, 2} + suite.nodes = []int64{0, 1, 2} + suite.segments = map[int64]*Segment{ + 1: SegmentFromInfo(&datapb.SegmentInfo{ + ID: 1, + CollectionID: suite.collection, + PartitionID: suite.partitions[0], + InsertChannel: "dmc0", + }), + 2: SegmentFromInfo(&datapb.SegmentInfo{ + ID: 2, + CollectionID: suite.collection, + PartitionID: suite.partitions[0], + InsertChannel: "dmc1", + }), + 3: SegmentFromInfo(&datapb.SegmentInfo{ + ID: 3, + CollectionID: suite.collection, + PartitionID: suite.partitions[1], + InsertChannel: "dmc0", + }), + 4: SegmentFromInfo(&datapb.SegmentInfo{ + ID: 4, + CollectionID: suite.collection, + PartitionID: suite.partitions[1], + InsertChannel: "dmc1", + }), + } +} + +func (suite *SegmentDistManagerSuite) SetupTest() { + suite.dist = NewSegmentDistManager() + // Distribution: + // node 0 contains channel segment 1, 2 + // node 1 contains channel segment 1, 2, 3, 4 + // node 2 contains channel segment 3, 4 + suite.dist.Update(suite.nodes[0], suite.segments[1].Clone(), suite.segments[2].Clone()) + suite.dist.Update(suite.nodes[1], + suite.segments[1].Clone(), + suite.segments[2].Clone(), + suite.segments[3].Clone(), + suite.segments[4].Clone()) + suite.dist.Update(suite.nodes[2], suite.segments[3].Clone(), suite.segments[4].Clone()) +} + +func (suite *SegmentDistManagerSuite) TestGetBy() { + dist := suite.dist + // Test GetByNode + for _, node := range suite.nodes { + segments := dist.GetByNode(node) + suite.AssertNode(segments, node) + } + + // Test GetByShard + for _, shard := range []string{"dmc0", "dmc1"} { + segments := dist.GetByShard(shard) + suite.AssertShard(segments, shard) + } + + // Test GetByCollection + segments := dist.GetByCollection(suite.collection) + suite.Len(segments, 8) + suite.AssertCollection(segments, suite.collection) + segments = dist.GetByCollection(-1) + suite.Len(segments, 0) + + // Test GetByNodeAndCollection + // 1. Valid node and valid collection + for _, node := range suite.nodes { + segments := dist.GetByCollectionAndNode(suite.collection, node) + suite.AssertNode(segments, node) + suite.AssertCollection(segments, suite.collection) + } + + // 2. Valid node and invalid collection + segments = dist.GetByCollectionAndNode(-1, suite.nodes[1]) + suite.Len(segments, 0) + + // 3. Invalid node and valid collection + segments = dist.GetByCollectionAndNode(suite.collection, -1) + suite.Len(segments, 0) +} + +func (suite *SegmentDistManagerSuite) AssertIDs(segments []*Segment, ids ...int64) bool { + for _, segment := range segments { + hasSegment := false + for _, id := range ids { + if segment.ID == id { + hasSegment = true + break + } + } + if !suite.True(hasSegment, "segment %v not in the given expected list %+v", segment.GetID(), ids) { + return false + } + } + return true +} + +func (suite *SegmentDistManagerSuite) AssertNode(segments []*Segment, node int64) bool { + for _, segment := range segments { + if !suite.Equal(node, segment.Node) { + return false + } + } + return true +} + +func (suite *SegmentDistManagerSuite) AssertCollection(segments []*Segment, collection int64) bool { + for _, segment := range segments { + if !suite.Equal(collection, segment.GetCollectionID()) { + return false + } + } + return true +} + +func (suite *SegmentDistManagerSuite) AssertShard(segments []*Segment, shard string) bool { + for _, segment := range segments { + if !suite.Equal(shard, segment.GetInsertChannel()) { + return false + } + } + return true +} + +func TestSegmentDistManager(t *testing.T) { + suite.Run(t, new(SegmentDistManagerSuite)) +} diff --git a/internal/querycoordv2/meta/store.go b/internal/querycoordv2/meta/store.go new file mode 100644 index 0000000000000..786e41a77a64c --- /dev/null +++ b/internal/querycoordv2/meta/store.go @@ -0,0 +1,282 @@ +package meta + +import ( + "errors" + "fmt" + + "github.com/golang/protobuf/proto" + "github.com/samber/lo" + clientv3 "go.etcd.io/etcd/client/v3" + + "github.com/milvus-io/milvus/internal/kv" + "github.com/milvus-io/milvus/internal/metastore" + "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/util" +) + +var ( + ErrInvalidKey = errors.New("invalid load info key") +) + +const ( + CollectionLoadInfoPrefix = "querycoord-collection-loadinfo" + PartitionLoadInfoPrefix = "querycoord-partition-loadinfo" + ReplicaPrefix = "querycoord-replica" + CollectionMetaPrefixV1 = "queryCoord-collectionMeta" + ReplicaMetaPrefixV1 = "queryCoord-ReplicaMeta" +) + +type WatchStoreChan = clientv3.WatchChan + +// Store is used to save and get from object storage. +type Store interface { + metastore.QueryCoordCatalog + WatchHandoffEvent(revision int64) WatchStoreChan + LoadHandoffWithRevision() ([]string, []string, int64, error) +} + +type metaStore struct { + cli kv.MetaKv +} + +func NewMetaStore(cli kv.MetaKv) metaStore { + return metaStore{ + cli: cli, + } +} + +func (s metaStore) SaveCollection(info *querypb.CollectionLoadInfo) error { + k := encodeCollectionLoadInfoKey(info.GetCollectionID()) + v, err := proto.Marshal(info) + if err != nil { + return err + } + return s.cli.Save(k, string(v)) +} + +func (s metaStore) SavePartition(info ...*querypb.PartitionLoadInfo) error { + kvs := make(map[string]string) + for _, partition := range info { + key := encodePartitionLoadInfoKey(partition.GetCollectionID(), partition.GetPartitionID()) + value, err := proto.Marshal(partition) + if err != nil { + return err + } + kvs[key] = string(value) + } + return s.cli.MultiSave(kvs) +} + +func (s metaStore) SaveReplica(replica *querypb.Replica) error { + key := encodeReplicaKey(replica.GetCollectionID(), replica.GetID()) + value, err := proto.Marshal(replica) + if err != nil { + return err + } + return s.cli.Save(key, string(value)) +} + +func (s metaStore) GetCollections() ([]*querypb.CollectionLoadInfo, error) { + _, values, err := s.cli.LoadWithPrefix(CollectionLoadInfoPrefix) + if err != nil { + return nil, err + } + ret := make([]*querypb.CollectionLoadInfo, 0, len(values)) + for _, v := range values { + info := querypb.CollectionLoadInfo{} + if err := proto.Unmarshal([]byte(v), &info); err != nil { + return nil, err + } + ret = append(ret, &info) + } + + collectionsV1, err := s.getCollectionsFromV1() + if err != nil { + return nil, err + } + ret = append(ret, collectionsV1...) + + return ret, nil +} + +// getCollectionsFromV1 recovers collections from 2.1 meta store +func (s metaStore) getCollectionsFromV1() ([]*querypb.CollectionLoadInfo, error) { + _, collectionValues, err := s.cli.LoadWithPrefix(CollectionMetaPrefixV1) + if err != nil { + return nil, err + } + ret := make([]*querypb.CollectionLoadInfo, 0, len(collectionValues)) + for _, value := range collectionValues { + collectionInfo := querypb.CollectionInfo{} + err = proto.Unmarshal([]byte(value), &collectionInfo) + if err != nil { + return nil, err + } + if collectionInfo.LoadType != querypb.LoadType_LoadCollection { + continue + } + ret = append(ret, &querypb.CollectionLoadInfo{ + CollectionID: collectionInfo.GetCollectionID(), + ReleasedPartitions: collectionInfo.GetReleasedPartitionIDs(), + ReplicaNumber: collectionInfo.GetReplicaNumber(), + Status: querypb.LoadStatus_Loaded, + }) + } + return ret, nil +} + +func (s metaStore) GetPartitions() (map[int64][]*querypb.PartitionLoadInfo, error) { + _, values, err := s.cli.LoadWithPrefix(PartitionLoadInfoPrefix) + if err != nil { + return nil, err + } + ret := make(map[int64][]*querypb.PartitionLoadInfo) + for _, v := range values { + info := querypb.PartitionLoadInfo{} + if err := proto.Unmarshal([]byte(v), &info); err != nil { + return nil, err + } + ret[info.GetCollectionID()] = append(ret[info.GetCollectionID()], &info) + } + + partitionsV1, err := s.getPartitionsFromV1() + if err != nil { + return nil, err + } + for _, partition := range partitionsV1 { + ret[partition.GetCollectionID()] = append(ret[partition.GetCollectionID()], partition) + } + + return ret, nil +} + +// getCollectionsFromV1 recovers collections from 2.1 meta store +func (s metaStore) getPartitionsFromV1() ([]*querypb.PartitionLoadInfo, error) { + _, collectionValues, err := s.cli.LoadWithPrefix(CollectionMetaPrefixV1) + if err != nil { + return nil, err + } + ret := make([]*querypb.PartitionLoadInfo, 0, len(collectionValues)) + for _, value := range collectionValues { + collectionInfo := querypb.CollectionInfo{} + err = proto.Unmarshal([]byte(value), &collectionInfo) + if err != nil { + return nil, err + } + if collectionInfo.LoadType != querypb.LoadType_LoadPartition { + continue + } + + for _, partition := range collectionInfo.GetPartitionIDs() { + ret = append(ret, &querypb.PartitionLoadInfo{ + CollectionID: collectionInfo.GetCollectionID(), + PartitionID: partition, + ReplicaNumber: collectionInfo.GetReplicaNumber(), + Status: querypb.LoadStatus_Loaded, + }) + } + } + return ret, nil +} + +func (s metaStore) GetReplicas() ([]*querypb.Replica, error) { + _, values, err := s.cli.LoadWithPrefix(ReplicaPrefix) + if err != nil { + return nil, err + } + ret := make([]*querypb.Replica, 0, len(values)) + for _, v := range values { + info := querypb.Replica{} + if err := proto.Unmarshal([]byte(v), &info); err != nil { + return nil, err + } + ret = append(ret, &info) + } + + replicasV1, err := s.getReplicasFromV1() + if err != nil { + return nil, err + } + ret = append(ret, replicasV1...) + + return ret, nil +} + +func (s metaStore) getReplicasFromV1() ([]*querypb.Replica, error) { + _, replicaValues, err := s.cli.LoadWithPrefix(ReplicaMetaPrefixV1) + if err != nil { + return nil, err + } + + ret := make([]*querypb.Replica, 0, len(replicaValues)) + for _, value := range replicaValues { + replicaInfo := milvuspb.ReplicaInfo{} + err = proto.Unmarshal([]byte(value), &replicaInfo) + if err != nil { + return nil, err + } + + ret = append(ret, &querypb.Replica{ + ID: replicaInfo.GetReplicaID(), + CollectionID: replicaInfo.GetCollectionID(), + Nodes: replicaInfo.GetNodeIds(), + }) + } + return ret, nil +} + +func (s metaStore) ReleaseCollection(id int64) error { + k := encodeCollectionLoadInfoKey(id) + return s.cli.Remove(k) +} + +func (s metaStore) ReleasePartition(collection int64, partitions ...int64) error { + keys := lo.Map(partitions, func(partition int64, _ int) string { + return encodePartitionLoadInfoKey(collection, partition) + }) + return s.cli.MultiRemove(keys) +} + +func (s metaStore) ReleaseReplicas(collectionID int64) error { + key := encodeCollectionReplicaKey(collectionID) + return s.cli.RemoveWithPrefix(key) +} + +func (s metaStore) ReleaseReplica(collection, replica int64) error { + key := encodeReplicaKey(collection, replica) + return s.cli.Remove(key) +} + +func (s metaStore) WatchHandoffEvent(revision int64) WatchStoreChan { + return s.cli.WatchWithRevision(util.HandoffSegmentPrefix, revision) +} + +func (s metaStore) RemoveHandoffEvent(info *querypb.SegmentInfo) error { + key := encodeHandoffEventKey(info.CollectionID, info.PartitionID, info.SegmentID) + return s.cli.Remove(key) +} + +func (s metaStore) LoadHandoffWithRevision() ([]string, []string, int64, error) { + return s.cli.LoadWithRevision(util.HandoffSegmentPrefix) +} + +func encodeCollectionLoadInfoKey(collection int64) string { + return fmt.Sprintf("%s/%d", CollectionLoadInfoPrefix, collection) +} + +func encodePartitionLoadInfoKey(collection, partition int64) string { + return fmt.Sprintf("%s/%d/%d", PartitionLoadInfoPrefix, collection, partition) +} + +func encodeReplicaKey(collection, replica int64) string { + return fmt.Sprintf("%s/%d/%d", ReplicaPrefix, collection, replica) +} + +func encodeCollectionReplicaKey(collection int64) string { + return fmt.Sprintf("%s/%d", ReplicaPrefix, collection) +} + +func encodeHandoffEventKey(collection, partition, segment int64) string { + return fmt.Sprintf("%s/%d/%d/%d", util.HandoffSegmentPrefix, collection, partition, segment) +} diff --git a/internal/querycoordv2/meta/store_test.go b/internal/querycoordv2/meta/store_test.go new file mode 100644 index 0000000000000..7a87fa9251d3f --- /dev/null +++ b/internal/querycoordv2/meta/store_test.go @@ -0,0 +1,27 @@ +package meta + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type StoreTestSuite struct { + suite.Suite + store metaStore +} + +func (suite *StoreTestSuite) SetupTest() { + //kv := memkv.NewMemoryKV() + //suite.store = NewMetaStore(kv) +} + +func (suite *StoreTestSuite) TearDownTest() {} + +func (suite *StoreTestSuite) TestLoadRelease() { + // TODO(sunby): add ut +} + +func TestStoreSuite(t *testing.T) { + suite.Run(t, new(StoreTestSuite)) +} diff --git a/internal/querycoordv2/meta/target_manager.go b/internal/querycoordv2/meta/target_manager.go new file mode 100644 index 0000000000000..93b562154adc3 --- /dev/null +++ b/internal/querycoordv2/meta/target_manager.go @@ -0,0 +1,186 @@ +package meta + +import ( + "sync" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/util/funcutil" + "go.uber.org/zap" +) + +type TargetManager struct { + rwmutex sync.RWMutex + + segments map[int64]*datapb.SegmentInfo + dmChannels map[string]*DmChannel +} + +func NewTargetManager() *TargetManager { + return &TargetManager{ + segments: make(map[int64]*datapb.SegmentInfo), + dmChannels: make(map[string]*DmChannel), + } +} + +// RemoveCollection removes all channels and segments in the given collection +func (mgr *TargetManager) RemoveCollection(collectionID int64) { + mgr.rwmutex.Lock() + defer mgr.rwmutex.Unlock() + + log.Info("remove collection from targets") + for _, segment := range mgr.segments { + if segment.CollectionID == collectionID { + mgr.removeSegment(segment.GetID()) + } + } + for _, dmChannel := range mgr.dmChannels { + if dmChannel.CollectionID == collectionID { + mgr.removeDmChannel(dmChannel.GetChannelName()) + } + } +} + +// RemovePartition removes all segment in the given partition, +// NOTE: this doesn't remove any channel even the given one is the only partition +func (mgr *TargetManager) RemovePartition(partitionID int64) { + mgr.rwmutex.Lock() + defer mgr.rwmutex.Unlock() + + log.Info("remove partition from targets", + zap.Int64("partitionID", partitionID)) + for _, segment := range mgr.segments { + if segment.GetPartitionID() == partitionID { + mgr.removeSegment(segment.GetID()) + } + } +} + +func (mgr *TargetManager) RemoveSegment(segmentID int64) { + mgr.rwmutex.Lock() + defer mgr.rwmutex.Unlock() + + delete(mgr.segments, segmentID) +} + +func (mgr *TargetManager) removeSegment(segmentID int64) { + delete(mgr.segments, segmentID) + log.Info("segment removed from targets") +} + +// AddSegment adds segment into target set, +// requires CollectionID, PartitionID, InsertChannel, SegmentID are set +func (mgr *TargetManager) AddSegment(segments ...*datapb.SegmentInfo) { + mgr.rwmutex.Lock() + defer mgr.rwmutex.Unlock() + + mgr.addSegment(segments...) +} + +func (mgr *TargetManager) addSegment(segments ...*datapb.SegmentInfo) { + for _, segment := range segments { + log.Info("add segment into targets", + zap.Int64("segmentID", segment.GetID()), + zap.Int64("collectionID", segment.GetCollectionID()), + ) + mgr.segments[segment.GetID()] = segment + } +} + +func (mgr *TargetManager) ContainSegment(id int64) bool { + mgr.rwmutex.RLock() + defer mgr.rwmutex.RUnlock() + + return mgr.containSegment(id) +} + +func (mgr *TargetManager) containSegment(id int64) bool { + _, ok := mgr.segments[id] + return ok +} + +func (mgr *TargetManager) GetSegmentsByCollection(collection int64, partitions ...int64) []*datapb.SegmentInfo { + mgr.rwmutex.RLock() + defer mgr.rwmutex.RUnlock() + + segments := make([]*datapb.SegmentInfo, 0) + for _, segment := range mgr.segments { + if segment.CollectionID == collection && + (len(partitions) == 0 || funcutil.SliceContain(partitions, segment.PartitionID)) { + segments = append(segments, segment) + } + } + + return segments +} + +func (mgr *TargetManager) HandoffSegment(dest *datapb.SegmentInfo, sources ...int64) { + mgr.rwmutex.Lock() + defer mgr.rwmutex.Unlock() + + // add dest to target + dest.CompactionFrom = sources + mgr.addSegment(dest) +} + +// AddDmChannel adds a channel into target set, +// requires CollectionID, ChannelName are set +func (mgr *TargetManager) AddDmChannel(channels ...*DmChannel) { + mgr.rwmutex.Lock() + defer mgr.rwmutex.Unlock() + + for _, channel := range channels { + log.Info("add channel into targets", + zap.String("channel", channel.GetChannelName())) + mgr.dmChannels[channel.ChannelName] = channel + } +} + +func (mgr *TargetManager) GetDmChannel(channel string) *DmChannel { + mgr.rwmutex.RLock() + defer mgr.rwmutex.RUnlock() + for _, ch := range mgr.dmChannels { + if ch.ChannelName == channel { + return ch + } + } + return nil +} + +func (mgr *TargetManager) ContainDmChannel(channel string) bool { + mgr.rwmutex.RLock() + defer mgr.rwmutex.RUnlock() + + _, ok := mgr.dmChannels[channel] + return ok +} + +func (mgr *TargetManager) removeDmChannel(channel string) { + delete(mgr.dmChannels, channel) + log.Info("remove channel from targets", zap.String("channel", channel)) +} + +func (mgr *TargetManager) GetDmChannelsByCollection(collectionID int64) []*DmChannel { + mgr.rwmutex.RLock() + defer mgr.rwmutex.RUnlock() + + channels := make([]*DmChannel, 0) + for _, channel := range mgr.dmChannels { + if channel.GetCollectionID() == collectionID { + channels = append(channels, channel) + } + } + return channels +} + +func (mgr *TargetManager) GetSegment(id int64) *datapb.SegmentInfo { + mgr.rwmutex.RLock() + defer mgr.rwmutex.RUnlock() + + for _, s := range mgr.segments { + if s.GetID() == id { + return s + } + } + return nil +} diff --git a/internal/querycoordv2/meta/target_manager_test.go b/internal/querycoordv2/meta/target_manager_test.go new file mode 100644 index 0000000000000..c3a413850561e --- /dev/null +++ b/internal/querycoordv2/meta/target_manager_test.go @@ -0,0 +1,163 @@ +package meta + +import ( + "testing" + + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/stretchr/testify/suite" +) + +type TargetManagerSuite struct { + suite.Suite + + // Data + collections []int64 + partitions map[int64][]int64 + channels map[int64][]string + segments map[int64]map[int64][]int64 // CollectionID, PartitionID -> Segments + // Derived data + allChannels []string + allSegments []int64 + + // Test object + mgr *TargetManager +} + +func (suite *TargetManagerSuite) SetupSuite() { + suite.collections = []int64{1000, 1001} + suite.partitions = map[int64][]int64{ + 1000: {100, 101}, + 1001: {102, 103}, + } + suite.channels = map[int64][]string{ + 1000: {"1000-dmc0", "1000-dmc1"}, + 1001: {"1001-dmc0", "1001-dmc1"}, + } + suite.segments = map[int64]map[int64][]int64{ + 1000: { + 100: {1, 2}, + 101: {3, 4}, + }, + 1001: { + 102: {5, 6}, + 103: {7, 8}, + }, + } + + suite.allChannels = make([]string, 0) + suite.allSegments = make([]int64, 0) + for _, channels := range suite.channels { + suite.allChannels = append(suite.allChannels, channels...) + } + for _, partitions := range suite.segments { + for _, segments := range partitions { + suite.allSegments = append(suite.allSegments, segments...) + } + } +} + +func (suite *TargetManagerSuite) SetupTest() { + suite.mgr = NewTargetManager() + for collection, channels := range suite.channels { + for _, channel := range channels { + suite.mgr.AddDmChannel(DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: collection, + ChannelName: channel, + })) + } + } + for collection, partitions := range suite.segments { + for partition, segments := range partitions { + for _, segment := range segments { + suite.mgr.AddSegment(&datapb.SegmentInfo{ + ID: segment, + CollectionID: collection, + PartitionID: partition, + }) + } + } + } +} + +func (suite *TargetManagerSuite) TestGet() { + mgr := suite.mgr + + for collection, channels := range suite.channels { + results := mgr.GetDmChannelsByCollection(collection) + suite.assertChannels(channels, results) + for _, channel := range channels { + suite.True(mgr.ContainDmChannel(channel)) + } + } + + for collection, partitions := range suite.segments { + collectionSegments := make([]int64, 0) + for partition, segments := range partitions { + results := mgr.GetSegmentsByCollection(collection, partition) + suite.assertSegments(segments, results) + for _, segment := range segments { + suite.True(mgr.ContainSegment(segment)) + } + collectionSegments = append(collectionSegments, segments...) + } + results := mgr.GetSegmentsByCollection(collection) + suite.assertSegments(collectionSegments, results) + } +} + +func (suite *TargetManagerSuite) TestRemove() { + mgr := suite.mgr + + for collection, partitions := range suite.segments { + // Remove first segment of each partition + for _, segments := range partitions { + mgr.RemoveSegment(segments[0]) + suite.False(mgr.ContainSegment(segments[0])) + } + + // Remove first partition of each collection + firstPartition := suite.partitions[collection][0] + mgr.RemovePartition(firstPartition) + segments := mgr.GetSegmentsByCollection(collection, firstPartition) + suite.Empty(segments) + } + + // Remove first collection + firstCollection := suite.collections[0] + mgr.RemoveCollection(firstCollection) + channels := mgr.GetDmChannelsByCollection(firstCollection) + suite.Empty(channels) + segments := mgr.GetSegmentsByCollection(firstCollection) + suite.Empty(segments) +} + +func (suite *TargetManagerSuite) assertChannels(expected []string, actual []*DmChannel) bool { + if !suite.Equal(len(expected), len(actual)) { + return false + } + + set := typeutil.NewSet(expected...) + for _, channel := range actual { + set.Remove(channel.ChannelName) + } + + return suite.Len(set, 0) +} + +func (suite *TargetManagerSuite) assertSegments(expected []int64, actual []*datapb.SegmentInfo) bool { + if !suite.Equal(len(expected), len(actual)) { + return false + } + + set := typeutil.NewUniqueSet(expected...) + for _, segment := range actual { + set.Remove(segment.ID) + } + + return suite.Len(set, 0) +} + +func TestTargetManager(t *testing.T) { + suite.Run(t, new(TargetManagerSuite)) +} diff --git a/internal/querycoordv2/mocks/mock_querynode.go b/internal/querycoordv2/mocks/mock_querynode.go new file mode 100644 index 0000000000000..cc39d7afd5565 --- /dev/null +++ b/internal/querycoordv2/mocks/mock_querynode.go @@ -0,0 +1,891 @@ +// Code generated by mockery v2.14.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + commonpb "github.com/milvus-io/milvus/internal/proto/commonpb" + + internalpb "github.com/milvus-io/milvus/internal/proto/internalpb" + + milvuspb "github.com/milvus-io/milvus/internal/proto/milvuspb" + + mock "github.com/stretchr/testify/mock" + + querypb "github.com/milvus-io/milvus/internal/proto/querypb" +) + +// MockQueryNodeServer is an autogenerated mock type for the QueryNodeServer type +type MockQueryNodeServer struct { + mock.Mock +} + +type MockQueryNodeServer_Expecter struct { + mock *mock.Mock +} + +func (_m *MockQueryNodeServer) EXPECT() *MockQueryNodeServer_Expecter { + return &MockQueryNodeServer_Expecter{mock: &_m.Mock} +} + +// GetComponentStates provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) GetComponentStates(_a0 context.Context, _a1 *internalpb.GetComponentStatesRequest) (*internalpb.ComponentStates, error) { + ret := _m.Called(_a0, _a1) + + var r0 *internalpb.ComponentStates + if rf, ok := ret.Get(0).(func(context.Context, *internalpb.GetComponentStatesRequest) *internalpb.ComponentStates); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*internalpb.ComponentStates) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *internalpb.GetComponentStatesRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_GetComponentStates_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetComponentStates' +type MockQueryNodeServer_GetComponentStates_Call struct { + *mock.Call +} + +// GetComponentStates is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *internalpb.GetComponentStatesRequest +func (_e *MockQueryNodeServer_Expecter) GetComponentStates(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_GetComponentStates_Call { + return &MockQueryNodeServer_GetComponentStates_Call{Call: _e.mock.On("GetComponentStates", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_GetComponentStates_Call) Run(run func(_a0 context.Context, _a1 *internalpb.GetComponentStatesRequest)) *MockQueryNodeServer_GetComponentStates_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*internalpb.GetComponentStatesRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_GetComponentStates_Call) Return(_a0 *internalpb.ComponentStates, _a1 error) *MockQueryNodeServer_GetComponentStates_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// GetDataDistribution provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) GetDataDistribution(_a0 context.Context, _a1 *querypb.GetDataDistributionRequest) (*querypb.GetDataDistributionResponse, error) { + ret := _m.Called(_a0, _a1) + + var r0 *querypb.GetDataDistributionResponse + if rf, ok := ret.Get(0).(func(context.Context, *querypb.GetDataDistributionRequest) *querypb.GetDataDistributionResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*querypb.GetDataDistributionResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *querypb.GetDataDistributionRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_GetDataDistribution_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetDataDistribution' +type MockQueryNodeServer_GetDataDistribution_Call struct { + *mock.Call +} + +// GetDataDistribution is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *querypb.GetDataDistributionRequest +func (_e *MockQueryNodeServer_Expecter) GetDataDistribution(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_GetDataDistribution_Call { + return &MockQueryNodeServer_GetDataDistribution_Call{Call: _e.mock.On("GetDataDistribution", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_GetDataDistribution_Call) Run(run func(_a0 context.Context, _a1 *querypb.GetDataDistributionRequest)) *MockQueryNodeServer_GetDataDistribution_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*querypb.GetDataDistributionRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_GetDataDistribution_Call) Return(_a0 *querypb.GetDataDistributionResponse, _a1 error) *MockQueryNodeServer_GetDataDistribution_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// GetMetrics provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) GetMetrics(_a0 context.Context, _a1 *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) { + ret := _m.Called(_a0, _a1) + + var r0 *milvuspb.GetMetricsResponse + if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.GetMetricsRequest) *milvuspb.GetMetricsResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*milvuspb.GetMetricsResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.GetMetricsRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_GetMetrics_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetMetrics' +type MockQueryNodeServer_GetMetrics_Call struct { + *mock.Call +} + +// GetMetrics is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *milvuspb.GetMetricsRequest +func (_e *MockQueryNodeServer_Expecter) GetMetrics(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_GetMetrics_Call { + return &MockQueryNodeServer_GetMetrics_Call{Call: _e.mock.On("GetMetrics", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_GetMetrics_Call) Run(run func(_a0 context.Context, _a1 *milvuspb.GetMetricsRequest)) *MockQueryNodeServer_GetMetrics_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*milvuspb.GetMetricsRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_GetMetrics_Call) Return(_a0 *milvuspb.GetMetricsResponse, _a1 error) *MockQueryNodeServer_GetMetrics_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// GetSegmentInfo provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) GetSegmentInfo(_a0 context.Context, _a1 *querypb.GetSegmentInfoRequest) (*querypb.GetSegmentInfoResponse, error) { + ret := _m.Called(_a0, _a1) + + var r0 *querypb.GetSegmentInfoResponse + if rf, ok := ret.Get(0).(func(context.Context, *querypb.GetSegmentInfoRequest) *querypb.GetSegmentInfoResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*querypb.GetSegmentInfoResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *querypb.GetSegmentInfoRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_GetSegmentInfo_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSegmentInfo' +type MockQueryNodeServer_GetSegmentInfo_Call struct { + *mock.Call +} + +// GetSegmentInfo is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *querypb.GetSegmentInfoRequest +func (_e *MockQueryNodeServer_Expecter) GetSegmentInfo(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_GetSegmentInfo_Call { + return &MockQueryNodeServer_GetSegmentInfo_Call{Call: _e.mock.On("GetSegmentInfo", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_GetSegmentInfo_Call) Run(run func(_a0 context.Context, _a1 *querypb.GetSegmentInfoRequest)) *MockQueryNodeServer_GetSegmentInfo_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*querypb.GetSegmentInfoRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_GetSegmentInfo_Call) Return(_a0 *querypb.GetSegmentInfoResponse, _a1 error) *MockQueryNodeServer_GetSegmentInfo_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// GetStatistics provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) GetStatistics(_a0 context.Context, _a1 *querypb.GetStatisticsRequest) (*internalpb.GetStatisticsResponse, error) { + ret := _m.Called(_a0, _a1) + + var r0 *internalpb.GetStatisticsResponse + if rf, ok := ret.Get(0).(func(context.Context, *querypb.GetStatisticsRequest) *internalpb.GetStatisticsResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*internalpb.GetStatisticsResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *querypb.GetStatisticsRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_GetStatistics_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetStatistics' +type MockQueryNodeServer_GetStatistics_Call struct { + *mock.Call +} + +// GetStatistics is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *querypb.GetStatisticsRequest +func (_e *MockQueryNodeServer_Expecter) GetStatistics(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_GetStatistics_Call { + return &MockQueryNodeServer_GetStatistics_Call{Call: _e.mock.On("GetStatistics", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_GetStatistics_Call) Run(run func(_a0 context.Context, _a1 *querypb.GetStatisticsRequest)) *MockQueryNodeServer_GetStatistics_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*querypb.GetStatisticsRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_GetStatistics_Call) Return(_a0 *internalpb.GetStatisticsResponse, _a1 error) *MockQueryNodeServer_GetStatistics_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// GetStatisticsChannel provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) GetStatisticsChannel(_a0 context.Context, _a1 *internalpb.GetStatisticsChannelRequest) (*milvuspb.StringResponse, error) { + ret := _m.Called(_a0, _a1) + + var r0 *milvuspb.StringResponse + if rf, ok := ret.Get(0).(func(context.Context, *internalpb.GetStatisticsChannelRequest) *milvuspb.StringResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*milvuspb.StringResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *internalpb.GetStatisticsChannelRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_GetStatisticsChannel_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetStatisticsChannel' +type MockQueryNodeServer_GetStatisticsChannel_Call struct { + *mock.Call +} + +// GetStatisticsChannel is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *internalpb.GetStatisticsChannelRequest +func (_e *MockQueryNodeServer_Expecter) GetStatisticsChannel(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_GetStatisticsChannel_Call { + return &MockQueryNodeServer_GetStatisticsChannel_Call{Call: _e.mock.On("GetStatisticsChannel", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_GetStatisticsChannel_Call) Run(run func(_a0 context.Context, _a1 *internalpb.GetStatisticsChannelRequest)) *MockQueryNodeServer_GetStatisticsChannel_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*internalpb.GetStatisticsChannelRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_GetStatisticsChannel_Call) Return(_a0 *milvuspb.StringResponse, _a1 error) *MockQueryNodeServer_GetStatisticsChannel_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// GetTimeTickChannel provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) GetTimeTickChannel(_a0 context.Context, _a1 *internalpb.GetTimeTickChannelRequest) (*milvuspb.StringResponse, error) { + ret := _m.Called(_a0, _a1) + + var r0 *milvuspb.StringResponse + if rf, ok := ret.Get(0).(func(context.Context, *internalpb.GetTimeTickChannelRequest) *milvuspb.StringResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*milvuspb.StringResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *internalpb.GetTimeTickChannelRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_GetTimeTickChannel_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetTimeTickChannel' +type MockQueryNodeServer_GetTimeTickChannel_Call struct { + *mock.Call +} + +// GetTimeTickChannel is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *internalpb.GetTimeTickChannelRequest +func (_e *MockQueryNodeServer_Expecter) GetTimeTickChannel(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_GetTimeTickChannel_Call { + return &MockQueryNodeServer_GetTimeTickChannel_Call{Call: _e.mock.On("GetTimeTickChannel", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_GetTimeTickChannel_Call) Run(run func(_a0 context.Context, _a1 *internalpb.GetTimeTickChannelRequest)) *MockQueryNodeServer_GetTimeTickChannel_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*internalpb.GetTimeTickChannelRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_GetTimeTickChannel_Call) Return(_a0 *milvuspb.StringResponse, _a1 error) *MockQueryNodeServer_GetTimeTickChannel_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// LoadSegments provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) LoadSegments(_a0 context.Context, _a1 *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { + ret := _m.Called(_a0, _a1) + + var r0 *commonpb.Status + if rf, ok := ret.Get(0).(func(context.Context, *querypb.LoadSegmentsRequest) *commonpb.Status); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*commonpb.Status) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *querypb.LoadSegmentsRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_LoadSegments_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LoadSegments' +type MockQueryNodeServer_LoadSegments_Call struct { + *mock.Call +} + +// LoadSegments is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *querypb.LoadSegmentsRequest +func (_e *MockQueryNodeServer_Expecter) LoadSegments(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_LoadSegments_Call { + return &MockQueryNodeServer_LoadSegments_Call{Call: _e.mock.On("LoadSegments", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_LoadSegments_Call) Run(run func(_a0 context.Context, _a1 *querypb.LoadSegmentsRequest)) *MockQueryNodeServer_LoadSegments_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*querypb.LoadSegmentsRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_LoadSegments_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryNodeServer_LoadSegments_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// Query provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) Query(_a0 context.Context, _a1 *querypb.QueryRequest) (*internalpb.RetrieveResults, error) { + ret := _m.Called(_a0, _a1) + + var r0 *internalpb.RetrieveResults + if rf, ok := ret.Get(0).(func(context.Context, *querypb.QueryRequest) *internalpb.RetrieveResults); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*internalpb.RetrieveResults) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *querypb.QueryRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_Query_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Query' +type MockQueryNodeServer_Query_Call struct { + *mock.Call +} + +// Query is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *querypb.QueryRequest +func (_e *MockQueryNodeServer_Expecter) Query(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_Query_Call { + return &MockQueryNodeServer_Query_Call{Call: _e.mock.On("Query", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_Query_Call) Run(run func(_a0 context.Context, _a1 *querypb.QueryRequest)) *MockQueryNodeServer_Query_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*querypb.QueryRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_Query_Call) Return(_a0 *internalpb.RetrieveResults, _a1 error) *MockQueryNodeServer_Query_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// ReleaseCollection provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) ReleaseCollection(_a0 context.Context, _a1 *querypb.ReleaseCollectionRequest) (*commonpb.Status, error) { + ret := _m.Called(_a0, _a1) + + var r0 *commonpb.Status + if rf, ok := ret.Get(0).(func(context.Context, *querypb.ReleaseCollectionRequest) *commonpb.Status); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*commonpb.Status) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *querypb.ReleaseCollectionRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_ReleaseCollection_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReleaseCollection' +type MockQueryNodeServer_ReleaseCollection_Call struct { + *mock.Call +} + +// ReleaseCollection is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *querypb.ReleaseCollectionRequest +func (_e *MockQueryNodeServer_Expecter) ReleaseCollection(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_ReleaseCollection_Call { + return &MockQueryNodeServer_ReleaseCollection_Call{Call: _e.mock.On("ReleaseCollection", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_ReleaseCollection_Call) Run(run func(_a0 context.Context, _a1 *querypb.ReleaseCollectionRequest)) *MockQueryNodeServer_ReleaseCollection_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*querypb.ReleaseCollectionRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_ReleaseCollection_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryNodeServer_ReleaseCollection_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// ReleasePartitions provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) ReleasePartitions(_a0 context.Context, _a1 *querypb.ReleasePartitionsRequest) (*commonpb.Status, error) { + ret := _m.Called(_a0, _a1) + + var r0 *commonpb.Status + if rf, ok := ret.Get(0).(func(context.Context, *querypb.ReleasePartitionsRequest) *commonpb.Status); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*commonpb.Status) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *querypb.ReleasePartitionsRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_ReleasePartitions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReleasePartitions' +type MockQueryNodeServer_ReleasePartitions_Call struct { + *mock.Call +} + +// ReleasePartitions is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *querypb.ReleasePartitionsRequest +func (_e *MockQueryNodeServer_Expecter) ReleasePartitions(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_ReleasePartitions_Call { + return &MockQueryNodeServer_ReleasePartitions_Call{Call: _e.mock.On("ReleasePartitions", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_ReleasePartitions_Call) Run(run func(_a0 context.Context, _a1 *querypb.ReleasePartitionsRequest)) *MockQueryNodeServer_ReleasePartitions_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*querypb.ReleasePartitionsRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_ReleasePartitions_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryNodeServer_ReleasePartitions_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// ReleaseSegments provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) ReleaseSegments(_a0 context.Context, _a1 *querypb.ReleaseSegmentsRequest) (*commonpb.Status, error) { + ret := _m.Called(_a0, _a1) + + var r0 *commonpb.Status + if rf, ok := ret.Get(0).(func(context.Context, *querypb.ReleaseSegmentsRequest) *commonpb.Status); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*commonpb.Status) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *querypb.ReleaseSegmentsRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_ReleaseSegments_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReleaseSegments' +type MockQueryNodeServer_ReleaseSegments_Call struct { + *mock.Call +} + +// ReleaseSegments is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *querypb.ReleaseSegmentsRequest +func (_e *MockQueryNodeServer_Expecter) ReleaseSegments(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_ReleaseSegments_Call { + return &MockQueryNodeServer_ReleaseSegments_Call{Call: _e.mock.On("ReleaseSegments", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_ReleaseSegments_Call) Run(run func(_a0 context.Context, _a1 *querypb.ReleaseSegmentsRequest)) *MockQueryNodeServer_ReleaseSegments_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*querypb.ReleaseSegmentsRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_ReleaseSegments_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryNodeServer_ReleaseSegments_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// Search provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) Search(_a0 context.Context, _a1 *querypb.SearchRequest) (*internalpb.SearchResults, error) { + ret := _m.Called(_a0, _a1) + + var r0 *internalpb.SearchResults + if rf, ok := ret.Get(0).(func(context.Context, *querypb.SearchRequest) *internalpb.SearchResults); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*internalpb.SearchResults) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *querypb.SearchRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_Search_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Search' +type MockQueryNodeServer_Search_Call struct { + *mock.Call +} + +// Search is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *querypb.SearchRequest +func (_e *MockQueryNodeServer_Expecter) Search(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_Search_Call { + return &MockQueryNodeServer_Search_Call{Call: _e.mock.On("Search", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_Search_Call) Run(run func(_a0 context.Context, _a1 *querypb.SearchRequest)) *MockQueryNodeServer_Search_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*querypb.SearchRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_Search_Call) Return(_a0 *internalpb.SearchResults, _a1 error) *MockQueryNodeServer_Search_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// ShowConfigurations provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) ShowConfigurations(_a0 context.Context, _a1 *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) { + ret := _m.Called(_a0, _a1) + + var r0 *internalpb.ShowConfigurationsResponse + if rf, ok := ret.Get(0).(func(context.Context, *internalpb.ShowConfigurationsRequest) *internalpb.ShowConfigurationsResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*internalpb.ShowConfigurationsResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *internalpb.ShowConfigurationsRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_ShowConfigurations_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ShowConfigurations' +type MockQueryNodeServer_ShowConfigurations_Call struct { + *mock.Call +} + +// ShowConfigurations is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *internalpb.ShowConfigurationsRequest +func (_e *MockQueryNodeServer_Expecter) ShowConfigurations(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_ShowConfigurations_Call { + return &MockQueryNodeServer_ShowConfigurations_Call{Call: _e.mock.On("ShowConfigurations", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_ShowConfigurations_Call) Run(run func(_a0 context.Context, _a1 *internalpb.ShowConfigurationsRequest)) *MockQueryNodeServer_ShowConfigurations_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*internalpb.ShowConfigurationsRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_ShowConfigurations_Call) Return(_a0 *internalpb.ShowConfigurationsResponse, _a1 error) *MockQueryNodeServer_ShowConfigurations_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// SyncDistribution provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) SyncDistribution(_a0 context.Context, _a1 *querypb.SyncDistributionRequest) (*commonpb.Status, error) { + ret := _m.Called(_a0, _a1) + + var r0 *commonpb.Status + if rf, ok := ret.Get(0).(func(context.Context, *querypb.SyncDistributionRequest) *commonpb.Status); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*commonpb.Status) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *querypb.SyncDistributionRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_SyncDistribution_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SyncDistribution' +type MockQueryNodeServer_SyncDistribution_Call struct { + *mock.Call +} + +// SyncDistribution is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *querypb.SyncDistributionRequest +func (_e *MockQueryNodeServer_Expecter) SyncDistribution(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_SyncDistribution_Call { + return &MockQueryNodeServer_SyncDistribution_Call{Call: _e.mock.On("SyncDistribution", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_SyncDistribution_Call) Run(run func(_a0 context.Context, _a1 *querypb.SyncDistributionRequest)) *MockQueryNodeServer_SyncDistribution_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*querypb.SyncDistributionRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_SyncDistribution_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryNodeServer_SyncDistribution_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// SyncReplicaSegments provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) SyncReplicaSegments(_a0 context.Context, _a1 *querypb.SyncReplicaSegmentsRequest) (*commonpb.Status, error) { + ret := _m.Called(_a0, _a1) + + var r0 *commonpb.Status + if rf, ok := ret.Get(0).(func(context.Context, *querypb.SyncReplicaSegmentsRequest) *commonpb.Status); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*commonpb.Status) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *querypb.SyncReplicaSegmentsRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_SyncReplicaSegments_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SyncReplicaSegments' +type MockQueryNodeServer_SyncReplicaSegments_Call struct { + *mock.Call +} + +// SyncReplicaSegments is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *querypb.SyncReplicaSegmentsRequest +func (_e *MockQueryNodeServer_Expecter) SyncReplicaSegments(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_SyncReplicaSegments_Call { + return &MockQueryNodeServer_SyncReplicaSegments_Call{Call: _e.mock.On("SyncReplicaSegments", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_SyncReplicaSegments_Call) Run(run func(_a0 context.Context, _a1 *querypb.SyncReplicaSegmentsRequest)) *MockQueryNodeServer_SyncReplicaSegments_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*querypb.SyncReplicaSegmentsRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_SyncReplicaSegments_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryNodeServer_SyncReplicaSegments_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// UnsubDmChannel provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) UnsubDmChannel(_a0 context.Context, _a1 *querypb.UnsubDmChannelRequest) (*commonpb.Status, error) { + ret := _m.Called(_a0, _a1) + + var r0 *commonpb.Status + if rf, ok := ret.Get(0).(func(context.Context, *querypb.UnsubDmChannelRequest) *commonpb.Status); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*commonpb.Status) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *querypb.UnsubDmChannelRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_UnsubDmChannel_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UnsubDmChannel' +type MockQueryNodeServer_UnsubDmChannel_Call struct { + *mock.Call +} + +// UnsubDmChannel is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *querypb.UnsubDmChannelRequest +func (_e *MockQueryNodeServer_Expecter) UnsubDmChannel(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_UnsubDmChannel_Call { + return &MockQueryNodeServer_UnsubDmChannel_Call{Call: _e.mock.On("UnsubDmChannel", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_UnsubDmChannel_Call) Run(run func(_a0 context.Context, _a1 *querypb.UnsubDmChannelRequest)) *MockQueryNodeServer_UnsubDmChannel_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*querypb.UnsubDmChannelRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_UnsubDmChannel_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryNodeServer_UnsubDmChannel_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// WatchDmChannels provides a mock function with given fields: _a0, _a1 +func (_m *MockQueryNodeServer) WatchDmChannels(_a0 context.Context, _a1 *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) { + ret := _m.Called(_a0, _a1) + + var r0 *commonpb.Status + if rf, ok := ret.Get(0).(func(context.Context, *querypb.WatchDmChannelsRequest) *commonpb.Status); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*commonpb.Status) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *querypb.WatchDmChannelsRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockQueryNodeServer_WatchDmChannels_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'WatchDmChannels' +type MockQueryNodeServer_WatchDmChannels_Call struct { + *mock.Call +} + +// WatchDmChannels is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *querypb.WatchDmChannelsRequest +func (_e *MockQueryNodeServer_Expecter) WatchDmChannels(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_WatchDmChannels_Call { + return &MockQueryNodeServer_WatchDmChannels_Call{Call: _e.mock.On("WatchDmChannels", _a0, _a1)} +} + +func (_c *MockQueryNodeServer_WatchDmChannels_Call) Run(run func(_a0 context.Context, _a1 *querypb.WatchDmChannelsRequest)) *MockQueryNodeServer_WatchDmChannels_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*querypb.WatchDmChannelsRequest)) + }) + return _c +} + +func (_c *MockQueryNodeServer_WatchDmChannels_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryNodeServer_WatchDmChannels_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +type mockConstructorTestingTNewMockQueryNodeServer interface { + mock.TestingT + Cleanup(func()) +} + +// NewMockQueryNodeServer creates a new instance of MockQueryNodeServer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMockQueryNodeServer(t mockConstructorTestingTNewMockQueryNodeServer) *MockQueryNodeServer { + mock := &MockQueryNodeServer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/querycoordv2/mocks/querynode.go b/internal/querycoordv2/mocks/querynode.go new file mode 100644 index 0000000000000..f8f198a887fc3 --- /dev/null +++ b/internal/querycoordv2/mocks/querynode.go @@ -0,0 +1,150 @@ +package mocks + +import ( + "context" + "net" + "sync" + "testing" + "time" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/commonpb" + querypb "github.com/milvus-io/milvus/internal/proto/querypb" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/util/sessionutil" + "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/stretchr/testify/mock" + clientv3 "go.etcd.io/etcd/client/v3" + "go.uber.org/zap" + "google.golang.org/grpc" +) + +type MockQueryNode struct { + *MockQueryNodeServer + + ID int64 + addr string + ctx context.Context + cancel context.CancelFunc + session *sessionutil.Session + server *grpc.Server + + rwmutex sync.RWMutex + channels map[int64][]string + channelVersion map[string]int64 + segments map[int64]map[string][]int64 + segmentVersion map[int64]int64 +} + +func NewMockQueryNode(t *testing.T, etcdCli *clientv3.Client) *MockQueryNode { + ctx, cancel := context.WithCancel(context.Background()) + node := &MockQueryNode{ + MockQueryNodeServer: NewMockQueryNodeServer(t), + ctx: ctx, + cancel: cancel, + session: sessionutil.NewSession(ctx, Params.EtcdCfg.MetaRootPath, etcdCli), + channels: make(map[int64][]string), + segments: make(map[int64]map[string][]int64), + } + + return node +} + +func (node *MockQueryNode) Start() error { + // Start gRPC server + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + return err + } + node.addr = lis.Addr().String() + node.server = grpc.NewServer() + querypb.RegisterQueryNodeServer(node.server, node) + go func() { + err = node.server.Serve(lis) + }() + + // Regiser + node.session.Init(typeutil.QueryNodeRole, node.addr, false, true) + node.ID = node.session.ServerID + node.session.Register() + log.Debug("mock QueryNode started", + zap.Int64("nodeID", node.ID), + zap.String("nodeAddr", node.addr)) + + successStatus := &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + } + node.EXPECT().GetDataDistribution(mock.Anything, mock.Anything).Return(&querypb.GetDataDistributionResponse{ + Status: successStatus, + NodeID: node.ID, + Channels: node.getAllChannels(), + Segments: node.getAllSegments(), + }, nil).Maybe() + + node.EXPECT().WatchDmChannels(mock.Anything, mock.Anything).Run(func(ctx context.Context, req *querypb.WatchDmChannelsRequest) { + node.rwmutex.Lock() + defer node.rwmutex.Unlock() + + node.channels[req.GetCollectionID()] = append(node.channels[req.GetCollectionID()], + req.GetInfos()[0].GetChannelName()) + }).Return(successStatus, nil).Maybe() + node.EXPECT().LoadSegments(mock.Anything, mock.Anything).Run(func(ctx context.Context, req *querypb.LoadSegmentsRequest) { + node.rwmutex.Lock() + defer node.rwmutex.Unlock() + + shardSegments, ok := node.segments[req.GetCollectionID()] + if !ok { + shardSegments = make(map[string][]int64) + node.segments[req.GetCollectionID()] = shardSegments + } + segment := req.GetInfos()[0] + shardSegments[segment.GetInsertChannel()] = append(shardSegments[segment.GetInsertChannel()], + segment.GetSegmentID()) + node.segmentVersion[segment.GetSegmentID()] = req.GetVersion() + }).Return(successStatus, nil).Maybe() + + return err +} + +func (node *MockQueryNode) Stop() { + node.cancel() + node.server.GracefulStop() + node.session.Revoke(time.Second) +} + +func (node *MockQueryNode) getAllChannels() []*querypb.ChannelVersionInfo { + node.rwmutex.RLock() + defer node.rwmutex.RUnlock() + + ret := make([]*querypb.ChannelVersionInfo, 0) + for collection, channels := range node.channels { + for _, channel := range channels { + ret = append(ret, &querypb.ChannelVersionInfo{ + Channel: channel, + Collection: collection, + Version: node.channelVersion[channel], + }) + } + } + return ret +} + +func (node *MockQueryNode) getAllSegments() []*querypb.SegmentVersionInfo { + node.rwmutex.RLock() + defer node.rwmutex.RUnlock() + + ret := make([]*querypb.SegmentVersionInfo, 0) + for collection, shardSegments := range node.segments { + for shard, segments := range shardSegments { + for _, segment := range segments { + ret = append(ret, &querypb.SegmentVersionInfo{ + ID: segment, + Collection: collection, + Channel: shard, + Version: node.segmentVersion[segment], + }) + } + } + } + return ret +} diff --git a/internal/querycoordv2/observers/collection_observer.go b/internal/querycoordv2/observers/collection_observer.go new file mode 100644 index 0000000000000..d7555b0623bc1 --- /dev/null +++ b/internal/querycoordv2/observers/collection_observer.go @@ -0,0 +1,247 @@ +package observers + +import ( + "context" + "time" + + "go.uber.org/zap" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/metrics" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" +) + +type CollectionObserver struct { + stopCh chan struct{} + + dist *meta.DistributionManager + meta *meta.Meta + targetMgr *meta.TargetManager +} + +func NewCollectionObserver( + dist *meta.DistributionManager, + meta *meta.Meta, + targetMgr *meta.TargetManager, +) *CollectionObserver { + return &CollectionObserver{ + stopCh: make(chan struct{}), + dist: dist, + meta: meta, + targetMgr: targetMgr, + } +} + +func (ob *CollectionObserver) Start(ctx context.Context) { + const observePeriod = time.Second + go func() { + ticker := time.NewTicker(observePeriod) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + log.Info("CollectionObserver stopped due to context canceled") + return + + case <-ob.stopCh: + log.Info("CollectionObserver stopped") + return + + case <-ticker.C: + ob.Observe() + } + } + }() +} + +func (ob *CollectionObserver) Stop() { + close(ob.stopCh) +} + +func (ob *CollectionObserver) Observe() { + ob.observeTimeout() + ob.observeLoadStatus() +} + +func (ob *CollectionObserver) observeTimeout() { + collections := ob.meta.CollectionManager.GetAllCollections() + for _, collection := range collections { + if collection.GetStatus() != querypb.LoadStatus_Loading || + time.Now().Before(collection.CreatedAt.Add(Params.QueryCoordCfg.LoadTimeoutSeconds)) { + continue + } + + log.Info("load collection timeout, cancel it", + zap.Int64("collectionID", collection.GetCollectionID()), + zap.Duration("loadTime", time.Since(collection.CreatedAt))) + ob.meta.CollectionManager.RemoveCollection(collection.GetCollectionID()) + ob.meta.ReplicaManager.RemoveCollection(collection.GetCollectionID()) + ob.targetMgr.RemoveCollection(collection.GetCollectionID()) + } + + partitions := utils.GroupPartitionsByCollection( + ob.meta.CollectionManager.GetAllPartitions()) + if len(partitions) > 0 { + log.Info("observes partitions timeout", zap.Int("partitionNum", len(partitions))) + } + for collection, partitions := range partitions { + log := log.With( + zap.Int64("collectionID", collection), + ) + for _, partition := range partitions { + if partition.GetStatus() != querypb.LoadStatus_Loading || + time.Now().Before(partition.CreatedAt.Add(Params.QueryCoordCfg.LoadTimeoutSeconds)) { + continue + } + + log.Info("load partition timeout, cancel all partitions", + zap.Int64("partitionID", partition.GetPartitionID()), + zap.Duration("loadTime", time.Since(partition.CreatedAt))) + // TODO(yah01): Now, releasing part of partitions is not allowed + ob.meta.CollectionManager.RemoveCollection(partition.GetCollectionID()) + ob.meta.ReplicaManager.RemoveCollection(partition.GetCollectionID()) + ob.targetMgr.RemoveCollection(partition.GetCollectionID()) + break + } + } +} + +func (ob *CollectionObserver) observeLoadStatus() { + collections := ob.meta.CollectionManager.GetAllCollections() + for _, collection := range collections { + if collection.LoadPercentage == 100 { + continue + } + ob.observeCollectionLoadStatus(collection) + } + + partitions := ob.meta.CollectionManager.GetAllPartitions() + if len(partitions) > 0 { + log.Info("observe partitions status", zap.Int("partitionNum", len(partitions))) + } + for _, partition := range partitions { + if partition.LoadPercentage == 100 { + continue + } + ob.observePartitionLoadStatus(partition) + } +} + +func (ob *CollectionObserver) observeCollectionLoadStatus(collection *meta.Collection) { + log := log.With(zap.Int64("collectionID", collection.GetCollectionID())) + + segmentTargets := ob.targetMgr.GetSegmentsByCollection(collection.GetCollectionID()) + channelTargets := ob.targetMgr.GetDmChannelsByCollection(collection.GetCollectionID()) + targetNum := len(segmentTargets) + len(channelTargets) + log.Info("collection targets", + zap.Int("segment-target-num", len(segmentTargets)), + zap.Int("channel-target-num", len(channelTargets)), + zap.Int("total-target-num", targetNum)) + if targetNum == 0 { + log.Info("collection released, skip it") + return + } + + loadedCount := 0 + for _, channel := range channelTargets { + group := utils.GroupNodesByReplica(ob.meta.ReplicaManager, + collection.GetCollectionID(), + ob.dist.LeaderViewManager.GetChannelDist(channel.GetChannelName())) + if len(group) >= int(collection.GetReplicaNumber()) { + loadedCount++ + } + } + subChannelCount := loadedCount + for _, segment := range segmentTargets { + group := utils.GroupNodesByReplica(ob.meta.ReplicaManager, + collection.GetCollectionID(), + ob.dist.LeaderViewManager.GetSealedSegmentDist(segment.GetID())) + if len(group) >= int(collection.GetReplicaNumber()) { + loadedCount++ + } + } + if loadedCount > 0 { + log.Info("collection load progress", + zap.Int("sub-channel-count", subChannelCount), + zap.Int("load-segment-count", loadedCount-subChannelCount), + ) + } + + updated := collection.Clone() + updated.LoadPercentage = int32(loadedCount * 100 / targetNum) + if loadedCount >= len(segmentTargets)+len(channelTargets) { + updated.Status = querypb.LoadStatus_Loaded + ob.meta.CollectionManager.UpdateCollection(updated) + + elapsed := time.Since(updated.CreatedAt) + metrics.QueryCoordLoadLatency.WithLabelValues().Observe(float64(elapsed.Milliseconds())) + } else { + ob.meta.CollectionManager.UpdateCollectionInMemory(updated) + } + if updated.LoadPercentage != collection.LoadPercentage { + log.Info("collection load status updated", + zap.Int32("loadPercentage", updated.LoadPercentage), + zap.Int32("collectionStatus", int32(updated.GetStatus()))) + } +} + +func (ob *CollectionObserver) observePartitionLoadStatus(partition *meta.Partition) { + log := log.With( + zap.Int64("collectionID", partition.GetCollectionID()), + zap.Int64("partitionID", partition.GetPartitionID()), + ) + + segmentTargets := ob.targetMgr.GetSegmentsByCollection(partition.GetCollectionID(), partition.GetPartitionID()) + channelTargets := ob.targetMgr.GetDmChannelsByCollection(partition.GetCollectionID()) + targetNum := len(segmentTargets) + len(channelTargets) + log.Info("partition targets", + zap.Int("segment-target-num", len(segmentTargets)), + zap.Int("channel-target-num", len(channelTargets)), + zap.Int("total-target-num", targetNum)) + if targetNum == 0 { + log.Info("partition released, skip it") + return + } + + loadedCount := 0 + for _, channel := range channelTargets { + group := utils.GroupNodesByReplica(ob.meta.ReplicaManager, + partition.GetCollectionID(), + ob.dist.LeaderViewManager.GetChannelDist(channel.GetChannelName())) + if len(group) >= int(partition.GetReplicaNumber()) { + loadedCount++ + } + } + subChannelCount := loadedCount + for _, segment := range segmentTargets { + group := utils.GroupNodesByReplica(ob.meta.ReplicaManager, + partition.GetCollectionID(), + ob.dist.LeaderViewManager.GetSealedSegmentDist(segment.GetID())) + if len(group) >= int(partition.GetReplicaNumber()) { + loadedCount++ + } + } + if loadedCount > 0 { + log.Info("partition load progress", + zap.Int("sub-channel-count", subChannelCount), + zap.Int("load-segment-count", loadedCount-subChannelCount)) + } + + partition = partition.Clone() + partition.LoadPercentage = int32(loadedCount * 100 / targetNum) + if loadedCount >= len(segmentTargets)+len(channelTargets) { + partition.Status = querypb.LoadStatus_Loaded + ob.meta.CollectionManager.PutPartition(partition) + + elapsed := time.Since(partition.CreatedAt) + metrics.QueryCoordLoadLatency.WithLabelValues().Observe(float64(elapsed.Milliseconds())) + } else { + ob.meta.CollectionManager.UpdatePartitionInMemory(partition) + } + log.Info("partition load status updated", + zap.Int32("loadPercentage", partition.LoadPercentage), + zap.Int32("partitionStatus", int32(partition.GetStatus()))) +} diff --git a/internal/querycoordv2/observers/collection_observer_test.go b/internal/querycoordv2/observers/collection_observer_test.go new file mode 100644 index 0000000000000..afb76a2af2867 --- /dev/null +++ b/internal/querycoordv2/observers/collection_observer_test.go @@ -0,0 +1,259 @@ +package observers + +import ( + "context" + "testing" + "time" + + "github.com/milvus-io/milvus/internal/kv" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/util/etcd" + "github.com/stretchr/testify/suite" + clientv3 "go.etcd.io/etcd/client/v3" +) + +type CollectionObserverSuite struct { + suite.Suite + + // Data + collections []int64 + partitions map[int64][]int64 // CollectionID -> PartitionIDs + channels map[int64][]*meta.DmChannel + segments map[int64][]*datapb.SegmentInfo // CollectionID -> []datapb.SegmentInfo + loadTypes map[int64]querypb.LoadType + replicaNumber map[int64]int32 + loadPercentage map[int64]int32 + nodes []int64 + + // Mocks + idAllocator func() (int64, error) + etcd *clientv3.Client + kv kv.MetaKv + store meta.Store + + // Dependencies + dist *meta.DistributionManager + meta *meta.Meta + targetMgr *meta.TargetManager + + // Test object + ob *CollectionObserver +} + +func (suite *CollectionObserverSuite) SetupSuite() { + Params.Init() + + suite.collections = []int64{100, 101} + suite.partitions = map[int64][]int64{ + 100: {10}, + 101: {11, 12}, + } + suite.channels = map[int64][]*meta.DmChannel{ + 100: { + meta.DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: 100, + ChannelName: "100-dmc0", + }), + meta.DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: 100, + ChannelName: "100-dmc1", + }), + }, + 101: { + meta.DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: 101, + ChannelName: "101-dmc0", + }), + meta.DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: 101, + ChannelName: "101-dmc1", + }), + }, + } + suite.segments = map[int64][]*datapb.SegmentInfo{ + 100: { + &datapb.SegmentInfo{ + ID: 1, + CollectionID: 100, + PartitionID: 10, + InsertChannel: "100-dmc0", + }, + &datapb.SegmentInfo{ + ID: 2, + CollectionID: 100, + PartitionID: 10, + InsertChannel: "100-dmc1", + }, + }, + 101: { + &datapb.SegmentInfo{ + ID: 3, + CollectionID: 101, + PartitionID: 11, + InsertChannel: "101-dmc0", + }, + &datapb.SegmentInfo{ + ID: 4, + CollectionID: 101, + PartitionID: 12, + InsertChannel: "101-dmc1", + }, + }, + } + suite.loadTypes = map[int64]querypb.LoadType{ + 100: querypb.LoadType_LoadCollection, + 101: querypb.LoadType_LoadPartition, + } + suite.replicaNumber = map[int64]int32{ + 100: 1, + 101: 1, + } + suite.loadPercentage = map[int64]int32{ + 100: 0, + 101: 50, + } + suite.nodes = []int64{1, 2, 3} +} + +func (suite *CollectionObserverSuite) SetupTest() { + // Mocks + var err error + suite.idAllocator = RandomIncrementIDAllocator() + log.Debug("create embedded etcd KV...") + config := GenerateEtcdConfig() + client, err := etcd.GetEtcdClient(&config) + suite.Require().NoError(err) + suite.kv = etcdkv.NewEtcdKV(client, Params.EtcdCfg.MetaRootPath+"-"+RandomMetaRootPath()) + suite.Require().NoError(err) + log.Debug("create meta store...") + suite.store = meta.NewMetaStore(suite.kv) + + // Dependencies + suite.dist = meta.NewDistributionManager() + suite.meta = meta.NewMeta(suite.idAllocator, suite.store) + suite.targetMgr = meta.NewTargetManager() + + // Test object + suite.ob = NewCollectionObserver( + suite.dist, + suite.meta, + suite.targetMgr, + ) + + suite.loadAll() +} + +func (suite *CollectionObserverSuite) TearDownTest() { + suite.ob.Stop() + suite.kv.Close() +} + +func (suite *CollectionObserverSuite) TestObserve() { + const ( + timeout = 2 * time.Second + ) + // Not timeout + Params.QueryCoordCfg.LoadTimeoutSeconds = timeout + suite.ob.Start(context.Background()) + + // Collection 100 loaded before timeout, + // collection 101 timeout + suite.dist.LeaderViewManager.Update(1, &meta.LeaderView{ + ID: 1, + CollectionID: 100, + Channel: "100-dmc0", + Segments: map[int64]int64{1: 1}, + }) + suite.dist.LeaderViewManager.Update(2, &meta.LeaderView{ + ID: 2, + CollectionID: 100, + Channel: "100-dmc1", + Segments: map[int64]int64{2: 2}, + }) + suite.Eventually(func() bool { + return suite.isCollectionLoaded(suite.collections[0]) && + suite.isCollectionTimeout(suite.collections[1]) + }, timeout*2, timeout/10) +} + +func (suite *CollectionObserverSuite) isCollectionLoaded(collection int64) bool { + exist := suite.meta.Exist(collection) + percentage := suite.meta.GetLoadPercentage(collection) + status := suite.meta.GetStatus(collection) + replicas := suite.meta.ReplicaManager.GetByCollection(collection) + channels := suite.targetMgr.GetDmChannelsByCollection(collection) + segments := suite.targetMgr.GetSegmentsByCollection(collection) + + return exist && + percentage == 100 && + status == querypb.LoadStatus_Loaded && + len(replicas) == int(suite.replicaNumber[collection]) && + len(channels) == len(suite.channels[collection]) && + len(segments) == len(suite.segments[collection]) +} + +func (suite *CollectionObserverSuite) isCollectionTimeout(collection int64) bool { + exist := suite.meta.Exist(collection) + replicas := suite.meta.ReplicaManager.GetByCollection(collection) + channels := suite.targetMgr.GetDmChannelsByCollection(collection) + segments := suite.targetMgr.GetSegmentsByCollection(collection) + + return !(exist || + len(replicas) > 0 || + len(channels) > 0 || + len(segments) > 0) +} + +func (suite *CollectionObserverSuite) loadAll() { + for _, collection := range suite.collections { + suite.load(collection) + } +} + +func (suite *CollectionObserverSuite) load(collection int64) { + // Mock meta data + replicas, err := suite.meta.ReplicaManager.Spawn(collection, suite.replicaNumber[collection]) + suite.NoError(err) + for _, replica := range replicas { + replica.AddNode(suite.nodes...) + } + err = suite.meta.ReplicaManager.Put(replicas...) + suite.NoError(err) + + if suite.loadTypes[collection] == querypb.LoadType_LoadCollection { + suite.meta.PutCollection(&meta.Collection{ + CollectionLoadInfo: &querypb.CollectionLoadInfo{ + CollectionID: collection, + ReplicaNumber: suite.replicaNumber[collection], + Status: querypb.LoadStatus_Loading, + }, + LoadPercentage: 0, + CreatedAt: time.Now(), + }) + } else { + for _, partition := range suite.partitions[collection] { + suite.meta.PutPartition(&meta.Partition{ + PartitionLoadInfo: &querypb.PartitionLoadInfo{ + CollectionID: collection, + PartitionID: partition, + ReplicaNumber: suite.replicaNumber[collection], + Status: querypb.LoadStatus_Loading, + }, + LoadPercentage: 0, + CreatedAt: time.Now(), + }) + } + } + + suite.targetMgr.AddDmChannel(suite.channels[collection]...) + suite.targetMgr.AddSegment(suite.segments[collection]...) +} + +func TestCollectionObserver(t *testing.T) { + suite.Run(t, new(CollectionObserverSuite)) +} diff --git a/internal/querycoordv2/observers/handoff_observer.go b/internal/querycoordv2/observers/handoff_observer.go new file mode 100644 index 0000000000000..af3f00e75437b --- /dev/null +++ b/internal/querycoordv2/observers/handoff_observer.go @@ -0,0 +1,372 @@ +package observers + +import ( + "context" + "sync" + "time" + + "github.com/golang/protobuf/proto" + "go.etcd.io/etcd/api/v3/mvccpb" + "go.uber.org/zap" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/internal/util/retry" + "github.com/milvus-io/milvus/internal/util/typeutil" +) + +type CollectionHandoffStatus int32 +type HandoffEventStatus int32 + +const ( + // CollectionHandoffStatusRegistered start receive handoff event + CollectionHandoffStatusRegistered CollectionHandoffStatus = iota + 1 + // CollectionHandoffStatusStarted start trigger handoff event + CollectionHandoffStatusStarted +) + +const ( + HandoffEventStatusReceived HandoffEventStatus = iota + 1 + HandoffEventStatusTriggered +) + +type HandoffEvent struct { + Segment *querypb.SegmentInfo + Status HandoffEventStatus +} + +type queue []int64 + +type HandoffObserver struct { + store meta.Store + c chan struct{} + wg sync.WaitGroup + meta *meta.Meta + dist *meta.DistributionManager + target *meta.TargetManager + revision int64 + + collectionStatus map[int64]CollectionHandoffStatus + handoffEventLock sync.RWMutex + handoffEvents map[int64]*HandoffEvent + // partition id -> queue + handoffSubmitOrders map[int64]queue +} + +func NewHandoffObserver(store meta.Store, meta *meta.Meta, dist *meta.DistributionManager, target *meta.TargetManager) *HandoffObserver { + return &HandoffObserver{ + store: store, + c: make(chan struct{}), + meta: meta, + dist: dist, + target: target, + collectionStatus: map[int64]CollectionHandoffStatus{}, + handoffEvents: map[int64]*HandoffEvent{}, + handoffSubmitOrders: map[int64]queue{}, + } +} + +func (ob *HandoffObserver) Register(collectionIDs ...int64) { + ob.handoffEventLock.Lock() + defer ob.handoffEventLock.Unlock() + + for _, collectionID := range collectionIDs { + ob.collectionStatus[collectionID] = CollectionHandoffStatusRegistered + } +} + +func (ob *HandoffObserver) Unregister(ctx context.Context, collectionIDs ...int64) { + ob.handoffEventLock.Lock() + defer ob.handoffEventLock.Unlock() + + for _, collectionID := range collectionIDs { + delete(ob.collectionStatus, collectionID) + } +} + +func (ob *HandoffObserver) StartHandoff(collectionIDs ...int64) { + ob.handoffEventLock.Lock() + defer ob.handoffEventLock.Unlock() + + for _, collectionID := range collectionIDs { + ob.collectionStatus[collectionID] = CollectionHandoffStatusStarted + } +} + +func (ob *HandoffObserver) reloadFromStore(ctx context.Context) error { + _, handoffReqValues, version, err := ob.store.LoadHandoffWithRevision() + if err != nil { + log.Error("reloadFromKV: LoadWithRevision from kv failed", zap.Error(err)) + return err + } + ob.revision = version + + for _, value := range handoffReqValues { + segmentInfo := &querypb.SegmentInfo{} + err := proto.Unmarshal([]byte(value), segmentInfo) + if err != nil { + log.Error("reloadFromKV: unmarshal failed", zap.Any("error", err.Error())) + return err + } + ob.tryHandoff(ctx, segmentInfo) + } + + return nil +} + +func (ob *HandoffObserver) Start(ctx context.Context) error { + log.Info("Start reload handoff event from etcd") + if err := ob.reloadFromStore(ctx); err != nil { + log.Error("handoff observer reload from kv failed", zap.Error(err)) + return err + } + log.Info("Finish reload handoff event from etcd") + + ob.wg.Add(1) + go ob.schedule(ctx) + + return nil +} + +func (ob *HandoffObserver) Stop() { + close(ob.c) + ob.wg.Wait() +} + +func (ob *HandoffObserver) schedule(ctx context.Context) { + defer ob.wg.Done() + log.Info("start watch Segment handoff loop") + ticker := time.NewTicker(Params.QueryCoordCfg.CheckHandoffInterval) + log.Info("handoff interval", zap.String("interval", Params.QueryCoordCfg.CheckHandoffInterval.String())) + watchChan := ob.store.WatchHandoffEvent(ob.revision + 1) + for { + select { + case <-ctx.Done(): + log.Info("close handoff handler due to context done!") + return + case <-ob.c: + log.Info("close handoff handler") + return + + case resp, ok := <-watchChan: + if !ok { + log.Error("watch Segment handoff loop failed because watch channel is closed!") + } + + if err := resp.Err(); err != nil { + log.Warn("receive error handoff event from etcd", + zap.Error(err)) + } + + for _, event := range resp.Events { + segmentInfo := &querypb.SegmentInfo{} + err := proto.Unmarshal(event.Kv.Value, segmentInfo) + if err != nil { + log.Error("failed to deserialize handoff event", zap.Error(err)) + continue + } + + switch event.Type { + case mvccpb.PUT: + ob.tryHandoff(ctx, segmentInfo) + default: + log.Warn("HandoffObserver: receive event", + zap.String("type", event.Type.String()), + zap.String("key", string(event.Kv.Key)), + ) + } + } + + case <-ticker.C: + for _, event := range ob.handoffEvents { + switch event.Status { + case HandoffEventStatusReceived: + ob.tryHandoff(ctx, event.Segment) + case HandoffEventStatusTriggered: + ob.tryRelease(ctx, event) + } + } + + ob.tryClean(ctx) + } + } +} + +func (ob *HandoffObserver) tryHandoff(ctx context.Context, segment *querypb.SegmentInfo) { + ob.handoffEventLock.Lock() + defer ob.handoffEventLock.Unlock() + log := log.With(zap.Int64("collectionID", segment.CollectionID), + zap.Int64("partitionID", segment.PartitionID), + zap.Int64("segmentID", segment.SegmentID)) + + partitionStatus, collectionRegistered := ob.collectionStatus[segment.CollectionID] + if Params.QueryCoordCfg.AutoHandoff && collectionRegistered { + if partitionStatus == CollectionHandoffStatusRegistered { + ob.handoffEvents[segment.SegmentID] = &HandoffEvent{ + Segment: segment, + Status: HandoffEventStatusReceived, + } + } else { + ob.handoffEvents[segment.SegmentID] = &HandoffEvent{ + Segment: segment, + Status: HandoffEventStatusTriggered, + } + ob.handoff(segment) + } + _, ok := ob.handoffSubmitOrders[segment.PartitionID] + if !ok { + ob.handoffSubmitOrders[segment.PartitionID] = make([]int64, 0) + } + ob.handoffSubmitOrders[segment.PartitionID] = append(ob.handoffSubmitOrders[segment.PartitionID], segment.SegmentID) + } else { + // ignore handoff task + log.Debug("handoff event trigger failed due to collection/partition is not loaded!") + ob.cleanEvent(ctx, segment) + } +} + +func (ob *HandoffObserver) handoff(segment *querypb.SegmentInfo) { + targets := ob.target.GetSegmentsByCollection(segment.GetCollectionID(), segment.GetPartitionID()) + // when handoff event load a Segment, it sobuld remove all recursive handoff compact from + uniqueSet := typeutil.NewUniqueSet() + recursiveCompactFrom := ob.getOverrideSegmentInfo(targets, segment.CompactionFrom...) + uniqueSet.Insert(recursiveCompactFrom...) + uniqueSet.Insert(segment.GetCompactionFrom()...) + + segmentInfo := &datapb.SegmentInfo{ + ID: segment.SegmentID, + CollectionID: segment.CollectionID, + PartitionID: segment.PartitionID, + InsertChannel: segment.GetDmChannel(), + State: segment.GetSegmentState(), + CreatedByCompaction: segment.GetCreatedByCompaction(), + CompactionFrom: uniqueSet.Collect(), + } + + log.Info("HandoffObserver: handoff Segment, register to target") + ob.target.HandoffSegment(segmentInfo, segmentInfo.CompactionFrom...) +} + +func (ob *HandoffObserver) isSegmentReleased(id int64) bool { + return len(ob.dist.LeaderViewManager.GetSegmentDist(id)) == 0 +} + +func (ob *HandoffObserver) isGrowingSegmentReleased(id int64) bool { + return len(ob.dist.LeaderViewManager.GetGrowingSegmentDist(id)) == 0 +} + +func (ob *HandoffObserver) isSealedSegmentLoaded(segment *querypb.SegmentInfo) bool { + // must be sealed Segment loaded in all replica, in case of handoff between growing and sealed + nodes := ob.dist.LeaderViewManager.GetSealedSegmentDist(segment.SegmentID) + replicas := utils.GroupNodesByReplica(ob.meta.ReplicaManager, segment.CollectionID, nodes) + return len(replicas) == len(ob.meta.ReplicaManager.GetByCollection(segment.CollectionID)) +} + +func (ob *HandoffObserver) getOverrideSegmentInfo(handOffSegments []*datapb.SegmentInfo, segmentIDs ...int64) []int64 { + overrideSegments := make([]int64, 0) + for _, segmentID := range segmentIDs { + for _, segmentInHandoff := range handOffSegments { + if segmentID == segmentInHandoff.ID { + toReleaseSegments := ob.getOverrideSegmentInfo(handOffSegments, segmentInHandoff.CompactionFrom...) + if len(toReleaseSegments) > 0 { + overrideSegments = append(overrideSegments, toReleaseSegments...) + } + + overrideSegments = append(overrideSegments, segmentID) + } + } + } + + return overrideSegments +} + +func (ob *HandoffObserver) tryRelease(ctx context.Context, event *HandoffEvent) { + segment := event.Segment + if ob.isSealedSegmentLoaded(segment) || !ob.isSegmentExistOnTarget(segment) { + compactSource := segment.CompactionFrom + + for _, toRelease := range compactSource { + // when handoff happens between growing and sealed, both with same Segment id, so can't remove from target here + if segment.CreatedByCompaction { + log.Info("HandoffObserver: remove compactFrom Segment", + zap.Int64("collectionID", segment.CollectionID), + zap.Int64("partitionID", segment.PartitionID), + zap.Int64("compactedSegment", segment.SegmentID), + zap.Int64("toReleaseSegment", toRelease), + ) + ob.target.RemoveSegment(toRelease) + } + } + } +} + +func (ob *HandoffObserver) tryClean(ctx context.Context) { + ob.handoffEventLock.Lock() + defer ob.handoffEventLock.Unlock() + + for partitionID, partitionSubmitOrder := range ob.handoffSubmitOrders { + pos := 0 + for _, segmentID := range partitionSubmitOrder { + event, ok := ob.handoffEvents[segmentID] + if !ok { + continue + } + + segment := event.Segment + if ob.isAllCompactFromReleased(segment) { + log.Info("HandoffObserver: clean handoff event after handoff finished!", + zap.Int64("collectionID", segment.CollectionID), + zap.Int64("partitionID", segment.PartitionID), + zap.Int64("segmentID", segment.SegmentID), + ) + err := ob.cleanEvent(ctx, segment) + if err == nil { + delete(ob.handoffEvents, segment.SegmentID) + } + pos++ + } else { + break + } + } + ob.handoffSubmitOrders[partitionID] = partitionSubmitOrder[pos:] + } +} + +func (ob *HandoffObserver) cleanEvent(ctx context.Context, segmentInfo *querypb.SegmentInfo) error { + log := log.With( + zap.Int64("collectionID", segmentInfo.CollectionID), + zap.Int64("partitionID", segmentInfo.PartitionID), + zap.Int64("segmentID", segmentInfo.SegmentID), + ) + + // add retry logic + err := retry.Do(ctx, func() error { + return ob.store.RemoveHandoffEvent(segmentInfo) + }, retry.Attempts(5)) + + if err != nil { + log.Warn("failed to clean handoff event from etcd", zap.Error(err)) + } + return err +} + +func (ob *HandoffObserver) isSegmentExistOnTarget(segmentInfo *querypb.SegmentInfo) bool { + return ob.target.ContainSegment(segmentInfo.SegmentID) +} + +func (ob *HandoffObserver) isAllCompactFromReleased(segmentInfo *querypb.SegmentInfo) bool { + if !segmentInfo.CreatedByCompaction { + return !ob.isGrowingSegmentReleased(segmentInfo.SegmentID) + } + + for _, segment := range segmentInfo.CompactionFrom { + if !ob.isSegmentReleased(segment) { + return false + } + } + return true +} diff --git a/internal/querycoordv2/observers/handoff_observer_test.go b/internal/querycoordv2/observers/handoff_observer_test.go new file mode 100644 index 0000000000000..e372570cfa83e --- /dev/null +++ b/internal/querycoordv2/observers/handoff_observer_test.go @@ -0,0 +1,535 @@ +package observers + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/golang/protobuf/proto" + "github.com/stretchr/testify/suite" + clientv3 "go.etcd.io/etcd/client/v3" + "go.uber.org/zap" + + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/util" + "github.com/milvus-io/milvus/internal/util/etcd" + "github.com/milvus-io/milvus/internal/util/typeutil" +) + +type HandoffObserverTestSuit struct { + suite.Suite + // Data + collection int64 + partition int64 + channel *meta.DmChannel + replicaNumber int32 + nodes []int64 + growingSegments []*datapb.SegmentInfo + sealedSegments []*datapb.SegmentInfo + + //Mocks + idAllocator func() (int64, error) + etcd *clientv3.Client + kv *etcdkv.EtcdKV + + //Dependency + store meta.Store + meta *meta.Meta + dist *meta.DistributionManager + target *meta.TargetManager + + // Test Object + observer *HandoffObserver +} + +func (suite *HandoffObserverTestSuit) SetupSuite() { + Params.Init() + + suite.collection = 100 + suite.partition = 10 + suite.channel = meta.DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: 100, + ChannelName: "100-dmc0", + }) + + suite.sealedSegments = []*datapb.SegmentInfo{ + { + ID: 1, + CollectionID: 100, + PartitionID: 10, + InsertChannel: "100-dmc0", + State: commonpb.SegmentState_Sealed, + }, + { + ID: 2, + CollectionID: 100, + PartitionID: 10, + InsertChannel: "100-dmc1", + State: commonpb.SegmentState_Sealed, + }, + } + suite.replicaNumber = 1 + suite.nodes = []int64{1, 2, 3} +} + +func (suite *HandoffObserverTestSuit) SetupTest() { + // Mocks + var err error + suite.idAllocator = RandomIncrementIDAllocator() + log.Debug("create embedded etcd KV...") + config := GenerateEtcdConfig() + client, err := etcd.GetEtcdClient(&config) + suite.Require().NoError(err) + suite.kv = etcdkv.NewEtcdKV(client, Params.EtcdCfg.MetaRootPath+"-"+RandomMetaRootPath()) + suite.Require().NoError(err) + log.Debug("create meta store...") + suite.store = meta.NewMetaStore(suite.kv) + + // Dependency + suite.meta = meta.NewMeta(suite.idAllocator, suite.store) + suite.dist = meta.NewDistributionManager() + suite.target = meta.NewTargetManager() + + // Test Object + suite.observer = NewHandoffObserver(suite.store, suite.meta, suite.dist, suite.target) + suite.observer.Register(suite.collection) + suite.observer.StartHandoff(suite.collection) + suite.load() +} + +func (suite *HandoffObserverTestSuit) TearDownTest() { + suite.observer.Stop() + suite.kv.Close() +} + +func (suite *HandoffObserverTestSuit) TestFlushingHandoff() { + // init leader view + suite.dist.LeaderViewManager.Update(1, &meta.LeaderView{ + ID: 1, + CollectionID: suite.collection, + Channel: suite.channel.ChannelName, + Segments: map[int64]int64{1: 1, 2: 2}, + GrowingSegments: typeutil.NewUniqueSet(3), + }) + + Params.QueryCoordCfg.CheckHandoffInterval = 1 * time.Second + err := suite.observer.Start(context.Background()) + suite.NoError(err) + + flushingSegment := &querypb.SegmentInfo{ + SegmentID: 3, + CollectionID: suite.collection, + PartitionID: suite.partition, + SegmentState: commonpb.SegmentState_Sealed, + } + suite.produceHandOffEvent(flushingSegment) + + suite.Eventually(func() bool { + return suite.target.ContainSegment(3) + }, 3*time.Second, 1*time.Second) + + // fake load CompactTo Segment + suite.dist.LeaderViewManager.Update(1, &meta.LeaderView{ + ID: 1, + CollectionID: suite.collection, + Channel: suite.channel.ChannelName, + Segments: map[int64]int64{1: 1, 2: 2, 3: 3}, + GrowingSegments: typeutil.NewUniqueSet(3), + }) + + suite.Eventually(func() bool { + return suite.target.ContainSegment(3) + }, 3*time.Second, 1*time.Second) + + // fake release CompactFrom Segment + suite.dist.LeaderViewManager.Update(1, &meta.LeaderView{ + ID: 1, + CollectionID: suite.collection, + Channel: suite.channel.ChannelName, + Segments: map[int64]int64{1: 1, 2: 2, 3: 3}, + }) + + suite.Eventually(func() bool { + return len(suite.dist.LeaderViewManager.GetGrowingSegmentDist(3)) == 0 + }, 3*time.Second, 1*time.Second) + + suite.Eventually(func() bool { + key := fmt.Sprintf("%s/%d/%d/%d", util.HandoffSegmentPrefix, suite.collection, suite.partition, 3) + value, err := suite.kv.Load(key) + return len(value) == 0 && err != nil + }, 3*time.Second, 1*time.Second) +} + +func (suite *HandoffObserverTestSuit) TestCompactHandoff() { + // init leader view + suite.dist.LeaderViewManager.Update(2, &meta.LeaderView{ + ID: 1, + CollectionID: suite.collection, + Channel: suite.channel.ChannelName, + Segments: map[int64]int64{1: 1, 2: 2}, + }) + + Params.QueryCoordCfg.CheckHandoffInterval = 1 * time.Second + err := suite.observer.Start(context.Background()) + suite.NoError(err) + + compactSegment := &querypb.SegmentInfo{ + SegmentID: 3, + CollectionID: suite.collection, + PartitionID: suite.partition, + SegmentState: commonpb.SegmentState_Sealed, + CompactionFrom: []int64{1}, + CreatedByCompaction: true, + } + suite.produceHandOffEvent(compactSegment) + + suite.Eventually(func() bool { + log.Info("", zap.Bool("contains", suite.target.ContainSegment(3))) + return suite.target.ContainSegment(3) + }, 3*time.Second, 1*time.Second) + + // fake load CompactTo Segment + suite.dist.LeaderViewManager.Update(2, &meta.LeaderView{ + ID: 1, + CollectionID: suite.collection, + Channel: suite.channel.ChannelName, + Segments: map[int64]int64{1: 1, 2: 2, 3: 3}, + }) + + suite.Eventually(func() bool { + log.Info("", zap.Bool("contains", suite.target.ContainSegment(1))) + return !suite.target.ContainSegment(1) + }, 3*time.Second, 1*time.Second) + + // fake release CompactFrom Segment + suite.dist.LeaderViewManager.Update(2, &meta.LeaderView{ + ID: 1, + CollectionID: suite.collection, + Channel: suite.channel.ChannelName, + Segments: map[int64]int64{2: 2, 3: 3}, + }) + + suite.Eventually(func() bool { + key := fmt.Sprintf("%s/%d/%d/%d", util.HandoffSegmentPrefix, suite.collection, suite.partition, 3) + value, err := suite.kv.Load(key) + return len(value) == 0 && err != nil + }, 3*time.Second, 1*time.Second) +} + +func (suite *HandoffObserverTestSuit) TestRecursiveHandoff() { + // init leader view + suite.dist.LeaderViewManager.Update(1, &meta.LeaderView{ + ID: 1, + CollectionID: suite.collection, + Channel: suite.channel.ChannelName, + Segments: map[int64]int64{1: 1, 2: 2}, + GrowingSegments: typeutil.NewUniqueSet(3), + }) + + Params.QueryCoordCfg.CheckHandoffInterval = 1 * time.Second + err := suite.observer.Start(context.Background()) + suite.NoError(err) + + flushingSegment := &querypb.SegmentInfo{ + SegmentID: 3, + CollectionID: suite.collection, + PartitionID: suite.partition, + SegmentState: commonpb.SegmentState_Sealed, + } + + compactSegment1 := &querypb.SegmentInfo{ + SegmentID: 4, + CollectionID: suite.collection, + PartitionID: suite.partition, + SegmentState: commonpb.SegmentState_Sealed, + CompactionFrom: []int64{3}, + CreatedByCompaction: true, + } + + compactSegment2 := &querypb.SegmentInfo{ + SegmentID: 5, + CollectionID: suite.collection, + PartitionID: suite.partition, + SegmentState: commonpb.SegmentState_Sealed, + CompactionFrom: []int64{4}, + CreatedByCompaction: true, + } + + suite.produceHandOffEvent(flushingSegment) + suite.produceHandOffEvent(compactSegment1) + suite.produceHandOffEvent(compactSegment2) + + // fake load CompactTo Segment + suite.dist.LeaderViewManager.Update(1, &meta.LeaderView{ + ID: 1, + CollectionID: suite.collection, + Channel: suite.channel.ChannelName, + Segments: map[int64]int64{1: 1, 2: 2, 5: 3}, + GrowingSegments: typeutil.NewUniqueSet(3), + }) + + suite.Eventually(func() bool { + return !suite.target.ContainSegment(3) && !suite.target.ContainSegment(4) + }, 3*time.Second, 1*time.Second) + + suite.Eventually(func() bool { + return suite.target.ContainSegment(1) && suite.target.ContainSegment(2) && suite.target.ContainSegment(5) + }, 3*time.Second, 1*time.Second) + + // fake release CompactFrom Segment + suite.dist.LeaderViewManager.Update(1, &meta.LeaderView{ + ID: 1, + CollectionID: suite.collection, + Channel: suite.channel.ChannelName, + Segments: map[int64]int64{1: 1, 2: 2, 5: 3}, + }) + + suite.Eventually(func() bool { + return !suite.target.ContainSegment(3) && !suite.target.ContainSegment(4) + }, 3*time.Second, 1*time.Second) + + suite.Eventually(func() bool { + return suite.target.ContainSegment(1) && suite.target.ContainSegment(2) && suite.target.ContainSegment(5) + }, 3*time.Second, 1*time.Second) + + suite.Eventually(func() bool { + return len(suite.dist.LeaderViewManager.GetGrowingSegmentDist(3)) == 0 + }, 3*time.Second, 1*time.Second) + + suite.Eventually(func() bool { + key := fmt.Sprintf("%s/%d/%d/%d", util.HandoffSegmentPrefix, suite.collection, suite.partition, 3) + value, err := suite.kv.Load(key) + return len(value) == 0 && err != nil + }, 3*time.Second, 1*time.Second) +} + +func (suite *HandoffObserverTestSuit) TestReloadHandoffEventOrder() { + // init leader view + suite.dist.LeaderViewManager.Update(1, &meta.LeaderView{ + ID: 1, + CollectionID: suite.collection, + Channel: suite.channel.ChannelName, + Segments: map[int64]int64{1: 1, 2: 2}, + GrowingSegments: typeutil.NewUniqueSet(3), + }) + + // fake handoff event from start + flushingSegment := &querypb.SegmentInfo{ + SegmentID: 3, + CollectionID: suite.collection, + PartitionID: suite.partition, + SegmentState: commonpb.SegmentState_Sealed, + } + compactSegment1 := &querypb.SegmentInfo{ + SegmentID: 9, + CollectionID: suite.collection, + PartitionID: suite.partition, + SegmentState: commonpb.SegmentState_Sealed, + CompactionFrom: []int64{3}, + CreatedByCompaction: true, + } + compactSegment2 := &querypb.SegmentInfo{ + SegmentID: 10, + CollectionID: suite.collection, + PartitionID: suite.partition, + SegmentState: commonpb.SegmentState_Sealed, + CompactionFrom: []int64{4}, + CreatedByCompaction: true, + } + + suite.produceHandOffEvent(flushingSegment) + suite.produceHandOffEvent(compactSegment1) + suite.produceHandOffEvent(compactSegment2) + + keys, _, _, err := suite.kv.LoadWithRevision(util.HandoffSegmentPrefix) + suite.NoError(err) + suite.Equal(true, strings.HasSuffix(keys[0], "3")) + suite.Equal(true, strings.HasSuffix(keys[1], "9")) + suite.Equal(true, strings.HasSuffix(keys[2], "10")) +} + +func (suite *HandoffObserverTestSuit) TestLoadHandoffEventFromStore() { + // init leader view + suite.dist.LeaderViewManager.Update(1, &meta.LeaderView{ + ID: 1, + CollectionID: suite.collection, + Channel: suite.channel.ChannelName, + Segments: map[int64]int64{1: 1, 2: 2}, + GrowingSegments: typeutil.NewUniqueSet(3), + }) + + // fake handoff event from start + flushingSegment := &querypb.SegmentInfo{ + SegmentID: 3, + CollectionID: suite.collection, + PartitionID: suite.partition, + SegmentState: commonpb.SegmentState_Sealed, + } + compactSegment1 := &querypb.SegmentInfo{ + SegmentID: 4, + CollectionID: suite.collection, + PartitionID: suite.partition, + SegmentState: commonpb.SegmentState_Sealed, + CompactionFrom: []int64{3}, + CreatedByCompaction: true, + } + compactSegment2 := &querypb.SegmentInfo{ + SegmentID: 5, + CollectionID: suite.collection, + PartitionID: suite.partition, + SegmentState: commonpb.SegmentState_Sealed, + CompactionFrom: []int64{4}, + CreatedByCompaction: true, + } + + suite.produceHandOffEvent(flushingSegment) + suite.produceHandOffEvent(compactSegment1) + suite.produceHandOffEvent(compactSegment2) + + Params.QueryCoordCfg.CheckHandoffInterval = 1 * time.Second + err := suite.observer.Start(context.Background()) + suite.NoError(err) + + // fake load CompactTo Segment + suite.dist.LeaderViewManager.Update(1, &meta.LeaderView{ + ID: 1, + CollectionID: suite.collection, + Channel: suite.channel.ChannelName, + Segments: map[int64]int64{1: 1, 2: 2, 3: 3, 4: 2, 5: 3}, + GrowingSegments: typeutil.NewUniqueSet(3), + }) + + suite.Eventually(func() bool { + return !suite.target.ContainSegment(3) && !suite.target.ContainSegment(4) + }, 3*time.Second, 1*time.Second) + + suite.Eventually(func() bool { + return suite.target.ContainSegment(1) && suite.target.ContainSegment(2) && suite.target.ContainSegment(5) + }, 3*time.Second, 1*time.Second) + + // fake release CompactFrom Segment + suite.dist.LeaderViewManager.Update(1, &meta.LeaderView{ + ID: 1, + CollectionID: suite.collection, + Channel: suite.channel.ChannelName, + Segments: map[int64]int64{1: 1, 2: 2, 5: 3}, + }) + + suite.Eventually(func() bool { + return !suite.target.ContainSegment(3) && !suite.target.ContainSegment(4) + }, 3*time.Second, 1*time.Second) + + suite.Eventually(func() bool { + return suite.target.ContainSegment(1) && suite.target.ContainSegment(2) && suite.target.ContainSegment(5) + }, 3*time.Second, 1*time.Second) + + suite.Eventually(func() bool { + return len(suite.dist.LeaderViewManager.GetGrowingSegmentDist(3)) == 0 + }, 3*time.Second, 1*time.Second) + + suite.Eventually(func() bool { + key := fmt.Sprintf("%s/%d/%d/%d", util.HandoffSegmentPrefix, suite.collection, suite.partition, 3) + value, err := suite.kv.Load(key) + return len(value) == 0 && err != nil + }, 3*time.Second, 1*time.Second) +} + +func (suite *HandoffObserverTestSuit) produceHandOffEvent(segmentInfo *querypb.SegmentInfo) { + key := fmt.Sprintf("%s/%d/%d/%d", util.HandoffSegmentPrefix, segmentInfo.CollectionID, segmentInfo.PartitionID, segmentInfo.SegmentID) + value, err := proto.Marshal(segmentInfo) + suite.NoError(err) + err = suite.kv.Save(key, string(value)) + suite.NoError(err) +} + +func (suite *HandoffObserverTestSuit) load() { + // Mock meta data + replicas, err := suite.meta.ReplicaManager.Spawn(suite.collection, suite.replicaNumber) + suite.NoError(err) + for _, replica := range replicas { + replica.AddNode(suite.nodes...) + } + err = suite.meta.ReplicaManager.Put(replicas...) + suite.NoError(err) + + err = suite.meta.PutCollection(&meta.Collection{ + CollectionLoadInfo: &querypb.CollectionLoadInfo{ + CollectionID: suite.collection, + ReplicaNumber: suite.replicaNumber, + Status: querypb.LoadStatus_Loaded, + }, + LoadPercentage: 0, + CreatedAt: time.Now(), + }) + suite.NoError(err) + + suite.target.AddDmChannel(suite.channel) + suite.target.AddSegment(suite.sealedSegments...) +} + +func (suite *HandoffObserverTestSuit) TestHandoffOnUnLoadedPartition() { + const ( + collectionID = 111 + loadedPartitionID = 1 + unloadedPartitionID = 2 + ) + err := suite.meta.PutPartition(&meta.Partition{ + PartitionLoadInfo: &querypb.PartitionLoadInfo{ + CollectionID: collectionID, + PartitionID: loadedPartitionID, + ReplicaNumber: suite.replicaNumber, + Status: querypb.LoadStatus_Loaded, + }, + }) + suite.NoError(err) + + // init leader view + suite.dist.LeaderViewManager.Update(2, &meta.LeaderView{ + ID: 1, + CollectionID: collectionID, + Channel: suite.channel.ChannelName, + Segments: map[int64]int64{1: 1, 2: 2}, + }) + + Params.QueryCoordCfg.CheckHandoffInterval = 1 * time.Second + err = suite.observer.Start(context.Background()) + suite.NoError(err) + + compactSegment := &querypb.SegmentInfo{ + SegmentID: 3, + CollectionID: collectionID, + PartitionID: unloadedPartitionID, + SegmentState: commonpb.SegmentState_Sealed, + CompactionFrom: []int64{2}, + CreatedByCompaction: true, + } + suite.produceHandOffEvent(compactSegment) + + suite.Eventually(func() bool { + log.Info("", zap.Bool("contains", suite.target.ContainSegment(3))) + return !suite.target.ContainSegment(3) + }, 3*time.Second, 1*time.Second) + + suite.Eventually(func() bool { + log.Info("", zap.Bool("contains", suite.target.ContainSegment(1))) + return suite.target.ContainSegment(1) && suite.target.ContainSegment(2) + }, 3*time.Second, 1*time.Second) + + suite.Eventually(func() bool { + key := fmt.Sprintf("%s/%d/%d/%d", util.HandoffSegmentPrefix, suite.collection, suite.partition, 3) + value, err := suite.kv.Load(key) + return len(value) == 0 && err != nil + }, 3*time.Second, 1*time.Second) +} + +func TestHandoffObserverSuit(t *testing.T) { + suite.Run(t, new(HandoffObserverTestSuit)) +} diff --git a/internal/querycoordv2/observers/leader_observer.go b/internal/querycoordv2/observers/leader_observer.go new file mode 100644 index 0000000000000..1b0a31fa35acd --- /dev/null +++ b/internal/querycoordv2/observers/leader_observer.go @@ -0,0 +1,158 @@ +package observers + +import ( + "context" + "sync" + "time" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "go.uber.org/zap" +) + +const interval = 1 * time.Second + +// LeaderObserver is to sync the distribution with leader +type LeaderObserver struct { + wg sync.WaitGroup + closeCh chan struct{} + dist *meta.DistributionManager + meta *meta.Meta + target *meta.TargetManager + cluster session.Cluster +} + +func (o *LeaderObserver) Start(ctx context.Context) { + o.wg.Add(1) + go func() { + defer o.wg.Done() + ticker := time.NewTicker(interval) + for { + select { + case <-o.closeCh: + log.Info("stop leader observer") + return + case <-ctx.Done(): + log.Info("stop leader observer due to ctx done") + return + case <-ticker.C: + o.observe(ctx) + } + } + }() +} + +func (o *LeaderObserver) Stop() { + close(o.closeCh) + o.wg.Wait() +} + +func (o *LeaderObserver) observe(ctx context.Context) { + o.observeSegmentsDist(ctx) +} + +func (o *LeaderObserver) observeSegmentsDist(ctx context.Context) { + collectionIDs := o.meta.CollectionManager.GetAll() + for _, cid := range collectionIDs { + o.observeCollection(ctx, cid) + } +} + +func (o *LeaderObserver) observeCollection(ctx context.Context, collection int64) { + replicas := o.meta.ReplicaManager.GetByCollection(collection) + for _, replica := range replicas { + leaders := o.dist.ChannelDistManager.GetShardLeadersByReplica(replica) + for ch, leaderID := range leaders { + leaderView := o.dist.LeaderViewManager.GetLeaderShardView(leaderID, ch) + if leaderView == nil { + continue + } + dists := o.dist.SegmentDistManager.GetByShard(ch) + needLoaded, needRemoved := o.findNeedLoadedSegments(leaderView, dists), + o.findNeedRemovedSegments(leaderView, dists) + o.sync(ctx, leaderView, append(needLoaded, needRemoved...)) + } + } +} + +func (o *LeaderObserver) findNeedLoadedSegments(leaderView *meta.LeaderView, dists []*meta.Segment) []*querypb.SyncAction { + ret := make([]*querypb.SyncAction, 0) + dists = utils.FindMaxVersionSegments(dists) + for _, s := range dists { + node, ok := leaderView.Segments[s.GetID()] + consistentOnLeader := ok && node == s.Node + if consistentOnLeader || !o.target.ContainSegment(s.GetID()) { + continue + } + ret = append(ret, &querypb.SyncAction{ + Type: querypb.SyncType_Set, + PartitionID: s.GetPartitionID(), + SegmentID: s.GetID(), + NodeID: s.Node, + }) + } + return ret +} + +func (o *LeaderObserver) findNeedRemovedSegments(leaderView *meta.LeaderView, dists []*meta.Segment) []*querypb.SyncAction { + ret := make([]*querypb.SyncAction, 0) + distMap := make(map[int64]struct{}) + for _, s := range dists { + distMap[s.GetID()] = struct{}{} + } + for sid := range leaderView.Segments { + _, ok := distMap[sid] + if ok || o.target.ContainSegment(sid) { + continue + } + ret = append(ret, &querypb.SyncAction{ + Type: querypb.SyncType_Remove, + SegmentID: sid, + }) + } + return ret +} + +func (o *LeaderObserver) sync(ctx context.Context, leaderView *meta.LeaderView, diffs []*querypb.SyncAction) { + log := log.With( + zap.Int64("leaderID", leaderView.ID), + zap.Int64("collectionID", leaderView.CollectionID), + zap.String("channel", leaderView.Channel), + ) + req := &querypb.SyncDistributionRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_SyncDistribution, + }, + CollectionID: leaderView.CollectionID, + Channel: leaderView.Channel, + Actions: diffs, + } + resp, err := o.cluster.SyncDistribution(ctx, leaderView.ID, req) + if err != nil { + log.Error("failed to sync distribution", zap.Error(err)) + return + } + + if resp.ErrorCode != commonpb.ErrorCode_Success { + log.Error("failed to sync distribution", zap.String("reason", resp.GetReason())) + } +} + +func NewLeaderObserver( + dist *meta.DistributionManager, + meta *meta.Meta, + targetMgr *meta.TargetManager, + cluster session.Cluster, +) *LeaderObserver { + return &LeaderObserver{ + closeCh: make(chan struct{}), + dist: dist, + meta: meta, + target: targetMgr, + cluster: cluster, + } +} diff --git a/internal/querycoordv2/observers/leader_observer_test.go b/internal/querycoordv2/observers/leader_observer_test.go new file mode 100644 index 0000000000000..6fa7914fad7ea --- /dev/null +++ b/internal/querycoordv2/observers/leader_observer_test.go @@ -0,0 +1,131 @@ +package observers + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "go.uber.org/atomic" + + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/internal/util/etcd" +) + +type LeaderObserverTestSuite struct { + suite.Suite + observer *LeaderObserver + kv *etcdkv.EtcdKV + mockCluster *session.MockCluster +} + +func (suite *LeaderObserverTestSuite) SetupSuite() { + Params.Init() +} + +func (suite *LeaderObserverTestSuite) SetupTest() { + var err error + config := GenerateEtcdConfig() + cli, err := etcd.GetEtcdClient(&config) + suite.Require().NoError(err) + suite.kv = etcdkv.NewEtcdKV(cli, config.MetaRootPath) + + // meta + store := meta.NewMetaStore(suite.kv) + idAllocator := RandomIncrementIDAllocator() + testMeta := meta.NewMeta(idAllocator, store) + + suite.mockCluster = session.NewMockCluster(suite.T()) + distManager := meta.NewDistributionManager() + targetManager := meta.NewTargetManager() + suite.observer = NewLeaderObserver(distManager, testMeta, targetManager, suite.mockCluster) +} + +func (suite *LeaderObserverTestSuite) TearDownTest() { + suite.observer.Stop() + suite.kv.Close() +} + +func (suite *LeaderObserverTestSuite) TestSyncLoadedSegments() { + observer := suite.observer + observer.meta.CollectionManager.PutCollection(utils.CreateTestCollection(1, 1)) + observer.meta.ReplicaManager.Put(utils.CreateTestReplica(1, 1, []int64{1, 2})) + observer.target.AddSegment(utils.CreateTestSegmentInfo(1, 1, 1, "test-insert-channel")) + observer.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 1, 1, "test-insert-channel")) + observer.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel")) + observer.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, []int64{})) + expectReq := &querypb.SyncDistributionRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_SyncDistribution, + }, + CollectionID: 1, + Channel: "test-insert-channel", + Actions: []*querypb.SyncAction{ + { + Type: querypb.SyncType_Set, + PartitionID: 1, + SegmentID: 1, + NodeID: 1, + }, + }, + } + called := atomic.NewBool(false) + suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), expectReq).Once(). + Run(func(args mock.Arguments) { called.Store(true) }). + Return(&commonpb.Status{}, nil) + + observer.Start(context.TODO()) + + suite.Eventually( + func() bool { + return called.Load() + }, + 10*time.Second, + 500*time.Millisecond, + ) +} + +func (suite *LeaderObserverTestSuite) TestSyncRemovedSegments() { + observer := suite.observer + observer.meta.CollectionManager.PutCollection(utils.CreateTestCollection(1, 1)) + observer.meta.ReplicaManager.Put(utils.CreateTestReplica(1, 1, []int64{1, 2})) + + observer.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel")) + observer.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{3: 2}, []int64{})) + + expectReq := &querypb.SyncDistributionRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_SyncDistribution, + }, + CollectionID: 1, + Channel: "test-insert-channel", + Actions: []*querypb.SyncAction{ + { + Type: querypb.SyncType_Remove, + SegmentID: 3, + }, + }, + } + ch := make(chan struct{}) + suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), expectReq).Once(). + Run(func(args mock.Arguments) { close(ch) }). + Return(&commonpb.Status{}, nil) + + observer.Start(context.TODO()) + + select { + case <-ch: + case <-time.After(2 * time.Second): + } +} + +func TestLeaderObserverSuite(t *testing.T) { + suite.Run(t, new(LeaderObserverTestSuite)) +} diff --git a/internal/querycoordv2/params/params.go b/internal/querycoordv2/params/params.go new file mode 100644 index 0000000000000..0d98b851ed50e --- /dev/null +++ b/internal/querycoordv2/params/params.go @@ -0,0 +1,44 @@ +package params + +import ( + "errors" + "math/rand" + "strconv" + "sync/atomic" + "time" + + "github.com/milvus-io/milvus/internal/util/paramtable" +) + +var Params paramtable.ComponentParam + +var ( + ErrFailedAllocateID = errors.New("failed to allocate ID") +) + +// GenerateEtcdConfig returns a etcd config with a random root path, +// NOTE: for test only +func GenerateEtcdConfig() paramtable.EtcdConfig { + config := Params.EtcdCfg + rand.Seed(time.Now().UnixNano()) + suffix := "-test-querycoord" + strconv.FormatInt(rand.Int63(), 10) + config.MetaRootPath = config.MetaRootPath + suffix + return config +} + +func RandomMetaRootPath() string { + return "test-query-coord-" + strconv.FormatInt(rand.Int63(), 10) +} + +func RandomIncrementIDAllocator() func() (int64, error) { + id := rand.Int63() / 2 + return func() (int64, error) { + return atomic.AddInt64(&id, 1), nil + } +} + +func ErrorIDAllocator() func() (int64, error) { + return func() (int64, error) { + return 0, ErrFailedAllocateID + } +} diff --git a/internal/querycoordv2/server.go b/internal/querycoordv2/server.go new file mode 100644 index 0000000000000..e9a4b6b492870 --- /dev/null +++ b/internal/querycoordv2/server.go @@ -0,0 +1,640 @@ +package querycoordv2 + +import ( + "context" + "errors" + "fmt" + "os" + "sort" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/samber/lo" + clientv3 "go.etcd.io/etcd/client/v3" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + + "github.com/milvus-io/milvus/internal/allocator" + "github.com/milvus-io/milvus/internal/common" + "github.com/milvus-io/milvus/internal/kv" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/internalpb" + "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/balance" + "github.com/milvus-io/milvus/internal/querycoordv2/checkers" + "github.com/milvus-io/milvus/internal/querycoordv2/dist" + "github.com/milvus-io/milvus/internal/querycoordv2/job" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/observers" + "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/task" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/internal/types" + "github.com/milvus-io/milvus/internal/util/dependency" + "github.com/milvus-io/milvus/internal/util/metricsinfo" + "github.com/milvus-io/milvus/internal/util/sessionutil" + "github.com/milvus-io/milvus/internal/util/tsoutil" + "github.com/milvus-io/milvus/internal/util/typeutil" +) + +var ( + // Only for re-export + Params = ¶ms.Params +) + +type Server struct { + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + status atomic.Value + etcdCli *clientv3.Client + session *sessionutil.Session + kv kv.MetaKv + idAllocator func() (int64, error) + factory dependency.Factory + metricsCacheManager *metricsinfo.MetricsCacheManager + chunkManager storage.ChunkManager + + // Coordinators + dataCoord types.DataCoord + rootCoord types.RootCoord + indexCoord types.IndexCoord + + // Meta + store meta.Store + meta *meta.Meta + dist *meta.DistributionManager + targetMgr *meta.TargetManager + broker meta.Broker + + // Session + cluster session.Cluster + nodeMgr *session.NodeManager + + // Schedulers + jobScheduler *job.Scheduler + taskScheduler task.Scheduler + + // HeartBeat + distController *dist.Controller + + // Checkers + checkerController *checkers.CheckerController + + // Observers + collectionObserver *observers.CollectionObserver + leaderObserver *observers.LeaderObserver + handoffObserver *observers.HandoffObserver + + balancer balance.Balance +} + +func NewQueryCoord(ctx context.Context, factory dependency.Factory) (*Server, error) { + ctx, cancel := context.WithCancel(ctx) + server := &Server{ + ctx: ctx, + cancel: cancel, + factory: factory, + } + server.UpdateStateCode(internalpb.StateCode_Abnormal) + return server, nil +} + +func (s *Server) Register() error { + s.session.Register() + go s.session.LivenessCheck(s.ctx, func() { + log.Error("QueryCoord disconnected from etcd, process will exit", zap.Int64("serverID", s.session.ServerID)) + if err := s.Stop(); err != nil { + log.Fatal("failed to stop server", zap.Error(err)) + } + // manually send signal to starter goroutine + if s.session.TriggerKill { + if p, err := os.FindProcess(os.Getpid()); err == nil { + p.Signal(syscall.SIGINT) + } + } + }) + return nil +} + +func (s *Server) Init() error { + log.Info("QueryCoord start init", + zap.String("meta-root-path", Params.EtcdCfg.MetaRootPath), + zap.String("address", Params.QueryCoordCfg.Address)) + + // Init QueryCoord session + s.session = sessionutil.NewSession(s.ctx, Params.EtcdCfg.MetaRootPath, s.etcdCli) + if s.session == nil { + return fmt.Errorf("failed to create session") + } + s.session.Init(typeutil.QueryCoordRole, Params.QueryCoordCfg.Address, true, true) + Params.QueryCoordCfg.SetNodeID(s.session.ServerID) + Params.SetLogger(s.session.ServerID) + s.factory.Init(Params) + + // Init KV + etcdKV := etcdkv.NewEtcdKV(s.etcdCli, Params.EtcdCfg.MetaRootPath) + s.kv = etcdKV + log.Debug("query coordinator try to connect etcd success") + + // Init ID allocator + idAllocatorKV := tsoutil.NewTSOKVBase(s.etcdCli, Params.EtcdCfg.KvRootPath, "querycoord-id-allocator") + idAllocator := allocator.NewGlobalIDAllocator("idTimestamp", idAllocatorKV) + err := idAllocator.Initialize() + if err != nil { + log.Error("query coordinator id allocator initialize failed", zap.Error(err)) + return err + } + s.idAllocator = func() (int64, error) { + return idAllocator.AllocOne() + } + + // Init metrics cache manager + s.metricsCacheManager = metricsinfo.NewMetricsCacheManager() + + // Init chunk manager + s.chunkManager, err = s.factory.NewVectorStorageChunkManager(s.ctx) + if err != nil { + log.Error("failed to init chunk manager", zap.Error(err)) + return err + } + + // Init meta + err = s.initMeta() + if err != nil { + return err + } + // Init session + log.Debug("init session") + s.nodeMgr = session.NewNodeManager() + s.cluster = session.NewCluster(s.nodeMgr) + + // Init schedulers + log.Debug("init schedulers") + s.jobScheduler = job.NewScheduler() + s.taskScheduler = task.NewScheduler( + s.ctx, + s.meta, + s.dist, + s.targetMgr, + s.broker, + s.cluster, + s.nodeMgr, + ) + + // Init heartbeat + log.Debug("init dist controller") + s.distController = dist.NewDistController( + s.cluster, + s.nodeMgr, + s.dist, + s.targetMgr, + s.taskScheduler, + ) + + // Init balancer + log.Debug("init balancer") + s.balancer = balance.NewRowCountBasedBalancer( + s.taskScheduler, + s.nodeMgr, + s.dist, + s.meta, + ) + + // Init checker controller + log.Debug("init checker controller") + s.checkerController = checkers.NewCheckerController( + s.meta, + s.dist, + s.targetMgr, + s.balancer, + s.taskScheduler, + ) + + // Init observers + s.initObserver() + + log.Info("QueryCoord init success") + return err +} + +func (s *Server) initMeta() error { + log.Debug("init meta") + s.store = meta.NewMetaStore(s.kv) + s.meta = meta.NewMeta(s.idAllocator, s.store) + + log.Debug("recover meta...") + err := s.meta.CollectionManager.Recover() + if err != nil { + log.Error("failed to recover collections") + return err + } + err = s.meta.ReplicaManager.Recover() + if err != nil { + log.Error("failed to recover replicas") + return err + } + + s.dist = &meta.DistributionManager{ + SegmentDistManager: meta.NewSegmentDistManager(), + ChannelDistManager: meta.NewChannelDistManager(), + LeaderViewManager: meta.NewLeaderViewManager(), + } + s.targetMgr = meta.NewTargetManager() + s.broker = meta.NewCoordinatorBroker( + s.dataCoord, + s.rootCoord, + s.indexCoord, + s.chunkManager, + ) + return nil +} + +func (s *Server) initObserver() { + log.Debug("init observers") + s.collectionObserver = observers.NewCollectionObserver( + s.dist, + s.meta, + s.targetMgr, + ) + s.leaderObserver = observers.NewLeaderObserver( + s.dist, + s.meta, + s.targetMgr, + s.cluster, + ) + s.handoffObserver = observers.NewHandoffObserver( + s.store, + s.meta, + s.dist, + s.targetMgr, + ) +} + +func (s *Server) Start() error { + log.Info("start watcher...") + sessions, revision, err := s.session.GetSessions(typeutil.QueryNodeRole) + if err != nil { + return err + } + for _, node := range sessions { + s.nodeMgr.Add(session.NewNodeInfo(node.ServerID, node.Address)) + } + s.checkReplicas() + for _, node := range sessions { + s.handleNodeUp(node.ServerID) + } + s.wg.Add(1) + go s.watchNodes(revision) + + log.Info("start recovering dist and target") + err = s.recover() + if err != nil { + return err + } + + log.Info("start cluster...") + s.cluster.Start(s.ctx) + + log.Info("start job scheduler...") + s.jobScheduler.Start(s.ctx) + + log.Info("start checker controller...") + s.checkerController.Start(s.ctx) + + log.Info("start observers...") + s.collectionObserver.Start(s.ctx) + s.leaderObserver.Start(s.ctx) + if err := s.handoffObserver.Start(s.ctx); err != nil { + log.Error("start handoff observer failed, exit...", zap.Error(err)) + panic(err.Error()) + } + + s.status.Store(internalpb.StateCode_Healthy) + log.Info("QueryCoord started") + + return nil +} + +func (s *Server) Stop() error { + s.cancel() + s.session.Revoke(time.Second) + + log.Info("stop cluster...") + s.cluster.Stop() + + log.Info("stop dist controller...") + s.distController.Stop() + + log.Info("stop checker controller...") + s.checkerController.Stop() + + log.Info("stop job scheduler...") + s.jobScheduler.Stop() + + log.Info("stop observers...") + s.collectionObserver.Stop() + s.leaderObserver.Stop() + s.handoffObserver.Stop() + + s.wg.Wait() + return nil +} + +// UpdateStateCode updates the status of the coord, including healthy, unhealthy +func (s *Server) UpdateStateCode(code internalpb.StateCode) { + s.status.Store(code) +} + +func (s *Server) GetComponentStates(ctx context.Context) (*internalpb.ComponentStates, error) { + nodeID := common.NotRegisteredID + if s.session != nil && s.session.Registered() { + nodeID = s.session.ServerID + } + serviceComponentInfo := &internalpb.ComponentInfo{ + // NodeID: Params.QueryCoordID, // will race with QueryCoord.Register() + NodeID: nodeID, + StateCode: s.status.Load().(internalpb.StateCode), + } + + return &internalpb.ComponentStates{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + }, + State: serviceComponentInfo, + //SubcomponentStates: subComponentInfos, + }, nil +} + +func (s *Server) GetStatisticsChannel(ctx context.Context) (*milvuspb.StringResponse, error) { + return &milvuspb.StringResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "", + }, + }, nil +} + +func (s *Server) GetTimeTickChannel(ctx context.Context) (*milvuspb.StringResponse, error) { + return &milvuspb.StringResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "", + }, + Value: Params.CommonCfg.QueryCoordTimeTick, + }, nil +} + +// SetEtcdClient sets etcd's client +func (s *Server) SetEtcdClient(etcdClient *clientv3.Client) { + s.etcdCli = etcdClient +} + +// SetRootCoord sets root coordinator's client +func (s *Server) SetRootCoord(rootCoord types.RootCoord) error { + if rootCoord == nil { + return errors.New("null RootCoord interface") + } + + s.rootCoord = rootCoord + return nil +} + +// SetDataCoord sets data coordinator's client +func (s *Server) SetDataCoord(dataCoord types.DataCoord) error { + if dataCoord == nil { + return errors.New("null DataCoord interface") + } + + s.dataCoord = dataCoord + return nil +} + +// SetIndexCoord sets index coordinator's client +func (s *Server) SetIndexCoord(indexCoord types.IndexCoord) error { + if indexCoord == nil { + return errors.New("null IndexCoord interface") + } + + s.indexCoord = indexCoord + return nil +} + +func (s *Server) recover() error { + // Recover target managers + group, ctx := errgroup.WithContext(s.ctx) + for _, collection := range s.meta.GetAll() { + collection := collection + group.Go(func() error { + return s.recoverCollectionTargets(ctx, collection) + }) + } + err := group.Wait() + if err != nil { + return err + } + + // Recover dist + s.distController.SyncAll(s.ctx) + + return nil +} + +func (s *Server) recoverCollectionTargets(ctx context.Context, collection int64) error { + var ( + partitions []int64 + err error + ) + if s.meta.GetLoadType(collection) == querypb.LoadType_LoadCollection { + partitions, err = s.broker.GetPartitions(ctx, collection) + if err != nil { + msg := "failed to get partitions from RootCoord" + log.Error(msg, zap.Error(err)) + return utils.WrapError(msg, err) + } + } else { + partitions = lo.Map(s.meta.GetPartitionsByCollection(collection), func(partition *meta.Partition, _ int) int64 { + return partition.GetPartitionID() + }) + } + + s.handoffObserver.Register(collection) + err = utils.RegisterTargets( + ctx, + s.targetMgr, + s.broker, + collection, + partitions, + ) + if err != nil { + return err + } + s.handoffObserver.StartHandoff(collection) + return nil +} + +func (s *Server) watchNodes(revision int64) { + defer s.wg.Done() + + eventChan := s.session.WatchServices(typeutil.QueryNodeRole, revision+1, nil) + for { + select { + case <-s.ctx.Done(): + log.Info("stop watching nodes, QueryCoord stopped") + return + + case event, ok := <-eventChan: + if !ok { + // ErrCompacted is handled inside SessionWatcher + log.Error("Session Watcher channel closed", zap.Int64("serverID", s.session.ServerID)) + go s.Stop() + if s.session.TriggerKill { + if p, err := os.FindProcess(os.Getpid()); err == nil { + p.Signal(syscall.SIGINT) + } + } + return + } + + switch event.EventType { + case sessionutil.SessionAddEvent: + nodeID := event.Session.ServerID + addr := event.Session.Address + log.Info("add node to NodeManager", + zap.Int64("nodeID", nodeID), + zap.String("nodeAddr", addr), + ) + s.nodeMgr.Add(session.NewNodeInfo(nodeID, addr)) + s.handleNodeUp(nodeID) + s.metricsCacheManager.InvalidateSystemInfoMetrics() + + case sessionutil.SessionDelEvent: + nodeID := event.Session.ServerID + log.Info("a node down, remove it", zap.Int64("nodeID", nodeID)) + s.nodeMgr.Remove(nodeID) + s.handleNodeDown(nodeID) + s.metricsCacheManager.InvalidateSystemInfoMetrics() + } + } + } +} + +func (s *Server) handleNodeUp(node int64) { + log := log.With(zap.Int64("nodeID", node)) + s.distController.StartDistInstance(s.ctx, node) + + for _, collection := range s.meta.CollectionManager.GetAll() { + log := log.With(zap.Int64("collectionID", collection)) + replica := s.meta.ReplicaManager.GetByCollectionAndNode(collection, node) + if replica == nil { + replicas := s.meta.ReplicaManager.GetByCollection(collection) + sort.Slice(replicas, func(i, j int) bool { + return replicas[i].Nodes.Len() < replicas[j].Nodes.Len() + }) + replica := replicas[0] + // TODO(yah01): this may fail, need a component to check whether a node is assigned + err := s.meta.ReplicaManager.AddNode(replica.GetID(), node) + if err != nil { + log.Warn("failed to assign node to replicas", + zap.Int64("replicaID", replica.GetID()), + zap.Error(err), + ) + } + log.Info("assign node to replica", + zap.Int64("replicaID", replica.GetID())) + } + } +} + +func (s *Server) handleNodeDown(node int64) { + log := log.With(zap.Int64("nodeID", node)) + s.distController.Remove(node) + + // Refresh the targets, to avoid consuming messages too early from channel + // FIXME(yah01): the leads to miss data, the segments flushed between the two check points + // are missed, it will recover for a while. + channels := s.dist.ChannelDistManager.GetByNode(node) + for _, channel := range channels { + partitions, err := utils.GetPartitions(s.meta.CollectionManager, + s.broker, + channel.GetCollectionID()) + if err != nil { + log.Warn("failed to refresh targets of collection", + zap.Int64("collectionID", channel.GetCollectionID()), + zap.Error(err)) + } + err = utils.RegisterTargets(s.ctx, + s.targetMgr, + s.broker, + channel.GetCollectionID(), + partitions) + if err != nil { + log.Warn("failed to refresh targets of collection", + zap.Int64("collectionID", channel.GetCollectionID()), + zap.Error(err)) + } + } + + // Clear dist + s.dist.LeaderViewManager.Update(node) + s.dist.ChannelDistManager.Update(node) + s.dist.SegmentDistManager.Update(node) + + // Clear meta + for _, collection := range s.meta.CollectionManager.GetAll() { + log := log.With(zap.Int64("collectionID", collection)) + replica := s.meta.ReplicaManager.GetByCollectionAndNode(collection, node) + if replica == nil { + continue + } + err := s.meta.ReplicaManager.RemoveNode(replica.GetID(), node) + if err != nil { + log.Warn("failed to remove node from collection's replicas", + zap.Int64("replicaID", replica.GetID()), + zap.Error(err), + ) + } + log.Info("remove node from replica", + zap.Int64("replicaID", replica.GetID())) + } + + // Clear tasks + s.taskScheduler.RemoveByNode(node) +} + +// checkReplicas checks whether replica contains offline node, and remove those nodes +func (s *Server) checkReplicas() { + for _, collection := range s.meta.CollectionManager.GetAll() { + log := log.With(zap.Int64("collectionID", collection)) + replicas := s.meta.ReplicaManager.GetByCollection(collection) + for _, replica := range replicas { + replica := replica.Clone() + toRemove := make([]int64, 0) + for node := range replica.Nodes { + if s.nodeMgr.Get(node) == nil { + toRemove = append(toRemove, node) + } + } + log := log.With( + zap.Int64("replicaID", replica.GetID()), + zap.Int64s("offlineNodes", toRemove), + ) + + log.Debug("some nodes are offline, remove them from replica") + if len(toRemove) > 0 { + replica.RemoveNode(toRemove...) + err := s.meta.ReplicaManager.Put(replica) + if err != nil { + log.Warn("failed to remove offline nodes from replica") + } + } + } + } +} diff --git a/internal/querycoordv2/server_test.go b/internal/querycoordv2/server_test.go new file mode 100644 index 0000000000000..76be31ea077ce --- /dev/null +++ b/internal/querycoordv2/server_test.go @@ -0,0 +1,342 @@ +package querycoordv2 + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/proto/schemapb" + "github.com/milvus-io/milvus/internal/querycoordv2/checkers" + "github.com/milvus-io/milvus/internal/querycoordv2/dist" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/mocks" + "github.com/milvus-io/milvus/internal/querycoordv2/observers" + "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/task" + "github.com/milvus-io/milvus/internal/util/dependency" + "github.com/milvus-io/milvus/internal/util/etcd" +) + +type ServerSuite struct { + suite.Suite + + // Data + collections []int64 + partitions map[int64][]int64 + channels map[int64][]string + segments map[int64]map[int64][]int64 // CollectionID, PartitionID -> Segments + loadTypes map[int64]querypb.LoadType + replicaNumber map[int64]int32 + + // Mocks + broker *meta.MockBroker + + server *Server + nodes []*mocks.MockQueryNode +} + +func (suite *ServerSuite) SetupSuite() { + Params.Init() + Params.EtcdCfg = params.GenerateEtcdConfig() + + suite.collections = []int64{1000, 1001} + suite.partitions = map[int64][]int64{ + 1000: {100, 101}, + 1001: {102, 103}, + } + suite.channels = map[int64][]string{ + 1000: {"1000-dmc0", "1000-dmc1"}, + 1001: {"1001-dmc0", "1001-dmc1"}, + } + suite.segments = map[int64]map[int64][]int64{ + 1000: { + 100: {1, 2}, + 101: {3, 4}, + }, + 1001: { + 102: {5, 6}, + 103: {7, 8}, + }, + } + suite.loadTypes = map[int64]querypb.LoadType{ + 1000: querypb.LoadType_LoadCollection, + 1001: querypb.LoadType_LoadPartition, + } + suite.replicaNumber = map[int64]int32{ + 1000: 1, + 1001: 3, + } + suite.nodes = make([]*mocks.MockQueryNode, 3) +} + +func (suite *ServerSuite) SetupTest() { + var err error + + suite.server, err = newQueryCoord() + suite.Require().NoError(err) + suite.hackServer() + err = suite.server.Start() + suite.NoError(err) + + for i := range suite.nodes { + suite.nodes[i] = mocks.NewMockQueryNode(suite.T(), suite.server.etcdCli) + err := suite.nodes[i].Start() + suite.Require().NoError(err) + ok := suite.waitNodeUp(suite.nodes[i], 5*time.Second) + suite.Require().True(ok) + } + + suite.loadAll() + for _, collection := range suite.collections { + suite.assertLoaded(collection) + suite.updateCollectionStatus(collection, querypb.LoadStatus_Loaded) + } +} + +func (suite *ServerSuite) TearDownTest() { + err := suite.server.Stop() + suite.Require().NoError(err) + for _, node := range suite.nodes { + if node != nil { + node.Stop() + } + } +} + +func (suite *ServerSuite) TestRecover() { + err := suite.server.Stop() + suite.NoError(err) + + suite.server, err = newQueryCoord() + suite.NoError(err) + suite.hackServer() + err = suite.server.Start() + suite.NoError(err) + + for _, collection := range suite.collections { + suite.assertLoaded(collection) + } +} + +func (suite *ServerSuite) TestNodeUp() { + newNode := mocks.NewMockQueryNode(suite.T(), suite.server.etcdCli) + newNode.EXPECT().GetDataDistribution(mock.Anything, mock.Anything).Return(&querypb.GetDataDistributionResponse{}, nil) + err := newNode.Start() + suite.NoError(err) + defer newNode.Stop() + + suite.Eventually(func() bool { + node := suite.server.nodeMgr.Get(newNode.ID) + if node == nil { + return false + } + for _, collection := range suite.collections { + replica := suite.server.meta.ReplicaManager.GetByCollectionAndNode(collection, newNode.ID) + if replica == nil { + return false + } + } + return true + }, 5*time.Second, time.Second) + +} + +func (suite *ServerSuite) TestNodeDown() { + downNode := suite.nodes[0] + downNode.Stop() + suite.nodes[0] = nil + + suite.Eventually(func() bool { + node := suite.server.nodeMgr.Get(downNode.ID) + if node != nil { + return false + } + for _, collection := range suite.collections { + replica := suite.server.meta.ReplicaManager.GetByCollectionAndNode(collection, downNode.ID) + if replica != nil { + return false + } + } + return true + }, 5*time.Second, time.Second) +} + +func (suite *ServerSuite) waitNodeUp(node *mocks.MockQueryNode, timeout time.Duration) bool { + start := time.Now() + for time.Since(start) < timeout { + if suite.server.nodeMgr.Get(node.ID) != nil { + return true + } + time.Sleep(500 * time.Millisecond) + } + return false +} + +func (suite *ServerSuite) loadAll() { + ctx := context.Background() + for _, collection := range suite.collections { + if suite.loadTypes[collection] == querypb.LoadType_LoadCollection { + req := &querypb.LoadCollectionRequest{ + CollectionID: collection, + ReplicaNumber: suite.replicaNumber[collection], + } + resp, err := suite.server.LoadCollection(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.ErrorCode) + } else { + req := &querypb.LoadPartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + ReplicaNumber: suite.replicaNumber[collection], + } + resp, err := suite.server.LoadPartitions(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.ErrorCode) + } + } +} + +func (suite *ServerSuite) assertLoaded(collection int64) { + suite.True(suite.server.meta.Exist(collection)) + for _, channel := range suite.channels[collection] { + suite.NotNil(suite.server.targetMgr.GetDmChannel(channel)) + } + for _, partitions := range suite.segments[collection] { + for _, segment := range partitions { + suite.NotNil(suite.server.targetMgr.GetSegment(segment)) + } + } +} + +func (suite *ServerSuite) expectGetRecoverInfo(collection int64) { + var ( + mu sync.Mutex + vChannels []*datapb.VchannelInfo + segmentBinlogs []*datapb.SegmentBinlogs + ) + + for partition, segments := range suite.segments[collection] { + segments := segments + suite.broker.EXPECT().GetRecoveryInfo(mock.Anything, collection, partition).Maybe().Return(func(ctx context.Context, collectionID, partitionID int64) []*datapb.VchannelInfo { + mu.Lock() + vChannels = []*datapb.VchannelInfo{} + for _, channel := range suite.channels[collection] { + vChannels = append(vChannels, &datapb.VchannelInfo{ + CollectionID: collection, + ChannelName: channel, + }) + } + segmentBinlogs = []*datapb.SegmentBinlogs{} + for _, segment := range segments { + segmentBinlogs = append(segmentBinlogs, &datapb.SegmentBinlogs{ + SegmentID: segment, + InsertChannel: suite.channels[collection][segment%2], + }) + } + return vChannels + }, + func(ctx context.Context, collectionID, partitionID int64) []*datapb.SegmentBinlogs { + return segmentBinlogs + }, + func(ctx context.Context, collectionID, partitionID int64) error { + mu.Unlock() + return nil + }, + ) + } +} + +func (suite *ServerSuite) updateCollectionStatus(collectionID int64, status querypb.LoadStatus) { + collection := suite.server.meta.GetCollection(collectionID) + if collection != nil { + collection := collection.Clone() + collection.LoadPercentage = 0 + if status == querypb.LoadStatus_Loaded { + collection.LoadPercentage = 100 + } + collection.CollectionLoadInfo.Status = status + suite.server.meta.UpdateCollection(collection) + } else { + partitions := suite.server.meta.GetPartitionsByCollection(collectionID) + for _, partition := range partitions { + partition := partition.Clone() + partition.LoadPercentage = 0 + if status == querypb.LoadStatus_Loaded { + partition.LoadPercentage = 100 + } + partition.PartitionLoadInfo.Status = status + suite.server.meta.UpdatePartition(partition) + } + } +} + +func (suite *ServerSuite) hackServer() { + suite.broker = meta.NewMockBroker(suite.T()) + suite.server.broker = suite.broker + suite.server.taskScheduler = task.NewScheduler( + suite.server.ctx, + suite.server.meta, + suite.server.dist, + suite.server.targetMgr, + suite.broker, + suite.server.cluster, + suite.server.nodeMgr, + ) + suite.server.handoffObserver = observers.NewHandoffObserver( + suite.server.store, + suite.server.meta, + suite.server.dist, + suite.server.targetMgr, + ) + suite.server.distController = dist.NewDistController( + suite.server.cluster, + suite.server.nodeMgr, + suite.server.dist, + suite.server.targetMgr, + suite.server.taskScheduler, + ) + suite.server.checkerController = checkers.NewCheckerController( + suite.server.meta, + suite.server.dist, + suite.server.targetMgr, + suite.server.balancer, + suite.server.taskScheduler, + ) + + suite.broker.EXPECT().GetCollectionSchema(mock.Anything, mock.Anything).Return(&schemapb.CollectionSchema{}, nil).Maybe() + for _, collection := range suite.collections { + if suite.loadTypes[collection] == querypb.LoadType_LoadCollection { + suite.broker.EXPECT().GetPartitions(mock.Anything, collection).Return(suite.partitions[collection], nil).Maybe() + } + suite.expectGetRecoverInfo(collection) + } + log.Debug("server hacked") +} + +func newQueryCoord() (*Server, error) { + server, err := NewQueryCoord(context.Background(), dependency.NewDefaultFactory(true)) + if err != nil { + return nil, err + } + + etcdCli, err := etcd.GetEtcdClient(&Params.EtcdCfg) + if err != nil { + return nil, err + } + server.SetEtcdClient(etcdCli) + + err = server.Init() + return server, err +} + +func TestServer(t *testing.T) { + suite.Run(t, new(ServerSuite)) +} diff --git a/internal/querycoordv2/services.go b/internal/querycoordv2/services.go new file mode 100644 index 0000000000000..5d7d539b359a5 --- /dev/null +++ b/internal/querycoordv2/services.go @@ -0,0 +1,686 @@ +package querycoordv2 + +import ( + "context" + "errors" + "fmt" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/metrics" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/internalpb" + "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/job" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/internal/util/metricsinfo" + "github.com/milvus-io/milvus/internal/util/timerecord" + "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/samber/lo" + "go.uber.org/zap" +) + +var ( + successStatus = utils.WrapStatus(commonpb.ErrorCode_Success, "") +) + +func (s *Server) ShowCollections(ctx context.Context, req *querypb.ShowCollectionsRequest) (*querypb.ShowCollectionsResponse, error) { + log := log.With(zap.Int64("msgID", req.GetBase().GetMsgID())) + + log.Info("show collections request received", zap.Int64s("collections", req.GetCollectionIDs())) + + if s.status.Load() != internalpb.StateCode_Healthy { + msg := "failed to show collections" + log.Warn(msg, zap.Error(ErrNotHealthy)) + return &querypb.ShowCollectionsResponse{ + Status: utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, ErrNotHealthy), + }, nil + } + + collectionSet := typeutil.NewUniqueSet(req.GetCollectionIDs()...) + if len(req.GetCollectionIDs()) == 0 { + for _, collection := range s.meta.GetAllCollections() { + collectionSet.Insert(collection.GetCollectionID()) + } + for _, partition := range s.meta.GetAllPartitions() { + collectionSet.Insert(partition.GetCollectionID()) + } + } + collections := collectionSet.Collect() + + resp := &querypb.ShowCollectionsResponse{ + Status: successStatus, + CollectionIDs: collections, + InMemoryPercentages: make([]int64, len(collectionSet)), + QueryServiceAvailable: make([]bool, len(collectionSet)), + } + for i, collectionID := range collections { + log := log.With(zap.Int64("collectionID", collectionID)) + + percentage := s.meta.CollectionManager.GetLoadPercentage(collectionID) + if percentage < 0 { + err := fmt.Errorf("collection %d has not been loaded to memory or load failed", collectionID) + log.Warn("show collection failed", zap.Error(err)) + return &querypb.ShowCollectionsResponse{ + Status: utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, err.Error()), + }, nil + } + resp.InMemoryPercentages[i] = int64(percentage) + resp.QueryServiceAvailable[i] = s.checkAnyReplicaAvailable(collectionID) + } + + return resp, nil +} + +func (s *Server) ShowPartitions(ctx context.Context, req *querypb.ShowPartitionsRequest) (*querypb.ShowPartitionsResponse, error) { + log := log.With( + zap.Int64("msgID", req.GetBase().GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + log.Info("show partitions", zap.Int64s("partitions", req.GetPartitionIDs())) + + if s.status.Load() != internalpb.StateCode_Healthy { + msg := "failed to show partitions" + log.Warn(msg, zap.Error(ErrNotHealthy)) + return &querypb.ShowPartitionsResponse{ + Status: utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, ErrNotHealthy), + }, nil + } + + // TODO(yah01): now, for load collection, the percentage of partition is equal to the percentage of collection, + // we can calculates the real percentage of partitions + partitions := req.GetPartitionIDs() + percentages := make([]int64, 0) + isReleased := false + switch s.meta.GetLoadType(req.GetCollectionID()) { + case querypb.LoadType_LoadCollection: + percentage := s.meta.GetLoadPercentage(req.GetCollectionID()) + if percentage < 0 { + isReleased = true + break + } + + if len(partitions) == 0 { + var err error + partitions, err = s.broker.GetPartitions(ctx, req.GetCollectionID()) + if err != nil { + msg := "failed to show partitions" + log.Warn(msg, zap.Error(err)) + return &querypb.ShowPartitionsResponse{ + Status: utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, err), + }, nil + } + } + for range partitions { + percentages = append(percentages, int64(percentage)) + } + + case querypb.LoadType_LoadPartition: + if len(partitions) == 0 { + partitions = lo.Map(s.meta.GetPartitionsByCollection(req.GetCollectionID()), func(partition *meta.Partition, _ int) int64 { + return partition.GetPartitionID() + }) + } + for _, partitionID := range partitions { + partition := s.meta.GetPartition(partitionID) + if partition == nil { + isReleased = true + break + } + percentages = append(percentages, int64(partition.LoadPercentage)) + } + + default: + isReleased = true + } + + if isReleased { + msg := fmt.Sprintf("collection %v has not been loaded into QueryNode", req.GetCollectionID()) + log.Warn(msg) + return &querypb.ShowPartitionsResponse{ + Status: utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg), + }, nil + } + + return &querypb.ShowPartitionsResponse{ + Status: successStatus, + PartitionIDs: partitions, + InMemoryPercentages: percentages, + }, nil +} + +func (s *Server) LoadCollection(ctx context.Context, req *querypb.LoadCollectionRequest) (*commonpb.Status, error) { + log := log.With( + zap.Int64("msgID", req.GetBase().GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + log.Info("load collection request received", + zap.Any("schema", req.Schema), + zap.Int32("replicaNumber", req.ReplicaNumber)) + metrics.QueryCoordLoadCount.WithLabelValues(metrics.TotalLabel).Inc() + + if s.status.Load() != internalpb.StateCode_Healthy { + msg := "failed to load collection" + log.Warn(msg, zap.Error(ErrNotHealthy)) + metrics.QueryCoordLoadCount.WithLabelValues(metrics.FailLabel).Inc() + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, ErrNotHealthy), nil + } + + loadJob := job.NewLoadCollectionJob(ctx, + req, + s.dist, + s.meta, + s.targetMgr, + s.broker, + s.nodeMgr, + s.handoffObserver, + ) + s.jobScheduler.Add(loadJob) + err := loadJob.Wait() + if err != nil && !errors.Is(err, job.ErrCollectionLoaded) { + msg := "failed to load collection" + log.Warn(msg, zap.Error(err)) + metrics.QueryCoordLoadCount.WithLabelValues(metrics.FailLabel).Inc() + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, err), nil + } + + metrics.QueryCoordLoadCount.WithLabelValues(metrics.SuccessLabel).Inc() + return successStatus, nil +} + +func (s *Server) ReleaseCollection(ctx context.Context, req *querypb.ReleaseCollectionRequest) (*commonpb.Status, error) { + log := log.With( + zap.Int64("msgID", req.GetBase().GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + log.Info("release collection request received") + metrics.QueryCoordReleaseCount.WithLabelValues(metrics.TotalLabel).Inc() + tr := timerecord.NewTimeRecorder("release-collection") + + if s.status.Load() != internalpb.StateCode_Healthy { + msg := "failed to release collection" + log.Warn(msg, zap.Error(ErrNotHealthy)) + metrics.QueryCoordReleaseCount.WithLabelValues(metrics.FailLabel).Inc() + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, ErrNotHealthy), nil + } + + releaseJob := job.NewReleaseCollectionJob(ctx, + req, + s.dist, + s.meta, + s.targetMgr, + s.handoffObserver, + ) + s.jobScheduler.Add(releaseJob) + err := releaseJob.Wait() + if err != nil { + msg := "failed to release collection" + log.Error(msg, zap.Error(err)) + metrics.QueryCoordReleaseCount.WithLabelValues(metrics.FailLabel).Inc() + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, err), nil + } + + log.Info("collection released") + metrics.QueryCoordReleaseCount.WithLabelValues(metrics.SuccessLabel).Inc() + metrics.QueryCoordReleaseLatency.WithLabelValues().Observe(float64(tr.ElapseSpan().Milliseconds())) + return successStatus, nil +} + +func (s *Server) LoadPartitions(ctx context.Context, req *querypb.LoadPartitionsRequest) (*commonpb.Status, error) { + log := log.With( + zap.Int64("msgID", req.GetBase().GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + log.Info("received load partitions request", + zap.Any("schema", req.Schema), + zap.Int32("replicaNumber", req.ReplicaNumber), + zap.Int64s("partitions", req.GetPartitionIDs())) + metrics.QueryCoordLoadCount.WithLabelValues(metrics.TotalLabel).Inc() + + if s.status.Load() != internalpb.StateCode_Healthy { + msg := "failed to load partitions" + log.Warn(msg, zap.Error(ErrNotHealthy)) + metrics.QueryCoordLoadCount.WithLabelValues(metrics.FailLabel).Inc() + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, ErrNotHealthy), nil + } + + loadJob := job.NewLoadPartitionJob(ctx, + req, + s.dist, + s.meta, + s.targetMgr, + s.broker, + s.nodeMgr, + s.handoffObserver, + ) + s.jobScheduler.Add(loadJob) + err := loadJob.Wait() + if err != nil && !errors.Is(err, job.ErrCollectionLoaded) { + msg := "failed to load partitions" + log.Warn(msg, zap.Error(err)) + metrics.QueryCoordLoadCount.WithLabelValues(metrics.FailLabel).Inc() + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, err), nil + } + + metrics.QueryCoordLoadCount.WithLabelValues(metrics.SuccessLabel).Inc() + return successStatus, nil +} + +func (s *Server) ReleasePartitions(ctx context.Context, req *querypb.ReleasePartitionsRequest) (*commonpb.Status, error) { + log := log.With( + zap.Int64("msgID", req.GetBase().GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + log.Info("release partitions", zap.Int64s("partitions", req.GetPartitionIDs())) + metrics.QueryCoordReleaseCount.WithLabelValues(metrics.TotalLabel).Inc() + + if s.status.Load() != internalpb.StateCode_Healthy { + msg := "failed to release partitions" + log.Warn(msg, zap.Error(ErrNotHealthy)) + metrics.QueryCoordReleaseCount.WithLabelValues(metrics.FailLabel).Inc() + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, ErrNotHealthy), nil + } + + if len(req.GetPartitionIDs()) == 0 { + msg := "partitions is empty" + log.Warn(msg) + metrics.QueryCoordReleaseCount.WithLabelValues(metrics.FailLabel).Inc() + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg), nil + } + + tr := timerecord.NewTimeRecorder("release-partitions") + releaseJob := job.NewReleasePartitionJob(ctx, + req, + s.dist, + s.meta, + s.targetMgr, + s.handoffObserver, + ) + s.jobScheduler.Add(releaseJob) + err := releaseJob.Wait() + if err != nil { + msg := "failed to release partitions" + log.Error(msg, zap.Error(err)) + metrics.QueryCoordReleaseCount.WithLabelValues(metrics.FailLabel).Inc() + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, err), nil + } + + metrics.QueryCoordReleaseCount.WithLabelValues(metrics.SuccessLabel).Inc() + metrics.QueryCoordReleaseLatency.WithLabelValues().Observe(float64(tr.ElapseSpan().Milliseconds())) + return successStatus, nil +} + +func (s *Server) GetPartitionStates(ctx context.Context, req *querypb.GetPartitionStatesRequest) (*querypb.GetPartitionStatesResponse, error) { + log := log.With( + zap.Int64("msgID", req.GetBase().GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + log.Info("get partition states", zap.Int64s("partitions", req.GetPartitionIDs())) + + if s.status.Load() != internalpb.StateCode_Healthy { + msg := "failed to get partition states" + log.Warn(msg, zap.Error(ErrNotHealthy)) + return &querypb.GetPartitionStatesResponse{ + Status: utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, ErrNotHealthy), + }, nil + } + + msg := "partition not loaded" + notLoadResp := &querypb.GetPartitionStatesResponse{ + Status: utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg), + } + + states := make([]*querypb.PartitionStates, 0, len(req.GetPartitionIDs())) + switch s.meta.GetLoadType(req.GetCollectionID()) { + case querypb.LoadType_LoadCollection: + collection := s.meta.GetCollection(req.GetCollectionID()) + state := querypb.PartitionState_PartialInMemory + if collection.LoadPercentage >= 100 { + state = querypb.PartitionState_InMemory + } + releasedPartitions := typeutil.NewUniqueSet(collection.GetReleasedPartitions()...) + for _, partition := range req.GetPartitionIDs() { + if releasedPartitions.Contain(partition) { + log.Warn(msg) + return notLoadResp, nil + } + states = append(states, &querypb.PartitionStates{ + PartitionID: partition, + State: state, + }) + } + + case querypb.LoadType_LoadPartition: + for _, partitionID := range req.GetPartitionIDs() { + partition := s.meta.GetPartition(partitionID) + if partition == nil { + log.Warn(msg, zap.Int64("partition", partitionID)) + return notLoadResp, nil + } + state := querypb.PartitionState_PartialInMemory + if partition.LoadPercentage >= 100 { + state = querypb.PartitionState_InMemory + } + states = append(states, &querypb.PartitionStates{ + PartitionID: partitionID, + State: state, + }) + } + + default: + log.Warn(msg) + return notLoadResp, nil + } + + return &querypb.GetPartitionStatesResponse{ + Status: successStatus, + PartitionDescriptions: states, + }, nil +} + +func (s *Server) GetSegmentInfo(ctx context.Context, req *querypb.GetSegmentInfoRequest) (*querypb.GetSegmentInfoResponse, error) { + log := log.With( + zap.Int64("msgID", req.GetBase().GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + log.Info("get segment info", zap.Int64s("segments", req.GetSegmentIDs())) + + if s.status.Load() != internalpb.StateCode_Healthy { + msg := "failed to get segment info" + log.Warn(msg, zap.Error(ErrNotHealthy)) + return &querypb.GetSegmentInfoResponse{ + Status: utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, ErrNotHealthy), + }, nil + } + + infos := make([]*querypb.SegmentInfo, 0, len(req.GetSegmentIDs())) + if len(req.GetSegmentIDs()) == 0 { + infos = s.getCollectionSegmentInfo(req.GetCollectionID()) + } else { + for _, segmentID := range req.GetSegmentIDs() { + segments := s.dist.SegmentDistManager.Get(segmentID) + if len(segments) == 0 { + msg := fmt.Sprintf("segment %v not found in any node", segmentID) + log.Warn(msg, zap.Int64("segment", segmentID)) + return &querypb.GetSegmentInfoResponse{ + Status: utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg), + }, nil + } + info := &querypb.SegmentInfo{} + utils.MergeMetaSegmentIntoSegmentInfo(info, segments...) + infos = append(infos, info) + } + } + + return &querypb.GetSegmentInfoResponse{ + Status: successStatus, + Infos: infos, + }, nil +} + +func (s *Server) LoadBalance(ctx context.Context, req *querypb.LoadBalanceRequest) (*commonpb.Status, error) { + log := log.With( + zap.Int64("msgID", req.GetBase().GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + log.Info("load balance request received", + zap.Int64s("source", req.GetSourceNodeIDs()), + zap.Int64s("dest", req.GetDstNodeIDs()), + zap.Int64s("segments", req.GetSealedSegmentIDs())) + + if s.status.Load() != internalpb.StateCode_Healthy { + msg := "failed to load balance" + log.Warn(msg, zap.Error(ErrNotHealthy)) + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, ErrNotHealthy), nil + } + + // Verify request + if len(req.GetSourceNodeIDs()) != 1 { + msg := "source nodes can only contain 1 node" + log.Warn(msg, zap.Int("source-nodes-num", len(req.GetSourceNodeIDs()))) + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg), nil + } + if s.meta.CollectionManager.GetLoadPercentage(req.GetCollectionID()) < 100 { + msg := "can't balance segments of not fully loaded collection" + log.Warn(msg) + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg), nil + } + srcNode := req.GetSourceNodeIDs()[0] + replica := s.meta.ReplicaManager.GetByCollectionAndNode(req.GetCollectionID(), srcNode) + if replica == nil { + msg := "source node not in any replica" + log.Warn(msg) + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg), nil + } + for _, dstNode := range req.GetDstNodeIDs() { + if !replica.Nodes.Contain(dstNode) { + msg := "destination nodes have to be in the same replica of source node" + log.Warn(msg) + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg), nil + } + } + + err := s.balanceSegments(ctx, req, replica) + if err != nil { + msg := "failed to balance segments" + log.Warn(msg, zap.Error(err)) + return utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, err), nil + } + return successStatus, nil +} + +func (s *Server) ShowConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) { + log := log.With( + zap.Int64("msgID", req.GetBase().GetMsgID()), + ) + + log.Debug("show configurations request received", zap.String("pattern", req.GetPattern())) + + if s.status.Load() != internalpb.StateCode_Healthy { + msg := "failed to show configurations" + log.Warn(msg, zap.Error(ErrNotHealthy)) + return &internalpb.ShowConfigurationsResponse{ + Status: utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, ErrNotHealthy), + }, nil + } + + prefix := "querycoord." + matchedConfig := Params.QueryCoordCfg.Base.GetByPattern(prefix + req.Pattern) + configList := make([]*commonpb.KeyValuePair, 0, len(matchedConfig)) + for key, value := range matchedConfig { + configList = append(configList, + &commonpb.KeyValuePair{ + Key: key, + Value: value, + }) + } + + return &internalpb.ShowConfigurationsResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "", + }, + Configuations: configList, + }, nil +} + +func (s *Server) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) { + log := log.With(zap.Int64("msgID", req.Base.GetMsgID())) + + log.Info("get metrics request received", + zap.String("metricType", req.GetRequest())) + + if s.status.Load() != internalpb.StateCode_Healthy { + msg := "failed to get metrics" + log.Warn(msg, zap.Error(ErrNotHealthy)) + return &milvuspb.GetMetricsResponse{ + Status: utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, ErrNotHealthy), + }, nil + } + + resp := &milvuspb.GetMetricsResponse{ + Status: successStatus, + ComponentName: metricsinfo.ConstructComponentName(typeutil.QueryCoordRole, + Params.QueryCoordCfg.GetNodeID()), + } + + metricType, err := metricsinfo.ParseMetricType(req.GetRequest()) + if err != nil { + msg := "failed to parse metric type" + log.Warn(msg, zap.Error(err)) + resp.Status = utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, err) + return resp, nil + } + + if metricType != metricsinfo.SystemInfoMetrics { + msg := "invalid metric type" + err := errors.New(metricsinfo.MsgUnimplementedMetric) + log.Warn(msg, zap.Error(err)) + resp.Status = utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, err) + return resp, nil + } + + metrics, err := s.metricsCacheManager.GetSystemInfoMetrics() + if err != nil { + log.Warn("failed to read metrics from cache, re-calculate it", zap.Error(err)) + metrics = resp + metrics.Response, err = s.getSystemInfoMetrics(ctx, req) + if err != nil { + msg := "failed to get system info metrics" + log.Warn(msg, zap.Error(err)) + resp.Status = utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, err) + return resp, nil + } + } + + s.metricsCacheManager.UpdateSystemInfoMetrics(metrics) + return metrics, nil +} + +func (s *Server) GetReplicas(ctx context.Context, req *milvuspb.GetReplicasRequest) (*milvuspb.GetReplicasResponse, error) { + log := log.With( + zap.Int64("msgID", req.Base.GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + log.Info("get replicas request received", zap.Bool("with-shard-nodes", req.GetWithShardNodes())) + + if s.status.Load() != internalpb.StateCode_Healthy { + msg := "failed to get replicas" + log.Warn(msg, zap.Error(ErrNotHealthy)) + return &milvuspb.GetReplicasResponse{ + Status: utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, ErrNotHealthy), + }, nil + } + + resp := &milvuspb.GetReplicasResponse{ + Status: successStatus, + Replicas: make([]*milvuspb.ReplicaInfo, 0), + } + + replicas := s.meta.ReplicaManager.GetByCollection(req.GetCollectionID()) + if len(replicas) == 0 { + msg := "failed to get replicas, collection not loaded" + log.Warn(msg) + resp.Status = utils.WrapStatus(commonpb.ErrorCode_MetaFailed, msg) + return resp, nil + } + + for _, replica := range replicas { + info, err := s.fillReplicaInfo(replica, req.GetWithShardNodes()) + if err != nil { + msg := "failed to get replica info" + log.Warn(msg, + zap.Int64("replica", replica.GetID()), + zap.Error(err)) + resp.Status = utils.WrapStatus(commonpb.ErrorCode_MetaFailed, msg, err) + } + resp.Replicas = append(resp.Replicas, info) + } + return resp, nil +} + +func (s *Server) GetShardLeaders(ctx context.Context, req *querypb.GetShardLeadersRequest) (*querypb.GetShardLeadersResponse, error) { + log := log.With( + zap.Int64("msgID", req.Base.GetMsgID()), + zap.Int64("collectionID", req.GetCollectionID()), + ) + + log.Info("get shard leaders request received") + if s.status.Load() != internalpb.StateCode_Healthy { + msg := "failed to get shard leaders" + log.Warn(msg, zap.Error(ErrNotHealthy)) + return &querypb.GetShardLeadersResponse{ + Status: utils.WrapStatus(commonpb.ErrorCode_UnexpectedError, msg, ErrNotHealthy), + }, nil + } + + resp := &querypb.GetShardLeadersResponse{ + Status: successStatus, + } + + if s.meta.CollectionManager.GetLoadPercentage(req.GetCollectionID()) < 100 { + msg := fmt.Sprintf("collection %v is not fully loaded", req.GetCollectionID()) + log.Warn(msg) + resp.Status = utils.WrapStatus(commonpb.ErrorCode_NoReplicaAvailable, msg) + return resp, nil + } + + channels := s.targetMgr.GetDmChannelsByCollection(req.GetCollectionID()) + if len(channels) == 0 { + msg := "failed to get channels" + log.Warn(msg, zap.Error(meta.ErrCollectionNotFound)) + resp.Status = utils.WrapStatus(commonpb.ErrorCode_MetaFailed, msg, meta.ErrCollectionNotFound) + return resp, nil + } + + for _, channel := range channels { + log := log.With(zap.String("channel", channel.GetChannelName())) + + leaders := s.dist.LeaderViewManager.GetLeadersByShard(channel.GetChannelName()) + ids := make([]int64, 0, len(leaders)) + addrs := make([]string, 0, len(leaders)) + for _, leader := range leaders { + info := s.nodeMgr.Get(leader.ID) + if info == nil { + continue + } + isAllNodeAvailable := true + for _, node := range leader.Segments { + if s.nodeMgr.Get(node) == nil { + isAllNodeAvailable = false + break + } + } + if !isAllNodeAvailable { + continue + } + ids = append(ids, info.ID()) + addrs = append(addrs, info.Addr()) + } + + if len(ids) == 0 { + msg := fmt.Sprintf("channel %s is not available in any replica", channel.GetChannelName()) + log.Warn(msg) + resp.Status = utils.WrapStatus(commonpb.ErrorCode_NoReplicaAvailable, msg) + resp.Shards = nil + return resp, nil + } + + resp.Shards = append(resp.Shards, &querypb.ShardLeadersList{ + ChannelName: channel.GetChannelName(), + NodeIds: ids, + NodeAddrs: addrs, + }) + } + return resp, nil +} diff --git a/internal/querycoordv2/services_test.go b/internal/querycoordv2/services_test.go new file mode 100644 index 0000000000000..a07c038d6fd37 --- /dev/null +++ b/internal/querycoordv2/services_test.go @@ -0,0 +1,990 @@ +package querycoordv2 + +import ( + "context" + "encoding/json" + "testing" + + "github.com/milvus-io/milvus/internal/kv" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/internalpb" + "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/balance" + "github.com/milvus-io/milvus/internal/querycoordv2/job" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/observers" + "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/task" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/internal/util/etcd" + "github.com/milvus-io/milvus/internal/util/metricsinfo" + "github.com/milvus-io/milvus/internal/util/sessionutil" + "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type ServiceSuite struct { + suite.Suite + + // Data + collections []int64 + partitions map[int64][]int64 + channels map[int64][]string + segments map[int64]map[int64][]int64 // CollectionID, PartitionID -> Segments + loadTypes map[int64]querypb.LoadType + replicaNumber map[int64]int32 + nodes []int64 + + // Dependencies + kv kv.MetaKv + store meta.Store + dist *meta.DistributionManager + meta *meta.Meta + targetMgr *meta.TargetManager + broker *meta.MockBroker + cluster *session.MockCluster + nodeMgr *session.NodeManager + jobScheduler *job.Scheduler + taskScheduler *task.MockScheduler + handoffObserver *observers.HandoffObserver + balancer balance.Balance + + // Test object + server *Server +} + +func (suite *ServiceSuite) SetupSuite() { + Params.Init() + + suite.collections = []int64{1000, 1001} + suite.partitions = map[int64][]int64{ + 1000: {100, 101}, + 1001: {102, 103}, + } + suite.channels = map[int64][]string{ + 1000: {"1000-dmc0", "1000-dmc1"}, + 1001: {"1001-dmc0", "1001-dmc1"}, + } + suite.segments = map[int64]map[int64][]int64{ + 1000: { + 100: {1, 2}, + 101: {3, 4}, + }, + 1001: { + 102: {5, 6}, + 103: {7, 8}, + }, + } + suite.loadTypes = map[int64]querypb.LoadType{ + 1000: querypb.LoadType_LoadCollection, + 1001: querypb.LoadType_LoadPartition, + } + suite.replicaNumber = map[int64]int32{ + 1000: 1, + 1001: 3, + } + suite.nodes = []int64{1, 2, 3, 4, 5, + 101, 102, 103, 104, 105} +} + +func (suite *ServiceSuite) SetupTest() { + config := params.GenerateEtcdConfig() + cli, err := etcd.GetEtcdClient(&config) + suite.Require().NoError(err) + suite.kv = etcdkv.NewEtcdKV(cli, config.MetaRootPath) + + suite.store = meta.NewMetaStore(suite.kv) + suite.dist = meta.NewDistributionManager() + suite.meta = meta.NewMeta(params.RandomIncrementIDAllocator(), suite.store) + suite.targetMgr = meta.NewTargetManager() + suite.broker = meta.NewMockBroker(suite.T()) + suite.nodeMgr = session.NewNodeManager() + for _, node := range suite.nodes { + suite.nodeMgr.Add(session.NewNodeInfo(node, "localhost")) + } + suite.cluster = session.NewMockCluster(suite.T()) + suite.jobScheduler = job.NewScheduler() + suite.taskScheduler = task.NewMockScheduler(suite.T()) + suite.jobScheduler.Start(context.Background()) + suite.handoffObserver = observers.NewHandoffObserver( + suite.store, + suite.meta, + suite.dist, + suite.targetMgr, + ) + suite.balancer = balance.NewRowCountBasedBalancer( + suite.taskScheduler, + suite.nodeMgr, + suite.dist, + suite.meta, + ) + + suite.server = &Server{ + kv: suite.kv, + store: suite.store, + session: sessionutil.NewSession(context.Background(), Params.EtcdCfg.MetaRootPath, cli), + metricsCacheManager: metricsinfo.NewMetricsCacheManager(), + dist: suite.dist, + meta: suite.meta, + targetMgr: suite.targetMgr, + broker: suite.broker, + nodeMgr: suite.nodeMgr, + cluster: suite.cluster, + jobScheduler: suite.jobScheduler, + taskScheduler: suite.taskScheduler, + balancer: suite.balancer, + handoffObserver: suite.handoffObserver, + } + suite.server.UpdateStateCode(internalpb.StateCode_Healthy) +} + +func (suite *ServiceSuite) TestShowCollections() { + suite.loadAll() + ctx := context.Background() + server := suite.server + collectionNum := len(suite.collections) + + // Test get all collections + req := &querypb.ShowCollectionsRequest{} + resp, err := server.ShowCollections(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.Status.ErrorCode) + suite.Len(resp.CollectionIDs, collectionNum) + for _, collection := range suite.collections { + suite.Contains(resp.CollectionIDs, collection) + } + + // Test get 1 collection + collection := suite.collections[0] + req.CollectionIDs = []int64{collection} + resp, err = server.ShowCollections(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.Status.ErrorCode) + suite.Len(resp.CollectionIDs, 1) + suite.Equal(collection, resp.CollectionIDs[0]) + + // Test when server is not healthy + server.UpdateStateCode(internalpb.StateCode_Initializing) + resp, err = server.ShowCollections(ctx, req) + suite.NoError(err) + suite.Contains(resp.Status.Reason, ErrNotHealthy.Error()) +} + +func (suite *ServiceSuite) TestShowPartitions() { + suite.loadAll() + ctx := context.Background() + server := suite.server + + for _, collection := range suite.collections { + partitions := suite.partitions[collection] + partitionNum := len(partitions) + + // Test get all partitions + req := &querypb.ShowPartitionsRequest{ + CollectionID: collection, + } + resp, err := server.ShowPartitions(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.Status.ErrorCode) + suite.Len(resp.PartitionIDs, partitionNum) + for _, partition := range partitions { + suite.Contains(resp.PartitionIDs, partition) + } + + // Test get 1 partition + req = &querypb.ShowPartitionsRequest{ + CollectionID: collection, + PartitionIDs: partitions[0:1], + } + resp, err = server.ShowPartitions(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.Status.ErrorCode) + suite.Len(resp.PartitionIDs, 1) + for _, partition := range partitions[0:1] { + suite.Contains(resp.PartitionIDs, partition) + } + } + + // Test when server is not healthy + req := &querypb.ShowPartitionsRequest{ + CollectionID: suite.collections[0], + } + server.UpdateStateCode(internalpb.StateCode_Initializing) + resp, err := server.ShowPartitions(ctx, req) + suite.NoError(err) + suite.Contains(resp.Status.Reason, ErrNotHealthy.Error()) +} + +func (suite *ServiceSuite) TestLoadCollection() { + ctx := context.Background() + server := suite.server + + // Test load all collections + for _, collection := range suite.collections { + suite.broker.EXPECT().GetPartitions(mock.Anything, collection).Return(suite.partitions[collection], nil) + suite.expectGetRecoverInfo(collection) + + req := &querypb.LoadCollectionRequest{ + CollectionID: collection, + } + resp, err := server.LoadCollection(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.ErrorCode) + suite.assertLoaded(collection) + } + + // Test load again + for _, collection := range suite.collections { + req := &querypb.LoadCollectionRequest{ + CollectionID: collection, + } + resp, err := server.LoadCollection(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.ErrorCode) + } + + // Test when server is not healthy + server.UpdateStateCode(internalpb.StateCode_Initializing) + req := &querypb.LoadCollectionRequest{ + CollectionID: suite.collections[0], + } + resp, err := server.LoadCollection(ctx, req) + suite.NoError(err) + suite.Contains(resp.Reason, ErrNotHealthy.Error()) +} + +func (suite *ServiceSuite) TestLoadCollectionFailed() { + suite.loadAll() + ctx := context.Background() + server := suite.server + + // Test load with different replica number + for _, collection := range suite.collections { + req := &querypb.LoadCollectionRequest{ + CollectionID: collection, + ReplicaNumber: suite.replicaNumber[collection] + 1, + } + resp, err := server.LoadCollection(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_UnexpectedError, resp.ErrorCode) + suite.Contains(resp.Reason, job.ErrLoadParameterMismatched.Error()) + } + + // Test load with partitions loaded + for _, collection := range suite.collections { + if suite.loadTypes[collection] != querypb.LoadType_LoadPartition { + continue + } + + req := &querypb.LoadCollectionRequest{ + CollectionID: collection, + } + resp, err := server.LoadCollection(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_UnexpectedError, resp.ErrorCode) + suite.Contains(resp.Reason, job.ErrLoadParameterMismatched.Error()) + } +} + +func (suite *ServiceSuite) TestLoadPartition() { + ctx := context.Background() + server := suite.server + + // Test load all partitions + for _, collection := range suite.collections { + suite.expectGetRecoverInfo(collection) + + req := &querypb.LoadPartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + } + resp, err := server.LoadPartitions(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.ErrorCode) + suite.assertLoaded(collection) + } + + // Test load again + for _, collection := range suite.collections { + req := &querypb.LoadPartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + } + resp, err := server.LoadPartitions(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.ErrorCode) + } + + // Test when server is not healthy + server.UpdateStateCode(internalpb.StateCode_Initializing) + req := &querypb.LoadPartitionsRequest{ + CollectionID: suite.collections[0], + PartitionIDs: suite.partitions[suite.collections[0]], + } + resp, err := server.LoadPartitions(ctx, req) + suite.NoError(err) + suite.Contains(resp.Reason, ErrNotHealthy.Error()) +} + +func (suite *ServiceSuite) TestLoadPartitionFailed() { + suite.loadAll() + ctx := context.Background() + server := suite.server + + // Test load with different replica number + for _, collection := range suite.collections { + req := &querypb.LoadPartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + ReplicaNumber: suite.replicaNumber[collection] + 1, + } + resp, err := server.LoadPartitions(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_UnexpectedError, resp.ErrorCode) + suite.Contains(resp.Reason, job.ErrLoadParameterMismatched.Error()) + } + + // Test load with collection loaded + for _, collection := range suite.collections { + if suite.loadTypes[collection] != querypb.LoadType_LoadCollection { + continue + } + req := &querypb.LoadPartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + } + resp, err := server.LoadPartitions(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_UnexpectedError, resp.ErrorCode) + suite.Contains(resp.Reason, job.ErrLoadParameterMismatched.Error()) + } + + // Test load with more partitions + for _, collection := range suite.collections { + if suite.loadTypes[collection] != querypb.LoadType_LoadPartition { + continue + } + req := &querypb.LoadPartitionsRequest{ + CollectionID: collection, + PartitionIDs: append(suite.partitions[collection], 999), + } + resp, err := server.LoadPartitions(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_UnexpectedError, resp.ErrorCode) + suite.Contains(resp.Reason, job.ErrLoadParameterMismatched.Error()) + } +} + +func (suite *ServiceSuite) TestReleaseCollection() { + suite.loadAll() + ctx := context.Background() + server := suite.server + + // Test release all collections + for _, collection := range suite.collections { + req := &querypb.ReleaseCollectionRequest{ + CollectionID: collection, + } + resp, err := server.ReleaseCollection(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.ErrorCode) + suite.assertReleased(collection) + } + + // Test release again + for _, collection := range suite.collections { + req := &querypb.ReleaseCollectionRequest{ + CollectionID: collection, + } + resp, err := server.ReleaseCollection(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.ErrorCode) + } + + // Test when server is not healthy + server.UpdateStateCode(internalpb.StateCode_Initializing) + req := &querypb.ReleaseCollectionRequest{ + CollectionID: suite.collections[0], + } + resp, err := server.ReleaseCollection(ctx, req) + suite.NoError(err) + suite.Contains(resp.Reason, ErrNotHealthy.Error()) +} + +func (suite *ServiceSuite) TestReleasePartition() { + suite.loadAll() + ctx := context.Background() + server := suite.server + + // Test release all partitions + for _, collection := range suite.collections { + req := &querypb.ReleasePartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection][0:1], + } + resp, err := server.ReleasePartitions(ctx, req) + suite.NoError(err) + if suite.loadTypes[collection] == querypb.LoadType_LoadCollection { + suite.Equal(commonpb.ErrorCode_UnexpectedError, resp.ErrorCode) + } else { + suite.Equal(commonpb.ErrorCode_Success, resp.ErrorCode) + } + suite.assertPartitionLoaded(collection, suite.partitions[collection][1:]...) + } + + // Test release again + for _, collection := range suite.collections { + req := &querypb.ReleasePartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection][0:1], + } + resp, err := server.ReleasePartitions(ctx, req) + suite.NoError(err) + if suite.loadTypes[collection] == querypb.LoadType_LoadCollection { + suite.Equal(commonpb.ErrorCode_UnexpectedError, resp.ErrorCode) + } else { + suite.Equal(commonpb.ErrorCode_Success, resp.ErrorCode) + } + suite.assertPartitionLoaded(collection, suite.partitions[collection][1:]...) + } + + // Test when server is not healthy + server.UpdateStateCode(internalpb.StateCode_Initializing) + req := &querypb.ReleasePartitionsRequest{ + CollectionID: suite.collections[0], + PartitionIDs: suite.partitions[suite.collections[0]][0:1], + } + resp, err := server.ReleasePartitions(ctx, req) + suite.NoError(err) + suite.Contains(resp.Reason, ErrNotHealthy.Error()) +} + +func (suite *ServiceSuite) TestGetPartitionStates() { + suite.loadAll() + ctx := context.Background() + server := suite.server + + // Test get partitions' state + for _, collection := range suite.collections { + req := &querypb.GetPartitionStatesRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + } + resp, err := server.GetPartitionStates(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.Status.ErrorCode) + suite.Len(resp.PartitionDescriptions, len(suite.partitions[collection])) + } + + // Test when server is not healthy + server.UpdateStateCode(internalpb.StateCode_Initializing) + req := &querypb.GetPartitionStatesRequest{ + CollectionID: suite.collections[0], + } + resp, err := server.GetPartitionStates(ctx, req) + suite.NoError(err) + suite.Contains(resp.Status.Reason, ErrNotHealthy.Error()) +} + +func (suite *ServiceSuite) TestGetSegmentInfo() { + suite.loadAll() + ctx := context.Background() + server := suite.server + + // Test get all segments + for i, collection := range suite.collections { + suite.updateSegmentDist(collection, int64(i)) + req := &querypb.GetSegmentInfoRequest{ + CollectionID: collection, + } + resp, err := server.GetSegmentInfo(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.Status.ErrorCode) + suite.assertSegments(collection, resp.GetInfos()) + } + + // Test get given segments + for _, collection := range suite.collections { + req := &querypb.GetSegmentInfoRequest{ + CollectionID: collection, + SegmentIDs: suite.getAllSegments(collection), + } + resp, err := server.GetSegmentInfo(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.Status.ErrorCode) + suite.assertSegments(collection, resp.GetInfos()) + } + + // Test when server is not healthy + server.UpdateStateCode(internalpb.StateCode_Initializing) + req := &querypb.GetSegmentInfoRequest{ + CollectionID: suite.collections[0], + } + resp, err := server.GetSegmentInfo(ctx, req) + suite.NoError(err) + suite.Contains(resp.Status.Reason, ErrNotHealthy.Error()) +} + +func (suite *ServiceSuite) TestLoadBalance() { + suite.loadAll() + ctx := context.Background() + server := suite.server + + // Test get balance first segment + for _, collection := range suite.collections { + replicas := suite.meta.ReplicaManager.GetByCollection(collection) + srcNode := replicas[0].GetNodes()[0] + dstNode := replicas[0].GetNodes()[1] + suite.updateCollectionStatus(collection, querypb.LoadStatus_Loaded) + suite.updateSegmentDist(collection, srcNode) + segments := suite.getAllSegments(collection) + req := &querypb.LoadBalanceRequest{ + CollectionID: collection, + SourceNodeIDs: []int64{srcNode}, + DstNodeIDs: []int64{dstNode}, + SealedSegmentIDs: segments, + } + suite.taskScheduler.EXPECT().Add(mock.Anything).Run(func(task task.Task) { + task.Cancel() + }).Return(nil) + resp, err := server.LoadBalance(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.ErrorCode) + } + + // Test when server is not healthy + server.UpdateStateCode(internalpb.StateCode_Initializing) + req := &querypb.LoadBalanceRequest{ + CollectionID: suite.collections[0], + SourceNodeIDs: []int64{1}, + DstNodeIDs: []int64{100 + 1}, + } + resp, err := server.LoadBalance(ctx, req) + suite.NoError(err) + suite.Contains(resp.Reason, ErrNotHealthy.Error()) +} + +func (suite *ServiceSuite) TestLoadBalanceFailed() { + suite.loadAll() + ctx := context.Background() + server := suite.server + + // Test load balance without source node + for _, collection := range suite.collections { + replicas := suite.meta.ReplicaManager.GetByCollection(collection) + dstNode := replicas[0].GetNodes()[1] + segments := suite.getAllSegments(collection) + req := &querypb.LoadBalanceRequest{ + CollectionID: collection, + DstNodeIDs: []int64{dstNode}, + SealedSegmentIDs: segments, + } + resp, err := server.LoadBalance(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_UnexpectedError, resp.ErrorCode) + suite.Contains(resp.Reason, "source nodes can only contain 1 node") + } + + // Test load balance with not fully loaded + for _, collection := range suite.collections { + replicas := suite.meta.ReplicaManager.GetByCollection(collection) + srcNode := replicas[0].GetNodes()[0] + dstNode := replicas[0].GetNodes()[1] + suite.updateCollectionStatus(collection, querypb.LoadStatus_Loading) + segments := suite.getAllSegments(collection) + req := &querypb.LoadBalanceRequest{ + CollectionID: collection, + SourceNodeIDs: []int64{srcNode}, + DstNodeIDs: []int64{dstNode}, + SealedSegmentIDs: segments, + } + resp, err := server.LoadBalance(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_UnexpectedError, resp.ErrorCode) + suite.Contains(resp.Reason, "can't balance segments of not fully loaded collection") + } + + // Test load balance with source node and dest node not in the same replica + for _, collection := range suite.collections { + if suite.replicaNumber[collection] <= 1 { + continue + } + + replicas := suite.meta.ReplicaManager.GetByCollection(collection) + srcNode := replicas[0].GetNodes()[0] + dstNode := replicas[1].GetNodes()[0] + suite.updateCollectionStatus(collection, querypb.LoadStatus_Loaded) + suite.updateSegmentDist(collection, srcNode) + segments := suite.getAllSegments(collection) + req := &querypb.LoadBalanceRequest{ + CollectionID: collection, + SourceNodeIDs: []int64{srcNode}, + DstNodeIDs: []int64{dstNode}, + SealedSegmentIDs: segments, + } + resp, err := server.LoadBalance(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_UnexpectedError, resp.ErrorCode) + suite.Contains(resp.Reason, "destination nodes have to be in the same replica of source node") + } + + // Test balance task failed + for _, collection := range suite.collections { + replicas := suite.meta.ReplicaManager.GetByCollection(collection) + srcNode := replicas[0].GetNodes()[0] + dstNode := replicas[0].GetNodes()[1] + suite.updateCollectionStatus(collection, querypb.LoadStatus_Loaded) + suite.updateSegmentDist(collection, srcNode) + segments := suite.getAllSegments(collection) + req := &querypb.LoadBalanceRequest{ + CollectionID: collection, + SourceNodeIDs: []int64{srcNode}, + DstNodeIDs: []int64{dstNode}, + SealedSegmentIDs: segments, + } + suite.taskScheduler.EXPECT().Add(mock.Anything).Run(func(balanceTask task.Task) { + balanceTask.SetErr(task.ErrTaskCanceled) + balanceTask.Cancel() + }).Return(nil) + resp, err := server.LoadBalance(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_UnexpectedError, resp.ErrorCode) + suite.Contains(resp.Reason, "failed to balance segments") + suite.Contains(resp.Reason, task.ErrTaskCanceled.Error()) + } +} + +func (suite *ServiceSuite) TestShowConfigurations() { + ctx := context.Background() + server := suite.server + + req := &internalpb.ShowConfigurationsRequest{ + Pattern: "Port", + } + resp, err := server.ShowConfigurations(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.Status.ErrorCode) + suite.Len(resp.Configuations, 1) + suite.Equal("querycoord.port", resp.Configuations[0].Key) + + // Test when server is not healthy + server.UpdateStateCode(internalpb.StateCode_Initializing) + req = &internalpb.ShowConfigurationsRequest{ + Pattern: "Port", + } + resp, err = server.ShowConfigurations(ctx, req) + suite.NoError(err) + suite.Contains(resp.Status.Reason, ErrNotHealthy.Error()) +} + +func (suite *ServiceSuite) TestGetMetrics() { + ctx := context.Background() + server := suite.server + + for _, node := range suite.nodes { + suite.cluster.EXPECT().GetMetrics(ctx, node, mock.Anything).Return(&milvuspb.GetMetricsResponse{ + Status: successStatus, + ComponentName: "QueryNode", + }, nil) + } + + metricReq := make(map[string]string) + metricReq[metricsinfo.MetricTypeKey] = "system_info" + req, err := json.Marshal(metricReq) + suite.NoError(err) + resp, err := server.GetMetrics(ctx, &milvuspb.GetMetricsRequest{ + Base: &commonpb.MsgBase{}, + Request: string(req), + }) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.Status.ErrorCode) + + // Test when server is not healthy + server.UpdateStateCode(internalpb.StateCode_Initializing) + resp, err = server.GetMetrics(ctx, &milvuspb.GetMetricsRequest{ + Base: &commonpb.MsgBase{}, + Request: string(req), + }) + suite.NoError(err) + suite.Contains(resp.Status.Reason, ErrNotHealthy.Error()) +} + +func (suite *ServiceSuite) TestGetReplicas() { + suite.loadAll() + ctx := context.Background() + server := suite.server + + for _, collection := range suite.collections { + suite.updateChannelDist(collection) + req := &milvuspb.GetReplicasRequest{ + CollectionID: collection, + } + resp, err := server.GetReplicas(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.Status.ErrorCode) + suite.EqualValues(suite.replicaNumber[collection], len(resp.Replicas)) + } + + // Test get with shard nodes + for _, collection := range suite.collections { + replicas := suite.meta.ReplicaManager.GetByCollection(collection) + for _, replica := range replicas { + suite.updateSegmentDist(collection, replica.GetNodes()[0]) + } + suite.updateChannelDist(collection) + req := &milvuspb.GetReplicasRequest{ + CollectionID: collection, + WithShardNodes: true, + } + resp, err := server.GetReplicas(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.Status.ErrorCode) + suite.EqualValues(suite.replicaNumber[collection], len(resp.Replicas)) + } + + // Test when server is not healthy + server.UpdateStateCode(internalpb.StateCode_Initializing) + req := &milvuspb.GetReplicasRequest{ + CollectionID: suite.collections[0], + } + resp, err := server.GetReplicas(ctx, req) + suite.NoError(err) + suite.Contains(resp.Status.Reason, ErrNotHealthy.Error()) +} + +func (suite *ServiceSuite) TestGetShardLeaders() { + suite.loadAll() + ctx := context.Background() + server := suite.server + + for _, collection := range suite.collections { + suite.updateCollectionStatus(collection, querypb.LoadStatus_Loaded) + suite.updateChannelDist(collection) + req := &querypb.GetShardLeadersRequest{ + CollectionID: collection, + } + resp, err := server.GetShardLeaders(ctx, req) + suite.NoError(err) + suite.Equal(commonpb.ErrorCode_Success, resp.Status.ErrorCode) + suite.Len(resp.Shards, len(suite.channels[collection])) + for _, shard := range resp.Shards { + suite.Len(shard.NodeIds, int(suite.replicaNumber[collection])) + } + } + + // Test when server is not healthy + server.UpdateStateCode(internalpb.StateCode_Initializing) + req := &querypb.GetShardLeadersRequest{ + CollectionID: suite.collections[0], + } + resp, err := server.GetShardLeaders(ctx, req) + suite.NoError(err) + suite.Contains(resp.Status.Reason, ErrNotHealthy.Error()) +} + +func (suite *ServiceSuite) loadAll() { + ctx := context.Background() + for _, collection := range suite.collections { + suite.expectGetRecoverInfo(collection) + if suite.loadTypes[collection] == querypb.LoadType_LoadCollection { + suite.broker.EXPECT().GetPartitions(mock.Anything, collection).Return(suite.partitions[collection], nil) + + req := &querypb.LoadCollectionRequest{ + CollectionID: collection, + ReplicaNumber: suite.replicaNumber[collection], + } + job := job.NewLoadCollectionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.jobScheduler.Add(job) + err := job.Wait() + suite.NoError(err) + suite.EqualValues(suite.replicaNumber[collection], suite.meta.GetReplicaNumber(collection)) + suite.True(suite.meta.Exist(collection)) + suite.NotNil(suite.meta.GetCollection(collection)) + } else { + req := &querypb.LoadPartitionsRequest{ + CollectionID: collection, + PartitionIDs: suite.partitions[collection], + ReplicaNumber: suite.replicaNumber[collection], + } + job := job.NewLoadPartitionJob( + ctx, + req, + suite.dist, + suite.meta, + suite.targetMgr, + suite.broker, + suite.nodeMgr, + suite.handoffObserver, + ) + suite.jobScheduler.Add(job) + err := job.Wait() + suite.NoError(err) + suite.EqualValues(suite.replicaNumber[collection], suite.meta.GetReplicaNumber(collection)) + suite.True(suite.meta.Exist(collection)) + suite.NotNil(suite.meta.GetPartitionsByCollection(collection)) + } + } +} + +func (suite *ServiceSuite) assertLoaded(collection int64) { + suite.True(suite.meta.Exist(collection)) + for _, channel := range suite.channels[collection] { + suite.NotNil(suite.targetMgr.GetDmChannel(channel)) + } + for _, partitions := range suite.segments[collection] { + for _, segment := range partitions { + suite.NotNil(suite.targetMgr.GetSegment(segment)) + } + } +} + +func (suite *ServiceSuite) assertPartitionLoaded(collection int64, partitions ...int64) { + suite.True(suite.meta.Exist(collection)) + for _, channel := range suite.channels[collection] { + suite.NotNil(suite.targetMgr.GetDmChannel(channel)) + } + partitionSet := typeutil.NewUniqueSet(partitions...) + for partition, segments := range suite.segments[collection] { + if !partitionSet.Contain(partition) { + continue + } + for _, segment := range segments { + suite.NotNil(suite.targetMgr.GetSegment(segment)) + } + } +} + +func (suite *ServiceSuite) assertReleased(collection int64) { + suite.False(suite.meta.Exist(collection)) + for _, channel := range suite.channels[collection] { + suite.Nil(suite.targetMgr.GetDmChannel(channel)) + } + for _, partitions := range suite.segments[collection] { + for _, segment := range partitions { + suite.Nil(suite.targetMgr.GetSegment(segment)) + } + } +} + +func (suite *ServiceSuite) assertSegments(collection int64, segments []*querypb.SegmentInfo) bool { + segmentSet := typeutil.NewUniqueSet( + suite.getAllSegments(collection)...) + if !suite.Len(segments, segmentSet.Len()) { + return false + } + for _, segment := range segments { + if !suite.Contains(segmentSet, segment.GetSegmentID()) { + return false + } + } + + return true +} + +func (suite *ServiceSuite) expectGetRecoverInfo(collection int64) { + vChannels := []*datapb.VchannelInfo{} + for _, channel := range suite.channels[collection] { + vChannels = append(vChannels, &datapb.VchannelInfo{ + CollectionID: collection, + ChannelName: channel, + }) + } + + for partition, segments := range suite.segments[collection] { + segmentBinlogs := []*datapb.SegmentBinlogs{} + for _, segment := range segments { + segmentBinlogs = append(segmentBinlogs, &datapb.SegmentBinlogs{ + SegmentID: segment, + InsertChannel: suite.channels[collection][segment%2], + }) + } + + suite.broker.EXPECT(). + GetRecoveryInfo(mock.Anything, collection, partition). + Return(vChannels, segmentBinlogs, nil) + } +} + +func (suite *ServiceSuite) getAllSegments(collection int64) []int64 { + allSegments := make([]int64, 0) + for _, segments := range suite.segments[collection] { + allSegments = append(allSegments, segments...) + } + return allSegments +} + +func (suite *ServiceSuite) updateSegmentDist(collection, node int64) { + metaSegments := make([]*meta.Segment, 0) + for partition, segments := range suite.segments[collection] { + for _, segment := range segments { + metaSegments = append(metaSegments, + utils.CreateTestSegment(collection, partition, segment, node, 1, "test-channel")) + } + } + suite.dist.SegmentDistManager.Update(node, metaSegments...) +} + +func (suite *ServiceSuite) updateChannelDist(collection int64) { + channels := suite.channels[collection] + replicas := suite.meta.ReplicaManager.GetByCollection(collection) + for _, replica := range replicas { + i := 0 + for _, node := range replica.GetNodes() { + suite.dist.ChannelDistManager.Update(node, meta.DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: collection, + ChannelName: channels[i], + })) + suite.dist.LeaderViewManager.Update(node, &meta.LeaderView{ + ID: node, + CollectionID: collection, + Channel: channels[i], + }) + i++ + if i >= len(channels) { + break + } + } + } +} + +func (suite *ServiceSuite) updateCollectionStatus(collectionID int64, status querypb.LoadStatus) { + collection := suite.meta.GetCollection(collectionID) + if collection != nil { + collection := collection.Clone() + collection.LoadPercentage = 0 + if status == querypb.LoadStatus_Loaded { + collection.LoadPercentage = 100 + } + collection.CollectionLoadInfo.Status = status + suite.meta.UpdateCollection(collection) + } else { + partitions := suite.meta.GetPartitionsByCollection(collectionID) + for _, partition := range partitions { + partition := partition.Clone() + partition.LoadPercentage = 0 + if status == querypb.LoadStatus_Loaded { + partition.LoadPercentage = 100 + } + partition.PartitionLoadInfo.Status = status + suite.meta.UpdatePartition(partition) + } + } +} + +func TestService(t *testing.T) { + suite.Run(t, new(ServiceSuite)) +} diff --git a/internal/querycoordv2/session/cluster.go b/internal/querycoordv2/session/cluster.go new file mode 100644 index 0000000000000..de3ea2b479735 --- /dev/null +++ b/internal/querycoordv2/session/cluster.go @@ -0,0 +1,298 @@ +package session + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + grpcquerynodeclient "github.com/milvus-io/milvus/internal/distributed/querynode/client" + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/util/typeutil" + "go.uber.org/zap" +) + +const ( + updateTickerDuration = 1 * time.Minute + segmentBufferSize = 16 + bufferFlushPeriod = 500 * time.Millisecond +) + +var ( + ErrNodeNotFound = errors.New("NodeNotFound") +) + +func WrapErrNodeNotFound(nodeID int64) error { + return fmt.Errorf("%w(%v)", ErrNodeNotFound, nodeID) +} + +type Cluster interface { + WatchDmChannels(ctx context.Context, nodeID int64, req *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) + UnsubDmChannel(ctx context.Context, nodeID int64, req *querypb.UnsubDmChannelRequest) (*commonpb.Status, error) + LoadSegments(ctx context.Context, nodeID int64, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) + ReleaseSegments(ctx context.Context, nodeID int64, req *querypb.ReleaseSegmentsRequest) (*commonpb.Status, error) + GetDataDistribution(ctx context.Context, nodeID int64, req *querypb.GetDataDistributionRequest) (*querypb.GetDataDistributionResponse, error) + GetMetrics(ctx context.Context, nodeID int64, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) + SyncDistribution(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) (*commonpb.Status, error) + Start(ctx context.Context) + Stop() +} + +type segmentIndex struct { + NodeID int64 + CollectionID int64 + Shard string +} + +// QueryCluster is used to send requests to QueryNodes and manage connections +type QueryCluster struct { + *clients + nodeManager *NodeManager + wg sync.WaitGroup + ch chan struct{} + + scheduler *typeutil.GroupScheduler[segmentIndex, *commonpb.Status] +} + +func NewCluster(nodeManager *NodeManager) *QueryCluster { + c := &QueryCluster{ + clients: newClients(), + nodeManager: nodeManager, + ch: make(chan struct{}), + scheduler: typeutil.NewGroupScheduler[segmentIndex, *commonpb.Status](), + } + c.wg.Add(1) + go c.updateLoop() + return c +} + +func (c *QueryCluster) Start(ctx context.Context) { + c.scheduler.Start(ctx) +} + +func (c *QueryCluster) Stop() { + c.clients.closeAll() + close(c.ch) + c.scheduler.Stop() + c.wg.Wait() +} + +func (c *QueryCluster) updateLoop() { + defer c.wg.Done() + ticker := time.NewTicker(updateTickerDuration) + for { + select { + case <-c.ch: + log.Info("cluster closed") + return + case <-ticker.C: + nodes := c.clients.getAllNodeIDs() + for _, id := range nodes { + if c.nodeManager.Get(id) == nil { + c.clients.close(id) + } + } + } + } +} + +func (c *QueryCluster) LoadSegments(ctx context.Context, nodeID int64, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { + // task := NewLoadSegmentsTask(c, nodeID, req) + // c.scheduler.Add(task) + // return task.Wait() + return c.loadSegments(ctx, nodeID, req) +} + +func (c *QueryCluster) loadSegments(ctx context.Context, nodeID int64, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { + var status *commonpb.Status + var err error + err1 := c.send(ctx, nodeID, func(cli *grpcquerynodeclient.Client) { + status, err = cli.LoadSegments(ctx, req) + }) + if err1 != nil { + return nil, err1 + } + return status, err +} + +func (c *QueryCluster) WatchDmChannels(ctx context.Context, nodeID int64, req *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) { + var status *commonpb.Status + var err error + err1 := c.send(ctx, nodeID, func(cli *grpcquerynodeclient.Client) { + status, err = cli.WatchDmChannels(ctx, req) + }) + if err1 != nil { + return nil, err1 + } + return status, err +} + +func (c *QueryCluster) UnsubDmChannel(ctx context.Context, nodeID int64, req *querypb.UnsubDmChannelRequest) (*commonpb.Status, error) { + var status *commonpb.Status + var err error + err1 := c.send(ctx, nodeID, func(cli *grpcquerynodeclient.Client) { + status, err = cli.UnsubDmChannel(ctx, req) + }) + if err1 != nil { + return nil, err1 + } + return status, err +} + +func (c *QueryCluster) ReleaseSegments(ctx context.Context, nodeID int64, req *querypb.ReleaseSegmentsRequest) (*commonpb.Status, error) { + var status *commonpb.Status + var err error + err1 := c.send(ctx, nodeID, func(cli *grpcquerynodeclient.Client) { + status, err = cli.ReleaseSegments(ctx, req) + }) + if err1 != nil { + return nil, err1 + } + return status, err +} + +func (c *QueryCluster) GetDataDistribution(ctx context.Context, nodeID int64, req *querypb.GetDataDistributionRequest) (*querypb.GetDataDistributionResponse, error) { + var resp *querypb.GetDataDistributionResponse + var err error + err1 := c.send(ctx, nodeID, func(cli *grpcquerynodeclient.Client) { + resp, err = cli.GetDataDistribution(ctx, req) + }) + if err1 != nil { + return nil, err1 + } + return resp, err +} + +func (c *QueryCluster) GetMetrics(ctx context.Context, nodeID int64, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) { + var ( + resp *milvuspb.GetMetricsResponse + err error + ) + err1 := c.send(ctx, nodeID, func(cli *grpcquerynodeclient.Client) { + resp, err = cli.GetMetrics(ctx, req) + }) + if err1 != nil { + return nil, err1 + } + return resp, err +} + +func (c *QueryCluster) SyncDistribution(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) (*commonpb.Status, error) { + var ( + resp *commonpb.Status + err error + ) + err1 := c.send(ctx, nodeID, func(cli *grpcquerynodeclient.Client) { + resp, err = cli.SyncDistribution(ctx, req) + }) + if err1 != nil { + return nil, err1 + } + return resp, err +} + +func (c *QueryCluster) send(ctx context.Context, nodeID int64, fn func(cli *grpcquerynodeclient.Client)) error { + node := c.nodeManager.Get(nodeID) + if node == nil { + return WrapErrNodeNotFound(nodeID) + } + + cli, err := c.clients.getOrCreate(ctx, node) + if err != nil { + return err + } + + fn(cli) + return nil +} + +type clients struct { + sync.RWMutex + clients map[int64]*grpcquerynodeclient.Client // nodeID -> client +} + +func (c *clients) getAllNodeIDs() []int64 { + c.RLock() + defer c.RUnlock() + + ret := make([]int64, 0, len(c.clients)) + for k := range c.clients { + ret = append(ret, k) + } + return ret +} + +func (c *clients) getOrCreate(ctx context.Context, node *NodeInfo) (*grpcquerynodeclient.Client, error) { + if cli := c.get(node.ID()); cli != nil { + return cli, nil + } + + newCli, err := createNewClient(context.Background(), node.Addr()) + if err != nil { + return nil, err + } + c.set(node.ID(), newCli) + return c.get(node.ID()), nil +} + +func createNewClient(ctx context.Context, addr string) (*grpcquerynodeclient.Client, error) { + newCli, err := grpcquerynodeclient.NewClient(ctx, addr) + if err != nil { + return nil, err + } + if err = newCli.Init(); err != nil { + return nil, err + } + if err = newCli.Start(); err != nil { + return nil, err + } + return newCli, nil +} + +func (c *clients) set(nodeID int64, client *grpcquerynodeclient.Client) { + c.Lock() + defer c.Unlock() + if _, ok := c.clients[nodeID]; ok { + if err := client.Stop(); err != nil { + log.Warn("close new created client error", zap.Int64("nodeID", nodeID), zap.Error(err)) + return + } + log.Info("use old client", zap.Int64("nodeID", nodeID)) + } + c.clients[nodeID] = client +} + +func (c *clients) get(nodeID int64) *grpcquerynodeclient.Client { + c.RLock() + defer c.RUnlock() + return c.clients[nodeID] +} + +func (c *clients) close(nodeID int64) { + c.Lock() + defer c.Unlock() + if cli, ok := c.clients[nodeID]; ok { + if err := cli.Stop(); err != nil { + log.Warn("error occurred during stopping client", zap.Int64("nodeID", nodeID), zap.Error(err)) + } + delete(c.clients, nodeID) + } +} + +func (c *clients) closeAll() { + c.Lock() + defer c.Unlock() + for nodeID, cli := range c.clients { + if err := cli.Stop(); err != nil { + log.Warn("error occurred during stopping client", zap.Int64("nodeID", nodeID), zap.Error(err)) + } + } +} + +func newClients() *clients { + return &clients{clients: make(map[int64]*grpcquerynodeclient.Client)} +} diff --git a/internal/querycoordv2/session/cluster_test.go b/internal/querycoordv2/session/cluster_test.go new file mode 100644 index 0000000000000..89e0b6f3adb6c --- /dev/null +++ b/internal/querycoordv2/session/cluster_test.go @@ -0,0 +1,294 @@ +package session + +import ( + "context" + "net" + "testing" + "time" + + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/mocks" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc" +) + +const bufSize = 1024 * 1024 + +type ClusterTestSuite struct { + suite.Suite + svrs []*grpc.Server + listeners []net.Listener + cluster *QueryCluster + nodeManager *NodeManager +} + +func (suite *ClusterTestSuite) SetupSuite() { + suite.setupServers() +} + +func (suite *ClusterTestSuite) TearDownSuite() { + for _, svr := range suite.svrs { + svr.GracefulStop() + } +} + +func (suite *ClusterTestSuite) SetupTest() { + suite.setupCluster() +} + +func (suite *ClusterTestSuite) TearDownTest() { + suite.cluster.Stop() +} +func (suite *ClusterTestSuite) setupServers() { + svrs := suite.createTestServers() + for _, svr := range svrs { + lis, err := net.Listen("tcp", ":0") + suite.NoError(err) + suite.listeners = append(suite.listeners, lis) + s := grpc.NewServer() + querypb.RegisterQueryNodeServer(s, svr) + go func() { + suite.Eventually(func() bool { + return s.Serve(lis) == nil + }, 10*time.Second, 100*time.Millisecond) + }() + suite.svrs = append(suite.svrs, s) + } + + // check server ready to serve + for _, lis := range suite.listeners { + conn, err := grpc.Dial(lis.Addr().String(), grpc.WithBlock(), grpc.WithInsecure()) + suite.NoError(err) + suite.NoError(conn.Close()) + } +} + +func (suite *ClusterTestSuite) setupCluster() { + suite.nodeManager = NewNodeManager() + for i, lis := range suite.listeners { + node := NewNodeInfo(int64(i), lis.Addr().String()) + suite.nodeManager.Add(node) + } + suite.cluster = NewCluster(suite.nodeManager) + suite.cluster.Start(context.Background()) +} + +func (suite *ClusterTestSuite) createTestServers() []querypb.QueryNodeServer { + // create 2 mock servers with 1 always return error + ret := make([]querypb.QueryNodeServer, 0, 2) + ret = append(ret, suite.createDefaultMockServer()) + ret = append(ret, suite.createFailedMockServer()) + return ret +} + +func (suite *ClusterTestSuite) createDefaultMockServer() querypb.QueryNodeServer { + succStatus := &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "", + } + svr := mocks.NewMockQueryNodeServer(suite.T()) + // TODO: register more mock methods + svr.EXPECT().LoadSegments( + mock.Anything, + mock.AnythingOfType("*querypb.LoadSegmentsRequest"), + ).Maybe().Return(succStatus, nil) + svr.EXPECT().WatchDmChannels( + mock.Anything, + mock.AnythingOfType("*querypb.WatchDmChannelsRequest"), + ).Maybe().Return(succStatus, nil) + svr.EXPECT().UnsubDmChannel( + mock.Anything, + mock.AnythingOfType("*querypb.UnsubDmChannelRequest"), + ).Maybe().Return(succStatus, nil) + svr.EXPECT().ReleaseSegments( + mock.Anything, + mock.AnythingOfType("*querypb.ReleaseSegmentsRequest"), + ).Maybe().Return(succStatus, nil) + svr.EXPECT().GetDataDistribution( + mock.Anything, + mock.AnythingOfType("*querypb.GetDataDistributionRequest"), + ).Maybe().Return(&querypb.GetDataDistributionResponse{Status: succStatus}, nil) + svr.EXPECT().GetMetrics( + mock.Anything, + mock.AnythingOfType("*milvuspb.GetMetricsRequest"), + ).Maybe().Return(&milvuspb.GetMetricsResponse{Status: succStatus}, nil) + svr.EXPECT().SyncDistribution( + mock.Anything, + mock.AnythingOfType("*querypb.SyncDistributionRequest"), + ).Maybe().Return(succStatus, nil) + return svr +} + +func (suite *ClusterTestSuite) createFailedMockServer() querypb.QueryNodeServer { + failStatus := &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "unexpected error", + } + svr := mocks.NewMockQueryNodeServer(suite.T()) + // TODO: register more mock methods + svr.EXPECT().LoadSegments( + mock.Anything, + mock.AnythingOfType("*querypb.LoadSegmentsRequest"), + ).Maybe().Return(failStatus, nil) + svr.EXPECT().WatchDmChannels( + mock.Anything, + mock.AnythingOfType("*querypb.WatchDmChannelsRequest"), + ).Maybe().Return(failStatus, nil) + svr.EXPECT().UnsubDmChannel( + mock.Anything, + mock.AnythingOfType("*querypb.UnsubDmChannelRequest"), + ).Maybe().Return(failStatus, nil) + svr.EXPECT().ReleaseSegments( + mock.Anything, + mock.AnythingOfType("*querypb.ReleaseSegmentsRequest"), + ).Maybe().Return(failStatus, nil) + svr.EXPECT().GetDataDistribution( + mock.Anything, + mock.AnythingOfType("*querypb.GetDataDistributionRequest"), + ).Maybe().Return(&querypb.GetDataDistributionResponse{Status: failStatus}, nil) + svr.EXPECT().GetMetrics( + mock.Anything, + mock.AnythingOfType("*milvuspb.GetMetricsRequest"), + ).Maybe().Return(&milvuspb.GetMetricsResponse{Status: failStatus}, nil) + svr.EXPECT().SyncDistribution( + mock.Anything, + mock.AnythingOfType("*querypb.SyncDistributionRequest"), + ).Maybe().Return(failStatus, nil) + return svr +} + +func (suite *ClusterTestSuite) TestLoadSegments() { + ctx := context.TODO() + status, err := suite.cluster.LoadSegments(ctx, 0, &querypb.LoadSegmentsRequest{ + Infos: []*querypb.SegmentLoadInfo{{}}, + }) + suite.NoError(err) + suite.Equal(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "", + }, status) + + status, err = suite.cluster.LoadSegments(ctx, 1, &querypb.LoadSegmentsRequest{ + Infos: []*querypb.SegmentLoadInfo{{}}, + }) + suite.NoError(err) + suite.Equal(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "unexpected error", + }, status) + + _, err = suite.cluster.LoadSegments(ctx, 3, &querypb.LoadSegmentsRequest{ + Infos: []*querypb.SegmentLoadInfo{{}}, + }) + suite.Error(err) + suite.IsType(WrapErrNodeNotFound(3), err) +} + +func (suite *ClusterTestSuite) TestWatchDmChannels() { + ctx := context.TODO() + status, err := suite.cluster.WatchDmChannels(ctx, 0, &querypb.WatchDmChannelsRequest{}) + suite.NoError(err) + suite.Equal(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "", + }, status) + + status, err = suite.cluster.WatchDmChannels(ctx, 1, &querypb.WatchDmChannelsRequest{}) + suite.NoError(err) + suite.Equal(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "unexpected error", + }, status) +} + +func (suite *ClusterTestSuite) TestUnsubDmChannel() { + ctx := context.TODO() + status, err := suite.cluster.UnsubDmChannel(ctx, 0, &querypb.UnsubDmChannelRequest{}) + suite.NoError(err) + suite.Equal(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "", + }, status) + + status, err = suite.cluster.UnsubDmChannel(ctx, 1, &querypb.UnsubDmChannelRequest{}) + suite.NoError(err) + suite.Equal(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "unexpected error", + }, status) +} + +func (suite *ClusterTestSuite) TestReleaseSegments() { + ctx := context.TODO() + status, err := suite.cluster.ReleaseSegments(ctx, 0, &querypb.ReleaseSegmentsRequest{}) + suite.NoError(err) + suite.Equal(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "", + }, status) + + status, err = suite.cluster.ReleaseSegments(ctx, 1, &querypb.ReleaseSegmentsRequest{}) + suite.NoError(err) + suite.Equal(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "unexpected error", + }, status) +} + +func (suite *ClusterTestSuite) TestGetDataDistribution() { + ctx := context.TODO() + resp, err := suite.cluster.GetDataDistribution(ctx, 0, &querypb.GetDataDistributionRequest{}) + suite.NoError(err) + suite.Equal(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "", + }, resp.GetStatus()) + + resp, err = suite.cluster.GetDataDistribution(ctx, 1, &querypb.GetDataDistributionRequest{}) + suite.NoError(err) + suite.Equal(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "unexpected error", + }, resp.GetStatus()) +} + +func (suite *ClusterTestSuite) TestGetMetrics() { + ctx := context.TODO() + resp, err := suite.cluster.GetMetrics(ctx, 0, &milvuspb.GetMetricsRequest{}) + suite.NoError(err) + suite.Equal(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "", + }, resp.GetStatus()) + + resp, err = suite.cluster.GetMetrics(ctx, 1, &milvuspb.GetMetricsRequest{}) + suite.NoError(err) + suite.Equal(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "unexpected error", + }, resp.GetStatus()) +} + +func (suite *ClusterTestSuite) TestSyncDistribution() { + ctx := context.TODO() + status, err := suite.cluster.SyncDistribution(ctx, 0, &querypb.SyncDistributionRequest{}) + suite.NoError(err) + suite.Equal(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "", + }, status) + + status, err = suite.cluster.SyncDistribution(ctx, 1, &querypb.SyncDistributionRequest{}) + suite.NoError(err) + suite.Equal(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "unexpected error", + }, status) +} + +func TestClusterSuite(t *testing.T) { + suite.Run(t, new(ClusterTestSuite)) +} diff --git a/internal/querycoordv2/session/mock_cluster.go b/internal/querycoordv2/session/mock_cluster.go new file mode 100644 index 0000000000000..c3f76da3b1a9e --- /dev/null +++ b/internal/querycoordv2/session/mock_cluster.go @@ -0,0 +1,434 @@ +// Code generated by mockery v2.14.0. DO NOT EDIT. + +package session + +import ( + context "context" + + commonpb "github.com/milvus-io/milvus/internal/proto/commonpb" + + milvuspb "github.com/milvus-io/milvus/internal/proto/milvuspb" + + mock "github.com/stretchr/testify/mock" + + querypb "github.com/milvus-io/milvus/internal/proto/querypb" +) + +// MockCluster is an autogenerated mock type for the Cluster type +type MockCluster struct { + mock.Mock +} + +type MockCluster_Expecter struct { + mock *mock.Mock +} + +func (_m *MockCluster) EXPECT() *MockCluster_Expecter { + return &MockCluster_Expecter{mock: &_m.Mock} +} + +// GetDataDistribution provides a mock function with given fields: ctx, nodeID, req +func (_m *MockCluster) GetDataDistribution(ctx context.Context, nodeID int64, req *querypb.GetDataDistributionRequest) (*querypb.GetDataDistributionResponse, error) { + ret := _m.Called(ctx, nodeID, req) + + var r0 *querypb.GetDataDistributionResponse + if rf, ok := ret.Get(0).(func(context.Context, int64, *querypb.GetDataDistributionRequest) *querypb.GetDataDistributionResponse); ok { + r0 = rf(ctx, nodeID, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*querypb.GetDataDistributionResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64, *querypb.GetDataDistributionRequest) error); ok { + r1 = rf(ctx, nodeID, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockCluster_GetDataDistribution_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetDataDistribution' +type MockCluster_GetDataDistribution_Call struct { + *mock.Call +} + +// GetDataDistribution is a helper method to define mock.On call +// - ctx context.Context +// - nodeID int64 +// - req *querypb.GetDataDistributionRequest +func (_e *MockCluster_Expecter) GetDataDistribution(ctx interface{}, nodeID interface{}, req interface{}) *MockCluster_GetDataDistribution_Call { + return &MockCluster_GetDataDistribution_Call{Call: _e.mock.On("GetDataDistribution", ctx, nodeID, req)} +} + +func (_c *MockCluster_GetDataDistribution_Call) Run(run func(ctx context.Context, nodeID int64, req *querypb.GetDataDistributionRequest)) *MockCluster_GetDataDistribution_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64), args[2].(*querypb.GetDataDistributionRequest)) + }) + return _c +} + +func (_c *MockCluster_GetDataDistribution_Call) Return(_a0 *querypb.GetDataDistributionResponse, _a1 error) *MockCluster_GetDataDistribution_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// GetMetrics provides a mock function with given fields: ctx, nodeID, req +func (_m *MockCluster) GetMetrics(ctx context.Context, nodeID int64, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) { + ret := _m.Called(ctx, nodeID, req) + + var r0 *milvuspb.GetMetricsResponse + if rf, ok := ret.Get(0).(func(context.Context, int64, *milvuspb.GetMetricsRequest) *milvuspb.GetMetricsResponse); ok { + r0 = rf(ctx, nodeID, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*milvuspb.GetMetricsResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64, *milvuspb.GetMetricsRequest) error); ok { + r1 = rf(ctx, nodeID, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockCluster_GetMetrics_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetMetrics' +type MockCluster_GetMetrics_Call struct { + *mock.Call +} + +// GetMetrics is a helper method to define mock.On call +// - ctx context.Context +// - nodeID int64 +// - req *milvuspb.GetMetricsRequest +func (_e *MockCluster_Expecter) GetMetrics(ctx interface{}, nodeID interface{}, req interface{}) *MockCluster_GetMetrics_Call { + return &MockCluster_GetMetrics_Call{Call: _e.mock.On("GetMetrics", ctx, nodeID, req)} +} + +func (_c *MockCluster_GetMetrics_Call) Run(run func(ctx context.Context, nodeID int64, req *milvuspb.GetMetricsRequest)) *MockCluster_GetMetrics_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64), args[2].(*milvuspb.GetMetricsRequest)) + }) + return _c +} + +func (_c *MockCluster_GetMetrics_Call) Return(_a0 *milvuspb.GetMetricsResponse, _a1 error) *MockCluster_GetMetrics_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// LoadSegments provides a mock function with given fields: ctx, nodeID, req +func (_m *MockCluster) LoadSegments(ctx context.Context, nodeID int64, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { + ret := _m.Called(ctx, nodeID, req) + + var r0 *commonpb.Status + if rf, ok := ret.Get(0).(func(context.Context, int64, *querypb.LoadSegmentsRequest) *commonpb.Status); ok { + r0 = rf(ctx, nodeID, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*commonpb.Status) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64, *querypb.LoadSegmentsRequest) error); ok { + r1 = rf(ctx, nodeID, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockCluster_LoadSegments_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LoadSegments' +type MockCluster_LoadSegments_Call struct { + *mock.Call +} + +// LoadSegments is a helper method to define mock.On call +// - ctx context.Context +// - nodeID int64 +// - req *querypb.LoadSegmentsRequest +func (_e *MockCluster_Expecter) LoadSegments(ctx interface{}, nodeID interface{}, req interface{}) *MockCluster_LoadSegments_Call { + return &MockCluster_LoadSegments_Call{Call: _e.mock.On("LoadSegments", ctx, nodeID, req)} +} + +func (_c *MockCluster_LoadSegments_Call) Run(run func(ctx context.Context, nodeID int64, req *querypb.LoadSegmentsRequest)) *MockCluster_LoadSegments_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64), args[2].(*querypb.LoadSegmentsRequest)) + }) + return _c +} + +func (_c *MockCluster_LoadSegments_Call) Return(_a0 *commonpb.Status, _a1 error) *MockCluster_LoadSegments_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// ReleaseSegments provides a mock function with given fields: ctx, nodeID, req +func (_m *MockCluster) ReleaseSegments(ctx context.Context, nodeID int64, req *querypb.ReleaseSegmentsRequest) (*commonpb.Status, error) { + ret := _m.Called(ctx, nodeID, req) + + var r0 *commonpb.Status + if rf, ok := ret.Get(0).(func(context.Context, int64, *querypb.ReleaseSegmentsRequest) *commonpb.Status); ok { + r0 = rf(ctx, nodeID, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*commonpb.Status) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64, *querypb.ReleaseSegmentsRequest) error); ok { + r1 = rf(ctx, nodeID, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockCluster_ReleaseSegments_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReleaseSegments' +type MockCluster_ReleaseSegments_Call struct { + *mock.Call +} + +// ReleaseSegments is a helper method to define mock.On call +// - ctx context.Context +// - nodeID int64 +// - req *querypb.ReleaseSegmentsRequest +func (_e *MockCluster_Expecter) ReleaseSegments(ctx interface{}, nodeID interface{}, req interface{}) *MockCluster_ReleaseSegments_Call { + return &MockCluster_ReleaseSegments_Call{Call: _e.mock.On("ReleaseSegments", ctx, nodeID, req)} +} + +func (_c *MockCluster_ReleaseSegments_Call) Run(run func(ctx context.Context, nodeID int64, req *querypb.ReleaseSegmentsRequest)) *MockCluster_ReleaseSegments_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64), args[2].(*querypb.ReleaseSegmentsRequest)) + }) + return _c +} + +func (_c *MockCluster_ReleaseSegments_Call) Return(_a0 *commonpb.Status, _a1 error) *MockCluster_ReleaseSegments_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// Start provides a mock function with given fields: ctx +func (_m *MockCluster) Start(ctx context.Context) { + _m.Called(ctx) +} + +// MockCluster_Start_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Start' +type MockCluster_Start_Call struct { + *mock.Call +} + +// Start is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockCluster_Expecter) Start(ctx interface{}) *MockCluster_Start_Call { + return &MockCluster_Start_Call{Call: _e.mock.On("Start", ctx)} +} + +func (_c *MockCluster_Start_Call) Run(run func(ctx context.Context)) *MockCluster_Start_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockCluster_Start_Call) Return() *MockCluster_Start_Call { + _c.Call.Return() + return _c +} + +// Stop provides a mock function with given fields: +func (_m *MockCluster) Stop() { + _m.Called() +} + +// MockCluster_Stop_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Stop' +type MockCluster_Stop_Call struct { + *mock.Call +} + +// Stop is a helper method to define mock.On call +func (_e *MockCluster_Expecter) Stop() *MockCluster_Stop_Call { + return &MockCluster_Stop_Call{Call: _e.mock.On("Stop")} +} + +func (_c *MockCluster_Stop_Call) Run(run func()) *MockCluster_Stop_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockCluster_Stop_Call) Return() *MockCluster_Stop_Call { + _c.Call.Return() + return _c +} + +// SyncDistribution provides a mock function with given fields: ctx, nodeID, req +func (_m *MockCluster) SyncDistribution(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) (*commonpb.Status, error) { + ret := _m.Called(ctx, nodeID, req) + + var r0 *commonpb.Status + if rf, ok := ret.Get(0).(func(context.Context, int64, *querypb.SyncDistributionRequest) *commonpb.Status); ok { + r0 = rf(ctx, nodeID, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*commonpb.Status) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64, *querypb.SyncDistributionRequest) error); ok { + r1 = rf(ctx, nodeID, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockCluster_SyncDistribution_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SyncDistribution' +type MockCluster_SyncDistribution_Call struct { + *mock.Call +} + +// SyncDistribution is a helper method to define mock.On call +// - ctx context.Context +// - nodeID int64 +// - req *querypb.SyncDistributionRequest +func (_e *MockCluster_Expecter) SyncDistribution(ctx interface{}, nodeID interface{}, req interface{}) *MockCluster_SyncDistribution_Call { + return &MockCluster_SyncDistribution_Call{Call: _e.mock.On("SyncDistribution", ctx, nodeID, req)} +} + +func (_c *MockCluster_SyncDistribution_Call) Run(run func(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest)) *MockCluster_SyncDistribution_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64), args[2].(*querypb.SyncDistributionRequest)) + }) + return _c +} + +func (_c *MockCluster_SyncDistribution_Call) Return(_a0 *commonpb.Status, _a1 error) *MockCluster_SyncDistribution_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// UnsubDmChannel provides a mock function with given fields: ctx, nodeID, req +func (_m *MockCluster) UnsubDmChannel(ctx context.Context, nodeID int64, req *querypb.UnsubDmChannelRequest) (*commonpb.Status, error) { + ret := _m.Called(ctx, nodeID, req) + + var r0 *commonpb.Status + if rf, ok := ret.Get(0).(func(context.Context, int64, *querypb.UnsubDmChannelRequest) *commonpb.Status); ok { + r0 = rf(ctx, nodeID, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*commonpb.Status) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64, *querypb.UnsubDmChannelRequest) error); ok { + r1 = rf(ctx, nodeID, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockCluster_UnsubDmChannel_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UnsubDmChannel' +type MockCluster_UnsubDmChannel_Call struct { + *mock.Call +} + +// UnsubDmChannel is a helper method to define mock.On call +// - ctx context.Context +// - nodeID int64 +// - req *querypb.UnsubDmChannelRequest +func (_e *MockCluster_Expecter) UnsubDmChannel(ctx interface{}, nodeID interface{}, req interface{}) *MockCluster_UnsubDmChannel_Call { + return &MockCluster_UnsubDmChannel_Call{Call: _e.mock.On("UnsubDmChannel", ctx, nodeID, req)} +} + +func (_c *MockCluster_UnsubDmChannel_Call) Run(run func(ctx context.Context, nodeID int64, req *querypb.UnsubDmChannelRequest)) *MockCluster_UnsubDmChannel_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64), args[2].(*querypb.UnsubDmChannelRequest)) + }) + return _c +} + +func (_c *MockCluster_UnsubDmChannel_Call) Return(_a0 *commonpb.Status, _a1 error) *MockCluster_UnsubDmChannel_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// WatchDmChannels provides a mock function with given fields: ctx, nodeID, req +func (_m *MockCluster) WatchDmChannels(ctx context.Context, nodeID int64, req *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) { + ret := _m.Called(ctx, nodeID, req) + + var r0 *commonpb.Status + if rf, ok := ret.Get(0).(func(context.Context, int64, *querypb.WatchDmChannelsRequest) *commonpb.Status); ok { + r0 = rf(ctx, nodeID, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*commonpb.Status) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int64, *querypb.WatchDmChannelsRequest) error); ok { + r1 = rf(ctx, nodeID, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockCluster_WatchDmChannels_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'WatchDmChannels' +type MockCluster_WatchDmChannels_Call struct { + *mock.Call +} + +// WatchDmChannels is a helper method to define mock.On call +// - ctx context.Context +// - nodeID int64 +// - req *querypb.WatchDmChannelsRequest +func (_e *MockCluster_Expecter) WatchDmChannels(ctx interface{}, nodeID interface{}, req interface{}) *MockCluster_WatchDmChannels_Call { + return &MockCluster_WatchDmChannels_Call{Call: _e.mock.On("WatchDmChannels", ctx, nodeID, req)} +} + +func (_c *MockCluster_WatchDmChannels_Call) Run(run func(ctx context.Context, nodeID int64, req *querypb.WatchDmChannelsRequest)) *MockCluster_WatchDmChannels_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64), args[2].(*querypb.WatchDmChannelsRequest)) + }) + return _c +} + +func (_c *MockCluster_WatchDmChannels_Call) Return(_a0 *commonpb.Status, _a1 error) *MockCluster_WatchDmChannels_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +type mockConstructorTestingTNewMockCluster interface { + mock.TestingT + Cleanup(func()) +} + +// NewMockCluster creates a new instance of MockCluster. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMockCluster(t mockConstructorTestingTNewMockCluster) *MockCluster { + mock := &MockCluster{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/querycoordv2/session/node_manager.go b/internal/querycoordv2/session/node_manager.go new file mode 100644 index 0000000000000..e655a15ca5ed5 --- /dev/null +++ b/internal/querycoordv2/session/node_manager.go @@ -0,0 +1,106 @@ +package session + +import "sync" + +type Manager interface { + Add(node *NodeInfo) + Remove(nodeID int64) + Get(nodeID int64) *NodeInfo + GetAll() []*NodeInfo +} + +type NodeManager struct { + mu sync.RWMutex + nodes map[int64]*NodeInfo +} + +func (m *NodeManager) Add(node *NodeInfo) { + m.mu.Lock() + defer m.mu.Unlock() + m.nodes[node.ID()] = node +} + +func (m *NodeManager) Remove(nodeID int64) { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.nodes, nodeID) +} + +func (m *NodeManager) Get(nodeID int64) *NodeInfo { + m.mu.RLock() + defer m.mu.RUnlock() + return m.nodes[nodeID] +} + +func (m *NodeManager) GetAll() []*NodeInfo { + m.mu.RLock() + defer m.mu.RUnlock() + ret := make([]*NodeInfo, 0, len(m.nodes)) + for _, n := range m.nodes { + ret = append(ret, n) + } + return ret +} + +func NewNodeManager() *NodeManager { + return &NodeManager{ + nodes: make(map[int64]*NodeInfo), + } +} + +type NodeInfo struct { + stats + mu sync.RWMutex + id int64 + addr string +} + +func (n *NodeInfo) ID() int64 { + return n.id +} + +func (n *NodeInfo) Addr() string { + return n.addr +} + +func (n *NodeInfo) SegmentCnt() int { + n.mu.RLock() + defer n.mu.RUnlock() + return n.stats.getSegmentCnt() +} + +func (n *NodeInfo) ChannelCnt() int { + n.mu.RLock() + defer n.mu.RUnlock() + return n.stats.getChannelCnt() +} + +func (n *NodeInfo) UpdateStats(opts ...StatsOption) { + n.mu.Lock() + for _, opt := range opts { + opt(n) + } + n.mu.Unlock() +} + +func NewNodeInfo(id int64, addr string) *NodeInfo { + return &NodeInfo{ + stats: newStats(), + id: id, + addr: addr, + } +} + +type StatsOption func(*NodeInfo) + +func WithSegmentCnt(cnt int) StatsOption { + return func(n *NodeInfo) { + n.setSegmentCnt(cnt) + } +} + +func WithChannelCnt(cnt int) StatsOption { + return func(n *NodeInfo) { + n.setChannelCnt(cnt) + } +} diff --git a/internal/querycoordv2/session/stats.go b/internal/querycoordv2/session/stats.go new file mode 100644 index 0000000000000..262bef3bbf6eb --- /dev/null +++ b/internal/querycoordv2/session/stats.go @@ -0,0 +1,26 @@ +package session + +type stats struct { + segmentCnt int + channelCnt int +} + +func (s *stats) setSegmentCnt(cnt int) { + s.segmentCnt = cnt +} + +func (s *stats) getSegmentCnt() int { + return s.segmentCnt +} + +func (s *stats) setChannelCnt(cnt int) { + s.channelCnt = cnt +} + +func (s *stats) getChannelCnt() int { + return s.channelCnt +} + +func newStats() stats { + return stats{} +} diff --git a/internal/querycoordv2/session/task.go b/internal/querycoordv2/session/task.go new file mode 100644 index 0000000000000..06023c3914b58 --- /dev/null +++ b/internal/querycoordv2/session/task.go @@ -0,0 +1,87 @@ +package session + +import ( + "context" + "time" + + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/internalpb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/util/typeutil" +) + +var _ typeutil.MergeableTask[segmentIndex, *commonpb.Status] = (*LoadSegmentsTask)(nil) + +type LoadSegmentsTask struct { + doneCh chan struct{} + cluster *QueryCluster + nodeID int64 + req *querypb.LoadSegmentsRequest + result *commonpb.Status + err error +} + +func NewLoadSegmentsTask(cluster *QueryCluster, nodeID int64, req *querypb.LoadSegmentsRequest) *LoadSegmentsTask { + return &LoadSegmentsTask{ + doneCh: make(chan struct{}), + cluster: cluster, + nodeID: nodeID, + req: req, + } +} + +func (task *LoadSegmentsTask) ID() segmentIndex { + return segmentIndex{ + NodeID: task.nodeID, + CollectionID: task.req.GetCollectionID(), + Shard: task.req.GetInfos()[0].GetInsertChannel(), + } +} + +func (task *LoadSegmentsTask) Execute() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + status, err := task.cluster.loadSegments(ctx, task.nodeID, task.req) + if err != nil { + task.err = err + return err + } + task.result = status + return nil +} + +func (task *LoadSegmentsTask) Merge(other typeutil.MergeableTask[segmentIndex, *commonpb.Status]) { + task.req.Infos = append(task.req.Infos, other.(*LoadSegmentsTask).req.GetInfos()...) + deltaPositions := make(map[string]*internalpb.MsgPosition) + for _, position := range task.req.DeltaPositions { + deltaPositions[position.GetChannelName()] = position + } + for _, position := range other.(*LoadSegmentsTask).req.GetDeltaPositions() { + merged, ok := deltaPositions[position.GetChannelName()] + if !ok || merged.GetTimestamp() > position.GetTimestamp() { + merged = position + } + deltaPositions[position.GetChannelName()] = merged + } + task.req.DeltaPositions = make([]*internalpb.MsgPosition, 0, len(deltaPositions)) + for _, position := range deltaPositions { + task.req.DeltaPositions = append(task.req.DeltaPositions, position) + } +} + +func (task *LoadSegmentsTask) SetResult(result *commonpb.Status) { + task.result = result +} + +func (task *LoadSegmentsTask) SetError(err error) { + task.err = err +} + +func (task *LoadSegmentsTask) Done() { + close(task.doneCh) +} + +func (task *LoadSegmentsTask) Wait() (*commonpb.Status, error) { + <-task.doneCh + return task.result, task.err +} diff --git a/internal/querycoordv2/task/action.go b/internal/querycoordv2/task/action.go new file mode 100644 index 0000000000000..aec13bd73ad61 --- /dev/null +++ b/internal/querycoordv2/task/action.go @@ -0,0 +1,124 @@ +package task + +import ( + "errors" + + "github.com/samber/lo" + "go.uber.org/atomic" + + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/util/typeutil" +) + +var ( + ErrActionCanceled = errors.New("ActionCanceled") + ErrActionRPCFailed = errors.New("ActionRPCFailed") + ErrActionStale = errors.New("ActionStale") +) + +type ActionType = int32 + +const ( + ActionTypeGrow ActionType = iota + 1 + ActionTypeReduce +) + +type Action interface { + Node() int64 + Type() ActionType + IsFinished(distMgr *meta.DistributionManager) bool +} + +type BaseAction struct { + nodeID UniqueID + typ ActionType + + onDone []func() +} + +func NewBaseAction(nodeID UniqueID, typ ActionType) *BaseAction { + return &BaseAction{ + nodeID: nodeID, + typ: typ, + } +} + +func (action *BaseAction) Node() int64 { + return action.nodeID +} + +func (action *BaseAction) Type() ActionType { + return action.typ +} + +type SegmentAction struct { + *BaseAction + + segmentID UniqueID + scope querypb.DataScope + + isReleaseCommitted atomic.Bool +} + +func NewSegmentAction(nodeID UniqueID, typ ActionType, segmentID UniqueID, onDone ...func()) *SegmentAction { + return NewSegmentActionWithScope(nodeID, typ, segmentID, querypb.DataScope_All, onDone...) +} +func NewSegmentActionWithScope(nodeID UniqueID, typ ActionType, segmentID UniqueID, scope querypb.DataScope, onDone ...func()) *SegmentAction { + base := NewBaseAction(nodeID, typ) + base.onDone = append(base.onDone, onDone...) + return &SegmentAction{ + BaseAction: base, + segmentID: segmentID, + scope: scope, + isReleaseCommitted: *atomic.NewBool(false), + } +} + +func (action *SegmentAction) SegmentID() UniqueID { + return action.segmentID +} + +func (action *SegmentAction) Scope() querypb.DataScope { + return action.scope +} + +func (action *SegmentAction) IsFinished(distMgr *meta.DistributionManager) bool { + if action.Type() == ActionTypeGrow { + nodes := distMgr.LeaderViewManager.GetSealedSegmentDist(action.SegmentID()) + return lo.Contains(nodes, action.Node()) + } + // FIXME: Now shard leader's segment view is a map of segment ID to node ID, + // loading segment replaces the node ID with the new one, + // which confuses the condition of finishing, + // the leader should return a map of segment ID to list of nodes, + // now, we just always commit the release task to executor once. + // NOTE: DO NOT create a task containing release action and the action is not the last action + + return action.isReleaseCommitted.Load() +} + +type ChannelAction struct { + *BaseAction + channelName string +} + +func NewChannelAction(nodeID UniqueID, typ ActionType, channelName string) *ChannelAction { + return &ChannelAction{ + BaseAction: NewBaseAction(nodeID, typ), + + channelName: channelName, + } +} + +func (action *ChannelAction) ChannelName() string { + return action.channelName +} + +func (action *ChannelAction) IsFinished(distMgr *meta.DistributionManager) bool { + nodes := distMgr.LeaderViewManager.GetChannelDist(action.ChannelName()) + hasNode := lo.Contains(nodes, action.Node()) + isGrow := action.Type() == ActionTypeGrow + + return hasNode == isGrow +} diff --git a/internal/querycoordv2/task/executor.go b/internal/querycoordv2/task/executor.go new file mode 100644 index 0000000000000..2538a956b80aa --- /dev/null +++ b/internal/querycoordv2/task/executor.go @@ -0,0 +1,319 @@ +package task + +import ( + "context" + "sync" + "time" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "go.uber.org/zap" +) + +const ( + actionTimeout = 10 * time.Second +) + +type actionIndex struct { + Task int64 + Step int +} + +type Executor struct { + meta *meta.Meta + dist *meta.DistributionManager + broker meta.Broker + targetMgr *meta.TargetManager + cluster session.Cluster + nodeMgr *session.NodeManager + + executingActions sync.Map +} + +func NewExecutor(meta *meta.Meta, + dist *meta.DistributionManager, + broker meta.Broker, + targetMgr *meta.TargetManager, + cluster session.Cluster, + nodeMgr *session.NodeManager) *Executor { + return &Executor{ + meta: meta, + dist: dist, + broker: broker, + targetMgr: targetMgr, + cluster: cluster, + nodeMgr: nodeMgr, + + executingActions: sync.Map{}, + } +} + +// Execute executes the given action, +// does nothing and returns false if the action is already committed, +// returns true otherwise. +func (ex *Executor) Execute(task Task, step int, action Action) bool { + index := actionIndex{ + Task: task.ID(), + Step: step, + } + _, exist := ex.executingActions.LoadOrStore(index, struct{}{}) + if exist { + return false + } + + log := log.With( + zap.Int64("task", task.ID()), + zap.Int("step", step), + zap.Int64("source", task.SourceID()), + ) + + go func() { + log.Info("execute the action of task") + switch action := action.(type) { + case *SegmentAction: + ex.executeSegmentAction(task.(*SegmentTask), action) + + case *ChannelAction: + ex.executeDmChannelAction(task.(*ChannelTask), action) + } + + ex.executingActions.Delete(index) + }() + + return true +} + +func (ex *Executor) executeSegmentAction(task *SegmentTask, action *SegmentAction) { + switch action.Type() { + case ActionTypeGrow: + ex.loadSegment(task, action) + + case ActionTypeReduce: + ex.releaseSegment(task, action) + } +} + +func (ex *Executor) loadSegment(task *SegmentTask, action *SegmentAction) { + log := log.With( + zap.Int64("task", task.ID()), + zap.Int64("collection", task.CollectionID()), + zap.Int64("segment", task.segmentID), + zap.Int64("node", action.Node()), + zap.Int64("source", task.SourceID()), + ) + + ctx, cancel := context.WithTimeout(task.Context(), actionTimeout) + defer cancel() + + schema, err := ex.broker.GetCollectionSchema(ctx, task.CollectionID()) + if err != nil { + log.Warn("failed to get schema of collection", zap.Error(err)) + return + } + partitions, err := utils.GetPartitions(ex.meta.CollectionManager, ex.broker, task.CollectionID()) + if err != nil { + log.Warn("failed to get partitions of collection", zap.Error(err)) + return + } + loadMeta := packLoadMeta( + ex.meta.GetLoadType(task.CollectionID()), + task.CollectionID(), + partitions..., + ) + segments, err := ex.broker.GetSegmentInfo(ctx, task.SegmentID()) + if err != nil || len(segments) == 0 { + log.Warn("failed to get segment info from DataCoord", zap.Error(err)) + return + } + segment := segments[0] + indexes, err := ex.broker.GetIndexInfo(ctx, task.CollectionID(), segment.GetID()) + if err != nil { + log.Warn("failed to get index of segment, will load without index") + } + loadInfo := utils.PackSegmentLoadInfo(segment, indexes) + + // Get shard leader for the given replica and segment + leader, ok := getShardLeader(ex.meta.ReplicaManager, ex.dist, task.CollectionID(), action.Node(), segment.GetInsertChannel()) + if !ok { + msg := "no shard leader for the segment to execute loading" + task.SetErr(utils.WrapError(msg, ErrTaskStale)) + log.Warn(msg, zap.String("shard", segment.GetInsertChannel())) + return + } + log = log.With(zap.Int64("shardLeader", leader)) + + deltaPositions, err := getSegmentDeltaPositions(ctx, ex.targetMgr, ex.broker, segment.GetCollectionID(), segment.GetPartitionID(), segment.GetInsertChannel()) + if err != nil { + log.Warn("failed to get delta positions of segment") + return + } + + req := packLoadSegmentRequest(task, action, schema, loadMeta, loadInfo, deltaPositions) + status, err := ex.cluster.LoadSegments(ctx, leader, req) + if err != nil { + log.Warn("failed to load segment, it may be a false failure", zap.Error(err)) + return + } + if status.ErrorCode != commonpb.ErrorCode_Success { + log.Warn("failed to load segment", zap.String("reason", status.GetReason())) + return + } +} + +func (ex *Executor) releaseSegment(task *SegmentTask, action *SegmentAction) { + defer action.isReleaseCommitted.Store(true) + + log := log.With( + zap.Int64("task", task.ID()), + zap.Int64("collection", task.CollectionID()), + zap.Int64("segment", task.segmentID), + zap.Int64("node", action.Node()), + zap.Int64("source", task.SourceID()), + ) + + ctx, cancel := context.WithTimeout(task.Context(), actionTimeout) + defer cancel() + + var targetSegment *meta.Segment + segments := ex.dist.SegmentDistManager.GetByNode(action.Node()) + for _, segment := range segments { + if segment.ID == task.SegmentID() { + targetSegment = segment + break + } + } + if targetSegment == nil { + log.Info("segment to release not found in distribution") + return + } + + req := packReleaseSegmentRequest(task, action, targetSegment.GetInsertChannel()) + + // Get shard leader for the given replica and segment + dstNode := action.Node() + if ex.meta.CollectionManager.Exist(task.CollectionID()) { + leader, ok := getShardLeader(ex.meta.ReplicaManager, ex.dist, task.CollectionID(), action.Node(), targetSegment.GetInsertChannel()) + if !ok { + log.Warn("no shard leader for the segment to execute loading", zap.String("shard", targetSegment.GetInsertChannel())) + return + } + dstNode = leader + log = log.With(zap.Int64("shardLeader", leader)) + req.NeedTransfer = true + } + status, err := ex.cluster.ReleaseSegments(ctx, dstNode, req) + if err != nil { + log.Warn("failed to release segment, it may be a false failure", zap.Error(err)) + return + } + if status.ErrorCode != commonpb.ErrorCode_Success { + log.Warn("failed to release segment", zap.String("reason", status.GetReason())) + return + } +} + +func (ex *Executor) executeDmChannelAction(task *ChannelTask, action *ChannelAction) { + switch action.Type() { + case ActionTypeGrow: + ex.subDmChannel(task, action) + + case ActionTypeReduce: + ex.unsubDmChannel(task, action) + } +} + +func (ex *Executor) subDmChannel(task *ChannelTask, action *ChannelAction) { + log := log.With( + zap.Int64("task", task.ID()), + zap.Int64("collection", task.CollectionID()), + zap.String("channel", task.Channel()), + zap.Int64("node", action.Node()), + zap.Int64("source", task.SourceID()), + ) + + ctx, cancel := context.WithTimeout(task.Context(), actionTimeout) + defer cancel() + + schema, err := ex.broker.GetCollectionSchema(ctx, task.CollectionID()) + if err != nil { + log.Warn("failed to get schema of collection") + return + } + partitions, err := utils.GetPartitions(ex.meta.CollectionManager, ex.broker, task.CollectionID()) + if err != nil { + log.Warn("failed to get partitions of collection") + return + } + loadMeta := packLoadMeta( + ex.meta.GetLoadType(task.CollectionID()), + task.CollectionID(), + partitions..., + ) + // DO NOT fetch channel info from DataCoord here, + // that may lead to leaking some data + // channels := make([]*datapb.VchannelInfo, 0, len(partitions)) + // for _, partition := range partitions { + // vchannels, _, err := ex.broker.GetRecoveryInfo(ctx, task.CollectionID(), partition) + // if err != nil { + // log.Warn("failed to get vchannel from DataCoord", zap.Error(err)) + // return + // } + + // for _, channel := range vchannels { + // if channel.ChannelName == action.ChannelName() { + // channels = append(channels, channel) + // } + // } + // } + // if len(channels) == 0 { + // log.Warn("no such channel in DataCoord") + // return + // } + + // dmChannel := utils.MergeDmChannelInfo(channels) + dmChannel := ex.targetMgr.GetDmChannel(action.ChannelName()) + req := packSubDmChannelRequest(task, action, schema, loadMeta, dmChannel) + err = fillSubDmChannelRequest(ctx, req, ex.broker) + if err != nil { + log.Warn("failed to subscribe DmChannel, failed to fill the request with segments", + zap.Error(err)) + return + } + status, err := ex.cluster.WatchDmChannels(ctx, action.Node(), req) + if err != nil { + log.Warn("failed to subscribe DmChannel, it may be a false failure", zap.Error(err)) + return + } + if status.ErrorCode != commonpb.ErrorCode_Success { + log.Warn("failed to subscribe DmChannel", zap.String("reason", status.GetReason())) + return + } + log.Info("subscribe DmChannel done") +} + +func (ex *Executor) unsubDmChannel(task *ChannelTask, action *ChannelAction) { + log := log.With( + zap.Int64("task", task.ID()), + zap.Int64("collection", task.CollectionID()), + zap.String("channel", task.Channel()), + zap.Int64("node", action.Node()), + zap.Int64("source", task.SourceID()), + ) + + ctx, cancel := context.WithTimeout(task.Context(), actionTimeout) + defer cancel() + + req := packUnsubDmChannelRequest(task, action) + status, err := ex.cluster.UnsubDmChannel(ctx, action.Node(), req) + if err != nil { + log.Warn("failed to unsubscribe DmChannel, it may be a false failure", zap.Error(err)) + return + } + if status.ErrorCode != commonpb.ErrorCode_Success { + log.Warn("failed to unsubscribe DmChannel", zap.String("reason", status.GetReason())) + return + } +} diff --git a/internal/querycoordv2/task/mock_scheduler.go b/internal/querycoordv2/task/mock_scheduler.go new file mode 100644 index 0000000000000..7d940f387e2c8 --- /dev/null +++ b/internal/querycoordv2/task/mock_scheduler.go @@ -0,0 +1,200 @@ +// Code generated by mockery v2.14.0. DO NOT EDIT. + +package task + +import mock "github.com/stretchr/testify/mock" + +// MockScheduler is an autogenerated mock type for the Scheduler type +type MockScheduler struct { + mock.Mock +} + +type MockScheduler_Expecter struct { + mock *mock.Mock +} + +func (_m *MockScheduler) EXPECT() *MockScheduler_Expecter { + return &MockScheduler_Expecter{mock: &_m.Mock} +} + +// Add provides a mock function with given fields: task +func (_m *MockScheduler) Add(task Task) error { + ret := _m.Called(task) + + var r0 error + if rf, ok := ret.Get(0).(func(Task) error); ok { + r0 = rf(task) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockScheduler_Add_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Add' +type MockScheduler_Add_Call struct { + *mock.Call +} + +// Add is a helper method to define mock.On call +// - task Task +func (_e *MockScheduler_Expecter) Add(task interface{}) *MockScheduler_Add_Call { + return &MockScheduler_Add_Call{Call: _e.mock.On("Add", task)} +} + +func (_c *MockScheduler_Add_Call) Run(run func(task Task)) *MockScheduler_Add_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(Task)) + }) + return _c +} + +func (_c *MockScheduler_Add_Call) Return(_a0 error) *MockScheduler_Add_Call { + _c.Call.Return(_a0) + return _c +} + +// Dispatch provides a mock function with given fields: node +func (_m *MockScheduler) Dispatch(node int64) { + _m.Called(node) +} + +// MockScheduler_Dispatch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Dispatch' +type MockScheduler_Dispatch_Call struct { + *mock.Call +} + +// Dispatch is a helper method to define mock.On call +// - node int64 +func (_e *MockScheduler_Expecter) Dispatch(node interface{}) *MockScheduler_Dispatch_Call { + return &MockScheduler_Dispatch_Call{Call: _e.mock.On("Dispatch", node)} +} + +func (_c *MockScheduler_Dispatch_Call) Run(run func(node int64)) *MockScheduler_Dispatch_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(int64)) + }) + return _c +} + +func (_c *MockScheduler_Dispatch_Call) Return() *MockScheduler_Dispatch_Call { + _c.Call.Return() + return _c +} + +// GetNodeChannelDelta provides a mock function with given fields: nodeID +func (_m *MockScheduler) GetNodeChannelDelta(nodeID int64) int { + ret := _m.Called(nodeID) + + var r0 int + if rf, ok := ret.Get(0).(func(int64) int); ok { + r0 = rf(nodeID) + } else { + r0 = ret.Get(0).(int) + } + + return r0 +} + +// MockScheduler_GetNodeChannelDelta_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetNodeChannelDelta' +type MockScheduler_GetNodeChannelDelta_Call struct { + *mock.Call +} + +// GetNodeChannelDelta is a helper method to define mock.On call +// - nodeID int64 +func (_e *MockScheduler_Expecter) GetNodeChannelDelta(nodeID interface{}) *MockScheduler_GetNodeChannelDelta_Call { + return &MockScheduler_GetNodeChannelDelta_Call{Call: _e.mock.On("GetNodeChannelDelta", nodeID)} +} + +func (_c *MockScheduler_GetNodeChannelDelta_Call) Run(run func(nodeID int64)) *MockScheduler_GetNodeChannelDelta_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(int64)) + }) + return _c +} + +func (_c *MockScheduler_GetNodeChannelDelta_Call) Return(_a0 int) *MockScheduler_GetNodeChannelDelta_Call { + _c.Call.Return(_a0) + return _c +} + +// GetNodeSegmentDelta provides a mock function with given fields: nodeID +func (_m *MockScheduler) GetNodeSegmentDelta(nodeID int64) int { + ret := _m.Called(nodeID) + + var r0 int + if rf, ok := ret.Get(0).(func(int64) int); ok { + r0 = rf(nodeID) + } else { + r0 = ret.Get(0).(int) + } + + return r0 +} + +// MockScheduler_GetNodeSegmentDelta_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetNodeSegmentDelta' +type MockScheduler_GetNodeSegmentDelta_Call struct { + *mock.Call +} + +// GetNodeSegmentDelta is a helper method to define mock.On call +// - nodeID int64 +func (_e *MockScheduler_Expecter) GetNodeSegmentDelta(nodeID interface{}) *MockScheduler_GetNodeSegmentDelta_Call { + return &MockScheduler_GetNodeSegmentDelta_Call{Call: _e.mock.On("GetNodeSegmentDelta", nodeID)} +} + +func (_c *MockScheduler_GetNodeSegmentDelta_Call) Run(run func(nodeID int64)) *MockScheduler_GetNodeSegmentDelta_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(int64)) + }) + return _c +} + +func (_c *MockScheduler_GetNodeSegmentDelta_Call) Return(_a0 int) *MockScheduler_GetNodeSegmentDelta_Call { + _c.Call.Return(_a0) + return _c +} + +// RemoveByNode provides a mock function with given fields: node +func (_m *MockScheduler) RemoveByNode(node int64) { + _m.Called(node) +} + +// MockScheduler_RemoveByNode_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RemoveByNode' +type MockScheduler_RemoveByNode_Call struct { + *mock.Call +} + +// RemoveByNode is a helper method to define mock.On call +// - node int64 +func (_e *MockScheduler_Expecter) RemoveByNode(node interface{}) *MockScheduler_RemoveByNode_Call { + return &MockScheduler_RemoveByNode_Call{Call: _e.mock.On("RemoveByNode", node)} +} + +func (_c *MockScheduler_RemoveByNode_Call) Run(run func(node int64)) *MockScheduler_RemoveByNode_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(int64)) + }) + return _c +} + +func (_c *MockScheduler_RemoveByNode_Call) Return() *MockScheduler_RemoveByNode_Call { + _c.Call.Return() + return _c +} + +type mockConstructorTestingTNewMockScheduler interface { + mock.TestingT + Cleanup(func()) +} + +// NewMockScheduler creates a new instance of MockScheduler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMockScheduler(t mockConstructorTestingTNewMockScheduler) *MockScheduler { + mock := &MockScheduler{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/querycoordv2/task/scheduler.go b/internal/querycoordv2/task/scheduler.go new file mode 100644 index 0000000000000..1310131ebe93f --- /dev/null +++ b/internal/querycoordv2/task/scheduler.go @@ -0,0 +1,677 @@ +package task + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + . "github.com/milvus-io/milvus/internal/util/typeutil" + "go.uber.org/zap" +) + +const ( + TaskTypeGrow Type = iota + 1 + TaskTypeReduce + TaskTypeMove + + taskPoolSize = 128 +) + +var ( + ErrConflictTaskExisted = errors.New("ConflictTaskExisted") + + // The task is canceled or timeout + ErrTaskCanceled = errors.New("TaskCanceled") + + // The target node is offline, + // or the target segment is not in TargetManager, + // or the target channel is not in TargetManager + ErrTaskStale = errors.New("TaskStale") + + // No enough memory to load segment + ErrResourceNotEnough = errors.New("ResourceNotEnough") + + ErrTaskQueueFull = errors.New("TaskQueueFull") +) + +type Type = int32 + +type replicaSegmentIndex struct { + ReplicaID int64 + SegmentID int64 +} + +type replicaChannelIndex struct { + ReplicaID int64 + Channel string +} + +type taskQueue struct { + // TaskPriority -> Tasks + buckets [][]Task + + cap int +} + +func newTaskQueue(cap int) *taskQueue { + return &taskQueue{ + buckets: make([][]Task, len(TaskPriorities)), + + cap: cap, + } +} + +func (queue *taskQueue) Len() int { + taskNum := 0 + for _, tasks := range queue.buckets { + taskNum += len(tasks) + } + + return taskNum +} + +func (queue *taskQueue) Cap() int { + return queue.cap +} + +func (queue *taskQueue) Add(task Task) bool { + if queue.Len() >= queue.Cap() { + return false + } + + queue.buckets[task.Priority()] = append(queue.buckets[task.Priority()], task) + return true +} + +func (queue *taskQueue) Remove(task Task) { + bucket := &queue.buckets[task.Priority()] + + for i := range *bucket { + if (*bucket)[i].ID() == task.ID() { + *bucket = append((*bucket)[:i], (*bucket)[i+1:]...) + break + } + } +} + +// Range iterates all tasks in the queue ordered by priority from high to low +func (queue *taskQueue) Range(fn func(task Task) bool) { + for priority := len(queue.buckets) - 1; priority >= 0; priority-- { + for i := range queue.buckets[priority] { + if !fn(queue.buckets[priority][i]) { + return + } + } + } +} + +type Scheduler interface { + Add(task Task) error + Dispatch(node int64) + RemoveByNode(node int64) + GetNodeSegmentDelta(nodeID int64) int + GetNodeChannelDelta(nodeID int64) int +} + +type taskScheduler struct { + rwmutex sync.RWMutex + ctx context.Context + executor *Executor + idAllocator func() UniqueID + + distMgr *meta.DistributionManager + meta *meta.Meta + targetMgr *meta.TargetManager + broker meta.Broker + nodeMgr *session.NodeManager + + tasks UniqueSet + segmentTasks map[replicaSegmentIndex]Task + channelTasks map[replicaChannelIndex]Task + processQueue *taskQueue + waitQueue *taskQueue +} + +func NewScheduler(ctx context.Context, + meta *meta.Meta, + distMgr *meta.DistributionManager, + targetMgr *meta.TargetManager, + broker meta.Broker, + cluster session.Cluster, + nodeMgr *session.NodeManager) *taskScheduler { + id := int64(0) + return &taskScheduler{ + ctx: ctx, + executor: NewExecutor(meta, distMgr, broker, targetMgr, cluster, nodeMgr), + idAllocator: func() UniqueID { + id++ + return id + }, + + distMgr: distMgr, + meta: meta, + targetMgr: targetMgr, + broker: broker, + nodeMgr: nodeMgr, + + tasks: make(UniqueSet), + segmentTasks: make(map[replicaSegmentIndex]Task), + channelTasks: make(map[replicaChannelIndex]Task), + processQueue: newTaskQueue(taskPoolSize), + waitQueue: newTaskQueue(taskPoolSize * 10), + } +} + +func (scheduler *taskScheduler) Add(task Task) error { + scheduler.rwmutex.Lock() + defer scheduler.rwmutex.Unlock() + + err := scheduler.preAdd(task) + if err != nil { + return err + } + + task.SetID(scheduler.idAllocator()) + scheduler.tasks.Insert(task.ID()) + switch task := task.(type) { + case *SegmentTask: + index := replicaSegmentIndex{task.ReplicaID(), task.segmentID} + scheduler.segmentTasks[index] = task + + case *ChannelTask: + index := replicaChannelIndex{task.ReplicaID(), task.channel} + scheduler.channelTasks[index] = task + } + if !scheduler.waitQueue.Add(task) { + log.Warn("failed to add task", zap.String("task", task.String())) + return nil + } + log.Info("task added", zap.String("task", task.String())) + return nil +} + +// check checks whether the task is valid to add, +// must hold lock +func (scheduler *taskScheduler) preAdd(task Task) error { + if scheduler.waitQueue.Len() >= scheduler.waitQueue.Cap() { + return ErrTaskQueueFull + } + + switch task := task.(type) { + case *SegmentTask: + index := replicaSegmentIndex{task.ReplicaID(), task.segmentID} + if old, ok := scheduler.segmentTasks[index]; ok { + if task.Priority() > old.Priority() { + log.Info("replace old task, the new one with higher priority", + zap.Int64("oldID", old.ID()), + zap.Int32("oldPrioprity", old.Priority()), + zap.Int64("newID", task.ID()), + zap.Int32("newPriority", task.Priority()), + ) + old.SetStatus(TaskStatusCanceled) + old.SetErr(utils.WrapError("replaced with the other one with higher priority", ErrTaskCanceled)) + scheduler.remove(old) + return nil + } + + return ErrConflictTaskExisted + } + + case *ChannelTask: + index := replicaChannelIndex{task.ReplicaID(), task.channel} + if old, ok := scheduler.channelTasks[index]; ok { + if task.Priority() > old.Priority() { + log.Info("replace old task, the new one with higher priority", + zap.Int64("oldID", old.ID()), + zap.Int32("oldPriority", old.Priority()), + zap.Int64("newID", task.ID()), + zap.Int32("newPriority", task.Priority()), + ) + old.SetStatus(TaskStatusCanceled) + old.SetErr(utils.WrapError("replaced with the other one with higher priority", ErrTaskCanceled)) + scheduler.remove(old) + return nil + } + + return ErrConflictTaskExisted + } + + default: + panic(fmt.Sprintf("preAdd: forget to process task type: %+v", task)) + } + + return nil +} + +func (scheduler *taskScheduler) promote(task Task) error { + log := log.With( + zap.Int64("collection", task.CollectionID()), + zap.Int64("task", task.ID()), + zap.Int64("source", task.SourceID()), + ) + err := scheduler.prePromote(task) + if err != nil { + log.Info("failed to promote task", zap.Error(err)) + return err + } + + if scheduler.processQueue.Add(task) { + task.SetStatus(TaskStatusStarted) + return nil + } + + return ErrTaskQueueFull +} + +func (scheduler *taskScheduler) tryPromoteAll() { + // Promote waiting tasks + toPromote := make([]Task, 0, scheduler.processQueue.Cap()-scheduler.processQueue.Len()) + toRemove := make([]Task, 0) + scheduler.waitQueue.Range(func(task Task) bool { + err := scheduler.promote(task) + if errors.Is(err, ErrTaskStale) { // Task canceled or stale + task.SetStatus(TaskStatusStale) + task.SetErr(err) + toRemove = append(toRemove, task) + } else if errors.Is(err, ErrTaskCanceled) { + task.SetStatus(TaskStatusCanceled) + task.SetErr(err) + toRemove = append(toRemove, task) + } else if err == nil { + toPromote = append(toPromote, task) + } + + return !errors.Is(err, ErrTaskQueueFull) + }) + + for _, task := range toPromote { + scheduler.waitQueue.Remove(task) + } + for _, task := range toRemove { + scheduler.remove(task) + } + + if len(toPromote) > 0 || len(toRemove) > 0 { + log.Debug("promoted tasks", + zap.Int("promotedNum", len(toPromote)), + zap.Int("toRemoveNum", len(toRemove))) + } +} + +func (scheduler *taskScheduler) prePromote(task Task) error { + if scheduler.checkCanceled(task) { + return ErrTaskCanceled + } else if scheduler.checkStale(task) { + return ErrTaskStale + } + + return nil +} + +func (scheduler *taskScheduler) Dispatch(node int64) { + select { + case <-scheduler.ctx.Done(): + log.Info("scheduler stopped") + + default: + scheduler.schedule(node) + } +} + +func (scheduler *taskScheduler) GetNodeSegmentDelta(nodeID int64) int { + scheduler.rwmutex.RLock() + defer scheduler.rwmutex.RUnlock() + + return calculateNodeDelta(nodeID, scheduler.segmentTasks) +} + +func (scheduler *taskScheduler) GetNodeChannelDelta(nodeID int64) int { + scheduler.rwmutex.RLock() + defer scheduler.rwmutex.RUnlock() + + return calculateNodeDelta(nodeID, scheduler.channelTasks) +} + +func calculateNodeDelta[K comparable, T ~map[K]Task](nodeID int64, tasks T) int { + delta := 0 + for _, task := range tasks { + for _, action := range task.Actions() { + if action.Node() != nodeID { + continue + } + if action.Type() == ActionTypeGrow { + delta++ + } else if action.Type() == ActionTypeReduce { + delta-- + } + } + } + return delta +} + +func (scheduler *taskScheduler) GetNodeSegmentCntDelta(nodeID int64) int { + scheduler.rwmutex.RLock() + defer scheduler.rwmutex.RUnlock() + + delta := 0 + for _, task := range scheduler.segmentTasks { + for _, action := range task.Actions() { + if action.Node() != nodeID { + continue + } + segmentAction := action.(*SegmentAction) + segment := scheduler.targetMgr.GetSegment(segmentAction.SegmentID()) + if action.Type() == ActionTypeGrow { + delta += int(segment.GetNumOfRows()) + } else { + delta -= int(segment.GetNumOfRows()) + } + } + } + return delta +} + +// schedule selects some tasks to execute, follow these steps for each started selected tasks: +// 1. check whether this task is stale, set status to failed if stale +// 2. step up the task's actions, set status to succeeded if all actions finished +// 3. execute the current action of task +func (scheduler *taskScheduler) schedule(node int64) { + scheduler.rwmutex.Lock() + defer scheduler.rwmutex.Unlock() + + if scheduler.tasks.Len() == 0 { + return + } + + log := log.With( + zap.Int64("nodeID", node), + ) + + scheduler.tryPromoteAll() + + log.Debug("process tasks related to node", + zap.Int("processing-task-num", scheduler.processQueue.Len()), + zap.Int("waiting-task-num", scheduler.waitQueue.Len()), + zap.Int("segment-task-num", len(scheduler.segmentTasks)), + zap.Int("channel-task-num", len(scheduler.channelTasks)), + ) + + // Process tasks + toRemove := make([]Task, 0) + scheduler.processQueue.Range(func(task Task) bool { + log.Debug("check task related", + zap.Int64("task", task.ID())) + if scheduler.isRelated(task, node) { + scheduler.process(task) + } else { + log.Debug("task not related, skip it", + zap.Int64("task", task.ID()), + zap.Int64("taskActionNode", task.Actions()[0].Node()), + ) + } + + if task.Status() != TaskStatusStarted { + toRemove = append(toRemove, task) + } + + return true + }) + + for _, task := range toRemove { + scheduler.remove(task) + } + + log.Info("processed tasks", + zap.Int("toRemoveNum", len(toRemove))) + + log.Debug("process tasks related to node done", + zap.Int("processing-task-num", scheduler.processQueue.Len()), + zap.Int("waiting-task-num", scheduler.waitQueue.Len()), + zap.Int("segment-task-num", len(scheduler.segmentTasks)), + zap.Int("channel-task-num", len(scheduler.channelTasks)), + ) +} + +func (scheduler *taskScheduler) isRelated(task Task, node int64) bool { + for _, action := range task.Actions() { + if action.Node() == node { + return true + } + if task, ok := task.(*SegmentTask); ok { + segment := scheduler.targetMgr.GetSegment(task.SegmentID()) + if segment == nil { + continue + } + replica := scheduler.meta.ReplicaManager.GetByCollectionAndNode(task.CollectionID(), action.Node()) + if replica == nil { + continue + } + leader, ok := scheduler.distMgr.GetShardLeader(replica, segment.GetInsertChannel()) + if !ok { + continue + } + if leader == node { + return true + } + } + } + return false +} + +// process processes the given task, +// return true if the task is started and succeeds to commit the current action +func (scheduler *taskScheduler) process(task Task) bool { + log := log.With( + zap.Int64("task", task.ID()), + zap.Int32("type", GetTaskType(task)), + zap.Int64("source", task.SourceID()), + ) + + if task.IsFinished(scheduler.distMgr) { + task.SetStatus(TaskStatusSucceeded) + } else if scheduler.checkCanceled(task) { + task.SetStatus(TaskStatusCanceled) + task.SetErr(ErrTaskCanceled) + } else if scheduler.checkStale(task) { + task.SetStatus(TaskStatusStale) + task.SetErr(ErrTaskStale) + } + + actions, step := task.Actions(), task.Step() + log = log.With(zap.Int("step", step)) + switch task.Status() { + case TaskStatusStarted: + if scheduler.executor.Execute(task, step, actions[step]) { + return true + } + + case TaskStatusSucceeded: + log.Info("task succeeded") + + case TaskStatusCanceled, TaskStatusStale: + log.Warn("failed to execute task", zap.Error(task.Err())) + + default: + panic(fmt.Sprintf("invalid task status: %v", task.Status())) + } + + return false +} + +func (scheduler *taskScheduler) RemoveByNode(node int64) { + scheduler.rwmutex.Lock() + defer scheduler.rwmutex.Unlock() + + for _, task := range scheduler.segmentTasks { + if scheduler.isRelated(task, node) { + scheduler.remove(task) + } + } + for _, task := range scheduler.channelTasks { + if scheduler.isRelated(task, node) { + scheduler.remove(task) + } + } +} + +func (scheduler *taskScheduler) remove(task Task) { + log := log.With( + zap.Int64("task", task.ID()), + ) + task.Cancel() + scheduler.tasks.Remove(task.ID()) + scheduler.waitQueue.Remove(task) + scheduler.processQueue.Remove(task) + + switch task := task.(type) { + case *SegmentTask: + index := replicaSegmentIndex{task.ReplicaID(), task.SegmentID()} + delete(scheduler.segmentTasks, index) + log = log.With(zap.Int64("segment", task.SegmentID())) + + case *ChannelTask: + index := replicaChannelIndex{task.ReplicaID(), task.Channel()} + delete(scheduler.channelTasks, index) + log = log.With(zap.String("channel", task.Channel())) + } + + log.Debug("task removed") +} + +func (scheduler *taskScheduler) checkCanceled(task Task) bool { + log := log.With( + zap.Int64("task", task.ID()), + zap.Int64("source", task.SourceID()), + ) + + select { + case <-task.Context().Done(): + log.Warn("the task is timeout or canceled") + return true + + default: + return false + } +} + +func (scheduler *taskScheduler) checkStale(task Task) bool { + log := log.With( + zap.Int64("task", task.ID()), + zap.Int64("source", task.SourceID()), + ) + + switch task := task.(type) { + case *SegmentTask: + if scheduler.checkSegmentTaskStale(task) { + return true + } + + case *ChannelTask: + if scheduler.checkChannelTaskStale(task) { + return true + } + + default: + panic(fmt.Sprintf("checkStale: forget to check task type: %+v", task)) + } + + for step, action := range task.Actions() { + log := log.With( + zap.Int64("nodeID", action.Node()), + zap.Int("step", step)) + + if scheduler.nodeMgr.Get(action.Node()) == nil { + log.Warn("the task is stale, the target node is offline") + return true + } + } + + return false +} + +func (scheduler *taskScheduler) checkSegmentTaskStale(task *SegmentTask) bool { + log := log.With( + zap.Int64("task", task.ID()), + zap.Int64("source", task.SourceID()), + ) + + for _, action := range task.Actions() { + switch action.Type() { + case ActionTypeGrow: + segment := scheduler.targetMgr.GetSegment(task.SegmentID()) + if segment == nil { + log.Warn("task stale due tu the segment to load not exists in targets", + zap.Int64("segment", task.segmentID)) + return true + } + replica := scheduler.meta.ReplicaManager.GetByCollectionAndNode(task.CollectionID(), action.Node()) + if replica == nil { + log.Warn("task stale due to replica not found") + return true + } + _, ok := scheduler.distMgr.GetShardLeader(replica, segment.GetInsertChannel()) + if !ok { + log.Warn("task stale due to leader not found") + return true + } + + case ActionTypeReduce: + // Do nothing here, + // the task should succeeded if the segment not exists + // sealed := scheduler.distMgr.SegmentDistManager.GetByNode(action.Node()) + // growing := scheduler.distMgr.LeaderViewManager.GetSegmentByNode(action.Node()) + // segments := make([]int64, 0, len(sealed)+len(growing)) + // for _, segment := range sealed { + // segments = append(segments, segment.GetID()) + // } + // segments = append(segments, growing...) + // if !funcutil.SliceContain(segments, task.SegmentID()) { + // log.Warn("the task is stale, the segment to release not exists in dist", + // zap.Int64("segment", task.segmentID)) + // return true + // } + } + } + return false +} + +func (scheduler *taskScheduler) checkChannelTaskStale(task *ChannelTask) bool { + log := log.With( + zap.Int64("task", task.ID()), + zap.Int64("source", task.SourceID()), + ) + + for _, action := range task.Actions() { + switch action.Type() { + case ActionTypeGrow: + if !scheduler.targetMgr.ContainDmChannel(task.Channel()) { + log.Warn("the task is stale, the channel to subscribe not exists in targets", + zap.String("channel", task.Channel())) + return true + } + + case ActionTypeReduce: + // Do nothing here, + // the task should succeeded if the channel not exists + // hasChannel := false + // views := scheduler.distMgr.LeaderViewManager.GetLeaderView(action.Node()) + // for _, view := range views { + // if view.Channel == task.Channel() { + // hasChannel = true + // break + // } + // } + // if !hasChannel { + // log.Warn("the task is stale, the channel to unsubscribe not exists in dist", + // zap.String("channel", task.Channel())) + // return true + // } + } + } + return false +} diff --git a/internal/querycoordv2/task/task.go b/internal/querycoordv2/task/task.go new file mode 100644 index 0000000000000..1c3e62ac3d5b7 --- /dev/null +++ b/internal/querycoordv2/task/task.go @@ -0,0 +1,303 @@ +package task + +import ( + "context" + "fmt" + "time" + + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/util/typeutil" + "go.uber.org/atomic" +) + +type Status = int32 +type Priority = int32 + +const ( + TaskStatusCreated Status = iota + 1 + TaskStatusStarted + TaskStatusSucceeded + TaskStatusCanceled + TaskStatusStale +) + +const ( + TaskPriorityLow = iota + TaskPriorityNormal + TaskPriorityHigh +) + +var ( + // All task priorities from low to high + TaskPriorities = []Priority{TaskPriorityLow, TaskPriorityNormal, TaskPriorityHigh} +) + +type Task interface { + Context() context.Context + SourceID() UniqueID + ID() UniqueID + CollectionID() UniqueID + ReplicaID() UniqueID + SetID(id UniqueID) + Status() Status + SetStatus(status Status) + Err() error + SetErr(err error) + Priority() Priority + SetPriority(priority Priority) + + Cancel() + Wait() error + Actions() []Action + Step() int + IsFinished(dist *meta.DistributionManager) bool + String() string +} + +type baseTask struct { + ctx context.Context + cancel context.CancelFunc + doneCh chan struct{} + canceled *atomic.Bool + + sourceID UniqueID // RequestID + id UniqueID // Set by scheduler + collectionID UniqueID + replicaID UniqueID + loadType querypb.LoadType + + status *atomic.Int32 + priority Priority + err error + actions []Action + step int + + successCallbacks []func() + failureCallbacks []func() +} + +func newBaseTask(ctx context.Context, timeout time.Duration, sourceID, collectionID, replicaID UniqueID) *baseTask { + ctx, cancel := context.WithTimeout(ctx, timeout) + + return &baseTask{ + sourceID: sourceID, + collectionID: collectionID, + replicaID: replicaID, + + status: atomic.NewInt32(TaskStatusStarted), + priority: TaskPriorityNormal, + ctx: ctx, + cancel: cancel, + doneCh: make(chan struct{}), + canceled: atomic.NewBool(false), + } +} + +func (task *baseTask) Context() context.Context { + return task.ctx +} + +func (task *baseTask) SourceID() UniqueID { + return task.sourceID +} + +func (task *baseTask) ID() UniqueID { + return task.id +} + +func (task *baseTask) SetID(id UniqueID) { + task.id = id +} + +func (task *baseTask) CollectionID() UniqueID { + return task.collectionID +} + +func (task *baseTask) ReplicaID() UniqueID { + return task.replicaID +} + +func (task *baseTask) LoadType() querypb.LoadType { + return task.loadType +} + +func (task *baseTask) Status() Status { + return task.status.Load() +} + +func (task *baseTask) SetStatus(status Status) { + task.status.Store(status) +} + +func (task *baseTask) Priority() Priority { + return task.priority +} + +func (task *baseTask) SetPriority(priority Priority) { + task.priority = priority +} + +func (task *baseTask) Err() error { + return task.err +} + +func (task *baseTask) SetErr(err error) { + task.err = err +} + +func (task *baseTask) Cancel() { + if task.canceled.CAS(false, true) { + task.cancel() + close(task.doneCh) + } +} + +func (task *baseTask) Wait() error { + <-task.doneCh + return task.err +} + +func (task *baseTask) Actions() []Action { + return task.actions +} + +func (task *baseTask) Step() int { + return task.step +} + +func (task *baseTask) IsFinished(distMgr *meta.DistributionManager) bool { + if task.Status() != TaskStatusStarted { + return false + } + + actions, step := task.Actions(), task.Step() + for step < len(actions) && actions[step].IsFinished(distMgr) { + task.step++ + step++ + } + + return task.Step() >= len(actions) +} + +func (task *baseTask) String() string { + var actionsStr string + for i, action := range task.actions { + if realAction, ok := action.(*SegmentAction); ok { + actionsStr += fmt.Sprintf(`{[type=%v][node=%d][streaming=%v]}`, action.Type(), action.Node(), realAction.Scope() == querypb.DataScope_Streaming) + } else { + actionsStr += fmt.Sprintf(`{[type=%v][node=%d]}`, action.Type(), action.Node()) + } + if i != len(task.actions)-1 { + actionsStr += ", " + } + } + return fmt.Sprintf( + "[id=%d] [type=%v] [collectionID=%d] [replicaID=%d] [priority=%d] [actionsCount=%d] [actions=%s]", + task.id, + GetTaskType(task), + task.collectionID, + task.replicaID, + task.priority, + len(task.actions), + actionsStr, + ) +} + +type SegmentTask struct { + *baseTask + + segmentID UniqueID +} + +// NewSegmentTask creates a SegmentTask with actions, +// all actions must process the same segment, +// empty actions is not allowed +func NewSegmentTask(ctx context.Context, + timeout time.Duration, + sourceID, + collectionID, + replicaID UniqueID, + actions ...Action) *SegmentTask { + if len(actions) == 0 { + panic("empty actions is not allowed") + } + + segmentID := int64(-1) + for _, action := range actions { + action, ok := action.(*SegmentAction) + if !ok { + panic("SegmentTask can only contain SegmentActions") + } + if segmentID == -1 { + segmentID = action.segmentID + } else if segmentID != action.SegmentID() { + panic("all actions must process the same segment") + } + } + + base := newBaseTask(ctx, timeout, sourceID, collectionID, replicaID) + base.actions = actions + return &SegmentTask{ + baseTask: base, + + segmentID: segmentID, + } +} + +func (task *SegmentTask) SegmentID() UniqueID { + return task.segmentID +} + +func (task *SegmentTask) String() string { + return fmt.Sprintf("%s [segmentID=%d]", task.baseTask.String(), task.segmentID) +} + +type ChannelTask struct { + *baseTask + + channel string +} + +// NewChannelTask creates a ChannelTask with actions, +// all actions must process the same channel, and the same type of channel +// empty actions is not allowed +func NewChannelTask(ctx context.Context, + timeout time.Duration, + sourceID, + collectionID, + replicaID UniqueID, + actions ...Action) *ChannelTask { + if len(actions) == 0 { + panic("empty actions is not allowed") + } + + channel := "" + for _, action := range actions { + channelAction, ok := action.(interface{ ChannelName() string }) + if !ok { + panic("ChannelTask must contain only ChannelAction") + } + if channel == "" { + channel = channelAction.ChannelName() + } else if channel != channelAction.ChannelName() { + panic("all actions must process the same channel") + } + } + + base := newBaseTask(ctx, timeout, sourceID, collectionID, replicaID) + base.actions = actions + return &ChannelTask{ + baseTask: base, + + channel: channel, + } +} + +func (task *ChannelTask) Channel() string { + return task.channel +} + +func (task *ChannelTask) String() string { + return fmt.Sprintf("%s [channel=%s]", task.baseTask.String(), task.channel) +} diff --git a/internal/querycoordv2/task/task_test.go b/internal/querycoordv2/task/task_test.go new file mode 100644 index 0000000000000..dde7d6d0d9190 --- /dev/null +++ b/internal/querycoordv2/task/task_test.go @@ -0,0 +1,856 @@ +package task + +import ( + "context" + "math/rand" + "testing" + "time" + + "github.com/milvus-io/milvus/internal/kv" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/proto/schemapb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/internal/util/etcd" + "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type distribution struct { + NodeID int64 + channels typeutil.Set[string] + segments typeutil.Set[int64] +} + +type TaskSuite struct { + suite.Suite + + // Data + collection int64 + replica int64 + subChannels []string + unsubChannels []string + moveChannels []string + growingSegments map[string]int64 + loadSegments []int64 + releaseSegments []int64 + moveSegments []int64 + distributions map[int64]*distribution + + // Dependencies + kv kv.MetaKv + store meta.Store + meta *meta.Meta + dist *meta.DistributionManager + target *meta.TargetManager + broker *meta.MockBroker + nodeMgr *session.NodeManager + cluster *session.MockCluster + + // Test object + scheduler *taskScheduler +} + +func (suite *TaskSuite) SetupSuite() { + Params.Init() + suite.collection = 1000 + suite.replica = 10 + suite.subChannels = []string{ + "sub-0", + "sub-1", + } + suite.unsubChannels = []string{ + "unsub-2", + "unsub-3", + } + suite.moveChannels = []string{ + "move-4", + "move-5", + } + suite.growingSegments = map[string]int64{ + "sub-0": 10, + "sub-1": 11, + } + suite.loadSegments = []int64{1, 2} + suite.releaseSegments = []int64{3, 4} + suite.moveSegments = []int64{5, 6} + suite.distributions = map[int64]*distribution{ + 1: { + NodeID: 1, + channels: typeutil.NewSet("unsub-2", "move-4"), + segments: typeutil.NewSet[int64](3, 5), + }, + 2: { + NodeID: 2, + channels: typeutil.NewSet("unsub-3", "move-5"), + segments: typeutil.NewSet[int64](4, 6), + }, + 3: { + NodeID: 3, + channels: typeutil.NewSet[string](), + segments: typeutil.NewSet[int64](), + }, + } +} + +func (suite *TaskSuite) SetupTest() { + config := GenerateEtcdConfig() + cli, err := etcd.GetEtcdClient(&config) + suite.Require().NoError(err) + + suite.kv = etcdkv.NewEtcdKV(cli, config.MetaRootPath) + suite.store = meta.NewMetaStore(suite.kv) + suite.meta = meta.NewMeta(RandomIncrementIDAllocator(), suite.store) + suite.dist = meta.NewDistributionManager() + suite.target = meta.NewTargetManager() + suite.broker = meta.NewMockBroker(suite.T()) + suite.nodeMgr = session.NewNodeManager() + suite.cluster = session.NewMockCluster(suite.T()) + + suite.scheduler = suite.newScheduler() +} + +func (suite *TaskSuite) BeforeTest(suiteName, testName string) { + for node := range suite.distributions { + suite.nodeMgr.Add(session.NewNodeInfo(node, "localhost")) + } + + switch testName { + case "TestSubscribeChannelTask", + "TestLoadSegmentTask", + "TestSegmentTaskStale", + "TestMoveSegmentTask": + suite.meta.PutCollection(&meta.Collection{ + CollectionLoadInfo: &querypb.CollectionLoadInfo{ + CollectionID: suite.collection, + ReplicaNumber: 1, + Status: querypb.LoadStatus_Loading, + }, + }) + suite.meta.ReplicaManager.Put( + utils.CreateTestReplica(suite.replica, suite.collection, []int64{1, 2, 3})) + } +} + +func (suite *TaskSuite) TestSubscribeChannelTask() { + ctx := context.Background() + timeout := 10 * time.Second + targetNode := int64(3) + partitions := []int64{100, 101} + + // Expect + suite.broker.EXPECT().GetCollectionSchema(mock.Anything, suite.collection). + Return(&schemapb.CollectionSchema{ + Name: "TestSubscribeChannelTask", + }, nil) + suite.broker.EXPECT().GetPartitions(mock.Anything, suite.collection). + Return([]int64{100, 101}, nil) + channels := make([]*datapb.VchannelInfo, 0, len(suite.subChannels)) + for _, channel := range suite.subChannels { + channels = append(channels, &datapb.VchannelInfo{ + CollectionID: suite.collection, + ChannelName: channel, + UnflushedSegmentIds: []int64{suite.growingSegments[channel]}, + }) + } + for channel, segment := range suite.growingSegments { + suite.broker.EXPECT().GetSegmentInfo(mock.Anything, segment). + Return([]*datapb.SegmentInfo{ + { + ID: segment, + CollectionID: suite.collection, + PartitionID: partitions[0], + InsertChannel: channel, + }, + }, nil) + } + // for _, partition := range partitions { + // suite.broker.EXPECT().GetRecoveryInfo(mock.Anything, suite.collection, partition). + // Return(channels, nil, nil) + // } + suite.cluster.EXPECT().WatchDmChannels(mock.Anything, targetNode, mock.Anything).Return(utils.WrapStatus(commonpb.ErrorCode_Success, ""), nil) + + // Test subscribe channel task + tasks := []Task{} + for _, channel := range suite.subChannels { + suite.target.AddDmChannel(meta.DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: suite.collection, + ChannelName: channel, + UnflushedSegmentIds: []int64{suite.growingSegments[channel]}, + })) + task := NewChannelTask( + ctx, + timeout, + 0, + suite.collection, + suite.replica, + NewChannelAction(targetNode, ActionTypeGrow, channel), + ) + tasks = append(tasks, task) + err := suite.scheduler.Add(task) + suite.NoError(err) + } + suite.AssertTaskNum(0, len(suite.subChannels), len(suite.subChannels), 0) + + // Process tasks + suite.dispatchAndWait(targetNode) + suite.AssertTaskNum(len(suite.subChannels), 0, len(suite.subChannels), 0) + + // Other nodes' HB can't trigger the procedure of tasks + suite.dispatchAndWait(targetNode + 1) + suite.AssertTaskNum(len(suite.subChannels), 0, len(suite.subChannels), 0) + + // Process tasks done + // Dist contains channels + views := make([]*meta.LeaderView, 0) + for _, channel := range suite.subChannels { + views = append(views, &meta.LeaderView{ + ID: targetNode, + CollectionID: suite.collection, + Channel: channel, + }) + } + suite.dist.LeaderViewManager.Update(targetNode, views...) + suite.dispatchAndWait(targetNode) + suite.AssertTaskNum(0, 0, 0, 0) + + Wait(ctx, timeout, tasks...) + for _, task := range tasks { + suite.Equal(TaskStatusSucceeded, task.Status()) + suite.NoError(task.Err()) + } +} + +func (suite *TaskSuite) TestUnsubscribeChannelTask() { + ctx := context.Background() + timeout := 10 * time.Second + targetNode := int64(1) + + // Expect + suite.cluster.EXPECT().UnsubDmChannel(mock.Anything, targetNode, mock.Anything).Return(utils.WrapStatus(commonpb.ErrorCode_Success, ""), nil) + + // Test unsubscribe channel task + tasks := []Task{} + for _, channel := range suite.unsubChannels { + suite.target.AddDmChannel(meta.DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: suite.collection, + ChannelName: channel, + })) + task := NewChannelTask( + ctx, + timeout, + 0, + suite.collection, + -1, + NewChannelAction(targetNode, ActionTypeReduce, channel), + ) + tasks = append(tasks, task) + err := suite.scheduler.Add(task) + suite.NoError(err) + } + // Only first channel exists + suite.dist.LeaderViewManager.Update(targetNode, &meta.LeaderView{ + ID: targetNode, + CollectionID: suite.collection, + Channel: suite.unsubChannels[0], + }) + suite.AssertTaskNum(0, len(suite.unsubChannels), len(suite.unsubChannels), 0) + + // ProcessTasks + suite.dispatchAndWait(targetNode) + suite.AssertTaskNum(1, 0, 1, 0) + + // Other nodes' HB can't trigger the procedure of tasks + suite.dispatchAndWait(targetNode + 1) + suite.AssertTaskNum(1, 0, 1, 0) + + // Update dist + suite.dist.LeaderViewManager.Update(targetNode) + suite.dispatchAndWait(targetNode) + suite.AssertTaskNum(0, 0, 0, 0) + + for _, task := range tasks { + suite.Equal(TaskStatusSucceeded, task.Status()) + suite.NoError(task.Err()) + } +} + +func (suite *TaskSuite) TestLoadSegmentTask() { + ctx := context.Background() + timeout := 10 * time.Second + targetNode := int64(3) + partition := int64(100) + channel := &datapb.VchannelInfo{ + CollectionID: suite.collection, + ChannelName: Params.CommonCfg.RootCoordDml + "-test", + } + + // Expect + suite.broker.EXPECT().GetCollectionSchema(mock.Anything, suite.collection).Return(&schemapb.CollectionSchema{ + Name: "TestSubscribeChannelTask", + }, nil) + suite.broker.EXPECT().GetPartitions(mock.Anything, suite.collection).Return([]int64{100, 101}, nil) + for _, segment := range suite.loadSegments { + suite.broker.EXPECT().GetSegmentInfo(mock.Anything, segment).Return([]*datapb.SegmentInfo{ + { + ID: segment, + CollectionID: suite.collection, + PartitionID: partition, + InsertChannel: channel.ChannelName, + }, + }, nil) + suite.broker.EXPECT().GetIndexInfo(mock.Anything, suite.collection, segment).Return(nil, nil) + } + // suite.broker.EXPECT().GetRecoveryInfo(mock.Anything, suite.collection, partition). + // Return([]*datapb.VchannelInfo{channel}, nil, nil) + suite.cluster.EXPECT().LoadSegments(mock.Anything, targetNode, mock.Anything).Return(utils.WrapStatus(commonpb.ErrorCode_Success, ""), nil) + + // Test load segment task + suite.dist.ChannelDistManager.Update(targetNode, meta.DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: suite.collection, + ChannelName: channel.ChannelName, + })) + tasks := []Task{} + for _, segment := range suite.loadSegments { + suite.target.AddSegment(&datapb.SegmentInfo{ + ID: segment, + CollectionID: suite.collection, + PartitionID: partition, + InsertChannel: channel.ChannelName, + }) + task := NewSegmentTask( + ctx, + timeout, + 0, + suite.collection, + suite.replica, + NewSegmentAction(targetNode, ActionTypeGrow, segment), + ) + tasks = append(tasks, task) + err := suite.scheduler.Add(task) + suite.NoError(err) + } + segmentsNum := len(suite.loadSegments) + suite.AssertTaskNum(0, segmentsNum, 0, segmentsNum) + + // Process tasks + suite.dispatchAndWait(targetNode) + suite.AssertTaskNum(segmentsNum, 0, 0, segmentsNum) + + // Other nodes' HB can't trigger the procedure of tasks + suite.dispatchAndWait(targetNode + 1) + suite.AssertTaskNum(segmentsNum, 0, 0, segmentsNum) + + // Process tasks done + // Dist contains channels + view := &meta.LeaderView{ + ID: targetNode, + CollectionID: suite.collection, + Segments: map[int64]int64{}, + } + for _, segment := range suite.loadSegments { + view.Segments[segment] = targetNode + } + suite.dist.LeaderViewManager.Update(targetNode, view) + suite.dispatchAndWait(targetNode) + suite.AssertTaskNum(0, 0, 0, 0) + + for _, task := range tasks { + suite.Equal(TaskStatusSucceeded, task.Status()) + suite.NoError(task.Err()) + } +} + +func (suite *TaskSuite) TestReleaseSegmentTask() { + ctx := context.Background() + timeout := 10 * time.Second + targetNode := int64(3) + partition := int64(100) + channel := &datapb.VchannelInfo{ + CollectionID: suite.collection, + ChannelName: Params.CommonCfg.RootCoordDml + "-test", + } + + // Expect + suite.cluster.EXPECT().ReleaseSegments(mock.Anything, targetNode, mock.Anything).Return(utils.WrapStatus(commonpb.ErrorCode_Success, ""), nil) + + // Test load segment task + view := &meta.LeaderView{ + ID: targetNode, + CollectionID: suite.collection, + Channel: channel.ChannelName, + Segments: make(map[int64]int64), + } + segments := make([]*meta.Segment, 0) + tasks := []Task{} + for _, segment := range suite.releaseSegments { + segments = append(segments, &meta.Segment{ + SegmentInfo: &datapb.SegmentInfo{ + ID: segment, + CollectionID: suite.collection, + PartitionID: partition, + InsertChannel: channel.ChannelName, + }, + }) + view.Segments[segment] = targetNode + task := NewSegmentTask( + ctx, + timeout, + 0, + suite.collection, + suite.replica, + NewSegmentAction(targetNode, ActionTypeReduce, segment), + ) + tasks = append(tasks, task) + err := suite.scheduler.Add(task) + suite.NoError(err) + } + suite.dist.SegmentDistManager.Update(targetNode, segments...) + suite.dist.LeaderViewManager.Update(targetNode, view) + + segmentsNum := len(suite.releaseSegments) + suite.AssertTaskNum(0, segmentsNum, 0, segmentsNum) + + // Process tasks + suite.dispatchAndWait(targetNode) + suite.AssertTaskNum(segmentsNum, 0, 0, segmentsNum) + + // Other nodes' HB can't trigger the procedure of tasks + suite.dispatchAndWait(targetNode + 1) + suite.AssertTaskNum(segmentsNum, 0, 0, segmentsNum) + + // Process tasks done + suite.dist.LeaderViewManager.Update(targetNode) + suite.dispatchAndWait(targetNode) + suite.AssertTaskNum(0, 0, 0, 0) + + for _, task := range tasks { + suite.Equal(TaskStatusSucceeded, task.Status()) + suite.NoError(task.Err()) + } +} + +func (suite *TaskSuite) TestMoveSegmentTask() { + ctx := context.Background() + timeout := 10 * time.Second + leader := int64(1) + sourceNode := int64(2) + targetNode := int64(3) + partition := int64(100) + channel := &datapb.VchannelInfo{ + CollectionID: suite.collection, + ChannelName: Params.CommonCfg.RootCoordDml + "-test", + } + + // Expect + suite.broker.EXPECT().GetCollectionSchema(mock.Anything, suite.collection).Return(&schemapb.CollectionSchema{ + Name: "TestMoveSegmentTask", + }, nil) + suite.broker.EXPECT().GetPartitions(mock.Anything, suite.collection).Return([]int64{100, 101}, nil) + for _, segment := range suite.moveSegments { + suite.broker.EXPECT().GetSegmentInfo(mock.Anything, segment).Return([]*datapb.SegmentInfo{ + { + ID: segment, + CollectionID: suite.collection, + PartitionID: partition, + InsertChannel: channel.ChannelName, + }, + }, nil) + suite.broker.EXPECT().GetIndexInfo(mock.Anything, suite.collection, segment).Return(nil, nil) + } + // suite.broker.EXPECT().GetRecoveryInfo(mock.Anything, suite.collection, partition). + // Return([]*datapb.VchannelInfo{channel}, nil, nil) + suite.cluster.EXPECT().LoadSegments(mock.Anything, leader, mock.Anything).Return(utils.WrapStatus(commonpb.ErrorCode_Success, ""), nil) + suite.cluster.EXPECT().ReleaseSegments(mock.Anything, leader, mock.Anything).Return(utils.WrapStatus(commonpb.ErrorCode_Success, ""), nil) + + // Test move segment task + suite.dist.ChannelDistManager.Update(leader, meta.DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: suite.collection, + ChannelName: channel.ChannelName, + })) + view := &meta.LeaderView{ + ID: leader, + CollectionID: suite.collection, + Channel: channel.ChannelName, + Segments: make(map[int64]int64), + } + tasks := []Task{} + segments := make([]*meta.Segment, 0) + for _, segment := range suite.moveSegments { + segments = append(segments, + utils.CreateTestSegment(suite.collection, partition, segment, sourceNode, 1, channel.ChannelName)) + suite.target.AddSegment(&datapb.SegmentInfo{ + ID: segment, + CollectionID: suite.collection, + PartitionID: partition, + InsertChannel: channel.ChannelName, + }) + view.Segments[segment] = sourceNode + + task := NewSegmentTask( + ctx, + timeout, + 0, + suite.collection, + suite.replica, + NewSegmentAction(targetNode, ActionTypeGrow, segment), + NewSegmentAction(sourceNode, ActionTypeReduce, segment), + ) + tasks = append(tasks, task) + err := suite.scheduler.Add(task) + suite.NoError(err) + } + suite.dist.SegmentDistManager.Update(sourceNode, segments...) + suite.dist.LeaderViewManager.Update(leader, view) + segmentsNum := len(suite.moveSegments) + suite.AssertTaskNum(0, segmentsNum, 0, segmentsNum) + + // Process tasks + suite.dispatchAndWait(leader) + suite.AssertTaskNum(segmentsNum, 0, 0, segmentsNum) + + // Other nodes' HB can't trigger the procedure of tasks + suite.dispatchAndWait(-1) + suite.AssertTaskNum(segmentsNum, 0, 0, segmentsNum) + + // Process tasks, target node contains the segment + view = view.Clone() + for _, segment := range suite.moveSegments { + view.Segments[segment] = targetNode + } + suite.dist.LeaderViewManager.Update(leader, view) + // First action done, execute the second action + suite.dispatchAndWait(leader) + // Check second action + suite.dispatchAndWait(leader) + suite.AssertTaskNum(0, 0, 0, 0) + + for _, task := range tasks { + suite.Equal(TaskStatusSucceeded, task.Status()) + suite.NoError(task.Err()) + } +} + +func (suite *TaskSuite) TestTaskCanceled() { + ctx := context.Background() + timeout := 10 * time.Second + targetNode := int64(1) + + // Expect + suite.cluster.EXPECT().UnsubDmChannel(mock.Anything, targetNode, mock.Anything).Return(utils.WrapStatus(commonpb.ErrorCode_Success, ""), nil) + + // Test unsubscribe channel task + tasks := []Task{} + for _, channel := range suite.unsubChannels { + suite.target.AddDmChannel(meta.DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: suite.collection, + ChannelName: channel, + })) + task := NewChannelTask( + ctx, + timeout, + 0, + suite.collection, + -1, + NewChannelAction(targetNode, ActionTypeReduce, channel), + ) + tasks = append(tasks, task) + err := suite.scheduler.Add(task) + suite.NoError(err) + } + // Only first channel exists + suite.dist.LeaderViewManager.Update(targetNode, &meta.LeaderView{ + ID: targetNode, + CollectionID: suite.collection, + Channel: suite.unsubChannels[0], + }) + suite.AssertTaskNum(0, len(suite.unsubChannels), len(suite.unsubChannels), 0) + + // ProcessTasks + suite.dispatchAndWait(targetNode) + suite.AssertTaskNum(1, 0, 1, 0) + + // Other nodes' HB can't trigger the procedure of tasks + suite.dispatchAndWait(targetNode + 1) + suite.AssertTaskNum(1, 0, 1, 0) + + // Cancel first task + tasks[0].Cancel() + suite.dispatchAndWait(targetNode) + suite.AssertTaskNum(0, 0, 0, 0) + + for i, task := range tasks { + if i == 0 { + suite.Equal(TaskStatusCanceled, task.Status()) + suite.ErrorIs(task.Err(), ErrTaskCanceled) + } else { + suite.Equal(TaskStatusSucceeded, task.Status()) + suite.NoError(task.Err()) + } + } +} + +func (suite *TaskSuite) TestSegmentTaskStale() { + ctx := context.Background() + timeout := 10 * time.Second + targetNode := int64(3) + partition := int64(100) + channel := &datapb.VchannelInfo{ + CollectionID: suite.collection, + ChannelName: Params.CommonCfg.RootCoordDml + "-test", + } + + // Expect + suite.broker.EXPECT().GetCollectionSchema(mock.Anything, suite.collection).Return(&schemapb.CollectionSchema{ + Name: "TestSegmentTaskStale", + }, nil) + suite.broker.EXPECT().GetPartitions(mock.Anything, suite.collection).Return([]int64{100, 101}, nil) + for _, segment := range suite.loadSegments { + suite.broker.EXPECT().GetSegmentInfo(mock.Anything, segment).Return([]*datapb.SegmentInfo{ + { + ID: segment, + CollectionID: suite.collection, + PartitionID: partition, + InsertChannel: channel.ChannelName, + }, + }, nil) + suite.broker.EXPECT().GetIndexInfo(mock.Anything, suite.collection, segment).Return(nil, nil) + } + // suite.broker.EXPECT().GetRecoveryInfo(mock.Anything, suite.collection, partition). + // Return([]*datapb.VchannelInfo{channel}, nil, nil) + suite.cluster.EXPECT().LoadSegments(mock.Anything, targetNode, mock.Anything).Return(utils.WrapStatus(commonpb.ErrorCode_Success, ""), nil) + + // Test load segment task + suite.meta.ReplicaManager.Put( + createReplica(suite.collection, targetNode)) + suite.dist.ChannelDistManager.Update(targetNode, meta.DmChannelFromVChannel(&datapb.VchannelInfo{ + CollectionID: suite.collection, + ChannelName: channel.ChannelName, + })) + tasks := []Task{} + for _, segment := range suite.loadSegments { + suite.target.AddSegment(&datapb.SegmentInfo{ + ID: segment, + CollectionID: suite.collection, + PartitionID: partition, + InsertChannel: channel.ChannelName, + }) + task := NewSegmentTask( + ctx, + timeout, + 0, + suite.collection, + suite.replica, + NewSegmentAction(targetNode, ActionTypeGrow, segment), + ) + tasks = append(tasks, task) + err := suite.scheduler.Add(task) + suite.NoError(err) + } + segmentsNum := len(suite.loadSegments) + suite.AssertTaskNum(0, segmentsNum, 0, segmentsNum) + + // Process tasks + suite.dispatchAndWait(targetNode) + suite.AssertTaskNum(segmentsNum, 0, 0, segmentsNum) + + // Other nodes' HB can't trigger the procedure of tasks + suite.dispatchAndWait(targetNode + 1) + suite.AssertTaskNum(segmentsNum, 0, 0, segmentsNum) + + // Process tasks done + // Dist contains channels, first task stale + view := &meta.LeaderView{ + ID: targetNode, + CollectionID: suite.collection, + Segments: map[int64]int64{}, + } + for _, segment := range suite.loadSegments[1:] { + view.Segments[segment] = targetNode + } + suite.dist.LeaderViewManager.Update(targetNode, view) + suite.target.RemoveSegment(suite.loadSegments[0]) + suite.dispatchAndWait(targetNode) + suite.AssertTaskNum(0, 0, 0, 0) + + for i, task := range tasks { + if i == 0 { + suite.Equal(TaskStatusStale, task.Status()) + suite.ErrorIs(ErrTaskStale, task.Err()) + } else { + suite.Equal(TaskStatusSucceeded, task.Status()) + suite.NoError(task.Err()) + } + } +} + +func (suite *TaskSuite) TestChannelTaskReplace() { + ctx := context.Background() + timeout := 10 * time.Second + targetNode := int64(3) + + for _, channel := range suite.subChannels { + task := NewChannelTask( + ctx, + timeout, + 0, + suite.collection, + suite.replica, + NewChannelAction(targetNode, ActionTypeGrow, channel), + ) + task.SetPriority(TaskPriorityNormal) + err := suite.scheduler.Add(task) + suite.NoError(err) + } + + // Task with the same replica and segment, + // but without higher priority can't be added + for _, channel := range suite.subChannels { + task := NewChannelTask( + ctx, + timeout, + 0, + suite.collection, + suite.replica, + NewChannelAction(targetNode, ActionTypeGrow, channel), + ) + task.SetPriority(TaskPriorityNormal) + err := suite.scheduler.Add(task) + suite.ErrorIs(err, ErrConflictTaskExisted) + task.SetPriority(TaskPriorityLow) + err = suite.scheduler.Add(task) + suite.ErrorIs(err, ErrConflictTaskExisted) + } + + // Replace the task with one with higher priority + for _, channel := range suite.subChannels { + task := NewChannelTask( + ctx, + timeout, + 0, + suite.collection, + suite.replica, + NewChannelAction(targetNode, ActionTypeGrow, channel), + ) + task.SetPriority(TaskPriorityHigh) + err := suite.scheduler.Add(task) + suite.NoError(err) + } + channelNum := len(suite.subChannels) + suite.AssertTaskNum(0, channelNum, channelNum, 0) +} + +func (suite *TaskSuite) TestSegmentTaskReplace() { + ctx := context.Background() + timeout := 10 * time.Second + targetNode := int64(3) + + for _, segment := range suite.loadSegments { + task := NewSegmentTask( + ctx, + timeout, + 0, + suite.collection, + suite.replica, + NewSegmentAction(targetNode, ActionTypeGrow, segment), + ) + task.SetPriority(TaskPriorityNormal) + err := suite.scheduler.Add(task) + suite.NoError(err) + } + + // Task with the same replica and segment, + // but without higher priority can't be added + for _, segment := range suite.loadSegments { + task := NewSegmentTask( + ctx, + timeout, + 0, + suite.collection, + suite.replica, + NewSegmentAction(targetNode, ActionTypeGrow, segment), + ) + task.SetPriority(TaskPriorityNormal) + err := suite.scheduler.Add(task) + suite.ErrorIs(err, ErrConflictTaskExisted) + task.SetPriority(TaskPriorityLow) + err = suite.scheduler.Add(task) + suite.ErrorIs(err, ErrConflictTaskExisted) + } + + // Replace the task with one with higher priority + for _, segment := range suite.loadSegments { + task := NewSegmentTask( + ctx, + timeout, + 0, + suite.collection, + suite.replica, + NewSegmentAction(targetNode, ActionTypeGrow, segment), + ) + task.SetPriority(TaskPriorityHigh) + err := suite.scheduler.Add(task) + suite.NoError(err) + } + segmentNum := len(suite.loadSegments) + suite.AssertTaskNum(0, segmentNum, 0, segmentNum) +} + +func (suite *TaskSuite) AssertTaskNum(process, wait, channel, segment int) { + scheduler := suite.scheduler + + suite.Equal(process, scheduler.processQueue.Len()) + suite.Equal(wait, scheduler.waitQueue.Len()) + suite.Len(scheduler.segmentTasks, segment) + suite.Len(scheduler.channelTasks, channel) + suite.Equal(len(scheduler.tasks), process+wait) + suite.Equal(len(scheduler.tasks), segment+channel) +} + +func (suite *TaskSuite) dispatchAndWait(node int64) { + suite.scheduler.Dispatch(node) + for { + count := 0 + suite.scheduler.executor.executingActions.Range(func(key, value any) bool { + count++ + return true + }) + if count == 0 { + break + } + time.Sleep(200 * time.Millisecond) + } +} + +func (suite *TaskSuite) newScheduler() *taskScheduler { + return NewScheduler( + context.Background(), + suite.meta, + suite.dist, + suite.target, + suite.broker, + suite.cluster, + suite.nodeMgr, + ) +} + +func createReplica(collection int64, nodes ...int64) *meta.Replica { + return &meta.Replica{ + Replica: &querypb.Replica{ + ID: rand.Int63()/2 + 1, + CollectionID: collection, + Nodes: nodes, + }, + Nodes: typeutil.NewUniqueSet(nodes...), + } +} + +func TestTask(t *testing.T) { + suite.Run(t, new(TaskSuite)) +} diff --git a/internal/querycoordv2/task/utils.go b/internal/querycoordv2/task/utils.go new file mode 100644 index 0000000000000..2e46d1c2fd32a --- /dev/null +++ b/internal/querycoordv2/task/utils.go @@ -0,0 +1,232 @@ +package task + +import ( + "context" + "time" + + "github.com/golang/protobuf/proto" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/internalpb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/proto/schemapb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/internal/util/funcutil" + "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/samber/lo" +) + +func Wait(ctx context.Context, timeout time.Duration, tasks ...Task) error { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + var err error + go func() { + for _, task := range tasks { + err = task.Wait() + if err != nil { + cancel() + break + } + } + cancel() + }() + <-ctx.Done() + + return err +} + +// GetTaskType returns the task's type, +// for now, only 3 types; +// - only 1 grow action -> Grow +// - only 1 reduce action -> Reduce +// - 1 grow action, and ends with 1 reduce action -> Move +func GetTaskType(task Task) Type { + if len(task.Actions()) > 1 { + return TaskTypeMove + } else if task.Actions()[0].Type() == ActionTypeGrow { + return TaskTypeGrow + } else { + return TaskTypeReduce + } +} + +func packLoadSegmentRequest( + task *SegmentTask, + action Action, + schema *schemapb.CollectionSchema, + loadMeta *querypb.LoadMetaInfo, + loadInfo *querypb.SegmentLoadInfo, + deltaPositions []*internalpb.MsgPosition, +) *querypb.LoadSegmentsRequest { + return &querypb.LoadSegmentsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_LoadSegments, + MsgID: task.SourceID(), + }, + Infos: []*querypb.SegmentLoadInfo{loadInfo}, + Schema: schema, + LoadMeta: loadMeta, + CollectionID: task.CollectionID(), + ReplicaID: task.ReplicaID(), + DeltaPositions: deltaPositions, + DstNodeID: action.Node(), + Version: time.Now().UnixNano(), + NeedTransfer: true, + } +} + +func packReleaseSegmentRequest(task *SegmentTask, action *SegmentAction, shard string) *querypb.ReleaseSegmentsRequest { + return &querypb.ReleaseSegmentsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_ReleaseSegments, + MsgID: task.SourceID(), + }, + + NodeID: action.Node(), + CollectionID: task.CollectionID(), + SegmentIDs: []int64{task.SegmentID()}, + Shard: shard, + Scope: action.Scope(), + NeedTransfer: false, + } +} + +func packLoadMeta(loadType querypb.LoadType, collectionID int64, partitions ...int64) *querypb.LoadMetaInfo { + return &querypb.LoadMetaInfo{ + LoadType: loadType, + CollectionID: collectionID, + PartitionIDs: partitions, + } +} + +func packSubDmChannelRequest( + task *ChannelTask, + action Action, + schema *schemapb.CollectionSchema, + loadMeta *querypb.LoadMetaInfo, + channel *meta.DmChannel, +) *querypb.WatchDmChannelsRequest { + return &querypb.WatchDmChannelsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_WatchDmChannels, + MsgID: task.SourceID(), + }, + NodeID: action.Node(), + CollectionID: task.CollectionID(), + Infos: []*datapb.VchannelInfo{channel.VchannelInfo}, + Schema: schema, + LoadMeta: loadMeta, + ReplicaID: task.ReplicaID(), + } +} + +func fillSubDmChannelRequest( + ctx context.Context, + req *querypb.WatchDmChannelsRequest, + broker meta.Broker, +) error { + segmentIDs := typeutil.NewUniqueSet() + for _, vchannel := range req.GetInfos() { + segmentIDs.Insert(vchannel.GetFlushedSegmentIds()...) + segmentIDs.Insert(vchannel.GetUnflushedSegmentIds()...) + segmentIDs.Insert(vchannel.GetDroppedSegmentIds()...) + } + + if segmentIDs.Len() == 0 { + return nil + } + + resp, err := broker.GetSegmentInfo(ctx, segmentIDs.Collect()...) + if err != nil { + return err + } + segmentInfos := make(map[int64]*datapb.SegmentInfo) + for _, info := range resp { + segmentInfos[info.GetID()] = info + } + req.SegmentInfos = segmentInfos + return nil +} + +func packUnsubDmChannelRequest(task *ChannelTask, action Action) *querypb.UnsubDmChannelRequest { + return &querypb.UnsubDmChannelRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_UnsubDmChannel, + MsgID: task.SourceID(), + }, + NodeID: action.Node(), + CollectionID: task.CollectionID(), + ChannelName: task.Channel(), + } +} + +func getShardLeader(replicaMgr *meta.ReplicaManager, distMgr *meta.DistributionManager, collectionID, nodeID int64, channel string) (int64, bool) { + replica := replicaMgr.GetByCollectionAndNode(collectionID, nodeID) + if replica == nil { + return 0, false + } + return distMgr.GetShardLeader(replica, channel) +} + +func getSegmentDeltaPositions(ctx context.Context, targetMgr *meta.TargetManager, broker meta.Broker, collectionID, partitionID int64, channel string) ([]*internalpb.MsgPosition, error) { + deltaChannelName, err := funcutil.ConvertChannelName(channel, Params.CommonCfg.RootCoordDml, Params.CommonCfg.RootCoordDelta) + if err != nil { + return nil, err + } + + // vchannels, _, err := broker.GetRecoveryInfo(ctx, collectionID, partitionID) + // if err != nil { + // return nil, err + // } + + deltaChannels := make([]*datapb.VchannelInfo, 0) + for _, info := range targetMgr.GetDmChannelsByCollection(collectionID) { + deltaChannelInfo, err := generatDeltaChannelInfo(info.VchannelInfo) + if err != nil { + return nil, err + } + if deltaChannelInfo.ChannelName == deltaChannelName { + deltaChannels = append(deltaChannels, deltaChannelInfo) + } + } + deltaChannels = mergeWatchDeltaChannelInfo(deltaChannels) + + return lo.Map(deltaChannels, func(channel *datapb.VchannelInfo, _ int) *internalpb.MsgPosition { + return channel.GetSeekPosition() + }), nil +} + +func generatDeltaChannelInfo(info *datapb.VchannelInfo) (*datapb.VchannelInfo, error) { + deltaChannelName, err := funcutil.ConvertChannelName(info.ChannelName, Params.CommonCfg.RootCoordDml, Params.CommonCfg.RootCoordDelta) + if err != nil { + return nil, err + } + deltaChannel := proto.Clone(info).(*datapb.VchannelInfo) + deltaChannel.ChannelName = deltaChannelName + deltaChannel.UnflushedSegmentIds = nil + deltaChannel.FlushedSegmentIds = nil + deltaChannel.DroppedSegmentIds = nil + return deltaChannel, nil +} + +func mergeWatchDeltaChannelInfo(infos []*datapb.VchannelInfo) []*datapb.VchannelInfo { + minPositions := make(map[string]int) + for index, info := range infos { + _, ok := minPositions[info.ChannelName] + if !ok { + minPositions[info.ChannelName] = index + } + minTimeStampIndex := minPositions[info.ChannelName] + if info.SeekPosition.GetTimestamp() < infos[minTimeStampIndex].SeekPosition.GetTimestamp() { + minPositions[info.ChannelName] = index + } + } + var result []*datapb.VchannelInfo + for _, index := range minPositions { + result = append(result, infos[index]) + } + + return result +} diff --git a/internal/querycoordv2/utils/checker.go b/internal/querycoordv2/utils/checker.go new file mode 100644 index 0000000000000..eaf833f953cc6 --- /dev/null +++ b/internal/querycoordv2/utils/checker.go @@ -0,0 +1,35 @@ +package utils + +import ( + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/util/typeutil" +) + +func FilterReleased[E interface{ GetCollectionID() int64 }](elems []E, collections []int64) []E { + collectionSet := typeutil.NewUniqueSet(collections...) + ret := make([]E, 0, len(elems)) + for i := range elems { + collection := elems[i].GetCollectionID() + if !collectionSet.Contain(collection) { + ret = append(ret, elems[i]) + } + } + return ret +} + +func FindMaxVersionSegments(segments []*meta.Segment) []*meta.Segment { + versions := make(map[int64]int64) + segMap := make(map[int64]*meta.Segment) + for _, s := range segments { + v, ok := versions[s.GetID()] + if !ok || v < s.Version { + versions[s.GetID()] = s.Version + segMap[s.GetID()] = s + } + } + ret := make([]*meta.Segment, 0) + for _, s := range segMap { + ret = append(ret, s) + } + return ret +} diff --git a/internal/querycoordv2/utils/meta.go b/internal/querycoordv2/utils/meta.go new file mode 100644 index 0000000000000..81f3f5f991612 --- /dev/null +++ b/internal/querycoordv2/utils/meta.go @@ -0,0 +1,149 @@ +package utils + +import ( + "context" + "fmt" + "math/rand" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/samber/lo" + "go.uber.org/zap" +) + +func GetReplicaNodesInfo(replicaMgr *meta.ReplicaManager, nodeMgr *session.NodeManager, replicaID int64) []*session.NodeInfo { + replica := replicaMgr.Get(replicaID) + if replica == nil { + return nil + } + + nodes := make([]*session.NodeInfo, 0, len(replica.Nodes)) + for node := range replica.Nodes { + nodes = append(nodes, nodeMgr.Get(node)) + } + return nodes +} + +func GetPartitions(collectionMgr *meta.CollectionManager, broker meta.Broker, collectionID int64) ([]int64, error) { + collection := collectionMgr.GetCollection(collectionID) + if collection != nil { + partitions, err := broker.GetPartitions(context.Background(), collectionID) + return partitions, err + } + + partitions := collectionMgr.GetPartitionsByCollection(collectionID) + if partitions != nil { + return lo.Map(partitions, func(partition *meta.Partition, i int) int64 { + return partition.PartitionID + }), nil + } + + // todo(yah01): replace this error with a defined error + return nil, fmt.Errorf("collection/partition not loaded") +} + +// GroupNodesByReplica groups nodes by replica, +// returns ReplicaID -> NodeIDs +func GroupNodesByReplica(replicaMgr *meta.ReplicaManager, collectionID int64, nodes []int64) map[int64][]int64 { + ret := make(map[int64][]int64) + replicas := replicaMgr.GetByCollection(collectionID) + for _, replica := range replicas { + for _, node := range nodes { + if replica.Nodes.Contain(node) { + ret[replica.ID] = append(ret[replica.ID], node) + } + } + } + return ret +} + +// GroupPartitionsByCollection groups partitions by collection, +// returns CollectionID -> Partitions +func GroupPartitionsByCollection(partitions []*meta.Partition) map[int64][]*meta.Partition { + ret := make(map[int64][]*meta.Partition, 0) + for _, partition := range partitions { + collection := partition.GetCollectionID() + ret[collection] = append(ret[collection], partition) + } + return ret +} + +// GroupSegmentsByReplica groups segments by replica, +// returns ReplicaID -> Segments +func GroupSegmentsByReplica(replicaMgr *meta.ReplicaManager, collectionID int64, segments []*meta.Segment) map[int64][]*meta.Segment { + ret := make(map[int64][]*meta.Segment) + replicas := replicaMgr.GetByCollection(collectionID) + for _, replica := range replicas { + for _, segment := range segments { + if replica.Nodes.Contain(segment.Node) { + ret[replica.ID] = append(ret[replica.ID], segment) + } + } + } + return ret +} + +// AssignNodesToReplicas assigns nodes to the given replicas, +// all given replicas must be the same collection, +// the given replicas have to be not in ReplicaManager +func AssignNodesToReplicas(nodeMgr *session.NodeManager, replicas ...*meta.Replica) { + replicaNumber := len(replicas) + nodes := nodeMgr.GetAll() + rand.Shuffle(len(nodes), func(i, j int) { + nodes[i], nodes[j] = nodes[j], nodes[i] + }) + + for i, node := range nodes { + replicas[i%replicaNumber].AddNode(node.ID()) + } +} + +// SpawnReplicas spawns replicas for given collection, assign nodes to them, and save them +func SpawnReplicas(replicaMgr *meta.ReplicaManager, nodeMgr *session.NodeManager, collection int64, replicaNumber int32) ([]*meta.Replica, error) { + replicas, err := replicaMgr.Spawn(collection, replicaNumber) + if err != nil { + return nil, err + } + AssignNodesToReplicas(nodeMgr, replicas...) + return replicas, replicaMgr.Put(replicas...) +} + +// RegisterTargets fetch channels and segments of given collection(partitions) from DataCoord, +// and then registers them on Target Manager +func RegisterTargets(ctx context.Context, + targetMgr *meta.TargetManager, + broker meta.Broker, + collection int64, partitions []int64) error { + dmChannels := make(map[string][]*datapb.VchannelInfo) + + for _, partitionID := range partitions { + log.Debug("get recovery info...", + zap.Int64("collectionID", collection), + zap.Int64("partitionID", partitionID)) + vChannelInfos, binlogs, err := broker.GetRecoveryInfo(ctx, collection, partitionID) + if err != nil { + return err + } + + // Register segments + for _, segmentBinlogs := range binlogs { + targetMgr.AddSegment(SegmentBinlogs2SegmentInfo( + collection, + partitionID, + segmentBinlogs)) + } + + for _, info := range vChannelInfos { + channelName := info.GetChannelName() + dmChannels[channelName] = append(dmChannels[channelName], info) + } + } + // Merge and register channels + for _, channels := range dmChannels { + dmChannel := MergeDmChannelInfo(channels) + targetMgr.AddDmChannel(dmChannel) + } + return nil +} diff --git a/internal/querycoordv2/utils/test.go b/internal/querycoordv2/utils/test.go new file mode 100644 index 0000000000000..f4b52aa0c280d --- /dev/null +++ b/internal/querycoordv2/utils/test.go @@ -0,0 +1,66 @@ +package utils + +import ( + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" + "github.com/milvus-io/milvus/internal/util/typeutil" +) + +func CreateTestLeaderView(id, collection int64, channel string, segments map[int64]int64, growings []int64) *meta.LeaderView { + return &meta.LeaderView{ + ID: id, + CollectionID: collection, + Channel: channel, + Segments: segments, + GrowingSegments: typeutil.NewUniqueSet(growings...), + } +} + +func CreateTestChannel(collection, node, version int64, channel string) *meta.DmChannel { + return &meta.DmChannel{ + VchannelInfo: &datapb.VchannelInfo{ + CollectionID: collection, + ChannelName: channel, + }, + Node: node, + Version: version, + } +} + +func CreateTestReplica(id, collectionID int64, nodes []int64) *meta.Replica { + return &meta.Replica{ + Replica: &querypb.Replica{ + ID: id, + CollectionID: collectionID, + Nodes: nodes, + }, + Nodes: typeutil.NewUniqueSet(nodes...), + } +} + +func CreateTestCollection(collection int64, replica int32) *meta.Collection { + return &meta.Collection{ + CollectionLoadInfo: &querypb.CollectionLoadInfo{ + CollectionID: collection, + ReplicaNumber: 3, + }, + } +} + +func CreateTestSegmentInfo(collection, partition, segment int64, channel string) *datapb.SegmentInfo { + return &datapb.SegmentInfo{ + ID: segment, + CollectionID: collection, + PartitionID: partition, + InsertChannel: channel, + } +} + +func CreateTestSegment(collection, partition, segment, node, version int64, channel string) *meta.Segment { + return &meta.Segment{ + SegmentInfo: CreateTestSegmentInfo(collection, partition, segment, channel), + Node: node, + Version: version, + } +} diff --git a/internal/querycoordv2/utils/types.go b/internal/querycoordv2/utils/types.go new file mode 100644 index 0000000000000..a4c60a6c7a4c4 --- /dev/null +++ b/internal/querycoordv2/utils/types.go @@ -0,0 +1,150 @@ +package utils + +import ( + "fmt" + + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/querycoordv2/meta" +) + +// WrapStatus wraps status with given error code, message and errors +func WrapStatus(code commonpb.ErrorCode, msg string, errs ...error) *commonpb.Status { + status := &commonpb.Status{ + ErrorCode: code, + Reason: msg, + } + + for _, err := range errs { + status.Reason = fmt.Sprintf("%s, err=%v", status.Reason, err) + } + + return status +} + +// WrapError wraps error with given message +func WrapError(msg string, err error) error { + return fmt.Errorf("%s[%w]", msg, err) +} + +func SegmentBinlogs2SegmentInfo(collectionID int64, partitionID int64, segmentBinlogs *datapb.SegmentBinlogs) *datapb.SegmentInfo { + return &datapb.SegmentInfo{ + ID: segmentBinlogs.GetSegmentID(), + CollectionID: collectionID, + PartitionID: partitionID, + InsertChannel: segmentBinlogs.GetInsertChannel(), + NumOfRows: segmentBinlogs.GetNumOfRows(), + Binlogs: segmentBinlogs.GetFieldBinlogs(), + Statslogs: segmentBinlogs.GetStatslogs(), + Deltalogs: segmentBinlogs.GetDeltalogs(), + } +} + +func MergeMetaSegmentIntoSegmentInfo(info *querypb.SegmentInfo, segments ...*meta.Segment) { + first := segments[0] + if info.GetSegmentID() == 0 { + *info = querypb.SegmentInfo{ + SegmentID: first.GetID(), + CollectionID: first.GetCollectionID(), + PartitionID: first.GetPartitionID(), + NumRows: first.GetNumOfRows(), + DmChannel: first.GetInsertChannel(), + NodeIds: make([]int64, 0), + SegmentState: commonpb.SegmentState_Sealed, + } + } + + for _, segment := range segments { + info.NodeIds = append(info.NodeIds, segment.Node) + } +} + +// packSegmentLoadInfo packs SegmentLoadInfo for given segment, +// packs with index if withIndex is true, this fetch indexes from IndexCoord +func PackSegmentLoadInfo(segment *datapb.SegmentInfo, indexes []*querypb.FieldIndexInfo) *querypb.SegmentLoadInfo { + loadInfo := &querypb.SegmentLoadInfo{ + SegmentID: segment.ID, + PartitionID: segment.PartitionID, + CollectionID: segment.CollectionID, + BinlogPaths: segment.Binlogs, + NumOfRows: segment.NumOfRows, + Statslogs: segment.Statslogs, + Deltalogs: segment.Deltalogs, + InsertChannel: segment.InsertChannel, + IndexInfos: indexes, + } + loadInfo.SegmentSize = calculateSegmentSize(loadInfo) + return loadInfo +} + +func calculateSegmentSize(segmentLoadInfo *querypb.SegmentLoadInfo) int64 { + segmentSize := int64(0) + + fieldIndex := make(map[int64]*querypb.FieldIndexInfo) + for _, index := range segmentLoadInfo.IndexInfos { + if index.EnableIndex { + fieldID := index.FieldID + fieldIndex[fieldID] = index + } + } + + for _, fieldBinlog := range segmentLoadInfo.BinlogPaths { + fieldID := fieldBinlog.FieldID + if index, ok := fieldIndex[fieldID]; ok { + segmentSize += index.IndexSize + } else { + segmentSize += getFieldSizeFromBinlog(fieldBinlog) + } + } + + // Get size of state data + for _, fieldBinlog := range segmentLoadInfo.Statslogs { + segmentSize += getFieldSizeFromBinlog(fieldBinlog) + } + + // Get size of delete data + for _, fieldBinlog := range segmentLoadInfo.Deltalogs { + segmentSize += getFieldSizeFromBinlog(fieldBinlog) + } + + return segmentSize +} + +func getFieldSizeFromBinlog(fieldBinlog *datapb.FieldBinlog) int64 { + fieldSize := int64(0) + for _, binlog := range fieldBinlog.Binlogs { + fieldSize += binlog.LogSize + } + + return fieldSize +} + +func MergeDmChannelInfo(infos []*datapb.VchannelInfo) *meta.DmChannel { + var dmChannel *meta.DmChannel + + for _, info := range infos { + if dmChannel == nil { + dmChannel = meta.DmChannelFromVChannel(info) + continue + } + + if info.SeekPosition.GetTimestamp() < dmChannel.SeekPosition.GetTimestamp() { + dmChannel.SeekPosition = info.SeekPosition + } + dmChannel.DroppedSegmentIds = append(dmChannel.DroppedSegmentIds, info.DroppedSegmentIds...) + dmChannel.UnflushedSegmentIds = append(dmChannel.UnflushedSegmentIds, info.UnflushedSegmentIds...) + dmChannel.FlushedSegmentIds = append(dmChannel.FlushedSegmentIds, info.FlushedSegmentIds...) + } + + return dmChannel +} + +func Replica2ReplicaInfo(replica *querypb.Replica) *milvuspb.ReplicaInfo { + return &milvuspb.ReplicaInfo{ + ReplicaID: replica.GetID(), + CollectionID: replica.GetCollectionID(), + NodeIds: replica.GetNodes(), + } +} diff --git a/internal/querynode/flow_graph_delete_node_test.go b/internal/querynode/flow_graph_delete_node_test.go index 72feac40c9561..376331374a9ff 100644 --- a/internal/querynode/flow_graph_delete_node_test.go +++ b/internal/querynode/flow_graph_delete_node_test.go @@ -38,6 +38,7 @@ func TestFlowGraphDeleteNode_delete(t *testing.T) { defaultPartitionID, defaultCollectionID, defaultDMLChannel, + defaultSegmentVersion, segmentTypeSealed) assert.NoError(t, err) @@ -59,6 +60,7 @@ func TestFlowGraphDeleteNode_delete(t *testing.T) { defaultPartitionID, defaultCollectionID, defaultDMLChannel, + defaultSegmentVersion, segmentTypeSealed) assert.NoError(t, err) @@ -91,6 +93,7 @@ func TestFlowGraphDeleteNode_delete(t *testing.T) { defaultPartitionID, defaultCollectionID, defaultDMLChannel, + defaultSegmentVersion, segmentTypeGrowing) assert.NoError(t, err) @@ -111,6 +114,7 @@ func TestFlowGraphDeleteNode_operate(t *testing.T) { defaultPartitionID, defaultCollectionID, defaultDMLChannel, + defaultSegmentVersion, segmentTypeSealed) assert.NoError(t, err) @@ -145,6 +149,7 @@ func TestFlowGraphDeleteNode_operate(t *testing.T) { defaultPartitionID, defaultCollectionID, defaultDMLChannel, + defaultSegmentVersion, segmentTypeSealed) assert.NoError(t, err) @@ -169,6 +174,7 @@ func TestFlowGraphDeleteNode_operate(t *testing.T) { defaultPartitionID, defaultCollectionID, defaultDMLChannel, + defaultSegmentVersion, segmentTypeSealed) assert.NoError(t, err) @@ -196,6 +202,7 @@ func TestFlowGraphDeleteNode_operate(t *testing.T) { defaultPartitionID, defaultCollectionID, defaultDMLChannel, + defaultSegmentVersion, segmentTypeSealed) assert.NoError(t, err) @@ -222,6 +229,7 @@ func TestFlowGraphDeleteNode_operate(t *testing.T) { defaultPartitionID, defaultCollectionID, defaultDMLChannel, + defaultSegmentVersion, segmentTypeSealed) assert.NoError(t, err) diff --git a/internal/querynode/flow_graph_insert_node.go b/internal/querynode/flow_graph_insert_node.go index 1a59a110597cc..b971cb378aa62 100644 --- a/internal/querynode/flow_graph_insert_node.go +++ b/internal/querynode/flow_graph_insert_node.go @@ -137,7 +137,7 @@ func (iNode *insertNode) Operate(in []flowgraph.Msg) []flowgraph.Msg { panic(err) } if !has { - err = iNode.metaReplica.addSegment(insertMsg.SegmentID, insertMsg.PartitionID, insertMsg.CollectionID, insertMsg.ShardName, segmentTypeGrowing) + err = iNode.metaReplica.addSegment(insertMsg.SegmentID, insertMsg.PartitionID, insertMsg.CollectionID, insertMsg.ShardName, 0, segmentTypeGrowing) if err != nil { // error occurs when collection or partition cannot be found, collection and partition should be created before err = fmt.Errorf("insertNode addSegment failed, err = %s", err) diff --git a/internal/querynode/flow_graph_insert_node_test.go b/internal/querynode/flow_graph_insert_node_test.go index d233b49df2f2c..bf05e7798183d 100644 --- a/internal/querynode/flow_graph_insert_node_test.go +++ b/internal/querynode/flow_graph_insert_node_test.go @@ -43,6 +43,7 @@ func getInsertNode() (*insertNode, error) { defaultPartitionID, defaultCollectionID, defaultDMLChannel, + defaultSegmentVersion, segmentTypeGrowing) if err != nil { return nil, err diff --git a/internal/querynode/impl.go b/internal/querynode/impl.go index 439cb6e84fe9a..c75794f95e00d 100644 --- a/internal/querynode/impl.go +++ b/internal/querynode/impl.go @@ -32,7 +32,6 @@ import ( "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/milvuspb" "github.com/milvus-io/milvus/internal/proto/querypb" - queryPb "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/util/metricsinfo" "github.com/milvus-io/milvus/internal/util/timerecord" "github.com/milvus-io/milvus/internal/util/typeutil" @@ -146,7 +145,7 @@ func (node *QueryNode) GetStatistics(ctx context.Context, req *querypb.GetStatis return ret, nil } -func (node *QueryNode) getStatisticsWithDmlChannel(ctx context.Context, req *queryPb.GetStatisticsRequest, dmlChannel string) (*internalpb.GetStatisticsResponse, error) { +func (node *QueryNode) getStatisticsWithDmlChannel(ctx context.Context, req *querypb.GetStatisticsRequest, dmlChannel string) (*internalpb.GetStatisticsResponse, error) { failRet := &internalpb.GetStatisticsResponse{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UnexpectedError, @@ -277,7 +276,7 @@ func (node *QueryNode) getStatisticsWithDmlChannel(ctx context.Context, req *que } // WatchDmChannels create consumers on dmChannels to receive Incremental data,which is the important part of real-time query -func (node *QueryNode) WatchDmChannels(ctx context.Context, in *queryPb.WatchDmChannelsRequest) (*commonpb.Status, error) { +func (node *QueryNode) WatchDmChannels(ctx context.Context, in *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) { code := node.stateCode.Load().(internalpb.StateCode) if code != internalpb.StateCode_Healthy { err := fmt.Errorf("query node %d is not ready", Params.QueryNodeCfg.GetNodeID()) @@ -287,7 +286,7 @@ func (node *QueryNode) WatchDmChannels(ctx context.Context, in *queryPb.WatchDmC } return status, nil } - dct := &watchDmChannelsTask{ + task := &watchDmChannelsTask{ baseTask: baseTask{ ctx: ctx, done: make(chan error), @@ -296,7 +295,7 @@ func (node *QueryNode) WatchDmChannels(ctx context.Context, in *queryPb.WatchDmC node: node, } - err := node.scheduler.queue.Enqueue(dct) + err := node.scheduler.queue.Enqueue(task) if err != nil { status := &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UnexpectedError, @@ -307,7 +306,7 @@ func (node *QueryNode) WatchDmChannels(ctx context.Context, in *queryPb.WatchDmC } log.Info("watchDmChannelsTask Enqueue done", zap.Int64("collectionID", in.CollectionID), zap.Int64("nodeID", Params.QueryNodeCfg.GetNodeID()), zap.Int64("replicaID", in.GetReplicaID())) waitFunc := func() (*commonpb.Status, error) { - err = dct.WaitToFinish() + err = task.WaitToFinish() if err != nil { status := &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UnexpectedError, @@ -316,6 +315,14 @@ func (node *QueryNode) WatchDmChannels(ctx context.Context, in *queryPb.WatchDmC log.Warn(err.Error()) return status, nil } + + sc, _ := node.ShardClusterService.getShardCluster(in.Infos[0].GetChannelName()) + sc.mutVersion.Lock() + defer sc.mutVersion.Unlock() + version := NewShardClusterVersion(sc.nextVersionID.Inc(), make(SegmentsStatus), nil) + sc.versions.Store(version.versionID, version) + sc.currentVersion = version + log.Info("watchDmChannelsTask WaitToFinish done", zap.Int64("collectionID", in.CollectionID), zap.Int64("nodeID", Params.QueryNodeCfg.GetNodeID())) return &commonpb.Status{ ErrorCode: commonpb.ErrorCode_Success, @@ -325,8 +332,57 @@ func (node *QueryNode) WatchDmChannels(ctx context.Context, in *queryPb.WatchDmC return waitFunc() } +func (node *QueryNode) UnsubDmChannel(ctx context.Context, req *querypb.UnsubDmChannelRequest) (*commonpb.Status, error) { + code := node.stateCode.Load().(internalpb.StateCode) + if code != internalpb.StateCode_Healthy { + err := fmt.Errorf("query node %d is not ready", Params.QueryNodeCfg.GetNodeID()) + status := &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: err.Error(), + } + return status, nil + } + dct := &releaseCollectionTask{ + baseTask: baseTask{ + ctx: ctx, + done: make(chan error), + }, + req: &querypb.ReleaseCollectionRequest{ + Base: req.GetBase(), + CollectionID: req.GetCollectionID(), + NodeID: req.GetNodeID(), + }, + node: node, + } + + err := node.scheduler.queue.Enqueue(dct) + if err != nil { + status := &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: err.Error(), + } + log.Warn(err.Error()) + return status, nil + } + log.Info("unsubDmChannel(ReleaseCollection) enqueue done", zap.Int64("collectionID", req.GetCollectionID())) + + func() { + err = dct.WaitToFinish() + if err != nil { + log.Warn(err.Error()) + return + } + log.Info("unsubDmChannel(ReleaseCollection) WaitToFinish done", zap.Int64("collectionID", req.GetCollectionID())) + }() + + status := &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + } + return status, nil +} + // LoadSegments load historical data into query node, historical data can be vector data or index -func (node *QueryNode) LoadSegments(ctx context.Context, in *queryPb.LoadSegmentsRequest) (*commonpb.Status, error) { +func (node *QueryNode) LoadSegments(ctx context.Context, in *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { code := node.stateCode.Load().(internalpb.StateCode) if code != internalpb.StateCode_Healthy { err := fmt.Errorf("query node %d is not ready", Params.QueryNodeCfg.GetNodeID()) @@ -336,7 +392,12 @@ func (node *QueryNode) LoadSegments(ctx context.Context, in *queryPb.LoadSegment } return status, nil } - dct := &loadSegmentsTask{ + + if in.GetNeedTransfer() { + return node.TransferLoad(ctx, in) + } + + task := &loadSegmentsTask{ baseTask: baseTask{ ctx: ctx, done: make(chan error), @@ -349,7 +410,7 @@ func (node *QueryNode) LoadSegments(ctx context.Context, in *queryPb.LoadSegment for _, info := range in.Infos { segmentIDs = append(segmentIDs, info.SegmentID) } - err := node.scheduler.queue.Enqueue(dct) + err := node.scheduler.queue.Enqueue(task) if err != nil { status := &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UnexpectedError, @@ -362,7 +423,7 @@ func (node *QueryNode) LoadSegments(ctx context.Context, in *queryPb.LoadSegment log.Info("loadSegmentsTask Enqueue done", zap.Int64("collectionID", in.CollectionID), zap.Int64s("segmentIDs", segmentIDs), zap.Int64("nodeID", Params.QueryNodeCfg.GetNodeID())) waitFunc := func() (*commonpb.Status, error) { - err = dct.WaitToFinish() + err = task.WaitToFinish() if err != nil { status := &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UnexpectedError, @@ -381,7 +442,7 @@ func (node *QueryNode) LoadSegments(ctx context.Context, in *queryPb.LoadSegment } // ReleaseCollection clears all data related to this collection on the querynode -func (node *QueryNode) ReleaseCollection(ctx context.Context, in *queryPb.ReleaseCollectionRequest) (*commonpb.Status, error) { +func (node *QueryNode) ReleaseCollection(ctx context.Context, in *querypb.ReleaseCollectionRequest) (*commonpb.Status, error) { code := node.stateCode.Load().(internalpb.StateCode) if code != internalpb.StateCode_Healthy { err := fmt.Errorf("query node %d is not ready", Params.QueryNodeCfg.GetNodeID()) @@ -427,7 +488,7 @@ func (node *QueryNode) ReleaseCollection(ctx context.Context, in *queryPb.Releas } // ReleasePartitions clears all data related to this partition on the querynode -func (node *QueryNode) ReleasePartitions(ctx context.Context, in *queryPb.ReleasePartitionsRequest) (*commonpb.Status, error) { +func (node *QueryNode) ReleasePartitions(ctx context.Context, in *querypb.ReleasePartitionsRequest) (*commonpb.Status, error) { code := node.stateCode.Load().(internalpb.StateCode) if code != internalpb.StateCode_Healthy { err := fmt.Errorf("query node %d is not ready", Params.QueryNodeCfg.GetNodeID()) @@ -473,7 +534,7 @@ func (node *QueryNode) ReleasePartitions(ctx context.Context, in *queryPb.Releas } // ReleaseSegments remove the specified segments from query node according segmentIDs, partitionIDs, and collectionID -func (node *QueryNode) ReleaseSegments(ctx context.Context, in *queryPb.ReleaseSegmentsRequest) (*commonpb.Status, error) { +func (node *QueryNode) ReleaseSegments(ctx context.Context, in *querypb.ReleaseSegmentsRequest) (*commonpb.Status, error) { code := node.stateCode.Load().(internalpb.StateCode) if code != internalpb.StateCode_Healthy { err := fmt.Errorf("query node %d is not ready", Params.QueryNodeCfg.GetNodeID()) @@ -484,24 +545,17 @@ func (node *QueryNode) ReleaseSegments(ctx context.Context, in *queryPb.ReleaseS return status, nil } - collection, err := node.metaReplica.getCollectionByID(in.CollectionID) - if err != nil { - status := &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_UnexpectedError, - Reason: fmt.Sprintf("cannot find collection %d when ReleaseSegments", in.CollectionID), - } - return status, nil + if in.GetNeedTransfer() { + return node.TransferRelease(ctx, in) } - collection.Lock() - defer collection.Unlock() for _, id := range in.SegmentIDs { switch in.GetScope() { - case queryPb.DataScope_Streaming: + case querypb.DataScope_Streaming: node.metaReplica.removeSegment(id, segmentTypeGrowing) - case queryPb.DataScope_Historical: + case querypb.DataScope_Historical: node.metaReplica.removeSegment(id, segmentTypeSealed) - case queryPb.DataScope_All: + case querypb.DataScope_All: node.metaReplica.removeSegment(id, segmentTypeSealed) node.metaReplica.removeSegment(id, segmentTypeGrowing) } @@ -514,11 +568,11 @@ func (node *QueryNode) ReleaseSegments(ctx context.Context, in *queryPb.ReleaseS } // GetSegmentInfo returns segment information of the collection on the queryNode, and the information includes memSize, numRow, indexName, indexID ... -func (node *QueryNode) GetSegmentInfo(ctx context.Context, in *queryPb.GetSegmentInfoRequest) (*queryPb.GetSegmentInfoResponse, error) { +func (node *QueryNode) GetSegmentInfo(ctx context.Context, in *querypb.GetSegmentInfoRequest) (*querypb.GetSegmentInfoResponse, error) { code := node.stateCode.Load().(internalpb.StateCode) if code != internalpb.StateCode_Healthy { err := fmt.Errorf("query node %d is not ready", Params.QueryNodeCfg.GetNodeID()) - res := &queryPb.GetSegmentInfoResponse{ + res := &querypb.GetSegmentInfoResponse{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UnexpectedError, Reason: err.Error(), @@ -526,7 +580,7 @@ func (node *QueryNode) GetSegmentInfo(ctx context.Context, in *queryPb.GetSegmen } return res, nil } - var segmentInfos []*queryPb.SegmentInfo + var segmentInfos []*querypb.SegmentInfo segmentIDs := make(map[int64]struct{}) for _, segmentID := range in.GetSegmentIDs() { @@ -536,7 +590,7 @@ func (node *QueryNode) GetSegmentInfo(ctx context.Context, in *queryPb.GetSegmen infos := node.metaReplica.getSegmentInfosByColID(in.CollectionID) segmentInfos = append(segmentInfos, filterSegmentInfo(infos, segmentIDs)...) - return &queryPb.GetSegmentInfoResponse{ + return &querypb.GetSegmentInfoResponse{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_Success, }, @@ -545,11 +599,11 @@ func (node *QueryNode) GetSegmentInfo(ctx context.Context, in *queryPb.GetSegmen } // filterSegmentInfo returns segment info which segment id in segmentIDs map -func filterSegmentInfo(segmentInfos []*queryPb.SegmentInfo, segmentIDs map[int64]struct{}) []*queryPb.SegmentInfo { +func filterSegmentInfo(segmentInfos []*querypb.SegmentInfo, segmentIDs map[int64]struct{}) []*querypb.SegmentInfo { if len(segmentIDs) == 0 { return segmentInfos } - filtered := make([]*queryPb.SegmentInfo, 0, len(segmentIDs)) + filtered := make([]*querypb.SegmentInfo, 0, len(segmentIDs)) for _, info := range segmentInfos { _, ok := segmentIDs[info.GetSegmentID()] if !ok { @@ -567,8 +621,8 @@ func (node *QueryNode) isHealthy() bool { } // Search performs replica search tasks. -func (node *QueryNode) Search(ctx context.Context, req *queryPb.SearchRequest) (*internalpb.SearchResults, error) { - log.Ctx(ctx).Debug("Received SearchRequest", +func (node *QueryNode) Search(ctx context.Context, req *querypb.SearchRequest) (*internalpb.SearchResults, error) { + log.Debug("Received SearchRequest", zap.Int64("msgID", req.GetReq().GetBase().GetMsgID()), zap.Strings("vChannels", req.GetDmlChannels()), zap.Int64s("segmentIDs", req.GetSegmentIDs()), @@ -622,7 +676,7 @@ func (node *QueryNode) Search(ctx context.Context, req *queryPb.SearchRequest) ( return ret, nil } -func (node *QueryNode) searchWithDmlChannel(ctx context.Context, req *queryPb.SearchRequest, dmlChannel string) (*internalpb.SearchResults, error) { +func (node *QueryNode) searchWithDmlChannel(ctx context.Context, req *querypb.SearchRequest, dmlChannel string) (*internalpb.SearchResults, error) { metrics.QueryNodeSQCount.WithLabelValues(fmt.Sprint(Params.QueryNodeCfg.GetNodeID()), metrics.SearchLabel, metrics.TotalLabel).Inc() failRet := &internalpb.SearchResults{ Status: &commonpb.Status{ @@ -774,7 +828,7 @@ func (node *QueryNode) searchWithDmlChannel(ctx context.Context, req *queryPb.Se return ret, nil } -func (node *QueryNode) queryWithDmlChannel(ctx context.Context, req *queryPb.QueryRequest, dmlChannel string) (*internalpb.RetrieveResults, error) { +func (node *QueryNode) queryWithDmlChannel(ctx context.Context, req *querypb.QueryRequest, dmlChannel string) (*internalpb.RetrieveResults, error) { metrics.QueryNodeSQCount.WithLabelValues(fmt.Sprint(Params.QueryNodeCfg.GetNodeID()), metrics.QueryLabel, metrics.TotalLabel).Inc() failRet := &internalpb.RetrieveResults{ Status: &commonpb.Status{ @@ -1077,3 +1131,107 @@ func (node *QueryNode) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsR Response: "", }, nil } + +func (node *QueryNode) GetDataDistribution(ctx context.Context, req *querypb.GetDataDistributionRequest) (*querypb.GetDataDistributionResponse, error) { + log := log.With( + zap.Int64("msg-id", req.GetBase().GetMsgID()), + zap.Int64("node-id", Params.QueryNodeCfg.GetNodeID()), + ) + if !node.isHealthy() { + log.Warn("QueryNode.GetMetrics failed", + zap.Error(errQueryNodeIsUnhealthy(Params.QueryNodeCfg.GetNodeID()))) + + return &querypb.GetDataDistributionResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: msgQueryNodeIsUnhealthy(Params.QueryNodeCfg.GetNodeID()), + }, + }, nil + } + + growingSegments := node.metaReplica.getGrowingSegments() + sealedSegments := node.metaReplica.getSealedSegments() + shardClusters := node.ShardClusterService.GetShardClusters() + + growinsgSegmentIDs := make([]int64, 0, len(growingSegments)) + for _, s := range growingSegments { + growinsgSegmentIDs = append(growinsgSegmentIDs, s.ID()) + } + + segmentVersionInfos := make([]*querypb.SegmentVersionInfo, 0, len(sealedSegments)) + for _, s := range sealedSegments { + info := &querypb.SegmentVersionInfo{ + ID: s.ID(), + Collection: s.collectionID, + Partition: s.partitionID, + Channel: s.vChannelID, + Version: s.version, + } + segmentVersionInfos = append(segmentVersionInfos, info) + } + + channelVersionInfos := make([]*querypb.ChannelVersionInfo, 0, len(shardClusters)) + leaderViews := make([]*querypb.LeaderView, 0, len(shardClusters)) + for _, sc := range shardClusters { + segmentInfos := sc.GetSegmentInfos() + mapping := make(map[int64]int64) + for _, info := range segmentInfos { + mapping[info.segmentID] = info.nodeID + } + view := &querypb.LeaderView{ + Collection: sc.collectionID, + Channel: sc.vchannelName, + SegmentNodePairs: mapping, + } + leaderViews = append(leaderViews, view) + + channelInfo := &querypb.ChannelVersionInfo{ + Channel: sc.vchannelName, + Collection: sc.collectionID, + Version: sc.getVersion(), + } + channelVersionInfos = append(channelVersionInfos, channelInfo) + } + + return &querypb.GetDataDistributionResponse{ + Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, + NodeID: node.session.ServerID, + GrowingSegmentIDs: growinsgSegmentIDs, + Segments: segmentVersionInfos, + Channels: channelVersionInfos, + LeaderViews: leaderViews, + }, nil +} + +func (node *QueryNode) SyncDistribution(ctx context.Context, req *querypb.SyncDistributionRequest) (*commonpb.Status, error) { + shardCluster, ok := node.ShardClusterService.getShardCluster(req.GetChannel()) + if !ok { + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "shard not exist", + }, nil + } + for _, action := range req.GetActions() { + switch action.GetType() { + case querypb.SyncType_Remove: + shardCluster.forceRemoveSegment(action.GetSegmentID()) + case querypb.SyncType_Set: + shardCluster.updateSegment(shardSegmentInfo{ + segmentID: action.GetSegmentID(), + partitionID: action.GetPartitionID(), + nodeID: action.GetNodeID(), + state: segmentStateLoaded, + }) + default: + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "unexpected action type", + }, nil + } + } + + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "", + }, nil +} diff --git a/internal/querynode/impl_test.go b/internal/querynode/impl_test.go index 9c2add7f26c1d..098293b5fe741 100644 --- a/internal/querynode/impl_test.go +++ b/internal/querynode/impl_test.go @@ -29,6 +29,7 @@ import ( "github.com/milvus-io/milvus/internal/common" "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/milvuspb" "github.com/milvus-io/milvus/internal/proto/querypb" @@ -100,6 +101,12 @@ func TestImpl_WatchDmChannels(t *testing.T) { CollectionID: defaultCollectionID, PartitionIDs: []UniqueID{defaultPartitionID}, Schema: schema, + Infos: []*datapb.VchannelInfo{ + { + CollectionID: 1000, + ChannelName: "1000-dmc0", + }, + }, } status, err := node.WatchDmChannels(ctx, req) @@ -502,7 +509,7 @@ func TestImpl_ReleaseSegments(t *testing.T) { status, err := node.ReleaseSegments(ctx, req) assert.NoError(t, err) - assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.ErrorCode) + assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) }) wg.Wait() } diff --git a/internal/querynode/impl_utils.go b/internal/querynode/impl_utils.go new file mode 100644 index 0000000000000..5551e5bbc3225 --- /dev/null +++ b/internal/querynode/impl_utils.go @@ -0,0 +1,58 @@ +package querynode + +import ( + "context" + + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/querypb" +) + +func (node *QueryNode) TransferLoad(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { + if len(req.GetInfos()) == 0 { + return &commonpb.Status{}, nil + } + + shard := req.GetInfos()[0].GetInsertChannel() + shardCluster, ok := node.ShardClusterService.getShardCluster(shard) + if !ok { + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_NotShardLeader, + Reason: "shard cluster not found, the leader may have changed", + }, nil + } + + req.NeedTransfer = false + err := shardCluster.loadSegments(ctx, req) + if err != nil { + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: err.Error(), + }, nil + } + + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + }, nil +} + +func (node *QueryNode) TransferRelease(ctx context.Context, req *querypb.ReleaseSegmentsRequest) (*commonpb.Status, error) { + shardCluster, ok := node.ShardClusterService.getShardCluster(req.GetShard()) + if !ok { + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_NotShardLeader, + Reason: "shard cluster not found, the leader may have changed", + }, nil + } + + req.NeedTransfer = false + err := shardCluster.releaseSegments(ctx, req) + if err != nil { + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: err.Error(), + }, nil + } + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + }, nil +} diff --git a/internal/querynode/meta_replica.go b/internal/querynode/meta_replica.go index ffc287a62fb2d..415593ebba3f0 100644 --- a/internal/querynode/meta_replica.go +++ b/internal/querynode/meta_replica.go @@ -86,7 +86,7 @@ type ReplicaInterface interface { // segment // addSegment add a new segment to collectionReplica - addSegment(segmentID UniqueID, partitionID UniqueID, collectionID UniqueID, vChannelID Channel, segType segmentType) error + addSegment(segmentID UniqueID, partitionID UniqueID, collectionID UniqueID, vChannelID Channel, version UniqueID, segType segmentType) error // setSegment adds a segment to collectionReplica setSegment(segment *Segment) error // removeSegment removes a segment from collectionReplica @@ -114,6 +114,9 @@ type ReplicaInterface interface { freeAll() // printReplica prints the collections, partitions and segments in the collectionReplica printReplica() + + getGrowingSegments() []*Segment + getSealedSegments() []*Segment } // collectionReplica is the data replication of memory data in query node. @@ -205,7 +208,7 @@ func (replica *metaReplica) removeCollectionPrivate(collectionID UniqueID) error // delete partitions for _, partitionID := range collection.partitionIDs { // ignore error, try to delete - _ = replica.removePartitionPrivate(partitionID, true) + _ = replica.removePartitionPrivate(partitionID) } deleteCollection(collection) @@ -393,12 +396,25 @@ func (replica *metaReplica) addPartitionPrivate(collectionID UniqueID, partition func (replica *metaReplica) removePartition(partitionID UniqueID) error { replica.mu.Lock() defer replica.mu.Unlock() - return replica.removePartitionPrivate(partitionID, false) + + partition, err := replica.getPartitionByIDPrivate(partitionID) + if err != nil { + return err + } + + collection, err := replica.getCollectionByIDPrivate(partition.collectionID) + if err != nil { + return err + } + collection.Lock() + defer collection.Unlock() + + return replica.removePartitionPrivate(partitionID) } // removePartitionPrivate is the private function in collectionReplica, to remove the partition from collectionReplica // `locked` flag indicates whether corresponding collection lock is accquired before calling this method -func (replica *metaReplica) removePartitionPrivate(partitionID UniqueID, locked bool) error { +func (replica *metaReplica) removePartitionPrivate(partitionID UniqueID) error { partition, err := replica.getPartitionByIDPrivate(partitionID) if err != nil { return err @@ -409,11 +425,6 @@ func (replica *metaReplica) removePartitionPrivate(partitionID UniqueID, locked return err } - if !locked { - collection.Lock() - defer collection.Unlock() - } - // delete segments ids, _ := partition.getSegmentIDs(segmentTypeGrowing) for _, segmentID := range ids { @@ -525,7 +536,7 @@ func (replica *metaReplica) getSegmentIDsPrivate(partitionID UniqueID, segType s //----------------------------------------------------------------------------------------------------- segment // addSegment add a new segment to collectionReplica -func (replica *metaReplica) addSegment(segmentID UniqueID, partitionID UniqueID, collectionID UniqueID, vChannelID Channel, segType segmentType) error { +func (replica *metaReplica) addSegment(segmentID UniqueID, partitionID UniqueID, collectionID UniqueID, vChannelID Channel, version UniqueID, segType segmentType) error { replica.mu.Lock() defer replica.mu.Unlock() @@ -533,7 +544,7 @@ func (replica *metaReplica) addSegment(segmentID UniqueID, partitionID UniqueID, if err != nil { return err } - seg, err := newSegment(collection, segmentID, partitionID, collectionID, vChannelID, segType, replica.cgoPool) + seg, err := newSegment(collection, segmentID, partitionID, collectionID, vChannelID, segType, version, replica.cgoPool) if err != nil { return err } @@ -596,6 +607,25 @@ func (replica *metaReplica) setSegment(segment *Segment) error { func (replica *metaReplica) removeSegment(segmentID UniqueID, segType segmentType) { replica.mu.Lock() defer replica.mu.Unlock() + + switch segType { + case segmentTypeGrowing: + if segment, ok := replica.growingSegments[segmentID]; ok { + if collection, ok := replica.collections[segment.collectionID]; ok { + collection.RLock() + defer collection.RUnlock() + } + } + case segmentTypeSealed: + if segment, ok := replica.sealedSegments[segmentID]; ok { + if collection, ok := replica.collections[segment.collectionID]; ok { + collection.RLock() + defer collection.RUnlock() + } + } + default: + panic(fmt.Sprintf("unsupported segment type %s", segType.String())) + } replica.removeSegmentPrivate(segmentID, segType) } @@ -749,6 +779,28 @@ func (replica *metaReplica) freeAll() { replica.sealedSegments = make(map[UniqueID]*Segment) } +func (replica *metaReplica) getGrowingSegments() []*Segment { + replica.mu.RLock() + defer replica.mu.RUnlock() + + ret := make([]*Segment, 0, len(replica.growingSegments)) + for _, s := range replica.growingSegments { + ret = append(ret, s) + } + return ret +} + +func (replica *metaReplica) getSealedSegments() []*Segment { + replica.mu.RLock() + defer replica.mu.RUnlock() + + ret := make([]*Segment, 0, len(replica.sealedSegments)) + for _, s := range replica.sealedSegments { + ret = append(ret, s) + } + return ret +} + // newCollectionReplica returns a new ReplicaInterface func newCollectionReplica(pool *concurrency.Pool) ReplicaInterface { var replica ReplicaInterface = &metaReplica{ diff --git a/internal/querynode/meta_replica_test.go b/internal/querynode/meta_replica_test.go index 10e5e9f0e35eb..1932ee93a872c 100644 --- a/internal/querynode/meta_replica_test.go +++ b/internal/querynode/meta_replica_test.go @@ -147,7 +147,7 @@ func TestMetaReplica_segment(t *testing.T) { const segmentNum = 3 for i := 0; i < segmentNum; i++ { - err := replica.addSegment(UniqueID(i), defaultPartitionID, defaultCollectionID, "", segmentTypeGrowing) + err := replica.addSegment(UniqueID(i), defaultPartitionID, defaultCollectionID, "", defaultSegmentVersion, segmentTypeGrowing) assert.NoError(t, err) targetSeg, err := replica.getSegmentByID(UniqueID(i), segmentTypeGrowing) assert.NoError(t, err) @@ -162,7 +162,7 @@ func TestMetaReplica_segment(t *testing.T) { const segmentNum = 3 for i := 0; i < segmentNum; i++ { - err := replica.addSegment(UniqueID(i), defaultPartitionID, defaultCollectionID, "", segmentTypeGrowing) + err := replica.addSegment(UniqueID(i), defaultPartitionID, defaultCollectionID, "", defaultSegmentVersion, segmentTypeGrowing) assert.NoError(t, err) targetSeg, err := replica.getSegmentByID(UniqueID(i), segmentTypeGrowing) assert.NoError(t, err) @@ -178,7 +178,7 @@ func TestMetaReplica_segment(t *testing.T) { const segmentNum = 3 for i := 0; i < segmentNum; i++ { - err := replica.addSegment(UniqueID(i), defaultPartitionID, defaultCollectionID, "", segmentTypeGrowing) + err := replica.addSegment(UniqueID(i), defaultPartitionID, defaultCollectionID, "", defaultSegmentVersion, segmentTypeGrowing) assert.NoError(t, err) targetSeg, err := replica.getSegmentByID(UniqueID(i), segmentTypeGrowing) assert.NoError(t, err) @@ -197,10 +197,10 @@ func TestMetaReplica_segment(t *testing.T) { assert.NoError(t, err) defer replica.freeAll() - err = replica.addSegment(defaultSegmentID, defaultPartitionID, defaultCollectionID, "", segmentTypeGrowing) + err = replica.addSegment(defaultSegmentID, defaultPartitionID, defaultCollectionID, "", defaultSegmentVersion, segmentTypeGrowing) assert.NoError(t, err) - err = replica.addSegment(defaultSegmentID, defaultPartitionID, defaultCollectionID, "", segmentTypeGrowing) + err = replica.addSegment(defaultSegmentID, defaultPartitionID, defaultCollectionID, "", defaultSegmentVersion, segmentTypeGrowing) assert.Error(t, err) }) @@ -210,7 +210,7 @@ func TestMetaReplica_segment(t *testing.T) { defer replica.freeAll() invalidType := commonpb.SegmentState_NotExist - err = replica.addSegment(defaultSegmentID, defaultPartitionID, defaultCollectionID, "", invalidType) + err = replica.addSegment(defaultSegmentID, defaultPartitionID, defaultCollectionID, "", defaultSegmentVersion, invalidType) assert.Error(t, err) _, err = replica.getSegmentByID(defaultSegmentID, invalidType) assert.Error(t, err) @@ -250,12 +250,12 @@ func TestMetaReplica_segment(t *testing.T) { }, } - segment1, err := newSegment(collection, UniqueID(1), defaultPartitionID, defaultCollectionID, "", segmentTypeGrowing, pool) + segment1, err := newSegment(collection, UniqueID(1), defaultPartitionID, defaultCollectionID, "", segmentTypeGrowing, defaultSegmentVersion, pool) assert.NoError(t, err) err = replica.setSegment(segment1) assert.NoError(t, err) - segment2, err := newSegment(collection, UniqueID(2), defaultPartitionID, defaultCollectionID, "", segmentTypeSealed, pool) + segment2, err := newSegment(collection, UniqueID(2), defaultPartitionID, defaultCollectionID, "", segmentTypeSealed, defaultSegmentVersion, pool) assert.NoError(t, err) segment2.setIndexedFieldInfo(fieldID, indexInfo) err = replica.setSegment(segment2) @@ -285,22 +285,22 @@ func TestMetaReplica_segment(t *testing.T) { replica.addPartition(defaultCollectionID, defaultPartitionID) replica.addPartition(defaultCollectionID, defaultPartitionID+1) - segment1, err := newSegment(collection, UniqueID(1), defaultPartitionID, defaultCollectionID, "channel1", segmentTypeGrowing, pool) + segment1, err := newSegment(collection, UniqueID(1), defaultPartitionID, defaultCollectionID, "channel1", segmentTypeGrowing, defaultSegmentVersion, pool) assert.NoError(t, err) err = replica.setSegment(segment1) assert.NoError(t, err) - segment2, err := newSegment(collection, UniqueID(2), defaultPartitionID+1, defaultCollectionID, "channel2", segmentTypeGrowing, pool) + segment2, err := newSegment(collection, UniqueID(2), defaultPartitionID+1, defaultCollectionID, "channel2", segmentTypeGrowing, defaultSegmentVersion, pool) assert.NoError(t, err) err = replica.setSegment(segment2) assert.NoError(t, err) - segment3, err := newSegment(collection, UniqueID(3), defaultPartitionID+1, defaultCollectionID, "channel2", segmentTypeGrowing, pool) + segment3, err := newSegment(collection, UniqueID(3), defaultPartitionID+1, defaultCollectionID, "channel2", segmentTypeGrowing, defaultSegmentVersion, pool) assert.NoError(t, err) err = replica.setSegment(segment3) assert.NoError(t, err) - segment4, err := newSegment(collection, UniqueID(4), defaultPartitionID, defaultCollectionID, "channel1", segmentTypeSealed, pool) + segment4, err := newSegment(collection, UniqueID(4), defaultPartitionID, defaultCollectionID, "channel1", segmentTypeSealed, defaultSegmentVersion, pool) assert.NoError(t, err) err = replica.setSegment(segment4) assert.NoError(t, err) diff --git a/internal/querynode/mock_test.go b/internal/querynode/mock_test.go index 50ddbe6fe2124..41b719814f017 100644 --- a/internal/querynode/mock_test.go +++ b/internal/querynode/mock_test.go @@ -75,6 +75,8 @@ const ( defaultSubName = "query-node-unittest-sub-name-0" defaultLocalStorage = "/tmp/milvus_test/querynode" + + defaultSegmentVersion = int64(1001) ) const ( @@ -1223,6 +1225,7 @@ func genSealedSegment(schema *schemapb.CollectionSchema, collectionID, vChannel, segmentTypeSealed, + defaultSegmentVersion, pool) if err != nil { return nil, err @@ -1316,6 +1319,7 @@ func genSimpleReplicaWithGrowingSegment() (ReplicaInterface, error) { defaultPartitionID, defaultCollectionID, defaultDMLChannel, + defaultSegmentVersion, segmentTypeGrowing) if err != nil { return nil, err diff --git a/internal/querynode/query_node_test.go b/internal/querynode/query_node_test.go index 076a250d6f628..1bce1c6dbe223 100644 --- a/internal/querynode/query_node_test.go +++ b/internal/querynode/query_node_test.go @@ -66,7 +66,7 @@ func initTestMeta(t *testing.T, node *QueryNode, collectionID UniqueID, segmentI err = node.metaReplica.addPartition(collection.ID(), defaultPartitionID) assert.NoError(t, err) - err = node.metaReplica.addSegment(segmentID, defaultPartitionID, collectionID, "", segmentTypeSealed) + err = node.metaReplica.addSegment(segmentID, defaultPartitionID, collectionID, "", defaultSegmentVersion, segmentTypeSealed) assert.NoError(t, err) } @@ -132,12 +132,12 @@ func startEmbedEtcdServer() (*embed.Etcd, error) { config.Dir = os.TempDir() config.LogLevel = "warn" config.LogOutputs = []string{"default"} - u, err := url.Parse("http://localhost:2389") + u, err := url.Parse("http://localhost:8989") if err != nil { return nil, err } config.LCUrls = []url.URL{*u} - u, err = url.Parse("http://localhost:2390") + u, err = url.Parse("http://localhost:8990") if err != nil { return nil, err } diff --git a/internal/querynode/segment.go b/internal/querynode/segment.go index 6eb19052f6c1e..6708cad8d04d8 100644 --- a/internal/querynode/segment.go +++ b/internal/querynode/segment.go @@ -78,6 +78,7 @@ type Segment struct { segmentID UniqueID partitionID UniqueID collectionID UniqueID + version UniqueID vChannelID Channel lastMemSize int64 @@ -165,7 +166,14 @@ func (s *Segment) hasLoadIndexForIndexedField(fieldID int64) bool { return false } -func newSegment(collection *Collection, segmentID UniqueID, partitionID UniqueID, collectionID UniqueID, vChannelID Channel, segType segmentType, pool *concurrency.Pool) (*Segment, error) { +func newSegment(collection *Collection, + segmentID UniqueID, + partitionID UniqueID, + collectionID UniqueID, + vChannelID Channel, + segType segmentType, + version UniqueID, + pool *concurrency.Pool) (*Segment, error) { /* CSegmentInterface NewSegment(CCollection collection, uint64_t segment_id, SegmentType seg_type); @@ -205,6 +213,7 @@ func newSegment(collection *Collection, segmentID UniqueID, partitionID UniqueID segmentID: segmentID, partitionID: partitionID, collectionID: collectionID, + version: version, vChannelID: vChannelID, indexedFieldInfos: make(map[UniqueID]*IndexedFieldInfo), diff --git a/internal/querynode/segment_loader.go b/internal/querynode/segment_loader.go index de3a5ce8cf024..0bebb91d4dee3 100644 --- a/internal/querynode/segment_loader.go +++ b/internal/querynode/segment_loader.go @@ -147,7 +147,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme return err } - segment, err := newSegment(collection, segmentID, partitionID, collectionID, vChannelID, segmentType, loader.cgoPool) + segment, err := newSegment(collection, segmentID, partitionID, collectionID, vChannelID, segmentType, req.GetVersion(), loader.cgoPool) if err != nil { log.Error("load segment failed when create new segment", zap.Int64("collectionID", collectionID), diff --git a/internal/querynode/segment_loader_test.go b/internal/querynode/segment_loader_test.go index 3de5b1940fffa..4b954b6a40d8c 100644 --- a/internal/querynode/segment_loader_test.go +++ b/internal/querynode/segment_loader_test.go @@ -184,6 +184,7 @@ func TestSegmentLoader_loadSegmentFieldsData(t *testing.T) { defaultCollectionID, defaultDMLChannel, segmentTypeSealed, + defaultSegmentVersion, pool) assert.Nil(t, err) @@ -337,7 +338,7 @@ func TestSegmentLoader_testLoadGrowing(t *testing.T) { collection, err := node.metaReplica.getCollectionByID(defaultCollectionID) assert.NoError(t, err) - segment, err := newSegment(collection, defaultSegmentID+1, defaultPartitionID, defaultCollectionID, defaultDMLChannel, segmentTypeGrowing, loader.cgoPool) + segment, err := newSegment(collection, defaultSegmentID+1, defaultPartitionID, defaultCollectionID, defaultDMLChannel, segmentTypeGrowing, defaultSegmentVersion, loader.cgoPool) assert.Nil(t, err) insertData, err := genInsertData(defaultMsgLength, collection.schema) @@ -366,7 +367,7 @@ func TestSegmentLoader_testLoadGrowing(t *testing.T) { collection, err := node.metaReplica.getCollectionByID(defaultCollectionID) assert.NoError(t, err) - segment, err := newSegment(collection, defaultSegmentID+1, defaultPartitionID, defaultCollectionID, defaultDMLChannel, segmentTypeGrowing, node.loader.cgoPool) + segment, err := newSegment(collection, defaultSegmentID+1, defaultPartitionID, defaultCollectionID, defaultDMLChannel, segmentTypeGrowing, defaultSegmentVersion, node.loader.cgoPool) assert.Nil(t, err) insertData, err := genInsertData(defaultMsgLength, collection.schema) diff --git a/internal/querynode/segment_test.go b/internal/querynode/segment_test.go index e437938d9ef35..d74a4391edf23 100644 --- a/internal/querynode/segment_test.go +++ b/internal/querynode/segment_test.go @@ -52,7 +52,7 @@ func TestSegment_newSegment(t *testing.T) { assert.Equal(t, collection.ID(), collectionID) segmentID := UniqueID(0) - segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, pool) + segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, defaultSegmentVersion, pool) assert.Nil(t, err) assert.Equal(t, segmentID, segment.segmentID) deleteSegment(segment) @@ -62,7 +62,7 @@ func TestSegment_newSegment(t *testing.T) { _, err = newSegment(collection, defaultSegmentID, defaultPartitionID, - collectionID, "", 100, pool) + collectionID, "", 100, defaultSegmentVersion, pool) assert.Error(t, err) }) } @@ -79,7 +79,7 @@ func TestSegment_deleteSegment(t *testing.T) { assert.Equal(t, collection.ID(), collectionID) segmentID := UniqueID(0) - segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, pool) + segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, defaultSegmentVersion, pool) assert.Equal(t, segmentID, segment.segmentID) assert.Nil(t, err) @@ -106,7 +106,7 @@ func TestSegment_getRowCount(t *testing.T) { assert.Equal(t, collection.ID(), collectionID) segmentID := UniqueID(0) - segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, pool) + segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, defaultSegmentVersion, pool) assert.Equal(t, segmentID, segment.segmentID) assert.Nil(t, err) @@ -151,7 +151,7 @@ func TestSegment_retrieve(t *testing.T) { assert.Equal(t, collection.ID(), collectionID) segmentID := UniqueID(0) - segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, pool) + segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, defaultSegmentVersion, pool) assert.Equal(t, segmentID, segment.segmentID) assert.Nil(t, err) @@ -238,7 +238,7 @@ func TestSegment_getDeletedCount(t *testing.T) { assert.Equal(t, collection.ID(), collectionID) segmentID := UniqueID(0) - segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, pool) + segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, defaultSegmentVersion, pool) assert.Equal(t, segmentID, segment.segmentID) assert.Nil(t, err) @@ -290,7 +290,7 @@ func TestSegment_getMemSize(t *testing.T) { assert.Equal(t, collection.ID(), collectionID) segmentID := UniqueID(0) - segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, pool) + segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, defaultSegmentVersion, pool) assert.Equal(t, segmentID, segment.segmentID) assert.Nil(t, err) @@ -328,7 +328,7 @@ func TestSegment_segmentInsert(t *testing.T) { collection := newCollection(collectionID, schema) assert.Equal(t, collection.ID(), collectionID) segmentID := UniqueID(0) - segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, pool) + segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, defaultSegmentVersion, pool) assert.Equal(t, segmentID, segment.segmentID) assert.Nil(t, err) @@ -375,7 +375,7 @@ func TestSegment_segmentDelete(t *testing.T) { assert.Equal(t, collection.ID(), collectionID) segmentID := UniqueID(0) - segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, pool) + segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, defaultSegmentVersion, pool) assert.Equal(t, segmentID, segment.segmentID) assert.Nil(t, err) @@ -472,7 +472,7 @@ func TestSegment_segmentPreInsert(t *testing.T) { assert.Equal(t, collection.ID(), collectionID) segmentID := UniqueID(0) - segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, pool) + segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, defaultSegmentVersion, pool) assert.Equal(t, segmentID, segment.segmentID) assert.Nil(t, err) @@ -494,7 +494,7 @@ func TestSegment_segmentPreDelete(t *testing.T) { assert.Equal(t, collection.ID(), collectionID) segmentID := UniqueID(0) - segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, pool) + segment, err := newSegment(collection, segmentID, defaultPartitionID, collectionID, "", segmentTypeGrowing, defaultSegmentVersion, pool) assert.Equal(t, segmentID, segment.segmentID) assert.Nil(t, err) @@ -542,6 +542,7 @@ func TestSegment_segmentLoadDeletedRecord(t *testing.T) { defaultCollectionID, defaultDMLChannel, segmentTypeSealed, + defaultSegmentVersion, pool) assert.Nil(t, err) ids := []int64{1, 2, 3} @@ -622,6 +623,7 @@ func TestSegment_BasicMetrics(t *testing.T) { defaultCollectionID, defaultDMLChannel, segmentTypeSealed, + defaultSegmentVersion, pool) assert.Nil(t, err) @@ -672,6 +674,7 @@ func TestSegment_fillIndexedFieldsData(t *testing.T) { defaultCollectionID, defaultDMLChannel, segmentTypeSealed, + defaultSegmentVersion, pool) assert.Nil(t, err) @@ -997,6 +1000,7 @@ func TestUpdateBloomFilter(t *testing.T) { defaultPartitionID, defaultCollectionID, defaultDMLChannel, + defaultSegmentVersion, segmentTypeSealed) assert.NoError(t, err) seg, err := replica.getSegmentByID(defaultSegmentID, segmentTypeSealed) @@ -1020,6 +1024,7 @@ func TestUpdateBloomFilter(t *testing.T) { defaultPartitionID, defaultCollectionID, defaultDMLChannel, + defaultSegmentVersion, segmentTypeSealed) assert.NoError(t, err) seg, err := replica.getSegmentByID(defaultSegmentID, segmentTypeSealed) diff --git a/internal/querynode/shard_cluster.go b/internal/querynode/shard_cluster.go index 87aa516354b26..c8d3e0bd64278 100644 --- a/internal/querynode/shard_cluster.go +++ b/internal/querynode/shard_cluster.go @@ -85,6 +85,7 @@ type shardQueryNode interface { GetStatistics(context.Context, *querypb.GetStatisticsRequest) (*internalpb.GetStatisticsResponse, error) Search(context.Context, *querypb.SearchRequest) (*internalpb.SearchResults, error) Query(context.Context, *querypb.QueryRequest) (*internalpb.RetrieveResults, error) + LoadSegments(ctx context.Context, in *querypb.LoadSegmentsRequest) (*commonpb.Status, error) ReleaseSegments(ctx context.Context, in *querypb.ReleaseSegmentsRequest) (*commonpb.Status, error) Stop() error } @@ -133,6 +134,7 @@ type ShardCluster struct { collectionID int64 replicaID int64 vchannelName string + version int64 nodeDetector ShardNodeDetector segmentDetector ShardSegmentDetector @@ -337,7 +339,7 @@ func (sc *ShardCluster) SyncSegments(distribution []*querypb.ReplicaSegmentsInfo sc.mutVersion.Lock() defer sc.mutVersion.Unlock() - version := NewShardClusterVersion(sc.nextVersionID.Inc(), allocations) + version := NewShardClusterVersion(sc.nextVersionID.Inc(), allocations, sc.currentVersion) sc.versions.Store(version.versionID, version) sc.currentVersion = version } @@ -401,6 +403,15 @@ func (sc *ShardCluster) removeSegment(evt shardSegmentInfo) { sc.healthCheck() } +func (sc *ShardCluster) forceRemoveSegment(segmentID int64) { + log := log.With(zap.String("shard", sc.vchannelName), zap.Int64("segmentID", segmentID)) + log.Info("remove segment") + sc.mut.Lock() + defer sc.mut.Unlock() + + delete(sc.segments, segmentID) +} + // init list all nodes and semgent states ant start watching func (sc *ShardCluster) init() { // list nodes @@ -656,6 +667,134 @@ func (sc *ShardCluster) HandoffSegments(info *querypb.SegmentChangeInfo) error { return nil } +func (sc *ShardCluster) loadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) error { + // add common log fields + log := log.With(zap.Int64("collectionID", sc.collectionID), + zap.Int64("replicaID", sc.replicaID), + zap.String("vchannel", sc.vchannelName), + zap.Int64("dstNodeID", req.GetDstNodeID())) + segmentIDs := make([]int64, 0, len(req.Infos)) + for _, info := range req.Infos { + segmentIDs = append(segmentIDs, info.SegmentID) + } + log = log.With(zap.Int64s("segmentIDs", segmentIDs)) + + // notify follower to load segment + node, ok := sc.getNode(req.GetDstNodeID()) + if !ok { + log.Warn("node not in cluster") + return fmt.Errorf("node not in cluster %d", req.GetDstNodeID()) + } + + resp, err := node.client.LoadSegments(ctx, req) + if err != nil { + log.Warn("failed to dispatch load segment request", zap.Error(err)) + return err + } + if resp.GetErrorCode() != commonpb.ErrorCode_Success { + log.Warn("follower load segment failed", zap.String("reason", resp.GetReason())) + return fmt.Errorf("follower %d failed to load segment, reason %s", req.DstNodeID, resp.GetReason()) + } + + // update allocation + for _, info := range req.Infos { + sc.updateSegment(shardSegmentInfo{nodeID: req.DstNodeID, segmentID: info.SegmentID, partitionID: info.PartitionID, state: segmentStateLoaded}) + } + + // notify handoff wait online if any + sc.segmentCond.L.Lock() + sc.segmentCond.Broadcast() + sc.segmentCond.L.Unlock() + + sc.mutVersion.Lock() + defer sc.mutVersion.Unlock() + + // update shardleader allocation view + allocations := sc.currentVersion.segments.Clone(filterNothing) + for _, info := range req.Infos { + allocations[info.SegmentID] = shardSegmentInfo{nodeID: req.DstNodeID, segmentID: info.SegmentID, partitionID: info.PartitionID, state: segmentStateLoaded} + } + + version := NewShardClusterVersion(sc.nextVersionID.Inc(), allocations, sc.currentVersion) + sc.versions.Store(version.versionID, version) + sc.currentVersion = version + + return nil +} + +func (sc *ShardCluster) releaseSegments(ctx context.Context, req *querypb.ReleaseSegmentsRequest) error { + // add common log fields + log := log.With(zap.Int64("collectionID", sc.collectionID), + zap.Int64("replicaID", sc.replicaID), + zap.String("vchannel", sc.vchannelName), + zap.Int64s("segmentIDs", req.GetSegmentIDs()), + zap.String("scope", req.GetScope().String())) + + if req.GetScope() != querypb.DataScope_Streaming { + offlineSegments := make(typeutil.UniqueSet) + offlineSegments.Insert(req.GetSegmentIDs()...) + + sc.mutVersion.Lock() + + var allocations SegmentsStatus + if sc.currentVersion != nil { + allocations = sc.currentVersion.segments.Clone(func(segmentID UniqueID, nodeID UniqueID) bool { + return nodeID == req.NodeID && offlineSegments.Contain(segmentID) + }) + } + + // generate a new version + versionID := sc.nextVersionID.Inc() + // remove offline segments in next version + // so incoming request will not have allocation of these segments + version := NewShardClusterVersion(versionID, allocations, sc.currentVersion) + sc.versions.Store(versionID, version) + + var lastVersionID int64 + // currentVersion shall be not nil + if sc.currentVersion != nil { + // wait for last version search done + <-sc.currentVersion.Expire() + lastVersionID = sc.currentVersion.versionID + } + + // set current version to new one + sc.currentVersion = version + sc.mutVersion.Unlock() + sc.cleanupVersion(lastVersionID) + + sc.mut.Lock() + for _, segmentID := range req.SegmentIDs { + info, ok := sc.segments[segmentID] + if ok { + // otherwise, segment is on another node, do nothing + if info.nodeID == req.NodeID { + delete(sc.segments, segmentID) + } + } + } + sc.mut.Unlock() + } + + // try to release segments from nodes + node, ok := sc.getNode(req.GetNodeID()) + if !ok { + log.Warn("node not in cluster", zap.Int64("nodeID", req.NodeID)) + return fmt.Errorf("node %d not in cluster ", req.NodeID) + } + + resp, err := node.client.ReleaseSegments(ctx, req) + if err != nil { + log.Warn("failed to dispatch release segment request", zap.Error(err)) + return err + } + if resp.GetErrorCode() != commonpb.ErrorCode_Success { + log.Warn("follower release segment failed", zap.String("reason", resp.GetReason())) + return fmt.Errorf("follower %d failed to release segment, reason %s", req.NodeID, resp.GetReason()) + } + return nil +} + // appendHandoff adds the change info into pending list and returns the token. func (sc *ShardCluster) applySegmentChange(info *querypb.SegmentChangeInfo, onlineSegmentIDs []UniqueID) int64 { @@ -663,9 +802,9 @@ func (sc *ShardCluster) applySegmentChange(info *querypb.SegmentChangeInfo, onli // first all online segment shall be tried, for flush-handoff only puts segment in onlineSegments // and we need to try all offlineSegments in case flush-compact-handoff case possibleGrowingToRemove := make([]UniqueID, 0, len(info.OfflineSegments)+len(onlineSegmentIDs)) - offlineSegments := typeutil.UniqueSet{} // map stores offline segment id for quick check + offlineNodes := make(map[int64]int64) for _, offline := range info.OfflineSegments { - offlineSegments.Insert(offline.GetSegmentID()) + offlineNodes[offline.GetSegmentID()] = offline.GetNodeID() possibleGrowingToRemove = append(possibleGrowingToRemove, offline.GetSegmentID()) } // add online segment ids to suspect list @@ -673,8 +812,9 @@ func (sc *ShardCluster) applySegmentChange(info *querypb.SegmentChangeInfo, onli // generate next version allocation sc.mut.RLock() - allocations := sc.segments.Clone(func(segmentID int64) bool { - return offlineSegments.Contain(segmentID) + allocations := sc.segments.Clone(func(segmentID int64, nodeID int64) bool { + offlineNodeID, ok := offlineNodes[segmentID] + return ok && offlineNodeID == nodeID }) sc.mut.RUnlock() @@ -685,7 +825,7 @@ func (sc *ShardCluster) applySegmentChange(info *querypb.SegmentChangeInfo, onli versionID := sc.nextVersionID.Inc() // remove offline segments in next version // so incoming request will not have allocation of these segments - version := NewShardClusterVersion(versionID, allocations) + version := NewShardClusterVersion(versionID, allocations, sc.currentVersion) sc.versions.Store(versionID, version) var lastVersionID int64 @@ -849,7 +989,7 @@ func (sc *ShardCluster) GetStatistics(ctx context.Context, req *querypb.GetStati // Search preforms search operation on shard cluster. func (sc *ShardCluster) Search(ctx context.Context, req *querypb.SearchRequest, withStreaming withStreaming) ([]*internalpb.SearchResults, error) { if !sc.serviceable() { - return nil, fmt.Errorf("ShardCluster for %s replicaID %d is no available", sc.vchannelName, sc.replicaID) + return nil, fmt.Errorf("ShardCluster for %s replicaID %d is no available, state %d, version %+v", sc.vchannelName, sc.replicaID, sc.state.Load(), sc.currentVersion) } if !funcutil.SliceContain(req.GetDmlChannels(), sc.vchannelName) { return nil, fmt.Errorf("ShardCluster for %s does not match request channels :%v", sc.vchannelName, req.GetDmlChannels()) @@ -859,7 +999,7 @@ func (sc *ShardCluster) Search(ctx context.Context, req *querypb.SearchRequest, segAllocs, versionID := sc.segmentAllocations(req.GetReq().GetPartitionIDs()) defer sc.finishUsage(versionID) - log.Debug("cluster segment distribution", zap.Int("len", len(segAllocs))) + log.Debug("cluster segment distribution", zap.Int("len", len(segAllocs)), zap.Int64s("partitionIDs", req.GetReq().GetPartitionIDs())) for nodeID, segmentIDs := range segAllocs { log.Debug("segments distribution", zap.Int64("nodeID", nodeID), zap.Int64s("segments", segmentIDs)) } @@ -901,7 +1041,7 @@ func (sc *ShardCluster) Search(ctx context.Context, req *querypb.SearchRequest, } node, ok := sc.getNode(nodeID) if !ok { // meta dismatch, report error - return nil, fmt.Errorf("ShardCluster for %s replicaID %d is no available", sc.vchannelName, sc.replicaID) + return nil, fmt.Errorf("ShardCluster for %s replicaID %d is no available, node %d not found", sc.vchannelName, sc.replicaID, nodeID) } wg.Add(1) go func() { @@ -1005,3 +1145,17 @@ func (sc *ShardCluster) Query(ctx context.Context, req *querypb.QueryRequest, wi return results, nil } + +func (sc *ShardCluster) GetSegmentInfos() []shardSegmentInfo { + ret := make([]shardSegmentInfo, 0, len(sc.segments)) + for _, info := range sc.segments { + ret = append(ret, info) + } + return ret +} + +func (sc *ShardCluster) getVersion() int64 { + sc.mut.RLock() + defer sc.mut.RUnlock() + return sc.version +} diff --git a/internal/querynode/shard_cluster_service.go b/internal/querynode/shard_cluster_service.go index 310706490efdd..49370981f4aed 100644 --- a/internal/querynode/shard_cluster_service.go +++ b/internal/querynode/shard_cluster_service.go @@ -18,7 +18,7 @@ import ( ) const ( - ReplicaMetaPrefix = "queryCoord-ReplicaMeta" + ReplicaMetaPrefix = "querycoord-replica" ) // shardQueryNodeWrapper wraps a querynode to shardQueryNode and preventing it been closed @@ -159,3 +159,12 @@ func (s *ShardClusterService) HandoffVChannelSegments(vchannel string, info *que } return err } + +func (s *ShardClusterService) GetShardClusters() []*ShardCluster { + ret := make([]*ShardCluster, 0) + s.clusters.Range(func(key, value any) bool { + ret = append(ret, value.(*ShardCluster)) + return true + }) + return ret +} diff --git a/internal/querynode/shard_cluster_service_test.go b/internal/querynode/shard_cluster_service_test.go index f665367d9a2e9..2ef72d83387f6 100644 --- a/internal/querynode/shard_cluster_service_test.go +++ b/internal/querynode/shard_cluster_service_test.go @@ -51,6 +51,7 @@ func TestShardClusterService_HandoffSegments(t *testing.T) { assert.NotPanics(t, func() { clusterService.HandoffSegments(defaultCollectionID, &querypb.SegmentChangeInfo{}) }) + clusterService.releaseShardCluster(defaultDMLChannel) } func TestShardClusterService_SyncReplicaSegments(t *testing.T) { diff --git a/internal/querynode/shard_cluster_test.go b/internal/querynode/shard_cluster_test.go index 960e176b114e0..cebaf21eab4c8 100644 --- a/internal/querynode/shard_cluster_test.go +++ b/internal/querynode/shard_cluster_test.go @@ -28,6 +28,7 @@ import ( "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" ) type mockNodeDetector struct { @@ -59,6 +60,8 @@ type mockShardQueryNode struct { searchErr error queryResult *internalpb.RetrieveResults queryErr error + loadSegmentsResults *commonpb.Status + loadSegmentsErr error releaseSegmentsResult *commonpb.Status releaseSegmentsErr error } @@ -75,6 +78,10 @@ func (m *mockShardQueryNode) Query(_ context.Context, _ *querypb.QueryRequest) ( return m.queryResult, m.queryErr } +func (m *mockShardQueryNode) LoadSegments(ctx context.Context, in *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { + return m.loadSegmentsResults, m.loadSegmentsErr +} + func (m *mockShardQueryNode) ReleaseSegments(ctx context.Context, in *querypb.ReleaseSegmentsRequest) (*commonpb.Status, error) { return m.releaseSegmentsResult, m.releaseSegmentsErr } @@ -2387,3 +2394,105 @@ func TestShardCluster_HandoffSegments(t *testing.T) { }) } + +type ShardClusterSuite struct { + suite.Suite + + collectionID int64 + otherCollectionID int64 + vchannelName string + otherVchannelName string + + replicaID int64 + + sc *ShardCluster +} + +func (suite *ShardClusterSuite) SetupSuite() { + suite.collectionID = int64(1) + suite.otherCollectionID = int64(2) + suite.vchannelName = "dml_1_1_v0" + suite.otherVchannelName = "dml_1_2_v0" + suite.replicaID = int64(0) +} + +func (suite *ShardClusterSuite) SetupTest() { + nodeEvents := []nodeEvent{ + { + nodeID: 1, + nodeAddr: "addr_1", + isLeader: true, + }, + { + nodeID: 2, + nodeAddr: "addr_2", + }, + } + + segmentEvents := []segmentEvent{ + { + segmentID: 1, + nodeIDs: []int64{1}, + state: segmentStateLoaded, + }, + { + segmentID: 2, + nodeIDs: []int64{2}, + state: segmentStateLoaded, + }, + } + suite.sc = NewShardCluster(suite.collectionID, suite.replicaID, suite.vchannelName, + &mockNodeDetector{ + initNodes: nodeEvents, + }, &mockSegmentDetector{ + initSegments: segmentEvents, + }, buildMockQueryNode) +} + +func (suite *ShardClusterSuite) TearDownTest() { + suite.sc.Close() + suite.sc = nil +} + +func (suite *ShardClusterSuite) TestReleaseSegments() { + type TestCase struct { + tag string + segmentIDs []int64 + nodeID int64 + scope querypb.DataScope + + expectError bool + } + + cases := []TestCase{ + { + tag: "normal release", + segmentIDs: []int64{2}, + nodeID: 2, + scope: querypb.DataScope_All, + expectError: false, + }, + } + + for _, test := range cases { + suite.Run(test.tag, func() { + suite.TearDownTest() + suite.SetupTest() + + err := suite.sc.releaseSegments(context.Background(), &querypb.ReleaseSegmentsRequest{ + NodeID: test.nodeID, + SegmentIDs: test.segmentIDs, + Scope: test.scope, + }) + if test.expectError { + suite.Error(err) + } else { + suite.NoError(err) + } + }) + } +} + +func TestShardClusterSuite(t *testing.T) { + suite.Run(t, new(ShardClusterSuite)) +} diff --git a/internal/querynode/shard_cluster_version.go b/internal/querynode/shard_cluster_version.go index 221b6e1de3bcd..0cded83e3a904 100644 --- a/internal/querynode/shard_cluster_version.go +++ b/internal/querynode/shard_cluster_version.go @@ -20,6 +20,9 @@ import ( "sync" "go.uber.org/atomic" + "go.uber.org/zap" + + "github.com/milvus-io/milvus/internal/log" ) // SegmentsStatus alias for map[int64]shardSegmentInfo. @@ -41,10 +44,10 @@ func (s SegmentsStatus) GetAllocations(partitionIDs []int64) map[int64][]int64 { } // Clone returns a copy of segments status data. -func (s SegmentsStatus) Clone(filter func(int64) bool) SegmentsStatus { +func (s SegmentsStatus) Clone(filter func(int64, int64) bool) SegmentsStatus { c := make(map[int64]shardSegmentInfo) for k, v := range s { - if filter(v.segmentID) { + if filter(v.segmentID, v.nodeID) { continue } c[k] = v @@ -53,26 +56,31 @@ func (s SegmentsStatus) Clone(filter func(int64) bool) SegmentsStatus { } // helper filter function that filters nothing -var filterNothing = func(int64) bool { return false } +var filterNothing = func(int64, int64) bool { return false } // ShardClusterVersion maintains a snapshot of sealed segments allocation. type ShardClusterVersion struct { - versionID int64 // identifier for version - segments SegmentsStatus // nodeID => []segmentID - current *atomic.Bool // is this version current - inUse *atomic.Int64 + versionID int64 // identifier for version + segments SegmentsStatus // nodeID => []segmentID + current *atomic.Bool // is this version current + inUse *atomic.Int64 + lastVersion *ShardClusterVersion + ch chan struct{} // signal channel to notify safe remove closeOnce sync.Once } // NewShardClusterVersion creates a version with id and allocation. -func NewShardClusterVersion(vID int64, status SegmentsStatus) *ShardClusterVersion { +func NewShardClusterVersion(vID int64, status SegmentsStatus, lastVersion *ShardClusterVersion) *ShardClusterVersion { + log.Info("Update shard cluster version", zap.Int64("newVersionID", vID), + zap.Any("newAllocation", status)) return &ShardClusterVersion{ - versionID: vID, - segments: status, - current: atomic.NewBool(true), // by default new version will be current - inUse: atomic.NewInt64(0), - ch: make(chan struct{}), + versionID: vID, + segments: status, + current: atomic.NewBool(true), // by default new version will be current + inUse: atomic.NewInt64(0), + ch: make(chan struct{}), + lastVersion: lastVersion, } } @@ -105,7 +113,12 @@ func (v *ShardClusterVersion) Expire() chan struct{} { func (v *ShardClusterVersion) checkSafeGC() { if !v.IsCurrent() && v.inUse.Load() == int64(0) { v.closeOnce.Do(func() { - close(v.ch) + go func() { + if v.lastVersion != nil { + <-v.lastVersion.Expire() + } + close(v.ch) + }() }) } } diff --git a/internal/querynode/shard_cluster_version_test.go b/internal/querynode/shard_cluster_version_test.go index 47102d8563843..8ffd2270ab828 100644 --- a/internal/querynode/shard_cluster_version_test.go +++ b/internal/querynode/shard_cluster_version_test.go @@ -18,6 +18,7 @@ package querynode import ( "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -34,7 +35,7 @@ func channelClose(ch chan struct{}) bool { func TestShardClusterVersion(t *testing.T) { t.Run("new version", func(t *testing.T) { - v := NewShardClusterVersion(1, SegmentsStatus{}) + v := NewShardClusterVersion(1, SegmentsStatus{}, nil) assert.True(t, v.IsCurrent()) assert.Equal(t, int64(1), v.versionID) @@ -42,19 +43,21 @@ func TestShardClusterVersion(t *testing.T) { }) t.Run("version expired", func(t *testing.T) { - v := NewShardClusterVersion(1, SegmentsStatus{}) + v := NewShardClusterVersion(1, SegmentsStatus{}, nil) assert.True(t, v.IsCurrent()) ch := v.Expire() assert.False(t, v.IsCurrent()) - assert.True(t, channelClose(ch)) + assert.Eventually(t, func() bool { + return channelClose(ch) + }, time.Second, time.Millisecond*10) }) t.Run("In use check", func(t *testing.T) { v := NewShardClusterVersion(1, SegmentsStatus{ 1: shardSegmentInfo{segmentID: 1, partitionID: 0, nodeID: 1}, 2: shardSegmentInfo{segmentID: 2, partitionID: 1, nodeID: 2}, - }) + }, nil) allocs := v.GetAllocation([]int64{1}) assert.EqualValues(t, map[int64][]int64{2: {2}}, allocs) @@ -65,6 +68,29 @@ func TestShardClusterVersion(t *testing.T) { v.FinishUsage() - assert.True(t, channelClose(ch)) + assert.Eventually(t, func() bool { + return channelClose(ch) + }, time.Second, time.Millisecond*10) + + }) + + t.Run("wait last version", func(t *testing.T) { + lastVersion := NewShardClusterVersion(1, SegmentsStatus{}, nil) + lastVersion.GetAllocation(nil) + currentVersion := NewShardClusterVersion(2, SegmentsStatus{}, lastVersion) + + ch := currentVersion.Expire() + + select { + case <-ch: + t.FailNow() + default: + } + + lastVersion.FinishUsage() + + assert.Eventually(t, func() bool { + return channelClose(ch) + }, time.Second, time.Millisecond*10) }) } diff --git a/internal/querynode/shard_node_detector.go b/internal/querynode/shard_node_detector.go index ce186c1e63504..07252e717c25a 100644 --- a/internal/querynode/shard_node_detector.go +++ b/internal/querynode/shard_node_detector.go @@ -22,7 +22,7 @@ import ( "github.com/golang/protobuf/proto" "github.com/milvus-io/milvus/internal/log" - "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" "go.etcd.io/etcd/api/v3/mvccpb" v3rpc "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" @@ -86,19 +86,12 @@ func (nd *etcdShardNodeDetector) watchNodes(collectionID int64, replicaID int64, continue } // skip replica not related - if info.GetCollectionID() != collectionID || info.GetReplicaID() != replicaID { + if info.GetCollectionID() != collectionID || info.GetID() != replicaID { continue } - // find the leader id for the shard replica - var leaderID int64 - for _, shardReplica := range info.GetShardReplicas() { - if shardReplica.GetDmChannelName() == vchannelName { - leaderID = shardReplica.GetLeaderID() - break - } - } + // generate node event - for _, nodeID := range info.GetNodeIds() { + for _, nodeID := range info.GetNodes() { addr, has := idAddr[nodeID] if !has { log.Warn("Node not found in session", zap.Int64("node id", nodeID)) @@ -108,7 +101,7 @@ func (nd *etcdShardNodeDetector) watchNodes(collectionID int64, replicaID int64, nodeID: nodeID, nodeAddr: addr, eventType: nodeAdd, - isLeader: nodeID == leaderID, + isLeader: nodeID == Params.QueryNodeCfg.GetNodeID(), }) } } @@ -168,13 +161,13 @@ func (nd *etcdShardNodeDetector) watch(ch clientv3.WatchChan, collectionID, repl func (nd *etcdShardNodeDetector) handlePutEvent(e *clientv3.Event, collectionID, replicaID int64) { var err error - var info, prevInfo *milvuspb.ReplicaInfo + var info, prevInfo *querypb.Replica info, err = nd.parseReplicaInfo(e.Kv.Value) if err != nil { log.Warn("failed to handle node event", zap.Any("event", e), zap.Error(err)) return } - if info.CollectionID != collectionID || info.ReplicaID != replicaID { + if info.CollectionID != collectionID || info.GetID() != replicaID { return } if e.PrevKv != nil { @@ -188,7 +181,7 @@ func (nd *etcdShardNodeDetector) handlePutEvent(e *clientv3.Event, collectionID, } // all node is added if prevInfo == nil { - for _, nodeID := range info.GetNodeIds() { + for _, nodeID := range info.GetNodes() { addr, ok := idAddr[nodeID] if !ok { log.Warn("node id not in session", zap.Int64("nodeID", nodeID)) @@ -205,12 +198,12 @@ func (nd *etcdShardNodeDetector) handlePutEvent(e *clientv3.Event, collectionID, // maybe binary search is better here currIDs := make(map[int64]struct{}) - for _, id := range info.GetNodeIds() { + for _, id := range info.GetNodes() { currIDs[id] = struct{}{} } oldIDs := make(map[int64]struct{}) - for _, id := range prevInfo.GetNodeIds() { + for _, id := range prevInfo.GetNodes() { oldIDs[id] = struct{}{} } @@ -254,7 +247,7 @@ func (nd *etcdShardNodeDetector) handleDelEvent(e *clientv3.Event, collectionID, return } // skip replica not related - if prevInfo.GetCollectionID() != collectionID || prevInfo.GetReplicaID() != replicaID { + if prevInfo.GetCollectionID() != collectionID || prevInfo.GetID() != replicaID { return } idAddr, err := nd.idAddr() @@ -262,7 +255,7 @@ func (nd *etcdShardNodeDetector) handleDelEvent(e *clientv3.Event, collectionID, log.Error("Etcd NodeDetector session map failed", zap.Error(err)) panic(err) } - for _, id := range prevInfo.GetNodeIds() { + for _, id := range prevInfo.GetNodes() { //best effort to notify offline addr := idAddr[id] nd.evtCh <- nodeEvent{ @@ -273,8 +266,8 @@ func (nd *etcdShardNodeDetector) handleDelEvent(e *clientv3.Event, collectionID, } } -func (nd *etcdShardNodeDetector) parseReplicaInfo(bs []byte) (*milvuspb.ReplicaInfo, error) { - info := &milvuspb.ReplicaInfo{} +func (nd *etcdShardNodeDetector) parseReplicaInfo(bs []byte) (*querypb.Replica, error) { + info := &querypb.Replica{} err := proto.Unmarshal(bs, info) return info, err } diff --git a/internal/querynode/shard_node_detector_test.go b/internal/querynode/shard_node_detector_test.go index 8321f5d24b988..43f42e26a5a85 100644 --- a/internal/querynode/shard_node_detector_test.go +++ b/internal/querynode/shard_node_detector_test.go @@ -25,7 +25,7 @@ import ( "time" "github.com/golang/protobuf/proto" - "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/util/funcutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -41,9 +41,9 @@ func TestEtcdShardNodeDetector_watch(t *testing.T) { type testCase struct { name string ids []int64 - oldRecords map[string]*milvuspb.ReplicaInfo + oldRecords map[string]*querypb.Replica oldGarbage map[string]string - updateRecords map[string]*milvuspb.ReplicaInfo + updateRecords map[string]*querypb.Replica updateGarbage map[string]string delRecords []string expectInitEvents []nodeEvent @@ -56,17 +56,11 @@ func TestEtcdShardNodeDetector_watch(t *testing.T) { { name: "init normal case", ids: []int64{1, 2}, - oldRecords: map[string]*milvuspb.ReplicaInfo{ + oldRecords: map[string]*querypb.Replica{ "replica_1": { CollectionID: 1, - ReplicaID: 1, - NodeIds: []int64{1, 2}, - ShardReplicas: []*milvuspb.ShardReplica{ - { - LeaderID: 1, - DmChannelName: "dml1", - }, - }, + ID: 1, + Nodes: []int64{1, 2}, }, }, oldGarbage: map[string]string{ @@ -77,7 +71,6 @@ func TestEtcdShardNodeDetector_watch(t *testing.T) { nodeID: 1, nodeAddr: "1", eventType: nodeAdd, - isLeader: true, }, { nodeID: 2, @@ -92,16 +85,16 @@ func TestEtcdShardNodeDetector_watch(t *testing.T) { { name: "normal case with other replica", ids: []int64{1, 2}, - oldRecords: map[string]*milvuspb.ReplicaInfo{ + oldRecords: map[string]*querypb.Replica{ "replica_1": { CollectionID: 1, - ReplicaID: 1, - NodeIds: []int64{1, 2}, + ID: 1, + Nodes: []int64{1, 2}, }, "replica_2": { CollectionID: 1, - ReplicaID: 2, - NodeIds: []int64{1, 2}, + ID: 2, + Nodes: []int64{1, 2}, }, }, expectInitEvents: []nodeEvent{ @@ -122,11 +115,11 @@ func TestEtcdShardNodeDetector_watch(t *testing.T) { { name: "init normal missing node", ids: []int64{1}, - oldRecords: map[string]*milvuspb.ReplicaInfo{ + oldRecords: map[string]*querypb.Replica{ "replica_1": { CollectionID: 1, - ReplicaID: 1, - NodeIds: []int64{1, 2}, + ID: 1, + Nodes: []int64{1, 2}, }, }, expectInitEvents: []nodeEvent{ @@ -142,11 +135,11 @@ func TestEtcdShardNodeDetector_watch(t *testing.T) { { name: "normal updates", ids: []int64{1, 2, 3}, - oldRecords: map[string]*milvuspb.ReplicaInfo{ + oldRecords: map[string]*querypb.Replica{ "replica_1": { CollectionID: 1, - ReplicaID: 1, - NodeIds: []int64{1}, + ID: 1, + Nodes: []int64{1}, }, }, expectInitEvents: []nodeEvent{ @@ -156,16 +149,16 @@ func TestEtcdShardNodeDetector_watch(t *testing.T) { eventType: nodeAdd, }, }, - updateRecords: map[string]*milvuspb.ReplicaInfo{ + updateRecords: map[string]*querypb.Replica{ "replica_1": { CollectionID: 1, - ReplicaID: 1, - NodeIds: []int64{2}, + ID: 1, + Nodes: []int64{2}, }, "replica_1_extra": { CollectionID: 1, - ReplicaID: 1, - NodeIds: []int64{3, 4}, + ID: 1, + Nodes: []int64{3, 4}, }, }, updateGarbage: map[string]string{ @@ -194,11 +187,11 @@ func TestEtcdShardNodeDetector_watch(t *testing.T) { { name: "normal updates with other replica", ids: []int64{1, 2}, - oldRecords: map[string]*milvuspb.ReplicaInfo{ + oldRecords: map[string]*querypb.Replica{ "replica_1": { CollectionID: 1, - ReplicaID: 1, - NodeIds: []int64{1}, + ID: 1, + Nodes: []int64{1}, }, }, expectInitEvents: []nodeEvent{ @@ -208,16 +201,16 @@ func TestEtcdShardNodeDetector_watch(t *testing.T) { eventType: nodeAdd, }, }, - updateRecords: map[string]*milvuspb.ReplicaInfo{ + updateRecords: map[string]*querypb.Replica{ "replica_1": { CollectionID: 1, - ReplicaID: 1, - NodeIds: []int64{2}, + ID: 1, + Nodes: []int64{2}, }, "replica_2": { CollectionID: 1, - ReplicaID: 2, - NodeIds: []int64{2}, + ID: 2, + Nodes: []int64{2}, }, }, updateGarbage: map[string]string{ @@ -241,16 +234,16 @@ func TestEtcdShardNodeDetector_watch(t *testing.T) { { name: "normal deletes", ids: []int64{1, 2}, - oldRecords: map[string]*milvuspb.ReplicaInfo{ + oldRecords: map[string]*querypb.Replica{ "replica_1": { CollectionID: 1, - ReplicaID: 1, - NodeIds: []int64{1}, + ID: 1, + Nodes: []int64{1}, }, "replica_2": { CollectionID: 1, - ReplicaID: 2, - NodeIds: []int64{2}, + ID: 2, + Nodes: []int64{2}, }, }, oldGarbage: map[string]string{ diff --git a/internal/querynode/task.go b/internal/querynode/task.go index 4e24d5d09ed59..33f2342068b5a 100644 --- a/internal/querynode/task.go +++ b/internal/querynode/task.go @@ -250,13 +250,8 @@ func (w *watchDmChannelsTask) Execute(ctx context.Context) (err error) { // remove growing segment if watch dmChannels failed defer func() { if err != nil { - collection, err2 := w.node.metaReplica.getCollectionByID(collectionID) - if err2 == nil { - collection.Lock() - defer collection.Unlock() - for _, segmentID := range unFlushedSegmentIDs { - w.node.metaReplica.removeSegment(segmentID, segmentTypeGrowing) - } + for _, segmentID := range unFlushedSegmentIDs { + w.node.metaReplica.removeSegment(segmentID, segmentTypeGrowing) } } }() diff --git a/internal/types/types.go b/internal/types/types.go index 13fff86b2a536..2285913108777 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1235,6 +1235,7 @@ type QueryNode interface { TimeTickProvider WatchDmChannels(ctx context.Context, req *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) + UnsubDmChannel(ctx context.Context, req *querypb.UnsubDmChannelRequest) (*commonpb.Status, error) // LoadSegments notifies QueryNode to load the sealed segments from storage. The load tasks are sync to this // rpc, QueryNode will return after all the sealed segments are loaded. // @@ -1257,6 +1258,8 @@ type QueryNode interface { ShowConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) // GetMetrics gets the metrics about QueryNode. GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) + GetDataDistribution(context.Context, *querypb.GetDataDistributionRequest) (*querypb.GetDataDistributionResponse, error) + SyncDistribution(context.Context, *querypb.SyncDistributionRequest) (*commonpb.Status, error) } // QueryNodeComponent is used by grpc server of QueryNode diff --git a/internal/util/funcutil/slice.go b/internal/util/funcutil/slice.go index 09e15607f569c..087cf3fc7cc40 100644 --- a/internal/util/funcutil/slice.go +++ b/internal/util/funcutil/slice.go @@ -16,39 +16,31 @@ package funcutil -import "reflect" +import ( + "reflect" -// SliceContain returns true if slice s contains item. -func SliceContain(s, item interface{}) bool { - ss := reflect.ValueOf(s) - if ss.Kind() != reflect.Slice { - panic("SliceContain expect a slice") - } + "github.com/milvus-io/milvus/internal/util/typeutil" +) - for i := 0; i < ss.Len(); i++ { - if ss.Index(i).Interface() == item { +// SliceContain returns true if slice s contains item. +func SliceContain[T comparable](s []T, item T) bool { + for i := range s { + if s[i] == item { return true } } - return false } // SliceSetEqual is used to compare two Slice -func SliceSetEqual(s1, s2 interface{}) bool { - ss1 := reflect.ValueOf(s1) - ss2 := reflect.ValueOf(s2) - if ss1.Kind() != reflect.Slice { - panic("expect a slice") - } - if ss2.Kind() != reflect.Slice { - panic("expect a slice") - } - if ss1.Len() != ss2.Len() { +func SliceSetEqual[T comparable](s1, s2 []T) bool { + if len(s1) != len(s2) { return false } - for i := 0; i < ss1.Len(); i++ { - if !SliceContain(s2, ss1.Index(i).Interface()) { + + set := typeutil.NewSet(s1...) + for i := range s2 { + if !set.Contain(s2[i]) { return false } } diff --git a/internal/util/funcutil/slice_test.go b/internal/util/funcutil/slice_test.go index b8e664b0b8fd6..10dd091e54ce0 100644 --- a/internal/util/funcutil/slice_test.go +++ b/internal/util/funcutil/slice_test.go @@ -24,15 +24,12 @@ import ( ) func Test_SliceContain(t *testing.T) { - invalid := "invalid" - assert.Panics(t, func() { SliceContain(invalid, 1) }) - strSlice := []string{"test", "for", "SliceContain"} intSlice := []int{1, 2, 3} cases := []struct { - s interface{} - item interface{} + s any + item any want bool }{ {strSlice, "test", true}, @@ -46,18 +43,20 @@ func Test_SliceContain(t *testing.T) { } for _, test := range cases { - if got := SliceContain(test.s, test.item); got != test.want { - t.Errorf("SliceContain(%v, %v) = %v", test.s, test.item, test.want) + switch test.item.(type) { + case string: + if got := SliceContain(test.s.([]string), test.item.(string)); got != test.want { + t.Errorf("SliceContain(%v, %v) = %v", test.s, test.item, test.want) + } + case int: + if got := SliceContain(test.s.([]int), test.item.(int)); got != test.want { + t.Errorf("SliceContain(%v, %v) = %v", test.s, test.item, test.want) + } } } } func Test_SliceSetEqual(t *testing.T) { - invalid := "invalid" - assert.Panics(t, func() { SliceSetEqual(invalid, 1) }) - temp := []int{1, 2, 3} - assert.Panics(t, func() { SliceSetEqual(temp, invalid) }) - cases := []struct { s1 interface{} s2 interface{} @@ -78,8 +77,15 @@ func Test_SliceSetEqual(t *testing.T) { } for _, test := range cases { - if got := SliceSetEqual(test.s1, test.s2); got != test.want { - t.Errorf("SliceSetEqual(%v, %v) = %v", test.s1, test.s2, test.want) + switch test.s1.(type) { + case string: + if got := SliceSetEqual(test.s1.([]string), test.s2.([]string)); got != test.want { + t.Errorf("SliceSetEqual(%v, %v) = %v", test.s1, test.s2, test.want) + } + case int: + if got := SliceSetEqual(test.s1.([]int), test.s2.([]int)); got != test.want { + t.Errorf("SliceSetEqual(%v, %v) = %v", test.s1, test.s2, test.want) + } } } } diff --git a/internal/util/mock/grpc_querynode_client.go b/internal/util/mock/grpc_querynode_client.go index 7906662f0ddfc..e7396f1d7626e 100644 --- a/internal/util/mock/grpc_querynode_client.go +++ b/internal/util/mock/grpc_querynode_client.go @@ -92,3 +92,15 @@ func (m *GrpcQueryNodeClient) GetMetrics(ctx context.Context, in *milvuspb.GetMe func (m *GrpcQueryNodeClient) ShowConfigurations(ctx context.Context, in *internalpb.ShowConfigurationsRequest, opts ...grpc.CallOption) (*internalpb.ShowConfigurationsResponse, error) { return &internalpb.ShowConfigurationsResponse{}, m.Err } + +func (m *GrpcQueryNodeClient) GetDataDistribution(ctx context.Context, in *querypb.GetDataDistributionRequest, opts ...grpc.CallOption) (*querypb.GetDataDistributionResponse, error) { + return &querypb.GetDataDistributionResponse{}, m.Err +} + +func (m *GrpcQueryNodeClient) SyncDistribution(ctx context.Context, in *querypb.SyncDistributionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) { + return &commonpb.Status{}, m.Err +} + +func (m *GrpcQueryNodeClient) UnsubDmChannel(ctx context.Context, req *querypb.UnsubDmChannelRequest, opts ...grpc.CallOption) (*commonpb.Status, error) { + return &commonpb.Status{}, m.Err +} diff --git a/internal/util/mock/querynode_client.go b/internal/util/mock/querynode_client.go index 3ac2b80bd8727..8e91e04d40df5 100644 --- a/internal/util/mock/querynode_client.go +++ b/internal/util/mock/querynode_client.go @@ -108,3 +108,15 @@ func (q QueryNodeClient) GetMetrics(ctx context.Context, req *milvuspb.GetMetric func (q QueryNodeClient) ShowConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) { return q.grpcClient.ShowConfigurations(ctx, req) } + +func (q QueryNodeClient) UnsubDmChannel(ctx context.Context, req *querypb.UnsubDmChannelRequest) (*commonpb.Status, error) { + return q.grpcClient.UnsubDmChannel(ctx, req) +} + +func (q QueryNodeClient) GetDataDistribution(ctx context.Context, req *querypb.GetDataDistributionRequest) (*querypb.GetDataDistributionResponse, error) { + return q.grpcClient.GetDataDistribution(ctx, req) +} + +func (q QueryNodeClient) SyncDistribution(ctx context.Context, req *querypb.SyncDistributionRequest) (*commonpb.Status, error) { + return q.grpcClient.SyncDistribution(ctx, req) +} diff --git a/internal/util/paramtable/component_param.go b/internal/util/paramtable/component_param.go index d56afa524f25b..60bb0144341b5 100644 --- a/internal/util/paramtable/component_param.go +++ b/internal/util/paramtable/component_param.go @@ -612,6 +612,12 @@ type queryCoordConfig struct { OverloadedMemoryThresholdPercentage float64 BalanceIntervalSeconds int64 MemoryUsageMaxDifferencePercentage float64 + CheckInterval time.Duration + ChannelTaskTimeout time.Duration + SegmentTaskTimeout time.Duration + DistPullInterval time.Duration + LoadTimeoutSeconds time.Duration + CheckHandoffInterval time.Duration } func (p *queryCoordConfig) init(base *BaseTable) { @@ -630,6 +636,12 @@ func (p *queryCoordConfig) init(base *BaseTable) { p.initOverloadedMemoryThresholdPercentage() p.initBalanceIntervalSeconds() p.initMemoryUsageMaxDifferencePercentage() + p.initCheckInterval() + p.initChannelTaskTimeout() + p.initSegmentTaskTimeout() + p.initDistPullInterval() + p.initLoadTimeoutSeconds() + p.initCheckHandoffInterval() } func (p *queryCoordConfig) initTaskRetryNum() { @@ -687,6 +699,60 @@ func (p *queryCoordConfig) initMemoryUsageMaxDifferencePercentage() { p.MemoryUsageMaxDifferencePercentage = float64(diffPercentage) / 100 } +func (p *queryCoordConfig) initCheckInterval() { + interval := p.Base.LoadWithDefault("queryCoord.checkInterval", "1000") + checkInterval, err := strconv.ParseInt(interval, 10, 64) + if err != nil { + panic(err) + } + p.CheckInterval = time.Duration(checkInterval) * time.Millisecond +} + +func (p *queryCoordConfig) initChannelTaskTimeout() { + timeout := p.Base.LoadWithDefault("queryCoord.channelTaskTimeout", "60000") + taskTimeout, err := strconv.ParseInt(timeout, 10, 64) + if err != nil { + panic(err) + } + p.ChannelTaskTimeout = time.Duration(taskTimeout) * time.Millisecond +} + +func (p *queryCoordConfig) initSegmentTaskTimeout() { + timeout := p.Base.LoadWithDefault("queryCoord.segmentTaskTimeout", "15000") + taskTimeout, err := strconv.ParseInt(timeout, 10, 64) + if err != nil { + panic(err) + } + p.SegmentTaskTimeout = time.Duration(taskTimeout) * time.Millisecond +} + +func (p *queryCoordConfig) initDistPullInterval() { + interval := p.Base.LoadWithDefault("queryCoord.distPullInterval", "500") + pullInterval, err := strconv.ParseInt(interval, 10, 64) + if err != nil { + panic(err) + } + p.DistPullInterval = time.Duration(pullInterval) * time.Millisecond +} + +func (p *queryCoordConfig) initLoadTimeoutSeconds() { + timeout := p.Base.LoadWithDefault("queryCoord.loadTimeoutSeconds", "600") + loadTimeout, err := strconv.ParseInt(timeout, 10, 64) + if err != nil { + panic(err) + } + p.LoadTimeoutSeconds = time.Duration(loadTimeout) * time.Second +} + +func (p *queryCoordConfig) initCheckHandoffInterval() { + interval := p.Base.LoadWithDefault("queryCoord.checkHandoffInterval", "5000") + checkHandoffInterval, err := strconv.ParseInt(interval, 10, 64) + if err != nil { + panic(err) + } + p.CheckHandoffInterval = time.Duration(checkHandoffInterval) * time.Millisecond +} + func (p *queryCoordConfig) SetNodeID(id UniqueID) { p.NodeID.Store(id) } diff --git a/internal/util/typeutil/group_scheduler.go b/internal/util/typeutil/group_scheduler.go new file mode 100644 index 0000000000000..1672c11dcd8e6 --- /dev/null +++ b/internal/util/typeutil/group_scheduler.go @@ -0,0 +1,151 @@ +package typeutil + +import ( + "context" + "sync" + "time" + + "github.com/milvus-io/milvus/internal/log" +) + +// GroupScheduler schedules requests, +// all requests within the same partition & node will run sequentially, +// with group commit +const ( + taskQueueCap = 16 + waitQueueCap = 128 +) + +type MergeableTask[K comparable, R any] interface { + ID() K + Execute() error + Merge(other MergeableTask[K, R]) + SetResult(R) + SetError(error) + Done() + Wait() (R, error) +} + +type GroupScheduler[K comparable, R any] struct { + stopCh chan struct{} + wg sync.WaitGroup + + processors *ConcurrentSet[K] // Tasks of having processor + queues map[K]chan MergeableTask[K, R] // TaskID -> Queue + waitQueue chan MergeableTask[K, R] +} + +func NewGroupScheduler[K comparable, R any]() *GroupScheduler[K, R] { + return &GroupScheduler[K, R]{ + stopCh: make(chan struct{}), + processors: NewConcurrentSet[K](), + queues: make(map[K]chan MergeableTask[K, R]), + waitQueue: make(chan MergeableTask[K, R], waitQueueCap), + } +} + +func (scheduler *GroupScheduler[K, R]) Start(ctx context.Context) { + scheduler.wg.Add(1) + go scheduler.schedule(ctx) +} + +func (scheduler *GroupScheduler[K, R]) Stop() { + close(scheduler.stopCh) + scheduler.wg.Wait() +} + +func (scheduler *GroupScheduler[K, R]) schedule(ctx context.Context) { + defer scheduler.wg.Done() + + ticker := time.NewTicker(500 * time.Millisecond) + go func() { + for { + select { + case <-ctx.Done(): + log.Info("GroupScheduler stopped due to context canceled") + return + + case <-scheduler.stopCh: + log.Info("GroupScheduler stopped") + return + + case task := <-scheduler.waitQueue: + queue, ok := scheduler.queues[task.ID()] + if !ok { + queue = make(chan MergeableTask[K, R], taskQueueCap) + scheduler.queues[task.ID()] = queue + } + outer: + for { + select { + case queue <- task: + break outer + default: // Queue full, flush and retry + scheduler.startProcessor(task.ID(), queue) + } + } + + case <-ticker.C: + for id, queue := range scheduler.queues { + if len(queue) > 0 { + scheduler.startProcessor(id, queue) + } else { + // Release resource if no job for the task + delete(scheduler.queues, id) + } + } + } + } + }() +} + +func (scheduler *GroupScheduler[K, R]) isStopped() bool { + select { + case <-scheduler.stopCh: + return true + default: + return false + } +} + +func (scheduler *GroupScheduler[K, R]) Add(job MergeableTask[K, R]) { + scheduler.waitQueue <- job +} + +func (scheduler *GroupScheduler[K, R]) startProcessor(id K, queue chan MergeableTask[K, R]) { + if scheduler.isStopped() { + return + } + if !scheduler.processors.Insert(id) { + return + } + + scheduler.wg.Add(1) + go scheduler.processQueue(id, queue) +} + +// processQueue processes tasks in the given queue, +// it only processes tasks with the number of the length of queue at the time, +// to avoid leaking goroutines +func (scheduler *GroupScheduler[K, R]) processQueue(id K, queue chan MergeableTask[K, R]) { + defer scheduler.wg.Done() + defer scheduler.processors.Remove(id) + + len := len(queue) + buffer := make([]MergeableTask[K, R], len) + for i := range buffer { + buffer[i] = <-queue + if i > 0 { + buffer[0].Merge(buffer[i]) + } + } + + buffer[0].Execute() + buffer[0].Done() + result, err := buffer[0].Wait() + for _, buffer := range buffer[1:] { + buffer.SetResult(result) + buffer.SetError(err) + buffer.Done() + } +} diff --git a/internal/util/typeutil/map.go b/internal/util/typeutil/map.go index 18f904ed2b107..630c2a73cd16d 100644 --- a/internal/util/typeutil/map.go +++ b/internal/util/typeutil/map.go @@ -1,5 +1,7 @@ package typeutil +import "sync" + // MergeMap merge one map to another func MergeMap(src map[string]string, dst map[string]string) map[string]string { for k, v := range src { @@ -16,3 +18,55 @@ func GetMapKeys(src map[string]string) []string { } return keys } + +type ConcurrentMap[K comparable, V any] struct { + inner sync.Map +} + +func NewConcurrentMap[K comparable, V any]() *ConcurrentMap[K, V] { + return &ConcurrentMap[K, V]{} +} + +func (m *ConcurrentMap[K, V]) Insert(key K, value V) { + m.inner.Store(key, value) +} + +func (m *ConcurrentMap[K, V]) Get(key K) (V, bool) { + var zeroValue V + value, ok := m.inner.Load(key) + if !ok { + return zeroValue, ok + } + return value.(V), true +} + +func (m *ConcurrentMap[K, V]) get(key K) (V, bool) { + var zeroValue V + value, ok := m.inner.Load(key) + if !ok { + return zeroValue, ok + } + return value.(V), true +} + +func (m *ConcurrentMap[K, V]) GetOrInsert(key K, value V) (V, bool) { + var zeroValue V + loaded, exist := m.inner.LoadOrStore(key, value) + if !exist { + return zeroValue, exist + } + return loaded.(V), true +} + +func (m *ConcurrentMap[K, V]) GetAndRemove(key K) (V, bool) { + var zeroValue V + value, ok := m.inner.LoadAndDelete(key) + if !ok { + return zeroValue, ok + } + return value.(V), true +} + +func (m *ConcurrentMap[K, V]) Remove(key K) { + m.inner.Delete(key) +} diff --git a/internal/util/typeutil/set.go b/internal/util/typeutil/set.go index 18974dca59c30..d408f976c03b5 100644 --- a/internal/util/typeutil/set.go +++ b/internal/util/typeutil/set.go @@ -16,46 +16,141 @@ package typeutil +import ( + "sync" +) + // UniqueSet is set type, which contains only UniqueIDs, // the underlying type is map[UniqueID]struct{}. // Create a UniqueSet instance with make(UniqueSet) like creating a map instance. -type UniqueSet map[UniqueID]struct{} +type UniqueSet = Set[UniqueID] + +func NewUniqueSet(ids ...UniqueID) UniqueSet { + set := make(UniqueSet) + set.Insert(ids...) + return set +} + +type Set[T comparable] map[T]struct{} + +func NewSet[T comparable](elements ...T) Set[T] { + set := make(Set[T]) + set.Insert(elements...) + return set +} // Insert elements into the set, // do nothing if the id existed -func (set UniqueSet) Insert(ids ...UniqueID) { - for i := range ids { - set[ids[i]] = struct{}{} +func (set Set[T]) Insert(elements ...T) { + for i := range elements { + set[elements[i]] = struct{}{} } } +// Intersection returns the intersection with the given set +func (set Set[T]) Intersection(other Set[T]) Set[T] { + ret := NewSet[T]() + for elem := range set { + if other.Contain(elem) { + ret.Insert(elem) + } + } + return ret +} + +// Union returns the union with the given set +func (set Set[T]) Union(other Set[T]) Set[T] { + ret := NewSet(set.Collect()...) + ret.Insert(other.Collect()...) + return ret +} + +// Complement returns the complement with the given set +func (set Set[T]) Complement(other Set[T]) Set[T] { + ret := NewSet(set.Collect()...) + ret.Remove(other.Collect()...) + return ret +} + // Check whether the elements exist -func (set UniqueSet) Contain(ids ...UniqueID) bool { - for i := range ids { - _, ok := set[ids[i]] +func (set Set[T]) Contain(elements ...T) bool { + for i := range elements { + _, ok := set[elements[i]] if !ok { return false } } - return true } // Remove elements from the set, // do nothing if set is nil or id not exists -func (set UniqueSet) Remove(ids ...UniqueID) { - for i := range ids { - delete(set, ids[i]) +func (set Set[T]) Remove(elements ...T) { + for i := range elements { + delete(set, elements[i]) } } // Get all elements in the set -func (set UniqueSet) Collect() []UniqueID { - ids := make([]UniqueID, 0, len(set)) +func (set Set[T]) Collect() []T { + elements := make([]T, 0, len(set)) + for elem := range set { + elements = append(elements, elem) + } + return elements +} + +// Len returns the number of elements in the set +func (set Set[T]) Len() int { + return len(set) +} + +type ConcurrentSet[T comparable] struct { + inner sync.Map +} - for id := range set { - ids = append(ids, id) +func NewConcurrentSet[T comparable]() *ConcurrentSet[T] { + return &ConcurrentSet[T]{} +} + +// Insert elements into the set, +// do nothing if the id existed +func (set *ConcurrentSet[T]) Upsert(elements ...T) { + for i := range elements { + set.inner.Store(elements[i], struct{}{}) } +} - return ids +func (set *ConcurrentSet[T]) Insert(element T) bool { + _, exist := set.inner.LoadOrStore(element, struct{}{}) + return !exist +} + +// Check whether the elements exist +func (set *ConcurrentSet[T]) Contain(elements ...T) bool { + for i := range elements { + _, ok := set.inner.Load(elements[i]) + if !ok { + return false + } + } + return true +} + +// Remove elements from the set, +// do nothing if set is nil or id not exists +func (set *ConcurrentSet[T]) Remove(elements ...T) { + for i := range elements { + set.inner.Delete(elements[i]) + } +} + +// Get all elements in the set +func (set *ConcurrentSet[T]) Collect() []T { + elements := make([]T, 0) + set.inner.Range(func(key, value any) bool { + elements = append(elements, key.(T)) + return true + }) + return elements } diff --git a/rules.go b/rules.go index bdec749333359..4c45aa7c5d1e9 100644 --- a/rules.go +++ b/rules.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build ignore // +build ignore package gorules @@ -219,30 +220,30 @@ func oddmathbits(m dsl.Matcher) { ).Report("odd math/bits expression: use bits.Len*() instead?") } -func floateq(m dsl.Matcher) { - m.Match( - "$x == $y", - "$x != $y", - ). - Where(m["x"].Type.Is("float32") && !m["x"].Const && !m["y"].Text.Matches("0(.0+)?") && !m.File().Name.Matches("floating_comparision.go")). - Report("floating point tested for equality") - - m.Match( - "$x == $y", - "$x != $y", - ). - Where(m["x"].Type.Is("float64") && !m["x"].Const && !m["y"].Text.Matches("0(.0+)?") && !m.File().Name.Matches("floating_comparision.go")). - Report("floating point tested for equality") - - m.Match("switch $x { $*_ }", "switch $*_; $x { $*_ }"). - Where(m["x"].Type.Is("float32")). - Report("floating point as switch expression") - - m.Match("switch $x { $*_ }", "switch $*_; $x { $*_ }"). - Where(m["x"].Type.Is("float64")). - Report("floating point as switch expression") - -} +// func floateq(m dsl.Matcher) { +// m.Match( +// "$x == $y", +// "$x != $y", +// ). +// Where(m["x"].Type.Is("float32") && !m["x"].Const && !m["y"].Text.Matches("0(.0+)?") && !m.File().Name.Matches("floating_comparision.go")). +// Report("floating point tested for equality") + +// m.Match( +// "$x == $y", +// "$x != $y", +// ). +// Where(m["x"].Type.Is("float64") && !m["x"].Const && !m["y"].Text.Matches("0(.0+)?") && !m.File().Name.Matches("floating_comparision.go")). +// Report("floating point tested for equality") + +// m.Match("switch $x { $*_ }", "switch $*_; $x { $*_ }"). +// Where(m["x"].Type.Is("float32")). +// Report("floating point as switch expression") + +// m.Match("switch $x { $*_ }", "switch $*_; $x { $*_ }"). +// Where(m["x"].Type.Is("float64")). +// Report("floating point as switch expression") + +// } func badexponent(m dsl.Matcher) { m.Match( diff --git a/scripts/proto_gen_go.sh b/scripts/proto_gen_go.sh index 43b3bbe68bf5d..6d6cca44ea149 100755 --- a/scripts/proto_gen_go.sh +++ b/scripts/proto_gen_go.sh @@ -51,7 +51,6 @@ mkdir -p indexpb mkdir -p datapb mkdir -p querypb mkdir -p planpb -mkdir -p querypbv2 ${protoc} --proto_path="${GOOGLE_PROTO_DIR}" --proto_path=. --go_out=plugins=grpc,paths=source_relative:./commonpb common.proto ${protoc} --proto_path="${GOOGLE_PROTO_DIR}" --proto_path=. --go_out=plugins=grpc,paths=source_relative:./schemapb schema.proto @@ -68,6 +67,5 @@ ${protoc} --proto_path="${GOOGLE_PROTO_DIR}" --proto_path=. --go_out=plugins=grp ${protoc} --proto_path="${GOOGLE_PROTO_DIR}" --proto_path=. --go_out=plugins=grpc,paths=source_relative:./querypb query_coord.proto ${protoc} --proto_path="${GOOGLE_PROTO_DIR}" --proto_path=. --go_out=plugins=grpc,paths=source_relative:./planpb plan.proto ${protoc} --proto_path="${GOOGLE_PROTO_DIR}" --proto_path=. --go_out=plugins=grpc,paths=source_relative:./segcorepb segcore.proto -${protoc} --proto_path="${GOOGLE_PROTO_DIR}" --proto_path=. --go_out=plugins=grpc,paths=source_relative:./querypbv2 query_coordv2.proto popd diff --git a/scripts/run_go_unittest.sh b/scripts/run_go_unittest.sh index f58b9e70c5af8..02ceb3e5a921c 100755 --- a/scripts/run_go_unittest.sh +++ b/scripts/run_go_unittest.sh @@ -67,6 +67,11 @@ go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/proxy/..." -failfast go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/proxy/..." -failfast } +function test_querycoordv2() +{ +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/querycoordv2/..." -failfast +} + function test_querynode() { go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/querynode/..." -failfast @@ -174,6 +179,7 @@ test_tso test_config test_util test_metastore +test_querycoordv2 }