Mishards 070 (milvus-io#1699)

* update mishards to 0.7.0 Signed-off-by: Yhz <[email protected]> * update mishards to 0.7.0 Signed-off-by: Yhz <[email protected]> * fix search bug and example pass Signed-off-by: Yhz <[email protected]> * add new api Signed-off-by: Yhz <[email protected]> * add segment stat Signed-off-by: Yhz <[email protected]> * fix table info issue Signed-off-by: yhz <[email protected]> * fix mishards api issue Signed-off-by: yhz <[email protected]> * update mishards Signed-off-by: Yhz <[email protected]> * update all_in_one docker images Signed-off-by: Yhz <[email protected]> * fix delete_by_id param parser & remove some comments Signed-off-by: Yhz <[email protected]> * update yaml config file Signed-off-by: Yhz <[email protected]> * update all_in_one config Signed-off-by: Yhz <[email protected]> * remove delete_by_range comments Signed-off-by: Yhz <[email protected]> * update cmd api Signed-off-by: Yhz <[email protected]> * add warning when search Signed-off-by: Yhz <[email protected]> * update service_handler Signed-off-by: Yhz <[email protected]> * update shrads requiremtns Signed-off-by: Yhz <[email protected]> * [skip ci] remove surplus log info Signed-off-by: Yhz <[email protected]>
yhmo · Mar 20, 2020 · 3c3617f · 3c3617f
1 parent fe2595f
commit 3c3617f
Show file tree

Hide file tree

Showing 14 changed files with 982 additions and 239 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -28,6 +28,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1649 Fix Milvus crash on old CPU 
 -   \#1653 IndexFlat (SSE) and IndexBinaryFlat performance improvement for small NQ
 -   \#1678 Remove CUSTOMIZATION macro 
+-   \#1698 Upgrade mishards to v0.7.0
 
 ## Task
 

diff --git a/shards/all_in_one/all_in_one.yml b/shards/all_in_one/all_in_one.yml
@@ -3,7 +3,7 @@ services:
     milvus_wr:
         runtime: nvidia
         restart: always
-        image: milvusdb/milvus:0.6.0-gpu-d120719-2b40dd
+        image: milvusdb/milvus:0.7.0-gpu-d031120-40c7e8
         ports:
             - "0.0.0.0:19540:19530"
         volumes:
@@ -13,7 +13,7 @@ services:
     milvus_ro:
         runtime: nvidia
         restart: always
-        image: milvusdb/milvus:0.6.0-gpu-d120719-2b40dd
+        image: milvusdb/milvus:0.7.0-gpu-d031120-40c7e8
         ports:
             - "0.0.0.0:19541:19530"
         volumes:

diff --git a/shards/all_in_one/ro_server.yml b/shards/all_in_one/ro_server.yml
diff --git a/shards/all_in_one/wr_server.yml b/shards/all_in_one/wr_server.yml
diff --git a/shards/all_in_one_with_mysql/all_in_one.yml b/shards/all_in_one_with_mysql/all_in_one.yml
@@ -18,7 +18,7 @@ services:
     milvus_wr:
         runtime: nvidia
         restart: always
-        image: milvusdb/milvus:0.6.0-gpu-d120719-2b40dd
+        image: milvusdb/milvus:0.7.0-gpu-d031120-40c7e8
         volumes:
             - /tmp/milvus/db:/var/lib/milvus/db
             - ./wr_server.yml:/var/lib/milvus/conf/server_config.yaml
@@ -29,7 +29,7 @@ services:
     milvus_ro:
         runtime: nvidia
         restart: always
-        image: milvusdb/milvus:0.6.0-gpu-d120719-2b40dd
+        image: milvusdb/milvus:0.7.0-gpu-d031120-40c7e8
         volumes:
             - /tmp/milvus/db:/var/lib/milvus/db
             - ./ro_server.yml:/var/lib/milvus/conf/server_config.yaml

diff --git a/shards/all_in_one_with_mysql/ro_server.yml b/shards/all_in_one_with_mysql/ro_server.yml
diff --git a/shards/all_in_one_with_mysql/wr_server.yml b/shards/all_in_one_with_mysql/wr_server.yml
diff --git a/shards/mishards/exception_codes.py b/shards/mishards/exception_codes.py
@@ -8,3 +8,4 @@
 INVALID_ARGUMENT_CODE = 20002
 INVALID_DATE_RANGE_CODE = 20003
 INVALID_TOPK_CODE = 20004
+SEARCH_PARAM_LOSS_CODE = 20005
diff --git a/shards/mishards/exceptions.py b/shards/mishards/exceptions.py
@@ -30,6 +30,10 @@ class InvalidTopKError(BaseException):
     code = codes.INVALID_TOPK_CODE
 
 
+class SearchParamError(BaseException):
+    code = codes.SEARCH_PARAM_LOSS_CODE
+
+
 class InvalidArgumentError(BaseException):
     code = codes.INVALID_ARGUMENT_CODE
 

diff --git a/shards/mishards/grpc_utils/grpc_args_parser.py b/shards/mishards/grpc_utils/grpc_args_parser.py
@@ -1,3 +1,4 @@
+import ujson
 from milvus import Status
 from functools import wraps
 
@@ -21,26 +22,30 @@ class GrpcArgsParser(object):
     @error_status
     def parse_proto_TableSchema(cls, param):
         _table_schema = {
-            'status': param.status,
-            'table_name': param.table_name,
+            'collection_name': param.table_name,
             'dimension': param.dimension,
             'index_file_size': param.index_file_size,
             'metric_type': param.metric_type
         }
 
-        return _table_schema
+        return param.status, _table_schema
 
     @classmethod
     @error_status
     def parse_proto_TableName(cls, param):
         return param.table_name
 
+    @classmethod
+    @error_status
+    def parse_proto_FlushParam(cls, param):
+        return list(param.table_name_array)
+
     @classmethod
     @error_status
     def parse_proto_Index(cls, param):
         _index = {
             'index_type': param.index_type,
-            'nlist': param.nlist
+            'params': param.extra_params[0].value
         }
 
         return _index
@@ -49,12 +54,14 @@ def parse_proto_Index(cls, param):
     @error_status
     def parse_proto_IndexParam(cls, param):
         _table_name = param.table_name
-        _status, _index = cls.parse_proto_Index(param.index)
+        _index_type = param.index_type
+        _index_param = {}
 
-        if not _status.OK():
-            raise Exception("Argument parse error")
+        for params in param.extra_params:
+            if params.key == 'params':
+                _index_param = ujson.loads(str(params.value))
 
-        return _table_name, _index
+        return _table_name, _index_type, _index_param
 
     @classmethod
     @error_status
@@ -63,14 +70,6 @@ def parse_proto_Command(cls, param):
 
         return _cmd
 
-    @classmethod
-    @error_status
-    def parse_proto_Range(cls, param):
-        _start_value = param.start_value
-        _end_value = param.end_value
-
-        return _start_value, _end_value
-
     @classmethod
     @error_status
     def parse_proto_RowRecord(cls, param):
@@ -79,32 +78,52 @@ def parse_proto_RowRecord(cls, param):
     @classmethod
     def parse_proto_PartitionParam(cls, param):
         _table_name = param.table_name
-        _partition_name = param.partition_name
         _tag = param.tag
 
-        return _table_name, _partition_name, _tag
+        return _table_name, _tag
 
     @classmethod
     @error_status
     def parse_proto_SearchParam(cls, param):
         _table_name = param.table_name
         _topk = param.topk
-        _nprobe = param.nprobe
-        _status, _range = cls.parse_proto_Range(param.query_range_array)
 
-        if not _status.OK():
-            raise Exception("Argument parse error")
+        if len(param.extra_params) == 0:
+            raise Exception("Search param loss")
+        _params = ujson.loads(str(param.extra_params[0].value))
+
+        _query_record_array = []
+        if param.query_record_array:
+            for record in param.query_record_array:
+                if record.float_data:
+                    _query_record_array.append(list(record.float_data))
+                else:
+                    _query_record_array.append(bytes(record.binary_data))
+        else:
+            raise Exception("Search argument parse error: record array is empty")
 
-        _row_record = param.query_record_array
+        return _table_name, _query_record_array, _topk, _params
 
-        return _table_name, _row_record, _range, _topk
+    @classmethod
+    @error_status
+    def parse_proto_DeleteByIDParam(cls, param):
+        _table_name = param.table_name
+        _id_array = list(param.id_array)
+
+        return _table_name, _id_array
 
     @classmethod
     @error_status
-    def parse_proto_DeleteByRangeParam(cls, param):
+    def parse_proto_VectorIdentity(cls, param):
         _table_name = param.table_name
-        _range = param.range
-        _start_value = _range.start_value
-        _end_value = _range.end_value
+        _id = param.id
+
+        return _table_name, _id
+
+    @classmethod
+    @error_status
+    def parse_proto_GetVectorIDsParam(cls, param):
+        _table__name = param.table_name
+        _segment_name = param.segment_name
 
-        return _table_name, _start_value, _end_value
+        return _table__name, _segment_name
diff --git a/shards/mishards/models.py b/shards/mishards/models.py
@@ -23,6 +23,7 @@ class TableFiles(db.Model):
 
     id = Column(BigInteger, primary_key=True, autoincrement=True)
     table_id = Column(String(50))
+    segment_id = Column(String(50))
     engine_type = Column(Integer)
     file_id = Column(String(50))
     file_type = Column(Integer)
@@ -31,6 +32,7 @@ class TableFiles(db.Model):
     updated_time = Column(BigInteger)
     created_on = Column(BigInteger)
     date = Column(Integer)
+    flush_lsn = Column(Integer)
 
     table = relationship(
         'Tables',
@@ -55,9 +57,10 @@ class Tables(db.Model):
     created_on = Column(Integer)
     flag = Column(Integer, default=0)
     index_file_size = Column(Integer)
+    index_params = Column(String(50))
     engine_type = Column(Integer)
-    nlist = Column(Integer)
     metric_type = Column(Integer)
+    flush_lsn = Column(Integer)
 
     def files_to_search(self, date_range=None):
         cond = or_(