backup: remove unnecessary conversion from vch to pch (#552)

Signed-off-by: huanghaoyuanhhy <[email protected]>

core/backup_impl_create_backup.go

@@ -774,13 +774,12 @@ func (b *BackupContext) writeBackupInfoMeta(ctx context.Context, id string) erro
 	log.Debug("segment meta", zap.String("value", string(output.SegmentMetaBytes)))
 
 	collectionBackups := backupInfo.GetCollectionBackups()
-	collectionPositions := make(map[string][]*backuppb.ChannelPosition, 0)
+	collectionPositions := make(map[string][]*backuppb.ChannelPosition)
 	for _, collectionBackup := range collectionBackups {
-		collectionCPs := make([]*backuppb.ChannelPosition, 0)
+		collectionCPs := make([]*backuppb.ChannelPosition, 0, len(collectionBackup.GetChannelCheckpoints()))
 		for vCh, position := range collectionBackup.GetChannelCheckpoints() {
-			pCh := strings.Split(vCh, "_")[0] + "_" + strings.Split(vCh, "_")[1]
 			collectionCPs = append(collectionCPs, &backuppb.ChannelPosition{
-				Name:     pCh,
+				Name:     vCh,
 				Position: position,
 			})
 		}

core/utils/convert_util.go

@@ -2,107 +2,11 @@ package utils
 
 import (
 	"encoding/base64"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"time"
 
 	"github.com/golang/protobuf/proto"
 	"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
-
-	"github.com/zilliztech/milvus-backup/core/proto/backuppb"
-)
-
-const (
-	logicalBits     = 18
-	logicalBitsMask = (1 << logicalBits) - 1
-	PARAMS          = "params"
 )
 
-// ComposeTS returns a timestamp composed of physical part and logical part
-func ComposeTS(physical, logical int64) uint64 {
-	return uint64((physical << logicalBits) + logical)
-}
-
-// ParseTS returns a timestamp composed of physical part and logical part
-func ParseTS(ts uint64) (time.Time, uint64) {
-	logical := ts & logicalBitsMask
-	physical := ts >> logicalBits
-	physicalTime := time.Unix(int64(physical/1000), int64(physical)%1000*time.Millisecond.Nanoseconds())
-	return physicalTime, logical
-}
-
-// kvPairToMap largely copied from internal/proxy/task.go#parseIndexParams
-func KVPairToMap(m []*backuppb.KeyValuePair) (map[string]string, error) {
-	params := make(map[string]string)
-	for _, kv := range m {
-		if kv.Key == PARAMS {
-			params, err := parseParamsMap(kv.Value)
-			if err != nil {
-				return nil, err
-			}
-			for k, v := range params {
-				params[k] = v
-			}
-		} else {
-			params[kv.Key] = kv.Value
-		}
-	}
-	return params, nil
-}
-
-// parseParamsMap parse the jsonic index parameters to map
-func parseParamsMap(mStr string) (map[string]string, error) {
-	buffer := make(map[string]interface{})
-	err := json.Unmarshal([]byte(mStr), &buffer)
-	if err != nil {
-		return nil, errors.New("Unmarshal params failed")
-	}
-	ret := make(map[string]string)
-	for key, value := range buffer {
-		valueStr := fmt.Sprintf("%v", value)
-		ret[key] = valueStr
-	}
-	return ret, nil
-}
-
-func MapToKVPair(dict map[string]string) []*backuppb.KeyValuePair {
-	kvs := make([]*backuppb.KeyValuePair, 0)
-
-	for key, value := range dict {
-		kvs = append(kvs, &backuppb.KeyValuePair{
-			Key:   key,
-			Value: value,
-		})
-	}
-	return kvs
-}
-
-// KvPairsMap converts common.KeyValuePair slices into map
-func KvPairsMap(kvps []*backuppb.KeyValuePair) map[string]string {
-	m := make(map[string]string)
-	for _, kvp := range kvps {
-		m[kvp.Key] = kvp.Value
-	}
-	return m
-}
-
-func ArrayToMap(strs []int64) map[int64]bool {
-	ret := make(map[int64]bool)
-	for _, value := range strs {
-		ret[value] = true
-	}
-	return ret
-}
-
-func MapKeyArray(dict map[int64]bool) []int64 {
-	arr := make([]int64, 0)
-	for k, _ := range dict {
-		arr = append(arr, k)
-	}
-	return arr
-}
-
 func Base64MsgPosition(position *msgpb.MsgPosition) string {
 	positionByte, err := proto.Marshal(position)
 	if err != nil {

core/utils/convert_util_test.go

@@ -1,13 +1,31 @@
 package utils
 
 import (
+	"encoding/base64"
 	"testing"
+
+	"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/protobuf/proto"
 )
 
-func TestTs(t *testing.T) {
-	ts := 443727974068387848
-	time, logical := ParseTS(uint64(ts))
-	println(time.Unix())
-	println(logical)
+func TestBase64MsgPosition(t *testing.T) {
+	position := &msgpb.MsgPosition{
+		ChannelName: "channel",
+		MsgID:       []byte("msg_id"),
+		Timestamp:   10,
+		MsgGroup:    "msg_group",
+	}
+
+	str := Base64MsgPosition(position)
+	decodeByte, err := base64.StdEncoding.DecodeString(str)
+	assert.NoError(t, err)
+	decodePos := &msgpb.MsgPosition{}
+	err = proto.Unmarshal(decodeByte, decodePos)
+	assert.NoError(t, err)
 
+	assert.Equal(t, position.ChannelName, decodePos.ChannelName)
+	assert.Equal(t, position.MsgID, decodePos.MsgID)
+	assert.Equal(t, position.Timestamp, decodePos.Timestamp)
+	assert.Equal(t, position.MsgGroup, decodePos.MsgGroup)
 }