add RangeList

README.md

@@ -26,6 +26,7 @@ Table of Contents
             * [删除](#删除)
             * [获取文件信息](#获取文件信息)
             * [获取文件列表](#获取文件列表)
+            * [获取获取指定时间段内增量文件列表](#获取指定时间段内增量文件列表)
          * [又拍云缓存刷新接口](#又拍云缓存刷新接口)
          * [又拍云表单上传接口](#又拍云表单上传接口)
          * [又拍云处理接口](#又拍云处理接口)
@@ -40,6 +41,7 @@ Table of Contents
             * [PutObjectConfig](#putobjectconfig)
             * [GetObjectConfig](#getobjectconfig)
             * [GetObjectsConfig](#getobjectsconfig)
+            * [RangeObjectsConfig](#rangeobjectsconfig)
             * [DeleteObjectConfig](#deleteobjectconfig)
             * [FormUploadConfig](#formuploadconfig)
             * [CommitTasksConfig](#committasksconfig)
@@ -152,6 +154,11 @@ func (up *UpYun) GetInfo(path string) (*FileInfo, error)
 func (up *UpYun) List(config *GetObjectsConfig) error
 ```
 
+#### 获取指定时间段内增量文件列表
+```go
+func (up *UpYun) RangeList(config *RangeObjectsConfig) error
+```
+
 ---
 
 ### 又拍云缓存刷新接口
@@ -309,6 +316,22 @@ type GetObjectsConfig struct {
 `GetObjectsConfig` 提供列目录所需的参数。当列目录结束后，SDK 会将 `ObjectsChan` 关闭掉。
 
 
+#### RangeObjectsConfig
+
+```go
+type RangeObjectsConfig struct {
+        StartTimestamp int64                    // 开始时间戳
+        EndTimestamp   int64                    // 结束时间戳
+        Headers        map[string]string        // 额外的 HTTP 请求头
+        ObjectsChan    chan *FileInfo           // 对象通道
+        QuitChan       chan bool                // 停止信号
+        MaxListTries   int                      // 列目录最大重试次数
+}
+```
+
+`RangeObjectsConfig` 提供获取指定时间段内增量文件所需的参数。当列目录结束后，SDK 会将 `ObjectsChan` 关闭掉。
+
+
 #### DeleteObjectConfig
 
 ```go

upyun/fileinfo.go

@@ -62,3 +62,47 @@ func parseHeaderToFileInfo(header http.Header, getinfo bool) *FileInfo {
 	}
 	return fInfo
 }
+
+func parseBodyToFileInfos(b []byte) (fInfos []*FileInfo) {
+	line := strings.Split(string(b), "\n")
+	for _, l := range line {
+		if len(l) == 0 {
+			continue
+		}
+		items := strings.Split(l, "\t")
+		if len(items) != 4 {
+			continue
+		}
+
+		fInfos = append(fInfos, &FileInfo{
+			Name:  items[0],
+			IsDir: items[1] == "F",
+			Size:  int64(parseStrToInt(items[2])),
+			Time:  time.Unix(parseStrToInt(items[3]), 0),
+		})
+	}
+	return
+}
+
+func parseRangeListToFileInfos(b []byte) (fInfos []*FileInfo) {
+	line := strings.Split(string(b), "\n")
+	for _, l := range line {
+		if len(l) == 0 {
+			continue
+		}
+		items := strings.Split(l, "\t")
+		if len(items) != 5 {
+			continue
+		}
+
+		fInfos = append(fInfos, &FileInfo{
+			Name:        items[0],
+			IsDir:       false,
+			ContentType: items[1],
+			Size:        int64(parseStrToInt(items[2])),
+			Time:        time.Unix(parseStrToInt(items[3]), 0),
+			MD5:         items[4],
+		})
+	}
+	return
+}

upyun/rest.go

@@ -54,6 +54,16 @@ type GetObjectsConfig struct {
 	try     int
 }
 
+// RangeObjectsConfig provides a configuration to RangeList method.
+type RangeObjectsConfig struct {
+	StartTimestamp int64
+	EndTimestamp   int64
+	ObjectsChan    chan *FileInfo
+	QuitChan       chan bool
+	Headers        map[string]string
+	MaxListTries   int
+}
+
 // PutObjectConfig provides a configuration to Put method.
 type PutObjectConfig struct {
 	Path            string
@@ -415,6 +425,65 @@ func (up *UpYun) List(config *GetObjectsConfig) error {
 	}
 }
 
+func (up *UpYun) RangeList(config *RangeObjectsConfig) error {
+	if config.ObjectsChan == nil {
+		return fmt.Errorf("ObjectsChan == nil")
+	}
+	if config.Headers == nil {
+		config.Headers = make(map[string]string)
+	}
+	if config.QuitChan == nil {
+		config.QuitChan = make(chan bool)
+	}
+	if _, exist := config.Headers["X-List-Limit"]; !exist {
+		config.Headers["X-List-Limit"] = "50"
+	}
+	if config.StartTimestamp != 0 {
+		config.Headers["X-List-Start"] = fmt.Sprint(config.StartTimestamp)
+	}
+	if config.EndTimestamp != 0 {
+		config.Headers["X-List-End"] = fmt.Sprint(config.EndTimestamp)
+	}
+	defer close(config.ObjectsChan)
+
+	try := 0
+	for {
+		resp, err := up.doRESTRequest(&restReqConfig{
+			method:  "GET",
+			uri:     "/?files",
+			headers: config.Headers,
+		})
+		if err != nil {
+			if _, ok := err.(net.Error); ok {
+				try++
+				if config.MaxListTries == 0 || try < config.MaxListTries {
+					continue
+				}
+			}
+			return err
+		}
+
+		b, err := ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+		if err != nil {
+			return fmt.Errorf("ioutil ReadAll: %v", err)
+		}
+
+		for _, fInfo := range parseRangeListToFileInfos(b) {
+			select {
+			case config.ObjectsChan <- fInfo:
+			case <-config.QuitChan:
+				return nil
+			}
+		}
+
+		config.Headers["X-List-Iter"] = resp.Header.Get("X-Upyun-List-Iter")
+		if config.Headers["X-List-Iter"] == "g2gCZAAEbmV4dGQAA2VvZg" {
+			return nil
+		}
+	}
+}
+
 func (up *UpYun) ModifyMetadata(config *ModifyMetadataConfig) error {
 	if config.Operation == "" {
 		config.Operation = "merge"

upyun/rest_test.go

@@ -10,6 +10,7 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 )
 
 var (
@@ -204,7 +205,6 @@ func TestIsNotExist(t *testing.T) {
 }
 
 func TestModifyMetadata(t *testing.T) {
-	//	time.Sleep(10 * time.Second)
 	err := up.ModifyMetadata(&ModifyMetadataConfig{
 		Path:      REST_FILE_1,
 		Operation: "replace",
@@ -216,6 +216,34 @@ func TestModifyMetadata(t *testing.T) {
 	Nil(t, err)
 }
 
+func TestRangeList(t *testing.T) {
+	ch := make(chan *FileInfo, 10)
+	files := []string{}
+
+	time.Sleep(5 * time.Second)
+
+	go func() {
+		err := up.RangeList(&RangeObjectsConfig{
+			StartTimestamp: START_TIME,
+			ObjectsChan:    ch,
+		})
+		Nil(t, err)
+	}()
+
+	for fInfo := range ch {
+		if strings.HasPrefix("/"+fInfo.Name, REST_DIR) {
+			files = append(files, "/"+fInfo.Name)
+		}
+	}
+
+	Equal(t, len(files), len(REST_OBJS))
+	sort.Strings(files)
+	sort.Strings(REST_OBJS)
+	for k := range REST_OBJS {
+		Equal(t, path.Join(REST_DIR, REST_OBJS[k]), files[k])
+	}
+}
+
 func TestDelete(t *testing.T) {
 	err := up.Delete(&DeleteObjectConfig{
 		Path: REST_DIR,

upyun/upyun_test.go

@@ -15,6 +15,7 @@ import (
 
 var (
 	ROOT       = MakeTmpPath()
+	START_TIME = time.Now().Unix()
 	NOTIFY_URL = os.Getenv("UPYUN_NOTIFY")
 )
 

upyun/utils.go

@@ -159,24 +159,3 @@ func md5File(f io.ReadSeeker) (string, error) {
 	}
 	return fmt.Sprintf("%x", hash.Sum(nil)), nil
 }
-
-func parseBodyToFileInfos(b []byte) (fInfos []*FileInfo) {
-	line := strings.Split(string(b), "\n")
-	for _, l := range line {
-		if len(l) == 0 {
-			continue
-		}
-		items := strings.Split(l, "\t")
-		if len(items) != 4 {
-			continue
-		}
-
-		fInfos = append(fInfos, &FileInfo{
-			Name:  items[0],
-			IsDir: items[1] == "F",
-			Size:  int64(parseStrToInt(items[2])),
-			Time:  time.Unix(parseStrToInt(items[3]), 0),
-		})
-	}
-	return
-}