From ee4544beeb35ed34a4d1989443ac35f5120181f7 Mon Sep 17 00:00:00 2001 From: "hongbo.mo" Date: Fri, 25 May 2018 11:20:27 +0800 Subject: [PATCH] add RangeList --- README.md | 23 +++++++++++++++ upyun/fileinfo.go | 44 +++++++++++++++++++++++++++++ upyun/rest.go | 69 +++++++++++++++++++++++++++++++++++++++++++++ upyun/rest_test.go | 30 +++++++++++++++++++- upyun/upyun_test.go | 1 + upyun/utils.go | 21 -------------- 6 files changed, 166 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 9b7704c..8ba52d2 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Table of Contents * [删除](#删除) * [获取文件信息](#获取文件信息) * [获取文件列表](#获取文件列表) + * [获取获取指定时间段内增量文件列表](#获取指定时间段内增量文件列表) * [又拍云缓存刷新接口](#又拍云缓存刷新接口) * [又拍云表单上传接口](#又拍云表单上传接口) * [又拍云处理接口](#又拍云处理接口) @@ -40,6 +41,7 @@ Table of Contents * [PutObjectConfig](#putobjectconfig) * [GetObjectConfig](#getobjectconfig) * [GetObjectsConfig](#getobjectsconfig) + * [RangeObjectsConfig](#rangeobjectsconfig) * [DeleteObjectConfig](#deleteobjectconfig) * [FormUploadConfig](#formuploadconfig) * [CommitTasksConfig](#committasksconfig) @@ -152,6 +154,11 @@ func (up *UpYun) GetInfo(path string) (*FileInfo, error) func (up *UpYun) List(config *GetObjectsConfig) error ``` +#### 获取指定时间段内增量文件列表 +```go +func (up *UpYun) RangeList(config *RangeObjectsConfig) error +``` + --- ### 又拍云缓存刷新接口 @@ -309,6 +316,22 @@ type GetObjectsConfig struct { `GetObjectsConfig` 提供列目录所需的参数。当列目录结束后，SDK 会将 `ObjectsChan` 关闭掉。 +#### RangeObjectsConfig + +```go +type RangeObjectsConfig struct { + StartTimestamp int64 // 开始时间戳 + EndTimestamp int64 // 结束时间戳 + Headers map[string]string // 额外的 HTTP 请求头 + ObjectsChan chan *FileInfo // 对象通道 + QuitChan chan bool // 停止信号 + MaxListTries int // 列目录最大重试次数 +} +``` + +`RangeObjectsConfig` 提供获取指定时间段内增量文件所需的参数。当列目录结束后，SDK 会将 `ObjectsChan` 关闭掉。 + + #### DeleteObjectConfig ```go diff --git a/upyun/fileinfo.go b/upyun/fileinfo.go index f7a7eae..6ad4ba8 100644 --- a/upyun/fileinfo.go +++ b/upyun/fileinfo.go @@ -62,3 +62,47 @@ func parseHeaderToFileInfo(header http.Header, getinfo bool) *FileInfo { } return fInfo } + +func parseBodyToFileInfos(b []byte) (fInfos []*FileInfo) { + line := strings.Split(string(b), "\n") + for _, l := range line { + if len(l) == 0 { + continue + } + items := strings.Split(l, "\t") + if len(items) != 4 { + continue + } + + fInfos = append(fInfos, &FileInfo{ + Name: items[0], + IsDir: items[1] == "F", + Size: int64(parseStrToInt(items[2])), + Time: time.Unix(parseStrToInt(items[3]), 0), + }) + } + return +} + +func parseRangeListToFileInfos(b []byte) (fInfos []*FileInfo) { + line := strings.Split(string(b), "\n") + for _, l := range line { + if len(l) == 0 { + continue + } + items := strings.Split(l, "\t") + if len(items) != 5 { + continue + } + + fInfos = append(fInfos, &FileInfo{ + Name: items[0], + IsDir: false, + ContentType: items[1], + Size: int64(parseStrToInt(items[2])), + Time: time.Unix(parseStrToInt(items[3]), 0), + MD5: items[4], + }) + } + return +} diff --git a/upyun/rest.go b/upyun/rest.go index 7d039fd..4cd7ea9 100644 --- a/upyun/rest.go +++ b/upyun/rest.go @@ -54,6 +54,16 @@ type GetObjectsConfig struct { try int } +// RangeObjectsConfig provides a configuration to RangeList method. +type RangeObjectsConfig struct { + StartTimestamp int64 + EndTimestamp int64 + ObjectsChan chan *FileInfo + QuitChan chan bool + Headers map[string]string + MaxListTries int +} + // PutObjectConfig provides a configuration to Put method. type PutObjectConfig struct { Path string @@ -415,6 +425,65 @@ func (up *UpYun) List(config *GetObjectsConfig) error { } } +func (up *UpYun) RangeList(config *RangeObjectsConfig) error { + if config.ObjectsChan == nil { + return fmt.Errorf("ObjectsChan == nil") + } + if config.Headers == nil { + config.Headers = make(map[string]string) + } + if config.QuitChan == nil { + config.QuitChan = make(chan bool) + } + if _, exist := config.Headers["X-List-Limit"]; !exist { + config.Headers["X-List-Limit"] = "50" + } + if config.StartTimestamp != 0 { + config.Headers["X-List-Start"] = fmt.Sprint(config.StartTimestamp) + } + if config.EndTimestamp != 0 { + config.Headers["X-List-End"] = fmt.Sprint(config.EndTimestamp) + } + defer close(config.ObjectsChan) + + try := 0 + for { + resp, err := up.doRESTRequest(&restReqConfig{ + method: "GET", + uri: "/?files", + headers: config.Headers, + }) + if err != nil { + if _, ok := err.(net.Error); ok { + try++ + if config.MaxListTries == 0 || try < config.MaxListTries { + continue + } + } + return err + } + + b, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return fmt.Errorf("ioutil ReadAll: %v", err) + } + + for _, fInfo := range parseRangeListToFileInfos(b) { + select { + case config.ObjectsChan <- fInfo: + case <-config.QuitChan: + return nil + } + } + + config.Headers["X-List-Iter"] = resp.Header.Get("X-Upyun-List-Iter") + if config.Headers["X-List-Iter"] == "g2gCZAAEbmV4dGQAA2VvZg" { + return nil + } + } +} + func (up *UpYun) ModifyMetadata(config *ModifyMetadataConfig) error { if config.Operation == "" { config.Operation = "merge" diff --git a/upyun/rest_test.go b/upyun/rest_test.go index 4979620..99dcb55 100644 --- a/upyun/rest_test.go +++ b/upyun/rest_test.go @@ -10,6 +10,7 @@ import ( "sort" "strings" "testing" + "time" ) var ( @@ -204,7 +205,6 @@ func TestIsNotExist(t *testing.T) { } func TestModifyMetadata(t *testing.T) { - // time.Sleep(10 * time.Second) err := up.ModifyMetadata(&ModifyMetadataConfig{ Path: REST_FILE_1, Operation: "replace", @@ -216,6 +216,34 @@ func TestModifyMetadata(t *testing.T) { Nil(t, err) } +func TestRangeList(t *testing.T) { + ch := make(chan *FileInfo, 10) + files := []string{} + + time.Sleep(5 * time.Second) + + go func() { + err := up.RangeList(&RangeObjectsConfig{ + StartTimestamp: START_TIME, + ObjectsChan: ch, + }) + Nil(t, err) + }() + + for fInfo := range ch { + if strings.HasPrefix("/"+fInfo.Name, REST_DIR) { + files = append(files, "/"+fInfo.Name) + } + } + + Equal(t, len(files), len(REST_OBJS)) + sort.Strings(files) + sort.Strings(REST_OBJS) + for k := range REST_OBJS { + Equal(t, path.Join(REST_DIR, REST_OBJS[k]), files[k]) + } +} + func TestDelete(t *testing.T) { err := up.Delete(&DeleteObjectConfig{ Path: REST_DIR, diff --git a/upyun/upyun_test.go b/upyun/upyun_test.go index 1cd4e70..1a11468 100644 --- a/upyun/upyun_test.go +++ b/upyun/upyun_test.go @@ -15,6 +15,7 @@ import ( var ( ROOT = MakeTmpPath() + START_TIME = time.Now().Unix() NOTIFY_URL = os.Getenv("UPYUN_NOTIFY") ) diff --git a/upyun/utils.go b/upyun/utils.go index 9042f60..40ee143 100644 --- a/upyun/utils.go +++ b/upyun/utils.go @@ -159,24 +159,3 @@ func md5File(f io.ReadSeeker) (string, error) { } return fmt.Sprintf("%x", hash.Sum(nil)), nil } - -func parseBodyToFileInfos(b []byte) (fInfos []*FileInfo) { - line := strings.Split(string(b), "\n") - for _, l := range line { - if len(l) == 0 { - continue - } - items := strings.Split(l, "\t") - if len(items) != 4 { - continue - } - - fInfos = append(fInfos, &FileInfo{ - Name: items[0], - IsDir: items[1] == "F", - Size: int64(parseStrToInt(items[2])), - Time: time.Unix(parseStrToInt(items[3]), 0), - }) - } - return -}