From ba51e53e110113fc666dce1dc1e36f7bf40a1cf2 Mon Sep 17 00:00:00 2001 From: Igor Makhlin Date: Tue, 4 Jun 2019 14:19:57 +0300 Subject: [PATCH] Extend v3io-go API to align with full data model (#18) --- pkg/dataplane/http/context.go | 29 ++++- pkg/dataplane/test/sync_test.go | 214 ++++++++++++++++++++++++++++++-- pkg/dataplane/types.go | 48 ++++--- 3 files changed, 258 insertions(+), 33 deletions(-) diff --git a/pkg/dataplane/http/context.go b/pkg/dataplane/http/context.go index 64db394..e847fa8 100644 --- a/pkg/dataplane/http/context.go +++ b/pkg/dataplane/http/context.go @@ -68,7 +68,7 @@ func NewContext(parentLogger logger.Logger, newContextInput *v3io.NewContextInpu } if httpEndpointFound && httpsEndpointFound { - return nil, errors.New("Cannot create a context with a mix of HTTP and HTTPS endpoints.") + return nil, errors.New("cannot create a context with a mix of HTTP and HTTPS endpoints") } requestChanLen := newContextInput.RequestChanLen @@ -81,7 +81,7 @@ func NewContext(parentLogger logger.Logger, newContextInput *v3io.NewContextInpu numWorkers = 8 } - tlsConfig := newContextInput.TlsConfig + tlsConfig := newContextInput.TLSConfig if tlsConfig == nil { tlsConfig = &tls.Config{InsecureSkipVerify: true} } @@ -166,15 +166,34 @@ func (c *context) GetContainerContents(getContainerContentsInput *v3io.GetContai func (c *context) GetContainerContentsSync(getContainerContentsInput *v3io.GetContainerContentsInput) (*v3io.Response, error) { getContainerContentOutput := v3io.GetContainerContentsOutput{} - query := "" + var queryBuilder strings.Builder if getContainerContentsInput.Path != "" { - query += "prefix=" + getContainerContentsInput.Path + queryBuilder.WriteString("prefix=") + queryBuilder.WriteString(getContainerContentsInput.Path) + } + + if getContainerContentsInput.DirectoriesOnly { + queryBuilder.WriteString("&prefix-only=1") + } + + if getContainerContentsInput.GetAllAttributes { + queryBuilder.WriteString("&prefix-info=1") + } + + if getContainerContentsInput.Marker != "" { + queryBuilder.WriteString("&marker=") + queryBuilder.WriteString(getContainerContentsInput.Marker) + } + + if getContainerContentsInput.Limit > 0 { + queryBuilder.WriteString("&max-keys=") + queryBuilder.WriteString(strconv.Itoa(getContainerContentsInput.Limit)) } return c.sendRequestAndXMLUnmarshal(&getContainerContentsInput.DataPlaneInput, http.MethodGet, "", - query, + queryBuilder.String(), nil, nil, &getContainerContentOutput) diff --git a/pkg/dataplane/test/sync_test.go b/pkg/dataplane/test/sync_test.go index 99f409b..e0a3a5c 100644 --- a/pkg/dataplane/test/sync_test.go +++ b/pkg/dataplane/test/sync_test.go @@ -3,6 +3,7 @@ package test import ( "fmt" "testing" + "time" "github.com/v3io/v3io-go/pkg/dataplane" "github.com/v3io/v3io-go/pkg/errors" @@ -34,15 +35,81 @@ func (suite *syncContainerTestSuite) TestGetContainers() { // get containers response, err := suite.container.GetContainersSync(&getContainersInput) suite.Require().NoError(err, "Failed to get containers") + response.Release() +} + +func (suite *syncContainerTestSuite) TestGetContainerContentsDefault() { + path := fmt.Sprintf("tmp/test/sync_test/TestGetContainerContentsDefault/%d/", time.Now().Unix()) + fileContent := "If you cannot do great things, do small things in a great way." + + // Create some content (directory and files) + putObjectInput := &v3io.PutObjectInput{} + for i := 0; i < 10; i++ { + if i < 5 { + // Create file with content + putObjectInput.Path = fmt.Sprintf("%sfile-%d.txt", path, i) + putObjectInput.Body = []byte(fileContent) + } else { + // create empty directory + putObjectInput.Path = fmt.Sprintf("%sdir-%d/", path, i) + putObjectInput.Body = nil + } + + // when run against a context + suite.populateDataPlaneInput(&putObjectInput.DataPlaneInput) + err := suite.container.PutObjectSync(putObjectInput) + suite.Require().NoError(err, "Failed to create test content") + } + + getContainerContentsInput := v3io.GetContainerContentsInput{ + Path: path, + } - getContainersOutput := response.Output.(*v3io.GetContainersOutput) - fmt.Println(getContainersOutput) + // when run against a context + suite.populateDataPlaneInput(&getContainerContentsInput.DataPlaneInput) + // get container contents + response, err := suite.container.GetContainerContentsSync(&getContainerContentsInput) + suite.Require().NoError(err, "Failed to get container contents") response.Release() + + getContainerContentsOutput := response.Output.(*v3io.GetContainerContentsOutput) + suite.Require().Equal(5, len(getContainerContentsOutput.Contents)) + + for _, content := range getContainerContentsOutput.Contents { + validateContent(suite, &content, len(fileContent), false) + } + + for _, prefix := range getContainerContentsOutput.CommonPrefixes { + validateCommonPrefix(suite, &prefix, false) + } + + suite.Require().Equal(5, len(getContainerContentsOutput.CommonPrefixes)) } -func (suite *syncContainerTestSuite) TestGetContainerContents() { - getContainerContentsInput := v3io.GetContainerContentsInput{} +func (suite *syncContainerTestSuite) TestGetContainerContentsFilesWithAllAttrs() { + path := fmt.Sprintf("tmp/test/sync_test/TestGetContainerContentsFilesWithAllAttrs/%d/", time.Now().Unix()) + fileContent := "If you cannot do great things, do small things in a great way." + + // Create some content (directory and files) + putObjectInput := &v3io.PutObjectInput{} + for i := 0; i < 10; i++ { + // Create file with content + putObjectInput.Path = path + fmt.Sprintf("file-%d.txt", i) + putObjectInput.Body = []byte(fileContent) + + // when run against a context + suite.populateDataPlaneInput(&putObjectInput.DataPlaneInput) + err := suite.container.PutObjectSync(putObjectInput) + suite.Require().NoError(err, "Failed to create test content") + } + + getContainerContentsInput := v3io.GetContainerContentsInput{ + Path: path, + GetAllAttributes: true, + DirectoriesOnly: false, + Limit: 5, + } // when run against a context suite.populateDataPlaneInput(&getContainerContentsInput.DataPlaneInput) @@ -50,11 +117,81 @@ func (suite *syncContainerTestSuite) TestGetContainerContents() { // get container contents response, err := suite.container.GetContainerContentsSync(&getContainerContentsInput) suite.Require().NoError(err, "Failed to get container contents") + response.Release() getContainerContentsOutput := response.Output.(*v3io.GetContainerContentsOutput) - fmt.Println(getContainerContentsOutput) + suite.Require().Equal(5, len(getContainerContentsOutput.Contents)) + suite.Require().Equal(path+"file-4.txt", getContainerContentsOutput.NextMarker) + suite.Require().Equal(true, getContainerContentsOutput.IsTruncated) + for _, content := range getContainerContentsOutput.Contents { + validateContent(suite, &content, len(fileContent), true) + } + // get remaining content + getContainerContentsInput.Marker = getContainerContentsOutput.NextMarker + // get container contents + response, err = suite.container.GetContainerContentsSync(&getContainerContentsInput) + suite.Require().NoError(err, "Failed to get container contents") response.Release() + + getContainerContentsOutput = response.Output.(*v3io.GetContainerContentsOutput) + suite.Require().Equal(5, len(getContainerContentsOutput.Contents)) + suite.Require().Equal(path+"file-9.txt", getContainerContentsOutput.NextMarker) + suite.Require().Equal(false, getContainerContentsOutput.IsTruncated) + + for _, content := range getContainerContentsOutput.Contents { + validateContent(suite, &content, len(fileContent), true) + } +} + +func (suite *syncContainerTestSuite) TestGetContainerContentsDirsWithAllAttrs() { + path := fmt.Sprintf("tmp/test/sync_test/TestGetContainerContentsDirsWithAllAttrs/%d/", time.Now().Unix()) + content := "If you cannot do great things, do small things in a great way." + + // Create some content (directory and files) + putObjectInput := &v3io.PutObjectInput{} + for i := 0; i < 10; i++ { + // create 2 files and 8 directories at the target path + if i < 2 { + // Create file with content + putObjectInput.Path = fmt.Sprintf("%sfile-%d.txt", path, i) + putObjectInput.Body = []byte(content) + } else { + // create empty directory + putObjectInput.Path = fmt.Sprintf("%sdir-%d/", path, i) + putObjectInput.Body = nil + } + + // when run against a context + suite.populateDataPlaneInput(&putObjectInput.DataPlaneInput) + err := suite.container.PutObjectSync(putObjectInput) + suite.Require().NoError(err, "Failed to create test content") + } + + getContainerContentsInput := v3io.GetContainerContentsInput{ + Path: path, + GetAllAttributes: true, + DirectoriesOnly: true, + Limit: 10, + } + + // when run against a context + suite.populateDataPlaneInput(&getContainerContentsInput.DataPlaneInput) + + // get container contents + response, err := suite.container.GetContainerContentsSync(&getContainerContentsInput) + suite.Require().NoError(err, "Failed to get container contents") + response.Release() + + getContainerContentsOutput := response.Output.(*v3io.GetContainerContentsOutput) + suite.Require().Empty(len(getContainerContentsOutput.Contents)) + suite.Require().Equal(8, len(getContainerContentsOutput.CommonPrefixes)) + suite.Require().Equal(path+"dir-9", getContainerContentsOutput.NextMarker) + suite.Require().Equal(false, getContainerContentsOutput.IsTruncated) + + for _, prefix := range getContainerContentsOutput.CommonPrefixes { + validateCommonPrefix(suite, &prefix, true) + } } type syncContextContainerTestSuite struct { @@ -131,7 +268,6 @@ func (suite *syncObjectTestSuite) TestObject() { suite.populateDataPlaneInput(&getObjectInput.DataPlaneInput) response, err = suite.container.GetObjectSync(getObjectInput) - suite.Require().NoError(err, "Failed to get") // make sure buckets is not empty @@ -438,7 +574,7 @@ func (suite *syncKVTestSuite) verifyItems(items map[string]map[string]interface{ func (suite *syncKVTestSuite) deleteItems(items map[string]map[string]interface{}) { // delete the items - for itemKey, _ := range items { + for itemKey := range items { input := v3io.DeleteObjectInput{ Path: "/emd0/" + itemKey, } @@ -495,11 +631,13 @@ type syncStreamTestSuite struct { func (suite *syncStreamTestSuite) SetupTest() { suite.testPath = "/stream-test" - suite.deleteAllStreamsInPath(suite.testPath) + err := suite.deleteAllStreamsInPath(suite.testPath) + suite.Require().NoError(err, "Failed to setup test suite") } func (suite *syncStreamTestSuite) TearDownTest() { - suite.deleteAllStreamsInPath(suite.testPath) + err := suite.deleteAllStreamsInPath(suite.testPath) + suite.Require().NoError(err, "Failed to tea down test suite") } func (suite *syncStreamTestSuite) TestStream() { @@ -636,8 +774,7 @@ func (suite *syncStreamTestSuite) deleteAllStreamsInPath(path string) error { if err != nil { return err } - - defer response.Release() + response.Release() // iterate over streams (prefixes) and delete them for _, commonPrefix := range response.Output.(*v3io.GetContainerContentsOutput).CommonPrefixes { @@ -647,7 +784,10 @@ func (suite *syncStreamTestSuite) deleteAllStreamsInPath(path string) error { suite.populateDataPlaneInput(&deleteStreamInput.DataPlaneInput) - suite.container.DeleteStreamSync(&deleteStreamInput) + err := suite.container.DeleteStreamSync(&deleteStreamInput) + if err != nil { + return err + } } return nil @@ -684,3 +824,53 @@ func TestSyncSuite(t *testing.T) { suite.Run(t, new(syncContextStreamTestSuite)) suite.Run(t, new(syncContainerStreamTestSuite)) } + +func validateContent(suite *syncContainerTestSuite, content *v3io.Content, expectedSize int, withPrefixInfo bool) { + // common + suite.Require().NotEmpty(content.Key) + suite.Require().NotEmpty(content.LastModified) + suite.Require().NotNil(content.Size) + suite.Require().Equal(expectedSize, *content.Size) + + if withPrefixInfo { + suite.Require().NotEmpty(content.AccessTime) + suite.Require().NotEmpty(content.CreatingTime) + suite.Require().NotEmpty(content.GID) + suite.Require().NotEmpty(content.UID) + suite.Require().NotEmpty(content.Mode) + suite.Require().NotEmpty(content.InodeNumber) + suite.Require().Nil(content.LastSequenceID) + } else { + suite.Require().Empty(content.AccessTime) + suite.Require().Empty(content.CreatingTime) + suite.Require().Empty(content.GID) + suite.Require().Empty(content.UID) + suite.Require().Nil(content.Mode) + suite.Require().Nil(content.InodeNumber) + suite.Require().Nil(content.LastSequenceID) + } +} + +func validateCommonPrefix(suite *syncContainerTestSuite, prefix *v3io.CommonPrefix, withPrefixInfo bool) { + // common + suite.Require().NotEmpty(prefix.Prefix) + + if withPrefixInfo { + suite.Require().NotEmpty(prefix.LastModified) + suite.Require().NotEmpty(prefix.AccessTime) + suite.Require().NotEmpty(prefix.CreatingTime) + suite.Require().NotEmpty(prefix.GID) + suite.Require().NotEmpty(prefix.UID) + suite.Require().NotEmpty(prefix.Mode) + suite.Require().NotEmpty(prefix.InodeNumber) + suite.Require().Equal(true, *prefix.InodeNumber > 0) + } else { + suite.Require().Empty(prefix.LastModified) + suite.Require().Empty(prefix.AccessTime) + suite.Require().Empty(prefix.CreatingTime) + suite.Require().Empty(prefix.GID) + suite.Require().Empty(prefix.UID) + suite.Require().Nil(prefix.Mode) + suite.Require().Nil(prefix.InodeNumber) + } +} diff --git a/pkg/dataplane/types.go b/pkg/dataplane/types.go index d772407..8e58121 100644 --- a/pkg/dataplane/types.go +++ b/pkg/dataplane/types.go @@ -20,6 +20,7 @@ import ( "context" "crypto/tls" "encoding/xml" + "os" "time" ) @@ -31,7 +32,7 @@ type NewContextInput struct { ClusterEndpoints []string NumWorkers int RequestChanLen int - TlsConfig *tls.Config + TLSConfig *tls.Config DialTimeout time.Duration } @@ -67,30 +68,45 @@ type DataPlaneOutput struct { type GetContainerContentsInput struct { DataPlaneInput - Path string + Path string + GetAllAttributes bool // if "true" return ALL available attributes + DirectoriesOnly bool // if "true" return directory entries only, otherwise return children of any kind + Limit int // max number of entries per request + Marker string // start from specific entry (e.g. to get next chunk) } type Content struct { - XMLName xml.Name `xml:"Contents"` - Key string `xml:"Key"` - Size int `xml:"Size"` - LastSequenceID int `xml:"LastSequenceId"` - ETag string `xml:"ETag"` - LastModified string `xml:"LastModified"` + Key string `xml:"Key"` + Size *int `xml:"Size"` // file size in bytes + LastSequenceID *int `xml:"LastSequenceId"` // greater than zero for shard files + LastModified string `xml:"LastModified"` // Date in format time.RFC3339: "2019-06-02T14:30:39.18Z" + + Mode *os.FileMode `xml:"Mode"` // uint32, e.g. 0100664 + AccessTime string `xml:"AccessTime"` // Date in format time.RFC3339: "2019-06-02T14:30:39.18Z" + CreatingTime string `xml:"CreatingTime"` // Date in format time.RFC3339: "2019-06-02T14:30:39.18Z" + GID string `xml:"GID"` // Hexadecimal representation of GID (e.g. "3e8" -> i.e. "0x3e8" == 1000) + UID string `xml:"UID"` // Hexadecimal representation of UID (e.g. "3e8" -> i.e. "0x3e8" == 1000) + InodeNumber *uint32 `xml:"InodeNumber"` // iNode number } type CommonPrefix struct { - CommonPrefixes xml.Name `xml:"CommonPrefixes"` - Prefix string `xml:"Prefix"` + Prefix string `xml:"Prefix"` // directory name + LastModified string `xml:"LastModified"` // Date in format time.RFC3339: "2019-06-02T14:30:39.18Z" + AccessTime string `xml:"AccessTime"` // Date in format time.RFC3339: "2019-06-02T14:30:39.18Z" + CreatingTime string `xml:"CreatingTime"` // Date in format time.RFC3339: "2019-06-02T14:30:39.18Z" + Mode *os.FileMode `xml:"Mode"` // uint32, e.g. 040775 + GID string `xml:"GID"` // Hexadecimal representation of GID (e.g. "3e8" -> i.e. "0x3e8" == 1000) + UID string `xml:"UID"` // Hexadecimal representation of UID (e.g. "3e8" -> i.e. "0x3e8" == 1000) + InodeNumber *uint32 `xml:"InodeNumber"` // iNode number } type GetContainerContentsOutput struct { - BucketName xml.Name `xml:"ListBucketResult"` - Name string `xml:"Name"` - NextMarker string `xml:"NextMarker"` - MaxKeys string `xml:"MaxKeys"` - Contents []Content `xml:"Contents"` - CommonPrefixes []CommonPrefix `xml:"CommonPrefixes"` + Name string `xml:"Name"` // Bucket name + NextMarker string `xml:"NextMarker"` // if not empty and isTruncated="true" - has more children (need another fetch to get them) + MaxKeys string `xml:"MaxKeys"` // max number of entries in single batch + Contents []Content `xml:"Contents"` // files + CommonPrefixes []CommonPrefix `xml:"CommonPrefixes"` // directories + IsTruncated bool `xml:"IsTruncated"` // "true" if has more content. Note, "NextMarker" should not be empty if "true" } type GetContainersInput struct {