Skip to content

Commit

Permalink
Extend v3io-go API to align with full data model (#18)
Browse files Browse the repository at this point in the history
  • Loading branch information
@imakhlin @pavius
imakhlin authored and pavius committed Jun 4, 2019
1 parent 088181c commit ba51e53
Show file tree
Hide file tree
Showing 3 changed files with 258 additions and 33 deletions.
29 changes: 24 additions & 5 deletions pkg/dataplane/http/context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func NewContext(parentLogger logger.Logger, newContextInput *v3io.NewContextInpu
}

if httpEndpointFound && httpsEndpointFound {
return nil, errors.New("Cannot create a context with a mix of HTTP and HTTPS endpoints.")
return nil, errors.New("cannot create a context with a mix of HTTP and HTTPS endpoints")
}

requestChanLen := newContextInput.RequestChanLen
Expand All @@ -81,7 +81,7 @@ func NewContext(parentLogger logger.Logger, newContextInput *v3io.NewContextInpu
numWorkers = 8
}

tlsConfig := newContextInput.TlsConfig
tlsConfig := newContextInput.TLSConfig
if tlsConfig == nil {
tlsConfig = &tls.Config{InsecureSkipVerify: true}
}
Expand Down Expand Up @@ -166,15 +166,34 @@ func (c *context) GetContainerContents(getContainerContentsInput *v3io.GetContai
func (c *context) GetContainerContentsSync(getContainerContentsInput *v3io.GetContainerContentsInput) (*v3io.Response, error) {
getContainerContentOutput := v3io.GetContainerContentsOutput{}

query := ""
var queryBuilder strings.Builder
if getContainerContentsInput.Path != "" {
query += "prefix=" + getContainerContentsInput.Path
queryBuilder.WriteString("prefix=")
queryBuilder.WriteString(getContainerContentsInput.Path)
}

if getContainerContentsInput.DirectoriesOnly {
queryBuilder.WriteString("&prefix-only=1")
}

if getContainerContentsInput.GetAllAttributes {
queryBuilder.WriteString("&prefix-info=1")
}

if getContainerContentsInput.Marker != "" {
queryBuilder.WriteString("&marker=")
queryBuilder.WriteString(getContainerContentsInput.Marker)
}

if getContainerContentsInput.Limit > 0 {
queryBuilder.WriteString("&max-keys=")
queryBuilder.WriteString(strconv.Itoa(getContainerContentsInput.Limit))
}

return c.sendRequestAndXMLUnmarshal(&getContainerContentsInput.DataPlaneInput,
http.MethodGet,
"",
query,
queryBuilder.String(),
nil,
nil,
&getContainerContentOutput)
Expand Down
214 changes: 202 additions & 12 deletions pkg/dataplane/test/sync_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package test
import (
"fmt"
"testing"
"time"

"github.com/v3io/v3io-go/pkg/dataplane"
"github.com/v3io/v3io-go/pkg/errors"
Expand Down Expand Up @@ -34,27 +35,163 @@ func (suite *syncContainerTestSuite) TestGetContainers() {
// get containers
response, err := suite.container.GetContainersSync(&getContainersInput)
suite.Require().NoError(err, "Failed to get containers")
response.Release()
}

func (suite *syncContainerTestSuite) TestGetContainerContentsDefault() {
path := fmt.Sprintf("tmp/test/sync_test/TestGetContainerContentsDefault/%d/", time.Now().Unix())
fileContent := "If you cannot do great things, do small things in a great way."

// Create some content (directory and files)
putObjectInput := &v3io.PutObjectInput{}
for i := 0; i < 10; i++ {
if i < 5 {
// Create file with content
putObjectInput.Path = fmt.Sprintf("%sfile-%d.txt", path, i)
putObjectInput.Body = []byte(fileContent)
} else {
// create empty directory
putObjectInput.Path = fmt.Sprintf("%sdir-%d/", path, i)
putObjectInput.Body = nil
}

// when run against a context
suite.populateDataPlaneInput(&putObjectInput.DataPlaneInput)
err := suite.container.PutObjectSync(putObjectInput)
suite.Require().NoError(err, "Failed to create test content")
}

getContainerContentsInput := v3io.GetContainerContentsInput{
Path: path,
}

getContainersOutput := response.Output.(*v3io.GetContainersOutput)
fmt.Println(getContainersOutput)
// when run against a context
suite.populateDataPlaneInput(&getContainerContentsInput.DataPlaneInput)

// get container contents
response, err := suite.container.GetContainerContentsSync(&getContainerContentsInput)
suite.Require().NoError(err, "Failed to get container contents")
response.Release()

getContainerContentsOutput := response.Output.(*v3io.GetContainerContentsOutput)
suite.Require().Equal(5, len(getContainerContentsOutput.Contents))

for _, content := range getContainerContentsOutput.Contents {
validateContent(suite, &content, len(fileContent), false)
}

for _, prefix := range getContainerContentsOutput.CommonPrefixes {
validateCommonPrefix(suite, &prefix, false)
}

suite.Require().Equal(5, len(getContainerContentsOutput.CommonPrefixes))
}

func (suite *syncContainerTestSuite) TestGetContainerContents() {
getContainerContentsInput := v3io.GetContainerContentsInput{}
func (suite *syncContainerTestSuite) TestGetContainerContentsFilesWithAllAttrs() {
path := fmt.Sprintf("tmp/test/sync_test/TestGetContainerContentsFilesWithAllAttrs/%d/", time.Now().Unix())
fileContent := "If you cannot do great things, do small things in a great way."

// Create some content (directory and files)
putObjectInput := &v3io.PutObjectInput{}
for i := 0; i < 10; i++ {
// Create file with content
putObjectInput.Path = path + fmt.Sprintf("file-%d.txt", i)
putObjectInput.Body = []byte(fileContent)

// when run against a context
suite.populateDataPlaneInput(&putObjectInput.DataPlaneInput)
err := suite.container.PutObjectSync(putObjectInput)
suite.Require().NoError(err, "Failed to create test content")
}

getContainerContentsInput := v3io.GetContainerContentsInput{
Path: path,
GetAllAttributes: true,
DirectoriesOnly: false,
Limit: 5,
}

// when run against a context
suite.populateDataPlaneInput(&getContainerContentsInput.DataPlaneInput)

// get container contents
response, err := suite.container.GetContainerContentsSync(&getContainerContentsInput)
suite.Require().NoError(err, "Failed to get container contents")
response.Release()

getContainerContentsOutput := response.Output.(*v3io.GetContainerContentsOutput)
fmt.Println(getContainerContentsOutput)
suite.Require().Equal(5, len(getContainerContentsOutput.Contents))
suite.Require().Equal(path+"file-4.txt", getContainerContentsOutput.NextMarker)
suite.Require().Equal(true, getContainerContentsOutput.IsTruncated)

for _, content := range getContainerContentsOutput.Contents {
validateContent(suite, &content, len(fileContent), true)
}
// get remaining content
getContainerContentsInput.Marker = getContainerContentsOutput.NextMarker
// get container contents
response, err = suite.container.GetContainerContentsSync(&getContainerContentsInput)
suite.Require().NoError(err, "Failed to get container contents")
response.Release()

getContainerContentsOutput = response.Output.(*v3io.GetContainerContentsOutput)
suite.Require().Equal(5, len(getContainerContentsOutput.Contents))
suite.Require().Equal(path+"file-9.txt", getContainerContentsOutput.NextMarker)
suite.Require().Equal(false, getContainerContentsOutput.IsTruncated)

for _, content := range getContainerContentsOutput.Contents {
validateContent(suite, &content, len(fileContent), true)
}
}

func (suite *syncContainerTestSuite) TestGetContainerContentsDirsWithAllAttrs() {
path := fmt.Sprintf("tmp/test/sync_test/TestGetContainerContentsDirsWithAllAttrs/%d/", time.Now().Unix())
content := "If you cannot do great things, do small things in a great way."

// Create some content (directory and files)
putObjectInput := &v3io.PutObjectInput{}
for i := 0; i < 10; i++ {
// create 2 files and 8 directories at the target path
if i < 2 {
// Create file with content
putObjectInput.Path = fmt.Sprintf("%sfile-%d.txt", path, i)
putObjectInput.Body = []byte(content)
} else {
// create empty directory
putObjectInput.Path = fmt.Sprintf("%sdir-%d/", path, i)
putObjectInput.Body = nil
}

// when run against a context
suite.populateDataPlaneInput(&putObjectInput.DataPlaneInput)
err := suite.container.PutObjectSync(putObjectInput)
suite.Require().NoError(err, "Failed to create test content")
}

getContainerContentsInput := v3io.GetContainerContentsInput{
Path: path,
GetAllAttributes: true,
DirectoriesOnly: true,
Limit: 10,
}

// when run against a context
suite.populateDataPlaneInput(&getContainerContentsInput.DataPlaneInput)

// get container contents
response, err := suite.container.GetContainerContentsSync(&getContainerContentsInput)
suite.Require().NoError(err, "Failed to get container contents")
response.Release()

getContainerContentsOutput := response.Output.(*v3io.GetContainerContentsOutput)
suite.Require().Empty(len(getContainerContentsOutput.Contents))
suite.Require().Equal(8, len(getContainerContentsOutput.CommonPrefixes))
suite.Require().Equal(path+"dir-9", getContainerContentsOutput.NextMarker)
suite.Require().Equal(false, getContainerContentsOutput.IsTruncated)

for _, prefix := range getContainerContentsOutput.CommonPrefixes {
validateCommonPrefix(suite, &prefix, true)
}
}

type syncContextContainerTestSuite struct {
Expand Down Expand Up @@ -131,7 +268,6 @@ func (suite *syncObjectTestSuite) TestObject() {
suite.populateDataPlaneInput(&getObjectInput.DataPlaneInput)

response, err = suite.container.GetObjectSync(getObjectInput)

suite.Require().NoError(err, "Failed to get")

// make sure buckets is not empty
Expand Down Expand Up @@ -438,7 +574,7 @@ func (suite *syncKVTestSuite) verifyItems(items map[string]map[string]interface{
func (suite *syncKVTestSuite) deleteItems(items map[string]map[string]interface{}) {

// delete the items
for itemKey, _ := range items {
for itemKey := range items {
input := v3io.DeleteObjectInput{
Path: "/emd0/" + itemKey,
}
Expand Down Expand Up @@ -495,11 +631,13 @@ type syncStreamTestSuite struct {

func (suite *syncStreamTestSuite) SetupTest() {
suite.testPath = "/stream-test"
suite.deleteAllStreamsInPath(suite.testPath)
err := suite.deleteAllStreamsInPath(suite.testPath)
suite.Require().NoError(err, "Failed to setup test suite")
}

func (suite *syncStreamTestSuite) TearDownTest() {
suite.deleteAllStreamsInPath(suite.testPath)
err := suite.deleteAllStreamsInPath(suite.testPath)
suite.Require().NoError(err, "Failed to tea down test suite")
}

func (suite *syncStreamTestSuite) TestStream() {
Expand Down Expand Up @@ -636,8 +774,7 @@ func (suite *syncStreamTestSuite) deleteAllStreamsInPath(path string) error {
if err != nil {
return err
}

defer response.Release()
response.Release()

// iterate over streams (prefixes) and delete them
for _, commonPrefix := range response.Output.(*v3io.GetContainerContentsOutput).CommonPrefixes {
Expand All @@ -647,7 +784,10 @@ func (suite *syncStreamTestSuite) deleteAllStreamsInPath(path string) error {

suite.populateDataPlaneInput(&deleteStreamInput.DataPlaneInput)

suite.container.DeleteStreamSync(&deleteStreamInput)
err := suite.container.DeleteStreamSync(&deleteStreamInput)
if err != nil {
return err
}
}

return nil
Expand Down Expand Up @@ -684,3 +824,53 @@ func TestSyncSuite(t *testing.T) {
suite.Run(t, new(syncContextStreamTestSuite))
suite.Run(t, new(syncContainerStreamTestSuite))
}

func validateContent(suite *syncContainerTestSuite, content *v3io.Content, expectedSize int, withPrefixInfo bool) {
// common
suite.Require().NotEmpty(content.Key)
suite.Require().NotEmpty(content.LastModified)
suite.Require().NotNil(content.Size)
suite.Require().Equal(expectedSize, *content.Size)

if withPrefixInfo {
suite.Require().NotEmpty(content.AccessTime)
suite.Require().NotEmpty(content.CreatingTime)
suite.Require().NotEmpty(content.GID)
suite.Require().NotEmpty(content.UID)
suite.Require().NotEmpty(content.Mode)
suite.Require().NotEmpty(content.InodeNumber)
suite.Require().Nil(content.LastSequenceID)
} else {
suite.Require().Empty(content.AccessTime)
suite.Require().Empty(content.CreatingTime)
suite.Require().Empty(content.GID)
suite.Require().Empty(content.UID)
suite.Require().Nil(content.Mode)
suite.Require().Nil(content.InodeNumber)
suite.Require().Nil(content.LastSequenceID)
}
}

func validateCommonPrefix(suite *syncContainerTestSuite, prefix *v3io.CommonPrefix, withPrefixInfo bool) {
// common
suite.Require().NotEmpty(prefix.Prefix)

if withPrefixInfo {
suite.Require().NotEmpty(prefix.LastModified)
suite.Require().NotEmpty(prefix.AccessTime)
suite.Require().NotEmpty(prefix.CreatingTime)
suite.Require().NotEmpty(prefix.GID)
suite.Require().NotEmpty(prefix.UID)
suite.Require().NotEmpty(prefix.Mode)
suite.Require().NotEmpty(prefix.InodeNumber)
suite.Require().Equal(true, *prefix.InodeNumber > 0)
} else {
suite.Require().Empty(prefix.LastModified)
suite.Require().Empty(prefix.AccessTime)
suite.Require().Empty(prefix.CreatingTime)
suite.Require().Empty(prefix.GID)
suite.Require().Empty(prefix.UID)
suite.Require().Nil(prefix.Mode)
suite.Require().Nil(prefix.InodeNumber)
}
}
Loading

0 comments on commit ba51e53

Please sign in to comment.