merge from databricks

.gitignore

@@ -3,3 +3,4 @@ goofys
 goofys.test
 xout
 s3proxy.jar
+.idea

api/api.go

@@ -120,7 +120,17 @@ func Mount(
 		err = fmt.Errorf("Mount: initialization failed")
 		return
 	}
-	server := fuseutil.NewFileSystemServer(FusePanicLogger{fs})
+	var fusefs fuseutil.FileSystem
+
+	if flags.BgInit {
+		fusefs = &LazyInitFileSystem{
+			Fs: fs,
+		}
+	} else {
+		fusefs = fs
+	}
+
+	server := fuseutil.NewFileSystemServer(FusePanicLogger{fusefs})
 
 	mfs, err = fuse.Mount(flags.MountPoint, server, mountCfg)
 	if err != nil {

api/common/conf_azure.go

@@ -47,6 +47,9 @@ type AZBlobConfig struct {
 
 	Container string
 	Prefix    string
+
+	RetryDuration time.Duration
+	MaxRetries    int32
 }
 
 func (config *AZBlobConfig) Init() {
@@ -83,16 +86,18 @@ func (config *AZBlobConfig) WithAuthorization() autorest.PrepareDecorator {
 }
 
 type ADLv1Config struct {
-	Endpoint   string
-	Authorizer autorest.Authorizer
+	Endpoint      string
+	Authorizer    autorest.Authorizer
+	RetryDuration time.Duration
 }
 
 func (config *ADLv1Config) Init() {
 }
 
 type ADLv2Config struct {
-	Endpoint   string
-	Authorizer autorest.Authorizer
+	Endpoint      string
+	Authorizer    autorest.Authorizer
+	RetryDuration time.Duration
 }
 
 type AzureAuthorizerConfig struct {
@@ -260,6 +265,31 @@ func azureAccountsClient(account string) (azblob.AccountsClient, error) {
 	}
 
 	c.BaseClient.Authorizer = authorizer
+	c.BaseClient.RequestInspector = func(p autorest.Preparer) autorest.Preparer {
+		return autorest.PreparerFunc(func(r *http.Request) (*http.Request, error) {
+			azbLog.Debugf("%v %v", r.Method, r.URL.String())
+			r, err := p.Prepare(r)
+			if err != nil {
+				azbLog.Error(err)
+			}
+			return r, err
+		})
+	}
+	c.BaseClient.ResponseInspector = func(p autorest.Responder) autorest.Responder {
+		return autorest.ResponderFunc(func(r *http.Response) error {
+			// seems like this is called twice per
+			// response and causes one extra debug log,
+			// but that's better than not logging it
+			azbLog.Debugf("%v %v %v",
+				r.Request.Method, r.Request.URL.String(), r.Status)
+			err := p.Respond(r)
+			if err != nil {
+				azbLog.Error(err)
+			}
+			return err
+		})
+	}
+
 	return c, nil
 }
 

api/common/conf_s3.go

@@ -19,6 +19,7 @@ import (
 	"encoding/base64"
 	"fmt"
 	"net/http"
+	"os"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/client"
@@ -55,6 +56,7 @@ type S3Config struct {
 	Session     *session.Session
 
 	BucketOwner string
+	MaxRetries  int
 }
 
 var s3Session *session.Session
@@ -77,6 +79,10 @@ func (c *S3Config) ToAwsConfig(flags *FlagStorage) (*aws.Config, error) {
 		Transport: &defaultHTTPTransport,
 		Timeout:   flags.HTTPTimeout,
 	})
+	if c.MaxRetries != 0 {
+		awsConfig.MaxRetries = &c.MaxRetries
+	}
+
 	if flags.DebugS3 {
 		awsConfig.LogLevel = aws.LogLevel(aws.LogDebug | aws.LogDebugWithRequestErrors)
 	}
@@ -145,6 +151,7 @@ func (c stsConfigProvider) ClientConfig(serviceName string, cfgs ...*aws.Config)
 	}
 	if c.StsEndpoint != "" {
 		config.Endpoint = c.StsEndpoint
+		config.SigningRegion = os.Getenv("AWS_REGION")
 	}
 
 	return config

api/common/config.go

@@ -32,6 +32,11 @@ type FlagStorage struct {
 	MountPointArg     string
 	MountPointCreated string
 
+	// analog of `-o bg` option in nfs, initialize the mount point
+	// right away but let the work of actually contacting remote
+	// happen in the background
+	BgInit bool
+
 	Cache    []string
 	DirMode  os.FileMode
 	FileMode os.FileMode