follow-up

Signed-off-by: Alex Aizman <[email protected]>

ais/backend/aws.go

@@ -19,7 +19,6 @@ import (
 
 	aiss3 "github.com/NVIDIA/aistore/ais/s3"
 	"github.com/NVIDIA/aistore/api/apc"
-	"github.com/NVIDIA/aistore/api/env"
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
 	"github.com/NVIDIA/aistore/cmn/debug"
@@ -641,7 +640,7 @@ func getBucketLocation(svc *s3.Client, bckName string) (region string, err error
 	}
 	region = string(resp.LocationConstraint)
 	if region == "" {
-		region = env.AwsDefaultRegion // Buckets in region `us-east-1` have a LocationConstraint of null.
+		region = cmn.AwsDefaultRegion // Buckets in region `us-east-1` have a LocationConstraint of null.
 	}
 	return
 }

ais/test/scripted_cli_test.go

@@ -179,11 +179,13 @@ func TestMultipartUploadLargeFilesScript(t *testing.T) {
 // remais-blob-download.sh
 func TestRemaisBlobDownloadScript(t *testing.T) {
 	tools.CheckSkip(t, &tools.SkipTestArgs{
-		Long: true,
+		RequiresRemoteCluster: true,
+		Long:                  true,
 	})
 	bck := cmn.Bck{
-		Name:     trand.String(10),
-		Ns:       cmn.Ns{UUID: tools.RemoteCluster.UUID},
+		Name: trand.String(10),
+		Ns:   cmn.Ns{UUID: tools.RemoteCluster.Alias},
+		// Ns:       cmn.Ns{UUID: tools.RemoteCluster.UUID},
 		Provider: apc.AIS,
 	}
 	tools.CreateBucket(t, proxyURL, bck, nil, true /*cleanup*/)

bench/tools/aisloader/client.go

@@ -17,7 +17,6 @@ import (
 
 	"github.com/NVIDIA/aistore/api"
 	"github.com/NVIDIA/aistore/api/apc"
-	"github.com/NVIDIA/aistore/api/env"
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
 	"github.com/NVIDIA/aistore/cmn/debug"
@@ -484,7 +483,7 @@ func initS3Svc() error {
 		cfg.BaseEndpoint = aws.String(s3Endpoint)
 	}
 	if cfg.Region == "" {
-		cfg.Region = env.AwsDefaultRegion // Buckets in region `us-east-1` have a LocationConstraint of null.
+		cfg.Region = cmn.AwsDefaultRegion // Buckets in region `us-east-1` have a LocationConstraint of null.
 	}
 
 	s3svc = s3.NewFromConfig(cfg)

bench/tools/aisloader/run.go

@@ -285,13 +285,12 @@ func Start(version, buildtime string) (err error) {
 	loggedUserToken = authn.LoadToken(runParams.tokenFile)
 	runParams.bp.Token = loggedUserToken
 	runParams.bp.UA = ua
+
 	var created bool
-	if err := setupBucket(runParams, &created); err != nil {
-		return err
-	}
-	if runParams.cached && runParams.bck.IsAIS() {
-		return fmt.Errorf("--cached option (to list \"cached\" objects only) applies to remote buckets (have %s)",
-			runParams.bck.Cname(""))
+	if !runParams.getConfig {
+		if err := setupBucket(runParams, &created); err != nil {
+			return err
+		}
 	}
 
 	if isDirectS3() {
@@ -975,9 +974,6 @@ func (s *sts) aggregate(other *sts) {
 }
 
 func setupBucket(runParams *params, created *bool) error {
-	if runParams.getConfig {
-		return nil
-	}
 	if strings.Contains(runParams.bck.Name, apc.BckProviderSeparator) {
 		bck, objName, err := cmn.ParseBckObjectURI(runParams.bck.Name, cmn.ParseURIOpts{})
 		if err != nil {
@@ -993,12 +989,26 @@ func setupBucket(runParams *params, created *bool) error {
 		}
 		runParams.bck = bck
 	}
-	if isDirectS3() && apc.ToScheme(runParams.bck.Provider) != apc.S3Scheme {
-		return fmt.Errorf("option --s3endpoint requires s3 bucket (have %s)", runParams.bck)
+
+	const cachedText = "--cached option (to list \"cached\" objects only) "
+
+	if isDirectS3() {
+		if apc.ToScheme(runParams.bck.Provider) != apc.S3Scheme {
+			return fmt.Errorf("option --s3endpoint requires s3 bucket (have %s)", runParams.bck)
+		}
+		if runParams.cached {
+			return errors.New(cachedText + "cannot be used together with --s3endpoint (direct S3 access)")
+		}
+	}
+	if runParams.putPct == 100 && runParams.cached {
+		return errors.New(cachedText + "is incompatible with 100% PUT workload")
 	}
 	if runParams.bck.Provider != apc.AIS {
 		return nil
 	}
+	if runParams.cached && !runParams.bck.IsRemote() {
+		return fmt.Errorf(cachedText+"applies to remote buckets (have %s)", runParams.bck.Cname(""))
+	}
 
 	//
 	// ais:// or ais://@remais

cmn/aws.go

@@ -0,0 +1,19 @@
+// Package cmn provides common constants, types, and utilities for AIS clients
+// and AIStore.
+/*
+ * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
+ */
+package cmn
+
+import "strings"
+
+const AwsDefaultRegion = "us-east-1"
+
+// from https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html
+// "The ETag may or may not be an MD5 digest of the object data. Whether or
+// not it is depends on how the object was created and how it is encrypted..."
+const AwsMultipartDelim = "-"
+
+func IsS3MultipartEtag(etag string) bool {
+	return strings.Contains(etag, AwsMultipartDelim)
+}

cmn/backend.go

@@ -23,15 +23,6 @@ func awsIsVersionSet(version *string) bool {
 	return version != nil && *version != "" && *version != "null"
 }
 
-// from https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html
-// "The ETag may or may not be an MD5 digest of the object data. Whether or
-// not it is depends on how the object was created and how it is encrypted..."
-const AwsMultipartDelim = "-"
-
-func IsS3MultipartEtag(etag string) bool {
-	return strings.Contains(etag, AwsMultipartDelim)
-}
-
 // unquote checksum, ETag, and version
 // e.g., https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
 func UnquoteCEV(val string) string { return strings.Trim(val, "\"") }

xact/xs/lrit.go

@@ -6,12 +6,9 @@
 package xs
 
 import (
-	"net/http"
-
 	"github.com/NVIDIA/aistore/api/apc"
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
-	"github.com/NVIDIA/aistore/cmn/debug"
 	"github.com/NVIDIA/aistore/cmn/nlog"
 	"github.com/NVIDIA/aistore/core"
 	"github.com/NVIDIA/aistore/core/meta"
@@ -160,6 +157,7 @@ func (r *lriterator) _range(wi lrwi, smap *meta.Smap) error {
 func (r *lriterator) _prefix(wi lrwi, smap *meta.Smap) error {
 	var (
 		err     error
+		errCode int
 		lst     *cmn.LsoResult
 		msg     = &apc.LsoMsg{Prefix: r.prefix, Props: apc.GetPropsStatus}
 		npg     = newNpgCtx(r.bck, msg, noopCb)
@@ -176,22 +174,16 @@ func (r *lriterator) _prefix(wi lrwi, smap *meta.Smap) error {
 			break
 		}
 		if bremote {
-			var errCode int
 			lst = &cmn.LsoResult{Entries: allocLsoEntries()}
 			errCode, err = core.T.Backend(r.bck).ListObjects(r.bck, msg, lst) // (TODO comment above)
-			if err != nil {
-				freeLsoEntries(lst.Entries)
-				if errCode == http.StatusNotFound && !cos.IsNotExist(err, 0) {
-					debug.Assert(false, err, errCode) // TODO: remove
-					err = cos.NewErrNotFound(nil, err.Error())
-				}
-			}
 		} else {
 			npg.page.Entries = allocLsoEntries()
 			err = npg.nextPageA()
 			lst = &npg.page
 		}
 		if err != nil {
+			nlog.Errorln(core.T.String()+":", err, errCode)
+			freeLsoEntries(lst.Entries)
 			return err
 		}
 		for _, be := range lst.Entries {

xact/xs/lso.go

@@ -527,7 +527,7 @@ func (r *LsoXact) doWalk(msg *apc.LsoMsg) {
 	opts.ValidateCallback = r.validateCb
 	if err := fs.WalkBck(opts); err != nil {
 		if err != filepath.SkipDir && err != errStopped {
-			nlog.Errorf("%s walk failed, err %v", r, err)
+			r.AddErr(err, 0)
 		}
 	}
 	close(r.walk.pageCh)