[config change] assorted feature flags now have bucket scope (major)

* bucket props: add feature flags and 3 (three) specific ones:
  - `Skip-Loading-VersionChecksum-MD`
  - `Fsync-PUT`
  - `Presigned-S3-Req`
* no need to change global cluster for that...
* move `NilValue`
* comment `AIS_*` environment vars
* CLI: handle feature and access (enumerated) flags; refactor

Signed-off-by: Alex Aizman <[email protected]>

ais/s3/presigned.go

@@ -37,6 +37,14 @@ type (
 	}
 )
 
+//////////////////
+// PresignedReq //
+//////////////////
+
+func NewPresignedReq(oreq *http.Request, lom *core.LOM, body io.ReadCloser, q url.Values) *PresignedReq {
+	return &PresignedReq{oreq, lom, body, q}
+}
+
 // FIXME: handle error cases
 func parseSignatureV4(query url.Values, header http.Header) (region string) {
 	if credentials := query.Get(HeaderCredentials); credentials != "" {
@@ -51,10 +59,6 @@ func parseSignatureV4(query url.Values, header http.Header) (region string) {
 	return region
 }
 
-func NewPresignedReq(oreq *http.Request, lom *core.LOM, body io.ReadCloser, q url.Values) *PresignedReq {
-	return &PresignedReq{oreq, lom, body, q}
-}
-
 func (pts *PresignedReq) Do(client *http.Client) (*PresignedResp, error) {
 	region := parseSignatureV4(pts.query, pts.oreq.Header)
 	if region == "" {
@@ -98,16 +102,12 @@ func (pts *PresignedReq) Do(client *http.Client) (*PresignedResp, error) {
 		return &PresignedResp{StatusCode: http.StatusBadRequest},
 			fmt.Errorf("failed to read response body: %v", err)
 	}
-	return &PresignedResp{
-		Body:       output,
-		Header:     resp.Header,
-		StatusCode: resp.StatusCode,
-	}, nil
+	return &PresignedResp{Body: output, Header: resp.Header, StatusCode: resp.StatusCode}, nil
 }
 
-///////////////////////////
+///////////////////
 // PresignedResp //
-///////////////////////////
+///////////////////
 
 // (compare w/ cmn/objattrs FromHeader)
 func (resp *PresignedResp) ObjAttrs() (oa *cmn.ObjAttrs) {

ais/test/regression_test.go

@@ -108,7 +108,7 @@ func TestListObjectsLocalGetLocation(t *testing.T) {
 			if i == 0 {
 				tlog.Logln("Modifying config to enforce intra-cluster access, expecting errors...\n")
 			}
-			tools.SetClusterConfig(t, cos.StrKVs{"features": feat.EnforceIntraClusterAccess.Value()})
+			tools.SetClusterConfig(t, cos.StrKVs{"features": feat.EnforceIntraClusterAccess.String()})
 			t.Cleanup(func() {
 				tools.SetClusterConfig(t, cos.StrKVs{"features": "0"})
 			})
@@ -196,7 +196,7 @@ func TestListObjectsCloudGetLocation(t *testing.T) {
 			if i == 0 {
 				tlog.Logln("Modifying config to enforce intra-cluster access, expecting errors...\n")
 			}
-			tools.SetClusterConfig(t, cos.StrKVs{"features": feat.EnforceIntraClusterAccess.Value()})
+			tools.SetClusterConfig(t, cos.StrKVs{"features": feat.EnforceIntraClusterAccess.String()})
 			_, err = api.GetObject(baseParams, m.bck, e.Name, nil)
 
 			if err == nil {

ais/test/s3_compat_test.go

@@ -75,7 +75,7 @@ func TestS3PassThroughPutGet(t *testing.T) {
 		objName = "object.txt"
 	)
 
-	tools.SetClusterConfig(t, cos.StrKVs{"features": feat.PresignedS3Req.Value()})
+	tools.SetClusterConfig(t, cos.StrKVs{"features": feat.PresignedS3Req.String()})
 	t.Cleanup(func() {
 		tools.SetClusterConfig(t, cos.StrKVs{"features": "0"})
 	})
@@ -116,7 +116,7 @@ func TestS3PassThroughMultipart(t *testing.T) {
 		objName = "object.txt"
 	)
 
-	tools.SetClusterConfig(t, cos.StrKVs{"features": feat.PresignedS3Req.Value()})
+	tools.SetClusterConfig(t, cos.StrKVs{"features": feat.PresignedS3Req.String()})
 	t.Cleanup(func() {
 		tools.SetClusterConfig(t, cos.StrKVs{"features": "0"})
 	})
@@ -192,7 +192,7 @@ func TestWriteThroughCacheNoColdGet(t *testing.T) {
 		objName = "object.txt"
 	)
 
-	tools.SetClusterConfig(t, cos.StrKVs{"features": feat.PresignedS3Req.Value()})
+	tools.SetClusterConfig(t, cos.StrKVs{"features": feat.PresignedS3Req.String()})
 	t.Cleanup(func() {
 		tools.SetClusterConfig(t, cos.StrKVs{"features": "0"})
 	})

ais/tgtobj.go

@@ -604,7 +604,6 @@ do:
 		var (
 			res    core.GetReaderResult
 			ckconf = goi.lom.CksumConf()
-			fast   = !cmn.Rom.Features().IsSet(feat.DisableFastColdGET) // can be disabled
 			loaded bool
 		)
 		if cs.IsNil() {
@@ -637,18 +636,16 @@ do:
 		}
 		goi.cold = true
 
-		// fast path limitations: read archived; compute more checksums (TODO: reduce)
-		fast = fast && goi.archive.filename == "" &&
-			(ckconf.Type == cos.ChecksumNone || (!ckconf.ValidateColdGet && !ckconf.EnableReadRange))
-
-		// fast path
-		if fast {
+		// two alternative ways to perform cold GET: "fast" and "regular"
+		// "fast" limitations: read archived; compute more checksums (TODO)
+		if goi.archive.filename == "" &&
+			(ckconf.Type == cos.ChecksumNone || (!ckconf.ValidateColdGet && !ckconf.EnableReadRange)) {
+			// fast path
 			err = goi.coldSeek(&res)
 			goi.unlocked = true // always
 			return 0, err
 		}
-
-		// regular path
+		// otherwise, regular path
 		errCode, err = goi._coldPut(&res)
 		if err != nil {
 			goi.unlocked = true

api/apc/const.go

@@ -19,6 +19,8 @@ const (
 	DeploymentDev = "dev"
 )
 
+const NilValue = "none" // features (flags), log modules, et al.
+
 // in re: "Slowloris Attack"
 const (
 	ReadHeaderTimeout    = 16 * time.Second

api/env/ais.go

@@ -1,10 +1,17 @@
 // Package env contains environment variables
 /*
- * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
  */
 package env
 
-// See also: docs/environment-vars.md
+// See also:
+// - "AIS_READ_HEADER_TIMEOUT"
+// - "AIS_DAEMON_ID"
+// - "AIS_CLUSTER_CIDR", "AIS_HOST_IP", "AIS_HOST_PORT"
+// - "AIS_TARGET_URL"
+//
+// See also:
+// - docs/environment-vars.md
 
 var (
 	AIS = struct {

cmd/cli/cli/bucket.go

@@ -297,7 +297,7 @@ func reformatBackendProps(c *cli.Context, nvs cos.StrKVs) (err error) {
 		goto validate
 	}
 
-	if v != NilValue {
+	if v != apc.NilValue {
 		if originBck, err = parseBckURI(c, v, true /*error only*/); err != nil {
 			return fmt.Errorf("invalid '%s=%s': expecting %q to be a valid bucket name",
 				cmn.PropBackendBck, v, v)

cmd/cli/cli/completions.go

@@ -37,10 +37,16 @@ const (
 var (
 	supportedBool = []string{"true", "false"}
 	propCmpls     = map[string][]string{
-		confLogModules:                        append(cos.Smodules, NilValue),
-		cmn.PropBucketAccessAttrs:             apc.SupportedPermissions(),
-		apc.HdrObjCksumType:                   cos.SupportedChecksums(),
-		feat.FeaturesPropName:                 append(feat.All, NilValue),
+		// log modules
+		confLogModules: append(cos.Smodules, apc.NilValue),
+		// checksums
+		apc.HdrObjCksumType: cos.SupportedChecksums(),
+		// access
+		cmn.PropBucketAccessAttrs: apc.SupportedPermissions(),
+		// feature flags
+		"cluster.features": append(feat.Cluster, apc.NilValue),
+		"bucket.features":  append(feat.Bucket, apc.NilValue),
+		// rest
 		"write_policy.data":                   apc.SupportedWritePolicy,
 		"write_policy.md":                     apc.SupportedWritePolicy,
 		"ec.compression":                      apc.SupportedCompression,
@@ -69,16 +75,37 @@ var (
 	}
 )
 
-func lastIsSmodule(c *cli.Context) bool { return _lastv(c, propCmpls[confLogModules]) }
-func lastIsAccess(c *cli.Context) bool  { return _lastv(c, propCmpls[cmn.PropBucketAccessAttrs]) }
-func lastIsFeature(c *cli.Context) bool { return _lastv(c, propCmpls[feat.FeaturesPropName]) }
+func lastIsSmodule(c *cli.Context) bool {
+	if argLast(c) == confLogModules {
+		return true
+	}
+	return _lastv(c, propCmpls[confLogModules])
+}
+
+func lastIsAccess(c *cli.Context) bool {
+	if argLast(c) == cmn.PropBucketAccessAttrs {
+		return true
+	}
+	return _lastv(c, propCmpls[cmn.PropBucketAccessAttrs])
+}
+
+func lastIsFeature(c *cli.Context, bucketScope bool) bool {
+	if argLast(c) == feat.PropName {
+		return true
+	}
+	if bucketScope {
+		return _lastv(c, propCmpls["bucket.features"])
+	}
+	return _lastv(c, propCmpls["cluster.features"])
+}
 
 // Returns true if the last arg is any of the enumerated constants
 func _lastv(c *cli.Context, values []string) bool {
 	if c.NArg() == 0 {
 		return false
 	}
 	lastArg := argLast(c)
+
 	for _, v := range values {
 		if v == lastArg {
 			return true
@@ -93,7 +120,14 @@ func _lastv(c *cli.Context, values []string) bool {
 // - features
 func smoduleCompletions(c *cli.Context) { remaining(c, propCmpls[confLogModules]) }
 func accessCompletions(c *cli.Context)  { remaining(c, propCmpls[cmn.PropBucketAccessAttrs]) }
-func featureCompletions(c *cli.Context) { remaining(c, propCmpls[feat.FeaturesPropName]) }
+
+func featureCompletions(c *cli.Context, bucketScope bool) {
+	if bucketScope {
+		remaining(c, propCmpls["bucket.features"])
+	} else {
+		remaining(c, propCmpls["cluster.features"])
+	}
+}
 
 func remaining(c *cli.Context, values []string) {
 	typedList := c.Args()
@@ -108,15 +142,15 @@ outer:
 	}
 }
 
-func propValueCompletion(c *cli.Context) bool {
+func propValueCompletion(c *cli.Context, bucketScope bool) bool {
 	switch {
 	case c.NArg() == 0:
 		return false
 	case lastIsAccess(c):
 		accessCompletions(c)
 		return true
-	case lastIsFeature(c):
-		featureCompletions(c)
+	case lastIsFeature(c, bucketScope):
+		featureCompletions(c, bucketScope)
 		return true
 	case lastIsSmodule(c):
 		smoduleCompletions(c)
@@ -299,7 +333,7 @@ func setCluConfigCompletions(c *cli.Context) {
 	}, cmn.IterOpts{Allowed: apc.Cluster})
 	debug.AssertNoErr(err)
 
-	if propValueCompletion(c) {
+	if propValueCompletion(c, false /*bucket scope*/) {
 		return
 	}
 	for _, prop := range propList {
@@ -310,7 +344,7 @@ func setCluConfigCompletions(c *cli.Context) {
 }
 
 func suggestUpdatableConfig(c *cli.Context) {
-	if propValueCompletion(c) {
+	if propValueCompletion(c, false /*bucket scope*/) {
 		return
 	}
 	scope := apc.Cluster
@@ -349,7 +383,7 @@ func (opts *bcmplop) buckets(c *cli.Context) {
 	)
 	additionalCompletions = opts.additionalCompletions
 	if c.NArg() > opts.firstBucketIdx && !opts.multiple {
-		if propValueCompletion(c) {
+		if propValueCompletion(c, true /*bucket scope*/) {
 			return
 		}
 		for _, f := range additionalCompletions {

cmd/cli/cli/config_hdlr.go

@@ -184,12 +184,12 @@ func setCluConfigHandler(c *cli.Context) error {
 		goto show
 	}
 	for k, v := range nvs {
-		if k == feat.FeaturesPropName {
-			featfl, err := parseFeatureFlags(v)
+		if k == feat.PropName {
+			featfl, _, err := parseFeatureFlags([]string{v}, 0)
 			if err != nil {
 				return fmt.Errorf("invalid feature flag %q", v)
 			}
-			nvs[k] = featfl.Value()
+			nvs[k] = featfl.String() // FormatUint
 		}
 		if k == confLogModules { // (ref 836)
 			if nvs[confLogLevel], err = parseLogModules(v); err != nil {
@@ -232,7 +232,7 @@ func parseLogModules(v string) (string, error) {
 		return "", V(err)
 	}
 	level, _ := config.Log.Level.Parse()
-	if v == "" || v == NilValue {
+	if v == "" || v == apc.NilValue {
 		config.Log.Level.Set(level, []string{""})
 	} else {
 		config.Log.Level.Set(level, splitCsv(v))

cmd/cli/cli/const.go

@@ -191,10 +191,8 @@ const (
 	tgtTotal = "------- Sum:"
 )
 
-const NilValue = "none"
-
 const (
-	defaultChunkSize = 10 * cos.MiB
+	dfltStdinChunkSize = 10 * cos.MiB
 )
 
 const (

cmd/cli/cli/object_hdlr.go

@@ -386,10 +386,10 @@ func putStdin(c *cli.Context, a *putargs) error {
 	}
 	if flagIsSet(c, chunkSizeFlag) && chunkSize == 0 {
 		return fmt.Errorf("chunk size (in %s) cannot be zero (%s recommended)",
-			qflprn(chunkSizeFlag), teb.FmtSize(defaultChunkSize, cos.UnitsIEC, 0))
+			qflprn(chunkSizeFlag), teb.FmtSize(dfltStdinChunkSize, cos.UnitsIEC, 0))
 	}
 	if chunkSize == 0 {
-		chunkSize = defaultChunkSize
+		chunkSize = dfltStdinChunkSize
 	}
 	if flagIsSet(c, verboseFlag) {
 		actionWarn(c, "To terminate input, press Ctrl-D two or more times")