From d8bd6dfd1f848ac6c8a1db450a616f3bc570d8e3 Mon Sep 17 00:00:00 2001 From: Andrew Kim Date: Mon, 4 Mar 2019 12:35:31 -0500 Subject: [PATCH] move generic feature gate code from k8s.io/apiserver to k8s.io/component-base Kubernetes-commit: b2831a686c31b5db72f3106b3688f00c40ba1b00 --- featuregate/feature_gate.go | 333 ++++++++++++++++++++++++ featuregate/feature_gate_test.go | 386 ++++++++++++++++++++++++++++ featuregate/testing/feature_gate.go | 44 ++++ 3 files changed, 763 insertions(+) create mode 100644 featuregate/feature_gate.go create mode 100644 featuregate/feature_gate_test.go create mode 100644 featuregate/testing/feature_gate.go diff --git a/featuregate/feature_gate.go b/featuregate/feature_gate.go new file mode 100644 index 00000000..0243fb37 --- /dev/null +++ b/featuregate/feature_gate.go @@ -0,0 +1,333 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package featuregate + +import ( + "fmt" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + + "github.com/spf13/pflag" + "k8s.io/klog" +) + +type Feature string + +const ( + flagName = "feature-gates" + + // allAlphaGate is a global toggle for alpha features. Per-feature key + // values override the default set by allAlphaGate. Examples: + // AllAlpha=false,NewFeature=true will result in newFeature=true + // AllAlpha=true,NewFeature=false will result in newFeature=false + allAlphaGate Feature = "AllAlpha" +) + +var ( + // The generic features. + defaultFeatures = map[Feature]FeatureSpec{ + allAlphaGate: {Default: false, PreRelease: Alpha}, + } + + // Special handling for a few gates. + specialFeatures = map[Feature]func(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool){ + allAlphaGate: setUnsetAlphaGates, + } +) + +type FeatureSpec struct { + // Default is the default enablement state for the feature + Default bool + // LockToDefault indicates that the feature is locked to its default and cannot be changed + LockToDefault bool + // PreRelease indicates the maturity level of the feature + PreRelease prerelease +} + +type prerelease string + +const ( + // Values for PreRelease. + Alpha = prerelease("ALPHA") + Beta = prerelease("BETA") + GA = prerelease("") + + // Deprecated + Deprecated = prerelease("DEPRECATED") +) + +// FeatureGate indicates whether a given feature is enabled or not +type FeatureGate interface { + // Enabled returns true if the key is enabled. + Enabled(key Feature) bool + // KnownFeatures returns a slice of strings describing the FeatureGate's known features. + KnownFeatures() []string + // DeepCopy returns a deep copy of the FeatureGate object, such that gates can be + // set on the copy without mutating the original. This is useful for validating + // config against potential feature gate changes before committing those changes. + DeepCopy() MutableFeatureGate +} + +// MutableFeatureGate parses and stores flag gates for known features from +// a string like feature1=true,feature2=false,... +type MutableFeatureGate interface { + FeatureGate + + // AddFlag adds a flag for setting global feature gates to the specified FlagSet. + AddFlag(fs *pflag.FlagSet) + // Set parses and stores flag gates for known features + // from a string like feature1=true,feature2=false,... + Set(value string) error + // SetFromMap stores flag gates for known features from a map[string]bool or returns an error + SetFromMap(m map[string]bool) error + // Add adds features to the featureGate. + Add(features map[Feature]FeatureSpec) error +} + +// featureGate implements FeatureGate as well as pflag.Value for flag parsing. +type featureGate struct { + special map[Feature]func(map[Feature]FeatureSpec, map[Feature]bool, bool) + + // lock guards writes to known, enabled, and reads/writes of closed + lock sync.Mutex + // known holds a map[Feature]FeatureSpec + known *atomic.Value + // enabled holds a map[Feature]bool + enabled *atomic.Value + // closed is set to true when AddFlag is called, and prevents subsequent calls to Add + closed bool +} + +func setUnsetAlphaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool) { + for k, v := range known { + if v.PreRelease == Alpha { + if _, found := enabled[k]; !found { + enabled[k] = val + } + } + } +} + +// Set, String, and Type implement pflag.Value +var _ pflag.Value = &featureGate{} + +func NewFeatureGate() *featureGate { + known := map[Feature]FeatureSpec{} + for k, v := range defaultFeatures { + known[k] = v + } + + knownValue := &atomic.Value{} + knownValue.Store(known) + + enabled := map[Feature]bool{} + enabledValue := &atomic.Value{} + enabledValue.Store(enabled) + + f := &featureGate{ + known: knownValue, + special: specialFeatures, + enabled: enabledValue, + } + return f +} + +// Set parses a string of the form "key1=value1,key2=value2,..." into a +// map[string]bool of known keys or returns an error. +func (f *featureGate) Set(value string) error { + m := make(map[string]bool) + for _, s := range strings.Split(value, ",") { + if len(s) == 0 { + continue + } + arr := strings.SplitN(s, "=", 2) + k := strings.TrimSpace(arr[0]) + if len(arr) != 2 { + return fmt.Errorf("missing bool value for %s", k) + } + v := strings.TrimSpace(arr[1]) + boolValue, err := strconv.ParseBool(v) + if err != nil { + return fmt.Errorf("invalid value of %s=%s, err: %v", k, v, err) + } + m[k] = boolValue + } + return f.SetFromMap(m) +} + +// SetFromMap stores flag gates for known features from a map[string]bool or returns an error +func (f *featureGate) SetFromMap(m map[string]bool) error { + f.lock.Lock() + defer f.lock.Unlock() + + // Copy existing state + known := map[Feature]FeatureSpec{} + for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + known[k] = v + } + enabled := map[Feature]bool{} + for k, v := range f.enabled.Load().(map[Feature]bool) { + enabled[k] = v + } + + for k, v := range m { + k := Feature(k) + featureSpec, ok := known[k] + if !ok { + return fmt.Errorf("unrecognized feature gate: %s", k) + } + if featureSpec.LockToDefault && featureSpec.Default != v { + return fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, featureSpec.Default) + } + enabled[k] = v + // Handle "special" features like "all alpha gates" + if fn, found := f.special[k]; found { + fn(known, enabled, v) + } + + if featureSpec.PreRelease == Deprecated { + klog.Warningf("Setting deprecated feature gate %s=%t. It will be removed in a future release.", k, v) + } else if featureSpec.PreRelease == GA { + klog.Warningf("Setting GA feature gate %s=%t. It will be removed in a future release.", k, v) + } + } + + // Persist changes + f.known.Store(known) + f.enabled.Store(enabled) + + klog.V(1).Infof("feature gates: %v", f.enabled) + return nil +} + +// String returns a string containing all enabled feature gates, formatted as "key1=value1,key2=value2,...". +func (f *featureGate) String() string { + pairs := []string{} + for k, v := range f.enabled.Load().(map[Feature]bool) { + pairs = append(pairs, fmt.Sprintf("%s=%t", k, v)) + } + sort.Strings(pairs) + return strings.Join(pairs, ",") +} + +func (f *featureGate) Type() string { + return "mapStringBool" +} + +// Add adds features to the featureGate. +func (f *featureGate) Add(features map[Feature]FeatureSpec) error { + f.lock.Lock() + defer f.lock.Unlock() + + if f.closed { + return fmt.Errorf("cannot add a feature gate after adding it to the flag set") + } + + // Copy existing state + known := map[Feature]FeatureSpec{} + for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + known[k] = v + } + + for name, spec := range features { + if existingSpec, found := known[name]; found { + if existingSpec == spec { + continue + } + return fmt.Errorf("feature gate %q with different spec already exists: %v", name, existingSpec) + } + + known[name] = spec + } + + // Persist updated state + f.known.Store(known) + + return nil +} + +// Enabled returns true if the key is enabled. +func (f *featureGate) Enabled(key Feature) bool { + if v, ok := f.enabled.Load().(map[Feature]bool)[key]; ok { + return v + } + return f.known.Load().(map[Feature]FeatureSpec)[key].Default +} + +// AddFlag adds a flag for setting global feature gates to the specified FlagSet. +func (f *featureGate) AddFlag(fs *pflag.FlagSet) { + f.lock.Lock() + // TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead? + // Not all components expose a feature gates flag using this AddFlag method, and + // in the future, all components will completely stop exposing a feature gates flag, + // in favor of componentconfig. + f.closed = true + f.lock.Unlock() + + known := f.KnownFeatures() + fs.Var(f, flagName, ""+ + "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ + "Options are:\n"+strings.Join(known, "\n")) +} + +// KnownFeatures returns a slice of strings describing the FeatureGate's known features. +// Deprecated and GA features are hidden from the list. +func (f *featureGate) KnownFeatures() []string { + var known []string + for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + if v.PreRelease == GA || v.PreRelease == Deprecated { + continue + } + known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, v.PreRelease, v.Default)) + } + sort.Strings(known) + return known +} + +// DeepCopy returns a deep copy of the FeatureGate object, such that gates can be +// set on the copy without mutating the original. This is useful for validating +// config against potential feature gate changes before committing those changes. +func (f *featureGate) DeepCopy() MutableFeatureGate { + // Copy existing state. + known := map[Feature]FeatureSpec{} + for k, v := range f.known.Load().(map[Feature]FeatureSpec) { + known[k] = v + } + enabled := map[Feature]bool{} + for k, v := range f.enabled.Load().(map[Feature]bool) { + enabled[k] = v + } + + // Store copied state in new atomics. + knownValue := &atomic.Value{} + knownValue.Store(known) + enabledValue := &atomic.Value{} + enabledValue.Store(enabled) + + // Construct a new featureGate around the copied state. + // Note that specialFeatures is treated as immutable by convention, + // and we maintain the value of f.closed across the copy. + return &featureGate{ + special: specialFeatures, + known: knownValue, + enabled: enabledValue, + closed: f.closed, + } +} diff --git a/featuregate/feature_gate_test.go b/featuregate/feature_gate_test.go new file mode 100644 index 00000000..18c4e354 --- /dev/null +++ b/featuregate/feature_gate_test.go @@ -0,0 +1,386 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package featuregate + +import ( + "fmt" + "strings" + "testing" + + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" +) + +func TestFeatureGateFlag(t *testing.T) { + // gates for testing + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + + tests := []struct { + arg string + expect map[Feature]bool + parseError string + }{ + { + arg: "", + expect: map[Feature]bool{ + allAlphaGate: false, + testAlphaGate: false, + testBetaGate: false, + }, + }, + { + arg: "fooBarBaz=true", + expect: map[Feature]bool{ + allAlphaGate: false, + testAlphaGate: false, + testBetaGate: false, + }, + parseError: "unrecognized feature gate: fooBarBaz", + }, + { + arg: "AllAlpha=false", + expect: map[Feature]bool{ + allAlphaGate: false, + testAlphaGate: false, + testBetaGate: false, + }, + }, + { + arg: "AllAlpha=true", + expect: map[Feature]bool{ + allAlphaGate: true, + testAlphaGate: true, + testBetaGate: false, + }, + }, + { + arg: "AllAlpha=banana", + expect: map[Feature]bool{ + allAlphaGate: false, + testAlphaGate: false, + testBetaGate: false, + }, + parseError: "invalid value of AllAlpha", + }, + { + arg: "AllAlpha=false,TestAlpha=true", + expect: map[Feature]bool{ + allAlphaGate: false, + testAlphaGate: true, + testBetaGate: false, + }, + }, + { + arg: "TestAlpha=true,AllAlpha=false", + expect: map[Feature]bool{ + allAlphaGate: false, + testAlphaGate: true, + testBetaGate: false, + }, + }, + { + arg: "AllAlpha=true,TestAlpha=false", + expect: map[Feature]bool{ + allAlphaGate: true, + testAlphaGate: false, + testBetaGate: false, + }, + }, + { + arg: "TestAlpha=false,AllAlpha=true", + expect: map[Feature]bool{ + allAlphaGate: true, + testAlphaGate: false, + testBetaGate: false, + }, + }, + { + arg: "TestBeta=true,AllAlpha=false", + expect: map[Feature]bool{ + allAlphaGate: false, + testAlphaGate: false, + testBetaGate: true, + }, + }, + } + for i, test := range tests { + fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError) + f := NewFeatureGate() + f.Add(map[Feature]FeatureSpec{ + testAlphaGate: {Default: false, PreRelease: Alpha}, + testBetaGate: {Default: false, PreRelease: Beta}, + }) + f.AddFlag(fs) + + err := fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)}) + if test.parseError != "" { + if !strings.Contains(err.Error(), test.parseError) { + t.Errorf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) + } + } else if err != nil { + t.Errorf("%d: Parse() Expected nil, Got %v", i, err) + } + for k, v := range test.expect { + if actual := f.enabled.Load().(map[Feature]bool)[k]; actual != v { + t.Errorf("%d: expected %s=%v, Got %v", i, k, v, actual) + } + } + } +} + +func TestFeatureGateOverride(t *testing.T) { + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + + // Don't parse the flag, assert defaults are used. + var f *featureGate = NewFeatureGate() + f.Add(map[Feature]FeatureSpec{ + testAlphaGate: {Default: false, PreRelease: Alpha}, + testBetaGate: {Default: false, PreRelease: Beta}, + }) + + f.Set("TestAlpha=true,TestBeta=true") + if f.Enabled(testAlphaGate) != true { + t.Errorf("Expected true") + } + if f.Enabled(testBetaGate) != true { + t.Errorf("Expected true") + } + + f.Set("TestAlpha=false") + if f.Enabled(testAlphaGate) != false { + t.Errorf("Expected false") + } + if f.Enabled(testBetaGate) != true { + t.Errorf("Expected true") + } +} + +func TestFeatureGateFlagDefaults(t *testing.T) { + // gates for testing + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + + // Don't parse the flag, assert defaults are used. + var f *featureGate = NewFeatureGate() + f.Add(map[Feature]FeatureSpec{ + testAlphaGate: {Default: false, PreRelease: Alpha}, + testBetaGate: {Default: true, PreRelease: Beta}, + }) + + if f.Enabled(testAlphaGate) != false { + t.Errorf("Expected false") + } + if f.Enabled(testBetaGate) != true { + t.Errorf("Expected true") + } +} + +func TestFeatureGateKnownFeatures(t *testing.T) { + // gates for testing + const ( + testAlphaGate Feature = "TestAlpha" + testBetaGate Feature = "TestBeta" + testGAGate Feature = "TestGA" + testDeprecatedGate Feature = "TestDeprecated" + ) + + // Don't parse the flag, assert defaults are used. + var f *featureGate = NewFeatureGate() + f.Add(map[Feature]FeatureSpec{ + testAlphaGate: {Default: false, PreRelease: Alpha}, + testBetaGate: {Default: true, PreRelease: Beta}, + testGAGate: {Default: true, PreRelease: GA}, + testDeprecatedGate: {Default: false, PreRelease: Deprecated}, + }) + + known := strings.Join(f.KnownFeatures(), " ") + + assert.Contains(t, known, testAlphaGate) + assert.Contains(t, known, testBetaGate) + assert.NotContains(t, known, testGAGate) + assert.NotContains(t, known, testDeprecatedGate) +} + +func TestFeatureGateSetFromMap(t *testing.T) { + // gates for testing + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + const testLockedTrueGate Feature = "TestLockedTrue" + const testLockedFalseGate Feature = "TestLockedFalse" + + tests := []struct { + name string + setmap map[string]bool + expect map[Feature]bool + setmapError string + }{ + { + name: "set TestAlpha and TestBeta true", + setmap: map[string]bool{ + "TestAlpha": true, + "TestBeta": true, + }, + expect: map[Feature]bool{ + testAlphaGate: true, + testBetaGate: true, + }, + }, + { + name: "set TestBeta true", + setmap: map[string]bool{ + "TestBeta": true, + }, + expect: map[Feature]bool{ + testAlphaGate: false, + testBetaGate: true, + }, + }, + { + name: "set TestAlpha false", + setmap: map[string]bool{ + "TestAlpha": false, + }, + expect: map[Feature]bool{ + testAlphaGate: false, + testBetaGate: false, + }, + }, + { + name: "set TestInvaild true", + setmap: map[string]bool{ + "TestInvaild": true, + }, + expect: map[Feature]bool{ + testAlphaGate: false, + testBetaGate: false, + }, + setmapError: "unrecognized feature gate:", + }, + { + name: "set locked gates", + setmap: map[string]bool{ + "TestLockedTrue": true, + "TestLockedFalse": false, + }, + expect: map[Feature]bool{ + testAlphaGate: false, + testBetaGate: false, + }, + }, + { + name: "set locked gates", + setmap: map[string]bool{ + "TestLockedTrue": false, + }, + expect: map[Feature]bool{ + testAlphaGate: false, + testBetaGate: false, + }, + setmapError: "cannot set feature gate TestLockedTrue to false, feature is locked to true", + }, + { + name: "set locked gates", + setmap: map[string]bool{ + "TestLockedFalse": true, + }, + expect: map[Feature]bool{ + testAlphaGate: false, + testBetaGate: false, + }, + setmapError: "cannot set feature gate TestLockedFalse to true, feature is locked to false", + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("SetFromMap %s", test.name), func(t *testing.T) { + f := NewFeatureGate() + f.Add(map[Feature]FeatureSpec{ + testAlphaGate: {Default: false, PreRelease: Alpha}, + testBetaGate: {Default: false, PreRelease: Beta}, + testLockedTrueGate: {Default: true, PreRelease: GA, LockToDefault: true}, + testLockedFalseGate: {Default: false, PreRelease: GA, LockToDefault: true}, + }) + err := f.SetFromMap(test.setmap) + if test.setmapError != "" { + if err == nil { + t.Errorf("expected error, got none") + } else if !strings.Contains(err.Error(), test.setmapError) { + t.Errorf("%d: SetFromMap(%#v) Expected err:%v, Got err:%v", i, test.setmap, test.setmapError, err) + } + } else if err != nil { + t.Errorf("%d: SetFromMap(%#v) Expected success, Got err:%v", i, test.setmap, err) + } + for k, v := range test.expect { + if actual := f.Enabled(k); actual != v { + t.Errorf("%d: SetFromMap(%#v) Expected %s=%v, Got %s=%v", i, test.setmap, k, v, k, actual) + } + } + }) + } +} + +func TestFeatureGateString(t *testing.T) { + // gates for testing + const testAlphaGate Feature = "TestAlpha" + const testBetaGate Feature = "TestBeta" + const testGAGate Feature = "TestGA" + + featuremap := map[Feature]FeatureSpec{ + testGAGate: {Default: true, PreRelease: GA}, + testAlphaGate: {Default: false, PreRelease: Alpha}, + testBetaGate: {Default: true, PreRelease: Beta}, + } + + tests := []struct { + setmap map[string]bool + expect string + }{ + { + setmap: map[string]bool{ + "TestAlpha": false, + }, + expect: "TestAlpha=false", + }, + { + setmap: map[string]bool{ + "TestAlpha": false, + "TestBeta": true, + }, + expect: "TestAlpha=false,TestBeta=true", + }, + { + setmap: map[string]bool{ + "TestGA": true, + "TestAlpha": false, + "TestBeta": true, + }, + expect: "TestAlpha=false,TestBeta=true,TestGA=true", + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("SetFromMap %s", test.expect), func(t *testing.T) { + f := NewFeatureGate() + f.Add(featuremap) + f.SetFromMap(test.setmap) + result := f.String() + if result != test.expect { + t.Errorf("%d: SetFromMap(%#v) Expected %s, Got %s", i, test.setmap, test.expect, result) + } + }) + } +} diff --git a/featuregate/testing/feature_gate.go b/featuregate/testing/feature_gate.go new file mode 100644 index 00000000..1e0b650e --- /dev/null +++ b/featuregate/testing/feature_gate.go @@ -0,0 +1,44 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "testing" + + "k8s.io/component-base/featuregate" +) + +// SetFeatureGateDuringTest sets the specified gate to the specified value, and returns a function that restores the original value. +// Failures to set or restore cause the test to fail. +// +// Example use: +// +// defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features., true)() +func SetFeatureGateDuringTest(tb testing.TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) func() { + originalValue := gate.Enabled(f) + + if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, value)); err != nil { + tb.Errorf("error setting %s=%v: %v", f, value, err) + } + + return func() { + if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, originalValue)); err != nil { + tb.Errorf("error restoring %s=%v: %v", f, originalValue, err) + } + } +}