From dfc31285f0c698ddcacd019a76be38bbd8651520 Mon Sep 17 00:00:00 2001 From: ccremer Date: Thu, 7 Jul 2022 14:28:00 +0200 Subject: [PATCH 1/7] Add business steps for finalizer --- go.mod | 5 +- operator/operatortest/envtest.go | 198 ++++++++++++++++++++++++++++ operator/steps/context.go | 34 +++++ operator/steps/finalizer.go | 37 ++++++ operator/steps/finalizer_it_test.go | 147 +++++++++++++++++++++ 5 files changed, 419 insertions(+), 2 deletions(-) create mode 100644 operator/operatortest/envtest.go create mode 100644 operator/steps/context.go create mode 100644 operator/steps/finalizer.go create mode 100644 operator/steps/finalizer_it_test.go diff --git a/go.mod b/go.mod index 00fab6e..6203728 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,10 @@ require ( github.com/ccremer/go-command-pipeline v0.18.0 github.com/go-logr/logr v1.2.3 github.com/go-logr/zapr v1.2.3 + github.com/stretchr/testify v1.8.0 github.com/urfave/cli/v2 v2.10.3 go.uber.org/zap v1.21.0 + k8s.io/api v0.24.2 k8s.io/apimachinery v0.24.2 k8s.io/client-go v0.24.2 sigs.k8s.io/controller-runtime v0.12.3 @@ -49,6 +51,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.12.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect @@ -56,7 +59,6 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/cobra v1.4.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.8.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect @@ -75,7 +77,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.24.2 // indirect k8s.io/apiextensions-apiserver v0.24.2 // indirect k8s.io/component-base v0.24.2 // indirect k8s.io/klog/v2 v2.60.1 // indirect diff --git a/operator/operatortest/envtest.go b/operator/operatortest/envtest.go new file mode 100644 index 0000000..1335668 --- /dev/null +++ b/operator/operatortest/envtest.go @@ -0,0 +1,198 @@ +package operatortest + +import ( + "context" + cloudscalev1 "github.com/vshn/appcat-service-s3/apis/cloudscale/v1" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/go-logr/logr" + "github.com/go-logr/zapr" + "github.com/stretchr/testify/suite" + "go.uber.org/zap/zaptest" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +var invalidNSNameCharacters = regexp.MustCompile("[^a-z0-9-]") + +// Suite is the common test suite for integration tests using envtest. +// It's expected that concrete suites use this suite as the base. +type Suite struct { + suite.Suite + + NS string + Client client.Client + Config *rest.Config + Env *envtest.Environment + Logger logr.Logger + Context context.Context + Scheme *runtime.Scheme +} + +// SetupSuite implements suite.SetupAllSuite. +// It is run before running all the tests in the suite and is used to start up a local Kubernetes API server. +func (ts *Suite) SetupSuite() { + ts.Logger = zapr.NewLogger(zaptest.NewLogger(ts.T())) + log.SetLogger(ts.Logger) + + ts.Context = context.Background() + + envtestAssets, ok := os.LookupEnv("KUBEBUILDER_ASSETS") + if !ok { + ts.FailNow("The environment variable KUBEBUILDER_ASSETS is undefined. Configure your IDE to set this variable when running the integration test.") + } + crdDir, ok := os.LookupEnv("ENVTEST_CRD_DIR") + if !ok { + ts.FailNow("The environment variable ENVTEST_CRD_DIR is undefined. Configure your IDE to set this variable when running the integration test.") + } + + info, err := os.Stat(envtestAssets) + absEnvtestAssets, _ := filepath.Abs(envtestAssets) + ts.Require().NoErrorf(err, "'%s' does not seem to exist. Check KUBEBUILDER_ASSETS and make sure you run `make test-integration` before you run this test in your IDE.", absEnvtestAssets) + ts.Require().Truef(info.IsDir(), "'%s' does not seem to be a directory. Check KUBEBUILDER_ASSETS and make sure you run `make test-integration` before you run this test in your IDE.", absEnvtestAssets) + + absCrds, _ := filepath.Abs(crdDir) + info, err = os.Stat(crdDir) + ts.Require().NoErrorf(err, "'%s' does not seem to exist. Make sure to set the working directory to the project root.", absCrds) + ts.Require().Truef(info.IsDir(), "'%s' does not seem to be a directory. Make sure to set the working directory to the project root.", absCrds) + + ts.Logger.Info("envtest directories", "crd", absCrds, "binary assets", absEnvtestAssets) + + testEnv := &envtest.Environment{ + ErrorIfCRDPathMissing: true, + CRDDirectoryPaths: []string{crdDir}, + BinaryAssetsDirectory: envtestAssets, + } + + config, err := testEnv.Start() + ts.Require().NoError(err) + ts.Require().NotNil(config) + + registerCommonCRDs(ts) + + k8sClient, err := client.New(config, client.Options{ + Scheme: ts.Scheme, + }) + ts.Require().NoError(err) + ts.Require().NotNil(k8sClient) + + ts.Env = testEnv + ts.Config = config + ts.Client = k8sClient +} + +func registerCommonCRDs(ts *Suite) { + ts.Scheme = runtime.NewScheme() + ts.RegisterScheme(cloudscalev1.SchemeBuilder.AddToScheme) + ts.RegisterScheme(corev1.SchemeBuilder.AddToScheme) + + // +kubebuilder:scaffold:scheme +} + +// RegisterScheme passes the current scheme to the given SchemeBuilder func. +func (ts *Suite) RegisterScheme(addToScheme func(s *runtime.Scheme) error) { + ts.Require().NoError(addToScheme(ts.Scheme)) +} + +// TearDownSuite implements suite.TearDownAllSuite. +// It is used to shut down the local envtest environment. +func (ts *Suite) TearDownSuite() { + err := ts.Env.Stop() + ts.Require().NoErrorf(err, "error while stopping test environment") + ts.Logger.Info("test environment stopped") +} + +// NewNS returns a new Namespace object with the given name. +// Note: The namespace is not actually created, use EnsureNS for this. +func (ts *Suite) NewNS(nsName string) *corev1.Namespace { + ts.Assert().Emptyf(validation.IsDNS1123Label(nsName), "'%s' does not appear to be a valid name for a namespace", nsName) + + return &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + }, + } +} + +// EnsureNS creates a new Namespace object using Suite.Client. +func (ts *Suite) EnsureNS(nsName string) { + ns := ts.NewNS(nsName) + ts.T().Logf("creating namespace '%s'", nsName) + err := ts.Client.Create(ts.Context, ns) + if err != nil && apierrors.IsAlreadyExists(err) { + return + } + ts.Require().NoError(err) +} + +// EnsureResources ensures that the given resources are existing in the suite. Each error will fail the test. +func (ts *Suite) EnsureResources(resources ...client.Object) { + for _, resource := range resources { + ts.T().Logf("creating resource '%s/%s'", resource.GetNamespace(), resource.GetName()) + ts.Require().NoError(ts.Client.Create(ts.Context, resource)) + } +} + +// UpdateResources ensures that the given resources are updated in the suite. Each error will fail the test. +func (ts *Suite) UpdateResources(resources ...client.Object) { + for _, resource := range resources { + ts.T().Logf("updating resource '%s/%s'", resource.GetNamespace(), resource.GetName()) + ts.Require().NoError(ts.Client.Update(ts.Context, resource)) + } +} + +// UpdateStatus ensures that the Status property of the given resources are updated in the suite. Each error will fail the test. +func (ts *Suite) UpdateStatus(resources ...client.Object) { + for _, resource := range resources { + ts.T().Logf("updating status '%s/%s'", resource.GetNamespace(), resource.GetName()) + ts.Require().NoError(ts.Client.Status().Update(ts.Context, resource)) + } +} + +// DeleteResources deletes the given resources are updated from the suite. Each error will fail the test. +func (ts *Suite) DeleteResources(resources ...client.Object) { + for _, resource := range resources { + ts.T().Logf("deleting '%s/%s'", resource.GetNamespace(), resource.GetName()) + ts.Require().NoError(ts.Client.Delete(ts.Context, resource)) + } +} + +// FetchResource fetches the given object name and stores the result in the given object. +// Test fails on errors. +func (ts *Suite) FetchResource(name types.NamespacedName, object client.Object) { + ts.Require().NoError(ts.Client.Get(ts.Context, name, object)) +} + +// FetchResources fetches resources and puts the items into the given list with the given list options. +// Test fails on errors. +func (ts *Suite) FetchResources(objectList client.ObjectList, opts ...client.ListOption) { + ts.Require().NoError(ts.Client.List(ts.Context, objectList, opts...)) +} + +// MapToRequest maps the given object into a reconcile Request. +func (ts *Suite) MapToRequest(object metav1.Object) ctrl.Request { + return ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: object.GetName(), + Namespace: object.GetNamespace(), + }, + } +} + +// SanitizeNameForNS first converts the given name to lowercase using strings.ToLower +// and then remove all characters but `a-z` (only lower case), `0-9` and the `-` (dash). +func (ts *Suite) SanitizeNameForNS(name string) string { + return invalidNSNameCharacters.ReplaceAllString(strings.ToLower(name), "") +} diff --git a/operator/steps/context.go b/operator/steps/context.go new file mode 100644 index 0000000..58d8655 --- /dev/null +++ b/operator/steps/context.go @@ -0,0 +1,34 @@ +package steps + +import ( + "context" + "fmt" + pipeline "github.com/ccremer/go-command-pipeline" + "reflect" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// clientKey identifies the Kubernetes client in the context. +type clientKey struct{} + +// SetClientInContext sets the given client in the context. +func SetClientInContext(ctx context.Context, c client.Client) { + pipeline.StoreInContext(ctx, clientKey{}, c) +} + +// GetClientFromContext returns the client from the context. +func GetClientFromContext(ctx context.Context) client.Client { + return GetFromContextOrPanic(ctx, clientKey{}).(client.Client) +} + +// GetFromContextOrPanic returns the object if the key exists. +// If the does not exist, then it panics. +// May return nil if the key exists but the value actually is nil. +func GetFromContextOrPanic(ctx context.Context, key any) any { + val, exists := pipeline.LoadFromContext(ctx, key) + if !exists { + keyName := reflect.TypeOf(key).Name() + panic(fmt.Errorf("key %q does not exist in the given context", keyName)) + } + return val +} diff --git a/operator/steps/finalizer.go b/operator/steps/finalizer.go new file mode 100644 index 0000000..82945e7 --- /dev/null +++ b/operator/steps/finalizer.go @@ -0,0 +1,37 @@ +package steps + +import ( + "context" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// AddFinalizerFn returns a func that adds the given finalizer to an object identified by `objKey` in the context. +// If the finalizer is already present, this step is a no-op. +// The object from context needs to be of a client.Object. +func AddFinalizerFn(objKey any, finalizer string) func(ctx context.Context) error { + return func(ctx context.Context) error { + kube := GetClientFromContext(ctx) + obj := GetFromContextOrPanic(ctx, objKey).(client.Object) + + if controllerutil.AddFinalizer(obj, finalizer) { + return kube.Update(ctx, obj) + } + return nil + } +} + +// RemoveFinalizerFn returns a func that removes the given finalizer from the object identified by `objKey` in the context. +// If the finalizer is not present, this step is a no-op. +// The object from context needs to be of a client.Object. +func RemoveFinalizerFn(objKey any, finalizer string) func(ctx context.Context) error { + return func(ctx context.Context) error { + kube := GetClientFromContext(ctx) + instance := GetFromContextOrPanic(ctx, objKey).(client.Object) + + if controllerutil.RemoveFinalizer(instance, finalizer) { + return kube.Update(ctx, instance) + } + return nil + } +} diff --git a/operator/steps/finalizer_it_test.go b/operator/steps/finalizer_it_test.go new file mode 100644 index 0000000..cdb9460 --- /dev/null +++ b/operator/steps/finalizer_it_test.go @@ -0,0 +1,147 @@ +//go:build integration + +package steps + +import ( + "context" + pipeline "github.com/ccremer/go-command-pipeline" + "github.com/stretchr/testify/suite" + "github.com/vshn/appcat-service-s3/operator/operatortest" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "testing" +) + +type FinalizerSuite struct { + operatortest.Suite +} + +func TestFinalizerSuite(t *testing.T) { + suite.Run(t, new(FinalizerSuite)) +} + +func (ts *FinalizerSuite) BeforeTest(suiteName, testName string) { + ts.Context = pipeline.MutableContext(context.Background()) + SetClientInContext(ts.Context, ts.Client) +} + +func (ts *FinalizerSuite) Test_AddFinalizer() { + tests := map[string]struct { + prepare func(resource client.Object) + givenName string + givenNamespace string + assert func(previousResource, resource client.Object) + }{ + "GivenResourceWithoutFinalizer_WhenAddingFinalizer_ThenExpectResourceUpdatedWithAddedFinalizer": { + prepare: func(resource client.Object) { + ts.EnsureNS("remove-finalizer") + ts.EnsureResources(resource) + ts.Assert().Empty(resource.GetFinalizers()) + }, + + givenName: "has-finalizer", + givenNamespace: "add-finalizer", + assert: func(previousResource, result client.Object) { + ts.Require().Len(result.GetFinalizers(), 1, "amount of finalizers") + ts.Assert().Equal("domain.io/finalizer", result.GetFinalizers()[0]) + ts.Assert().NotEqual(previousResource.GetResourceVersion(), result.GetResourceVersion(), "resource version should change") + }, + }, + "GivenResourceWithExistingFinalizer_WhenAddingFinalizer_ThenExpectResourceUnchanged": { + prepare: func(resource client.Object) { + resource.SetFinalizers([]string{"domain.io/finalizer"}) + ts.EnsureNS("add-finalizer") + ts.EnsureResources(resource) + }, + + givenName: "no-finalizer", + givenNamespace: "add-finalizer", + assert: func(previousResource, result client.Object) { + ts.Require().Len(result.GetFinalizers(), 1, "amount of finalizers") + ts.Assert().Equal("domain.io/finalizer", result.GetFinalizers()[0]) + ts.Assert().Equal(previousResource.GetResourceVersion(), result.GetResourceVersion(), "resource version should be equal") + }, + }, + } + for name, tc := range tests { + ts.Run(name, func() { + type resourceKey struct{} + // Arrange + resource := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: tc.givenName, Namespace: tc.givenNamespace}, + } + pipeline.StoreInContext(ts.Context, resourceKey{}, resource) + tc.prepare(resource) + previousVersion := resource.DeepCopy() + + // Act + err := AddFinalizerFn(resourceKey{}, "domain.io/finalizer")(ts.Context) + ts.Require().NoError(err) + + // Assert + result := &corev1.ConfigMap{} + ts.FetchResource(client.ObjectKeyFromObject(resource), result) + tc.assert(previousVersion, result) + }) + } +} + +func (ts *FinalizerSuite) Test_RemoveFinalizer() { + tests := map[string]struct { + prepare func(resource client.Object) + givenName string + givenNamespace string + assert func(previousResource, resource client.Object) + }{ + "GivenResourceWithFinalizer_WhenDeletingFinalizer_ThenExpectResourceUpdatedWithRemovedFinalizer": { + prepare: func(resource client.Object) { + resource.SetFinalizers([]string{"domain.io/finalizer"}) + ts.EnsureNS("remove-finalizer") + ts.EnsureResources(resource) + ts.Assert().NotEmpty(resource.GetFinalizers()) + }, + + givenName: "has-finalizer", + givenNamespace: "remove-finalizer", + assert: func(previousResource, result client.Object) { + ts.Assert().Empty(result.GetFinalizers()) + ts.Assert().NotEqual(previousResource.GetResourceVersion(), result.GetResourceVersion(), "resource version should change") + }, + }, + "GivenResourceWithoutFinalizer_WhenDeletingFinalizer_ThenExpectResourceUnchanged": { + prepare: func(resource client.Object) { + ts.EnsureNS("remove-finalizer") + ts.EnsureResources(resource) + }, + + givenName: "no-finalizer", + givenNamespace: "remove-finalizer", + assert: func(previousResource, result client.Object) { + ts.Assert().Empty(result.GetFinalizers()) + ts.Assert().Equal(previousResource.GetResourceVersion(), result.GetResourceVersion(), "resource version should be equal") + }, + }, + } + for name, tc := range tests { + ts.Run(name, func() { + type resourceKey struct{} + // Arrange + resource := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: tc.givenName, Namespace: tc.givenNamespace}, + } + pipeline.StoreInContext(ts.Context, resourceKey{}, resource) + tc.prepare(resource) + previousVersion := resource.DeepCopy() + + // Act + err := RemoveFinalizerFn(resourceKey{}, "domain.io/finalizer")(ts.Context) + ts.Require().NoError(err) + + // Assert + result := &corev1.ConfigMap{} + ts.FetchResource(client.ObjectKeyFromObject(resource), result) + tc.assert(previousVersion, result) + }) + } +} From 354fdc81288c12be8346be1c37d5ffa5027aa81c Mon Sep 17 00:00:00 2001 From: ccremer Date: Thu, 7 Jul 2022 17:37:44 +0200 Subject: [PATCH 2/7] Fix remaining wrong names --- generate_sample.go | 4 ++-- operator/cloudscale/controller.go | 10 +++++----- operator/cloudscale/setup.go | 2 +- operator/operator.go | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/generate_sample.go b/generate_sample.go index 95e13de..86df73a 100644 --- a/generate_sample.go +++ b/generate_sample.go @@ -30,11 +30,11 @@ func main() { } func generateCloudscaleObjectsUserSample() { - spec := newPostgresqlStandaloneSample() + spec := newObjectsUserSample() serialize(spec, true) } -func newPostgresqlStandaloneSample() *cloudscalev1.ObjectsUser { +func newObjectsUserSample() *cloudscalev1.ObjectsUser { return &cloudscalev1.ObjectsUser{ TypeMeta: metav1.TypeMeta{ APIVersion: cloudscalev1.ObjectsUserGroupVersionKind.GroupVersion().String(), diff --git a/operator/cloudscale/controller.go b/operator/cloudscale/controller.go index 8cc0d7b..80fe854 100644 --- a/operator/cloudscale/controller.go +++ b/operator/cloudscale/controller.go @@ -20,13 +20,13 @@ var userFinalizer = "s3.appcat.vshn.io/user-protection" // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update -// PostgresStandaloneReconciler reconciles cloudscalev1.ObjectsUser. -type PostgresStandaloneReconciler struct { +// ObjectsUserReconciler reconciles cloudscalev1.ObjectsUser. +type ObjectsUserReconciler struct { client client.Client } // Reconcile implements reconcile.Reconciler. -func (r *PostgresStandaloneReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { +func (r *ObjectsUserReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { ctx = pipeline.MutableContext(ctx) obj := &cloudscalev1.ObjectsUser{} log := ctrl.LoggerFrom(ctx) @@ -47,14 +47,14 @@ func (r *PostgresStandaloneReconciler) Reconcile(ctx context.Context, request re } // Provision reconciles the given object. -func (r *PostgresStandaloneReconciler) Provision(ctx context.Context, instance *cloudscalev1.ObjectsUser) (reconcile.Result, error) { +func (r *ObjectsUserReconciler) Provision(ctx context.Context) (reconcile.Result, error) { log := ctrl.LoggerFrom(ctx) log.Info("Provisioning instance") return reconcile.Result{}, nil } // Delete prepares the given object for deletion. -func (r *PostgresStandaloneReconciler) Delete(ctx context.Context) (reconcile.Result, error) { +func (r *ObjectsUserReconciler) Delete(ctx context.Context) (reconcile.Result, error) { log := ctrl.LoggerFrom(ctx) log.Info("Deleting instance") return reconcile.Result{RequeueAfter: 1 * time.Second}, nil diff --git a/operator/cloudscale/setup.go b/operator/cloudscale/setup.go index 5e21d50..0c428be 100644 --- a/operator/cloudscale/setup.go +++ b/operator/cloudscale/setup.go @@ -15,7 +15,7 @@ func SetupController(mgr ctrl.Manager) error { Named(name). For(&cloudscalev1.ObjectsUser{}). WithEventFilter(predicate.Or(predicate.GenerationChangedPredicate{})). - Complete(&PostgresStandaloneReconciler{ + Complete(&ObjectsUserReconciler{ client: mgr.GetClient(), }) } diff --git a/operator/operator.go b/operator/operator.go index 16a6887..395482e 100644 --- a/operator/operator.go +++ b/operator/operator.go @@ -5,7 +5,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" ) -// SetupControllers creates all Postgresql controllers with the supplied logger and adds them to the supplied manager. +// SetupControllers creates all controllers with the supplied logger and adds them to the supplied manager. func SetupControllers(mgr ctrl.Manager) error { for _, setup := range []func(ctrl.Manager) error{ cloudscale.SetupController, From 7782a61d11d885b8487ba1e17bcde15d4c0caca9 Mon Sep 17 00:00:00 2001 From: ccremer Date: Thu, 7 Jul 2022 17:39:14 +0200 Subject: [PATCH 3/7] Implement creating objects users on cloudscale --- apis/cloudscale/v1/objectsuser_types.go | 8 ++ apis/conditions/conditions.go | 47 ++++++++ .../ROOT/examples/cloudscale_objectsuser.yaml | 3 +- generate_sample.go | 6 +- go.mod | 3 +- go.sum | 4 + operator/cloudscale/client.go | 113 ++++++++++++++++++ operator/cloudscale/client_it_test.go | 59 +++++++++ operator/cloudscale/controller.go | 27 ++++- operator/cloudscale/provision.go | 85 +++++++++++++ operator/steps/status.go | 16 +++ operator_command.go | 8 +- ...dscale.s3.appcat.vshn.io_objectsusers.yaml | 12 ++ ...udscale.s3.appcat.vshn.io_objectsuser.yaml | 3 +- 14 files changed, 382 insertions(+), 12 deletions(-) create mode 100644 apis/conditions/conditions.go create mode 100644 operator/cloudscale/client.go create mode 100644 operator/cloudscale/client_it_test.go create mode 100644 operator/cloudscale/provision.go create mode 100644 operator/steps/status.go diff --git a/apis/cloudscale/v1/objectsuser_types.go b/apis/cloudscale/v1/objectsuser_types.go index 57d8658..ab01b67 100644 --- a/apis/cloudscale/v1/objectsuser_types.go +++ b/apis/cloudscale/v1/objectsuser_types.go @@ -8,11 +8,19 @@ import ( // ObjectsUserSpec defines the desired state of an ObjectsUser. type ObjectsUserSpec struct { + // +kubebuilder:validation:Required + // +kubebuilder:validation:Pattern=`[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*` + + // SecretRef contains the name of the Secret where the credentials of the ObjectsUser are stored. + // Must be a name that Kubernetes accepts as Secret name (lowercase RFC 1123 subdomain). + SecretRef string `json:"secretRef"` } // ObjectsUserStatus represents the observed state of a ObjectsUser. type ObjectsUserStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty"` + // UserID is the unique ID as generated by cloudscale.ch. + UserID string `json:"userID,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/conditions/conditions.go b/apis/conditions/conditions.go new file mode 100644 index 0000000..4cf0e9e --- /dev/null +++ b/apis/conditions/conditions.go @@ -0,0 +1,47 @@ +package conditions + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +// Reasons that give more context to conditions +const ( + ReasonAvailable = "Available" + ReasonProvisioningFailed = "ProvisioningFailed" +) + +const ( + // TypeReady indicates that a resource is ready for use. + TypeReady = "Ready" + // TypeFailed indicates that a resource has failed the provisioning. + TypeFailed = "Failed" +) + +// Ready creates a condition with TypeReady, ReasonAvailable and empty message. +func Ready() metav1.Condition { + return metav1.Condition{ + Type: TypeReady, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.Now(), + Reason: ReasonAvailable, + } +} + +// NotReady creates a condition with TypeReady, ReasonAvailable and empty message. +func NotReady() metav1.Condition { + return metav1.Condition{ + Type: TypeReady, + Status: metav1.ConditionFalse, + LastTransitionTime: metav1.Now(), + Reason: ReasonAvailable, + } +} + +// Failed creates a condition with TypeFailed, ReasonProvisioningFailed and the error message. +func Failed(err error) metav1.Condition { + return metav1.Condition{ + Type: TypeFailed, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.Now(), + Reason: ReasonProvisioningFailed, + Message: err.Error(), + } +} diff --git a/docs/modules/ROOT/examples/cloudscale_objectsuser.yaml b/docs/modules/ROOT/examples/cloudscale_objectsuser.yaml index f7dbbb4..3956d33 100644 --- a/docs/modules/ROOT/examples/cloudscale_objectsuser.yaml +++ b/docs/modules/ROOT/examples/cloudscale_objectsuser.yaml @@ -3,4 +3,5 @@ kind: ObjectsUser metadata: name: my-cloudscale-user namespace: default -spec: {} +spec: + secretRef: my-cloudscale-user-credentials diff --git a/generate_sample.go b/generate_sample.go index 86df73a..ff9323c 100644 --- a/generate_sample.go +++ b/generate_sample.go @@ -41,8 +41,10 @@ func newObjectsUserSample() *cloudscalev1.ObjectsUser { Kind: cloudscalev1.ObjectsUserKind, }, ObjectMeta: metav1.ObjectMeta{Name: "my-cloudscale-user", Namespace: "default", Generation: 1}, - Spec: cloudscalev1.ObjectsUserSpec{}, - Status: cloudscalev1.ObjectsUserStatus{}, + Spec: cloudscalev1.ObjectsUserSpec{ + SecretRef: "my-cloudscale-user-credentials", + }, + Status: cloudscalev1.ObjectsUserStatus{}, } } diff --git a/go.mod b/go.mod index 6203728..a912b96 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,13 @@ go 1.18 require ( github.com/ccremer/go-command-pipeline v0.18.0 + github.com/cloudscale-ch/cloudscale-go-sdk/v2 v2.0.1 github.com/go-logr/logr v1.2.3 github.com/go-logr/zapr v1.2.3 github.com/stretchr/testify v1.8.0 github.com/urfave/cli/v2 v2.10.3 go.uber.org/zap v1.21.0 + golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 k8s.io/api v0.24.2 k8s.io/apimachinery v0.24.2 k8s.io/client-go v0.24.2 @@ -64,7 +66,6 @@ require ( go.uber.org/multierr v1.6.0 // indirect golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect - golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect diff --git a/go.sum b/go.sum index 4e20d77..ab6656a 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,7 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/ccremer/go-command-pipeline v0.18.0 h1:QJWlz+/KtBRIpv46T3zrW58w5DDaLZTnULka/7pJtQM= github.com/ccremer/go-command-pipeline v0.18.0/go.mod h1:fhQHl6aNWFKU1qKNTd26Zn5PHlcPqTylygOu845uejA= +github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= @@ -89,6 +90,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudscale-ch/cloudscale-go-sdk/v2 v2.0.1 h1:v0jzg+Wk2sGErKH8CTGl4+stWOJINmu1Xc3RNcBB0cM= +github.com/cloudscale-ch/cloudscale-go-sdk/v2 v2.0.1/go.mod h1:0oHKsCRkSUkAnnCBkfLxU+BaMa9s0ZicXYfYvOIJQ1E= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -618,6 +621,7 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacp golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190517181255-950ef44c6e07/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/operator/cloudscale/client.go b/operator/cloudscale/client.go new file mode 100644 index 0000000..2abc4c7 --- /dev/null +++ b/operator/cloudscale/client.go @@ -0,0 +1,113 @@ +package cloudscale + +import ( + "context" + "fmt" + pipeline "github.com/ccremer/go-command-pipeline" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v2" + cloudscalev1 "github.com/vshn/appcat-service-s3/apis/cloudscale/v1" + "github.com/vshn/appcat-service-s3/operator/steps" + "golang.org/x/oauth2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// APIToken is the authentication token to use against cloudscale.ch API +var APIToken string + +// CloudscaleClientKey identifies the cloudscale client in the context. +type CloudscaleClientKey struct{} + +// CreateCloudscaleClientFn creates a new client using the API token provided. +func CreateCloudscaleClientFn(apiToken string) func(ctx context.Context) error { + return func(ctx context.Context) error { + tc := oauth2.NewClient(ctx, oauth2.StaticTokenSource(&oauth2.Token{AccessToken: apiToken})) + csClient := cloudscalesdk.NewClient(tc) + pipeline.StoreInContext(ctx, CloudscaleClientKey{}, csClient) + return nil + } +} + +// CloudscaleUserKey identifies the User object from cloudscale SDK in the context. +type CloudscaleUserKey struct{} + +// CreateObjectsUserFn creates a new objects user in the project associated with the API token. +func CreateObjectsUserFn() func(ctx context.Context) error { + return func(ctx context.Context) error { + csClient := steps.GetFromContextOrPanic(ctx, CloudscaleClientKey{}).(*cloudscalesdk.Client) + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) + log := controllerruntime.LoggerFrom(ctx) + + displayName := fmt.Sprintf("%s.%s", user.Namespace, user.Name) + + csUser, err := csClient.ObjectsUsers.Create(ctx, &cloudscalesdk.ObjectsUserRequest{ + DisplayName: displayName, + }) + user.Status.UserID = csUser.ID + + pipeline.StoreInContext(ctx, CloudscaleUserKey{}, csUser) + return logIfNotError(err, log, 1, "created objects user in cloudscale") + } +} + +// GetObjectsUserFn fetches an existing objects user from the project associated with the API token. +func GetObjectsUserFn() func(ctx context.Context) error { + return func(ctx context.Context) error { + csClient := steps.GetFromContextOrPanic(ctx, CloudscaleClientKey{}).(*cloudscalesdk.Client) + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) + log := controllerruntime.LoggerFrom(ctx) + + csUser, err := csClient.ObjectsUsers.Get(ctx, user.Status.UserID) + + pipeline.StoreInContext(ctx, CloudscaleUserKey{}, csUser) + return logIfNotError(err, log, 1, "fetched objects user in cloudscale") + } +} + +// UserCredentialSecretKey identifies the credential Secret in the context. +type UserCredentialSecretKey struct{} + +// EnsureCredentialSecretFn creates the credential secret. +func EnsureCredentialSecretFn() func(ctx context.Context) error { + return func(ctx context.Context) error { + kube := steps.GetClientFromContext(ctx) + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) + csUser := steps.GetFromContextOrPanic(ctx, CloudscaleUserKey{}).(*cloudscalesdk.ObjectsUser) + log := controllerruntime.LoggerFrom(ctx) + + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: user.Spec.SecretRef, Namespace: user.Namespace}} + + if keyErr := checkUserForKeys(csUser); keyErr != nil { + return keyErr + } + + // See https://www.cloudscale.ch/en/api/v1#objects-users + + _, err := controllerruntime.CreateOrUpdate(ctx, kube, secret, func() error { + secret.Labels = labels.Merge(secret.Labels, getCommonLabels(user.Name)) + if secret.StringData == nil { + secret.StringData = map[string]string{} + } + secret.StringData["AWS_ACCESS_KEY_ID"] = csUser.Keys[0]["access_key"] + secret.StringData["AWS_SECRET_ACCESS_KEY"] = csUser.Keys[0]["secret_key"] + controllerutil.AddFinalizer(secret, userFinalizer) + return controllerutil.SetOwnerReference(user, secret, kube.Scheme()) + }) + + pipeline.StoreInContext(ctx, UserCredentialSecretKey{}, secret) + return logIfNotError(err, log, 1, "ensured credential secret", "secretName", user.Spec.SecretRef) + } +} + +func checkUserForKeys(user *cloudscalesdk.ObjectsUser) error { + if len(user.Keys) == 0 { + return fmt.Errorf("the returned objects user has no key pairs: %q", user.ID) + } + if val, exists := user.Keys[0]["secret_key"]; exists && val == "" { + return fmt.Errorf("the returned objects user %q has no secret_key. Does the API token have enough permissions?", user.ID) + } + return nil +} diff --git a/operator/cloudscale/client_it_test.go b/operator/cloudscale/client_it_test.go new file mode 100644 index 0000000..c35e9e7 --- /dev/null +++ b/operator/cloudscale/client_it_test.go @@ -0,0 +1,59 @@ +//go:build integration + +package cloudscale + +import ( + "context" + pipeline "github.com/ccremer/go-command-pipeline" + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v2" + "github.com/stretchr/testify/suite" + cloudscalev1 "github.com/vshn/appcat-service-s3/apis/cloudscale/v1" + "github.com/vshn/appcat-service-s3/operator/operatortest" + "github.com/vshn/appcat-service-s3/operator/steps" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "testing" +) + +type CloudscaleClientSuite struct { + operatortest.Suite +} + +func TestFinalizerSuite(t *testing.T) { + suite.Run(t, new(CloudscaleClientSuite)) +} + +func (ts *CloudscaleClientSuite) BeforeTest(suiteName, testName string) { + ts.Context = pipeline.MutableContext(context.Background()) + steps.SetClientInContext(ts.Context, ts.Client) +} + +func (ts *CloudscaleClientSuite) Test_EnsureCredentialSecretFn() { + // Arrange + user := &cloudscalev1.ObjectsUser{ + ObjectMeta: metav1.ObjectMeta{Name: "user", Namespace: "namespace", UID: "uid"}, + Spec: cloudscalev1.ObjectsUserSpec{SecretRef: "secret"}} + pipeline.StoreInContext(ts.Context, ObjectsUserKey{}, user) + + csUser := &cloudscalesdk.ObjectsUser{ + Keys: []map[string]string{{"access_key": "access", "secret_key": "secret"}}, + } + pipeline.StoreInContext(ts.Context, CloudscaleUserKey{}, csUser) + + ts.EnsureNS(user.Namespace) + + // Act + err := EnsureCredentialSecretFn()(ts.Context) + ts.Require().NoError(err) + + // Assert + result := &corev1.Secret{} + ts.FetchResource(types.NamespacedName{Namespace: user.Namespace, Name: "secret"}, result) + ts.Require().Len(result.Data, 2, "amount of keys") + ts.Assert().Equal("access", string(result.Data["AWS_ACCESS_KEY_ID"]), "access key value") + ts.Assert().Equal("secret", string(result.Data["AWS_SECRET_ACCESS_KEY"]), "secret key value") + ts.Assert().Equal(types.UID("uid"), result.OwnerReferences[0].UID, "owner reference set") + ts.Assert().Contains(result.Finalizers, userFinalizer, "finalizer present") + ts.Assert().NotNil(pipeline.MustLoadFromContext(ts.Context, UserCredentialSecretKey{}), "secret stored in context") +} diff --git a/operator/cloudscale/controller.go b/operator/cloudscale/controller.go index 80fe854..013568e 100644 --- a/operator/cloudscale/controller.go +++ b/operator/cloudscale/controller.go @@ -2,7 +2,9 @@ package cloudscale import ( "context" + "github.com/go-logr/logr" cloudscalev1 "github.com/vshn/appcat-service-s3/apis/cloudscale/v1" + "github.com/vshn/appcat-service-s3/operator/steps" "time" pipeline "github.com/ccremer/go-command-pipeline" @@ -25,12 +27,15 @@ type ObjectsUserReconciler struct { client client.Client } +// ObjectsUserKey identifies the ObjectsUser in the context. +type ObjectsUserKey struct{} + // Reconcile implements reconcile.Reconciler. func (r *ObjectsUserReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { ctx = pipeline.MutableContext(ctx) - obj := &cloudscalev1.ObjectsUser{} log := ctrl.LoggerFrom(ctx) log.V(1).Info("Reconciling") + obj := &cloudscalev1.ObjectsUser{} err := r.client.Get(ctx, request.NamespacedName, obj) if err != nil && apierrors.IsNotFound(err) { // doesn't exist anymore, nothing to do @@ -40,22 +45,34 @@ func (r *ObjectsUserReconciler) Reconcile(ctx context.Context, request reconcile // some other error return reconcile.Result{}, err } + pipeline.StoreInContext(ctx, ObjectsUserKey{}, obj) + steps.SetClientInContext(ctx, r.client) if !obj.DeletionTimestamp.IsZero() { return r.Delete(ctx) } - return r.Provision(ctx, obj) + return r.Provision(ctx) } // Provision reconciles the given object. func (r *ObjectsUserReconciler) Provision(ctx context.Context) (reconcile.Result, error) { log := ctrl.LoggerFrom(ctx) - log.Info("Provisioning instance") - return reconcile.Result{}, nil + log.Info("Provisioning resource") + p := NewObjectsUserPipeline() + err := p.Run(ctx) + return reconcile.Result{}, err } // Delete prepares the given object for deletion. func (r *ObjectsUserReconciler) Delete(ctx context.Context) (reconcile.Result, error) { log := ctrl.LoggerFrom(ctx) - log.Info("Deleting instance") + log.Info("Deleting resource") return reconcile.Result{RequeueAfter: 1 * time.Second}, nil } + +func logIfNotError(err error, log logr.Logger, level int, msg string, keysAndValues ...any) error { + if err != nil { + return err + } + log.V(level).Info(msg, keysAndValues...) + return nil +} diff --git a/operator/cloudscale/provision.go b/operator/cloudscale/provision.go new file mode 100644 index 0000000..4d693eb --- /dev/null +++ b/operator/cloudscale/provision.go @@ -0,0 +1,85 @@ +package cloudscale + +import ( + "context" + "fmt" + pipeline "github.com/ccremer/go-command-pipeline" + cloudscalev1 "github.com/vshn/appcat-service-s3/apis/cloudscale/v1" + "github.com/vshn/appcat-service-s3/operator/steps" + "k8s.io/apimachinery/pkg/labels" + "strings" +) + +// ObjectsUserPipeline provisions ObjectsUsers on cloudscale.ch +type ObjectsUserPipeline struct { +} + +// NewObjectsUserPipeline returns a new instance of ObjectsUserPipeline. +func NewObjectsUserPipeline() *ObjectsUserPipeline { + return &ObjectsUserPipeline{} +} + +// Run executes the business logic. +func (p *ObjectsUserPipeline) Run(ctx context.Context) error { + pipe := pipeline.NewPipeline(). + WithSteps( + pipeline.NewStepFromFunc("add finalizer", steps.AddFinalizerFn(ObjectsUserKey{}, userFinalizer)), + pipeline.NewStepFromFunc("validate spec", validateSpec()), + pipeline.NewStepFromFunc("create client", CreateCloudscaleClientFn(APIToken)), + pipeline.IfOrElse(isObjectsUserIDKnown(), + pipeline.NewStepFromFunc("fetch objects user", GetObjectsUserFn()), + pipeline.NewPipeline().WithNestedSteps("new user", + pipeline.NewStepFromFunc("create objects user", CreateObjectsUserFn()), + pipeline.NewStepFromFunc("set user in status", steps.UpdateStatusFn(ObjectsUserKey{})), + ), + ), + pipeline.NewStepFromFunc("ensure credential secret", EnsureCredentialSecretFn()), + pipeline.NewStepFromFunc("set status condition", markUserReadyFn()), + ). + ) + result := pipe.RunWithContext(ctx) + if result.IsFailed() { + // TODO: Add failed condition + // TODO: Emit event + return result.Err() + } + return nil +} + +func validateSpec() func(ctx context.Context) error { + return func(ctx context.Context) error { + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) + if user.Spec.SecretRef == "" { + return fmt.Errorf("spec.secretRef cannot be empty") + } + return nil + } +} + +func isObjectsUserIDKnown() func(ctx context.Context) bool { + return func(ctx context.Context) bool { + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) + return user.Status.UserID != "" + } +} + +func getCommonLabels(instanceName string) labels.Set { + // https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/ + return labels.Set{ + "app.kubernetes.io/instance": instanceName, + "app.kubernetes.io/managed-by": cloudscalev1.Group, + "app.kubernetes.io/created-by": fmt.Sprintf("controller-%s", strings.ToLower(cloudscalev1.ObjectsUserKind)), + } +} + +func markUserReadyFn() func(ctx context.Context) error { + return func(ctx context.Context) error { + kube := steps.GetClientFromContext(ctx) + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) + + meta.SetStatusCondition(&user.Status.Conditions, conditions.Ready()) + meta.RemoveStatusCondition(&user.Status.Conditions, conditions.TypeFailed) + return kube.Status().Update(ctx, user) + } + +} diff --git a/operator/steps/status.go b/operator/steps/status.go new file mode 100644 index 0000000..8590c12 --- /dev/null +++ b/operator/steps/status.go @@ -0,0 +1,16 @@ +package steps + +import ( + "context" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// UpdateStatusFn returns a func that updates the status of the object identified by key retrieved from the context. +func UpdateStatusFn(objKey any) func(ctx context.Context) error { + return func(ctx context.Context) error { + kube := GetClientFromContext(ctx) + obj := GetFromContextOrPanic(ctx, objKey).(client.Object) + + return kube.Status().Update(ctx, obj) + } +} diff --git a/operator_command.go b/operator_command.go index edbfa2e..bf4e3f8 100644 --- a/operator_command.go +++ b/operator_command.go @@ -2,15 +2,15 @@ package main import ( "context" + pipeline "github.com/ccremer/go-command-pipeline" "github.com/vshn/appcat-service-s3/apis" "github.com/vshn/appcat-service-s3/operator" + "github.com/vshn/appcat-service-s3/operator/cloudscale" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection/resourcelock" "sigs.k8s.io/controller-runtime/pkg/manager" "time" - pipeline "github.com/ccremer/go-command-pipeline" - "github.com/urfave/cli/v2" ctrl "sigs.k8s.io/controller-runtime" ) @@ -36,6 +36,10 @@ func newOperatorCommand() *cli.Command { Usage: "Use leader election for the controller manager.", Destination: &command.LeaderElectionEnabled, }, + &cli.StringFlag{Name: "cloudscale-api-token", Value: "", EnvVars: []string{"CLOUDSCALE_API_TOKEN"}, + Usage: "Token to use against cloudscale.ch API to provision ObjectsUsers.", + Destination: &cloudscale.APIToken, + }, }, } } diff --git a/package/crds/cloudscale.s3.appcat.vshn.io_objectsusers.yaml b/package/crds/cloudscale.s3.appcat.vshn.io_objectsusers.yaml index e2e3515..c0d6f38 100644 --- a/package/crds/cloudscale.s3.appcat.vshn.io_objectsusers.yaml +++ b/package/crds/cloudscale.s3.appcat.vshn.io_objectsusers.yaml @@ -44,6 +44,15 @@ spec: type: object spec: description: ObjectsUserSpec defines the desired state of an ObjectsUser. + properties: + secretRef: + description: SecretRef contains the name of the Secret where the credentials + of the ObjectsUser are stored. Must be a name that Kubernetes accepts + as Secret name (lowercase RFC 1123 subdomain). + pattern: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' + type: string + required: + - secretRef type: object status: description: ObjectsUserStatus represents the observed state of a ObjectsUser. @@ -116,6 +125,9 @@ spec: - type type: object type: array + userID: + description: UserID is the unique ID as generated by cloudscale.ch. + type: string type: object required: - spec diff --git a/package/samples/cloudscale.s3.appcat.vshn.io_objectsuser.yaml b/package/samples/cloudscale.s3.appcat.vshn.io_objectsuser.yaml index 9318c43..3d314e2 100644 --- a/package/samples/cloudscale.s3.appcat.vshn.io_objectsuser.yaml +++ b/package/samples/cloudscale.s3.appcat.vshn.io_objectsuser.yaml @@ -5,5 +5,6 @@ metadata: generation: 1 name: my-cloudscale-user namespace: default -spec: {} +spec: + secretRef: my-cloudscale-user-credentials status: {} From 39f2475818bd16a7b8323d0b98128172387a6948 Mon Sep 17 00:00:00 2001 From: ccremer Date: Fri, 8 Jul 2022 11:03:20 +0200 Subject: [PATCH 4/7] Emit Kubernetes events --- apis/conditions/builder.go | 34 ++++++++++++++++++++++++ operator/cloudscale/controller.go | 5 +++- operator/cloudscale/provision.go | 43 +++++++++++++++++++++++-------- operator/cloudscale/setup.go | 3 ++- operator/steps/context.go | 14 ++++++++++ 5 files changed, 86 insertions(+), 13 deletions(-) create mode 100644 apis/conditions/builder.go diff --git a/apis/conditions/builder.go b/apis/conditions/builder.go new file mode 100644 index 0000000..fcb1683 --- /dev/null +++ b/apis/conditions/builder.go @@ -0,0 +1,34 @@ +package conditions + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ConditionBuilder builds Conditions using various properties. +type ConditionBuilder struct { + condition metav1.Condition +} + +// Builder returns a new ConditionBuilder instance. +func Builder() *ConditionBuilder { + return &ConditionBuilder{} +} + +// With initializes the condition with the given value. +// Returns itself for convenience. +func (b *ConditionBuilder) With(condition metav1.Condition) *ConditionBuilder { + b.condition = condition + return b +} + +// WithMessage sets the condition message. +// Returns itself for convenience. +func (b *ConditionBuilder) WithMessage(message string) *ConditionBuilder { + b.condition.Message = message + return b +} + +// Build returns the condition. +func (b *ConditionBuilder) Build() metav1.Condition { + return b.condition +} diff --git a/operator/cloudscale/controller.go b/operator/cloudscale/controller.go index 013568e..8a8ac87 100644 --- a/operator/cloudscale/controller.go +++ b/operator/cloudscale/controller.go @@ -5,6 +5,7 @@ import ( "github.com/go-logr/logr" cloudscalev1 "github.com/vshn/appcat-service-s3/apis/cloudscale/v1" "github.com/vshn/appcat-service-s3/operator/steps" + "k8s.io/client-go/tools/record" "time" pipeline "github.com/ccremer/go-command-pipeline" @@ -24,7 +25,8 @@ var userFinalizer = "s3.appcat.vshn.io/user-protection" // ObjectsUserReconciler reconciles cloudscalev1.ObjectsUser. type ObjectsUserReconciler struct { - client client.Client + client client.Client + recorder record.EventRecorder } // ObjectsUserKey identifies the ObjectsUser in the context. @@ -47,6 +49,7 @@ func (r *ObjectsUserReconciler) Reconcile(ctx context.Context, request reconcile } pipeline.StoreInContext(ctx, ObjectsUserKey{}, obj) steps.SetClientInContext(ctx, r.client) + steps.SetEventRecorderInContext(ctx, r.recorder) if !obj.DeletionTimestamp.IsZero() { return r.Delete(ctx) } diff --git a/operator/cloudscale/provision.go b/operator/cloudscale/provision.go index 4d693eb..552530d 100644 --- a/operator/cloudscale/provision.go +++ b/operator/cloudscale/provision.go @@ -5,8 +5,13 @@ import ( "fmt" pipeline "github.com/ccremer/go-command-pipeline" cloudscalev1 "github.com/vshn/appcat-service-s3/apis/cloudscale/v1" + "github.com/vshn/appcat-service-s3/apis/conditions" "github.com/vshn/appcat-service-s3/operator/steps" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/labels" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "strings" ) @@ -24,34 +29,50 @@ func (p *ObjectsUserPipeline) Run(ctx context.Context) error { pipe := pipeline.NewPipeline(). WithSteps( pipeline.NewStepFromFunc("add finalizer", steps.AddFinalizerFn(ObjectsUserKey{}, userFinalizer)), - pipeline.NewStepFromFunc("validate spec", validateSpec()), pipeline.NewStepFromFunc("create client", CreateCloudscaleClientFn(APIToken)), pipeline.IfOrElse(isObjectsUserIDKnown(), pipeline.NewStepFromFunc("fetch objects user", GetObjectsUserFn()), pipeline.NewPipeline().WithNestedSteps("new user", pipeline.NewStepFromFunc("create objects user", CreateObjectsUserFn()), pipeline.NewStepFromFunc("set user in status", steps.UpdateStatusFn(ObjectsUserKey{})), + pipeline.NewStepFromFunc("emit event", emitSuccessEventFn()), ), ), pipeline.NewStepFromFunc("ensure credential secret", EnsureCredentialSecretFn()), pipeline.NewStepFromFunc("set status condition", markUserReadyFn()), ). - ) + WithFinalizer(errorHandler()) result := pipe.RunWithContext(ctx) - if result.IsFailed() { - // TODO: Add failed condition - // TODO: Emit event + return result.Err() +} + +func errorHandler() pipeline.ResultHandler { + return func(ctx context.Context, result pipeline.Result) error { + if result.IsSuccessful() { + return nil + } + kube := steps.GetClientFromContext(ctx) + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) + log := controllerruntime.LoggerFrom(ctx) + recorder := steps.GetEventRecorderFromContext(ctx) + + meta.SetStatusCondition(&user.Status.Conditions, conditions.NotReady()) + meta.SetStatusCondition(&user.Status.Conditions, conditions.Failed(result.Err())) + err := kube.Status().Update(ctx, user) + if err != nil { + log.V(1).Error(err, "updating status failed") + } + recorder.Event(user, v1.EventTypeWarning, "Failed", result.Err().Error()) return result.Err() } - return nil } -func validateSpec() func(ctx context.Context) error { +func emitSuccessEventFn() func(ctx context.Context) error { return func(ctx context.Context) error { - user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) - if user.Spec.SecretRef == "" { - return fmt.Errorf("spec.secretRef cannot be empty") - } + recorder := steps.GetEventRecorderFromContext(ctx) + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(client.Object) + + recorder.Event(user, v1.EventTypeNormal, "Created", "ObjectsUser successfully created") return nil } } diff --git a/operator/cloudscale/setup.go b/operator/cloudscale/setup.go index 0c428be..11c2bd6 100644 --- a/operator/cloudscale/setup.go +++ b/operator/cloudscale/setup.go @@ -16,6 +16,7 @@ func SetupController(mgr ctrl.Manager) error { For(&cloudscalev1.ObjectsUser{}). WithEventFilter(predicate.Or(predicate.GenerationChangedPredicate{})). Complete(&ObjectsUserReconciler{ - client: mgr.GetClient(), + client: mgr.GetClient(), + recorder: mgr.GetEventRecorderFor(name), }) } diff --git a/operator/steps/context.go b/operator/steps/context.go index 58d8655..01aaffc 100644 --- a/operator/steps/context.go +++ b/operator/steps/context.go @@ -4,6 +4,7 @@ import ( "context" "fmt" pipeline "github.com/ccremer/go-command-pipeline" + "k8s.io/client-go/tools/record" "reflect" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -11,6 +12,9 @@ import ( // clientKey identifies the Kubernetes client in the context. type clientKey struct{} +// eventRecorderKey identifies the Kubernetes event recorder in the context. +type eventRecorderKey struct{} + // SetClientInContext sets the given client in the context. func SetClientInContext(ctx context.Context, c client.Client) { pipeline.StoreInContext(ctx, clientKey{}, c) @@ -21,6 +25,16 @@ func GetClientFromContext(ctx context.Context) client.Client { return GetFromContextOrPanic(ctx, clientKey{}).(client.Client) } +// SetEventRecorderInContext sets the given recorder in the context. +func SetEventRecorderInContext(ctx context.Context, eventRecorder record.EventRecorder) { + pipeline.StoreInContext(ctx, eventRecorderKey{}, eventRecorder) +} + +// GetEventRecorderFromContext returns the recorder from the context. +func GetEventRecorderFromContext(ctx context.Context) record.EventRecorder { + return GetFromContextOrPanic(ctx, eventRecorderKey{}).(record.EventRecorder) +} + // GetFromContextOrPanic returns the object if the key exists. // If the does not exist, then it panics. // May return nil if the key exists but the value actually is nil. From c5c46d0babc9003696c5d795296b92dd11b75e01 Mon Sep 17 00:00:00 2001 From: ccremer Date: Fri, 8 Jul 2022 11:52:13 +0200 Subject: [PATCH 5/7] Improve logging --- .github/workflows/test.yml | 2 +- operator/cloudscale/client.go | 6 +- operator/cloudscale/controller.go | 26 +++++++++ operator/cloudscale/controller_it_test.go | 69 +++++++++++++++++++++++ operator/cloudscale/provision.go | 46 ++++++--------- operator/steps/finalizer_it_test.go | 2 +- test/local.mk | 6 -- 7 files changed, 116 insertions(+), 41 deletions(-) create mode 100644 operator/cloudscale/controller_it_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ae66151..a5b71a6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: ${{ runner.os }}-go- - name: Run tests - run: make test + run: make test-integration - name: Upload code coverage report to Code Climate uses: paambaati/codeclimate-action@v3.0.0 diff --git a/operator/cloudscale/client.go b/operator/cloudscale/client.go index 2abc4c7..bfd2ab8 100644 --- a/operator/cloudscale/client.go +++ b/operator/cloudscale/client.go @@ -49,7 +49,7 @@ func CreateObjectsUserFn() func(ctx context.Context) error { user.Status.UserID = csUser.ID pipeline.StoreInContext(ctx, CloudscaleUserKey{}, csUser) - return logIfNotError(err, log, 1, "created objects user in cloudscale") + return logIfNotError(err, log, 1, "Created objects user in cloudscale") } } @@ -63,7 +63,7 @@ func GetObjectsUserFn() func(ctx context.Context) error { csUser, err := csClient.ObjectsUsers.Get(ctx, user.Status.UserID) pipeline.StoreInContext(ctx, CloudscaleUserKey{}, csUser) - return logIfNotError(err, log, 1, "fetched objects user in cloudscale") + return logIfNotError(err, log, 1, "Fetched objects user in cloudscale") } } @@ -98,7 +98,7 @@ func EnsureCredentialSecretFn() func(ctx context.Context) error { }) pipeline.StoreInContext(ctx, UserCredentialSecretKey{}, secret) - return logIfNotError(err, log, 1, "ensured credential secret", "secretName", user.Spec.SecretRef) + return logIfNotError(err, log, 1, "Ensured credential secret", "secretName", user.Spec.SecretRef) } } diff --git a/operator/cloudscale/controller.go b/operator/cloudscale/controller.go index 8a8ac87..0772c2e 100644 --- a/operator/cloudscale/controller.go +++ b/operator/cloudscale/controller.go @@ -4,7 +4,10 @@ import ( "context" "github.com/go-logr/logr" cloudscalev1 "github.com/vshn/appcat-service-s3/apis/cloudscale/v1" + "github.com/vshn/appcat-service-s3/apis/conditions" "github.com/vshn/appcat-service-s3/operator/steps" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/client-go/tools/record" "time" @@ -13,6 +16,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + controllerruntime "sigs.k8s.io/controller-runtime" ) var userFinalizer = "s3.appcat.vshn.io/user-protection" @@ -79,3 +84,24 @@ func logIfNotError(err error, log logr.Logger, level int, msg string, keysAndVal log.V(level).Info(msg, keysAndValues...) return nil } + +func errorHandler() pipeline.ResultHandler { + return func(ctx context.Context, result pipeline.Result) error { + if result.IsSuccessful() { + return nil + } + kube := steps.GetClientFromContext(ctx) + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) + log := controllerruntime.LoggerFrom(ctx) + recorder := steps.GetEventRecorderFromContext(ctx) + + meta.SetStatusCondition(&user.Status.Conditions, conditions.NotReady()) + meta.SetStatusCondition(&user.Status.Conditions, conditions.Failed(result.Err())) + err := kube.Status().Update(ctx, user) + if err != nil { + log.V(1).Error(err, "updating status failed") + } + recorder.Event(user, v1.EventTypeWarning, "Failed", result.Err().Error()) + return result.Err() + } +} diff --git a/operator/cloudscale/controller_it_test.go b/operator/cloudscale/controller_it_test.go new file mode 100644 index 0000000..3aae447 --- /dev/null +++ b/operator/cloudscale/controller_it_test.go @@ -0,0 +1,69 @@ +//go:build integration + +package cloudscale + +import ( + "context" + "fmt" + pipeline "github.com/ccremer/go-command-pipeline" + "github.com/stretchr/testify/suite" + cloudscalev1 "github.com/vshn/appcat-service-s3/apis/cloudscale/v1" + "github.com/vshn/appcat-service-s3/apis/conditions" + "github.com/vshn/appcat-service-s3/operator/operatortest" + "github.com/vshn/appcat-service-s3/operator/steps" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "testing" +) + +type ControllerSuite struct { + operatortest.Suite +} + +func TestControllerSuite(t *testing.T) { + suite.Run(t, new(ControllerSuite)) +} + +func (ts *ControllerSuite) BeforeTest(suiteName, testName string) { + ts.Context = pipeline.MutableContext(context.Background()) + steps.SetClientInContext(ts.Context, ts.Client) +} + +func (ts *ControllerSuite) Test_ErrorHandler() { + // Arrange + user := &cloudscalev1.ObjectsUser{ + ObjectMeta: metav1.ObjectMeta{Name: "user", Namespace: "namespace"}, + Spec: cloudscalev1.ObjectsUserSpec{SecretRef: "irrelevant-but-required"}} + pipeline.StoreInContext(ts.Context, ObjectsUserKey{}, user) + ts.EnsureNS(user.Namespace) + ts.EnsureResources(user) + + mgr, err := ctrl.NewManager(ts.Config, ctrl.Options{Scheme: ts.Scheme}) + ts.Require().NoError(err) + + recorder := mgr.GetEventRecorderFor("controller") + steps.SetEventRecorderInContext(ts.Context, recorder) + + // Act + result := pipeline.NewPipeline(). + WithSteps(pipeline.NewStepFromFunc("create error", func(ctx context.Context) error { + return fmt.Errorf("error") + })). + WithFinalizer(errorHandler()). + RunWithContext(ts.Context) + + // Assert + updatedUser := &cloudscalev1.ObjectsUser{} + ts.FetchResource(client.ObjectKeyFromObject(user), updatedUser) + ts.Assert().EqualError(result.Err(), `step "create error" failed: error`, "error returned") + ts.Require().Len(user.Status.Conditions, 2, "amount of conditions") + + readyCondition := user.Status.Conditions[0] + ts.Assert().Equal(metav1.ConditionFalse, readyCondition.Status) + ts.Assert().Equal(conditions.TypeReady, readyCondition.Type) + failedCondition := user.Status.Conditions[1] + ts.Assert().Equal(metav1.ConditionTrue, failedCondition.Status) + ts.Assert().Equal(conditions.TypeFailed, failedCondition.Type) + ts.Assert().Equal(`step "create error" failed: error`, failedCondition.Message, "error message in condition") +} diff --git a/operator/cloudscale/provision.go b/operator/cloudscale/provision.go index 552530d..645fa75 100644 --- a/operator/cloudscale/provision.go +++ b/operator/cloudscale/provision.go @@ -26,7 +26,7 @@ func NewObjectsUserPipeline() *ObjectsUserPipeline { // Run executes the business logic. func (p *ObjectsUserPipeline) Run(ctx context.Context) error { - pipe := pipeline.NewPipeline(). + pipe := pipeline.NewPipeline().WithBeforeHooks(debugLogger(ctx)). WithSteps( pipeline.NewStepFromFunc("add finalizer", steps.AddFinalizerFn(ObjectsUserKey{}, userFinalizer)), pipeline.NewStepFromFunc("create client", CreateCloudscaleClientFn(APIToken)), @@ -46,24 +46,10 @@ func (p *ObjectsUserPipeline) Run(ctx context.Context) error { return result.Err() } -func errorHandler() pipeline.ResultHandler { - return func(ctx context.Context, result pipeline.Result) error { - if result.IsSuccessful() { - return nil - } - kube := steps.GetClientFromContext(ctx) +func isObjectsUserIDKnown() func(ctx context.Context) bool { + return func(ctx context.Context) bool { user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) - log := controllerruntime.LoggerFrom(ctx) - recorder := steps.GetEventRecorderFromContext(ctx) - - meta.SetStatusCondition(&user.Status.Conditions, conditions.NotReady()) - meta.SetStatusCondition(&user.Status.Conditions, conditions.Failed(result.Err())) - err := kube.Status().Update(ctx, user) - if err != nil { - log.V(1).Error(err, "updating status failed") - } - recorder.Event(user, v1.EventTypeWarning, "Failed", result.Err().Error()) - return result.Err() + return user.Status.UserID != "" } } @@ -77,10 +63,14 @@ func emitSuccessEventFn() func(ctx context.Context) error { } } -func isObjectsUserIDKnown() func(ctx context.Context) bool { - return func(ctx context.Context) bool { +func markUserReadyFn() func(ctx context.Context) error { + return func(ctx context.Context) error { + kube := steps.GetClientFromContext(ctx) user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) - return user.Status.UserID != "" + + meta.SetStatusCondition(&user.Status.Conditions, conditions.Ready()) + meta.RemoveStatusCondition(&user.Status.Conditions, conditions.TypeFailed) + return kube.Status().Update(ctx, user) } } @@ -93,14 +83,10 @@ func getCommonLabels(instanceName string) labels.Set { } } -func markUserReadyFn() func(ctx context.Context) error { - return func(ctx context.Context) error { - kube := steps.GetClientFromContext(ctx) - user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) - - meta.SetStatusCondition(&user.Status.Conditions, conditions.Ready()) - meta.RemoveStatusCondition(&user.Status.Conditions, conditions.TypeFailed) - return kube.Status().Update(ctx, user) +func debugLogger(ctx context.Context) []pipeline.Listener { + log := controllerruntime.LoggerFrom(ctx) + hook := func(step pipeline.Step) { + log.V(2).Info(`Entering step "` + step.Name + `"`) } - + return []pipeline.Listener{hook} } diff --git a/operator/steps/finalizer_it_test.go b/operator/steps/finalizer_it_test.go index cdb9460..08d54ba 100644 --- a/operator/steps/finalizer_it_test.go +++ b/operator/steps/finalizer_it_test.go @@ -35,7 +35,7 @@ func (ts *FinalizerSuite) Test_AddFinalizer() { }{ "GivenResourceWithoutFinalizer_WhenAddingFinalizer_ThenExpectResourceUpdatedWithAddedFinalizer": { prepare: func(resource client.Object) { - ts.EnsureNS("remove-finalizer") + ts.EnsureNS("add-finalizer") ts.EnsureResources(resource) ts.Assert().Empty(resource.GetFinalizers()) }, diff --git a/test/local.mk b/test/local.mk index e3b3f02..93be168 100644 --- a/test/local.mk +++ b/test/local.mk @@ -10,12 +10,6 @@ $(setup_envtest_bin): $@ $(ENVTEST_ADDITIONAL_FLAGS) use '$(ENVTEST_K8S_VERSION)!' chmod -R +w $(kind_dir)/k8s -ifeq ($(shell uname -s),Darwin) - b64 := base64 -else - b64 := base64 -w0 -endif - .PHONY: local-install local-install: export KUBECONFIG = $(KIND_KUBECONFIG) local-install: kind-load-image install-crd ## Install Operator in local cluster From 6d954ded9f6b8ae50c09a7bd47fcae85abe312bc Mon Sep 17 00:00:00 2001 From: ccremer Date: Fri, 8 Jul 2022 14:19:03 +0200 Subject: [PATCH 6/7] Add readme --- README.md | 30 +++++++++++++++++++ .../appcat-service-s3/templates/secret.yaml | 2 +- test/local.mk | 2 ++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cb7aa28..d83cdaf 100644 --- a/README.md +++ b/README.md @@ -14,3 +14,33 @@ VSHN opinionated operator to deploy S3 resources on supported cloud providers. https://vshn.github.io/appcat-service-s3/ + +## Local Development + +### Requirements + +* `docker` +* `go` +* `helm` +* `kubectl` +* `yq` +* `sed` (or `gsed` for Mac) + +Some other requirements (e.g. `kind`) will be compiled on-the-fly and put in the local cache dir `.kind` as needed. + +### Common make targets + +* `make build` to build the binary and docker image +* `make generate` to (re)generate additional code artifacts +* `make test` run test suite +* `make local-install` to install the operator in local cluster +* `make install-samples` to run the provider in local cluster and apply a sample instance +* `make run-operator` to run the code in operator mode against your current kubecontext + +See all targets with `make help` + +### QuickStart Demonstration + +1. Get an API token cloudscale.ch +1. `export CLOUDSCALE_API_TOKEN=` +1. `make local-install install-samples` diff --git a/charts/appcat-service-s3/templates/secret.yaml b/charts/appcat-service-s3/templates/secret.yaml index f80a5e9..a7bc699 100644 --- a/charts/appcat-service-s3/templates/secret.yaml +++ b/charts/appcat-service-s3/templates/secret.yaml @@ -5,6 +5,6 @@ metadata: name: {{ include "appcat-service-s3.fullname" . }} labels: {{- include "appcat-service-s3.labels" . | nindent 4 }} -data: +stringData: CLOUDSCALE_API_TOKEN: {{ .Values.tokens.cloudscale }} {{- end -}} diff --git a/test/local.mk b/test/local.mk index 93be168..164f991 100644 --- a/test/local.mk +++ b/test/local.mk @@ -13,11 +13,13 @@ $(setup_envtest_bin): .PHONY: local-install local-install: export KUBECONFIG = $(KIND_KUBECONFIG) local-install: kind-load-image install-crd ## Install Operator in local cluster + yq -n e '.tokens.cloudscale=strenv(CLOUDSCALE_API_TOKEN)' > $(kind_dir)/.credentials.yaml helm upgrade --install appcat-service-s3 charts/appcat-service-s3 \ --create-namespace --namespace appcat-service-s3-system \ --set "operator.args[0]=--log-level=2" \ --set "operator.args[1]=operator" \ --set podAnnotations.date="$(shell date)" \ + --values $(kind_dir)/.credentials.yaml \ --wait $(local_install_args) .PHONY: kind-run-operator From 728fed8914f8e0531229618e2f0141a573415e5e Mon Sep 17 00:00:00 2001 From: ccremer Date: Fri, 8 Jul 2022 14:56:17 +0200 Subject: [PATCH 7/7] Simplify step functions --- operator/cloudscale/client.go | 96 +++++++++++++-------------- operator/cloudscale/client_it_test.go | 2 +- operator/cloudscale/provision.go | 46 ++++++------- 3 files changed, 66 insertions(+), 78 deletions(-) diff --git a/operator/cloudscale/client.go b/operator/cloudscale/client.go index bfd2ab8..cf4744f 100644 --- a/operator/cloudscale/client.go +++ b/operator/cloudscale/client.go @@ -34,72 +34,66 @@ func CreateCloudscaleClientFn(apiToken string) func(ctx context.Context) error { // CloudscaleUserKey identifies the User object from cloudscale SDK in the context. type CloudscaleUserKey struct{} -// CreateObjectsUserFn creates a new objects user in the project associated with the API token. -func CreateObjectsUserFn() func(ctx context.Context) error { - return func(ctx context.Context) error { - csClient := steps.GetFromContextOrPanic(ctx, CloudscaleClientKey{}).(*cloudscalesdk.Client) - user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) - log := controllerruntime.LoggerFrom(ctx) +// CreateObjectsUser creates a new objects user in the project associated with the API token. +func CreateObjectsUser(ctx context.Context) error { + csClient := steps.GetFromContextOrPanic(ctx, CloudscaleClientKey{}).(*cloudscalesdk.Client) + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) + log := controllerruntime.LoggerFrom(ctx) - displayName := fmt.Sprintf("%s.%s", user.Namespace, user.Name) + displayName := fmt.Sprintf("%s.%s", user.Namespace, user.Name) - csUser, err := csClient.ObjectsUsers.Create(ctx, &cloudscalesdk.ObjectsUserRequest{ - DisplayName: displayName, - }) - user.Status.UserID = csUser.ID + csUser, err := csClient.ObjectsUsers.Create(ctx, &cloudscalesdk.ObjectsUserRequest{ + DisplayName: displayName, + }) + user.Status.UserID = csUser.ID - pipeline.StoreInContext(ctx, CloudscaleUserKey{}, csUser) - return logIfNotError(err, log, 1, "Created objects user in cloudscale") - } + pipeline.StoreInContext(ctx, CloudscaleUserKey{}, csUser) + return logIfNotError(err, log, 1, "Created objects user in cloudscale") } -// GetObjectsUserFn fetches an existing objects user from the project associated with the API token. -func GetObjectsUserFn() func(ctx context.Context) error { - return func(ctx context.Context) error { - csClient := steps.GetFromContextOrPanic(ctx, CloudscaleClientKey{}).(*cloudscalesdk.Client) - user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) - log := controllerruntime.LoggerFrom(ctx) +// GetObjectsUser fetches an existing objects user from the project associated with the API token. +func GetObjectsUser(ctx context.Context) error { + csClient := steps.GetFromContextOrPanic(ctx, CloudscaleClientKey{}).(*cloudscalesdk.Client) + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) + log := controllerruntime.LoggerFrom(ctx) - csUser, err := csClient.ObjectsUsers.Get(ctx, user.Status.UserID) + csUser, err := csClient.ObjectsUsers.Get(ctx, user.Status.UserID) - pipeline.StoreInContext(ctx, CloudscaleUserKey{}, csUser) - return logIfNotError(err, log, 1, "Fetched objects user in cloudscale") - } + pipeline.StoreInContext(ctx, CloudscaleUserKey{}, csUser) + return logIfNotError(err, log, 1, "Fetched objects user in cloudscale") } // UserCredentialSecretKey identifies the credential Secret in the context. type UserCredentialSecretKey struct{} -// EnsureCredentialSecretFn creates the credential secret. -func EnsureCredentialSecretFn() func(ctx context.Context) error { - return func(ctx context.Context) error { - kube := steps.GetClientFromContext(ctx) - user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) - csUser := steps.GetFromContextOrPanic(ctx, CloudscaleUserKey{}).(*cloudscalesdk.ObjectsUser) - log := controllerruntime.LoggerFrom(ctx) - - secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: user.Spec.SecretRef, Namespace: user.Namespace}} +// EnsureCredentialSecret creates the credential secret. +func EnsureCredentialSecret(ctx context.Context) error { + kube := steps.GetClientFromContext(ctx) + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) + csUser := steps.GetFromContextOrPanic(ctx, CloudscaleUserKey{}).(*cloudscalesdk.ObjectsUser) + log := controllerruntime.LoggerFrom(ctx) - if keyErr := checkUserForKeys(csUser); keyErr != nil { - return keyErr - } + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: user.Spec.SecretRef, Namespace: user.Namespace}} - // See https://www.cloudscale.ch/en/api/v1#objects-users - - _, err := controllerruntime.CreateOrUpdate(ctx, kube, secret, func() error { - secret.Labels = labels.Merge(secret.Labels, getCommonLabels(user.Name)) - if secret.StringData == nil { - secret.StringData = map[string]string{} - } - secret.StringData["AWS_ACCESS_KEY_ID"] = csUser.Keys[0]["access_key"] - secret.StringData["AWS_SECRET_ACCESS_KEY"] = csUser.Keys[0]["secret_key"] - controllerutil.AddFinalizer(secret, userFinalizer) - return controllerutil.SetOwnerReference(user, secret, kube.Scheme()) - }) - - pipeline.StoreInContext(ctx, UserCredentialSecretKey{}, secret) - return logIfNotError(err, log, 1, "Ensured credential secret", "secretName", user.Spec.SecretRef) + if keyErr := checkUserForKeys(csUser); keyErr != nil { + return keyErr } + + // See https://www.cloudscale.ch/en/api/v1#objects-users + + _, err := controllerruntime.CreateOrUpdate(ctx, kube, secret, func() error { + secret.Labels = labels.Merge(secret.Labels, getCommonLabels(user.Name)) + if secret.StringData == nil { + secret.StringData = map[string]string{} + } + secret.StringData["AWS_ACCESS_KEY_ID"] = csUser.Keys[0]["access_key"] + secret.StringData["AWS_SECRET_ACCESS_KEY"] = csUser.Keys[0]["secret_key"] + controllerutil.AddFinalizer(secret, userFinalizer) + return controllerutil.SetOwnerReference(user, secret, kube.Scheme()) + }) + + pipeline.StoreInContext(ctx, UserCredentialSecretKey{}, secret) + return logIfNotError(err, log, 1, "Ensured credential secret", "secretName", user.Spec.SecretRef) } func checkUserForKeys(user *cloudscalesdk.ObjectsUser) error { diff --git a/operator/cloudscale/client_it_test.go b/operator/cloudscale/client_it_test.go index c35e9e7..ea283df 100644 --- a/operator/cloudscale/client_it_test.go +++ b/operator/cloudscale/client_it_test.go @@ -44,7 +44,7 @@ func (ts *CloudscaleClientSuite) Test_EnsureCredentialSecretFn() { ts.EnsureNS(user.Namespace) // Act - err := EnsureCredentialSecretFn()(ts.Context) + err := EnsureCredentialSecret(ts.Context) ts.Require().NoError(err) // Assert diff --git a/operator/cloudscale/provision.go b/operator/cloudscale/provision.go index 645fa75..0a44204 100644 --- a/operator/cloudscale/provision.go +++ b/operator/cloudscale/provision.go @@ -30,48 +30,42 @@ func (p *ObjectsUserPipeline) Run(ctx context.Context) error { WithSteps( pipeline.NewStepFromFunc("add finalizer", steps.AddFinalizerFn(ObjectsUserKey{}, userFinalizer)), pipeline.NewStepFromFunc("create client", CreateCloudscaleClientFn(APIToken)), - pipeline.IfOrElse(isObjectsUserIDKnown(), - pipeline.NewStepFromFunc("fetch objects user", GetObjectsUserFn()), + pipeline.IfOrElse(isObjectsUserIDKnown, + pipeline.NewStepFromFunc("fetch objects user", GetObjectsUser), pipeline.NewPipeline().WithNestedSteps("new user", - pipeline.NewStepFromFunc("create objects user", CreateObjectsUserFn()), + pipeline.NewStepFromFunc("create objects user", CreateObjectsUser), pipeline.NewStepFromFunc("set user in status", steps.UpdateStatusFn(ObjectsUserKey{})), - pipeline.NewStepFromFunc("emit event", emitSuccessEventFn()), + pipeline.NewStepFromFunc("emit event", emitSuccessEvent), ), ), - pipeline.NewStepFromFunc("ensure credential secret", EnsureCredentialSecretFn()), - pipeline.NewStepFromFunc("set status condition", markUserReadyFn()), + pipeline.NewStepFromFunc("ensure credential secret", EnsureCredentialSecret), + pipeline.NewStepFromFunc("set status condition", markUserReady), ). WithFinalizer(errorHandler()) result := pipe.RunWithContext(ctx) return result.Err() } -func isObjectsUserIDKnown() func(ctx context.Context) bool { - return func(ctx context.Context) bool { - user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) - return user.Status.UserID != "" - } +func isObjectsUserIDKnown(ctx context.Context) bool { + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) + return user.Status.UserID != "" } -func emitSuccessEventFn() func(ctx context.Context) error { - return func(ctx context.Context) error { - recorder := steps.GetEventRecorderFromContext(ctx) - user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(client.Object) +func emitSuccessEvent(ctx context.Context) error { + recorder := steps.GetEventRecorderFromContext(ctx) + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(client.Object) - recorder.Event(user, v1.EventTypeNormal, "Created", "ObjectsUser successfully created") - return nil - } + recorder.Event(user, v1.EventTypeNormal, "Created", "ObjectsUser successfully created") + return nil } -func markUserReadyFn() func(ctx context.Context) error { - return func(ctx context.Context) error { - kube := steps.GetClientFromContext(ctx) - user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) +func markUserReady(ctx context.Context) error { + kube := steps.GetClientFromContext(ctx) + user := steps.GetFromContextOrPanic(ctx, ObjectsUserKey{}).(*cloudscalev1.ObjectsUser) - meta.SetStatusCondition(&user.Status.Conditions, conditions.Ready()) - meta.RemoveStatusCondition(&user.Status.Conditions, conditions.TypeFailed) - return kube.Status().Update(ctx, user) - } + meta.SetStatusCondition(&user.Status.Conditions, conditions.Ready()) + meta.RemoveStatusCondition(&user.Status.Conditions, conditions.TypeFailed) + return kube.Status().Update(ctx, user) } func getCommonLabels(instanceName string) labels.Set {