diff --git a/controllers/storagecluster/provider_server.go b/controllers/storagecluster/provider_server.go index 8516abc57c..e79edb0d96 100644 --- a/controllers/storagecluster/provider_server.go +++ b/controllers/storagecluster/provider_server.go @@ -45,9 +45,6 @@ func (o *ocsProviderServer) ensureCreated(r *StorageClusterReconciler, instance if !instance.Spec.AllowRemoteStorageConsumers { r.Log.Info("Spec.AllowRemoteStorageConsumers is disabled") - if err := r.verifyNoStorageConsumerExist(instance); err != nil { - return reconcile.Result{}, err - } return o.ensureDeleted(r, instance) } @@ -88,6 +85,10 @@ func (o *ocsProviderServer) ensureDeleted(r *StorageClusterReconciler, instance // NOTE: Do not add the check + if err := r.verifyNoStorageConsumerExist(instance, false /* skip local */); err != nil { + return reconcile.Result{}, err + } + var finalErr error for _, resource := range []client.Object{ diff --git a/controllers/storagecluster/reconcile.go b/controllers/storagecluster/reconcile.go index b4f4880e7b..5ae001c94e 100644 --- a/controllers/storagecluster/reconcile.go +++ b/controllers/storagecluster/reconcile.go @@ -391,6 +391,7 @@ func (r *StorageClusterReconciler) reconcilePhases( // preserve list order objs = []resourceManager{ &ocsProviderServer{}, + &storageClient{}, &backingStorageClasses{}, &ocsTopologyMap{}, &ocsStorageQuota{}, diff --git a/controllers/storagecluster/storageclient.go b/controllers/storagecluster/storageclient.go new file mode 100644 index 0000000000..9fefee2e8a --- /dev/null +++ b/controllers/storagecluster/storageclient.go @@ -0,0 +1,63 @@ +package storagecluster + +import ( + "fmt" + + ocsclientv1a1 "github.com/red-hat-storage/ocs-client-operator/api/v1alpha1" + ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" + "github.com/red-hat-storage/ocs-operator/v4/services" + kerrors "k8s.io/apimachinery/pkg/api/errors" + cutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + storageClientName = "local-storageclient" + + tokenLifetimeInHours = 48 + onboardingPrivateKeyFilePath = "/etc/private-key/key" +) + +type storageClient struct{} + +var _ resourceManager = &storageClient{} + +func (s *storageClient) ensureCreated(r *StorageClusterReconciler, storagecluster *ocsv1.StorageCluster) (reconcile.Result, error) { + + if !storagecluster.Spec.AllowRemoteStorageConsumers { + r.Log.Info("Spec.AllowRemoteStorageConsumers is disabled") + return s.ensureDeleted(r, storagecluster) + } + + storageClient := &ocsclientv1a1.StorageClient{} + storageClient.Name = storageClientName + storageClient.Namespace = r.OperatorNamespace + _, err := cutil.CreateOrUpdate(r.ctx, r.Client, storageClient, func() error { + if storageClient.Status.ConsumerID == "" { + token, err := services.GenerateOnboardingToken(tokenLifetimeInHours, onboardingPrivateKeyFilePath) + if err != nil { + return fmt.Errorf("unable to generate onboarding token: %v", err) + } + storageClient.Spec.OnboardingTicket = token + } + storageClient.Spec.StorageProviderEndpoint = storagecluster.Status.StorageProviderEndpoint + return nil + }) + if err != nil { + r.Log.Error(err, "Failed to create local StorageClient CR") + return reconcile.Result{}, nil + } + + return reconcile.Result{}, nil +} + +func (s *storageClient) ensureDeleted(r *StorageClusterReconciler, _ *ocsv1.StorageCluster) (reconcile.Result, error) { + storageClient := &ocsclientv1a1.StorageClient{} + storageClient.Name = storageClientName + storageClient.Namespace = r.OperatorNamespace + if err := r.Delete(r.ctx, storageClient); err != nil && !kerrors.IsNotFound(err) { + r.Log.Error(err, "Failed to initiate deletion of local StorageClient CR") + return reconcile.Result{}, err + } + return reconcile.Result{}, nil +} diff --git a/controllers/storagecluster/storagecluster_controller.go b/controllers/storagecluster/storagecluster_controller.go index 0e821efc7b..d682970196 100644 --- a/controllers/storagecluster/storagecluster_controller.go +++ b/controllers/storagecluster/storagecluster_controller.go @@ -9,6 +9,7 @@ import ( nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1" "github.com/operator-framework/operator-lib/conditions" + ocsclientv1a1 "github.com/red-hat-storage/ocs-client-operator/api/v1alpha1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" "github.com/red-hat-storage/ocs-operator/v4/controllers/util" @@ -181,7 +182,8 @@ func (r *StorageClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { }, enqueueStorageClusterRequest, ). - Watches(&ocsv1alpha1.StorageConsumer{}, enqueueStorageClusterRequest, builder.WithPredicates(ocsClientOperatorVersionPredicate)) + Watches(&ocsv1alpha1.StorageConsumer{}, enqueueStorageClusterRequest, builder.WithPredicates(ocsClientOperatorVersionPredicate)). + Watches(&ocsclientv1a1.StorageClient{}, enqueueStorageClusterRequest) if os.Getenv("SKIP_NOOBAA_CRD_WATCH") != "true" { builder.Owns(&nbv1.NooBaa{}) } diff --git a/controllers/storagecluster/storagecluster_controller_test.go b/controllers/storagecluster/storagecluster_controller_test.go index 2748e7b5ab..cd63cfe60d 100644 --- a/controllers/storagecluster/storagecluster_controller_test.go +++ b/controllers/storagecluster/storagecluster_controller_test.go @@ -17,6 +17,7 @@ import ( openshiftv1 "github.com/openshift/api/template/v1" conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + ocsclientv1a1 "github.com/red-hat-storage/ocs-client-operator/api/v1alpha1" api "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" "github.com/red-hat-storage/ocs-operator/v4/controllers/defaults" @@ -1213,6 +1214,11 @@ func createFakeScheme(t *testing.T) *runtime.Scheme { assert.Fail(t, "failed to add ocsv1alpha1 scheme") } + err = ocsclientv1a1.AddToScheme(scheme) + if err != nil { + assert.Fail(t, "failed to add ocsclientv1a1 scheme") + } + return scheme } diff --git a/controllers/storagecluster/uninstall_reconciler.go b/controllers/storagecluster/uninstall_reconciler.go index 5827b6fa18..8fb70c692f 100644 --- a/controllers/storagecluster/uninstall_reconciler.go +++ b/controllers/storagecluster/uninstall_reconciler.go @@ -4,11 +4,13 @@ import ( "context" "encoding/json" "fmt" + "strings" nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" "github.com/red-hat-storage/ocs-operator/v4/controllers/defaults" + "github.com/red-hat-storage/ocs-operator/v4/controllers/util" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -270,8 +272,8 @@ func (r *StorageClusterReconciler) reconcileUninstallAnnotations(sc *ocsv1.Stora return false, nil } -// verifyNoStorageConsumerExist verifies there are no storageConsumers on the same namespace -func (r *StorageClusterReconciler) verifyNoStorageConsumerExist(instance *ocsv1.StorageCluster) error { +// verifyNoStorageConsumerExist verifies there are no storageConsumers on the same namespace conditionally +func (r *StorageClusterReconciler) verifyNoStorageConsumerExist(instance *ocsv1.StorageCluster, skipLocalConsumer bool) error { storageConsumers := &ocsv1alpha1.StorageConsumerList{} err := r.Client.List(context.TODO(), storageConsumers, &client.ListOptions{Namespace: instance.Namespace}) @@ -279,6 +281,19 @@ func (r *StorageClusterReconciler) verifyNoStorageConsumerExist(instance *ocsv1. return err } + if len(storageConsumers.Items) == 1 && skipLocalConsumer { + clusterID := util.GetClusterID(r.ctx, r.Client, &r.Log) + if clusterID == "" { + return fmt.Errorf("failed to get cluster id") + } + + consumer := &storageConsumers.Items[0] + if strings.HasSuffix(consumer.Name, clusterID) { + r.Log.Info("Only local storage consumer exist") + return nil + } + } + if len(storageConsumers.Items) != 0 { err = fmt.Errorf("Failed to cleanup provider resources. StorageConsumers are present in the %s namespace. "+ "Offboard all consumer clusters for the provider cleanup to proceed", instance.Namespace) @@ -296,7 +311,7 @@ func (r *StorageClusterReconciler) deleteResources(sc *ocsv1.StorageCluster) (re // we do not check if it is a provider before checking storageConsumers CR because of corner case // where user can mark instance.Spec.AllowRemoteStorageConsumers as false and mark CR for deletion immediately. // Which will trigger a deletion without having instance.Spec.AllowRemoteStorageConsumers as true. - err := r.verifyNoStorageConsumerExist(sc) + err := r.verifyNoStorageConsumerExist(sc, true /* skip local */) if err != nil { return reconcile.Result{}, err } @@ -325,9 +340,10 @@ func (r *StorageClusterReconciler) deleteResources(sc *ocsv1.StorageCluster) (re } objs := []resourceManager{ + &storageClient{}, &ocsExternalResources{}, - &ocsProviderServer{}, &ocsNoobaaSystem{}, + &ocsProviderServer{}, &ocsCephRGWRoutes{}, &ocsCephObjectStoreUsers{}, &ocsCephObjectStores{}, diff --git a/controllers/storagecluster/uninstall_reconciler_test.go b/controllers/storagecluster/uninstall_reconciler_test.go index 551c5f4b0a..903b660e4f 100644 --- a/controllers/storagecluster/uninstall_reconciler_test.go +++ b/controllers/storagecluster/uninstall_reconciler_test.go @@ -843,7 +843,7 @@ func TestVerifyNoStorageConsumerExist(t *testing.T) { err := r.Client.Create(context.TODO(), storageConsumer) assert.NoError(t, err) - err = r.verifyNoStorageConsumerExist(instance) + err = r.verifyNoStorageConsumerExist(instance, false /* skip local */) assert.Error(t, err) expectedErr := fmt.Errorf("Failed to cleanup provider resources. StorageConsumers are present in the namespace. " + "Offboard all consumer clusters for the provider cleanup to proceed") @@ -854,7 +854,7 @@ func TestVerifyNoStorageConsumerExist(t *testing.T) { r, instance := createSetup() - err := r.verifyNoStorageConsumerExist(instance) + err := r.verifyNoStorageConsumerExist(instance, false /* skip local */) assert.NoError(t, err) }) } diff --git a/go.mod b/go.mod index 8ed0895cf1..85339df839 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/operator-framework/operator-lifecycle-manager v0.26.0 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.70.0 github.com/prometheus-operator/prometheus-operator/pkg/client v0.70.0 + github.com/red-hat-storage/ocs-client-operator v0.0.0-20240216124345-1b9b7fb23b8d github.com/red-hat-storage/ocs-operator/api/v4 v4.0.0-00010101000000-000000000000 github.com/rook/rook/pkg/apis v0.0.0-20231215165123-32de0fb5f69b github.com/stretchr/testify v1.8.4 diff --git a/main.go b/main.go index 711a6a5381..2fa40f2947 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,7 @@ import ( apiv2 "github.com/operator-framework/api/pkg/operators/v2" "github.com/operator-framework/operator-lib/conditions" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + ocsclientv1a1 "github.com/red-hat-storage/ocs-client-operator/api/v1alpha1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" "github.com/red-hat-storage/ocs-operator/v4/controllers/namespace" @@ -92,6 +93,7 @@ func init() { utilruntime.Must(clusterv1alpha1.AddToScheme(scheme)) utilruntime.Must(operatorsv1alpha1.AddToScheme(scheme)) utilruntime.Must(nadscheme.AddToScheme(scheme)) + utilruntime.Must(ocsclientv1a1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } diff --git a/services/tokengenerator.go b/services/tokengenerator.go new file mode 100644 index 0000000000..73347b9156 --- /dev/null +++ b/services/tokengenerator.go @@ -0,0 +1,73 @@ +package services + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "fmt" + "os" + "time" + + "github.com/google/uuid" +) + +// GenerateOnboardingToken generates a token valid for a duration of "tokenLifetimeInHours". +// The token content is predefined and signed by the private key which'll be read from supplied "privateKeyPath". +func GenerateOnboardingToken(tokenLifetimeInHours int, privateKeyPath string) (string, error) { + tokenExpirationDate := time.Now(). + Add(time.Duration(tokenLifetimeInHours) * time.Hour). + Unix() + + payload, err := json.Marshal(OnboardingTicket{ + ID: uuid.New().String(), + ExpirationDate: tokenExpirationDate, + }) + if err != nil { + return "", fmt.Errorf("failed to marshal the payload: %v", err) + } + + encodedPayload := base64.StdEncoding.EncodeToString(payload) + // Before signing, we need to hash our message + // The hash is what we actually sign + msgHash := sha256.New() + _, err = msgHash.Write(payload) + if err != nil { + return "", fmt.Errorf("failed to hash onboarding token payload: %v", err) + } + + privateKey, err := readAndDecodePrivateKey(privateKeyPath) + if err != nil { + return "", fmt.Errorf("failed to read and decode private key: %v", err) + } + + msgHashSum := msgHash.Sum(nil) + // In order to generate the signature, we provide a random number generator, + // our private key, the hashing algorithm that we used, and the hash sum + // of our message + signature, err := rsa.SignPKCS1v15(rand.Reader, privateKey, crypto.SHA256, msgHashSum) + if err != nil { + return "", fmt.Errorf("failed to sign private key: %v", err) + } + + encodedSignature := base64.StdEncoding.EncodeToString(signature) + return fmt.Sprintf("%s.%s", encodedPayload, encodedSignature), nil +} + +func readAndDecodePrivateKey(privateKeyPath string) (*rsa.PrivateKey, error) { + pemString, err := os.ReadFile(privateKeyPath) + if err != nil { + return nil, fmt.Errorf("failed to read private key: %v", err) + } + + Block, _ := pem.Decode(pemString) + privateKey, err := x509.ParsePKCS1PrivateKey(Block.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse private key: %v", err) + } + return privateKey, nil +} diff --git a/services/ux-backend/handlers/onboardingtokens/handler.go b/services/ux-backend/handlers/onboardingtokens/handler.go index 8560b8428c..fe9112e396 100644 --- a/services/ux-backend/handlers/onboardingtokens/handler.go +++ b/services/ux-backend/handlers/onboardingtokens/handler.go @@ -1,20 +1,9 @@ package onboardingtokens import ( - "crypto" - "crypto/rand" - "crypto/rsa" - "crypto/sha256" - "crypto/x509" - "encoding/base64" - "encoding/json" - "encoding/pem" "fmt" "net/http" - "os" - "time" - "github.com/google/uuid" "github.com/red-hat-storage/ocs-operator/v4/services" "github.com/red-hat-storage/ocs-operator/v4/services/ux-backend/handlers" "k8s.io/klog/v2" @@ -34,7 +23,7 @@ func HandleMessage(w http.ResponseWriter, r *http.Request, tokenLifetimeInHours } func handlePost(w http.ResponseWriter, tokenLifetimeInHours int) { - if onboardingToken, err := generateOnboardingToken(tokenLifetimeInHours); err != nil { + if onboardingToken, err := services.GenerateOnboardingToken(tokenLifetimeInHours, onboardingPrivateKeyFilePath); err != nil { klog.Errorf("failed to get onboardig token: %v", err) w.WriteHeader(http.StatusInternalServerError) w.Header().Set("Content-Type", handlers.ContentTypeTextPlain) @@ -63,57 +52,3 @@ func handleUnsupportedMethod(w http.ResponseWriter, r *http.Request) { klog.Errorf("failed write data to response writer: %v", err) } } - -func generateOnboardingToken(tokenLifetimeInHours int) (string, error) { - tokenExpirationDate := time.Now(). - Add(time.Duration(tokenLifetimeInHours) * time.Hour). - Unix() - - payload, err := json.Marshal(services.OnboardingTicket{ - ID: uuid.New().String(), - ExpirationDate: tokenExpirationDate, - }) - if err != nil { - return "", fmt.Errorf("failed to marshal the payload: %v", err) - } - - encodedPayload := base64.StdEncoding.EncodeToString(payload) - // Before signing, we need to hash our message - // The hash is what we actually sign - msgHash := sha256.New() - _, err = msgHash.Write(payload) - if err != nil { - return "", fmt.Errorf("failed to hash onboarding token payload: %v", err) - } - - privateKey, err := readAndDecodeOnboardingPrivateKey() - if err != nil { - return "", fmt.Errorf("failed to read and decode private key: %v", err) - } - - msgHashSum := msgHash.Sum(nil) - signature, err := rsa.SignPKCS1v15(rand.Reader, privateKey, crypto.SHA256, msgHashSum) - if err != nil { - return "", fmt.Errorf("failed to sign private key: %v", err) - } - - encodedSignature := base64.StdEncoding.EncodeToString(signature) - return fmt.Sprintf("%s.%s", encodedPayload, encodedSignature), nil -} - -func readAndDecodeOnboardingPrivateKey() (*rsa.PrivateKey, error) { - pemString, err := os.ReadFile(onboardingPrivateKeyFilePath) - if err != nil { - return nil, fmt.Errorf("failed to read onboarding private key: %v", err) - } - - // In order to generate the signature, we provide a random number generator, - // our private key, the hashing algorithm that we used, and the hash sum - // of our message - Block, _ := pem.Decode(pemString) - privateKey, err := x509.ParsePKCS1PrivateKey(Block.Bytes) - if err != nil { - return nil, fmt.Errorf("failed to parse private key: %v", err) - } - return privateKey, nil -}