local_files auth fix (#232)

* Merged main
Added ExternalRegions.Seeds test

* `local_files` auth fixes

* Reverted skaffold.yaml changes

* Updated to latest mockgen

* Updated gitignore to include intellij project files
  and config samples
Removed gitignore of node_modules as there are
  0 nodes files in this project as of now

* Updated mocks after updating mockgen

* Mainly committing so we can talk about this approach to passing JMX credentials around

.gitignore

@@ -25,11 +25,13 @@ tiller
 
 # editor and IDE paraphernalia
 .idea
+**/*.iml
 *.swp
 *.swo
 *~
 
-node_modules
+# config samples
+config/samples
 
 # macOS custom attributes
 **/.DS_Store

api/v1alpha1/cassandracluster_types.go

@@ -41,6 +41,8 @@ const (
 	CassandraDefaultPassword       = "cassandra"
 	CassandraOperatorAdminRole     = "admin-role"
 	CassandraOperatorAdminPassword = "admin-password"
+	CassandraOperatorJmxUsername   = "jmx-username"
+	CassandraOperatorJmxPassword   = "jmx-password"
 	CassandraLocalhost             = "127.0.0.1"
 
 	ProberServicePort    = 80

cassandra/.bashrc

@@ -1,4 +1,8 @@
 if [[ -f /etc/cassandra-auth-config/admin-role ]]; then
-    alias nodetool="nodetool $TLS_ARG -u \"$(cat /etc/cassandra-auth-config/admin-role)\" -pw \"$(cat /etc/cassandra-auth-config/admin-password)\"";
-    alias cqlsh="cqlsh $TLS_ARG --cqlshrc /etc/cassandra-auth-config/cqlshrc";
+  if [[ -f /etc/cassandra-auth-config/jmxremote.password ]]; then
+    alias nodetool='nodetool $TLS_ARG -u "$(cat /etc/cassandra-auth-config/jmxremote.password | cut -f1 -d'"'"' '"'"')" -pw "$(cat /etc/cassandra-auth-config/jmxremote.password | cut -f2 -d'"'"' '"'"')"'
+  else
+    alias nodetool='nodetool $TLS_ARG -u "$(cat /etc/cassandra-auth-config/admin-role)" -pw "$(cat /etc/cassandra-auth-config/admin-password)"'
+  fi
+  alias cqlsh='cqlsh $TLS_ARG -u "$(cat /etc/cassandra-auth-config/admin-role)" -p "$(cat /etc/cassandra-auth-config/admin-password)"'
 fi

controllers/admin_auth.go

@@ -17,7 +17,6 @@ import (
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
@@ -89,40 +88,36 @@ func (r *CassandraClusterReconciler) reconcileAdminAuth(ctx context.Context, cc
 }
 
 func (r *CassandraClusterReconciler) createClusterAdminSecrets(ctx context.Context, cc *dbv1alpha1.CassandraCluster) error {
-	baseAdminSecret := &v1.Secret{}
-	err := r.Get(ctx, types.NamespacedName{Namespace: cc.Namespace, Name: cc.Spec.AdminRoleSecretName}, baseAdminSecret)
+	baseAdminSecret, err := r.adminRoleSecret(ctx, cc)
 	if err != nil {
-		return errors.Wrap(err, "can't get base admin secret")
+		return err
 	}
 
-	secretRoleName := string(baseAdminSecret.Data[dbv1alpha1.CassandraOperatorAdminRole])
-	secretRolePassword := string(baseAdminSecret.Data[dbv1alpha1.CassandraOperatorAdminPassword])
-
-	if len(secretRoleName) == 0 || len(secretRolePassword) == 0 {
-		return errors.New("admin role or password is empty")
+	secretRoleName, secretRolePassword, err := extractCredentials(baseAdminSecret)
+	if err != nil {
+		return err
 	}
 
 	desiredRoleName := dbv1alpha1.CassandraDefaultRole
 	desiredRolePassword := dbv1alpha1.CassandraDefaultPassword
+	desiredSecretData := baseAdminSecret.Data
 
-	if cc.Spec.Cassandra.Persistence.Enabled {
-		pvcs := &v1.PersistentVolumeClaimList{}
-		err = r.List(ctx, pvcs, client.InNamespace(cc.Namespace), client.MatchingLabels(labels.ComponentLabels(cc, dbv1alpha1.CassandraClusterComponentCassandra)))
-		if err != nil {
-			return errors.Wrap(err, "can't get pvcs")
-		}
-
-		if len(pvcs.Items) > 0 { // cluster existed before. Use the credentials from the provided secret to recreate the cluster.
-			r.Log.Info("PVCs found. Assuming cluster existed before. Using credentials from secret %s", cc.Spec.AdminRoleSecretName)
-			desiredRoleName = secretRoleName
-			desiredRolePassword = secretRolePassword
-		}
+	cqlClient, err := r.CqlClient(newCassandraConfig(cc, secretRoleName, secretRolePassword))
+	if err == nil {
+		r.Log.Info("Admin role has already been initialized")
+		cqlClient.CloseSession()
+		desiredRoleName = secretRoleName
+		desiredRolePassword = secretRolePassword
 	}
 
-	desiredSecretData := baseAdminSecret.Data
 	desiredSecretData[dbv1alpha1.CassandraOperatorAdminRole] = []byte(desiredRoleName)
 	desiredSecretData[dbv1alpha1.CassandraOperatorAdminPassword] = []byte(desiredRolePassword)
 
+	if cc.Spec.JMX.Authentication == jmxAuthenticationLocalFiles {
+		desiredSecretData[dbv1alpha1.CassandraOperatorJmxUsername] = []byte(secretRoleName)
+		desiredSecretData[dbv1alpha1.CassandraOperatorJmxPassword] = []byte(secretRolePassword)
+	}
+
 	err = r.reconcileAdminSecrets(ctx, cc, desiredSecretData)
 	if err != nil {
 		return errors.Wrap(err, "failed to reconcile active admin secrets")
@@ -131,6 +126,24 @@ func (r *CassandraClusterReconciler) createClusterAdminSecrets(ctx context.Conte
 	return nil
 }
 
+func extractCredentials(baseAdminSecret *v1.Secret) (string, string, error) {
+	secretRoleName := string(baseAdminSecret.Data[dbv1alpha1.CassandraOperatorAdminRole])
+	secretRolePassword := string(baseAdminSecret.Data[dbv1alpha1.CassandraOperatorAdminPassword])
+	if len(secretRoleName) == 0 || len(secretRolePassword) == 0 {
+		return "", "", errors.New("admin role or password is empty")
+	}
+	return secretRoleName, secretRolePassword, nil
+}
+
+func (r *CassandraClusterReconciler) adminRoleSecret(ctx context.Context, cc *dbv1alpha1.CassandraCluster) (*v1.Secret, error) {
+	baseAdminSecret := &v1.Secret{}
+	err := r.Get(ctx, types.NamespacedName{Namespace: cc.Namespace, Name: cc.Spec.AdminRoleSecretName}, baseAdminSecret)
+	if err != nil {
+		return nil, errors.Wrap(err, "can't get base admin secret")
+	}
+	return baseAdminSecret, nil
+}
+
 func (r *CassandraClusterReconciler) handleAdminRoleChange(ctx context.Context, cc *dbv1alpha1.CassandraCluster, actualBaseAdminSecret, actualActiveAdminSecret *v1.Secret) error {
 	r.Log.Info("Updating admin role")
 	cassandraOperatorAdminRole := string(actualActiveAdminSecret.Data[dbv1alpha1.CassandraOperatorAdminRole])
@@ -166,6 +179,11 @@ func (r *CassandraClusterReconciler) handleAdminRoleChange(ctx context.Context,
 	r.Log.Info("Logged in successfully with new credentials. Updating active admin secret.")
 	defer cqlClientTestCon.CloseSession()
 
+	if cc.Spec.JMX.Authentication == jmxAuthenticationLocalFiles {
+		actualBaseAdminSecret.Data[dbv1alpha1.CassandraOperatorJmxUsername] = []byte(newCassandraOperatorAdminRole)
+		actualBaseAdminSecret.Data[dbv1alpha1.CassandraOperatorJmxPassword] = []byte(newCassandraOperatorAdminPassword)
+	}
+
 	r.Events.Normal(cc, events.EventAdminRoleChanged, "admin role has been successfully changed")
 	err = r.reconcileAdminSecrets(ctx, cc, actualBaseAdminSecret.Data)
 	if err != nil {
@@ -258,7 +276,7 @@ func (r *CassandraClusterReconciler) reconcileActiveAdminSecret(ctx context.Cont
 	return nil
 }
 
-func (r *CassandraClusterReconciler) reconcileAdminAuthConfigSecret(ctx context.Context, cc *dbv1alpha1.CassandraCluster, cassandraAdminRole, cassandraAdminPassword string) error {
+func (r *CassandraClusterReconciler) reconcileAdminAuthConfigSecret(ctx context.Context, cc *dbv1alpha1.CassandraCluster, secretData map[string][]byte) error {
 	desiredAdminAuthConfigSecret := &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      names.AdminAuthConfigSecret(cc.Name),
@@ -270,16 +288,18 @@ func (r *CassandraClusterReconciler) reconcileAdminAuthConfigSecret(ctx context.
 
 	data := make(map[string][]byte)
 
+	cassandraAdminRole := string(secretData[dbv1alpha1.CassandraOperatorAdminRole])
+	cassandraAdminPassword := string(secretData[dbv1alpha1.CassandraOperatorAdminPassword])
+
 	if cc.Spec.JMX.Authentication == jmxAuthenticationLocalFiles {
-		data["jmxremote.password"] = []byte(fmt.Sprintf("%s %s\n", cassandraAdminRole, cassandraAdminPassword))
+		jmxUsername := string(secretData[dbv1alpha1.CassandraOperatorJmxUsername])
+		jmxPassword := string(secretData[dbv1alpha1.CassandraOperatorJmxPassword])
+		data["jmxremote.password"] = []byte(fmt.Sprintf("%s %s\n", jmxUsername, jmxPassword))
 		// jmxremote.access file is not hot-reload in runtime, so we need to set the cassandra role before the start
 		data["jmxremote.access"] = []byte(fmt.Sprintf(`%s readwrite \
 create javax.management.monitor.*, javax.management.timer.* \
 unregister
-%s readwrite \
-create javax.management.monitor.*, javax.management.timer.* \
-unregister
-`, cassandraAdminRole, dbv1alpha1.CassandraOperatorAdminRole))
+`, jmxUsername))
 	}
 
 	cqlshConfig := fmt.Sprintf(`

controllers/prober.go

@@ -233,6 +233,10 @@ func (r *CassandraClusterReconciler) reconcileProberService(ctx context.Context,
 }
 
 func proberContainer(cc *dbv1alpha1.CassandraCluster) v1.Container {
+	adminSecret := names.ActiveAdminSecret(cc.Name)
+	if cc.Spec.JMX.Authentication == jmxAuthenticationLocalFiles {
+		adminSecret = cc.Spec.AdminRoleSecretName
+	}
 	return v1.Container{
 		Name:            "prober",
 		Image:           cc.Spec.Prober.Image,
@@ -245,7 +249,7 @@ func proberContainer(cc *dbv1alpha1.CassandraCluster) v1.Container {
 			{Name: "SERVER_PORT", Value: strconv.Itoa(dbv1alpha1.ProberContainerPort)},
 			{Name: "JMX_POLL_PERIOD_SECONDS", Value: "10"},
 			{Name: "JMX_PORT", Value: fmt.Sprintf("%d", dbv1alpha1.JmxPort)},
-			{Name: "ADMIN_SECRET_NAME", Value: names.ActiveAdminSecret(cc.Name)},
+			{Name: "ADMIN_SECRET_NAME", Value: adminSecret},
 		},
 		Ports: []v1.ContainerPort{
 			{

controllers/role_admin.go

@@ -36,6 +36,11 @@ func (r *CassandraClusterReconciler) reconcileAdminRole(ctx context.Context, cc
 		}
 	}
 
+	if cc.Spec.JMX.Authentication == jmxAuthenticationLocalFiles {
+		adminRoleSecret.Data[dbv1alpha1.CassandraOperatorJmxUsername] = []byte(cassandraOperatorAdminRole)
+		adminRoleSecret.Data[dbv1alpha1.CassandraOperatorJmxPassword] = []byte(cassandraOperatorAdminPassword)
+	}
+
 	r.Log.Debug("Establishing cql session with role " + cassandraOperatorAdminRole)
 	cqlClient, err := r.CqlClient(newCassandraConfig(cc, cassandraOperatorAdminRole, cassandraOperatorAdminPassword))
 	if err == nil { // operator admin role exists
@@ -73,7 +78,6 @@ func (r *CassandraClusterReconciler) reconcileAdminRole(ctx context.Context, cc
 	})
 
 	r.Events.Normal(cc, events.EventAdminRoleCreated, "secure admin role is created")
-
 	err = r.reconcileAdminSecrets(ctx, cc, adminRoleSecret.Data)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to update admin secrets with new password")
@@ -118,10 +122,7 @@ func (r *CassandraClusterReconciler) reconcileAdminSecrets(ctx context.Context,
 		return errors.Wrap(err, "failed to update active admin secret")
 	}
 
-	newOperatorAdminRole := string(secretData[dbv1alpha1.CassandraOperatorAdminRole])
-	newOperatorAdminPassword := string(secretData[dbv1alpha1.CassandraOperatorAdminPassword])
-
-	if err = r.reconcileAdminAuthConfigSecret(ctx, cc, newOperatorAdminRole, newOperatorAdminPassword); err != nil {
+	if err = r.reconcileAdminAuthConfigSecret(ctx, cc, secretData); err != nil {
 		return errors.Wrap(err, "failed to reconcile admin auth secret")
 	}
 

skaffold.yaml

@@ -27,7 +27,7 @@ build:
       template: "{{.USER}}"
   local:
     push: true
-    concurrency: 1
+    concurrency: 4
 deploy:
   helm:
     releases: