Refactor CLI handling to reduce code duplication and reuse framework for topics, userscram and ACL

molecule/all_auth/converge.yml

@@ -17,6 +17,7 @@
 
     # Enabling Kafka Broker Authentication
     amq_streams_broker_auth_enabled: true
+    amq_streams_broker_acl_enabled: true
     amq_streams_broker_auth_scram_enabled: true
     amq_streams_broker_auth_listeners:
       - AUTHENTICATED:SASL_PLAINTEXT
@@ -59,12 +60,6 @@
   roles:
     - role: amq_streams_zookeeper
   tasks:
-    - name: "Ensure Zookeeper is running and available."
-      ansible.builtin.include_role:
-        name: amq_streams_zookeeper
-      vars:
-        amq_streams_common_skip_download: true
-
     - name: "Ensure Broker is running and available."
       ansible.builtin.include_role:
         name: amq_streams_broker
@@ -93,6 +88,23 @@
       vars:
         topic_name: "{{ topic.name }}"
 
+    - name: "ACL: allow principal on {{ topic.name }} for {{ amq_streams_broker_auth_plain_users[0].username }}"
+      ansible.builtin.include_role:
+        name: amq_streams_broker
+        tasks_from: acl/allow_deny_principal.yml
+      vars:
+        user: "{{ amq_streams_broker_auth_plain_users[0].username }}"
+        topic_name: 'otherTopic'
+
+    - name: "ACL: deny principal on {{ topic.name }} for {{ amq_streams_broker_auth_plain_users[0].username }}"
+      ansible.builtin.include_role:
+        name: amq_streams_broker
+        tasks_from: acl/allow_deny_principal.yml
+      vars:
+        user: "{{ amq_streams_broker_auth_plain_users[0].username }}"
+        acl_switch: deny
+        topic_name: 'otherTopic'
+
     - name: "Delete topics"
       ansible.builtin.include_role:
         name: amq_streams_broker

molecule/default/converge.yml

@@ -19,6 +19,33 @@
       vars:
         amq_streams_common_skip_download: true
 
+    - name: "Ensure AMQ Streams Connect is running and available."
+      ansible.builtin.include_role:
+        name: amq_streams_connect
+      vars:
+        connectors:
+          - { name: "file", path: "connectors/file.yml" }
+
+    - ansible.builtin.wait_for:
+        timeout: 300
+      delegate_to: localhost
+
+    - name: "Validate that deployment is functional."
+      ansible.builtin.include_role:
+        name: amq_streams_zookeeper
+        tasks_from: validate.yml
+
+    - name: "Validate that Broker deployment is functional."
+      ansible.builtin.include_role:
+        name: amq_streams_broker
+        tasks_from: validate.yml
+
+    - name: "Validate that Connect deployment is functional."
+      ansible.builtin.include_role:
+        name: amq_streams_connect
+        tasks_from: validate.yml
+
+  post_tasks:
     - name: "Ensures topics exist."
       ansible.builtin.include_role:
         name: amq_streams_broker
@@ -57,30 +84,3 @@
       vars:
         connectors:
           - { name: "file", path: "connectors/file.yml" }
-  post_tasks:
-    - name: "Display numbers of Zookeeper instances managed by Ansible."
-      ansible.builtin.debug:
-        msg: "Numbers of Zookeeper instances: {{ amq_streams_zookeeper_instance_count }}."
-      when:
-        - amq_streams_zookeeper_instance_count_enabled is defined and amq_streams_zookeeper_instance_count_enabled
-
-    - name: "Display numbers of broker instances managed by Ansible."
-      ansible.builtin.debug:
-        msg: "Numbers of broker instances: {{ amq_streams_broker_instance_count }}."
-      when:
-        - amq_streams_broker_instance_count_enabled is defined and amq_streams_broker_instance_count_enabled
-
-    - name: "Validate that deployment is functional."
-      ansible.builtin.include_role:
-        name: amq_streams_zookeeper
-        tasks_from: validate.yml
-
-    - name: "Validate that Broker deployment is functional."
-      ansible.builtin.include_role:
-        name: amq_streams_broker
-        tasks_from: validate.yml
-
-    - name: "Validate that Connect deployment is functional."
-      ansible.builtin.include_role:
-        name: amq_streams_connect
-        tasks_from: validate.yml

playbooks/describe_topic.yml

@@ -0,0 +1,16 @@
+---
+- name: "Describe topic {{ topic_name }}"
+  hosts: brokers
+  collections:
+    - middleware_automation.amq_streams
+  pre_tasks:
+    - name: "Ensure required parameters has been provided"
+      ansible.builtin.assert:
+        that:
+          - topic_name is defined and topic_name | length > 0
+        quiet: true
+  tasks:
+    - name: "Describe created topic."
+      ansible.builtin.include_role:
+        name: amq_streams_broker
+        tasks_from: topic/describe.yml

roles/amq_streams_broker/defaults/main.yml

@@ -1,6 +1,7 @@
 ---
 amq_streams_broker_create_topic_script: "{{ amq_streams_common_home }}/bin/kafka-topics.sh"
 amq_streams_broker_kafka_config_script: "{{ amq_streams_common_home }}/bin/kafka-configs.sh"
+amq_streams_broker_kafka_acl_script: "{{ amq_streams_common_home }}/bin/kafka-acls.sh"
 amq_streams_broker_enabled: true
 amq_streams_broker_server_start: "{{ amq_streams_common_home }}/bin/kafka-server-start.sh"
 amq_streams_broker_config: "/etc/amq_streams_broker.properties"

roles/amq_streams_broker/tasks/acl/allow_deny_principal.yml

@@ -0,0 +1,16 @@
+---
+- name: "Ensure required parameters are provided."
+  ansible.builtin.assert:
+    that:
+      - user is defined
+      - topic_name is defined
+    quiet: true
+
+- name: "Change ACL on {{ user }} for {{ topic_name }}."
+  block:
+    - name: "Use bootstrap server to create topic {{ topic_name }}"
+      ansible.builtin.include_tasks: cli/bootstrap.yml
+      vars:
+        main_argument: "--{{ acl_switch | default('allow') }}-principal User:{{ user }} --operation Write --topic {{ topic_name }}"
+        bootstrap_operator: "--add"
+        script: "{{ amq_streams_broker_kafka_acl_script }}"

roles/amq_streams_broker/tasks/topic/bootstrap.yml → roles/amq_streams_broker/tasks/cli/bootstrap.yml

@@ -2,12 +2,15 @@
 - name: "Ensure required parameter(s) are provided."
   ansible.builtin.assert:
     that:
-      - amq_streams_broker is defined
-      - amq_streams_broker.topic is defined
-      - amq_streams_broker.topic.script is defined
-      - topic_name is defined
+      - main_argument is defined
     quiet: True
 
+- name: "Set script to {{ amq_streams_broker_create_topic_script }} - if not defined."
+  ansible.builtin.set_fact:
+    script: "{{ amq_streams_broker_create_topic_script }}"
+  when:
+    - not script is defined
+
 - name: "Set bootstrap server host (if not defined)."
   ansible.builtin.set_fact:
     bootstrap_server_host: "{{ amq_streams_broker.bootstrap_server.host }}"
@@ -50,13 +53,23 @@
         src: "{{ amq_streams_broker_admin_cli_config_template }}"
         dest: "{{ amq_streams_broker_admin_cli_config_file }}"
 
-    - name: "Set command-config argument to run operation '{{ bootstrap_operator }}' for topic: '{{ topic_name }}'."
+    - name: "Set command-config argument to run operation '{{ bootstrap_operator }}' for topic: '{{ main_argument }}'."
       ansible.builtin.set_fact:
         bootstrap_command_config_operator_option: "--command-config {{ amq_streams_broker_admin_cli_config_file }}"
 
-- name: "Run operation '{{ bootstrap_operator }}' for topic: '{{ topic_name }}'."
+- name: "Run operation '{{ bootstrap_operator }}' for with: '{{ main_argument }}'."
   block:
-    - name: "Execute create request for topic '{{ topic_name }}'."
-      ansible.builtin.command: "{{ amq_streams_broker.topic.script }} {{ bootstrap_operator }} {{ bootstrap_operator_options | default('') }} {{ bootstrap_command_config_operator_option | default('')}} --topic {{ topic_name }} --bootstrap-server {{ bootstrap_server_host }}:{{ bootstrap_server_port }}"
+    - name: "Build command line"
+      ansible.builtin.set_fact:
+        command_line: "{{ script }} {{ bootstrap_operator }} {{ bootstrap_operator_options | default('') }} {{ main_argument }} {{ bootstrap_command_config_operator_option | default('')}}  --bootstrap-server {{ bootstrap_server_host }}:{{ bootstrap_server_port }}"
+
+    - name: "Print command line for debugging - if requested"
+      ansible.builtin.debug:
+        var: command_line
+      when:
+        - amq_streams_broker_bootstrap_debug is defined and amq_streams_broker_bootstrap_debug
+
+    - name: "Execute {{ script }} request for operation {{ bootstrap_operator }}."
+      ansible.builtin.command: "{{ command_line }}" #"{{ script }} {{ bootstrap_operator }} {{ main_argument }} {{ bootstrap_command_config_operator_option | default('')}}  --bootstrap-server {{ bootstrap_server_host }}:{{ bootstrap_server_port }}"
       register: operation_result
       changed_when: False

roles/amq_streams_broker/tasks/topic/create.yml

@@ -14,8 +14,9 @@
         - topic_replication_factor is defined
 
     - name: "Use bootstrap server to create topic {{ topic_name }}"
-      ansible.builtin.include_tasks: topic/bootstrap.yml
+      ansible.builtin.include_tasks: cli/bootstrap.yml
       vars:
+        main_argument: "--topic {{ topic_name }}"
         bootstrap_operator: "--create"
         bootstrap_operator_options: "{{ bootstrap_create_operator_partitions_option | default('') }} {{ bootstrap_create_operator_replication_factor_option  | default('') }}"
   rescue:

roles/amq_streams_broker/tasks/topic/delete.yml

@@ -2,8 +2,9 @@
 - name: "Delete topic using the bootstrap server."
   block:
     - name: "Use bootstrap server to delete topic {{ topic_name }}"
-      ansible.builtin.include_tasks: topic/bootstrap.yml
+      ansible.builtin.include_tasks: cli/bootstrap.yml
       vars:
+        main_argument: "--topic {{ topic_name }}"
         bootstrap_operator: "--delete"
   rescue:
     - name: "Set expected error message"

roles/amq_streams_broker/tasks/topic/describe.yml

@@ -1,7 +1,8 @@
 ---
 - name: "Get description of topic using the bootstrap server."
-  ansible.builtin.include_tasks: topic/bootstrap.yml
+  ansible.builtin.include_tasks: cli/bootstrap.yml
   vars:
+    main_argument: "--topic {{ topic_name }}"
     bootstrap_operator: "--describe"
 
 - name: "Verify that information on {{ topic_name }} could be retrieved without any errors."

roles/amq_streams_broker/tasks/user-scram/create.yml

@@ -8,8 +8,10 @@
         - user_password is defined
 
     - name: "Use bootstrap server to create user {{ user_username }}"
-      ansible.builtin.include_tasks: user-scram/bootstrap.yml
+      ansible.builtin.include_tasks: cli/bootstrap.yml
       vars:
+        script: "{{ amq_streams_broker_kafka_config_script }}"
+        main_argument: "--entity-type users --entity-name {{ user_username }}"
         bootstrap_operator: "--alter"
         bootstrap_operator_options: "{{ bootstrap_create_operator_password_option | default('') }}"
   rescue:

roles/amq_streams_broker/tasks/user-scram/delete.yml

@@ -8,8 +8,10 @@
         - user_username is defined
 
     - name: "Use bootstrap server to delete user {{ user_username }}"
-      ansible.builtin.include_tasks: user-scram/bootstrap.yml
+      ansible.builtin.include_tasks: cli/bootstrap.yml
       vars:
+        script: "{{ amq_streams_broker_kafka_config_script }}"
+        main_argument: "--entity-type users --entity-name {{ user_username }}"
         bootstrap_operator: "--alter"
         bootstrap_operator_options: "{{ bootstrap_delete_operator_password_option | default('') }}"
   rescue:

roles/amq_streams_broker/tasks/user-scram/describe.yml

@@ -1,7 +1,9 @@
 ---
 - name: "Get description of user using the bootstrap server."
-  ansible.builtin.include_tasks: user-scram/bootstrap.yml
+  ansible.builtin.include_tasks: cli/bootstrap.yml
   vars:
+    main_argument: "--entity-type users --entity-name {{ user_username }}"
+    script: "{{ amq_streams_broker_kafka_config_script }}"
     bootstrap_operator: "--describe"
 
 - name: "Verify that information on {{ user_username }} could be retrieved without any errors."

roles/amq_streams_broker/templates/server.properties.j2

@@ -183,3 +183,9 @@ zookeeper.session.timeout.ms={{ amq_streams_broker_zookeeper_session_timeout_ms
 # We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
 # However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
 group.initial.rebalance.delay.ms={{ amq_streams_broker_group_initial_rebalance_delay_ms }}
+
+{%if amq_streams_broker_acl_enabled is defined and amq_streams_broker_acl_enabled %}
+authorizer.class.name=kafka.security.authorizer.AclAuthorizer
+super.users=User:admin
+allow.everyone.if.no.acl.found=true
+{% endif %}

roles/amq_streams_broker/vars/main.yml

@@ -26,5 +26,3 @@ amq_streams_broker:
   bootstrap_server:
     host: "{{ amq_streams_broker_bootstrap_server_host }}"
     port: "{{ amq_streams_broker_bootstrap_server_port }}"
-  topic:
-    script: "{{ amq_streams_broker_create_topic_script }}"

roles/amq_streams_common/defaults/main.yml

@@ -10,7 +10,7 @@ amq_streams_common_download_group: "{{ amq_streams_common_group | default(omit)
 amq_streams_common_install_dir: /opt
 amq_streams_common_path_to_archive_file: "{{ amq_streams_common_download_dir }}/{{ amq_streams_common_archive_file }}"
 amq_streams_common_download_node: localhost
-amq_streams_common_systctl_update_enable: false
+amq_streams_common_systctl_update_enabled: false
 amq_streams_common_prereqs_dependencies:
   - tar
 amq_streams_common_openjdk_version: 17

roles/amq_streams_common/tasks/os_tuning.yml

@@ -1,24 +1,28 @@
 ---
-- name: "Ensure required parameters are provided."
-  ansible.builtin.assert:
-    that:
-      - sysctl_vm_max_map_count is defined
-    quiet: true
-
-- name: "Update vm.max_map_count to {{ sysctl_vm_max_map_count }}."
-  ansible.posix.sysctl:
-    name: vm.max_map_count
-    value: "{{ sysctl_vm_max_map_count }}"
-    state: present
+- name: "Tuned OS - if requested amq_streams_common_systctl_update_enabled is true."
   when:
-    - amq_streams_common_systctl_update_enable is defined and amq_streams_common_systctl_update_enable
+    - amq_streams_common_systctl_update_enabled is defined and amq_streams_common_systctl_update_enabled
+  block:
+    - name: "Ensure required parameters are provided."
+      ansible.builtin.assert:
+        that:
+          - sysctl_vm_max_map_count is defined
+        quiet: true
 
-- name: "Ensure that {{ ulimit_user }} ulimit is set to {{ ulimit_limit }}."
-  ansible.builtin.template:
-    src: templates/ulimit.j2
-    dest: /etc/security/limits.d/{{ ulimit_user }}
-    owner: root
-    group: root
-    mode: 0755
-  when:
-    - ulimit_limit is defined and ulimit_user is defined
+    - name: "Update vm.max_map_count to {{ sysctl_vm_max_map_count }}."
+      ansible.posix.sysctl:
+        name: vm.max_map_count
+        value: "{{ sysctl_vm_max_map_count }}"
+        state: present
+      when:
+        - amq_streams_common_systctl_update_enable is defined and amq_streams_common_systctl_update_enable
+
+    - name: "Ensure that {{ ulimit_user }} ulimit is set to {{ ulimit_limit }}."
+      ansible.builtin.template:
+        src: templates/ulimit.j2
+        dest: /etc/security/limits.d/{{ ulimit_user }}
+        owner: root
+        group: root
+        mode: 0755
+      when:
+        - ulimit_limit is defined and ulimit_user is defined