Merge pull request #220 from stackhpc/upstream/yoga-2023-12-11

Synchronise yoga with upstream
stackhpc · Dec 11, 2023 · 958737b · 958737b
2 parents 72475f7 + 7229b2b
commit 958737b
Show file tree

Hide file tree

Showing 25 changed files with 257 additions and 43 deletions.
diff --git a/ansible/group_vars/all/infra-vms b/ansible/group_vars/all/infra-vms
@@ -48,7 +48,7 @@ infra_vm_root_format: qcow2
 # or
 # https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2
 # when os_distribution is "rocky" and os_release is "9"
-# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2"
+# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2"
 # otherwise.
 infra_vm_root_image: >-
   {%- if os_distribution == 'ubuntu' %}
@@ -58,7 +58,7 @@ infra_vm_root_image: >-
   {%- elif os_distribution == 'rocky' and os_release == '9' %}
   https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2
   {%- else -%}
-  https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2
+  https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2
   {%- endif %}
 
 # Capacity of the infra VM data volume.

diff --git a/ansible/group_vars/all/kolla b/ansible/group_vars/all/kolla
@@ -203,11 +203,11 @@ overcloud_container_image_regex_map:
     enabled: "{{ kolla_enable_multipathd | bool }}"
   - regex: ^murano
     enabled: "{{ kolla_enable_murano | bool }}"
-  - regex: neutron-server
+  - regex: "neutron-\\(server\\|metadata-agent\\)"
     enabled: "{{ kolla_enable_neutron | bool }}"
   # Neutron SFC agent not currently supported on CentOS binary builds.
-  - regex: "neutron-\\(dhcp\\|l3\\|metadata\\|linuxbridge\\|openvswitch\\)-agent"
-    enabled: "{{ kolla_enable_neutron | bool }}"
+  - regex: "neutron-\\(dhcp\\|l3\\|linuxbridge\\|openvswitch\\)-agent"
+    enabled: "{{ kolla_enable_neutron | bool and not kolla_enable_ovn | bool}}"
   - regex: neutron-mlnx-agent
     enabled: "{{ kolla_enable_neutron_mlnx | bool }}"
   - regex: neutron-sriov-agent

diff --git a/ansible/group_vars/all/seed-vm b/ansible/group_vars/all/seed-vm
@@ -48,7 +48,7 @@ seed_vm_root_format: qcow2
 # or
 # https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2
 # when os_distribution is "rocky" and os_release is "9"
-# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2"
+# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2"
 # otherwise.
 seed_vm_root_image: >-
   {%- if os_distribution == 'ubuntu' %}
@@ -58,7 +58,7 @@ seed_vm_root_image: >-
   {%- elif os_distribution == 'rocky' and os_release == '9' %}
   https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2
   {%- else -%}
-  https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2
+  https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2
   {%- endif %}
 
 # Capacity of the seed VM data volume.

diff --git a/ansible/kayobe-ansible-user.yml b/ansible/kayobe-ansible-user.yml
@@ -13,28 +13,23 @@
     - kayobe-ansible-user
   tasks:
     - name: Check whether the host is accessible via SSH
-      local_action:
-        module: command ssh -o BatchMode=yes -p {{ ssh_port }} {{ ssh_user }}@{{ ssh_host }} hostname
-      failed_when: false
+      raw: hostname
+      ignore_unreachable: true
       changed_when: false
       check_mode: no
       register: ssh_result
-      vars:
-        ssh_user: "{{ ansible_user }}"
-        ssh_host: "{{ ansible_host | default(inventory_hostname) }}"
-        ssh_port: "{{ ansible_ssh_port | default('22') }}"
 
     - name: Group hosts requiring kayobe user bootstrapping
       group_by:
-        key: kayobe_user_bootstrap_required_{{ ssh_result.rc != 0 }}
+        key: kayobe_user_bootstrap_required_{{ ssh_result.unreachable | default(false) }}
       changed_when: false
 
     - name: Display a message when bootstrapping is required
       debug:
         msg: >
           Cannot access host via SSH using Kayobe Ansible user account -
           attempting bootstrap
-      when: ssh_result.rc != 0
+      when: ssh_result.unreachable | default(false)
 
 - name: Ensure python is installed
   hosts: kayobe_user_bootstrap_required_True

diff --git a/ansible/kolla-ansible.yml b/ansible/kolla-ansible.yml
@@ -97,7 +97,6 @@
         kolla_inspector_default_gateway: "{{ inspection_net_name | net_inspection_gateway or inspection_net_name | net_gateway }}"
         kolla_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}"
         kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}"
-        kolla_enable_host_ntp: false
         docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}"
         kolla_globals_paths_extra:
           - "{{ kayobe_config_path }}"

diff --git a/ansible/kolla-bifrost-hostvars.yml b/ansible/kolla-bifrost-hostvars.yml
@@ -22,7 +22,7 @@
       # If the admin network does not have a gateway defined and seed SNAT is
       # enabled, use the seed as a gateway to allow external access until other
       # networks have been configured. Otherwise, do not set any gateway.
-      ipv4_gateway: "{{ admin_oc_net_name | net_gateway or admin_oc_net_name | net_ip(seed_host) if seed_enable_snat | bool }}"
+      ipv4_gateway: "{{ (admin_oc_net_name | net_gateway) or (admin_oc_net_name | net_ip(seed_host) if seed_enable_snat | bool) }}"
       ipv4_nameserver: "{{ resolv_nameservers }}"
       network_mtu: "{{ admin_oc_net_name | net_mtu or '1500' }}"
       vlan_id: "{{ '' if admin_oc_net_name == provision_oc_net_name else (admin_oc_net_name | net_vlan) }}"

diff --git a/ansible/roles/kolla-ansible/defaults/main.yml b/ansible/roles/kolla-ansible/defaults/main.yml
@@ -278,12 +278,6 @@ kolla_ansible_custom_passwords: {}
 kolla_external_tls_cert:
 kolla_internal_tls_cert:
 
-###############################################################################
-# NTP
-
-# Whether to enable the NTP daemon.
-kolla_enable_host_ntp:
-
 ###############################################################################
 # Docker configuration.
 

diff --git a/ansible/roles/kolla-ansible/templates/kolla/globals.yml b/ansible/roles/kolla-ansible/templates/kolla/globals.yml
@@ -554,10 +554,6 @@ grafana_admin_username: "{{ grafana_local_admin_user_name }}"
 # Ansible to repeat this.
 change_selinux: false
 
-{% if kolla_enable_host_ntp is not none %}
-enable_host_ntp: {{ kolla_enable_host_ntp | bool }}
-{% endif %}
-
 # Kayobe performs creation of the Kolla Ansible user account, so there is no
 # need for Kolla Ansible to repeat this.
 create_kolla_user: false

diff --git a/ansible/roles/kolla-ansible/vars/main.yml b/ansible/roles/kolla-ansible/vars/main.yml
@@ -109,6 +109,7 @@ kolla_feature_flags:
   - gnocchi
   - gnocchi_statsd
   - grafana
+  - grafana_external
   - hacluster
   - haproxy
   - haproxy_memcached
@@ -148,6 +149,7 @@ kolla_feature_flags:
   - keystone_federation
   - keystone_horizon_policy_file
   - kibana
+  - kibana_external
   - kuryr
   - loadbalancer
   - magnum
@@ -160,6 +162,8 @@ kolla_feature_flags:
   - mariabackup
   - mariadb
   - masakari
+  - masakari_hostmonitor
+  - masakari_instancemonitor
   - memcached
   - mistral
   - monasca
@@ -202,6 +206,7 @@ kolla_feature_flags:
   - placement
   - prometheus
   - prometheus_alertmanager
+  - prometheus_alertmanager_external
   - prometheus_blackbox_exporter
   - prometheus_cadvisor
   - prometheus_ceph_mgr_exporter
@@ -211,6 +216,7 @@ kolla_feature_flags:
   - prometheus_haproxy_exporter
   - prometheus_libvirt_exporter
   - prometheus_memcached_exporter
+  - prometheus_msteams
   - prometheus_mysqld_exporter
   - prometheus_node_exporter
   - prometheus_openstack_exporter

diff --git a/ansible/roles/kolla-bifrost/tasks/main.yml b/ansible/roles/kolla-bifrost/tasks/main.yml
@@ -1,4 +1,12 @@
 ---
+- name: Check if inspection allocation is defined
+  assert:
+    that:
+      - kolla_bifrost_dhcp_pool_start | length > 0
+      - kolla_bifrost_dhcp_pool_end | length > 0
+      - kolla_bifrost_dhcp_pool_mask | length > 0
+    fail_msg: "Inspection allocation pool for provisioning network is not properly defined"
+
 - name: Ensure the Kolla Bifrost configuration directories exist
   file:
     path: "{{ kolla_node_custom_config_path }}/bifrost"

diff --git a/dev/functions b/dev/functions
@@ -232,22 +232,22 @@ function upgrade_kayobe_venv {
 # Deployment
 
 function is_deploy_image_built_locally {
-    ipa_build_images=$(kayobe configuration dump --host controllers[0] --var-name ipa_build_images)
+    ipa_build_images=$(kayobe configuration dump --host localhost --var-name ipa_build_images)
     to_bool "$ipa_build_images"
 }
 
 function is_ironic_enabled {
-    ironic_enabled=$(kayobe configuration dump --host controllers[0] --var-name kolla_enable_ironic)
+    ironic_enabled=$(kayobe configuration dump --host localhost --var-name kolla_enable_ironic)
     to_bool "$ironic_enabled"
 }
 
 function is_overcloud_host_image_built_by_dib {
-    overcloud_dib_build_host_images=$(kayobe configuration dump --host controllers[0] --var-name overcloud_dib_build_host_images)
+    overcloud_dib_build_host_images=$(kayobe configuration dump --host localhost --var-name overcloud_dib_build_host_images)
     to_bool "$overcloud_dib_build_host_images"
 }
 
 function is_cinder_enabled {
-    flag="$(run_kayobe configuration dump --host controllers[0] --var-name kolla_enable_cinder)"
+    flag="$(run_kayobe configuration dump --host localhost --var-name kolla_enable_cinder)"
     to_bool "$flag"
 }
 

diff --git a/doc/source/contributor/automated.rst b/doc/source/contributor/automated.rst
@@ -81,6 +81,30 @@ This can be added using the following commands::
     sudo ip l set eth1 up
     sudo ip l set eth1 master breth1
 
+Configuration
+-------------
+
+Enable TLS
+^^^^^^^^^^
+
+Apply the following configuration if you wish to enable TLS for the OpenStack
+API:
+
+Set the following option in ``config/src/kayobe-config/etc/kayobe/kolla.yml``:
+
+.. code-block:: yaml
+
+   kolla_enable_tls_internal: "yes"
+
+Set the following options in
+``config/src/kayobe-config/etc/kayobe/kolla/globals.yml``:
+
+.. code-block:: yaml
+
+   kolla_copy_ca_into_containers: "yes"
+   openstack_cacert: "{% if os_distribution == 'ubuntu' %}/etc/ssl/certs/ca-certificates.crt{% else %}/etc/pki/tls/certs/ca-bundle.crt{% endif %}"
+   kolla_admin_openrc_cacert: "{% if os_distribution == 'ubuntu' %}/etc/ssl/certs/ca-certificates.crt{% else %}/etc/pki/tls/certs/ca-bundle.crt{% endif %}"
+
 Usage
 -----
 
@@ -101,6 +125,10 @@ its dependencies in a Python virtual environment::
    changes will not been seen until you reinstall the package. To do this you
    can run ``./dev/install.sh``.
 
+If you are using TLS and wish to generate self-signed certificates::
+
+    export KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES=1
+
 Run the ``dev/overcloud-deploy.sh`` script to deploy the OpenStack control
 plane::
 

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
@@ -160,7 +160,7 @@ Editable source installation
 ----------------------------
 
 From Kayobe 5.0.0 onwards it is possible to create an `editable install
-<https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs>`__
+<https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs>`__
 of Kayobe. In an editable install, any changes to the Kayobe source tree will
 immediately be visible when running any Kayobe commands.  To create an editable
 install, add the ``-e`` flag::

diff --git a/etc/kayobe/infra-vms.yml b/etc/kayobe/infra-vms.yml
@@ -37,7 +37,7 @@
 # or
 # https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2
 # when os_distribution is "rocky" and os_release is "9"
-# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2"
+# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2"
 # otherwise.
 #infra_vm_root_image:
 

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
@@ -299,6 +299,7 @@
 #kolla_enable_gnocchi:
 #kolla_enable_gnocchi_statsd:
 #kolla_enable_grafana:
+#kolla_enable_grafana_external:
 #kolla_enable_hacluster:
 #kolla_enable_haproxy:
 #kolla_enable_haproxy_memcached:
@@ -338,6 +339,7 @@
 #kolla_enable_keystone_federation:
 #kolla_enable_keystone_horizon_policy_file:
 #kolla_enable_kibana:
+#kolla_enable_kibana_external:
 #kolla_enable_kuryr:
 #kolla_enable_loadbalancer:
 #kolla_enable_magnum:
@@ -350,6 +352,8 @@
 #kolla_enable_mariabackup:
 #kolla_enable_mariadb:
 #kolla_enable_masakari:
+#kolla_enable_masakari_hostmonitor:
+#kolla_enable_masakari_instancemonitor:
 #kolla_enable_memcached:
 #kolla_enable_mistral:
 #kolla_enable_monasca:
@@ -392,6 +396,7 @@
 #kolla_enable_placement:
 #kolla_enable_prometheus:
 #kolla_enable_prometheus_alertmanager:
+#kolla_enable_prometheus_alertmanager_external:
 #kolla_enable_prometheus_blackbox_exporter:
 #kolla_enable_prometheus_cadvisor:
 #kolla_enable_prometheus_ceph_mgr_exporter:
@@ -401,6 +406,7 @@
 #kolla_enable_prometheus_haproxy_exporter:
 #kolla_enable_prometheus_libvirt_exporter:
 #kolla_enable_prometheus_memcached_exporter:
+#kolla_enable_prometheus_msteams:
 #kolla_enable_prometheus_mysqld_exporter:
 #kolla_enable_prometheus_node_exporter:
 #kolla_enable_prometheus_openstack_exporter:

diff --git a/etc/kayobe/seed-vm.yml b/etc/kayobe/seed-vm.yml
@@ -31,7 +31,7 @@
 # or
 # https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2
 # when os_distribution is "rocky" and os_release is "9"
-# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2"
+# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2"
 # otherwise.
 #seed_vm_root_image:
 

diff --git a/kayobe/ansible.py b/kayobe/ansible.py
@@ -21,6 +21,8 @@
 import sys
 import tempfile
 
+from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
+
 from kayobe import exception
 from kayobe import utils
 from kayobe import vault
@@ -260,6 +262,18 @@ def run_playbook(parsed_args, playbook, *args, **kwargs):
     return run_playbooks(parsed_args, [playbook], *args, **kwargs)
 
 
+def _sanitise_hostvar(var):
+    """Sanitise a host variable."""
+    if isinstance(var, AnsibleVaultEncryptedUnicode):
+        return "******"
+    # Recursively sanitise dicts and lists.
+    if isinstance(var, dict):
+        return {k: _sanitise_hostvar(v) for k, v in var.items()}
+    if isinstance(var, list):
+        return [_sanitise_hostvar(v) for v in var]
+    return var
+
+
 def config_dump(parsed_args, host=None, hosts=None, var_name=None,
                 facts=None, extra_vars=None, tags=None, verbose_level=None):
     dump_dir = tempfile.mkdtemp()
@@ -285,15 +299,16 @@ def config_dump(parsed_args, host=None, hosts=None, var_name=None,
             LOG.debug("Found dump file %s", path)
             inventory_hostname, ext = os.path.splitext(path)
             if ext == ".yml":
-                hvars = utils.read_yaml_file(os.path.join(dump_dir, path))
+                dump_file = os.path.join(dump_dir, path)
+                hvars = utils.read_config_dump_yaml_file(dump_file)
                 if host:
                     return hvars
                 else:
                     hostvars[inventory_hostname] = hvars
             else:
                 LOG.warning("Unexpected extension on config dump file %s",
                             path)
-        return hostvars
+        return {k: _sanitise_hostvar(v) for k, v in hostvars.items()}
     finally:
         shutil.rmtree(dump_dir)