diff --git a/OpenStack-Rabbit-Consumer/rabbit_consumer/aq_api.py b/OpenStack-Rabbit-Consumer/rabbit_consumer/aq_api.py index 90f7b88b..4300d9ad 100644 --- a/OpenStack-Rabbit-Consumer/rabbit_consumer/aq_api.py +++ b/OpenStack-Rabbit-Consumer/rabbit_consumer/aq_api.py @@ -8,7 +8,7 @@ from urllib3.util.retry import Retry from rabbit_consumer.consumer_config import ConsumerConfig -from rabbit_consumer.image_metadata import ImageMetadata +from rabbit_consumer.aq_metadata import AqMetadata from rabbit_consumer.openstack_address import OpenstackAddress from rabbit_consumer.rabbit_message import RabbitMessage from rabbit_consumer.vm_data import VmData @@ -84,7 +84,7 @@ def setup_requests( return response.text -def aq_make(addresses: List[OpenstackAddress], image_meta: ImageMetadata) -> None: +def aq_make(addresses: List[OpenstackAddress], image_meta: AqMetadata) -> None: """ Runs AQ make against a list of addresses passed to build the default personality """ @@ -111,7 +111,7 @@ def aq_make(addresses: List[OpenstackAddress], image_meta: ImageMetadata) -> Non setup_requests(url, "post", "Make Template: ", params) -def aq_manage(addresses: List[OpenstackAddress], image_meta: ImageMetadata) -> None: +def aq_manage(addresses: List[OpenstackAddress], image_meta: AqMetadata) -> None: """ Manages the list of Aquilon addresses passed to it back to the production domain """ @@ -159,7 +159,7 @@ def delete_machine(machine_name: str) -> None: def create_host( - image_meta: ImageMetadata, addresses: List[OpenstackAddress], machine_name: str + image_meta: AqMetadata, addresses: List[OpenstackAddress], machine_name: str ) -> None: """ Creates a host in Aquilon diff --git a/OpenStack-Rabbit-Consumer/rabbit_consumer/aq_metadata.py b/OpenStack-Rabbit-Consumer/rabbit_consumer/aq_metadata.py new file mode 100644 index 00000000..7eca57d6 --- /dev/null +++ b/OpenStack-Rabbit-Consumer/rabbit_consumer/aq_metadata.py @@ -0,0 +1,50 @@ +import logging +from dataclasses import dataclass +from typing import Dict + +from mashumaro import DataClassDictMixin +from mashumaro.config import BaseConfig + +logger = logging.getLogger(__name__) + + +@dataclass +class AqMetadata(DataClassDictMixin): + """ + Deserialised metadata that is set either on an Openstack image + or a VM's metadata + """ + + aq_archetype: str + # Aq domain can hold either a domain or sandbox reference + aq_domain: str + + aq_personality: str + aq_os_version: str + aq_os: str + + # pylint: disable=too-few-public-methods + class Config(BaseConfig): + """ + Sets the aliases for the metadata keys + """ + + aliases = { + "aq_archetype": "AQ_ARCHETYPE", + "aq_domain": "AQ_DOMAIN", + "aq_personality": "AQ_PERSONALITY", + "aq_os_version": "AQ_OSVERSION", + "aq_os": "AQ_OS", + } + + def override_from_vm_meta(self, vm_meta: Dict[str, str]): + """ + Overrides the values in the metadata with the values from the VM's + metadata + """ + for attr, alias in self.Config.aliases.items(): + if alias in vm_meta: + setattr(self, attr, vm_meta[alias]) + + if "AQ_SANDBOX" in vm_meta: + self.aq_domain = vm_meta["AQ_SANDBOX"] diff --git a/OpenStack-Rabbit-Consumer/rabbit_consumer/image_metadata.py b/OpenStack-Rabbit-Consumer/rabbit_consumer/image_metadata.py deleted file mode 100644 index e7d7e7b2..00000000 --- a/OpenStack-Rabbit-Consumer/rabbit_consumer/image_metadata.py +++ /dev/null @@ -1,20 +0,0 @@ -import logging -from dataclasses import dataclass, field - -from mashumaro import DataClassDictMixin, field_options - -logger = logging.getLogger(__name__) - - -@dataclass -class ImageMetadata(DataClassDictMixin): - """ - Deserialised metadata that is set on OpenStack images - """ - - aq_archetype: str = field(metadata=field_options(alias="AQ_ARCHETYPE")) - aq_domain: str = field(metadata=field_options(alias="AQ_DOMAIN")) - - aq_personality: str = field(metadata=field_options(alias="AQ_PERSONALITY")) - aq_os_version: str = field(metadata=field_options(alias="AQ_OSVERSION")) - aq_os: str = field(metadata=field_options(alias="AQ_OS")) diff --git a/OpenStack-Rabbit-Consumer/rabbit_consumer/message_consumer.py b/OpenStack-Rabbit-Consumer/rabbit_consumer/message_consumer.py index 6e39e967..a4134d91 100644 --- a/OpenStack-Rabbit-Consumer/rabbit_consumer/message_consumer.py +++ b/OpenStack-Rabbit-Consumer/rabbit_consumer/message_consumer.py @@ -9,7 +9,7 @@ from rabbit_consumer import openstack_api from rabbit_consumer.aq_api import verify_kerberos_ticket from rabbit_consumer.consumer_config import ConsumerConfig -from rabbit_consumer.image_metadata import ImageMetadata +from rabbit_consumer.aq_metadata import AqMetadata from rabbit_consumer.openstack_address import OpenstackAddress from rabbit_consumer.rabbit_message import RabbitMessage, MessageEventType from rabbit_consumer.vm_data import VmData @@ -21,17 +21,28 @@ } -def is_aq_managed_image(rabbit_message: RabbitMessage) -> Optional[ImageMetadata]: +def is_aq_managed_image(vm_data: VmData) -> bool: """ Check to see if the metadata in the message contains entries that suggest it is for an Aquilon VM. """ - image = openstack_api.get_image(VmData.from_message(rabbit_message)) + image = openstack_api.get_image(vm_data) if "AQ_OS" not in image.metadata: logger.debug("Skipping non-Aquilon image: %s", image.name) - return None + return False + return True + + +def get_aq_build_metadata(vm_data: VmData) -> AqMetadata: + """ + Gets the Aq Metadata from either the image or VM (where + VM metadata takes precedence) to determine the AQ params + """ + image = openstack_api.get_image(vm_data) + image_meta = AqMetadata.from_dict(image.metadata) - image_meta = ImageMetadata.from_dict(image.metadata) + vm_metadata = openstack_api.get_server_metadata(vm_data) + image_meta.override_from_vm_meta(vm_metadata) return image_meta @@ -111,7 +122,7 @@ def check_machine_valid(rabbit_message: RabbitMessage) -> bool: ) return False - if not is_aq_managed_image(rabbit_message): + if not is_aq_managed_image(vm_data): logger.debug("Ignoring non AQ Image: %s", rabbit_message) return False @@ -131,7 +142,7 @@ def handle_create_machine(rabbit_message: RabbitMessage) -> None: vm_data = VmData.from_message(rabbit_message) - image_meta = is_aq_managed_image(rabbit_message) + image_meta = get_aq_build_metadata(vm_data) network_details = openstack_api.get_server_networks(vm_data) if not network_details or not network_details[0].hostname: diff --git a/OpenStack-Rabbit-Consumer/rabbit_consumer/openstack_api.py b/OpenStack-Rabbit-Consumer/rabbit_consumer/openstack_api.py index 05f6a381..6e27094a 100644 --- a/OpenStack-Rabbit-Consumer/rabbit_consumer/openstack_api.py +++ b/OpenStack-Rabbit-Consumer/rabbit_consumer/openstack_api.py @@ -71,7 +71,7 @@ def get_server_networks(vm_data: VmData) -> List[OpenstackAddress]: return OpenstackAddress.get_internal_networks(server.addresses) -def get_metadata(vm_data: VmData) -> dict: +def get_server_metadata(vm_data: VmData) -> dict: """ Gets the metadata from Openstack for the virtual machine. """ diff --git a/OpenStack-Rabbit-Consumer/test/fixtures.py b/OpenStack-Rabbit-Consumer/test/fixtures.py index 0b225038..39409831 100644 --- a/OpenStack-Rabbit-Consumer/test/fixtures.py +++ b/OpenStack-Rabbit-Consumer/test/fixtures.py @@ -2,7 +2,7 @@ import pytest -from rabbit_consumer.image_metadata import ImageMetadata +from rabbit_consumer.aq_metadata import AqMetadata from rabbit_consumer.openstack_address import OpenstackAddress from rabbit_consumer.rabbit_message import RabbitMessage, RabbitMeta, RabbitPayload from rabbit_consumer.vm_data import VmData @@ -14,7 +14,7 @@ def fixture_image_metadata(): Creates an ImageMetadata object with mock data which represent an example OpenStack image """ - return ImageMetadata( + return AqMetadata( aq_archetype="archetype_mock", aq_domain="domain_mock", aq_personality="personality_mock", diff --git a/OpenStack-Rabbit-Consumer/test/test_aq_metadata.py b/OpenStack-Rabbit-Consumer/test/test_aq_metadata.py new file mode 100644 index 00000000..1960ab30 --- /dev/null +++ b/OpenStack-Rabbit-Consumer/test/test_aq_metadata.py @@ -0,0 +1,75 @@ +from typing import Dict + +import pytest + +from rabbit_consumer.aq_metadata import AqMetadata + + +@pytest.fixture(name="image_metadata") +def fixture_image_metadata() -> Dict[str, str]: + """ + Creates a dictionary with mock data + which represents an example OpenStack image's metadata + """ + return { + "AQ_ARCHETYPE": "archetype_mock", + "AQ_DOMAIN": "domain_mock", + "AQ_PERSONALITY": "personality_mock", + "AQ_OS": "os_mock", + "AQ_OSVERSION": "osversion_mock", + } + + +def test_aq_metadata_from_initial_dict(image_metadata): + """ + Tests creating an AQ metadata object from an initial dictionary + """ + returned = AqMetadata.from_dict(image_metadata) + + assert returned.aq_archetype == "archetype_mock" + assert returned.aq_domain == "domain_mock" + assert returned.aq_personality == "personality_mock" + assert returned.aq_os == "os_mock" + assert returned.aq_os_version == "osversion_mock" + + +def test_aq_metadata_override_all(image_metadata): + """ + Tests overriding all values in an AQ metadata object + """ + returned = AqMetadata.from_dict(image_metadata) + returned.override_from_vm_meta( + { + "AQ_ARCHETYPE": "archetype_mock_override", + "AQ_DOMAIN": "domain_mock_override", + "AQ_PERSONALITY": "personality_mock_override", + } + ) + + assert returned.aq_archetype == "archetype_mock_override" + assert returned.aq_domain == "domain_mock_override" + assert returned.aq_personality == "personality_mock_override" + + # Check the original values are still there + assert returned.aq_os == "os_mock" + assert returned.aq_os_version == "osversion_mock" + + +def test_aq_metadata_override_sandbox(image_metadata): + """ + Tests overriding the sandbox value in an AQ metadata object + maps correctly onto the domain value + """ + returned = AqMetadata.from_dict(image_metadata) + returned.override_from_vm_meta( + { + "AQ_SANDBOX": "sandbox_mock", + } + ) + # This should be the only value that has changed + assert returned.aq_domain == "sandbox_mock" + + assert returned.aq_archetype == "archetype_mock" + assert returned.aq_personality == "personality_mock" + assert returned.aq_os == "os_mock" + assert returned.aq_os_version == "osversion_mock" diff --git a/OpenStack-Rabbit-Consumer/test/test_message_consumer.py b/OpenStack-Rabbit-Consumer/test/test_message_consumer.py index d24c26f9..2bcf9bd9 100644 --- a/OpenStack-Rabbit-Consumer/test/test_message_consumer.py +++ b/OpenStack-Rabbit-Consumer/test/test_message_consumer.py @@ -1,4 +1,4 @@ -from unittest.mock import Mock, NonCallableMock, patch, call +from unittest.mock import Mock, NonCallableMock, patch, call, MagicMock import pytest @@ -21,6 +21,7 @@ SUPPORTED_MESSAGE_TYPES, check_machine_valid, is_aq_managed_image, + get_aq_build_metadata, ) from rabbit_consumer.vm_data import VmData @@ -217,11 +218,13 @@ def test_consume_create_machine_hostnames_good_path( with ( patch("rabbit_consumer.message_consumer.VmData") as data_patch, patch("rabbit_consumer.message_consumer.check_machine_valid") as check_machine, - patch("rabbit_consumer.message_consumer.is_aq_managed_image") as is_managed, + patch( + "rabbit_consumer.message_consumer.get_aq_build_metadata" + ) as get_image_meta, patch("rabbit_consumer.message_consumer.delete_machine") as delete_machine, ): check_machine.return_value = True - is_managed.return_value = image_metadata + get_image_meta.return_value = image_metadata handle_create_machine(rabbit_message) @@ -272,13 +275,14 @@ def test_check_machine_valid(openstack_api, is_aq_managed): """ mock_message = NonCallableMock() is_aq_managed.return_value = True + + vm_data = VmData.from_message(mock_message) + openstack_api.check_machine_exists.return_value = True assert check_machine_valid(mock_message) - is_aq_managed.assert_called_once_with(mock_message) - openstack_api.check_machine_exists.assert_called_once_with( - VmData.from_message(mock_message) - ) + is_aq_managed.assert_called_once_with(vm_data) + openstack_api.check_machine_exists.assert_called_once_with(vm_data) @patch("rabbit_consumer.message_consumer.is_aq_managed_image") @@ -290,13 +294,12 @@ def test_check_machine_invalid_image(openstack_api, is_aq_managed): mock_message = NonCallableMock() is_aq_managed.return_value = False openstack_api.check_machine_exists.return_value = True + vm_data = VmData.from_message(mock_message) assert not check_machine_valid(mock_message) - openstack_api.check_machine_exists.assert_called_once_with( - VmData.from_message(mock_message) - ) - is_aq_managed.assert_called_once_with(mock_message) + openstack_api.check_machine_exists.assert_called_once_with(vm_data) + is_aq_managed.assert_called_once_with(vm_data) @patch("rabbit_consumer.message_consumer.is_aq_managed_image") @@ -316,30 +319,45 @@ def test_check_machine_invalid_machine(openstack_api, is_aq_managed): ) -@patch("rabbit_consumer.message_consumer.VmData") -@patch("rabbit_consumer.message_consumer.ImageMetadata") @patch("rabbit_consumer.message_consumer.openstack_api") -def test_is_aq_managed_image(openstack_api, image_meta, vm_data): +def test_is_aq_managed_image(openstack_api, vm_data): """ Test that the function returns True when the image is AQ managed """ - mock_message = NonCallableMock() openstack_api.get_image.return_value.metadata = {"AQ_OS": "True"} - assert is_aq_managed_image(mock_message) == image_meta.from_dict.return_value - openstack_api.get_image.assert_called_once_with(vm_data.from_message.return_value) + assert is_aq_managed_image(vm_data) + openstack_api.get_image.assert_called_once_with(vm_data) @patch("rabbit_consumer.message_consumer.VmData") -@patch("rabbit_consumer.message_consumer.ImageMetadata") @patch("rabbit_consumer.message_consumer.openstack_api") -def test_is_aq_managed_image_missing_key(openstack_api, image_meta, vm_data): +def test_is_aq_managed_image_missing_key(openstack_api, vm_data): """ Test that the function returns False when the image is not AQ managed """ - mock_message = NonCallableMock() openstack_api.get_image.return_value.metadata = {} - assert not is_aq_managed_image(mock_message) - openstack_api.get_image.assert_called_once_with(vm_data.from_message.return_value) - image_meta.from_dict.assert_not_called() + assert not is_aq_managed_image(vm_data) + openstack_api.get_image.assert_called_once_with(vm_data) + + +@patch("rabbit_consumer.message_consumer.AqMetadata") +@patch("rabbit_consumer.message_consumer.openstack_api") +def test_get_aq_build_metadata(openstack_api, aq_metadata_class, vm_data): + """ + Test that the function returns the correct metadata + """ + aq_metadata_obj: MagicMock = get_aq_build_metadata(vm_data) + + # We should first construct from an image + assert aq_metadata_obj == aq_metadata_class.from_dict.return_value + aq_metadata_class.from_dict.assert_called_once_with( + openstack_api.get_image.return_value.metadata + ) + + # Then override with an object + openstack_api.get_server_metadata.assert_called_once_with(vm_data) + aq_metadata_obj.override_from_vm_meta.assert_called_once_with( + openstack_api.get_server_metadata.return_value + ) diff --git a/OpenStack-Rabbit-Consumer/version.txt b/OpenStack-Rabbit-Consumer/version.txt index c043eea7..276cbf9e 100644 --- a/OpenStack-Rabbit-Consumer/version.txt +++ b/OpenStack-Rabbit-Consumer/version.txt @@ -1 +1 @@ -2.2.1 +2.3.0 diff --git a/charts/rabbit-consumer/Chart.yaml b/charts/rabbit-consumer/Chart.yaml index 689701ce..ea3f791a 100644 --- a/charts/rabbit-consumer/Chart.yaml +++ b/charts/rabbit-consumer/Chart.yaml @@ -6,10 +6,11 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.3.1 +version: 1.4.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v2.2.1" +appVersion: "v2.3.0" +