From 20ff1a957ed678875066070b4a17601ed31f39d1 Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Thu, 7 Nov 2024 14:59:44 +0000 Subject: [PATCH 1/9] Support versioning on required systems --- brewtils/plugin.py | 100 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 2 deletions(-) diff --git a/brewtils/plugin.py b/brewtils/plugin.py index 87458ad2..eaa31408 100644 --- a/brewtils/plugin.py +++ b/brewtils/plugin.py @@ -4,6 +4,7 @@ import logging import logging.config import os +import re import signal import sys import threading @@ -12,7 +13,7 @@ import appdirs from box import Box from datetime import datetime, timezone -from packaging.version import Version +from packaging.version import InvalidVersion, parse, Version from requests import ConnectionError as RequestsConnectionError import brewtils @@ -393,10 +394,105 @@ def get_timestamp(add_time: int = None): return current_timestamp + add_time return current_timestamp + def parse_require_version(self, require): + """ + Follows npm version symbols + ^ accept updates to minor and patch releases only. ^0.13.0: 0.13.1, 0.14.0 + ~ accept updates to patch releases only. ~0.13.0: 0.13.1 (not 0.14.0) + > accept updates to any version greater than specified. >0.13.0: 0.13.1, 0.14.1, 1.1.1 + < accept updates to any version less than specified. <3.0.0: 2.0.0, 2.9.0 + >= accept any version greater than or equal to specified. >=3.0.0: 3.0.0, 4.1.0 + <= accept any version less than or equal to specified. <=3.0.0: 3.0.0, 2.9.0 + = accept only the exact specified version. =3.0.0: 3.0.0, (not 3.0.1) + Future: regex version matching + """ + name_version_list = re.findall(r"(\w+)([\^|~|>|<|=])(.*)", require) + if len(name_version_list) > 1: + raise ValueError("Failed to parse name and version") + elif len(name_version_list) == 1: + return name_version_list[0] + else: + return require, None, None + + def get_system_matching_version(self, require): + name_version = self.parse_require_version(require) + require_name = name_version[0] + require_type = name_version[1] + require_version = name_version[2] + if require_version: + try: + parsed_version = str(parse(require_version)) + if require_type == "=": + system = self._ez_client.find_unique_system( + name=require_name, version=parsed_version, local=True + ) + else: + pattern = re.compile( + r"^(?:([0-9]+)\.([0-9]+)\.([0-9]+)){1}(?:\.([A-Za-z0-9]+)){0,1}$" + ) + match = re.search(pattern, parsed_version) + if match: + major, minor, _, _ = match.groups() + if require_type == "^" and major: + system = self._ez_client.find_unique_system( + name=require_name, + version__startswith=f"{major}.", + version__gte=parsed_version, + filter_latest=True, + local=True, + ) + elif require_type == "~" and major and minor: + system = self._ez_client.find_unique_system( + name=require_name, + version__startswith=f"{major}.{minor}.", + version__gte=parsed_version, + filter_latest=True, + local=True, + ) + elif require_type == ">": + system = self._ez_client.find_unique_system( + name=require_name, + version__gt=parsed_version, + filter_latest=True, + local=True, + ) + elif require_type == ">=": + system = self._ez_client.find_unique_system( + name=require_name, + version__gte=parsed_version, + filter_latest=True, + local=True, + ) + elif require_type == "<": + system = self._ez_client.find_unique_system( + name=require_name, + version__lt=parsed_version, + filter_latest=True, + local=True, + ) + elif require_type == "<=": + system = self._ez_client.find_unique_system( + name=require_name, + version__lte=parsed_version, + filter_latest=True, + local=True, + ) + except InvalidVersion: + # TODO: Regex. Switch to use version__regex when mongoengine>=0.24.0 + system = self._ez_client.find_unique_system( + name=require_name, filter_latest=True, local=True + ) + else: + system = self._ez_client.find_unique_system( + name=require_name, filter_latest=True, local=True + ) + + return system + def get_system_dependency(self, require, timeout=300): wait_time = 0.1 while timeout > 0: - system = self._ez_client.find_unique_system(name=require, local=True) + system = self.get_system_matching_version(require) if ( system and system.instances From c5053c55411061d13653bab64c061069a7dddf71 Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Fri, 8 Nov 2024 15:14:06 +0000 Subject: [PATCH 2/9] Update plugin dependency check to use filter_running --- brewtils/plugin.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/brewtils/plugin.py b/brewtils/plugin.py index eaa31408..8c0cb2c6 100644 --- a/brewtils/plugin.py +++ b/brewtils/plugin.py @@ -15,6 +15,7 @@ from datetime import datetime, timezone from packaging.version import InvalidVersion, parse, Version from requests import ConnectionError as RequestsConnectionError +from typing import Optional import brewtils from brewtils.config import load_config @@ -414,7 +415,11 @@ def parse_require_version(self, require): else: return require, None, None - def get_system_matching_version(self, require): + def get_system_matching_version(self, require, **kwargs) -> Optional[System]: + """ + Get system matching version or None. + Kwargs to accept named parameters like filter_latest, filter_running, and local + """ name_version = self.parse_require_version(require) require_name = name_version[0] require_type = name_version[1] @@ -424,7 +429,7 @@ def get_system_matching_version(self, require): parsed_version = str(parse(require_version)) if require_type == "=": system = self._ez_client.find_unique_system( - name=require_name, version=parsed_version, local=True + name=require_name, version=parsed_version, **kwargs ) else: pattern = re.compile( @@ -438,53 +443,49 @@ def get_system_matching_version(self, require): name=require_name, version__startswith=f"{major}.", version__gte=parsed_version, - filter_latest=True, - local=True, + **kwargs, ) elif require_type == "~" and major and minor: system = self._ez_client.find_unique_system( name=require_name, version__startswith=f"{major}.{minor}.", version__gte=parsed_version, - filter_latest=True, - local=True, + **kwargs, ) elif require_type == ">": system = self._ez_client.find_unique_system( name=require_name, version__gt=parsed_version, - filter_latest=True, - local=True, + **kwargs, ) elif require_type == ">=": system = self._ez_client.find_unique_system( name=require_name, version__gte=parsed_version, - filter_latest=True, - local=True, + **kwargs, ) elif require_type == "<": system = self._ez_client.find_unique_system( name=require_name, version__lt=parsed_version, - filter_latest=True, - local=True, + **kwargs, ) elif require_type == "<=": system = self._ez_client.find_unique_system( name=require_name, version__lte=parsed_version, - filter_latest=True, - local=True, + **kwargs, ) except InvalidVersion: # TODO: Regex. Switch to use version__regex when mongoengine>=0.24.0 system = self._ez_client.find_unique_system( - name=require_name, filter_latest=True, local=True + name=require_name, + **kwargs, ) else: system = self._ez_client.find_unique_system( - name=require_name, filter_latest=True, local=True + name=require_name, + **kwargs, ) return system @@ -492,12 +493,11 @@ def get_system_matching_version(self, require): def get_system_dependency(self, require, timeout=300): wait_time = 0.1 while timeout > 0: - system = self.get_system_matching_version(require) - if ( - system - and system.instances - and any("RUNNING" == instance.status for instance in system.instances) - ): + system = self.get_system_matching_version( + require, filter_latest=True, filter_running=True, local=True + ) + if system: + self.logger.debug(f"Found system: {system}") return system self.logger.error( f"Waiting {wait_time:.1f} seconds before next attempt for {self._system} " From 12efe313c372f65e2d4c3aab62e70de8eef62154 Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Thu, 14 Nov 2024 14:24:09 +0000 Subject: [PATCH 3/9] fix regex --- brewtils/plugin.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/brewtils/plugin.py b/brewtils/plugin.py index 8c0cb2c6..09163eb3 100644 --- a/brewtils/plugin.py +++ b/brewtils/plugin.py @@ -15,7 +15,6 @@ from datetime import datetime, timezone from packaging.version import InvalidVersion, parse, Version from requests import ConnectionError as RequestsConnectionError -from typing import Optional import brewtils from brewtils.config import load_config @@ -407,7 +406,7 @@ def parse_require_version(self, require): = accept only the exact specified version. =3.0.0: 3.0.0, (not 3.0.1) Future: regex version matching """ - name_version_list = re.findall(r"(\w+)([\^|~|>|<|=])(.*)", require) + name_version_list = re.findall(r"(\w+)([\^~]|[<>]=?|=)(.*)", require) if len(name_version_list) > 1: raise ValueError("Failed to parse name and version") elif len(name_version_list) == 1: @@ -415,7 +414,7 @@ def parse_require_version(self, require): else: return require, None, None - def get_system_matching_version(self, require, **kwargs) -> Optional[System]: + def get_system_matching_version(self, require, **kwargs): """ Get system matching version or None. Kwargs to accept named parameters like filter_latest, filter_running, and local From d6d157be7f661c31255754e1bf621b5a77b204f4 Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Thu, 14 Nov 2024 14:37:48 +0000 Subject: [PATCH 4/9] changelog --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cb72571d..4a317d7a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Brewtils Changelog ================== +3.29.0 +------ +TBD + +- Updated plugin class to accept version contraints for required dependencies. Contraints follow npm semantic versioning syntax. + 3.28.0 ------ 10/9/24 From 9bb67dfea9cb7b1c26aadd2a44eee4ca7e5f7a2c Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:33:27 +0000 Subject: [PATCH 5/9] Update requires versioning to use python packaging version specifiers --- CHANGELOG.rst | 2 +- brewtils/plugin.py | 109 ++++++++------------------------------ brewtils/test/fixtures.py | 27 ++++++++++ test/plugin_test.py | 91 +++++++++++++++++++++++++++++++ 4 files changed, 140 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4a317d7a..682f440a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,7 +5,7 @@ Brewtils Changelog ------ TBD -- Updated plugin class to accept version contraints for required dependencies. Contraints follow npm semantic versioning syntax. +- Updated plugin class to accept version contraints for required dependencies. Contraints follow python packaging version specifiers. 3.28.0 ------ diff --git a/brewtils/plugin.py b/brewtils/plugin.py index 09163eb3..97a74127 100644 --- a/brewtils/plugin.py +++ b/brewtils/plugin.py @@ -13,7 +13,8 @@ import appdirs from box import Box from datetime import datetime, timezone -from packaging.version import InvalidVersion, parse, Version +from packaging.requirements import Requirement +from packaging.version import Version from requests import ConnectionError as RequestsConnectionError import brewtils @@ -394,97 +395,29 @@ def get_timestamp(add_time: int = None): return current_timestamp + add_time return current_timestamp - def parse_require_version(self, require): - """ - Follows npm version symbols - ^ accept updates to minor and patch releases only. ^0.13.0: 0.13.1, 0.14.0 - ~ accept updates to patch releases only. ~0.13.0: 0.13.1 (not 0.14.0) - > accept updates to any version greater than specified. >0.13.0: 0.13.1, 0.14.1, 1.1.1 - < accept updates to any version less than specified. <3.0.0: 2.0.0, 2.9.0 - >= accept any version greater than or equal to specified. >=3.0.0: 3.0.0, 4.1.0 - <= accept any version less than or equal to specified. <=3.0.0: 3.0.0, 2.9.0 - = accept only the exact specified version. =3.0.0: 3.0.0, (not 3.0.1) - Future: regex version matching - """ - name_version_list = re.findall(r"(\w+)([\^~]|[<>]=?|=)(.*)", require) - if len(name_version_list) > 1: - raise ValueError("Failed to parse name and version") - elif len(name_version_list) == 1: - return name_version_list[0] - else: - return require, None, None - def get_system_matching_version(self, require, **kwargs): - """ - Get system matching version or None. - Kwargs to accept named parameters like filter_latest, filter_running, and local - """ - name_version = self.parse_require_version(require) - require_name = name_version[0] - require_type = name_version[1] - require_version = name_version[2] + system = None + req = Requirement(require) + require_name = req.name + require_version = req.specifier if require_version: - try: - parsed_version = str(parse(require_version)) - if require_type == "=": - system = self._ez_client.find_unique_system( - name=require_name, version=parsed_version, **kwargs - ) - else: - pattern = re.compile( - r"^(?:([0-9]+)\.([0-9]+)\.([0-9]+)){1}(?:\.([A-Za-z0-9]+)){0,1}$" - ) - match = re.search(pattern, parsed_version) - if match: - major, minor, _, _ = match.groups() - if require_type == "^" and major: - system = self._ez_client.find_unique_system( - name=require_name, - version__startswith=f"{major}.", - version__gte=parsed_version, - **kwargs, - ) - elif require_type == "~" and major and minor: - system = self._ez_client.find_unique_system( - name=require_name, - version__startswith=f"{major}.{minor}.", - version__gte=parsed_version, - **kwargs, - ) - elif require_type == ">": - system = self._ez_client.find_unique_system( - name=require_name, - version__gt=parsed_version, - **kwargs, - ) - elif require_type == ">=": - system = self._ez_client.find_unique_system( - name=require_name, - version__gte=parsed_version, - **kwargs, - ) - elif require_type == "<": - system = self._ez_client.find_unique_system( - name=require_name, - version__lt=parsed_version, - **kwargs, - ) - elif require_type == "<=": - system = self._ez_client.find_unique_system( - name=require_name, - version__lte=parsed_version, - **kwargs, - ) - except InvalidVersion: - # TODO: Regex. Switch to use version__regex when mongoengine>=0.24.0 - system = self._ez_client.find_unique_system( - name=require_name, - **kwargs, + systems = self._ez_client.find_systems(name=require_name, **kwargs) + valid_versions = list( + require_version.filter( + [str(Version(system.version)) for system in systems] ) + ) + if valid_versions: + system_candidates = [ + system for system in systems if system.version in valid_versions + ] + system = system_candidates[0] + for system_candidate in system_candidates: + if Version(system_candidate.version) > Version(system.version): + system = system_candidate else: system = self._ez_client.find_unique_system( - name=require_name, - **kwargs, + name=require_name, filter_latest=True, **kwargs ) return system @@ -493,7 +426,7 @@ def get_system_dependency(self, require, timeout=300): wait_time = 0.1 while timeout > 0: system = self.get_system_matching_version( - require, filter_latest=True, filter_running=True, local=True + require, filter_running=True, local=True ) if system: self.logger.debug(f"Found system: {system}") diff --git a/brewtils/test/fixtures.py b/brewtils/test/fixtures.py index 367335e7..3f53f885 100644 --- a/brewtils/test/fixtures.py +++ b/brewtils/test/fixtures.py @@ -293,6 +293,33 @@ def bg_system_2(system_dict, bg_instance, bg_command, bg_command_2): dict_copy["commands"] = [bg_command, bg_command_2] return System(**dict_copy) +@pytest.fixture +def bg_system_3(system_dict, bg_instance, bg_command, bg_command_2): + """A system with a different version.""" + dict_copy = copy.deepcopy(system_dict) + dict_copy["version"] = "2.1.0" + dict_copy["instances"] = [bg_instance] + dict_copy["commands"] = [bg_command, bg_command_2] + return System(**dict_copy) + +@pytest.fixture +def bg_system_4(system_dict, bg_instance, bg_command, bg_command_2): + """A system with a different version.""" + dict_copy = copy.deepcopy(system_dict) + dict_copy["version"] = "2.1.1" + dict_copy["instances"] = [bg_instance] + dict_copy["commands"] = [bg_command, bg_command_2] + return System(**dict_copy) + +@pytest.fixture +def bg_system_5(system_dict, bg_instance, bg_command, bg_command_2): + """A system with a different version.""" + dict_copy = copy.deepcopy(system_dict) + dict_copy["version"] = "3.0.0" + dict_copy["instances"] = [bg_instance] + dict_copy["commands"] = [bg_command, bg_command_2] + return System(**dict_copy) + @pytest.fixture def child_request_dict(ts_epoch): diff --git a/test/plugin_test.py b/test/plugin_test.py index 51ec4688..5626a0a3 100644 --- a/test/plugin_test.py +++ b/test/plugin_test.py @@ -940,3 +940,94 @@ def test_remote_plugin(self): assert "'RemotePlugin'" in str(warning) assert "'Plugin'" in str(warning) assert "4.0" in str(warning) + +class TestDependencies(object): + # 1.0.0 bg_system + # 2.0.0 bg_system_2 + # 2.1.0 bg_system_3 + # 2.1.1 bg_system_4 + # 3.0.0 bg_system_5 + def test_no_specifier(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + p = Plugin(bg_host="localhost", system=bg_system) + p._ez_client.find_unique_system.return_value=bg_system_5 + # Expect 3.0.0 as latest valid version + # Expect 3.0.0 + assert p.get_system_dependency("system").version == bg_system_5.version + + def test_equals(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + p = Plugin(bg_host="localhost", system=bg_system) + p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + # Expect 2.1.1 valid + # Expect 2.1.1 + assert p.get_system_dependency("system==2.1.0").version == bg_system_3.version + + def test_compatible_release(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + p = Plugin(bg_host="localhost", system=bg_system) + p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + # Expect 2.1.0, 2.1.1 valid + # Expect 2.1.1 + assert p.get_system_dependency("system~=2.1.0").version == bg_system_4.version + + def test_wildcard_minor(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + p = Plugin(bg_host="localhost", system=bg_system) + p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + # Expect 1.0.0, 2.0.0, 2.1.1, 3.0.0 valid + # Expect 3.0.0 + assert p.get_system_dependency("system==2.*").version == bg_system_4.version + + def test_wildcard_patch(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + p = Plugin(bg_host="localhost", system=bg_system) + p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + # Expect 1.0.0, 2.0.0, 2.1.1, 3.0.0 valid + # Expect 3.0.0 + assert p.get_system_dependency("system==2.1.*").version == bg_system_4.version + + def test_excludes(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + p = Plugin(bg_host="localhost", system=bg_system) + p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + # Expect 1.0.0, 2.0.0, 2.1.1, 3.0.0 valid + # Expect 3.0.0 + assert p.get_system_dependency("system!=2.1.0").version == bg_system_5.version + + def test_gt(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + p = Plugin(bg_host="localhost", system=bg_system) + p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + # Expect 2.1.1, 3.0.0 valid + # Expect 3.0.0 + assert p.get_system_dependency("system>2.1.0").version == bg_system_5.version + + def test_gte(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + p = Plugin(bg_host="localhost", system=bg_system) + p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + # Expect 2.1.0, 2.1.1, 3.0.0 valid + # Expect 3.0.0 + assert p.get_system_dependency("system>=2.1.0").version == bg_system_5.version + + def test_lt(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + p = Plugin(bg_host="localhost", system=bg_system) + p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + # Expect 1.0.0, 2.0.0 valid + # Expect 2.0.0 + assert p.get_system_dependency("system<2.1.0").version == bg_system_2.version + + def test_lte(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + p = Plugin(bg_host="localhost", system=bg_system) + p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + # Expect 1.0.0, 2.0.0, 2.1.0 valid + # Expect 2.1.0 + assert p.get_system_dependency("system<=2.1.0").version == bg_system_3.version + + def test_range(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + p = Plugin(bg_host="localhost", system=bg_system) + p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + # Expect 1.0.0 valid + # Expect 1.0.0 + assert p.get_system_dependency("system<2.0.0,>=1").version == bg_system.version + + + def test_combo(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + p = Plugin(bg_host="localhost", system=bg_system) + p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + # Expect 1.0.0 valid + # Expect 1.0.0 + assert p.get_system_dependency("system==2.*,<2.1.1,!=2.1.0").version == bg_system_2.version From e68fb11ea9606740d12c9bf90634e240f0e42726 Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:47:43 +0000 Subject: [PATCH 6/9] linting --- brewtils/test/fixtures.py | 3 + test/plugin_test.py | 131 +++++++++++++++++++++++++++++++------- 2 files changed, 111 insertions(+), 23 deletions(-) diff --git a/brewtils/test/fixtures.py b/brewtils/test/fixtures.py index 3f53f885..0c84996a 100644 --- a/brewtils/test/fixtures.py +++ b/brewtils/test/fixtures.py @@ -293,6 +293,7 @@ def bg_system_2(system_dict, bg_instance, bg_command, bg_command_2): dict_copy["commands"] = [bg_command, bg_command_2] return System(**dict_copy) + @pytest.fixture def bg_system_3(system_dict, bg_instance, bg_command, bg_command_2): """A system with a different version.""" @@ -302,6 +303,7 @@ def bg_system_3(system_dict, bg_instance, bg_command, bg_command_2): dict_copy["commands"] = [bg_command, bg_command_2] return System(**dict_copy) + @pytest.fixture def bg_system_4(system_dict, bg_instance, bg_command, bg_command_2): """A system with a different version.""" @@ -311,6 +313,7 @@ def bg_system_4(system_dict, bg_instance, bg_command, bg_command_2): dict_copy["commands"] = [bg_command, bg_command_2] return System(**dict_copy) + @pytest.fixture def bg_system_5(system_dict, bg_instance, bg_command, bg_command_2): """A system with a different version.""" diff --git a/test/plugin_test.py b/test/plugin_test.py index 5626a0a3..af0c87c7 100644 --- a/test/plugin_test.py +++ b/test/plugin_test.py @@ -941,93 +941,178 @@ def test_remote_plugin(self): assert "'Plugin'" in str(warning) assert "4.0" in str(warning) + class TestDependencies(object): # 1.0.0 bg_system # 2.0.0 bg_system_2 # 2.1.0 bg_system_3 # 2.1.1 bg_system_4 # 3.0.0 bg_system_5 - def test_no_specifier(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + def test_no_specifier( + plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 + ): p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_unique_system.return_value=bg_system_5 + p._ez_client.find_unique_system.return_value = bg_system_5 # Expect 3.0.0 as latest valid version # Expect 3.0.0 assert p.get_system_dependency("system").version == bg_system_5.version - def test_equals(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + def test_equals( + plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 + ): p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + p._ez_client.find_systems.return_value = [ + bg_system, + bg_system_2, + bg_system_3, + bg_system_4, + bg_system_5, + ] # Expect 2.1.1 valid # Expect 2.1.1 assert p.get_system_dependency("system==2.1.0").version == bg_system_3.version - - def test_compatible_release(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + + def test_compatible_release( + plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 + ): p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + p._ez_client.find_systems.return_value = [ + bg_system, + bg_system_2, + bg_system_3, + bg_system_4, + bg_system_5, + ] # Expect 2.1.0, 2.1.1 valid # Expect 2.1.1 assert p.get_system_dependency("system~=2.1.0").version == bg_system_4.version - def test_wildcard_minor(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + def test_wildcard_minor( + plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 + ): p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + p._ez_client.find_systems.return_value = [ + bg_system, + bg_system_2, + bg_system_3, + bg_system_4, + bg_system_5, + ] # Expect 1.0.0, 2.0.0, 2.1.1, 3.0.0 valid # Expect 3.0.0 assert p.get_system_dependency("system==2.*").version == bg_system_4.version - def test_wildcard_patch(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + def test_wildcard_patch( + plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 + ): p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + p._ez_client.find_systems.return_value = [ + bg_system, + bg_system_2, + bg_system_3, + bg_system_4, + bg_system_5, + ] # Expect 1.0.0, 2.0.0, 2.1.1, 3.0.0 valid # Expect 3.0.0 assert p.get_system_dependency("system==2.1.*").version == bg_system_4.version - def test_excludes(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + def test_excludes( + plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 + ): p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + p._ez_client.find_systems.return_value = [ + bg_system, + bg_system_2, + bg_system_3, + bg_system_4, + bg_system_5, + ] # Expect 1.0.0, 2.0.0, 2.1.1, 3.0.0 valid # Expect 3.0.0 assert p.get_system_dependency("system!=2.1.0").version == bg_system_5.version def test_gt(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + p._ez_client.find_systems.return_value = [ + bg_system, + bg_system_2, + bg_system_3, + bg_system_4, + bg_system_5, + ] # Expect 2.1.1, 3.0.0 valid # Expect 3.0.0 assert p.get_system_dependency("system>2.1.0").version == bg_system_5.version def test_gte(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + p._ez_client.find_systems.return_value = [ + bg_system, + bg_system_2, + bg_system_3, + bg_system_4, + bg_system_5, + ] # Expect 2.1.0, 2.1.1, 3.0.0 valid # Expect 3.0.0 assert p.get_system_dependency("system>=2.1.0").version == bg_system_5.version def test_lt(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + p._ez_client.find_systems.return_value = [ + bg_system, + bg_system_2, + bg_system_3, + bg_system_4, + bg_system_5, + ] # Expect 1.0.0, 2.0.0 valid # Expect 2.0.0 assert p.get_system_dependency("system<2.1.0").version == bg_system_2.version def test_lte(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + p._ez_client.find_systems.return_value = [ + bg_system, + bg_system_2, + bg_system_3, + bg_system_4, + bg_system_5, + ] # Expect 1.0.0, 2.0.0, 2.1.0 valid # Expect 2.1.0 assert p.get_system_dependency("system<=2.1.0").version == bg_system_3.version - def test_range(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + def test_range( + plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 + ): p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + p._ez_client.find_systems.return_value = [ + bg_system, + bg_system_2, + bg_system_3, + bg_system_4, + bg_system_5, + ] # Expect 1.0.0 valid # Expect 1.0.0 assert p.get_system_dependency("system<2.0.0,>=1").version == bg_system.version - - def test_combo(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): + def test_combo( + plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 + ): p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value=[bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5] + p._ez_client.find_systems.return_value = [ + bg_system, + bg_system_2, + bg_system_3, + bg_system_4, + bg_system_5, + ] # Expect 1.0.0 valid # Expect 1.0.0 - assert p.get_system_dependency("system==2.*,<2.1.1,!=2.1.0").version == bg_system_2.version + assert ( + p.get_system_dependency("system==2.*,<2.1.1,!=2.1.0").version + == bg_system_2.version + ) From 6bfc402dbf59703a91c9c3ff8c83bb95d178c6d4 Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:20:26 +0000 Subject: [PATCH 7/9] update tests --- brewtils/plugin.py | 23 ++-- brewtils/test/fixtures.py | 10 ++ test/plugin_test.py | 243 +++++++++++++++++--------------------- 3 files changed, 128 insertions(+), 148 deletions(-) diff --git a/brewtils/plugin.py b/brewtils/plugin.py index 71f0069a..4b195232 100644 --- a/brewtils/plugin.py +++ b/brewtils/plugin.py @@ -502,25 +502,24 @@ def get_system_matching_version(self, require, **kwargs): req = Requirement(require) require_name = req.name require_version = req.specifier + systems = self._ez_client.find_systems(name=require_name, **kwargs) if require_version: - systems = self._ez_client.find_systems(name=require_name, **kwargs) valid_versions = list( require_version.filter( [str(Version(system.version)) for system in systems] ) ) - if valid_versions: - system_candidates = [ - system for system in systems if system.version in valid_versions - ] - system = system_candidates[0] - for system_candidate in system_candidates: - if Version(system_candidate.version) > Version(system.version): - system = system_candidate else: - system = self._ez_client.find_unique_system( - name=require_name, filter_latest=True, **kwargs - ) + valid_versions = [str(Version(system.version)) for system in systems] + + if valid_versions: + system_candidates = [ + system for system in systems if system.version in valid_versions + ] + system = system_candidates[0] + for system_candidate in system_candidates: + if Version(system_candidate.version) > Version(system.version): + system = system_candidate return system diff --git a/brewtils/test/fixtures.py b/brewtils/test/fixtures.py index 0c84996a..645fc4fe 100644 --- a/brewtils/test/fixtures.py +++ b/brewtils/test/fixtures.py @@ -324,6 +324,16 @@ def bg_system_5(system_dict, bg_instance, bg_command, bg_command_2): return System(**dict_copy) +@pytest.fixture +def bg_system_6(system_dict, bg_instance, bg_command, bg_command_2): + """A system with a different version.""" + dict_copy = copy.deepcopy(system_dict) + dict_copy["version"] = "3.0.0.dev0" + dict_copy["instances"] = [bg_instance] + dict_copy["commands"] = [bg_command, bg_command_2] + return System(**dict_copy) + + @pytest.fixture def child_request_dict(ts_epoch): """A child request represented as a dictionary.""" diff --git a/test/plugin_test.py b/test/plugin_test.py index b10e6e7d..86d65258 100644 --- a/test/plugin_test.py +++ b/test/plugin_test.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- +import copy import logging import logging.config import os import warnings +from packaging.requirements import InvalidRequirement +from packaging.version import InvalidVersion import pytest from mock import ANY, MagicMock, Mock @@ -369,6 +372,7 @@ def test_success( return_value=(admin_processor, request_processor) ) plugin._ez_client.find_unique_system = Mock(return_value=bg_system) + plugin._ez_client.find_systems = Mock(return_value=[bg_system]) plugin._startup() assert admin_processor.startup.called is True @@ -390,6 +394,7 @@ def test_success_no_ns( return_value=(admin_processor, request_processor) ) plugin._ez_client.find_unique_system = Mock(return_value=bg_system) + plugin._ez_client.find_systems = Mock(return_value=[bg_system]) plugin._startup() assert admin_processor.startup.called is True @@ -968,77 +973,77 @@ class TestDependencies(object): # 2.1.0 bg_system_3 # 2.1.1 bg_system_4 # 3.0.0 bg_system_5 - def test_no_specifier( - plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 - ): - p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_unique_system.return_value = bg_system_5 - # Expect 3.0.0 as latest valid version - # Expect 3.0.0 - assert p.get_system_dependency("system").version == bg_system_5.version - - def test_equals( - plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 - ): - p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value = [ - bg_system, - bg_system_2, - bg_system_3, - bg_system_4, - bg_system_5, - ] - # Expect 2.1.1 valid - # Expect 2.1.1 - assert p.get_system_dependency("system==2.1.0").version == bg_system_3.version - - def test_compatible_release( - plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 - ): - p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value = [ - bg_system, - bg_system_2, - bg_system_3, - bg_system_4, - bg_system_5, - ] - # Expect 2.1.0, 2.1.1 valid - # Expect 2.1.1 - assert p.get_system_dependency("system~=2.1.0").version == bg_system_4.version - - def test_wildcard_minor( - plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 - ): + # 3.0.0.dev0 bg_system_6 + @pytest.mark.parametrize( + "latest,versions", + [ + ("1.0.0", ["1.0.0"]), + ("2.0.0", ["1.0.0", "2.0.0"]), + ("1.2.0", ["1.0.0", "1.2.0"]), + ("1.0.0", ["1.0.0", "0.2.1rc1"]), + ("1.0.0rc1", ["1.0.0rc1", "0.2.1"]), + ("1.0.0rc1", ["1.0.0rc1", "0.2.1rc1"]), + ("1.0", ["1.0", "0.2.1"]), + ("1.0.0", ["1.0.0rc1", "1.0.0"]), + ], + ) + def test_determine_latest(client, bg_system, versions, latest): p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value = [ - bg_system, - bg_system_2, - bg_system_3, - bg_system_4, - bg_system_5, - ] - # Expect 1.0.0, 2.0.0, 2.1.1, 3.0.0 valid - # Expect 3.0.0 - assert p.get_system_dependency("system==2.*").version == bg_system_4.version + system_versions = [] + for version in versions: + s = copy.deepcopy(bg_system) + s.version = version + system_versions.append(s) + p._ez_client.find_systems.return_value = system_versions + assert p.get_system_dependency("system").version == latest - def test_wildcard_patch( - plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 - ): + @pytest.mark.parametrize( + "latest,versions", + [ + ("b", ["a", "b"]), + ("1.0.0", ["a", "b", "1.0.0"]), + ], + ) + def test_determine_latest_failures(client, bg_system, versions, latest): p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value = [ - bg_system, - bg_system_2, - bg_system_3, - bg_system_4, - bg_system_5, - ] - # Expect 1.0.0, 2.0.0, 2.1.1, 3.0.0 valid - # Expect 3.0.0 - assert p.get_system_dependency("system==2.1.*").version == bg_system_4.version + system_versions = [] + for version in versions: + s = copy.deepcopy(bg_system) + s.version = version + system_versions.append(s) + p._ez_client.find_systems.return_value = system_versions + with pytest.raises(InvalidVersion): + assert p.get_system_dependency("system").version == latest - def test_excludes( - plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 + @pytest.mark.parametrize( + "version_spec,latest", + [ + ("system", "3.0.0"), # test no specifier ignores pre-release + ("system==3.0.0.dev", "3.0.0.dev0"), # test version parsing + ("system==2.1.0", "2.1.0"), # test equals + ("system==3", "3.0.0"), # test equals no dev + ("system~=2.1.0", "2.1.1"), # test compatible release + ("system==2.*", "2.1.1"), # test minor wildcard + ("system==2.1.*", "2.1.1"), # test patch wildcard + ("system!=2.1.0", "3.0.0"), # test excludes + ("system>2.1.0", "3.0.0"), # test greater than + ("system>=2.1.0", "3.0.0"), # test greater than or equal + ("system<2.1.0", "2.0.0"), # test less than + ("system<=2.1.0", "2.1.0"), # test less than or equal + ("system<2.0.0,>=1", "1.0.0"), # test range + ("system==2.*,<2.1.1,!=2.1.0", "2.0.0"), # test combination + ], + ) + def test_version_specifier( + plugin, + bg_system, + bg_system_2, + bg_system_3, + bg_system_4, + bg_system_5, + bg_system_6, + version_spec, + latest, ): p = Plugin(bg_host="localhost", system=bg_system) p._ez_client.find_systems.return_value = [ @@ -1047,65 +1052,18 @@ def test_excludes( bg_system_3, bg_system_4, bg_system_5, + bg_system_6, ] - # Expect 1.0.0, 2.0.0, 2.1.1, 3.0.0 valid - # Expect 3.0.0 - assert p.get_system_dependency("system!=2.1.0").version == bg_system_5.version - - def test_gt(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): - p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value = [ - bg_system, - bg_system_2, - bg_system_3, - bg_system_4, - bg_system_5, - ] - # Expect 2.1.1, 3.0.0 valid - # Expect 3.0.0 - assert p.get_system_dependency("system>2.1.0").version == bg_system_5.version - - def test_gte(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): - p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value = [ - bg_system, - bg_system_2, - bg_system_3, - bg_system_4, - bg_system_5, - ] - # Expect 2.1.0, 2.1.1, 3.0.0 valid - # Expect 3.0.0 - assert p.get_system_dependency("system>=2.1.0").version == bg_system_5.version - - def test_lt(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): - p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value = [ - bg_system, - bg_system_2, - bg_system_3, - bg_system_4, - bg_system_5, - ] - # Expect 1.0.0, 2.0.0 valid - # Expect 2.0.0 - assert p.get_system_dependency("system<2.1.0").version == bg_system_2.version - - def test_lte(plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5): - p = Plugin(bg_host="localhost", system=bg_system) - p._ez_client.find_systems.return_value = [ - bg_system, - bg_system_2, - bg_system_3, - bg_system_4, - bg_system_5, - ] - # Expect 1.0.0, 2.0.0, 2.1.0 valid - # Expect 2.1.0 - assert p.get_system_dependency("system<=2.1.0").version == bg_system_3.version - - def test_range( - plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 + assert p.get_system_dependency(version_spec).version == latest + + def test_no_match( + plugin, + bg_system, + bg_system_2, + bg_system_3, + bg_system_4, + bg_system_5, + bg_system_6, ): p = Plugin(bg_host="localhost", system=bg_system) p._ez_client.find_systems.return_value = [ @@ -1114,13 +1072,29 @@ def test_range( bg_system_3, bg_system_4, bg_system_5, + bg_system_6, ] - # Expect 1.0.0 valid - # Expect 1.0.0 - assert p.get_system_dependency("system<2.0.0,>=1").version == bg_system.version + p._wait = Mock(return_value=None) + with pytest.raises(PluginValidationError): + assert p.get_system_dependency("system==3.0.1.dev0").version - def test_combo( - plugin, bg_system, bg_system_2, bg_system_3, bg_system_4, bg_system_5 + @pytest.mark.parametrize( + "version_spec", + [ + "system==*", + "system==a", + "system$$3.0.0", + ], + ) + def test_invalid_requirement( + plugin, + bg_system, + bg_system_2, + bg_system_3, + bg_system_4, + bg_system_5, + bg_system_6, + version_spec, ): p = Plugin(bg_host="localhost", system=bg_system) p._ez_client.find_systems.return_value = [ @@ -1129,10 +1103,7 @@ def test_combo( bg_system_3, bg_system_4, bg_system_5, + bg_system_6, ] - # Expect 1.0.0 valid - # Expect 1.0.0 - assert ( - p.get_system_dependency("system==2.*,<2.1.1,!=2.1.0").version - == bg_system_2.version - ) + with pytest.raises(InvalidRequirement): + p.get_system_dependency(version_spec) From 0898cc3b98abf1423a6bad04b72c5c553ff53297 Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Fri, 22 Nov 2024 12:39:36 +0000 Subject: [PATCH 8/9] fix comparison --- brewtils/plugin.py | 10 ++++++---- test/plugin_test.py | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/brewtils/plugin.py b/brewtils/plugin.py index 4b195232..4cfb829c 100644 --- a/brewtils/plugin.py +++ b/brewtils/plugin.py @@ -514,7 +514,9 @@ def get_system_matching_version(self, require, **kwargs): if valid_versions: system_candidates = [ - system for system in systems if system.version in valid_versions + system + for system in systems + if str(Version(system.version)) in valid_versions ] system = system_candidates[0] for system_candidate in system_candidates: @@ -530,9 +532,9 @@ def get_system_dependency(self, require, timeout=300): require, filter_running=True, local=True ) if system: - self.logger.debug(f"Found system: {system}") + self._logger.debug(f"Found system: {system}") return system - self.logger.error( + self._logger.error( f"Waiting {wait_time:.1f} seconds before next attempt for {self._system} " f"dependency for {require}" ) @@ -547,7 +549,7 @@ def get_system_dependency(self, require, timeout=300): def await_dependencies(self, requires, config): for req in requires: system = self.get_system_dependency(req, config.requires_timeout) - self.logger.debug( + self._logger.debug( f"Resolved system {system} for {req}: {config.name} {config.instance_name}" ) diff --git a/test/plugin_test.py b/test/plugin_test.py index 86d65258..eae93048 100644 --- a/test/plugin_test.py +++ b/test/plugin_test.py @@ -985,6 +985,8 @@ class TestDependencies(object): ("1.0.0rc1", ["1.0.0rc1", "0.2.1rc1"]), ("1.0", ["1.0", "0.2.1"]), ("1.0.0", ["1.0.0rc1", "1.0.0"]), + ("3.0.0.dev0", ["3.0.0.dev0", "3.0.0.dev"]), + ("3.0.0.dev", ["3.0.0.dev", "2.0.0"]), ], ) def test_determine_latest(client, bg_system, versions, latest): From 2df9260766be6c152514f70e6c8217ad1d0227ab Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Fri, 22 Nov 2024 15:01:06 +0000 Subject: [PATCH 9/9] remove unused import --- brewtils/plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/brewtils/plugin.py b/brewtils/plugin.py index 4cfb829c..0d0e0213 100644 --- a/brewtils/plugin.py +++ b/brewtils/plugin.py @@ -4,7 +4,6 @@ import logging import logging.config import os -import re import signal import sys import threading