Latest and version pinning clash (#1601)

* #1599 - disable additional package version filtering if pinned version filtering is applied

* #1599 - disable additional package version filtering if pinned version filtering is applied (remove commented code)

* #1599 - add unit test for combined pinned and latest version filtering as well as clarifying comments in documentation

* #1599 - add changelog entry for combined pinned and latest version filtering

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update src/bandersnatch_filter_plugins/allowlist_name.py

* #1599 - clean added unit test

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Cooper Lees <[email protected]>

CHANGES.md

@@ -1,3 +1,7 @@
+# 6.5.0
+
+- `project_requirements_pinned` with a pinned version (range) disables additional release filter for this package `PR #1601`
+
 # 6.4.0
 
 - Move JSON Simple API to version 1.1 (as per PEP700) `PR #1557`

docs/filtering_configuration.md

@@ -13,6 +13,10 @@ converted.
 E.g. to Blocklist [discord.py](https://pypi.org/project/discord.py/) the string 'discord-py'
 is correct, but 'discord.PY' will also work.
 
+Plugins for release version filtering usually act in a way, that releases are only downloaded if all filter plugin rules are satisfied.
+An exception to this rule is the `project_requirements_pinned` filter: if there is a version number/range specified no other filter are applied.
+This allows smaller mirrors with newest versions and specifically needed ones.
+
 ## Plugins Enabling
 
 The plugins setting is a list of plugins to enable.

src/bandersnatch/filter.py

@@ -66,6 +66,17 @@ def initialize_plugin(self) -> None:
         # and check_match methods that are called in the fast path.
         pass
 
+    def pinned_version_exists(self, metadata: dict) -> bool:
+        """
+        Check if version specifier exist.
+
+        Returns
+        =======
+        bool:
+            True if version specifier exist, False otherwise
+        """
+        return False
+
     def filter(self, metadata: dict) -> bool:
         """
         Check if the plugin matches based on the package's metadata.

src/bandersnatch/package.py

@@ -96,14 +96,36 @@ def filter_all_releases(self, release_filters: list["Filter"]) -> bool:
         Filter releases and removes releases that fail the filters
         """
         releases = list(self.releases.keys())
-        for version in releases:
-            release_data = {
-                "version": version,
-                "releases": self.releases,
-                "info": self.info,
-            }
-            if not all(plugin.filter(release_data) for plugin in release_filters):
-                del self.releases[version]
+        release_data = {
+            "info": self.info,
+        }
+        pinned_version = False
+        pinned_plugin = -1
+        for plugin in release_filters:
+            pinned_plugin += 1
+            if plugin.name == "project_requirements_pinned":
+                if plugin.pinned_version_exists(release_data):
+                    pinned_version = True
+                    break
+        if pinned_version:
+            pinned_filter = release_filters[pinned_plugin]
+            for version in releases:
+                release_data = {
+                    "version": version,
+                    "releases": self.releases,
+                    "info": self.info,
+                }
+                if not pinned_filter.filter(release_data):
+                    del self.releases[version]
+        else:
+            for version in releases:
+                release_data = {
+                    "version": version,
+                    "releases": self.releases,
+                    "info": self.info,
+                }
+                if not all(plugin.filter(release_data) for plugin in release_filters):
+                    del self.releases[version]
         if releases:
             return True
         return False

src/bandersnatch/tests/plugins/test_allowlist_name.py

@@ -406,6 +406,40 @@ def test__filter__matches__release(self) -> None:
 
         self.assertEqual({"1.2.0": {}}, pkg.releases)
 
+    def test__filter__matches__release_latest(self) -> None:
+        with open(Path(self.tempdir.name) / "requirements.txt", "w") as fh:
+            fh.write("""\
+foo==1.2.0             # via -r requirements.in
+""")
+
+        mock_config(f"""\
+[mirror]
+storage-backend = filesystem
+
+[plugins]
+enabled =
+    project_requirements
+    project_requirements_pinned
+    latest_release
+[latest_release]
+keep = 2
+[allowlist]
+requirements_path = {self.tempdir.name}
+requirements =
+    requirements.txt
+""")
+
+        mirror = BandersnatchMirror(Path("."), Master(url="https://foo.bar.com"))
+        pkg = Package("foo", 1)
+        pkg._metadata = {
+            "info": {"name": "foo"},
+            "releases": {"1.2.0": {}, "1.2.1": {}, "1.2.2": {}},
+        }
+
+        pkg.filter_all_releases(mirror.filters.filter_release_plugins())
+
+        self.assertEqual({"1.2.0": {}}, pkg.releases)
+
     def test__filter__find_files(self) -> None:
         absolute_file_path = Path(self.tempdir.name) / "requirements.txt"
         with open(absolute_file_path, "w") as fh:

src/bandersnatch_filter_plugins/allowlist_name.py

@@ -202,6 +202,14 @@ def _determine_filtered_package_requirements(self) -> list[Requirement]:
             package_lines = []
         return list(_parse_package_lines(package_lines))
 
+    def pinned_version_exists(self, metadata: dict) -> bool:
+        name = canonicalize_name(metadata["info"]["name"])
+        for requirement in self.allowlist_release_requirements:
+            if name != requirement.name:
+                continue
+            return len(requirement.specifier) > 0
+        return False
+
     def filter(self, metadata: dict) -> bool:
         """
         Returns False if version fails the filter,