Use the configurable digest_name for simple JSON (#1442)

* Use the configurable digest_name for simple JSON

PEP 691 specifies that each file's hashes dictionary must contain only
hash algorithms that can be passed, without arguments, to hashlib.new().
PyPI has started using a "blake2b_256" key for digests, which is blake2b
with the digest size set to 32 bytes, and makes it so that contents of
"digests" can't just be reused for the hashes dictionary.

This commit uses the value of configuration option digest_name to pick
a single supported key/value pair from the digests for use in the hashes
dictionary.

* Fix format value lookup

list.sort() modifies a list in-place and returns None.

* Define digests supported by Simple API files

* Update simple API tests for digest changes

* Validate digest_name config using simple module

* Update digest name test in test_main module

Switch InvalidDigestFormat base class to ValueError.

* Add PR #1442 changelog entry

* Fix StrEnum usage on Python >=3.11

CHANGES.md

@@ -1,3 +1,10 @@
+# Unreleased
+
+## Bug Fixes
+
+- Fix digest used for file hashes in PEP 691 simple JSON file output `PR #1442`
+  - The `digest_name` setting from configuration (default value: `sha256`) will now be used for both HTML and JSON files.
+
 # 6.2.0
 
 ## New Features

src/bandersnatch/configuration.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 from typing import Any, Dict, List, NamedTuple, Optional, Type
 
-from .simple import SimpleFormat, get_format_value
+from .simple import SimpleDigest, SimpleFormat, get_digest_value, get_format_value
 
 logger = logging.getLogger("bandersnatch")
 
@@ -131,15 +131,17 @@ def validate_config_values(  # noqa: C901
     logger.info(f"Selected storage backend: {storage_backend_name}")
 
     try:
-        digest_name = config.get("mirror", "digest_name")
+        digest_name = get_digest_value(config.get("mirror", "digest_name"))
     except configparser.NoOptionError:
-        digest_name = "sha256"
-    if digest_name not in ("md5", "sha256"):
-        raise ValueError(
-            f"Supplied digest_name {digest_name} is not supported! Please "
-            + "update digest_name to one of ('sha256', 'md5') in the [mirror] "
-            + "section."
+        digest_name = SimpleDigest.SHA256
+        logger.debug(f"Using digest {digest_name} by default ...")
+    except ValueError as e:
+        logger.error(
+            f"Supplied digest_name {config.get('mirror', 'digest_name')} is "
+            + "not supported! Please update the digest_name in the [mirror] "
+            + "section of your config to a supported digest value."
         )
+        raise e
 
     try:
         cleanup = config.getboolean("mirror", "cleanup")

src/bandersnatch/simple.py

@@ -1,6 +1,7 @@
 import html
 import json
 import logging
+import sys
 from enum import Enum, auto
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Union
@@ -11,6 +12,11 @@
 if TYPE_CHECKING:
     from .storage import Storage
 
+if sys.version_info >= (3, 11):
+    from enum import StrEnum
+else:
+    from .utils import StrEnum
+
 
 class SimpleFormats(NamedTuple):
     html: str
@@ -23,6 +29,16 @@ class SimpleFormat(Enum):
     JSON = auto()
 
 
+class SimpleDigests(NamedTuple):
+    sha256: str
+    md5: str
+
+
+class SimpleDigest(StrEnum):
+    SHA256 = "sha256"
+    MD5 = "md5"
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -32,17 +48,34 @@ class InvalidSimpleFormat(KeyError):
     pass
 
 
+class InvalidDigestFormat(ValueError):
+    """We don't have a valid digest choice from configuration"""
+
+    pass
+
+
 def get_format_value(format: str) -> SimpleFormat:
     try:
         return SimpleFormat[format.upper()]
     except KeyError:
-        valid_formats = [v.name for v in SimpleFormat].sort()
+        valid_formats = sorted([v.name for v in SimpleFormat])
         raise InvalidSimpleFormat(
             f"{format.upper()} is not a valid Simple API format. "
             + f"Valid Options: {valid_formats}"
         )
 
 
+def get_digest_value(digest: str) -> SimpleDigest:
+    try:
+        return SimpleDigest[digest.upper()]
+    except KeyError:
+        valid_digests = sorted([v.name for v in SimpleDigest])
+        raise InvalidDigestFormat(
+            f"{digest} is not a valid Simple API file hash digest. "
+            + f"Valid Options: {valid_digests}"
+        )
+
+
 class SimpleAPI:
     """Handle all Simple API file generation"""
 
@@ -56,12 +89,16 @@ def __init__(
         storage_backend: "Storage",
         format: Union[SimpleFormat, str],
         diff_file_list: List[Path],
-        digest_name: str,
+        digest_name: Union[SimpleDigest, str],
         hash_index: bool,
         root_uri: Optional[str],
     ) -> None:
         self.diff_file_list = diff_file_list
-        self.digest_name = digest_name
+        self.digest_name = (
+            get_digest_value(digest_name)
+            if isinstance(digest_name, str)
+            else digest_name
+        )
         self.format = get_format_value(format) if isinstance(format, str) else format
         self.hash_index = hash_index
         self.root_uri = root_uri
@@ -190,8 +227,7 @@ def generate_json_simple_page(
                 {
                     "filename": r["filename"],
                     "hashes": {
-                        digest_name: digest_hash
-                        for digest_name, digest_hash in r["digests"].items()
+                        self.digest_name: r["digests"][self.digest_name],
                     },
                     "requires-python": r.get("requires_python", ""),
                     "url": self._file_url_to_local_url(r["url"]),

src/bandersnatch/tests/test_main.py

@@ -128,7 +128,7 @@ def test_main_throws_exception_on_unsupported_digest_name(
     with pytest.raises(ValueError) as e:
         main(asyncio.new_event_loop())
 
-    assert "foobar is not supported" in str(e.value)
+    assert "foobar is not a valid" in str(e.value)
 
 
 @pytest.fixture

src/bandersnatch/tests/test_simple.py

@@ -6,8 +6,15 @@
 import pytest
 
 from bandersnatch import utils
+from bandersnatch.configuration import validate_config_values
 from bandersnatch.package import Package
-from bandersnatch.simple import InvalidSimpleFormat, SimpleAPI, SimpleFormat
+from bandersnatch.simple import (
+    InvalidDigestFormat,
+    InvalidSimpleFormat,
+    SimpleAPI,
+    SimpleDigest,
+    SimpleFormat,
+)
 from bandersnatch.storage import Storage
 from bandersnatch.tests.test_simple_fixtures import (
     EXPECTED_SIMPLE_GLOBAL_JSON_PRETTY,
@@ -20,16 +27,35 @@
 
 def test_format_invalid() -> None:
     with pytest.raises(InvalidSimpleFormat):
-        SimpleAPI(Storage(), "l33t", [], "digest", False, None)
+        SimpleAPI(Storage(), "l33t", [], "sha256", False, None)
 
 
 def test_format_valid() -> None:
-    s = SimpleAPI(Storage(), "ALL", [], "digest", False, None)
+    s = SimpleAPI(Storage(), "ALL", [], "sha256", False, None)
     assert s.format == SimpleFormat.ALL
 
 
+def test_digest_invalid() -> None:
+    with pytest.raises(InvalidDigestFormat):
+        SimpleAPI(Storage(), "ALL", [], "digest", False, None)
+
+
+def test_digest_valid() -> None:
+    s = SimpleAPI(Storage(), "ALL", [], "md5", False, None)
+    assert s.digest_name == SimpleDigest.MD5
+
+
+def test_digest_config_default() -> None:
+    c = ConfigParser()
+    c.add_section("mirror")
+    config = validate_config_values(c)
+    s = SimpleAPI(Storage(), "ALL", [], config.digest_name, False, None)
+    assert config.digest_name.upper() in [v.name for v in SimpleDigest]
+    assert s.digest_name == SimpleDigest.SHA256
+
+
 def test_json_package_page() -> None:
-    s = SimpleAPI(Storage(), SimpleFormat.JSON, [], "digest", False, None)
+    s = SimpleAPI(Storage(), SimpleFormat.JSON, [], SimpleDigest.SHA256, False, None)
     p = Package("69")
     p._metadata = SIXTYNINE_METADATA
     assert EXPECTED_SIMPLE_SIXTYNINE_JSON == s.generate_json_simple_page(p)
@@ -44,7 +70,7 @@ def test_json_index_page() -> None:
     c.add_section("mirror")
     c["mirror"]["workers"] = "1"
     s = SimpleAPI(
-        FilesystemStorage(config=c), SimpleFormat.ALL, [], "digest", False, None
+        FilesystemStorage(config=c), SimpleFormat.ALL, [], "sha256", False, None
     )
     with TemporaryDirectory() as td:
         td_path = Path(td)

src/bandersnatch/tests/test_simple_fixtures.py

@@ -109,7 +109,7 @@
 }
 
 EXPECTED_SIMPLE_SIXTYNINE_JSON = """\
-{"files": [{"filename": "69-0.69.tar.gz", "hashes": {"md5": "4328d962656395fbd3e730c9d30bb48c", "sha256": "5c11f48399f9b1bca802751513f1f97bff6ce97e6facb576b7729e1351453c10"}, "requires-python": ">=3.6", "url": "../../packages/d3/cc/95dc5434362bd333a1fec275231775d748315b26edf1e7e568e6f8660238/69-0.69.tar.gz", "yanked": false}, {"filename": "69-6.9.tar.gz", "hashes": {"md5": "ff4bf804ef3722a1fd8853a8a32513d4", "sha256": "0c8deb7c8574787283c3fc08b714ee63fd6752a38d13515a9d8508798d428597"}, "requires-python": ">=3.6", "url": "../../packages/7b/6e/7c4ce77c6ca092e94e19b78282b459e7f8270362da655cbc6a75eeb9cdd7/69-6.9.tar.gz", "yanked": false}], "meta": {"api-version": "1.0", "_last-serial": "10333928"}, "name": "69"}\
+{"files": [{"filename": "69-0.69.tar.gz", "hashes": {"sha256": "5c11f48399f9b1bca802751513f1f97bff6ce97e6facb576b7729e1351453c10"}, "requires-python": ">=3.6", "url": "../../packages/d3/cc/95dc5434362bd333a1fec275231775d748315b26edf1e7e568e6f8660238/69-0.69.tar.gz", "yanked": false}, {"filename": "69-6.9.tar.gz", "hashes": {"sha256": "0c8deb7c8574787283c3fc08b714ee63fd6752a38d13515a9d8508798d428597"}, "requires-python": ">=3.6", "url": "../../packages/7b/6e/7c4ce77c6ca092e94e19b78282b459e7f8270362da655cbc6a75eeb9cdd7/69-6.9.tar.gz", "yanked": false}], "meta": {"api-version": "1.0", "_last-serial": "10333928"}, "name": "69"}\
 """
 
 EXPECTED_SIMPLE_SIXTYNINE_JSON_PRETTY = """\
@@ -118,7 +118,6 @@
         {
             "filename": "69-0.69.tar.gz",
             "hashes": {
-                "md5": "4328d962656395fbd3e730c9d30bb48c",
                 "sha256": "5c11f48399f9b1bca802751513f1f97bff6ce97e6facb576b7729e1351453c10"
             },
             "requires-python": ">=3.6",
@@ -128,7 +127,6 @@
         {
             "filename": "69-6.9.tar.gz",
             "hashes": {
-                "md5": "ff4bf804ef3722a1fd8853a8a32513d4",
                 "sha256": "0c8deb7c8574787283c3fc08b714ee63fd6752a38d13515a9d8508798d428597"
             },
             "requires-python": ">=3.6",

src/bandersnatch/utils.py

@@ -9,6 +9,7 @@
 import sys
 import tempfile
 from datetime import datetime
+from enum import Enum
 from pathlib import Path
 from typing import IO, Any, Generator, List, Set, Union
 from urllib.parse import urlparse
@@ -37,6 +38,15 @@ def user_agent() -> str:
 WINDOWS = bool(platform.system() == "Windows")
 
 
+class StrEnum(str, Enum):
+    """Enumeration class where members can be treated as strings."""
+
+    value: str
+
+    def __str__(self) -> str:
+        return self.value
+
+
 def make_time_stamp() -> str:
     """Helper function that returns a timestamp suitable for use
     in a filename on any OS"""