Use uv to manage and generate dependencies (#4232)

- Use frozen dependencies with hashes to ensure reproducible dependency
installation in a secure way.
- Uses `uv` and pre-commit to automatically generate the locked
requirements files from pyproject.toml.
- While the translation dependencies can be installed directly with
`uv`, uv doesn’t seem to be able to add `--find-link=…` to the exported
`requirements/translate.txt`, for that reason the translation
dependencies are left as they are and won’t be automatically exported
via pre-commit.
- So whenever `uv lock —upgrade` is run, it will freeze the latest patch
versions.
- The production dependencies are specified with a `~` which means `>=`
at the last digit of requirements.
- The dev dependencies have `>=` to make them easy to update, if
something breaks in dev either fix it or go back to the old frozen
dependencies.
- Moved generated dependencies in requirements/ folder.
- Use `--no-deps` where applicable while installing these generated
requirements as the sub-dependencies are already resolved.

.github/workflows/deploy-docs.yml

@@ -29,4 +29,4 @@ jobs:
           CUSTOM_DOMAIN: docs.hypha.app
           CONFIG_FILE: mkdocs.yml
           EXTRA_PACKAGES: build-base
-          REQUIREMENTS: requirements-docs.txt
+          REQUIREMENTS: requirements/docs.txt

.github/workflows/hypha-ci.yml

@@ -75,8 +75,7 @@ jobs:
       - name: Install python dependencies
         run: |
           uv venv
-          uv pip install -r requirements-dev.txt
-          uv pip install -r requirements-translate.txt
+          uv pip install -r requirements/dev.txt -r requirements/translate.txt
 
       - name: Check Django migrations
         if: matrix.group == 1

.pre-commit-config.yaml

@@ -1,4 +1,40 @@
 repos:
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    # uv version.
+    rev: 0.5.2
+    hooks:
+      - id: uv-export
+        name: uv-export requirements.txt
+        args:
+          [
+            "--frozen",
+            "--no-group",
+            "dev",
+            "--no-group",
+            "docs",
+            "-o",
+            "requirements/prod.txt",
+          ]
+      - id: uv-export
+        name: uv-export requirements/dev.txt
+        args: ["--frozen", "--group", "dev", "-o", "requirements/dev.txt"]
+      - id: uv-export
+        name: uv-export requirements/docs.txt
+        args:
+          ["--frozen", "--only-group", "docs", "-o", "requirements/docs.txt"]
+
+      # This is disabled because this doesn't export --find-links. The requirements-translate is maintained manually.
+      # - id: uv-export
+      #   name: uv-export requirements/translate.txt
+      #   args:
+      #     [
+      #       "--frozen",
+      #       "--only-group",
+      #       "translate",
+      #       "-o",
+      #       "requirements/translate.txt",
+      #     ]
+
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.7.0
     hooks:

Makefile

@@ -79,9 +79,9 @@ download-esm-modules:  ## Download ECMAScript modules for the project
 	@touch $@
 
 
-.cache/py-packages: requirements-dev.txt requirements-docs.txt  ## Install Python packages for development and documentation
+.cache/py-packages: requirements/dev.txt requirements/docs.txt  ## Install Python packages for development and documentation
 	@mkdir -p $$(dirname $@)
-	$(PIP) install -r requirements-dev.txt -r requirements-docs.txt
+	$(PIP) install --no-deps -r requirements/dev.txt -r requirements/docs.txt
 	@touch $@
 
 

docker/Dockerfile.dev

@@ -14,8 +14,8 @@ ENV PYTHONUNBUFFERED 1
 RUN sudo chown -R circleci:circleci /usr/local
 
 # Install python dependencies.
-COPY requirements.txt requirements-dev.txt /usr/local/hypha/
-RUN pip3 install --quiet -r requirements-dev.txt
+COPY requirements.txt requirements/dev.txt /usr/local/hypha/
+RUN pip3 install --quiet -r requirements/dev.txt
 
 # Run entrypoint.sh.
 ENTRYPOINT ["/usr/local/hypha/docker/entrypoint.dev.sh"]

docs/setup/administrators/machine-translations.md

@@ -8,7 +8,7 @@ Hypha has the ability to utilize [argostranslate](https://github.com/argosopente
 As referenced in the [production deployment guide](../deployment/production/stand-alone.md), it is required to install the dependencies needed for machine translation dependencies via
 
 ```bash
-python3 -m pip install -r requirements-translate.txt
+python3 -m pip install -r requirements/translate.txt
 ```
 
 This requirements file will specifically attempt to install the CPU version of [PyTorch](https://pytorch.org/) if available on the detected platform to play better with heroku (doesn't support GPU processing) and to minimize package bloat (CPU package is ~300MB less than the normal GPU). Depending on your use case, you may want to adjust this.
@@ -26,4 +26,4 @@ python3 manage.py install_languages ar_en fr_en
 
 ## Enabling on the system
 
-To enable machine translations on an instance, the proper configuration variables need to be set. These can be found in the [configuration options](configuration.md#hypha-custom-settings)
+To enable machine translations on an instance, the proper configuration variables need to be set. These can be found in the [configuration options](configuration.md#hypha-custom-settings)

docs/setup/deployment/development/stand-alone.md

@@ -275,13 +275,13 @@ Activate your virtual environment and install dependencies:
 
 ```shell
 source venv/bin/activate
-python3 -m pip install -r requirements-dev.txt
+python3 -m pip install --no-deps -r requirements/dev.txt
 ```
 
 If utilizing application machine translations, install the required dependencies:
 
 ```shell
-python3 -m pip install -r requirements-translate.txt
+python3 -m pip install -r requirements/translate.txt
 ```
 
 Run:

docs/setup/deployment/production/stand-alone.md

@@ -102,13 +102,13 @@ Inside your activated virtual environment you will use plain `python` and `pip`
 Next, install the required packages using:
 
 ```shell
-python3 -m pip install -r requirements.txt
+python3 -m pip install --no-deps -r requirements.txt
 ```
 
 If utilizing application machine translations, install the required dependencies:
 
 ```shell
-python3 -m pip install -r requirements-translate.txt
+python3 -m pip install -r requirements/translate.txt
 ```
 
 ### Install Node packages
@@ -277,4 +277,3 @@ SOCIAL_AUTH_GOOGLE_OAUTH2_KEY:                 [KEY]
 SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET:              [KEY]
 SOCIAL_AUTH_GOOGLE_OAUTH2_WHITELISTED_DOMAINS: example.org
 ```
-

hypha/apply/funds/tests/test_views.py

@@ -64,15 +64,15 @@ def prepare_form_data(submission, **kwargs):
 
 
 def check_form_fields_equality(form_fields_1, form_fields_2):
-    for form_field_1, form_field_2 in zip(form_fields_1, form_fields_2):
+    for form_field_1, form_field_2 in zip(form_fields_1, form_fields_2, strict=False):
         if form_field_1.block != form_field_2.block:  # The block types must be the same
             return False
 
         # Check if the form field values are StructValues (composite types)
         if isinstance(form_field_1.value, wagtail.blocks.StructValue):
             # Iterate through both StructValue fields and compare their key-value pairs
             for (key_1, field_value_1), (key_2, field_value_2) in zip(
-                form_field_1.value.items(), form_field_2.value.items()
+                form_field_1.value.items(), form_field_2.value.items(), strict=False
             ):
                 if key_1 != key_2:  # Keys (field_labels, etc.) must match
                     return False
@@ -86,7 +86,7 @@ def check_form_fields_equality(form_fields_1, form_fields_2):
                     # If the values are ListValues (e.g., multiple choice), compare their individual elements
                     if isinstance(field_value_1, wagtail.blocks.list_block.ListValue):
                         for item_value_1, item_value_2 in zip(
-                            field_value_1, field_value_2
+                            field_value_1, field_value_2, strict=False
                         ):
                             if item_value_1 != item_value_2:
                                 return False

hypha/settings/base.py

@@ -187,7 +187,7 @@
 LANGUAGE_CODE = env.str("LANGUAGE_CODE", "en")
 
 # Machine translation settings
-# NOTE: Ensure the packages in `requirements-translate.txt` have been installed!
+# NOTE: Ensure the packages in `requirements/translate.txt` have been installed!
 APPLICATION_TRANSLATIONS_ENABLED = env.bool("APPLICATION_TRANSLATIONS_ENABLED", False)
 
 # Number of seconds that password reset and account activation links are valid (default 259200, 3 days).

pyproject.toml

@@ -2,52 +2,150 @@
 name = "hypha"
 description = "A open source submission management platform to receive and manage applications for funding."
 readme = "README.md"
+version = "5.19.0"
+requires-python = ">=3.10"
+license = { text = "BSD-3-Clause" }
 
-[tool.pytest.ini_options]
-DJANGO_SETTINGS_MODULE = 'hypha.settings.test'
-addopts = [
-    '-n=auto',
-    '--failed-first',
+dependencies = [
+    "scout-apm~=3.1.0",
+    "sentry-sdk~=2.8.0",
+    "Babel~=2.14.0",
+    "boto3~=1.34.66",
+    "celery~=5.3.6",
+    "click~=8.1.7",
+    "dj-database-url~=2.1.0",
+    "django-anymail~=10.3",
+    "django-basic-auth-ip-whitelist~=0.5",
+    "django-countries~=7.5.1",
+    "django-elevate~=2.0.3",
+    "django-extensions~=3.2.3",
+    "django-file-form~=3.6.0",
+    "django-filter==23.5",
+    "django-formtools~=2.5.1",
+    "django-fsm~=2.8.1",
+    "django-heroku~=0.3.1",
+    "django-hijack~=3.4.5",
+    "django-htmx~=1.17",
+    "django-nh3~=0.1.1",
+    "django-pagedown~=2.2.1",
+    "django-ratelimit~=4.1.0",
+    "django-referrer-policy~=1.0",
+    "django-role-permissions~=3.2.0",
+    "django-select2~=8.1.2",
+    "django-slack~=5.19.0",
+    "django-storages~=1.14.2",
+    "django-tables2~=2.7.0",
+    "django-tinymce~=4.1.0",
+    "django-two-factor-auth~=1.16.0",
+    "django-web-components~=0.2.0",
+    "django~=4.2.0",
+    "djangorestframework-api-key~=3.0.0",
+    "djangorestframework~=3.15.2",
+    "drf-nested-routers~=0.93.5",
+    "djp~=0.3.1",
+    "environs~=11.0.0",
+    "gunicorn~=22.0.0",
+    "heroicons~=2.6.0",
+    "python-docx~=1.1.0",
+    "htmldocx~=0.0.6",
+    "lark~=1.1.9",
+    "mistune~=3.0.2",
+    "more-itertools~=10.2.0",
+    "nh3~=0.2.18",
+    "phonenumberslite~=8.13.32",
+    "Pillow~=10.3.0",
+    "psycopg[binary]~=3.1.18",
+    "pwned-passwords-django~=2.1",
+    "qrcode~=7.4.2",
+    "reportlab~=4.0.9",
+    "setuptools>=75.1.0",
+    "social_auth_app_django~=5.4.1",
+    "svgwrite~=1.4.3",
+    "tablib~=3.5.0",
+    "tomd~=0.1.3",
+    "wagtail~=5.2.6",
+    "whitenoise~=6.6.0",
+    "xhtml2pdf==0.2.15",
+    "python-bidi~=0.4.2",
+    "xmltodict~=0.13.0",
+]
+
+[dependency-groups]
+"dev" = [
+    "coverage>=7.4.4",
+    "django-browser-reload>=1.16.0",
+    "django-coverage-plugin>=3.1.0",
+    "django-debug-toolbar>=4.3.0",
+    "django-dynamic-fixture>=4.0.1",
+    "djhtml>=3.0.5",
+    "dslr>=0.4.0",
+    "factory_boy>=3.3.1",
+    "Faker>=28.4.1",
+    "freezegun>=1.5.1",
+    "model-bakery>=1.17.0",
+    "pre-commit>=4.0.1",
+    "pytest-cov>=4.1.0",
+    "pytest-django>=4.9.0",
+    "pytest-split>=0.10.0",
+    "pytest-xdist[psutil]>=3.6.1",
+    "responses>=0.25.0",
+    "ruff>=0.7.0",
+    "time-machine>=2.16.0",
+    "wagtail-factories>=4.2.1",
+    "Werkzeug>=3.0.6",
 ]
-python_files = [
-    'tests.py',
-    'test_*.py',
-    '*_tests.py',
+"docs" = [
+    "mkdocs-material>=9.5.5",
+    "mkdocs-awesome-pages-plugin>=2.9.2",
+    "mkdocs-git-revision-date-localized-plugin>=1.2.2",
+    "mkdocs-macros-plugin>=1.0.4",
+    "mkdocstrings[python]>=0.24.0",
+    "mkdocs-gen-files>=0.5.0",
+    "griffe>=0.49.0",
 ]
-testpaths = [
-    "hypha"
+
+# Requirements for machine translations
+# NOTE: If you update this section please manual update `requirements/translate.txt` as well.
+"translate" = [
+    # Only install the CPU version of torch when available (linux)
+    "torch==2.3.1+cpu; sys_platform == 'linux'",
+    "argostranslate~=1.9.6",
 ]
+
+[tool.uv]
+default-groups = ["dev", "docs"]
+find-links = ["https://download.pytorch.org/whl/torch_stable.html"]
+
+[tool.pytest.ini_options]
+DJANGO_SETTINGS_MODULE = 'hypha.settings.test'
+addopts = ['-n=auto', '--failed-first']
+python_files = ['tests.py', 'test_*.py', '*_tests.py']
+testpaths = ["hypha"]
 filterwarnings = [
     'ignore::DeprecationWarning',
     'ignore::PendingDeprecationWarning',
 ]
 
 [tool.coverage.run]
 parallel = true
-plugins = [
-    'django_coverage_plugin'
-]
-omit = [
-    '*migrations*',
-    '*test*',
-    'hypa/public/*',
-]
+plugins = ['django_coverage_plugin']
+omit = ['*migrations*', '*test*', 'hypa/public/*']
 
 # https://github.com/charliermarsh/ruff#ruff
 [tool.ruff.lint]
 ignore = [
-    "E501",  # line too long
-    "C901",  # too complex
+    "E501", # line too long
+    "C901", # too complex
     # 'F821',
     # 'W605',
 ]
 select = [
-    'C',  # flake8-comprehensions
-    'B',  # flake8-bugbear
-    'E',  # pycodestyle errors
-    'F',  # pyflakes
-    'I',  # iSort
-    'W',  # pycodestyle warnings
+    'C', # flake8-comprehensions
+    'B', # flake8-bugbear
+    'E', # pycodestyle errors
+    'F', # pyflakes
+    'I', # iSort
+    'W', # pycodestyle warnings
 ]
 
 [tool.ruff.lint.per-file-ignores]