Upgrade Python dependencies

* Add make rules to manage AMAUATs Python deps
* Update submodules
* Update GitHub CI workflows
* Upgrade pre-commit dependencies
* Remove vcrpy

.github/workflows/acceptance-test.yml

@@ -30,7 +30,7 @@ jobs:
             browser: Firefox
     steps:
       - name: "Check out repository"
-        uses: "actions/checkout@v3"
+        uses: "actions/checkout@v4"
         with:
           submodules: true
       - name: "Create external volumes"

.github/workflows/test.yml

@@ -13,35 +13,32 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        rule: ["mcp-server", "mcp-client", "dashboard", "archivematica-common"]
-        python-version: ["3.8", "3.9", "3.10"]
+        rule: [
+          "mcp-server",
+          "mcp-client",
+          "dashboard",
+          "archivematica-common",
+        ]
+        python-version: [
+          "3.8",
+          "3.9",
+          "3.10",
+        ]
         include:
           - rule: "storage-service"
             python-version: "3.9"
           - rule: "migrations"
             python-version: "3.9"
     steps:
       - name: "Check out repository"
-        uses: "actions/checkout@v3"
+        uses: "actions/checkout@v4"
       - name: "Check out the archivematica-storage-service submodule"
         run: |
           git submodule update --init hack/submodules/archivematica-storage-service/
       - name: "Set up buildx"
-        uses: "docker/setup-buildx-action@v2"
-        id: buildx
-        with:
-          install: true
-      - name: "Set up docker cache"
-        uses: "actions/cache@v3"
-        with:
-          path: /tmp/.docker-cache-old
-          key: ${{ runner.os }}-docker-${{ matrix.python-version }}-${{ matrix.rule }}-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-docker-${{ matrix.python-version }}-${{ matrix.rule }}-
-            ${{ runner.os }}-docker-${{ matrix.python-version }}-
-            ${{ runner.os }}-docker-
+        uses: "docker/setup-buildx-action@v3"
       - name: "Build archivematica-tests image"
-        uses: "docker/build-push-action@v4"
+        uses: "docker/build-push-action@v5"
         with:
           context: .
           file: ./hack/Dockerfile
@@ -51,13 +48,14 @@ jobs:
           tags: archivematica-tests:latest
           push: false
           load: true
-          cache-from: type=local,src=/tmp/.docker-cache-old
-          cache-to: type=local,dest=/tmp/.docker-cache-new,mode=max
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
       - name: "Create external volumes"
         run: |
           make -C hack/ create-volumes
       - name: "Run make rule"
         run: |
+          docker buildx install
           make -C hack/ test-${{ matrix.rule }}
         env:
           TOXARGS: -vv
@@ -71,10 +69,6 @@ jobs:
           fail_ci_if_error: false
           verbose: true
           name: ${{ matrix.rule }}
-      - name: "Set newest docker cache"
-        run: |
-          rm -rf /tmp/.docker-cache-old
-          mv /tmp/.docker-cache-new /tmp/.docker-cache-old
   frontend:
     name: "Test frontend"
     runs-on: "ubuntu-22.04"
@@ -85,16 +79,12 @@ jobs:
       - name: "Check out repository"
         uses: "actions/checkout@v3"
       - name: "Set up Node JS"
-        uses: "actions/setup-node@v3"
-        with:
-          node-version: "8"
-      - name: "Cache npm packages"
-        uses: "actions/cache@v3"
+        uses: "actions/setup-node@v4"
         with:
-          path: "~/.npm"
-          key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}"
-          restore-keys: |
-            ${{ runner.os }}-node-
+          node-version: "14"
+          cache: "npm"
+          cache-dependency-path: |
+            **/package-lock.json
       - name: "Install frontend dependencies"
         run: |
           npm install

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.5.0
   hooks:
   - id: pretty-format-json
     args: [--no-ensure-ascii, --autofix]
@@ -10,22 +10,22 @@ repos:
         src/(MCPClient/MCPServer|dashboard)/osdeps/.*\.json
       )
 - repo: https://github.com/asottile/pyupgrade
-  rev: v3.10.1
+  rev: v3.15.0
   hooks:
   - id: pyupgrade
     args: [--py38-plus]
 - repo: https://github.com/asottile/reorder_python_imports
-  rev: v3.10.0
+  rev: v3.12.0
   hooks:
   - id: reorder-python-imports
     args: [--py38-plus]
 - repo: https://github.com/adamchainz/django-upgrade
-  rev: "1.14.1"
+  rev: "1.15.0"
   hooks:
   - id: django-upgrade
     args: [--target-version, "3.2"]
 - repo: https://github.com/psf/black
-  rev: "23.7.0"
+  rev: "23.10.1"
   hooks:
   - id: black
     args: [--safe, --quiet]

hack/Makefile

@@ -218,6 +218,26 @@ upgrade-requirements-ss:  ## Run pip-upgrade for Storage Service
 		--entrypoint bash archivematica-storage-service \
 			-c "make pip-upgrade"
 
+define amauat_make
+	docker compose \
+		-f docker-compose.yml \
+		-f docker-compose.acceptance-tests.yml \
+		run \
+			--workdir /src \
+			-e XDG_CACHE_HOME=/tmp/pip-cache \
+			--rm \
+			--no-deps \
+			--user $(CALLER_UID):$(CALLER_GID) \
+			--entrypoint bash archivematica-acceptance-tests \
+				-c "make $(1)"
+endef
+
+compile-requirements-amauat: test-at-build  ## Run pip-compile for AMAUATs
+	$(call amauat_make,pip-compile)
+
+upgrade-requirements-amauat: test-at-build  ## Run pip-upgrade for AMAUATs
+	$(call amauat_make,pip-upgrade)
+
 db:  ## Connect to the MySQL server using the CLI.
 	docker compose exec mysql mysql -hlocalhost -uroot -p12345
 

hack/docker-compose.acceptance-tests.yml

@@ -13,6 +13,7 @@ services:
     security_opt:
       - "seccomp:unconfined"
     volumes:
+      - "./submodules/archivematica-acceptance-tests/:/src/"
       - "/dev/shm:/dev/shm"
       - "./submodules/archivematica-sampledata:/home/archivematica/archivematica-sampledata:ro"
       - "archivematica_pipeline_data:/var/archivematica/sharedDirectory:ro"

requirements-dev.in

@@ -8,4 +8,3 @@ pytest-django
 pytest-mock
 pytest-randomly
 tox
-vcrpy