From 4afa5d14cc277621c5a8c26b8c62185f1662172c Mon Sep 17 00:00:00 2001 From: Daniel Cosme Date: Mon, 18 Mar 2024 10:25:39 -0400 Subject: [PATCH 01/32] Add GHA to build and push images to docker hub --- .github/workflows/push-images.yml | 46 +++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/push-images.yml diff --git a/.github/workflows/push-images.yml b/.github/workflows/push-images.yml new file mode 100644 index 0000000000..ea67a51825 --- /dev/null +++ b/.github/workflows/push-images.yml @@ -0,0 +1,46 @@ +name: "Push images to Docker Hub" +on: workflow_dispatch +jobs: + build: + name: "Builds images and push them to Docker Hub" + runs-on: "ubuntu-22.04" + steps: + - name: "Check out repository" + uses: "actions/checkout@v4" + - name: "Set up buildx" + uses: "docker/setup-buildx-action@v3" + with: + version: latest + driver-opts: image=moby/buildkit:v0.13.0 + - name: "Login to Docker Hub" + uses: docker/login-action@v3 + with: + username: artefactual + password: ${{ secrets.DOCKER_HUB_TOKEN }} + - name: "Build and Push Dashboard" + uses: docker/build-push-action@v5 + with: + context: . + push: true + load: true + file: ./hack/Dockerfile + target: "archivematica-dashboard" + tags: artefactual/archivematica-dashboard:latest + - name: "Build and Push MCP-client" + uses: docker/build-push-action@v5 + with: + context: . + push: true + load: true + file: ./hack/Dockerfile + target: "archivematica-mcp-client" + tags: artefactual/archivematica-mcp-client:latest + - name: "Build and Push MCP-server" + uses: docker/build-push-action@v5 + with: + context: . + push: true + load: true + file: ./hack/Dockerfile + target: "archivematica-mcp-server" + tags: artefactual/archivematica-mcp-server:latest \ No newline at end of file From 6b5dbc7cbc5c0ce96f6311c91274ad7acb482be6 Mon Sep 17 00:00:00 2001 From: Daniel Cosme Date: Mon, 18 Mar 2024 10:26:21 -0400 Subject: [PATCH 02/32] Make Docker images smaller --- hack/Dockerfile | 145 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 113 insertions(+), 32 deletions(-) diff --git a/hack/Dockerfile b/hack/Dockerfile index bcef69af74..9d9b346aa9 100644 --- a/hack/Dockerfile +++ b/hack/Dockerfile @@ -150,6 +150,35 @@ ARG USER_ID=1000 ARG GROUP_ID=1000 ARG PYENV_DIR=/pyenv +RUN set -ex \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + coreutils \ + clamav \ + libldap2-dev \ + libsasl2-dev \ + && rm -rf /var/lib/apt/lists/* /var/cache/apt/* + +RUN set -ex \ + && groupadd --gid ${GROUP_ID} --system archivematica \ + && useradd --uid ${USER_ID} --gid ${GROUP_ID} --home-dir /var/archivematica --system archivematica \ + && mkdir -p /var/archivematica/sharedDirectory \ + && chown -R archivematica:archivematica /var/archivematica + +# Download ClamAV virus signatures +RUN freshclam --quiet + +USER archivematica + +COPY --chown=${USER_ID}:${GROUP_ID} --from=pyenv-builder --link ${PYENV_DIR} ${PYENV_DIR} +COPY --chown=${USER_ID}:${GROUP_ID} --link . /src + +# ----------------------------------------------------------------------------- + +FROM base AS archivematica-mcp-client + +USER root + RUN set -ex \ && curl --retry 3 -fsSL https://packages.archivematica.org/1.16.x/key.asc | gpg --dearmor -o /etc/apt/keyrings/archivematica-1.16.x.gpg \ && echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/archivematica-1.16.x.gpg] http://packages.archivematica.org/1.16.x/ubuntu-externals jammy main" > /etc/apt/sources.list.d/archivematica-external.list \ @@ -159,66 +188,36 @@ RUN set -ex \ && apt-get install -y --no-install-recommends \ atool \ bulk-extractor \ - clamav \ - coreutils \ ffmpeg \ fits \ - g++ \ - gcc \ gearman \ - gettext \ ghostscript \ hashdeep \ imagemagick \ inkscape \ jhove \ - libffi-dev \ libimage-exiftool-perl \ - libldap2-dev \ - libmysqlclient-dev \ - libsasl2-dev \ - libssl-dev \ - libxml2-dev \ - libxslt1-dev \ logapp \ md5deep \ + nailgun \ mediaconch \ mediainfo \ - nailgun \ nfs-common \ openjdk-8-jre-headless \ p7zip-full \ pbzip2 \ pst-utils \ - python3-lxml \ rsync \ siegfried \ sleuthkit \ tesseract-ocr \ tree \ - unar \ unrar-free \ uuid \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/* -RUN set -ex \ - && groupadd --gid ${GROUP_ID} --system archivematica \ - && useradd --uid ${USER_ID} --gid ${GROUP_ID} --home-dir /var/archivematica --system archivematica \ - && mkdir -p /var/archivematica/sharedDirectory \ - && chown -R archivematica:archivematica /var/archivematica - -# Download ClamAV virus signatures -RUN freshclam --quiet - USER archivematica -COPY --chown=${USER_ID}:${GROUP_ID} --from=pyenv-builder --link ${PYENV_DIR} ${PYENV_DIR} -COPY --chown=${USER_ID}:${GROUP_ID} --link . /src - -# ----------------------------------------------------------------------------- - -FROM base AS archivematica-mcp-client - ENV DJANGO_SETTINGS_MODULE settings.common ENV PYTHONPATH /src/src/MCPClient/lib/:/src/src/MCPClient/lib/clientScripts:/src/src/archivematicaCommon/lib/:/src/src/dashboard/src/ ENV ARCHIVEMATICA_MCPCLIENT_ARCHIVEMATICACLIENTMODULES /src/src/MCPClient/lib/archivematicaClientModules @@ -251,6 +250,19 @@ ARG PYTHON_VERSION=3.9 USER root +RUN set -ex \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + gcc \ + g++ \ + gettext \ + libffi-dev \ + libxml2-dev \ + libxslt1-dev \ + unar \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* /var/cache/apt/* + RUN set -ex \ && internalDirs=' \ /src/src/dashboard/frontend \ @@ -284,7 +296,76 @@ ENTRYPOINT ["pyenv", "exec", "python3", "-m", "gunicorn", "--config=/src/src/das # ----------------------------------------------------------------------------- -FROM base AS archivematica-tests +FROM base-builder as archivematica-tests + +ARG USER_ID=1000 +ARG GROUP_ID=1000 +ARG PYENV_DIR=/pyenv + +RUN set -ex \ + && curl --retry 3 -fsSL https://packages.archivematica.org/1.16.x/key.asc | gpg --dearmor -o /etc/apt/keyrings/archivematica-1.16.x.gpg \ + && echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/archivematica-1.16.x.gpg] http://packages.archivematica.org/1.16.x/ubuntu-externals jammy main" > /etc/apt/sources.list.d/archivematica-external.list \ + && curl --retry 3 -so /tmp/repo-mediaarea_1.0-21_all.deb -L https://mediaarea.net/repo/deb/repo-mediaarea_1.0-21_all.deb \ + && dpkg -i /tmp/repo-mediaarea_1.0-21_all.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + atool \ + bulk-extractor \ + clamav \ + coreutils \ + ffmpeg \ + fits \ + g++ \ + gcc \ + gearman \ + gettext \ + ghostscript \ + hashdeep \ + imagemagick \ + inkscape \ + jhove \ + libffi-dev \ + libimage-exiftool-perl \ + libldap2-dev \ + libmysqlclient-dev \ + libsasl2-dev \ + libssl-dev \ + libxml2-dev \ + libxslt1-dev \ + logapp \ + md5deep \ + mediaconch \ + mediainfo \ + nailgun \ + nfs-common \ + openjdk-8-jre-headless \ + p7zip-full \ + pbzip2 \ + pst-utils \ + python3-lxml \ + rsync \ + siegfried \ + sleuthkit \ + tesseract-ocr \ + tree \ + unar \ + unrar-free \ + uuid \ + && rm -rf /var/lib/apt/lists/* /var/cache/apt/* + +RUN set -ex \ + && groupadd --gid ${GROUP_ID} --system archivematica \ + && useradd --uid ${USER_ID} --gid ${GROUP_ID} --home-dir /var/archivematica --system archivematica \ + && mkdir -p /var/archivematica/sharedDirectory \ + && chown -R archivematica:archivematica /var/archivematica + +# Download ClamAV virus signatures +RUN freshclam --quiet + +USER archivematica + +COPY --chown=${USER_ID}:${GROUP_ID} --from=pyenv-builder --link ${PYENV_DIR} ${PYENV_DIR} +COPY --chown=${USER_ID}:${GROUP_ID} --link . /src # ----------------------------------------------------------------------------- From 550e541da9b70a9050594b3b4c8ea00883c13e68 Mon Sep 17 00:00:00 2001 From: Daniel Cosme Date: Wed, 3 Apr 2024 11:52:19 -0400 Subject: [PATCH 03/32] WIP: feedback from code review --- hack/Dockerfile | 73 ++++++++----------------------------------------- 1 file changed, 12 insertions(+), 61 deletions(-) diff --git a/hack/Dockerfile b/hack/Dockerfile index 9d9b346aa9..d642be3546 100644 --- a/hack/Dockerfile +++ b/hack/Dockerfile @@ -146,17 +146,14 @@ ENTRYPOINT ["npm", "run", "test-single-run"] FROM base-builder as base -ARG USER_ID=1000 -ARG GROUP_ID=1000 -ARG PYENV_DIR=/pyenv +ARG USER_ID +ARG GROUP_ID +ARG PYENV_DIR RUN set -ex \ && apt-get update \ && apt-get install -y --no-install-recommends \ coreutils \ - clamav \ - libldap2-dev \ - libsasl2-dev \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/* RUN set -ex \ @@ -165,9 +162,6 @@ RUN set -ex \ && mkdir -p /var/archivematica/sharedDirectory \ && chown -R archivematica:archivematica /var/archivematica -# Download ClamAV virus signatures -RUN freshclam --quiet - USER archivematica COPY --chown=${USER_ID}:${GROUP_ID} --from=pyenv-builder --link ${PYENV_DIR} ${PYENV_DIR} @@ -186,6 +180,7 @@ RUN set -ex \ && dpkg -i /tmp/repo-mediaarea_1.0-21_all.deb \ && apt-get update \ && apt-get install -y --no-install-recommends \ + clamav \ atool \ bulk-extractor \ ffmpeg \ @@ -216,6 +211,9 @@ RUN set -ex \ uuid \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/* +# Download ClamAV virus signatures +RUN freshclam --quiet + USER archivematica ENV DJANGO_SETTINGS_MODULE settings.common @@ -298,9 +296,9 @@ ENTRYPOINT ["pyenv", "exec", "python3", "-m", "gunicorn", "--config=/src/src/das FROM base-builder as archivematica-tests -ARG USER_ID=1000 -ARG GROUP_ID=1000 -ARG PYENV_DIR=/pyenv +ARG USER_ID +ARG GROUP_ID +ARG PYENV_DIR RUN set -ex \ && curl --retry 3 -fsSL https://packages.archivematica.org/1.16.x/key.asc | gpg --dearmor -o /etc/apt/keyrings/archivematica-1.16.x.gpg \ @@ -309,64 +307,17 @@ RUN set -ex \ && dpkg -i /tmp/repo-mediaarea_1.0-21_all.deb \ && apt-get update \ && apt-get install -y --no-install-recommends \ - atool \ - bulk-extractor \ - clamav \ - coreutils \ - ffmpeg \ - fits \ - g++ \ gcc \ - gearman \ - gettext \ - ghostscript \ - hashdeep \ - imagemagick \ - inkscape \ - jhove \ - libffi-dev \ - libimage-exiftool-perl \ - libldap2-dev \ - libmysqlclient-dev \ - libsasl2-dev \ - libssl-dev \ - libxml2-dev \ - libxslt1-dev \ - logapp \ - md5deep \ - mediaconch \ - mediainfo \ - nailgun \ - nfs-common \ - openjdk-8-jre-headless \ + media-types \ p7zip-full \ pbzip2 \ pst-utils \ python3-lxml \ rsync \ - siegfried \ - sleuthkit \ - tesseract-ocr \ - tree \ unar \ - unrar-free \ - uuid \ + tox \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/* -RUN set -ex \ - && groupadd --gid ${GROUP_ID} --system archivematica \ - && useradd --uid ${USER_ID} --gid ${GROUP_ID} --home-dir /var/archivematica --system archivematica \ - && mkdir -p /var/archivematica/sharedDirectory \ - && chown -R archivematica:archivematica /var/archivematica - -# Download ClamAV virus signatures -RUN freshclam --quiet - -USER archivematica - -COPY --chown=${USER_ID}:${GROUP_ID} --from=pyenv-builder --link ${PYENV_DIR} ${PYENV_DIR} -COPY --chown=${USER_ID}:${GROUP_ID} --link . /src - # ----------------------------------------------------------------------------- FROM ${TARGET} From 7221d265950c314c7fef0f657c9cd6ea0b7c91cd Mon Sep 17 00:00:00 2001 From: Daniel Cosme Date: Wed, 3 Apr 2024 11:53:37 -0400 Subject: [PATCH 04/32] WIP: fix typos in GH workflow --- .github/workflows/push-images.yml | 15 ++++++++++----- hack/Dockerfile | 1 - 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push-images.yml b/.github/workflows/push-images.yml index ea67a51825..28b4d4a143 100644 --- a/.github/workflows/push-images.yml +++ b/.github/workflows/push-images.yml @@ -1,8 +1,13 @@ name: "Push images to Docker Hub" -on: workflow_dispatch +#on: workflow_dispatch +on: + push: + branches: + - "qa/**" + - "stable/**" jobs: build: - name: "Builds images and push them to Docker Hub" + name: "Build and push images" runs-on: "ubuntu-22.04" steps: - name: "Check out repository" @@ -26,7 +31,7 @@ jobs: file: ./hack/Dockerfile target: "archivematica-dashboard" tags: artefactual/archivematica-dashboard:latest - - name: "Build and Push MCP-client" + - name: "Build and Push MCPClient" uses: docker/build-push-action@v5 with: context: . @@ -35,7 +40,7 @@ jobs: file: ./hack/Dockerfile target: "archivematica-mcp-client" tags: artefactual/archivematica-mcp-client:latest - - name: "Build and Push MCP-server" + - name: "Build and Push MCPServer" uses: docker/build-push-action@v5 with: context: . @@ -43,4 +48,4 @@ jobs: load: true file: ./hack/Dockerfile target: "archivematica-mcp-server" - tags: artefactual/archivematica-mcp-server:latest \ No newline at end of file + tags: artefactual/archivematica-mcp-server:latest diff --git a/hack/Dockerfile b/hack/Dockerfile index d642be3546..e60d921010 100644 --- a/hack/Dockerfile +++ b/hack/Dockerfile @@ -315,7 +315,6 @@ RUN set -ex \ python3-lxml \ rsync \ unar \ - tox \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/* # ----------------------------------------------------------------------------- From d207c9151e4c6a0bae157ca5d0a65632fbb1b91d Mon Sep 17 00:00:00 2001 From: Daniel Cosme Date: Wed, 3 Apr 2024 12:45:30 -0400 Subject: [PATCH 05/32] WIP: trigger CI --- .github/workflows/push-images.yml | 3 +-- hack/Dockerfile | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/push-images.yml b/.github/workflows/push-images.yml index 28b4d4a143..d912d1a7ad 100644 --- a/.github/workflows/push-images.yml +++ b/.github/workflows/push-images.yml @@ -3,8 +3,7 @@ name: "Push images to Docker Hub" on: push: branches: - - "qa/**" - - "stable/**" + - "dev/trim-dockerfile" jobs: build: name: "Build and push images" diff --git a/hack/Dockerfile b/hack/Dockerfile index e60d921010..80222a8aa9 100644 --- a/hack/Dockerfile +++ b/hack/Dockerfile @@ -19,6 +19,7 @@ ENV PYTHONUNBUFFERED 1 RUN set -ex \ && apt-get update \ && apt-get install -y --no-install-recommends \ + # coreutils \ ca-certificates \ curl \ git \ @@ -150,12 +151,6 @@ ARG USER_ID ARG GROUP_ID ARG PYENV_DIR -RUN set -ex \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - coreutils \ - && rm -rf /var/lib/apt/lists/* /var/cache/apt/* - RUN set -ex \ && groupadd --gid ${GROUP_ID} --system archivematica \ && useradd --uid ${USER_ID} --gid ${GROUP_ID} --home-dir /var/archivematica --system archivematica \ @@ -317,6 +312,11 @@ RUN set -ex \ unar \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/* +USER archivematica + +COPY --chown=${USER_ID}:${GROUP_ID} --from=pyenv-builder --link ${PYENV_DIR} ${PYENV_DIR} +COPY --chown=${USER_ID}:${GROUP_ID} --link . /src + # ----------------------------------------------------------------------------- FROM ${TARGET} From f6a5367b18e92801f7c44871f2318a93768027f2 Mon Sep 17 00:00:00 2001 From: Daniel Cosme Date: Wed, 3 Apr 2024 14:10:33 -0400 Subject: [PATCH 06/32] Remove coreutils comment --- hack/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/hack/Dockerfile b/hack/Dockerfile index 80222a8aa9..0283357b81 100644 --- a/hack/Dockerfile +++ b/hack/Dockerfile @@ -19,7 +19,6 @@ ENV PYTHONUNBUFFERED 1 RUN set -ex \ && apt-get update \ && apt-get install -y --no-install-recommends \ - # coreutils \ ca-certificates \ curl \ git \ From 1efafce32fc944fb6fa8d3bca2426627c2e8e8cf Mon Sep 17 00:00:00 2001 From: Daniel Cosme Date: Wed, 3 Apr 2024 14:38:26 -0400 Subject: [PATCH 07/32] Revert back Github action trigger to manual --- .github/workflows/push-images.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/push-images.yml b/.github/workflows/push-images.yml index d912d1a7ad..9eb3e565f7 100644 --- a/.github/workflows/push-images.yml +++ b/.github/workflows/push-images.yml @@ -1,9 +1,5 @@ name: "Push images to Docker Hub" -#on: workflow_dispatch -on: - push: - branches: - - "dev/trim-dockerfile" +on: workflow_dispatch jobs: build: name: "Build and push images" From b3f5b831a10ccd36776d4ef1d9e82b06d835ebd4 Mon Sep 17 00:00:00 2001 From: Daniel Cosme Date: Mon, 15 Apr 2024 11:44:58 -0400 Subject: [PATCH 08/32] Add SSH package to mcp-client --- hack/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/hack/Dockerfile b/hack/Dockerfile index 0283357b81..4c0e2999f6 100644 --- a/hack/Dockerfile +++ b/hack/Dockerfile @@ -203,6 +203,7 @@ RUN set -ex \ tree \ unrar-free \ uuid \ + ssh \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/* # Download ClamAV virus signatures From dabcad0150b9d19daa14ad588ea2e6cad620deac Mon Sep 17 00:00:00 2001 From: Daniel Cosme Date: Wed, 1 May 2024 12:06:47 -0400 Subject: [PATCH 09/32] WIP: trim build time dependencies from final images --- hack/Dockerfile | 50 ++++++++++++++++++++++++++++++++------ hack/build_for_registry.sh | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 8 deletions(-) create mode 100755 hack/build_for_registry.sh diff --git a/hack/Dockerfile b/hack/Dockerfile index 4c0e2999f6..bb04d7325e 100644 --- a/hack/Dockerfile +++ b/hack/Dockerfile @@ -144,11 +144,38 @@ ENTRYPOINT ["npm", "run", "test-single-run"] # ----------------------------------------------------------------------------- -FROM base-builder as base +FROM ubuntu:${UBUNTU_VERSION} AS base +ARG PYENV_DIR=/pyenv ARG USER_ID ARG GROUP_ID -ARG PYENV_DIR + +ENV DEBIAN_FRONTEND noninteractive +ENV PYTHONUNBUFFERED 1 + +RUN set -ex \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + # gnupg \ + # curl \ + # git \ + # libldap2-dev \ + libmysqlclient-dev \ + # libsasl2-dev \ + # libsqlite3-dev \ + locales \ + # pkg-config \ + tzdata \ + && rm -rf /var/lib/apt/lists/* /var/cache/apt/* + +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +ENV PYENV_ROOT=${PYENV_DIR}/data +ENV PATH=$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH RUN set -ex \ && groupadd --gid ${GROUP_ID} --system archivematica \ @@ -161,12 +188,24 @@ USER archivematica COPY --chown=${USER_ID}:${GROUP_ID} --from=pyenv-builder --link ${PYENV_DIR} ${PYENV_DIR} COPY --chown=${USER_ID}:${GROUP_ID} --link . /src + # ----------------------------------------------------------------------------- FROM base AS archivematica-mcp-client USER root +RUN set -ex \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + gnupg \ + curl \ + && rm -rf /var/lib/apt/lists/* /var/cache/apt/* + +# Purge +# - gnupg +# - git + RUN set -ex \ && curl --retry 3 -fsSL https://packages.archivematica.org/1.16.x/key.asc | gpg --dearmor -o /etc/apt/keyrings/archivematica-1.16.x.gpg \ && echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/archivematica-1.16.x.gpg] http://packages.archivematica.org/1.16.x/ubuntu-externals jammy main" > /etc/apt/sources.list.d/archivematica-external.list \ @@ -174,7 +213,6 @@ RUN set -ex \ && dpkg -i /tmp/repo-mediaarea_1.0-21_all.deb \ && apt-get update \ && apt-get install -y --no-install-recommends \ - clamav \ atool \ bulk-extractor \ ffmpeg \ @@ -206,9 +244,6 @@ RUN set -ex \ ssh \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/* -# Download ClamAV virus signatures -RUN freshclam --quiet - USER archivematica ENV DJANGO_SETTINGS_MODULE settings.common @@ -246,8 +281,7 @@ USER root RUN set -ex \ && apt-get update \ && apt-get install -y --no-install-recommends \ - gcc \ - g++ \ + # g++ \ gettext \ libffi-dev \ libxml2-dev \ diff --git a/hack/build_for_registry.sh b/hack/build_for_registry.sh new file mode 100755 index 0000000000..9d8b915f5a --- /dev/null +++ b/hack/build_for_registry.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -xeo pipefail + +DIR=$(dirname "$(readlink -f "$0")") +ROOT_DIR=$(pwd) + +docker build \ + --target "archivematica-mcp-server" \ + --tag "archivematica-mcp-server" \ + --file $DIR/Dockerfile . + +docker build \ + --target "archivematica-mcp-client" \ + --tag "archivematica-mcp-client" \ + --file $DIR/Dockerfile . + +docker build \ + --target "archivematica-dashboard" \ + --tag "archivematica-dashboard" \ + --file $DIR/Dockerfile . + +if [[ ! -z "${PUSH_DOCKER}" ]] +then + docker image tag archivematica-mcp-server "artefactual/archivematica-mcp-server:latest" + docker image tag archivematica-mcp-client "artefactual/archivematica-mcp-client:latest" + docker image tag archivematica-dashboard "artefactual/archivematica-dashboard:latest" + + docker image tag archivematica-mcp-server "artefactual/archivematica-mcp-server:qa.1.x" + docker image tag archivematica-mcp-client "artefactual/archivematica-mcp-client:qa.1.x" + docker image tag archivematica-dashboard "artefactual/archivematica-dashboard:qa.1.x" + + # docker push --all-tags artefactual/archivematica-dashboard + # docker push --all-tags artefactual/archivematica-mcp-client + # docker push --all-tags artefactual/archivematica-mcp-server + docker push artefactual/archivematica-dashboard:qa.1.x + docker push artefactual/archivematica-mcp-client:qa.1.x + docker push artefactual/archivematica-mcp-server:qa.1.x + + docker push artefactual/archivematica-dashboard:latest + docker push artefactual/archivematica-mcp-client:latest + docker push artefactual/archivematica-mcp-server:latest +fi + From 3e1bd3b2388188482d610ea1efa7f6dbb4565492 Mon Sep 17 00:00:00 2001 From: Douglas Cerna Date: Thu, 2 May 2024 09:01:50 -0600 Subject: [PATCH 10/32] Extend elasticSearchFunctions test coverage --- .../test_index_aip_and_files_METS.xml | 1352 +++++++++++++++ .../test_elasticsearch_functions.py | 1492 +++++++++++------ 2 files changed, 2369 insertions(+), 475 deletions(-) create mode 100644 tests/archivematicaCommon/fixtures/test_index_aip_and_files_METS.xml diff --git a/tests/archivematicaCommon/fixtures/test_index_aip_and_files_METS.xml b/tests/archivematicaCommon/fixtures/test_index_aip_and_files_METS.xml new file mode 100644 index 0000000000..c4fc084e83 --- /dev/null +++ b/tests/archivematicaCommon/fixtures/test_index_aip_and_files_METS.xml @@ -0,0 +1,1352 @@ + + + + + + + + + UUID + d7a641c0-daa4-4ec6-b531-660deea920f6 + + pictures-with-dublincore-d7a641c0-daa4-4ec6-b531-660deea920f6 + + + + + + + + + Pictures with DublinCore + Archival Information Package + + + + + + + + + + + UUID + 8518d9a7-cb66-4530-953d-8ac2c554e5f2 + + + 0 + + sha256 + a469c730e705d757d66f53f38bb4455e89d5691a3d87fc7bc069b91fa2a50d46 + + 1361321 + + + JPEG + 1.01 + + + PRONOM + fmt/43 + + + + 2023-09-21T23:42:01Z + + + + + + + + + + + + + + + + + + + + + + + + + + 12.40 + [minor] Possibly incorrect maker notes offsets (fix by -74?) + Landing_zone.jpg + /var/archivematica/sharedDirectory/watchedDirectories/workFlowDecisions/extractPackagesChoice/pictures-with-dublincore-f08c3cbb-3b39-4997-bc1c-74de2fa69eb0/objects + 1329 KiB + 2023:09:21 09:42:01-07:00 + 2024:04:28 20:50:25-07:00 + 2024:04:28 20:50:23-07:00 + -rwxrwxr-- + JPEG + jpg + image/jpeg + Little-endian (Intel, II) + 3648 + 2736 + Baseline DCT, Huffman coding + 8 + 3 + YCbCr4:2:0 (2 2) + 1.01 + inches + 72 + 72 + Panasonic + DMC-FS20 + Horizontal (normal) + 72 + 72 + inches + GIMP 2.4.7 + 2008:09:15 11:01:51 + Co-sited + 1/250 + 4.0 + Landscape + 100 + 0221 + 2008:09:10 16:51:18 + 2008:09:10 16:51:18 + Y, Cb, Cr, - + 4 + 0 + 3.3 + Multi-segment + Unknown + Off, Did not fire + 5.2 mm + 0100 + sRGB + 3648 + 2736 + One-chip color area + Digital Camera + Directly photographed + Normal + Auto + Auto + 0 + 30 mm + Standard + None + Normal + Normal + Normal + High + 0.1.1.2 + Auto + Auto + Tracking + On, Mode 2 + Off + Scenery + No + (Binary data 8200 bytes, use -b option to extract) + 0 + +AAAAAAAAAAAAAAAAAAAAAA== + + 0260 + n/a + Off + 00:00:52.22 + Off + 0 + High + Standard + Off + Horizontal (normal) + Enabled but Not Used + Normal + + Standard + Off + n/a + Medium + Destination + Off + n/a + 5 + Off + 0 + Inf 0.00015 + 2 + 4 0 14674 56 + 2 7 4 0 + 0121 + Scenery + No + No + 2030 + 1054 + 1800 + Off + Off + +Aw== + + R98 - DCF basic file (sRGB) + 0100 + 0250 + JPEG (old-style) + Horizontal (normal) + 180 + 180 + inches + 10050 + 8789 + Co-sited + (Binary data 8789 bytes, use -b option to extract) + 4.0 + 1.70778 + 3648x2736 + 10.0 + 1.925996 + 5.8 + 1/250 + Scenery (intelligent auto) + 0.005 mm + 61.9 deg + 5.2 mm (35 mm equivalent: 30.0 mm) + 1.30 m + 12.0 + + + + MediaInfoLib + + + 349 + 1 + General + General + 0 + 1 + JPEG + JPEG + JPEG + /var/archivematica/sharedDirectory/watchedDirectories/workFlowDecisions/extractPackagesChoice/pictures-with-dublincore-f08c3cbb-3b39-4997-bc1c-74de2fa69eb0/objects/Landing_zone.jpg + /var/archivematica/sharedDirectory/watchedDirectories/workFlowDecisions/extractPackagesChoice/pictures-with-dublincore-f08c3cbb-3b39-4997-bc1c-74de2fa69eb0/objects + Landing_zone.jpg + Landing_zone + jpg + JPEG + JPEG + h3d jpeg jpg jpe jps mpo + JPEG + image/jpeg + 1361321 + 1.30 MiB + 1 MiB + 1.3 MiB + 1.30 MiB + 1.298 MiB + 0 + 0.00 Byte (0%) + Byte0 + 0.0 Byte + 0.00 Byte + 0.000 Byte + 0.00 Byte (0%) + 0.00000 + 2023-09-21 16:42:01 UTC + 2023-09-21 09:42:01 + + + 124 + 1 + Image + Image + 0 + JPEG + JPEG + JPEG + image/jpeg + 3648 + 3 648 pixels + 2736 + 2 736 pixels + YUV + 4:2:0 + 8 + 8 bits + Lossy + Lossy + 1361321 + 1.30 MiB (100%) + 1 MiB + 1.3 MiB + 1.30 MiB + 1.298 MiB + 1.30 MiB (100%) + 1.00000 + + + + + + %transferDirectory%objects/Landing zone.jpg + + + + + + + + + + UUID + a03ca4bf-fd40-42d9-9972-a6bffd597095 + + ingestion + 2024-04-29T03:50:21.847564+00:00 + + + + + + + + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + UUID + 1270366a-19d4-449f-87f5-5c32f7816cda + + message digest calculation + 2024-04-29T03:50:21.915253+00:00 + + program="python"; module="hashlib.sha256()" + + + + + a469c730e705d757d66f53f38bb4455e89d5691a3d87fc7bc069b91fa2a50d46 + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + UUID + 9afc763e-1986-41c2-931d-2beb14769137 + + virus check + 2024-04-29T03:50:22.674126+00:00 + + program="ClamAV (clamd)"; version="ClamAV 0.103.11"; virusDefinitions="27259/Sun Apr 28 08:22:36 2024" + + + Pass + + + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + UUID + 07b83b4e-f578-4e7f-a874-b6669ae2e4c0 + + filename change + 2024-04-29T03:50:23.816149+00:00 + + prohibited characters removed: program="change_names"; version="1.10.96655da16f29f2ac6ebd3eb4b48a689d82efe76d" + + + + + Original name="%transferDirectory%objects/Landing zone.jpg"; new name="%transferDirectory%objects/Landing_zone.jpg" + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + UUID + 4a54765f-8755-4a27-be3e-2c0e5d130bd4 + + format identification + 2024-04-29T03:50:25.776406+00:00 + + program="Siegfried"; version="1.9.6" + + + Positive + + fmt/43 + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + UUID + f4a7a90e-57b7-4492-8477-9da8b8044da8 + + validation + 2024-04-29T03:50:28.635310+00:00 + + program="JHOVE"; version="1.26" + + + pass + + format="JPEG"; version="1.01"; result="Well-Formed and valid" + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + preservation system + Archivematica-1.15.1 + + Archivematica + software + + + + + + + + + + repository code + AM qa/1.x SS qa/0x + + AM qa/1.x SS qa/0x + organization + + + + + + + + + + Archivematica user pk + 1 + + username="test", first_name="", last_name="" + Archivematica user + + + + + + + + + + + + UUID + 34c0e0c1-58f1-4025-91b9-2d58ba161593 + + + 0 + + sha256 + 3cd6050858efb22b71630eba9e99e6ffef68c59f4c03bb79e5ae6c4edf3c88d8 + + 4261562 + + + Tagged Image File Format + + + + + + + + + 2024-04-29T03:50:32Z + + + %SIPDirectory%objects/MARBLES-34c0e0c1-58f1-4025-91b9-2d58ba161593.tif + + derivation + has source + + UUID + f333c372-5cf6-4346-b342-ea40373d07d7 + + + UUID + fb445513-51aa-4cdd-b6a3-26fa70dcd77f + + + + + + + + + + + + UUID + d0b90430-8648-43d0-b954-8c295dd14260 + + creation + 2024-04-29T03:50:32.692103+00:00 + + + + + + + + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + UUID + c5aaaa36-be28-450d-8907-92c01d64af2c + + message digest calculation + 2024-04-29T03:50:32.733500+00:00 + + program="python"; module="hashlib.sha256()" + + + + + 3cd6050858efb22b71630eba9e99e6ffef68c59f4c03bb79e5ae6c4edf3c88d8 + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + UUID + a541b242-ee28-4ef7-90d9-03b99e8df139 + + validation + 2024-04-29T03:50:33.383765+00:00 + + program="JHOVE"; version="1.26" + + + pass + + format="TIFF"; version="5.0"; result="Well-Formed and valid" + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + preservation system + Archivematica-1.15.1 + + Archivematica + software + + + + + + + + + + repository code + AM qa/1.x SS qa/0x + + AM qa/1.x SS qa/0x + organization + + + + + + + + + + Archivematica user pk + 1 + + username="test", first_name="", last_name="" + Archivematica user + + + + + + + + + + + + UUID + f333c372-5cf6-4346-b342-ea40373d07d7 + + + 0 + + sha256 + 91a5ddca3637590c2ddb50da5feb73ff0b8a98cd09a98afb79adc2cf70bc6220 + + 4261301 + + + Truevision TGA Bitmap + 2.0 + + + PRONOM + fmt/402 + + + + 2023-09-21T23:42:01Z + + + + + + + + + + + + + + + + + + + + + + + + + MediaInfoLib + + + 349 + 1 + General + General + 0 + 1 + Raw + Raw + Raw + /var/archivematica/sharedDirectory/watchedDirectories/workFlowDecisions/extractPackagesChoice/pictures-with-dublincore-f08c3cbb-3b39-4997-bc1c-74de2fa69eb0/objects/MARBLES.TGA + /var/archivematica/sharedDirectory/watchedDirectories/workFlowDecisions/extractPackagesChoice/pictures-with-dublincore-f08c3cbb-3b39-4997-bc1c-74de2fa69eb0/objects + MARBLES.TGA + MARBLES + TGA + TGA + TGA + tga + TGA + 2 + image/tga + 4261301 + 4.06 MiB + 4 MiB + 4.1 MiB + 4.06 MiB + 4.064 MiB + 2023-09-21 16:42:01 UTC + 2023-09-21 09:42:01 + + + 124 + 1 + Image + Image + 0 + Raw + Raw + Raw + 2 + 1419 + 1 419 pixels + 1001 + 1 001 pixels + RGB + 24 + 24 bits + + + + + + %transferDirectory%objects/MARBLES.TGA + + derivation + is source of + + UUID + 34c0e0c1-58f1-4025-91b9-2d58ba161593 + + + UUID + fb445513-51aa-4cdd-b6a3-26fa70dcd77f + + + + + + + + + + + + UUID + 114ee05e-7448-456e-802f-6cda5c2b00ab + + ingestion + 2024-04-29T03:50:21.857621+00:00 + + + + + + + + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + UUID + 2e0c7cb0-9b05-4a74-b741-d6bf9e550f58 + + message digest calculation + 2024-04-29T03:50:21.922880+00:00 + + program="python"; module="hashlib.sha256()" + + + + + 91a5ddca3637590c2ddb50da5feb73ff0b8a98cd09a98afb79adc2cf70bc6220 + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + UUID + a70c05a3-40bc-4cc7-a1f7-a5cc0a3789e7 + + virus check + 2024-04-29T03:50:22.686031+00:00 + + program="ClamAV (clamd)"; version="ClamAV 0.103.11"; virusDefinitions="27259/Sun Apr 28 08:22:36 2024" + + + Pass + + + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + UUID + e0eef09b-cf22-490b-bf54-ec00187712b5 + + format identification + 2024-04-29T03:50:25.738444+00:00 + + program="Siegfried"; version="1.9.6" + + + Positive + + fmt/402 + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + UUID + fb445513-51aa-4cdd-b6a3-26fa70dcd77f + + normalization + 2024-04-29T03:50:32.741862+00:00 + + ArchivematicaFPRCommandID="a34ddc9b-c922-4bb6-8037-bbe713332175"; program="convert"; version="Version: ImageMagick 6.9.11-60 Q16 x86_64 2021-01-25 https://imagemagick.org" + + + + + + %SIPDirectory%objects/MARBLES-34c0e0c1-58f1-4025-91b9-2d58ba161593.tif + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + preservation system + Archivematica-1.15.1 + + Archivematica + software + + + + + + + + + + repository code + AM qa/1.x SS qa/0x + + AM qa/1.x SS qa/0x + organization + + + + + + + + + + Archivematica user pk + 1 + + username="test", first_name="", last_name="" + Archivematica user + + + + + + + + + + + + UUID + adca4c81-7a50-4bcf-ae48-d13822c860bd + + + 0 + + sha256 + f3909d6cd379df21bdc432edee6318fc4d1132ddb1c8f5f68fe704c76c46f976 + + 65139 + + + XML + 1.0 + + + PRONOM + fmt/101 + + + + 2024-04-29T03:51:03Z + + + %SIPDirectory%objects/submissionDocumentation/transfer-pictures-with-dublincore-f08c3cbb-3b39-4997-bc1c-74de2fa69eb0/METS.xml + + + + + + + + + + UUID + 91b12ecb-2a79-4897-84e5-bd35821f73b0 + + ingestion + 2024-04-29T03:51:03.933994+00:00 + + + + + + + + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + UUID + e2e3a522-019b-4935-a7f8-554422a34af8 + + message digest calculation + 2024-04-29T03:51:03.968583+00:00 + + program="python"; module="hashlib.sha256()" + + + + + f3909d6cd379df21bdc432edee6318fc4d1132ddb1c8f5f68fe704c76c46f976 + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + UUID + 8d57f933-f420-4e53-91ea-4d24fb958119 + + virus check + 2024-04-29T03:51:05.319694+00:00 + + program="ClamAV (clamd)"; version="ClamAV 0.103.11"; virusDefinitions="27259/Sun Apr 28 08:22:36 2024" + + + Pass + + + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + UUID + c149c5a6-08f8-4bff-bb39-6c43d6a17896 + + format identification + 2024-04-29T03:51:06.790549+00:00 + + program="Siegfried"; version="1.9.6" + + + Positive + + fmt/101 + + + + preservation system + Archivematica-1.15.1 + + + repository code + AM qa/1.x SS qa/0x + + + Archivematica user pk + 1 + + + + + + + + + + + preservation system + Archivematica-1.15.1 + + Archivematica + software + + + + + + + + + + repository code + AM qa/1.x SS qa/0x + + AM qa/1.x SS qa/0x + organization + + + + + + + + + + Archivematica user pk + 1 + + username="test", first_name="", last_name="" + Archivematica user + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/archivematicaCommon/test_elasticsearch_functions.py b/tests/archivematicaCommon/test_elasticsearch_functions.py index 31c976a87c..b63033f8d7 100644 --- a/tests/archivematicaCommon/test_elasticsearch_functions.py +++ b/tests/archivematicaCommon/test_elasticsearch_functions.py @@ -1,510 +1,516 @@ +import datetime import os -import unittest +import pathlib +import uuid from unittest import mock -from unittest.mock import ANY -from unittest.mock import patch import elasticSearchFunctions import pytest +from components import helpers +from django.utils.timezone import make_aware from lxml import etree from main.models import Directory +from main.models import File from main.models import Identifier from main.models import SIP +from main.models import Transfer THIS_DIR = os.path.dirname(os.path.abspath(__file__)) -class TestElasticSearchFunctions(unittest.TestCase): - def setUp(self): - with mock.patch("elasticsearch.transport.Transport.perform_request"): - elasticSearchFunctions.setup("elasticsearch:9200") - self.client = elasticSearchFunctions.get_client() - self.aip_uuid = "b34521a3-1c63-43dd-b901-584416f36c91" - self.file_uuid = "268421a7-a986-4fa0-95c1-54176e508210" +@pytest.fixture +def es_client(): + with mock.patch("elasticsearch.transport.Transport.perform_request"): + elasticSearchFunctions.setup("elasticsearch:9200") + return elasticSearchFunctions.get_client() - @mock.patch( - "elasticsearch.transport.Transport.perform_request", - side_effect=[ - { - "took": 2, - "timed_out": False, - "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, - "hits": { - "total": 1, - "max_score": 0.2876821, - "hits": [ - { - "_index": "aips", - "_type": "_doc", - "_id": "lBsZBWgBn49OAVhMXeO8", - "_score": 0.2876821, - "_source": {"uuid": "b34521a3-1c63-43dd-b901-584416f36c91"}, - } - ], - }, - }, - { - "took": 8, - "timed_out": False, + +@mock.patch( + "elasticsearch.transport.Transport.perform_request", + side_effect=[ + { + "took": 2, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": { "total": 1, - "deleted": 1, - "batches": 1, - "version_conflicts": 0, - "noops": 0, - "retries": {"bulk": 0, "search": 0}, - "throttled_millis": 0, - "requests_per_second": -1.0, - "throttled_until_millis": 0, - "failures": [], - }, - { - "took": 0, - "timed_out": False, - "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, - "hits": {"total": 0, "max_score": None, "hits": []}, + "max_score": 0.2876821, + "hits": [ + { + "_index": "aips", + "_type": "_doc", + "_id": "lBsZBWgBn49OAVhMXeO8", + "_score": 0.2876821, + "_source": {"uuid": "b34521a3-1c63-43dd-b901-584416f36c91"}, + } + ], }, - ], + }, + { + "took": 8, + "timed_out": False, + "total": 1, + "deleted": 1, + "batches": 1, + "version_conflicts": 0, + "noops": 0, + "retries": {"bulk": 0, "search": 0}, + "throttled_millis": 0, + "requests_per_second": -1.0, + "throttled_until_millis": 0, + "failures": [], + }, + { + "took": 0, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": {"total": 0, "max_score": None, "hits": []}, + }, + ], +) +def test_delete_aip(perform_request, es_client): + aip_uuid = "b34521a3-1c63-43dd-b901-584416f36c91" + + # Verify AIP exists + results = es_client.search( + index="aips", + body={"query": {"term": {"uuid": aip_uuid}}}, + _source="uuid", ) - def test_delete_aip(self, perform_request): - # Verify AIP exists - results = self.client.search( - index="aips", - body={"query": {"term": {"uuid": self.aip_uuid}}}, - _source="uuid", - ) - assert results["hits"]["total"] == 1 - assert results["hits"]["hits"][0]["_source"]["uuid"] == self.aip_uuid - # Delete AIP - elasticSearchFunctions.delete_aip(self.client, self.aip_uuid) - # Verify AIP gone - results = self.client.search( - index="aips", - body={"query": {"term": {"uuid": self.aip_uuid}}}, - _source="uuid", - ) - assert results["hits"]["total"] == 0 + assert results["hits"]["total"] == 1 + assert results["hits"]["hits"][0]["_source"]["uuid"] == aip_uuid - @mock.patch( - "elasticsearch.transport.Transport.perform_request", - side_effect=[ - { - "took": 1, - "timed_out": False, - "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, - "hits": { - "total": 2, - "max_score": 0.2876821, - "hits": [ - { - "_index": "aipfiles", - "_type": "_doc", - "_id": "lRsZBWgBn49OAVhMXuMC", - "_score": 0.2876821, - "_source": { - "origin": "1a14043f-68ef-4bfe-a129-e2e4cdbe391b" - }, - }, - { - "_index": "aipfiles", - "_type": "_doc", - "_id": "lhsZBWgBn49OAVhMXuMh", - "_score": 0.2876821, - "_source": { - "origin": "1a14043f-68ef-4bfe-a129-e2e4cdbe391b" - }, - }, - ], - }, - }, - { - "took": 11, - "timed_out": False, + # Delete AIP + elasticSearchFunctions.delete_aip(es_client, "b34521a3-1c63-43dd-b901-584416f36c91") + + # Verify AIP gone + results = es_client.search( + index="aips", + body={"query": {"term": {"uuid": aip_uuid}}}, + _source="uuid", + ) + assert results["hits"]["total"] == 0 + + +@mock.patch( + "elasticsearch.transport.Transport.perform_request", + side_effect=[ + { + "took": 1, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": { "total": 2, - "deleted": 2, - "batches": 1, - "version_conflicts": 0, - "noops": 0, - "retries": {"bulk": 0, "search": 0}, - "throttled_millis": 0, - "requests_per_second": -1.0, - "throttled_until_millis": 0, - "failures": [], - }, - { - "took": 0, - "timed_out": False, - "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, - "hits": {"total": 0, "max_score": None, "hits": []}, + "max_score": 0.2876821, + "hits": [ + { + "_index": "aipfiles", + "_type": "_doc", + "_id": "lRsZBWgBn49OAVhMXuMC", + "_score": 0.2876821, + "_source": {"origin": "1a14043f-68ef-4bfe-a129-e2e4cdbe391b"}, + }, + { + "_index": "aipfiles", + "_type": "_doc", + "_id": "lhsZBWgBn49OAVhMXuMh", + "_score": 0.2876821, + "_source": {"origin": "1a14043f-68ef-4bfe-a129-e2e4cdbe391b"}, + }, + ], }, - ], + }, + { + "took": 11, + "timed_out": False, + "total": 2, + "deleted": 2, + "batches": 1, + "version_conflicts": 0, + "noops": 0, + "retries": {"bulk": 0, "search": 0}, + "throttled_millis": 0, + "requests_per_second": -1.0, + "throttled_until_millis": 0, + "failures": [], + }, + { + "took": 0, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": {"total": 0, "max_score": None, "hits": []}, + }, + ], +) +def test_delete_aip_files(perform_request, es_client): + aip_uuid = "b34521a3-1c63-43dd-b901-584416f36c91" + + # Verify AIP files exist + results = es_client.search( + index="aipfiles", body={"query": {"term": {"AIPUUID": aip_uuid}}} ) - def test_delete_aip_files(self, perform_request): - # Verify AIP files exist - results = self.client.search( - index="aipfiles", body={"query": {"term": {"AIPUUID": self.aip_uuid}}} - ) - assert results["hits"]["total"] == 2 - # Delete AIP files - elasticSearchFunctions.delete_aip_files(self.client, self.aip_uuid) - # Verify AIP files gone - results = self.client.search( - index="aipfiles", body={"query": {"term": {"AIPUUID": self.aip_uuid}}} - ) - assert results["hits"]["total"] == 0 + assert results["hits"]["total"] == 2 - assert perform_request.mock_calls == [ - mock.call( - "GET", - "/aipfiles/_search", - params={}, - body={ - "query": { - "term": {"AIPUUID": "b34521a3-1c63-43dd-b901-584416f36c91"} - } - }, - ), - mock.call( - "POST", - "/aipfiles/_delete_by_query", - params={}, - body={ - "query": { - "term": {"AIPUUID": "b34521a3-1c63-43dd-b901-584416f36c91"} - } - }, - ), - mock.call( - "GET", - "/aipfiles/_search", - params={}, - body={ - "query": { - "term": {"AIPUUID": "b34521a3-1c63-43dd-b901-584416f36c91"} - } - }, - ), - ] + # Delete AIP files + elasticSearchFunctions.delete_aip_files(es_client, aip_uuid) + # Verify AIP files gone + results = es_client.search( + index="aipfiles", body={"query": {"term": {"AIPUUID": aip_uuid}}} + ) + assert results["hits"]["total"] == 0 - @mock.patch( - "elasticsearch.transport.Transport.perform_request", - side_effect=[ - { - "took": 1, - "timed_out": False, - "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, - "hits": { - "total": 1, - "max_score": 0.6931472, - "hits": [ - { - "_index": "transferfiles", - "_type": "_doc", - "_id": "mBsZBWgBn49OAVhMh-OV", - "_score": 0.6931472, - "_source": { - "accessionid": "", - "status": "backlog", - "sipuuid": "17b168b6-cbba-4f43-8838-a53360238acb", - "tags": [], - "file_extension": "jpg", - "relative_path": "test-17b168b6-cbba-4f43-8838-a53360238acb/objects/Landing_zone.jpg", - "bulk_extractor_reports": [], - "origin": "1a14043f-68ef-4bfe-a129-e2e4cdbe391b", - "size": 1.2982568740844727, - "modification_date": "2018-12-11", - "created": 1546273029.7313669, - "format": [], - "ingestdate": "2018-12-31", - "filename": "Landing_zone.jpg", - "fileuuid": "268421a7-a986-4fa0-95c1-54176e508210", - }, - } - ], - }, + assert perform_request.mock_calls == [ + mock.call( + "GET", + "/aipfiles/_search", + params={}, + body={ + "query": {"term": {"AIPUUID": "b34521a3-1c63-43dd-b901-584416f36c91"}} }, - { - "_index": "transferfiles", - "_type": "_doc", - "_id": "mBsZBWgBn49OAVhMh-OV", - "_version": 2, - "result": "updated", - "forced_refresh": True, - "_shards": {"total": 2, "successful": 1, "failed": 0}, - "_seq_no": 2, - "_primary_term": 1, + ), + mock.call( + "POST", + "/aipfiles/_delete_by_query", + params={}, + body={ + "query": {"term": {"AIPUUID": "b34521a3-1c63-43dd-b901-584416f36c91"}} }, - { - "took": 2, - "timed_out": False, - "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, - "hits": { - "total": 1, - "max_score": 0.47000363, - "hits": [ - { - "_index": "transferfiles", - "_type": "_doc", - "_id": "mBsZBWgBn49OAVhMh-OV", - "_score": 0.47000363, - "_source": {"tags": ["test"]}, - } - ], - }, + ), + mock.call( + "GET", + "/aipfiles/_search", + params={}, + body={ + "query": {"term": {"AIPUUID": "b34521a3-1c63-43dd-b901-584416f36c91"}} }, - ], - ) - def test_set_get_tags(self, perform_request): - elasticSearchFunctions.set_file_tags(self.client, self.file_uuid, ["test"]) - assert elasticSearchFunctions.get_file_tags(self.client, self.file_uuid) == [ - "test" - ] + ), + ] - assert perform_request.mock_calls == [ - mock.call( - "GET", - "/transferfiles/_search", - params={"size": "10000"}, - body={ - "query": { - "term": {"fileuuid": "268421a7-a986-4fa0-95c1-54176e508210"} + +@mock.patch( + "elasticsearch.transport.Transport.perform_request", + side_effect=[ + { + "took": 1, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": { + "total": 1, + "max_score": 0.6931472, + "hits": [ + { + "_index": "transferfiles", + "_type": "_doc", + "_id": "mBsZBWgBn49OAVhMh-OV", + "_score": 0.6931472, + "_source": { + "accessionid": "", + "status": "backlog", + "sipuuid": "17b168b6-cbba-4f43-8838-a53360238acb", + "tags": [], + "file_extension": "jpg", + "relative_path": "test-17b168b6-cbba-4f43-8838-a53360238acb/objects/Landing_zone.jpg", + "bulk_extractor_reports": [], + "origin": "1a14043f-68ef-4bfe-a129-e2e4cdbe391b", + "size": 1.2982568740844727, + "modification_date": "2018-12-11", + "created": 1546273029.7313669, + "format": [], + "ingestdate": "2018-12-31", + "filename": "Landing_zone.jpg", + "fileuuid": "268421a7-a986-4fa0-95c1-54176e508210", + }, } - }, - ), - mock.call( - "POST", - "/transferfiles/_doc/mBsZBWgBn49OAVhMh-OV/_update", - params={}, - body={"doc": {"tags": ["test"]}}, - ), - mock.call( - "GET", - "/transferfiles/_search", - params={"_source": b"tags"}, - body={ - "query": { - "term": {"fileuuid": "268421a7-a986-4fa0-95c1-54176e508210"} + ], + }, + }, + { + "_index": "transferfiles", + "_type": "_doc", + "_id": "mBsZBWgBn49OAVhMh-OV", + "_version": 2, + "result": "updated", + "forced_refresh": True, + "_shards": {"total": 2, "successful": 1, "failed": 0}, + "_seq_no": 2, + "_primary_term": 1, + }, + { + "took": 2, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": { + "total": 1, + "max_score": 0.47000363, + "hits": [ + { + "_index": "transferfiles", + "_type": "_doc", + "_id": "mBsZBWgBn49OAVhMh-OV", + "_score": 0.47000363, + "_source": {"tags": ["test"]}, } - }, - ), - ] + ], + }, + }, + ], +) +def test_set_get_tags(perform_request, es_client): + file_uuid = "268421a7-a986-4fa0-95c1-54176e508210" + elasticSearchFunctions.set_file_tags(es_client, file_uuid, ["test"]) + assert elasticSearchFunctions.get_file_tags(es_client, file_uuid) == ["test"] - @mock.patch( - "elasticsearch.transport.Transport.perform_request", - side_effect=[ - { - "took": 1, - "timed_out": False, - "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, - "hits": {"total": 0, "max_score": None, "hits": []}, - } - ], - ) - def test_list_tags_fails_when_file_cant_be_found(self, perform_request): - with pytest.raises(elasticSearchFunctions.EmptySearchResultError): - elasticSearchFunctions.get_file_tags(self.client, "no_such_file") - perform_request.assert_called_once_with( + assert perform_request.mock_calls == [ + mock.call( + "GET", + "/transferfiles/_search", + params={"size": "10000"}, + body={ + "query": {"term": {"fileuuid": "268421a7-a986-4fa0-95c1-54176e508210"}} + }, + ), + mock.call( + "POST", + "/transferfiles/_doc/mBsZBWgBn49OAVhMh-OV/_update", + params={}, + body={"doc": {"tags": ["test"]}}, + ), + mock.call( "GET", "/transferfiles/_search", params={"_source": b"tags"}, - body={"query": {"term": {"fileuuid": "no_such_file"}}}, - ) + body={ + "query": {"term": {"fileuuid": "268421a7-a986-4fa0-95c1-54176e508210"}} + }, + ), + ] + + +@mock.patch( + "elasticsearch.transport.Transport.perform_request", + side_effect=[ + { + "took": 1, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": {"total": 0, "max_score": None, "hits": []}, + } + ], +) +def test_list_tags_fails_when_file_cant_be_found(perform_request, es_client): + with pytest.raises(elasticSearchFunctions.EmptySearchResultError): + elasticSearchFunctions.get_file_tags(es_client, "no_such_file") + perform_request.assert_called_once_with( + "GET", + "/transferfiles/_search", + params={"_source": b"tags"}, + body={"query": {"term": {"fileuuid": "no_such_file"}}}, + ) - @mock.patch( - "elasticsearch.transport.Transport.perform_request", - side_effect=[ - { - "took": 0, - "timed_out": False, - "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, - "hits": {"total": 0, "max_score": None, "hits": []}, - } - ], + +@mock.patch( + "elasticsearch.transport.Transport.perform_request", + side_effect=[ + { + "took": 0, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": {"total": 0, "max_score": None, "hits": []}, + } + ], +) +def test_set_tags_fails_when_file_cant_be_found(perform_request, es_client): + with pytest.raises(elasticSearchFunctions.EmptySearchResultError): + elasticSearchFunctions.set_file_tags(es_client, "no_such_file", []) + perform_request.assert_called_once_with( + "GET", + "/transferfiles/_search", + params={"size": "10000"}, + body={"query": {"term": {"fileuuid": "no_such_file"}}}, ) - def test_set_tags_fails_when_file_cant_be_found(self, perform_request): - with pytest.raises(elasticSearchFunctions.EmptySearchResultError): - elasticSearchFunctions.set_file_tags(self.client, "no_such_file", []) - perform_request.assert_called_once_with( - "GET", - "/transferfiles/_search", - params={"size": "10000"}, - body={"query": {"term": {"fileuuid": "no_such_file"}}}, - ) - @pytest.mark.django_db - @mock.patch("elasticSearchFunctions.get_dashboard_uuid") - @mock.patch("elasticSearchFunctions.bulk") - def test_index_mets_file_metadata( - self, dummy_helpers_bulk, dummy_get_dashboard_uuid - ): - # Set up mocked functions - dummy_get_dashboard_uuid.return_value = "test-uuid" - indexed_data = {} - - def _bulk(client, actions, stats_only=False, *args, **kwargs): - for item in actions: - try: - dmd_section = item["_source"]["METS"]["dmdSec"] - metadata_container = dmd_section["mets:xmlData_dict"] - dc = metadata_container["dcterms:dublincore_dict"] - except (KeyError, IndexError): - dc = None - indexed_data[item["_source"]["filePath"]] = dc - - dummy_helpers_bulk.side_effect = _bulk - - # This METS file is a cut-down version of the AIP METS produced - # using the SampleTransfers/DemoTransfer - mets_file_path = os.path.join( - THIS_DIR, "fixtures", "test_index_metadata-METS.xml" - ) - mets_object_id = "771aa252-7930-4e68-b73e-f91416b1d4a4" - uuid = "f42a260a-9b53-4555-847e-8a4329c81662" - sipName = f"DemoTransfer-{uuid}" - identifiers = [] - elasticSearchFunctions._index_aip_files( - client=self.client, - uuid=uuid, - mets=etree.parse(mets_file_path).getroot(), - name=sipName, - identifiers=identifiers, - ) - assert dummy_helpers_bulk.call_count == 1 +@pytest.mark.django_db +@mock.patch("elasticSearchFunctions.get_dashboard_uuid") +@mock.patch("elasticSearchFunctions.bulk") +def test_index_mets_file_metadata(bulk, get_dashboard_uuid, es_client): + # Set up mocked functions + get_dashboard_uuid.return_value = "test-uuid" + indexed_data = {} - # ES should have indexed 12 files - # - 5 content files - # - 5 checksum and csv files in the metadata directory - # - 2 files generated in the transfer process - assert len(indexed_data) == 12 + def _bulk(client, actions, stats_only=False, *args, **kwargs): + for item in actions: + try: + dmd_section = item["_source"]["METS"]["dmdSec"] + metadata_container = dmd_section["mets:xmlData_dict"] + dc = metadata_container["dcterms:dublincore_dict"] + except (KeyError, IndexError): + dc = None + indexed_data[item["_source"]["filePath"]] = dc - # Metadata should have been indexed only for these content - # files because they are listed in the metadata.csv file - content_files_with_metadata = ( - { - "path": ( - "objects/View_from_lookout_over_Queenstown_" - "towards_the_Remarkables_in_spring.jpg" - ), - "title": ( - "Morning view from lookout over Queenstown " - "towards the Remarkables in spring" - ), - "creator": "Pseudopanax at English Wikipedia", - }, - { - "path": "objects/beihai.tif", - "title": "Beihai, Guanxi, China, 1988", - "creator": ( - "NASA/GSFC/METI/ERSDAC/JAROS and U.S./Japan " "ASTER Science Team" - ), - }, - { - "path": "objects/bird.mp3", - "title": "14000 Caen, France - Bird in my garden", - "creator": "Nicolas Germain", - }, - { - "path": "objects/ocr-image.png", - "title": "OCR image", - "creator": "Tesseract", - }, - ) - for file_metadata in content_files_with_metadata: - dc = indexed_data[file_metadata["path"]] - assert dc["dc:title"] == file_metadata["title"] - assert dc["dc:creator"] == file_metadata["creator"] - - # There is no metadata for this content file because - # it was not listed in the metadata.csv file - assert indexed_data["objects/piiTestDataCreditCardNumbers.txt"] is None - - # Checksum and csv files in the metadata directory - # won't have dublin core metadata indexed - files_in_metadata_directory = ( - "checksum.md5", - "checksum.sha1", - "checksum.sha256", - "metadata.csv", - "rights.csv", - ) - for filename in files_in_metadata_directory: - path = "objects/metadata/transfers/DemoTransfer-{}/{}".format( - mets_object_id, filename - ) - assert indexed_data[path] is None - - # Neither will the generated files during the transfer process - generated_files = ("dc.json", "directory_tree.txt") - for filename in generated_files: - path = "objects/metadata/transfers/DemoTransfer-{}/{}".format( - mets_object_id, filename - ) - assert indexed_data[path] is None - - @pytest.mark.django_db - @mock.patch("elasticSearchFunctions.bulk") - def test_index_mets_file_metadata_with_utf8(self, dummy_helpers_bulk): - def _bulk(client, actions, stats_only=False, *args, **kwargs): - pass - - dummy_helpers_bulk.side_effect = _bulk - mets_file_path = os.path.join( - THIS_DIR, "fixtures", "test_index_metadata-METS-utf8.xml" - ) - elasticSearchFunctions._index_aip_files( - client=self.client, - uuid="", - mets=etree.parse(mets_file_path).getroot(), - name="", - identifiers=[], - ) + bulk.side_effect = _bulk - @patch("elasticSearchFunctions.create_indexes_if_needed") - def test_default_setup(self, patch): - elasticSearchFunctions.setup("elasticsearch:9200") - patch.assert_called_with( - ANY, ["aips", "aipfiles", "transfers", "transferfiles"] - ) + # This METS file is a cut-down version of the AIP METS produced + # using the SampleTransfers/DemoTransfer + mets_file_path = os.path.join(THIS_DIR, "fixtures", "test_index_metadata-METS.xml") + mets_object_id = "771aa252-7930-4e68-b73e-f91416b1d4a4" + aip_uuid = "f42a260a-9b53-4555-847e-8a4329c81662" + sipName = f"DemoTransfer-{aip_uuid}" + identifiers = [] + elasticSearchFunctions._index_aip_files( + client=es_client, + uuid=aip_uuid, + mets=etree.parse(mets_file_path).getroot(), + name=sipName, + identifiers=identifiers, + ) - @patch("elasticSearchFunctions.create_indexes_if_needed") - def test_only_aips_setup(self, patch): - elasticSearchFunctions.setup("elasticsearch:9200", enabled=["aips"]) - patch.assert_called_with(ANY, ["aips", "aipfiles"]) - - @patch("elasticSearchFunctions.create_indexes_if_needed") - def test_only_transfers_setup(self, patch): - elasticSearchFunctions.setup("elasticsearch:9200", enabled=["transfers"]) - patch.assert_called_with(ANY, ["transfers", "transferfiles"]) - - @patch("elasticSearchFunctions.create_indexes_if_needed") - def test_no_indexes_setup(self, patch): - elasticSearchFunctions.setup("elasticsearch:9200", enabled=[]) - elasticSearchFunctions.setup("elasticsearch:9200", enabled=["unknown"]) - patch.assert_not_called() - - @patch("elasticsearch.client.indices.IndicesClient.create") - @patch("elasticsearch.client.indices.IndicesClient.exists", return_value=True) - def test_create_indexes_already_created(self, mock, patch): - elasticSearchFunctions.create_indexes_if_needed( - self.client, ["aips", "aipfiles", "transfers", "transferfiles"] + assert bulk.call_count == 1 + + # ES should have indexed 12 files + # - 5 content files + # - 5 checksum and csv files in the metadata directory + # - 2 files generated in the transfer process + assert len(indexed_data) == 12 + + # Metadata should have been indexed only for these content + # files because they are listed in the metadata.csv file + content_files_with_metadata = ( + { + "path": ( + "objects/View_from_lookout_over_Queenstown_" + "towards_the_Remarkables_in_spring.jpg" + ), + "title": ( + "Morning view from lookout over Queenstown " + "towards the Remarkables in spring" + ), + "creator": "Pseudopanax at English Wikipedia", + }, + { + "path": "objects/beihai.tif", + "title": "Beihai, Guanxi, China, 1988", + "creator": ( + "NASA/GSFC/METI/ERSDAC/JAROS and U.S./Japan " "ASTER Science Team" + ), + }, + { + "path": "objects/bird.mp3", + "title": "14000 Caen, France - Bird in my garden", + "creator": "Nicolas Germain", + }, + { + "path": "objects/ocr-image.png", + "title": "OCR image", + "creator": "Tesseract", + }, + ) + for file_metadata in content_files_with_metadata: + dc = indexed_data[file_metadata["path"]] + assert dc["dc:title"] == file_metadata["title"] + assert dc["dc:creator"] == file_metadata["creator"] + + # There is no metadata for this content file because + # it was not listed in the metadata.csv file + assert indexed_data["objects/piiTestDataCreditCardNumbers.txt"] is None + + # Checksum and csv files in the metadata directory + # won't have dublin core metadata indexed + files_in_metadata_directory = ( + "checksum.md5", + "checksum.sha1", + "checksum.sha256", + "metadata.csv", + "rights.csv", + ) + for filename in files_in_metadata_directory: + path = "objects/metadata/transfers/DemoTransfer-{}/{}".format( + mets_object_id, filename ) - patch.assert_not_called() + assert indexed_data[path] is None - @patch("elasticsearch.client.indices.IndicesClient.create") - @patch("elasticsearch.client.indices.IndicesClient.exists", return_value=False) - def test_create_indexes_creation_calls(self, mock, patch): - elasticSearchFunctions.create_indexes_if_needed( - self.client, ["aips", "aipfiles", "transfers", "transferfiles"] + # Neither will the generated files during the transfer process + generated_files = ("dc.json", "directory_tree.txt") + for filename in generated_files: + path = "objects/metadata/transfers/DemoTransfer-{}/{}".format( + mets_object_id, filename ) - assert patch.call_count == 4 + assert indexed_data[path] is None - @patch("elasticsearch.client.indices.IndicesClient.create") - @patch("elasticsearch.client.indices.IndicesClient.exists", return_value=False) - def test_create_indexes_wrong_index(self, mock, patch): - elasticSearchFunctions.create_indexes_if_needed( - self.client, ["aips", "aipfiles", "unknown"] - ) - assert patch.call_count == 2 + +@pytest.mark.django_db +@mock.patch("elasticSearchFunctions.bulk") +def test_index_mets_file_metadata_with_utf8(bulk, es_client): + def _bulk(client, actions, stats_only=False, *args, **kwargs): + pass + + bulk.side_effect = _bulk + mets_file_path = os.path.join( + THIS_DIR, "fixtures", "test_index_metadata-METS-utf8.xml" + ) + elasticSearchFunctions._index_aip_files( + client=es_client, + uuid="", + mets=etree.parse(mets_file_path).getroot(), + name="", + identifiers=[], + ) + + +@mock.patch("elasticSearchFunctions.create_indexes_if_needed") +def test_default_setup(create_indexes_if_needed): + elasticSearchFunctions.setup("elasticsearch:9200") + create_indexes_if_needed.assert_called_with( + mock.ANY, ["aips", "aipfiles", "transfers", "transferfiles"] + ) + + +@mock.patch("elasticSearchFunctions.create_indexes_if_needed") +def test_only_aips_setup(create_indexes_if_needed): + elasticSearchFunctions.setup("elasticsearch:9200", enabled=["aips"]) + create_indexes_if_needed.assert_called_with(mock.ANY, ["aips", "aipfiles"]) + + +@mock.patch("elasticSearchFunctions.create_indexes_if_needed") +def test_only_transfers_setup(create_indexes_if_needed): + elasticSearchFunctions.setup("elasticsearch:9200", enabled=["transfers"]) + create_indexes_if_needed.assert_called_with( + mock.ANY, ["transfers", "transferfiles"] + ) + + +@mock.patch("elasticSearchFunctions.create_indexes_if_needed") +def test_no_indexes_setup(create_indexes_if_needed): + elasticSearchFunctions.setup("elasticsearch:9200", enabled=[]) + elasticSearchFunctions.setup("elasticsearch:9200", enabled=["unknown"]) + create_indexes_if_needed.assert_not_called() + + +@mock.patch("elasticsearch.client.indices.IndicesClient.create") +@mock.patch("elasticsearch.client.indices.IndicesClient.exists", return_value=True) +def test_create_indexes_already_created(exists, create, es_client): + elasticSearchFunctions.create_indexes_if_needed( + es_client, ["aips", "aipfiles", "transfers", "transferfiles"] + ) + create.assert_not_called() + + +@mock.patch("elasticsearch.client.indices.IndicesClient.create") +@mock.patch("elasticsearch.client.indices.IndicesClient.exists", return_value=False) +def test_create_indexes_creation_calls(exists, create, es_client): + elasticSearchFunctions.create_indexes_if_needed( + es_client, ["aips", "aipfiles", "transfers", "transferfiles"] + ) + assert create.call_count == 4 + + +@mock.patch("elasticsearch.client.indices.IndicesClient.create") +@mock.patch("elasticsearch.client.indices.IndicesClient.exists", return_value=False) +def test_create_indexes_wrong_index(exists, create, es_client): + elasticSearchFunctions.create_indexes_if_needed( + es_client, ["aips", "aipfiles", "unknown"] + ) + assert create.call_count == 2 fileuuid_premisv3 = ( @@ -570,12 +576,7 @@ def test_create_indexes_wrong_index(self, mock, patch): @mock.patch("elasticSearchFunctions.get_dashboard_uuid") @mock.patch("elasticSearchFunctions.bulk") def test_index_aipfile_fileuuid( - dummy_helpers_bulk, - dummy_get_dashboard_uuid, - metsfile, - fileuuid_dict, - aipuuid, - aipname, + bulk, get_dashboard_uuid, metsfile, fileuuid_dict, aipuuid, aipname ): """Check AIP file uuids are being correctly parsed from METS files. @@ -584,7 +585,7 @@ def test_index_aipfile_fileuuid( from the METS """ - dummy_get_dashboard_uuid.return_value = "test-uuid" + get_dashboard_uuid.return_value = "test-uuid" indexed_data = {} @@ -592,7 +593,7 @@ def _bulk(client, actions, stats_only=False, *args, **kwargs): for item in actions: indexed_data[item["_source"]["filePath"]] = item["_source"]["FILEUUID"] - dummy_helpers_bulk.side_effect = _bulk + bulk.side_effect = _bulk elasticSearchFunctions._index_aip_files( client=None, @@ -630,9 +631,7 @@ def _bulk(client, actions, stats_only=False, *args, **kwargs): ) @mock.patch("elasticSearchFunctions.get_dashboard_uuid") @mock.patch("elasticSearchFunctions.bulk") -def test_index_aipfile_dmdsec( - dummy_helpers_bulk, dummy_get_dashboard_uuid, metsfile, dmdsec_dict -): +def test_index_aipfile_dmdsec(bulk, get_dashboard_uuid, metsfile, dmdsec_dict): """Check AIP file dmdSec is correctly parsed from METS files. Mock _try_to_index() with a function that populates a dict @@ -640,7 +639,7 @@ def test_index_aipfile_dmdsec( from the METS """ - dummy_get_dashboard_uuid.return_value = "test-uuid" + get_dashboard_uuid.return_value = "test-uuid" indexed_data = {} @@ -654,7 +653,7 @@ def _bulk(client, actions, stats_only=False, *args, **kwargs): dc = None indexed_data[item["_source"]["filePath"]] = dc - dummy_helpers_bulk.side_effect = _bulk + bulk.side_effect = _bulk elasticSearchFunctions._index_aip_files( client=None, @@ -900,3 +899,546 @@ def test_get_metadata( ) == expected_metadata ) + + +def test_index_aip_and_files_logs_error_if_mets_does_not_exist( + es_client, tmp_path, caplog +): + printfn = mock.Mock() + aip_uuid = uuid.uuid4() + aip_stored_path = tmp_path / "aip.7z" + mets_staging_path = tmp_path / "mets.XML" + expected_error_message = f"METS file does not exist at: {mets_staging_path}" + + result = elasticSearchFunctions.index_aip_and_files( + es_client, + str(aip_uuid), + str(aip_stored_path), + str(mets_staging_path), + "aip", + 1024, + printfn=printfn, + ) + assert result == 1 + + assert [r.message for r in caplog.records] == [expected_error_message] + printfn.assert_called_once_with(expected_error_message, file=mock.ANY) + + +@pytest.mark.django_db +@mock.patch("elasticsearch.Elasticsearch.index") +@mock.patch( + "elasticsearch.client.cluster.ClusterClient.health", + return_value={"status": "green"}, +) +@mock.patch("elasticSearchFunctions._index_aip_files") +def test_index_aip_and_files(_index_aip_files, health, index, es_client, tmp_path): + dashboard_uuid = uuid.uuid4() + helpers.set_setting("dashboard_uuid", str(dashboard_uuid)) + printfn = mock.Mock() + aip_name = "aip" + aip_uuid = str(uuid.uuid4()) + aip_stored_path = tmp_path / "aip.7z" + mets_staging_path = pathlib.Path( + THIS_DIR, "fixtures", "test_index_aip_and_files_METS.xml" + ) + expected_file_count = 3 + accession_ids = "accession_ids" + _index_aip_files.return_value = (expected_file_count, accession_ids) + + result = elasticSearchFunctions.index_aip_and_files( + es_client, + str(aip_uuid), + str(aip_stored_path), + str(mets_staging_path), + aip_name, + 1024 * 1024 * 10, + printfn=printfn, + ) + assert result == 0 + + health.assert_called_once() + index.assert_called_once_with( + body={ + "AICID": None, + "accessionids": accession_ids, + "countAIPsinAIC": None, + "created": 1714362668, + "encrypted": False, + "filePath": str(aip_stored_path), + "file_count": expected_file_count, + "identifiers": [], + "isPartOf": "", + "location": "", + "name": aip_name, + "origin": str(dashboard_uuid), + "size": 10.0, + "status": elasticSearchFunctions.STATUS_UPLOADED, + "transferMetadata": [ + { + "__DIRECTORY_LABEL__": "objects", + "dc:title": "Pictures with DublinCore", + "dc:type": "Archival Information Package", + }, + ], + "uuid": str(aip_uuid), + }, + doc_type="_doc", + index="aips", + ) + assert printfn.mock_calls == [ + mock.call(f"AIP UUID: {aip_uuid}"), + mock.call("Indexing AIP files ..."), + mock.call(f"Files indexed: {expected_file_count}"), + mock.call("Indexing AIP ..."), + mock.call("Done."), + ] + + +def test_index_transfer_and_files_logs_error_if_transfer_path_does_not_exist( + es_client, tmp_path, caplog +): + printfn = mock.Mock() + transfer_uuid = uuid.uuid4() + transfer_path = tmp_path / "transfer" + expected_error_message = f"Transfer does not exist at: {transfer_path}" + + result = elasticSearchFunctions.index_transfer_and_files( + es_client, + str(transfer_uuid), + str(transfer_path), + 1024, + printfn=printfn, + ) + assert result == 1 + + assert [r.message for r in caplog.records] == [expected_error_message] + printfn.assert_called_once_with(expected_error_message, file=mock.ANY) + + +@pytest.mark.django_db +@pytest.fixture +def transfer(tmp_path): + transfer_dir = tmp_path / "transfer" + transfer_dir.mkdir() + + (transfer_dir / "processingMCP.xml").touch() + + return Transfer.objects.create( + # The trailing slash is expected by index_transfer_and_files. + currentlocation=f"{transfer_dir}/", + accessionid="accession_id", + ) + + +@pytest.mark.django_db +@pytest.fixture +def transfer_file(transfer): + filename = "file.txt" + (pathlib.Path(transfer.currentlocation) / filename).touch() + + result = File.objects.create( + transfer=transfer, currentlocation=f"%transferDirectory%{filename}".encode() + ) + + # enteredsystem is an auto_now DateTimeField. This resets its value. + dt = make_aware(datetime.datetime(2024, 1, 1)) + result.enteredsystem = dt + result.modificationtime = dt + result.save() + + return result + + +@pytest.mark.django_db +@mock.patch("elasticsearch.Elasticsearch.index") +@mock.patch( + "elasticsearch.client.cluster.ClusterClient.health", + return_value={"status": "green"}, +) +def test_index_transfer_and_files(health, index, es_client, transfer, transfer_file): + dashboard_uuid = uuid.uuid4() + helpers.set_setting("dashboard_uuid", str(dashboard_uuid)) + printfn = mock.Mock() + expected_transfer_name = "transfer" + expected_file_count = 1 + expected_file_name = "file.txt" + expected_date = "2024-01-01" + expected_status = "backlog" + + result = elasticSearchFunctions.index_transfer_and_files( + es_client, + str(transfer.uuid), + str(transfer.currentlocation), + 1024, + printfn=printfn, + ) + assert result == 0 + + assert health.mock_calls == [mock.call(), mock.call()] + assert index.mock_calls == [ + mock.call( + body={ + "filename": expected_file_name, + "relative_path": f"{expected_transfer_name}/{expected_file_name}", + "fileuuid": str(transfer_file.uuid), + "sipuuid": str(transfer.uuid), + "accessionid": transfer.accessionid, + "status": expected_status, + "origin": str(dashboard_uuid), + "ingestdate": expected_date, + "created": mock.ANY, + "modification_date": expected_date, + "size": 0.0, + "tags": [], + "file_extension": "txt", + "bulk_extractor_reports": [], + "format": [], + "pending_deletion": False, + }, + index="transferfiles", + doc_type="_doc", + ), + mock.call( + body={ + "accessionid": transfer.accessionid, + "file_count": expected_file_count, + "ingest_date": expected_date, + "name": expected_transfer_name, + "pending_deletion": False, + "size": 1024, + "status": expected_status, + "uuid": str(transfer.uuid), + }, + doc_type="_doc", + index="transfers", + ), + ] + # Cannot compare using mock_calls here because the use of os.listdir in + # _list_files_in_dir returns files in arbitrary order. + printfn.assert_has_calls( + [ + mock.call(f"Transfer UUID: {transfer.uuid}"), + mock.call("Indexing Transfer files ..."), + mock.call( + f"Indexing {expected_transfer_name}/{expected_file_name} (UUID: {transfer_file.uuid})" + ), + mock.call(f"Skipping indexing {expected_transfer_name}/processingMCP.xml"), + mock.call(f"Files indexed: {expected_file_count}"), + mock.call("Indexing Transfer ..."), + mock.call("Done."), + ], + any_order=True, + ) + + +@mock.patch( + "elasticSearchFunctions.search_all_results", + return_value={ + "hits": { + "hits": [ + { + "_source": { + "filename": "file.txt", + "fileuuid": "f704ab10-d52e-482f-af4a-cc21111f8df4", + }, + } + ], + }, + }, +) +def test_get_transfer_file_info_when_search_returns_a_single_document( + search_all_results, es_client +): + field = "fileuuid" + value = "f704ab10-d52e-482f-af4a-cc21111f8df4" + + result = elasticSearchFunctions.get_transfer_file_info(es_client, field, value) + + assert result == { + "filename": "file.txt", + "fileuuid": "f704ab10-d52e-482f-af4a-cc21111f8df4", + } + + search_all_results.assert_called_once_with( + es_client, + body={"query": {"term": {field: value}}}, + index=elasticSearchFunctions.TRANSFER_FILES_INDEX, + ) + + +@mock.patch( + "elasticSearchFunctions.search_all_results", + return_value={ + "hits": { + "hits": [ + { + "_source": { + "fileuuid": "f704ab10-d52e-482f-af4a-cc21111f8df4", + "filename": "foo.txt", + } + }, + { + "_source": { + "fileuuid": "f704ab10-d52e-482f-af4a-cc21111f8df4", + "filename": "bar.txt", + } + }, + ], + }, + }, +) +def test_get_transfer_file_info_logs_multiple_results(search_all_results, es_client): + field = "fileuuid" + value = "f704ab10-d52e-482f-af4a-cc21111f8df4" + + result = elasticSearchFunctions.get_transfer_file_info(es_client, field, value) + + assert result == {field: value, "filename": "foo.txt"} + + search_all_results.assert_called_once_with( + es_client, + body={"query": {"term": {field: value}}}, + index=elasticSearchFunctions.TRANSFER_FILES_INDEX, + ) + + +@pytest.mark.django_db +@mock.patch("elasticSearchFunctions.get_client") +@mock.patch("elasticSearchFunctions._document_ids_from_field_query") +def test_remove_backlog_transfer_files( + _document_ids_from_field_query, client, transfer +): + file_doc_id = str(uuid.uuid4()) + _document_ids_from_field_query.return_value = [file_doc_id] + + elasticSearchFunctions.remove_backlog_transfer_files(client, transfer.uuid) + + _document_ids_from_field_query.assert_called_once_with( + client, elasticSearchFunctions.TRANSFER_FILES_INDEX, "sipuuid", transfer.uuid + ) + client.delete.assert_called_once_with( + index=elasticSearchFunctions.TRANSFER_FILES_INDEX, + doc_type=elasticSearchFunctions.DOC_TYPE, + id=file_doc_id, + ) + + +@pytest.mark.django_db +@mock.patch("elasticSearchFunctions.get_client") +@mock.patch("elasticSearchFunctions._document_ids_from_field_query") +def test_remove_sip_transfer_files( + _document_ids_from_field_query, client, transfer, transfer_file +): + file_doc_id = str(uuid.uuid4()) + _document_ids_from_field_query.return_value = [file_doc_id] + + elasticSearchFunctions.remove_sip_transfer_files(client, transfer.uuid) + + _document_ids_from_field_query.assert_called_once_with( + client, elasticSearchFunctions.TRANSFER_FILES_INDEX, "sipuuid", transfer.uuid + ) + client.delete.assert_called_once_with( + index=elasticSearchFunctions.TRANSFER_FILES_INDEX, + doc_type=elasticSearchFunctions.DOC_TYPE, + id=file_doc_id, + ) + + +@mock.patch("elasticSearchFunctions.get_client") +@mock.patch("elasticSearchFunctions._document_ids_from_field_query", return_value=[]) +def test_mark_aip_stored_logs_error(_document_ids_from_field_query, client, caplog): + aip_uuid = str(uuid.uuid4()) + + elasticSearchFunctions.mark_aip_stored(es_client, aip_uuid) + + _document_ids_from_field_query.assert_called_once_with( + es_client, + elasticSearchFunctions.AIPS_INDEX, + elasticSearchFunctions.ES_FIELD_UUID, + aip_uuid, + ) + assert [r.message for r in caplog.records] == [ + f"Unable to find document with UUID {aip_uuid} in index {elasticSearchFunctions.AIPS_INDEX}" + ] + + +@mock.patch("elasticSearchFunctions.get_client") +@mock.patch("elasticSearchFunctions._document_ids_from_field_query") +def test_mark_aip_stored(_document_ids_from_field_query, client): + aip_uuid = str(uuid.uuid4()) + aip_doc_id = str(uuid.uuid4()) + _document_ids_from_field_query.return_value = [aip_doc_id] + + elasticSearchFunctions.mark_aip_stored(client, aip_uuid) + + client.update.assert_called_once_with( + body={ + "doc": { + elasticSearchFunctions.ES_FIELD_STATUS: elasticSearchFunctions.STATUS_UPLOADED + } + }, + index=elasticSearchFunctions.AIPS_INDEX, + doc_type=elasticSearchFunctions.DOC_TYPE, + id=aip_doc_id, + ) + _document_ids_from_field_query.assert_called_once_with( + client, + elasticSearchFunctions.AIPS_INDEX, + elasticSearchFunctions.ES_FIELD_UUID, + aip_uuid, + ) + + +@pytest.mark.parametrize( + "helper,package_index,files_index,package_uuid_field,field,value", + ( + ( + elasticSearchFunctions.mark_aip_deletion_requested, + elasticSearchFunctions.AIPS_INDEX, + elasticSearchFunctions.AIP_FILES_INDEX, + "AIPUUID", + elasticSearchFunctions.ES_FIELD_STATUS, + elasticSearchFunctions.STATUS_DELETE_REQUESTED, + ), + ( + elasticSearchFunctions.mark_backlog_deletion_requested, + elasticSearchFunctions.TRANSFERS_INDEX, + elasticSearchFunctions.TRANSFER_FILES_INDEX, + "sipuuid", + "pending_deletion", + True, + ), + ( + elasticSearchFunctions.revert_aip_deletion_request, + elasticSearchFunctions.AIPS_INDEX, + elasticSearchFunctions.AIP_FILES_INDEX, + "AIPUUID", + elasticSearchFunctions.ES_FIELD_STATUS, + elasticSearchFunctions.STATUS_UPLOADED, + ), + ( + elasticSearchFunctions.revert_backlog_deletion_request, + elasticSearchFunctions.TRANSFERS_INDEX, + elasticSearchFunctions.TRANSFER_FILES_INDEX, + "sipuuid", + "pending_deletion", + False, + ), + ), + ids=[ + "mark_aip_deletion_requested", + "mark_backlog_deletion_requested", + "revert_aip_deletion_request", + "revert_backlog_deletion_request", + ], +) +@mock.patch("elasticSearchFunctions.get_client") +@mock.patch("elasticSearchFunctions._document_ids_from_field_query") +def test_update_helpers( + _document_ids_from_field_query, + client, + helper, + package_index, + files_index, + package_uuid_field, + field, + value, +): + package_uuid = str(uuid.uuid4()) + package_doc_id = str(uuid.uuid4()) + file_doc_id = str(uuid.uuid4()) + _document_ids_from_field_query.side_effect = [[package_doc_id], [file_doc_id]] + + helper(client, package_uuid) + + assert client.update.mock_calls == [ + mock.call( + body={"doc": {field: value}}, + index=package_index, + doc_type=elasticSearchFunctions.DOC_TYPE, + id=package_doc_id, + ), + mock.call( + body={"doc": {field: value}}, + index=files_index, + doc_type=elasticSearchFunctions.DOC_TYPE, + id=file_doc_id, + ), + ] + assert _document_ids_from_field_query.mock_calls == [ + mock.call( + client, package_index, elasticSearchFunctions.ES_FIELD_UUID, package_uuid + ), + mock.call(client, files_index, package_uuid_field, package_uuid), + ] + + +def test_augment_raw_search_results(): + raw_results = { + "hits": { + "hits": [ + {"_id": "123", "_source": {"filename": "foo.txt"}}, + {"_id": "456", "_source": {"filename": "bar.txt"}}, + ], + }, + } + + result = elasticSearchFunctions.augment_raw_search_results(raw_results) + + assert result == [ + {"document_id": "123", "filename": "foo.txt"}, + {"document_id": "456", "filename": "bar.txt"}, + ] + + +@mock.patch("elasticSearchFunctions.get_client") +def test_try_to_index_fails_with_invalid_maximum_retries(client): + with pytest.raises(ValueError, match="max_tries must be 1 or greater"): + elasticSearchFunctions._try_to_index( + client, {}, elasticSearchFunctions.AIPS_INDEX, max_tries=0 + ) + + +@mock.patch("elasticSearchFunctions.get_client") +def test_try_to_index_retries_after_error(client): + error = Exception("error") + client.index.side_effect = [error, None] + printfn = mock.Mock() + data = {} + index = elasticSearchFunctions.AIPS_INDEX + + elasticSearchFunctions._try_to_index( + client, data, index, wait_between_tries=0, printfn=printfn + ) + + assert client.index.mock_calls == [ + mock.call(body=data, index=index, doc_type=elasticSearchFunctions.DOC_TYPE), + mock.call(body=data, index=index, doc_type=elasticSearchFunctions.DOC_TYPE), + ] + assert printfn.mock_calls == [ + mock.call("ERROR: error trying to index."), + mock.call(error), + ] + + +@mock.patch("elasticSearchFunctions.get_client") +def test_try_to_index_raises_exception_after_retries(client): + error = Exception("error") + client.index.side_effect = [error, None] + printfn = mock.Mock() + data = {} + index = elasticSearchFunctions.AIPS_INDEX + + with pytest.raises(Exception, match="error"): + elasticSearchFunctions._try_to_index( + client, data, index, wait_between_tries=0, max_tries=1, printfn=printfn + ) + + assert client.index.mock_calls == [ + mock.call(body=data, index=index, doc_type=elasticSearchFunctions.DOC_TYPE), + ] + assert printfn.mock_calls == [ + mock.call("ERROR: error trying to index."), + mock.call(error), + ] From 58e30b371fd4351e2c059f6fd4f007207d1e10b0 Mon Sep 17 00:00:00 2001 From: "Douglas Cerna (Soy Douglas)" Date: Tue, 30 Apr 2024 17:56:17 +0200 Subject: [PATCH 11/32] Remove comment from JHOVE validation migration --- src/dashboard/src/fpr/migrations/0040_update_jhove_validation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/dashboard/src/fpr/migrations/0040_update_jhove_validation.py b/src/dashboard/src/fpr/migrations/0040_update_jhove_validation.py index c062de5566..adff577fb7 100644 --- a/src/dashboard/src/fpr/migrations/0040_update_jhove_validation.py +++ b/src/dashboard/src/fpr/migrations/0040_update_jhove_validation.py @@ -1,4 +1,3 @@ -"""Update commands and rules for Python 3 compatibility.""" from django.db import migrations JHOVE_TOOL_ID = "085d8690-93b7-4d31-84f7-2c5f4cbf6735" From ba0db52e575f8e14f3e26c82490ff495afb292d1 Mon Sep 17 00:00:00 2001 From: "Douglas Cerna (Soy Douglas)" Date: Tue, 30 Apr 2024 17:56:46 +0200 Subject: [PATCH 12/32] Fix default thumbnail normalization command This adds a data migration for replacing the existing command with a fixed version of its Python script. --- .../0043_update_default_thumbnail_command.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 src/dashboard/src/fpr/migrations/0043_update_default_thumbnail_command.py diff --git a/src/dashboard/src/fpr/migrations/0043_update_default_thumbnail_command.py b/src/dashboard/src/fpr/migrations/0043_update_default_thumbnail_command.py new file mode 100644 index 0000000000..511b6122b3 --- /dev/null +++ b/src/dashboard/src/fpr/migrations/0043_update_default_thumbnail_command.py @@ -0,0 +1,107 @@ +from django.db import migrations + +OLD_DEFAULT_THUMBNAIL_CMD_UUID = "95149bc4-0620-4c20-964c-1d6c34b9400e" + +DEFAULT_THUMBNAIL_RULES = ("3a19f9a3-c5d5-4934-9286-13b3ad6c24d3",) + +NEW_DEFAULT_THUMBNAIL_CMD_UUID = "484d3a8f-9e59-4912-a5b8-f8a2deb3466a" +NEW_DEFAULT_THUMBNAIL_CMD = """ +import argparse +import base64 +import sys + +# http://i.imgur.com/ijwSkff.jpg +DEFAULT_THUMBNAIL = \"\"\" +/9j/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB +AQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB +AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAAwADADASIAAhEBAxEB/8QAHAABAQACAgMA +AAAAAAAAAAAACAAFCQEKAwQG/8QANBAAAAUDAgIFCwUAAAAAAAAAAQMEBQYAAgcICRESExQ5eLcV +GSExV1iVmMHS1xZScYWn/8QAFgEBAQEAAAAAAAAAAAAAAAAAAAEC/8QAKBEAAgADBgUFAAAAAAAA +AAAAAAECESESMVFhkcEiQUJxoVJystHS/9oADAMBAAIRAxEAPwDelul5RzWOsiAYeh2d8yYlgjdp +mTZLBtxBOF+PVTjLXvKcmi7iufHJjLKXPye5mj7MQhQu5yxIymJVKhjKbTnl/vdQn13Pfvpa1vmN +mf30vNzvtGIj3KWDx0n9F2gML13Pfvpa1vmNmf314znLPJBJp9+tLWwNhJRhtwW6jJkNw2l2jfcF +oCYADcIAPABEA4+sQD01nq9VcUaciWEkcnTGpVBRPSCNpfSmFX2F842gNwWc4hzCACIW8RABH0UA +xdv3cZl2K2rHuPNW85dpbjLIrexrYhqAmLkscHTGsslRFi+yIZZkj48Ozu6QV6VKRvbMivKm0uBv +Sq9reDUuNFrVZivsa10WSpLlNjhDVBXjT4dJWxsjLVHFR9kua1qNyKbW5Mg6yY3AxKxKAy5PapAm ++43oLuXlPvEsDR3HbLOsLKUyksm0c5BaXVQx4zxmom+OnWTOVy2UQ6Mx56g8SGAqHDqKW2SxxQfL +LXOJ86VrugaBtURZrPeYisizHj2TWK1B8Tud9oxEe5SweOk/ou0otzvtGIj3KWDx0n9F2qCqqqoC +pC7VicpPuS5UEouwvrGkZ5PM5LQt5zByliEq6+7gAcbhAq0BuH0iAAHqAKPVI3a07SPJfc+efFjF +FZivg9z+MRVdF23RldzvtGIj3KWDx0n9F2tru4Lt+ah9ROf4VnnT9P8AEjGvb8PW4ik0fysErbiC +EjXNHmZNrwyuMXj8sFzNdTpGqRL0S1AyeRbWNIemWPnl5QQxC7zWe5H7TNH4f3OWPricarbXS3mr +O8SfgSzS12TDnVSM81nuSe0zR/8AGcr/AImq81nuSe0zR/8AGcr/AImqWn6ItYP0JLFefoOdIzaz +HjuSZND9uj94D/V8TXfWuB2styQfVk7SAH8PGVvriUaY+3ht4ahNNeoTIef8/wCQ8UyBxkGKQxdH +o9i79TuKe9O4yaNSNxdXVxkcah1zX5LuhyBMhQpkD/e9Xv7goUODEWxpEj1Ktw8LUnOtnBqSk3Wv +a+ook6pzWeKfNZH/2Q==\"\"\" + +def main(target): + with open(target, 'wb') as f: + f.write(base64.b64decode(DEFAULT_THUMBNAIL)) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--output-location', required=True) + args, _ = parser.parse_known_args() + + sys.exit(main(args.output_location)) +""" + + +def data_migration_up(apps, schema_editor): + FPCommand = apps.get_model("fpr", "FPCommand") + FPRule = apps.get_model("fpr", "FPRule") + + # Get the old command + old_command = FPCommand.objects.get(uuid=OLD_DEFAULT_THUMBNAIL_CMD_UUID) + + # Replace the existing command with the following + FPCommand.objects.create( + uuid=NEW_DEFAULT_THUMBNAIL_CMD_UUID, + replaces_id=OLD_DEFAULT_THUMBNAIL_CMD_UUID, + tool_id=old_command.tool_id, + enabled=old_command.enabled, + command=NEW_DEFAULT_THUMBNAIL_CMD, + script_type=old_command.script_type, + command_usage=old_command.command_usage, + description=old_command.description, + output_location=old_command.output_location, + output_format_id=old_command.output_format_id, + event_detail_command_id=old_command.event_detail_command_id, + verification_command_id=old_command.verification_command_id, + ) + + # Update existing rules + FPRule.objects.filter(uuid__in=DEFAULT_THUMBNAIL_RULES).update( + command_id=NEW_DEFAULT_THUMBNAIL_CMD_UUID, + ) + + # Disable the old command + old_command.enabled = False + old_command.save() + + +def data_migration_down(apps, schema_editor): + FPCommand = apps.get_model("fpr", "FPCommand") + FPRule = apps.get_model("fpr", "FPRule") + + # The order matters. We make sure that the rules point to the previous + # command before the latter is deleted. Otherwise our rules would be + # deleted by Django's on cascade mechanism + FPRule.objects.filter(uuid__in=DEFAULT_THUMBNAIL_RULES).update( + command_id=OLD_DEFAULT_THUMBNAIL_CMD_UUID, + ) + + # Enable the old command. At this point we do not know if the + # command was in fact enabled before the migration was run, so + # this may have unexpected consequences + old_command = FPCommand.objects.get(uuid=OLD_DEFAULT_THUMBNAIL_CMD_UUID) + old_command.enabled = True + old_command.save() + + # Delete the new command + FPCommand.objects.filter(uuid=NEW_DEFAULT_THUMBNAIL_CMD_UUID).delete() + + +class Migration(migrations.Migration): + dependencies = [("fpr", "0042_update_idtools")] + + operations = [migrations.RunPython(data_migration_up, data_migration_down)] From fb0990297b97e2cb9f9ade097055b72b222bbb83 Mon Sep 17 00:00:00 2001 From: "Douglas Cerna (Soy Douglas)" Date: Tue, 7 May 2024 17:50:58 +0200 Subject: [PATCH 13/32] Fix SIP arrangement from ArchivesSpace pane --- src/dashboard/src/components/access/urls.py | 1 + src/dashboard/src/components/access/views.py | 4 +- tests/dashboard/test_access.py | 207 +++++++++++++++++++ 3 files changed, 210 insertions(+), 2 deletions(-) diff --git a/src/dashboard/src/components/access/urls.py b/src/dashboard/src/components/access/urls.py index 3cb41408ff..5e9937fa16 100644 --- a/src/dashboard/src/components/access/urls.py +++ b/src/dashboard/src/components/access/urls.py @@ -19,6 +19,7 @@ re_path( r"archivesspace/(?P[A-Za-z0-9-_]+)/copy_from_arrange/$", views.access_arrange_start_sip, + name="access_arrange_start_sip", ), re_path( r"archivesspace/(?P[A-Za-z0-9-_]+)/create_directory_within_arrange/$", diff --git a/src/dashboard/src/components/access/views.py b/src/dashboard/src/components/access/views.py index c47c873c42..e624f3ea09 100644 --- a/src/dashboard/src/components/access/views.py +++ b/src/dashboard/src/components/access/views.py @@ -97,7 +97,7 @@ def _get_sip(func): @wraps(func) def wrapper(request, mapping): arrange = SIPArrange.objects.get( - arrange_path=os.path.join(mapping.arrange_path, "") + arrange_path=os.path.join(mapping.arrange_path, "").encode() ) if arrange.sip is None: arrange.sip = SIP.objects.create(uuid=(uuid.uuid4()), currentpath=None) @@ -421,7 +421,7 @@ def access_arrange_start_sip(client, request, mapping): """ try: arrange = SIPArrange.objects.get( - arrange_path=os.path.join(mapping.arrange_path, "") + arrange_path=os.path.join(mapping.arrange_path, "").encode() ) except SIPArrange.DoesNotExist: response = { diff --git a/tests/dashboard/test_access.py b/tests/dashboard/test_access.py index abc5497e4f..718b6fb37a 100644 --- a/tests/dashboard/test_access.py +++ b/tests/dashboard/test_access.py @@ -1,7 +1,9 @@ import json import pathlib +from unittest import mock import archivematicaFunctions +import pytest from components import helpers from django.test import TestCase from django.test.client import Client @@ -56,3 +58,208 @@ def test_arrange_contents(self): in response_dict["entries"] ) assert len(response_dict["entries"]) == 1 + + +@pytest.mark.django_db +@pytest.fixture +def dashboard_uuid(): + helpers.set_setting("dashboard_uuid", "test-uuid") + + +def _encode_record_id(record_id): + return record_id.replace("/", "") + + +@pytest.mark.django_db +def test_access_arrange_start_sip_fails_if_arrange_mapping_does_not_exist( + dashboard_uuid, admin_client +): + record_id = "/repositories/2/archival_objects/1" + + response = admin_client.get( + reverse( + "access:access_arrange_start_sip", + kwargs={"record_id": _encode_record_id(record_id)}, + ) + ) + assert response.status_code == 404 + + result = json.loads(response.content.decode()) + assert result == { + "message": f"No SIP Arrange mapping exists for record {_encode_record_id(record_id)}", + "success": False, + } + + +@pytest.mark.django_db +@mock.patch("components.access.views.get_as_system_client") +def test_access_arrange_start_sip_fails_if_arrange_does_not_exist( + get_as_system_client, dashboard_uuid, admin_client +): + record_id = "/repositories/2/archival_objects/1" + models.SIPArrangeAccessMapping.objects.create( + arrange_path="/foobar", + system=models.SIPArrangeAccessMapping.ARCHIVESSPACE, + identifier=_encode_record_id(record_id), + ) + + response = admin_client.get( + reverse( + "access:access_arrange_start_sip", + kwargs={"record_id": _encode_record_id(record_id)}, + ) + ) + assert response.status_code == 404 + + result = json.loads(response.content.decode()) + assert result == { + "message": f"No SIP Arrange object exists for record {_encode_record_id(record_id)}", + "success": False, + } + + +@pytest.mark.django_db +@mock.patch( + "components.access.views.get_as_system_client", + return_value=mock.Mock( + **{ + "get_record.side_effect": [ + # Archival object. + {"resource": {"ref": "/repositories/2/resources/10"}}, + # Resource. + {"linked_agents": []}, + ] + } + ), +) +def test_access_arrange_start_sip_fails_if_resource_creators_cannot_be_fetched( + get_as_system_client, dashboard_uuid, admin_client +): + record_id = "/repositories/2/archival_objects/1" + mapping = models.SIPArrangeAccessMapping.objects.create( + arrange_path="/foobar", + system=models.SIPArrangeAccessMapping.ARCHIVESSPACE, + identifier=_encode_record_id(record_id), + ) + models.SIPArrange.objects.create( + arrange_path=f"{pathlib.Path(mapping.arrange_path)}/".encode() + ) + + response = admin_client.get( + reverse( + "access:access_arrange_start_sip", + kwargs={"record_id": _encode_record_id(record_id)}, + ) + ) + assert response.status_code == 502 + + result = json.loads(response.content.decode()) + assert result == { + "message": "Unable to fetch ArchivesSpace creator", + "success": False, + } + + +@pytest.mark.django_db +@mock.patch("components.filesystem_ajax.views.copy_from_arrange_to_completed_common") +@mock.patch("components.access.views.get_as_system_client") +def test_access_arrange_start_sip( + get_as_system_client, + copy_from_arrange_to_completed_common, + dashboard_uuid, + admin_client, + caplog, +): + # Mock expected responses from ArchivesSpace. + archival_object = { + "resource": {"ref": "/repositories/2/resources/10"}, + "notes": [{"type": "odd", "subnotes": [{"content": "A note"}]}], + "display_string": "Object, 2024", + "parent": {"ref": "/repositories/2/resources/1"}, + } + resource = {"linked_agents": [{"ref": "/agents/people/3", "role": "creator"}]} + creator = {"display_name": {"sort_name": "Foo, Bar"}} + parent = {"title": "Parent resource"} + digital_object = {"id": "do"} + get_as_system_client.return_value = mock.Mock( + **{ + "get_record.side_effect": [archival_object, resource, creator, parent], + "add_digital_object.side_effect": [ + digital_object, + ], + } + ) + + # Mock interaction with copy_from_arrange_to_completed_common. + sip = models.SIP.objects.create() + expected_status_code = 201 + expected_response_content = {"message": "SIP created.", "sip_uuid": str(sip.uuid)} + copy_from_arrange_to_completed_common.return_value = ( + expected_status_code, + expected_response_content, + ) + + # Set database fixtures. + record_id = "/repositories/2/archival_objects/1" + mapping = models.SIPArrangeAccessMapping.objects.create( + arrange_path="/foobar", + system=models.SIPArrangeAccessMapping.ARCHIVESSPACE, + identifier=_encode_record_id(record_id), + ) + models.SIPArrange.objects.create( + arrange_path=f"{pathlib.Path(mapping.arrange_path)}/".encode() + ) + models.ArchivesSpaceDigitalObject.objects.create( + resourceid=_encode_record_id(record_id), started=False + ) + + response = admin_client.post( + reverse( + "access:access_arrange_start_sip", + kwargs={"record_id": _encode_record_id(record_id)}, + ), + data=json.dumps({}), + content_type="application/json", + ) + assert response.status_code == expected_status_code + + result = json.loads(response.content.decode()) + assert result == expected_response_content + + assert models.DublinCore.objects.count() == 1 + assert ( + models.DublinCore.objects.filter( + metadataappliestotype_id=models.MetadataAppliesToType.SIP_TYPE, + metadataappliestoidentifier=sip.uuid, + title=archival_object["display_string"], + creator=creator["display_name"]["sort_name"], + description=archival_object["notes"][0]["subnotes"][0]["content"], + rights=" ".join( + [ + "This content may be under copyright.", + "Researchers are responsible for determining the", + "appropriate use or reuse of materials.", + ] + ), + relation=parent["title"], + ).count() + == 1 + ) + + assert models.ArchivesSpaceDigitalObject.objects.count() == 1 + assert ( + models.ArchivesSpaceDigitalObject.objects.filter( + resourceid=_encode_record_id(record_id), + started=True, + remoteid=digital_object["id"], + sip_id=expected_response_content["sip_uuid"], + ).count() + == 1 + ) + + assert [r.message for r in caplog.records] == [ + f"archival object {archival_object}", + f"resource {resource}", + f"creator {creator}", + f"New SIP UUID {sip.uuid}", + ] From 79ab3d526345ef17d8d6ce9dae07cb60b7c06680 Mon Sep 17 00:00:00 2001 From: Douglas Cerna Date: Wed, 8 May 2024 15:40:16 -0600 Subject: [PATCH 14/32] Fix failure reports index --- src/dashboard/src/templates/_pager.html | 8 +++--- .../administration/test_administration.py | 25 ++++++++++++++++--- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/dashboard/src/templates/_pager.html b/src/dashboard/src/templates/_pager.html index e9eb32d0ae..e28353686c 100644 --- a/src/dashboard/src/templates/_pager.html +++ b/src/dashboard/src/templates/_pager.html @@ -4,11 +4,11 @@
{% if page.has_previous %} - {% ifnotequal page.previous_page_number 1 %} + {% if not page.previous_page_number == 1 %} {% trans "First" %} - {% endifnotequal %} + {% endif %} {% trans "Previous" %} @@ -34,11 +34,11 @@ {% trans "Next" %} - {% ifnotequal page.next_page_number page.paginator.num_pages %} + {% if not page.next_page_number == page.paginator.num_pages %} {% trans "Last" %} - {% endifnotequal %} + {% endif %} {% endif %} diff --git a/tests/dashboard/components/administration/test_administration.py b/tests/dashboard/components/administration/test_administration.py index 9cfb18b134..15abeb5fff 100644 --- a/tests/dashboard/components/administration/test_administration.py +++ b/tests/dashboard/components/administration/test_administration.py @@ -1,13 +1,19 @@ +import uuid + import pytest from components import helpers from django.urls import reverse from main.models import Report +@pytest.fixture @pytest.mark.django_db -def test_admin_set_language(admin_client): - helpers.set_setting("dashboard_uuid", "test-uuid") +def dashboard_uuid(): + helpers.set_setting("dashboard_uuid", str(uuid.uuid4())) + +@pytest.mark.django_db +def test_admin_set_language(dashboard_uuid, admin_client): response = admin_client.get(reverse("administration:admin_set_language")) assert response.status_code == 200 @@ -17,8 +23,7 @@ def test_admin_set_language(admin_client): @pytest.mark.django_db -def test_failure_report_delete(admin_client): - helpers.set_setting("dashboard_uuid", "test-uuid") +def test_failure_report_delete(dashboard_uuid, admin_client): report = Report.objects.create(content="my report") response = admin_client.post( @@ -30,3 +35,15 @@ def test_failure_report_delete(admin_client): assert "No reports found." in response.content.decode() assert Report.objects.count() == 0 + + +@pytest.mark.django_db +def test_failure_report(dashboard_uuid, admin_client): + report = Report.objects.create(content="my report") + + response = admin_client.get(reverse("administration:reports_failures_index")) + assert response.status_code == 200 + + content = response.content.decode() + assert "

Failure report

" in content + assert reverse("administration:failure_report", args=[report.pk]) in content From 34aa3c8031fa3731344ecb4216f54c433b3364c1 Mon Sep 17 00:00:00 2001 From: "Douglas Cerna (Soy Douglas)" Date: Wed, 8 May 2024 20:54:48 +0200 Subject: [PATCH 15/32] Remove references to Binder --- src/MCPServer/lib/assets/workflow.json | 12 ++++++------ .../components/administration/forms_dip_upload.py | 10 +++++----- src/dashboard/src/media/js/ingest.js | 9 ++------- .../src/templates/administration/dips_atom_edit.html | 4 ++-- .../src/templates/administration/sidebar.html | 2 +- src/dashboard/src/templates/ingest/grid.html | 3 +-- 6 files changed, 17 insertions(+), 23 deletions(-) diff --git a/src/MCPServer/lib/assets/workflow.json b/src/MCPServer/lib/assets/workflow.json index 48c43dcc57..8f0ce2e563 100644 --- a/src/MCPServer/lib/assets/workflow.json +++ b/src/MCPServer/lib/assets/workflow.json @@ -69,9 +69,9 @@ }, "0fe9842f-9519-4067-a691-8a363132ae24": { "description": { - "en": "Upload DIP to AtoM/Binder", - "no": "Last opp DIP til AtoM/Binder", - "pt_BR": "Carregar DIP no AtoM/Binder" + "en": "Upload DIP to AtoM", + "no": "Last opp DIP til AtoM", + "pt_BR": "Carregar DIP no AtoM" }, "link_id": "7f975ba6-2185-434c-b507-2911f3c77213" }, @@ -6746,9 +6746,9 @@ "replacements": [] }, "description": { - "en": "Choose config for AtoM/Binder DIP upload", - "no": "Velg konfigurasjon for AtoM/Binder DIP opplasting", - "pt_BR": "Escolher configuração para carregar DIP no AtoM/Binder" + "en": "Choose config for AtoM DIP upload", + "no": "Velg konfigurasjon for AtoM DIP opplasting", + "pt_BR": "Escolher configuração para carregar DIP no AtoM" }, "exit_codes": { "0": { diff --git a/src/dashboard/src/components/administration/forms_dip_upload.py b/src/dashboard/src/components/administration/forms_dip_upload.py index 394125f87c..4b62fac4de 100644 --- a/src/dashboard/src/components/administration/forms_dip_upload.py +++ b/src/dashboard/src/components/administration/forms_dip_upload.py @@ -90,24 +90,24 @@ class AtomConfigForm(forms.Form): url = forms.CharField( label=_("Upload URL"), help_text=_( - "URL where the AtoM/Binder index.php frontend lives, SWORD services path will be appended." + "URL where the AtoM index.php frontend lives, SWORD services path will be appended." ), ) email = forms.CharField( label=_("Login email"), - help_text=_("E-mail account used to log into AtoM/Binder."), + help_text=_("E-mail account used to log into AtoM."), ) password = forms.CharField( - label=_("Login password"), help_text=_("Password used to log into AtoM/Binder.") + label=_("Login password"), help_text=_("Password used to log into AtoM.") ) version = forms.ChoiceField( - label=_("AtoM/Binder version"), choices=((1, "1.x"), (2, "2.x")) + label=_("AtoM version"), choices=((1, "1.x"), (2, "2.x")) ) rsync_target = forms.CharField( required=False, label=_("Rsync target"), help_text=_( - "The DIP can be sent with Rsync to a remote host before is deposited in AtoM/Binder. This is the destination value passed to Rsync (see man 1 rsync). For example: foobar.com:~/dips/." + "The DIP can be sent with Rsync to a remote host before is deposited in AtoM. This is the destination value passed to Rsync (see man 1 rsync). For example: foobar.com:~/dips/." ), ) rsync_command = forms.CharField( diff --git a/src/dashboard/src/media/js/ingest.js b/src/dashboard/src/media/js/ingest.js index 18bbe20fb7..efac18ebc7 100644 --- a/src/dashboard/src/media/js/ingest.js +++ b/src/dashboard/src/media/js/ingest.js @@ -313,8 +313,8 @@ $(function() return false; } - // "Upload DIP to AtoM/Binder" chain matched by its UUID. - // If no identifier for the AtoM or Binder SWORD V1 deposit endpoint + // "Upload DIP to AtoM" chain matched by its UUID. + // If no identifier for the AtoM SWORD V1 deposit endpoint // provided at start of transfer, display a modal dialog to request // such here. if (chainId == '0fe9842f-9519-4067-a691-8a363132ae24') @@ -407,11 +407,6 @@ $(function() .modal('show'); } else { - // The access system ID that the user supplies at the start of - // transfer must contain the correct target prefix if the upload - // is to Binder, i.e., the 'ar:' prefix for an artwork record and - // the 'tr:' prefix for a technical record. This is explained in - // the modal dialog help text. See templates/ingest/grid.html. var xhr = $.ajax(url, { type: 'POST', data: {'target': this.model.sip.attributes.access_system_id}, diff --git a/src/dashboard/src/templates/administration/dips_atom_edit.html b/src/dashboard/src/templates/administration/dips_atom_edit.html index be8627ebf6..81c7e3d1c9 100644 --- a/src/dashboard/src/templates/administration/dips_atom_edit.html +++ b/src/dashboard/src/templates/administration/dips_atom_edit.html @@ -35,9 +35,9 @@
{% csrf_token %} -

{% trans "AtoM/Binder DIP upload" %}

+

{% trans "AtoM DIP upload" %}

-

{% trans "The settings below configure DIP uploading to AtoM/Binder." %}

+

{% trans "The settings below configure DIP uploading to AtoM." %}

{% include "_form.html" %} diff --git a/src/dashboard/src/templates/administration/sidebar.html b/src/dashboard/src/templates/administration/sidebar.html index 56fe1703d2..6826129f96 100644 --- a/src/dashboard/src/templates/administration/sidebar.html +++ b/src/dashboard/src/templates/administration/sidebar.html @@ -31,7 +31,7 @@
  • {% trans "DIP upload" %}
    • diff --git a/src/dashboard/src/templates/ingest/grid.html b/src/dashboard/src/templates/ingest/grid.html index cf24aefd3c..575c47d7b6 100644 --- a/src/dashboard/src/templates/ingest/grid.html +++ b/src/dashboard/src/templates/ingest/grid.html @@ -155,10 +155,9 @@

    {% trans "Upload DIP" %}

    -

    {% trans "Please provide the identifier required by the AtoM or Binder SWORD V1 deposit endpoint." %}

    +

    {% trans "Please provide the identifier required by the AtoM SWORD V1 deposit endpoint." %}

    • {% trans "AtoM: In AtoM, this is the permalink or slug of the target archival description where the DIP will be attached as a child. For example, if the URL of the archival description is 'http://myAtoM.ca/newsletters-2;rad', then you would enter 'newsletters-2'." %}
      • -
    • {% trans "Binder: In Binder, this is an identifier that indicates which resource or technology record the DIP will be attached to. Resources are preceded by 'ar:' (e.g. 'ar:123') and technology records are preceded by 'tr:' (e.g. 'tr:456')." %}
    From 159a1ecfe46cee351b283a244d05656e794d3cf1 Mon Sep 17 00:00:00 2001 From: "Douglas Cerna (Soy Douglas)" Date: Thu, 9 May 2024 04:30:58 +0200 Subject: [PATCH 16/32] Upgrade Python requirements --- requirements-dev.txt | 31 +++++++++++++++---------------- requirements.txt | 12 ++++++------ 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9395b7ed8a..5b8c0475cd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -47,11 +47,11 @@ click==8.1.7 # via pip-tools colorama==0.4.6 # via tox -coverage[toml]==7.4.4 +coverage[toml]==7.5.1 # via # -r requirements-dev.in # pytest-cov -cryptography==42.0.5 +cryptography==42.0.7 # via # -r requirements.txt # josepy @@ -59,7 +59,7 @@ cryptography==42.0.5 # pyopenssl distlib==0.3.8 # via virtualenv -django==4.2.11 +django==4.2.13 # via # -r requirements.txt # django-auth-ldap @@ -80,13 +80,13 @@ django-prometheus==2.3.1 # via -r requirements.txt django-shibboleth-remoteuser @ git+https://github.com/artefactual-labs/django-shibboleth-remoteuser.git@f08a7864d6130416c352981ccf318fff0fd5be58 # via -r requirements.txt -django-tastypie==0.14.6 +django-tastypie==0.14.7 # via -r requirements.txt elasticsearch==6.8.2 # via -r requirements.txt -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via pytest -filelock==3.13.4 +filelock==3.14.0 # via # tox # virtualenv @@ -123,7 +123,7 @@ josepy==1.14.0 # via # -r requirements.txt # mozilla-django-oidc -jsonschema==4.21.1 +jsonschema==4.22.0 # via -r requirements.txt jsonschema-specifications==2023.12.1 # via @@ -163,11 +163,11 @@ packaging==24.0 # tox pip-tools==7.4.1 # via -r requirements-dev.in -platformdirs==4.2.0 +platformdirs==4.2.1 # via # tox # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via # pytest # tox @@ -194,11 +194,11 @@ pyopenssl==24.1.0 # josepy pyproject-api==1.6.1 # via tox -pyproject-hooks==1.0.0 +pyproject-hooks==1.1.0 # via # build # pip-tools -pytest==8.1.1 +pytest==8.2.0 # via # -r requirements-dev.in # pytest-cov @@ -230,7 +230,7 @@ python-mimeparse==1.6.0 # via # -r requirements.txt # django-tastypie -referencing==0.34.0 +referencing==0.35.1 # via # -r requirements.txt # jsonschema @@ -243,7 +243,7 @@ requests==2.31.0 # mozilla-django-oidc # opf-fido # python-cas -rpds-py==0.18.0 +rpds-py==0.18.1 # via # -r requirements.txt # jsonschema @@ -264,10 +264,9 @@ tomli==2.0.1 # coverage # pip-tools # pyproject-api - # pyproject-hooks # pytest # tox -tox==4.14.2 +tox==4.15.0 # via -r requirements-dev.in typing-extensions==4.11.0 # via @@ -281,7 +280,7 @@ urllib3==2.2.1 # amclient # elasticsearch # requests -virtualenv==20.25.2 +virtualenv==20.26.1 # via tox wheel==0.43.0 # via pip-tools diff --git a/requirements.txt b/requirements.txt index 84c2eb3455..e61051c241 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,12 +28,12 @@ charset-normalizer==3.3.2 # via requests clamd==1.0.2 # via -r requirements.in -cryptography==42.0.5 +cryptography==42.0.7 # via # josepy # mozilla-django-oidc # pyopenssl -django==4.2.11 +django==4.2.13 # via # -r requirements.in # django-auth-ldap @@ -54,7 +54,7 @@ django-prometheus==2.3.1 # via -r requirements.in django-shibboleth-remoteuser @ git+https://github.com/artefactual-labs/django-shibboleth-remoteuser.git@f08a7864d6130416c352981ccf318fff0fd5be58 # via -r requirements.in -django-tastypie==0.14.6 +django-tastypie==0.14.7 # via -r requirements.in elasticsearch==6.8.2 # via -r requirements.in @@ -76,7 +76,7 @@ inotify-simple==1.3.5 # via -r requirements.in josepy==1.14.0 # via mozilla-django-oidc -jsonschema==4.21.1 +jsonschema==4.22.0 # via -r requirements.in jsonschema-specifications==2023.12.1 # via jsonschema @@ -126,7 +126,7 @@ python-ldap==3.4.4 # django-auth-ldap python-mimeparse==1.6.0 # via django-tastypie -referencing==0.34.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications @@ -138,7 +138,7 @@ requests==2.31.0 # mozilla-django-oidc # opf-fido # python-cas -rpds-py==0.18.0 +rpds-py==0.18.1 # via # jsonschema # referencing From 44ae43e75aefd997a1695b7e224e410456c7b4a3 Mon Sep 17 00:00:00 2001 From: "Douglas Cerna (Soy Douglas)" Date: Thu, 9 May 2024 05:41:20 +0200 Subject: [PATCH 17/32] Update submodules --- hack/submodules/archivematica-acceptance-tests | 2 +- hack/submodules/archivematica-storage-service | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/submodules/archivematica-acceptance-tests b/hack/submodules/archivematica-acceptance-tests index 4bd7f46827..ea4155d5f6 160000 --- a/hack/submodules/archivematica-acceptance-tests +++ b/hack/submodules/archivematica-acceptance-tests @@ -1 +1 @@ -Subproject commit 4bd7f46827f2cfe2e7bc69e899f1c8bf4a01c501 +Subproject commit ea4155d5f6fa77e612422955878d102e96db95b2 diff --git a/hack/submodules/archivematica-storage-service b/hack/submodules/archivematica-storage-service index 19896224a9..f4ff4f3ac4 160000 --- a/hack/submodules/archivematica-storage-service +++ b/hack/submodules/archivematica-storage-service @@ -1 +1 @@ -Subproject commit 19896224a9e946e9760c8934825ceca09c992a35 +Subproject commit f4ff4f3ac4bfeb4ba1a2ed895b76189a7f57c408 From b08560e657ebb8b9210d5757179718a9730fd6c2 Mon Sep 17 00:00:00 2001 From: "Douglas Cerna (Soy Douglas)" Date: Wed, 15 May 2024 23:46:12 +0200 Subject: [PATCH 18/32] Update archivematica-storage-service submodule --- hack/submodules/archivematica-storage-service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/submodules/archivematica-storage-service b/hack/submodules/archivematica-storage-service index f4ff4f3ac4..262800a2ff 160000 --- a/hack/submodules/archivematica-storage-service +++ b/hack/submodules/archivematica-storage-service @@ -1 +1 @@ -Subproject commit f4ff4f3ac4bfeb4ba1a2ed895b76189a7f57c408 +Subproject commit 262800a2ff546966e65ade4d51ea9bfdfef23bb5 From fb498dcfc4f5157aa88db43762ef37672c4cbb63 Mon Sep 17 00:00:00 2001 From: "Douglas Cerna (Soy Douglas)" Date: Fri, 17 May 2024 14:14:21 +0200 Subject: [PATCH 19/32] Bump version to 1.17.0 --- src/archivematicaCommon/lib/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/archivematicaCommon/lib/version.py b/src/archivematicaCommon/lib/version.py index 8eb56a2a63..86d32957f3 100644 --- a/src/archivematicaCommon/lib/version.py +++ b/src/archivematicaCommon/lib/version.py @@ -1,4 +1,4 @@ -ARCHIVEMATICA_VERSION = (1, 16, 0) +ARCHIVEMATICA_VERSION = (1, 17, 0) def get_version(): From d5344ce0cad206c4dcc8c9444154d5d4b0d0c219 Mon Sep 17 00:00:00 2001 From: "Douglas Cerna (Soy Douglas)" Date: Tue, 21 May 2024 01:26:31 +0200 Subject: [PATCH 20/32] Upgrade Python requirements --- requirements-dev.txt | 14 +++++++------- requirements.txt | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 5b8c0475cd..b5be786399 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -131,7 +131,7 @@ jsonschema-specifications==2023.12.1 # jsonschema lazy-paged-sequence==0.3 # via -r requirements.txt -lxml==5.2.1 +lxml==5.2.2 # via # -r requirements.txt # ammcpc @@ -163,7 +163,7 @@ packaging==24.0 # tox pip-tools==7.4.1 # via -r requirements-dev.in -platformdirs==4.2.1 +platformdirs==4.2.2 # via # tox # virtualenv @@ -198,7 +198,7 @@ pyproject-hooks==1.1.0 # via # build # pip-tools -pytest==8.2.0 +pytest==8.2.1 # via # -r requirements-dev.in # pytest-cov @@ -235,7 +235,7 @@ referencing==0.35.1 # -r requirements.txt # jsonschema # jsonschema-specifications -requests==2.31.0 +requests==2.32.1 # via # -r requirements.txt # agentarchives @@ -280,13 +280,13 @@ urllib3==2.2.1 # amclient # elasticsearch # requests -virtualenv==20.26.1 +virtualenv==20.26.2 # via tox wheel==0.43.0 # via pip-tools whitenoise==6.6.0 # via -r requirements.txt -zipp==3.18.1 +zipp==3.18.2 # via # -r requirements.txt # importlib-metadata @@ -295,7 +295,7 @@ zope-event==5.0 # via # -r requirements.txt # gevent -zope-interface==6.3 +zope-interface==6.4 # via # -r requirements.txt # gevent diff --git a/requirements.txt b/requirements.txt index e61051c241..0b6aa5f0ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -82,7 +82,7 @@ jsonschema-specifications==2023.12.1 # via jsonschema lazy-paged-sequence==0.3 # via -r requirements.in -lxml==5.2.1 +lxml==5.2.2 # via # -r requirements.in # ammcpc @@ -130,7 +130,7 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.31.0 +requests==2.32.1 # via # -r requirements.in # agentarchives @@ -160,13 +160,13 @@ urllib3==2.2.1 # requests whitenoise==6.6.0 # via -r requirements.in -zipp==3.18.1 +zipp==3.18.2 # via # importlib-metadata # importlib-resources zope-event==5.0 # via gevent -zope-interface==6.3 +zope-interface==6.4 # via gevent # The following packages are considered to be unsafe in a requirements file: From c73d1b336b9f4303600ea428cf2a1d6f482e73b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Garc=C3=ADa=20Crespo?= Date: Sat, 20 Apr 2024 16:31:44 +0000 Subject: [PATCH 21/32] Remove dead code in MCPServer --- src/MCPServer/lib/server/jobs/chain.py | 38 +++++-------------- src/MCPServer/lib/server/mcp.py | 11 +++--- src/MCPServer/lib/server/processing_config.py | 22 ----------- src/MCPServer/lib/server/queues.py | 4 -- tests/MCPServer/test_mcp.py | 4 +- 5 files changed, 17 insertions(+), 62 deletions(-) diff --git a/src/MCPServer/lib/server/jobs/chain.py b/src/MCPServer/lib/server/jobs/chain.py index 4ae760444f..81a927d8b7 100644 --- a/src/MCPServer/lib/server/jobs/chain.py +++ b/src/MCPServer/lib/server/jobs/chain.py @@ -10,8 +10,6 @@ """ import logging -from django.utils import timezone -from server.jobs.client import ClientScriptJob from server.jobs.client import DirectoryClientScriptJob from server.jobs.client import FilesClientScriptJob from server.jobs.client import OutputClientScriptJob @@ -73,7 +71,6 @@ def __init__(self, package, chain, workflow, starting_link=None): self.package = package self.chain = chain self.workflow = workflow - self.started_on = timezone.now() self.initial_link = starting_link or self.chain.link self.current_link = None @@ -106,35 +103,20 @@ def __next__(self): except KeyError: next_link = None - if next_link: - self.current_link = next_link - job_class = get_job_class_for_link(self.current_link) - self.current_job = job_class(self, self.current_link, self.package) - return self.current_job - else: + # Chain completion. + if not next_link: self.current_link = None self.current_job = None - self.chain_completed() + logger.debug( + "Done with chain %s for package %s", self.id, self.package.uuid + ) raise StopIteration + self.current_link = next_link + job_class = get_job_class_for_link(self.current_link) + self.current_job = job_class(self, self.current_link, self.package) + return self.current_job + @property def id(self): return self.chain.id - - def job_completed(self): - logger.debug( - "%s %s done with exit code %s", - self.current_job.__class__.__name__, - self.current_job.uuid, - self.current_job.exit_code, - ) - if isinstance(self.current_job, ClientScriptJob): - self.current_job.update_status_from_exit_code() - else: - self.current_job.mark_complete() - - def chain_completed(self): - """Log chain completion""" - logger.debug( - "Done with chain %s for package %s", self.chain.id, self.package.uuid - ) diff --git a/src/MCPServer/lib/server/mcp.py b/src/MCPServer/lib/server/mcp.py index bef08b0519..d573f0c939 100644 --- a/src/MCPServer/lib/server/mcp.py +++ b/src/MCPServer/lib/server/mcp.py @@ -51,18 +51,17 @@ def watched_dir_handler(package_queue, path, watched_dir): logger.debug("Starting chain for %s", path) package = None - package_type = watched_dir["unit_type"] - watched_dir_path = watched_dir["path"] + package_type = watched_dir.unit_type is_dir = os.path.isdir(path) if package_type == "SIP" and is_dir: - package = SIP.get_or_create_from_db_by_path(path, watched_dir_path) + package = SIP.get_or_create_from_db_by_path(path, watched_dir.path) elif package_type == "DIP" and is_dir: - package = DIP.get_or_create_from_db_by_path(path, watched_dir_path) + package = DIP.get_or_create_from_db_by_path(path, watched_dir.path) elif package_type == "Transfer" and is_dir: - package = Transfer.get_or_create_from_db_by_path(path, watched_dir_path) + package = Transfer.get_or_create_from_db_by_path(path, watched_dir.path) elif package_type == "Transfer" and not is_dir: - package = Transfer.get_or_create_from_db_by_path(path, watched_dir_path) + package = Transfer.get_or_create_from_db_by_path(path, watched_dir.path) else: raise ValueError(f"Unexpected unit type given for file {path}") diff --git a/src/MCPServer/lib/server/processing_config.py b/src/MCPServer/lib/server/processing_config.py index 73dda1282c..5ad8137849 100644 --- a/src/MCPServer/lib/server/processing_config.py +++ b/src/MCPServer/lib/server/processing_config.py @@ -7,7 +7,6 @@ import abc import logging import os -import shutil import storageService as storage_service from django.conf import settings @@ -300,27 +299,6 @@ def get_processing_fields(workflow, lang="en"): return [field.to_dict(workflow, lang) for field in processing_fields] -def copy_processing_config(processing_config, destination_path): - if processing_config is None: - return - - src = os.path.join( - settings.SHARED_DIRECTORY, - "sharedMicroServiceTasksConfigs/processingMCPConfigs", - "%sProcessingMCP.xml" % processing_config, - ) - dest = os.path.join(destination_path, "processingMCP.xml") - try: - shutil.copyfile(src, dest) - except OSError: - logger.warning( - "Processing configuration could not be copied: (from=%s to=%s)", - src, - dest, - exc_info=True, - ) - - def load_processing_xml(package_path): processing_file_path = os.path.join(package_path, settings.PROCESSING_XML_FILE) diff --git a/src/MCPServer/lib/server/queues.py b/src/MCPServer/lib/server/queues.py index c6d37744ff..9afcf92264 100644 --- a/src/MCPServer/lib/server/queues.py +++ b/src/MCPServer/lib/server/queues.py @@ -186,10 +186,6 @@ def process_one_job(self, timeout=None): return result - def stop(self): - """Trigger queue shutdown.""" - self.shutdown_event.set() - def _package_completed_callback(self, package, link_id, future): """Marks the package as inactive and schedules a new package. diff --git a/tests/MCPServer/test_mcp.py b/tests/MCPServer/test_mcp.py index 49951db65c..a7ce22a791 100644 --- a/tests/MCPServer/test_mcp.py +++ b/tests/MCPServer/test_mcp.py @@ -24,7 +24,7 @@ def test_watched_dir_handler_creates_transfer_if_it_does_not_exist(mocker, tmpdi job_chain_mock.return_value = iter(["some_chain_link"]) mocker.patch("server.mcp.JobChain", job_chain_mock) package_queue = mocker.Mock() - watched_dir = mocker.MagicMock(**{"__getitem__.return_value": "Transfer"}) + watched_dir = mocker.MagicMock(unit_type="Transfer") # Mock a known UUID for the new transfer. transfer_uuid = uuid.uuid4() @@ -56,7 +56,7 @@ def test_watched_dir_handler_creates_transfer_for_file(mocker, tmpdir): job_chain_mock.return_value = iter(["some_chain_link"]) mocker.patch("server.mcp.JobChain", job_chain_mock) package_queue = mocker.Mock() - watched_dir = mocker.MagicMock(**{"__getitem__.return_value": "Transfer"}) + watched_dir = mocker.MagicMock(unit_type="Transfer") # Mock a known UUID for the new transfer. transfer_uuid = uuid.uuid4() From 1ae0dc7f696e6baa0321aff9e188d1e21f8da5c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Garc=C3=ADa=20Crespo?= Date: Mon, 22 Apr 2024 08:38:17 +0000 Subject: [PATCH 22/32] Remove unused file filter --- src/MCPServer/lib/assets/workflow-schema-v1.json | 6 ------ src/MCPServer/lib/server/jobs/client.py | 6 ------ src/MCPServer/lib/server/packages.py | 13 ++----------- tests/MCPServer/test_package.py | 10 +++++----- 4 files changed, 7 insertions(+), 28 deletions(-) diff --git a/src/MCPServer/lib/assets/workflow-schema-v1.json b/src/MCPServer/lib/assets/workflow-schema-v1.json index 62a44cdbd9..c5da26d29a 100644 --- a/src/MCPServer/lib/assets/workflow-schema-v1.json +++ b/src/MCPServer/lib/assets/workflow-schema-v1.json @@ -292,9 +292,6 @@ "filter_file_end": { "type": "string" }, - "filter_file_start": { - "type": "string" - }, "filter_subdir": { "type": "string" }, @@ -332,9 +329,6 @@ "filter_file_end": { "type": "string" }, - "filter_file_start": { - "type": "string" - }, "filter_subdir": { "type": "string" }, diff --git a/src/MCPServer/lib/server/jobs/client.py b/src/MCPServer/lib/server/jobs/client.py index a82ad3d0e8..8d785bb3d3 100644 --- a/src/MCPServer/lib/server/jobs/client.py +++ b/src/MCPServer/lib/server/jobs/client.py @@ -167,11 +167,6 @@ class FilesClientScriptJob(ClientScriptJob): A job with many tasks, one per file. """ - @property - def filter_file_start(self): - """Returns path prefix to filter files on, as defined in the workflow.""" - return self.link.config.get("filter_file_start", "") - @property def filter_file_end(self): """Returns path suffix to filter files on, as defined in the workflow.""" @@ -215,7 +210,6 @@ def filter_subdir(self): def submit_tasks(self): """Iterate through all matching files for the package, and submit tasks.""" for file_replacements in self.package.files( - filter_filename_start=self.filter_file_start, filter_filename_end=self.filter_file_end, filter_subdir=self.filter_subdir, ): diff --git a/src/MCPServer/lib/server/packages.py b/src/MCPServer/lib/server/packages.py index ccde7d64d8..d33d15b99c 100644 --- a/src/MCPServer/lib/server/packages.py +++ b/src/MCPServer/lib/server/packages.py @@ -646,18 +646,13 @@ def get_replacement_mapping(self, filter_subdir_path=None): return mapping - def files( - self, filter_filename_start=None, filter_filename_end=None, filter_subdir=None - ): + def files(self, filter_filename_end=None, filter_subdir=None): """Generator that yields all files associated with the package or that should be associated with a package. """ with auto_close_old_connections(): queryset = self.base_queryset - if filter_filename_start: - # TODO: regex filter - raise NotImplementedError("filter_filename_start is not implemented") if filter_filename_end: queryset = queryset.filter( currentlocation__endswith=filter_filename_end @@ -683,12 +678,8 @@ def files( for basedir, _, files in os.walk(start_path): for file_name in files: - if ( - filter_filename_start - and not file_name.startswith(filter_filename_start) - ) or ( + if filter_filename_end and not file_name.endswith( filter_filename_end - and not file_name.endswith(filter_filename_end) ): continue file_path = os.path.join(basedir, file_name) diff --git a/tests/MCPServer/test_package.py b/tests/MCPServer/test_package.py index 068db7b078..fda8289791 100644 --- a/tests/MCPServer/test_package.py +++ b/tests/MCPServer/test_package.py @@ -185,7 +185,7 @@ def test_reload_file_list(tmp_path): # One file will be returned from the database with a UUID, another from # the filesystem without a UUID. - for _file_count, file_info in enumerate(transfer.files(None, None, "/objects"), 1): + for _file_count, file_info in enumerate(transfer.files(None, "/objects"), 1): assert "%fileUUID%" in file_info assert "%fileGrpUse%" in file_info assert "%relativeLocation%" in file_info @@ -215,7 +215,7 @@ def test_reload_file_list(tmp_path): sub_dir.mkdir() new_file = sub_dir / "another_new_file.txt" new_file.touch() - for _file_count, file_info in enumerate(transfer.files(None, None, "/objects"), 1): + for _file_count, file_info in enumerate(transfer.files(None, "/objects"), 1): if file_info.get("%fileUUID%") != "None": continue file_path = Path( @@ -237,7 +237,7 @@ def test_reload_file_list(tmp_path): # Now the database is updated, we will still have the same file count, but # all objects will be returned from the database and we will have uuids. - for _file_count, file_info in enumerate(transfer.files(None, None, "/objects"), 1): + for _file_count, file_info in enumerate(transfer.files(None, "/objects"), 1): if file_info.get("%fileUUID%") == "None": raise AssertionError( "Non-database entries returned from package.files(): {}".format( @@ -253,7 +253,7 @@ def test_reload_file_list(tmp_path): new_file = objects_path / file_ new_file.touch() new_count = 0 - for _file_count, file_info in enumerate(transfer.files(None, None, "/objects"), 1): + for _file_count, file_info in enumerate(transfer.files(None, "/objects"), 1): if file_info.get("%fileUUID%") == "None": new_count += 1 assert new_count == 5 @@ -289,7 +289,7 @@ def test_package_files_with_non_ascii_names(tmp_path): models.File.objects.create(**kwargs) # Assert only one file is returned - result = list(transfer.files(None, None, "/objects")) + result = list(transfer.files(None, "/objects")) assert len(result) == 1 # And it is the file we just created From 6927afe2347a20bf2bf09c51fb41d4df5b8652f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Garc=C3=ADa=20Crespo?= Date: Tue, 23 Apr 2024 02:38:08 +0000 Subject: [PATCH 23/32] Use json.loads in OutputClientScriptJob --- src/MCPServer/lib/server/jobs/client.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/MCPServer/lib/server/jobs/client.py b/src/MCPServer/lib/server/jobs/client.py index 8d785bb3d3..02fb9d2a08 100644 --- a/src/MCPServer/lib/server/jobs/client.py +++ b/src/MCPServer/lib/server/jobs/client.py @@ -3,6 +3,7 @@ """ import abc import ast +import json import logging from dbconns import auto_close_old_connections @@ -235,13 +236,14 @@ def submit_tasks(self): class OutputClientScriptJob(ClientScriptJob): - """Retrieves output (e.g. a set of choices) from mcp client, for use in a decision. + """Retrieves output (e.g. a set of choices) from MCPClient, for use in a decision. Output is returned via stderr, and parsed via eval. It is stored for access by the next job on the `generated_choices` attribute of the job chain, which is used for only this purpose. - TODO: Remove from workflow if possible. + TODO: Remove from workflow if possible - this is only used by + getAipStorageLocations_v0.0 (twice). """ # We always need output for this type of job @@ -251,8 +253,8 @@ def task_completed_callback(self, task): logger.debug("stdout emitted by client: %s", task.stdout) try: - choices = ast.literal_eval(task.stdout) - except (ValueError, SyntaxError): + choices = json.loads(task.stdout) + except (TypeError, ValueError): logger.exception("Unable to parse output %s", task.stdout) choices = {} From 54f9de70674fa085bc2dbffd8f7d4dfce7c718ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Garc=C3=ADa=20Crespo?= Date: Wed, 10 Apr 2024 10:18:27 +0000 Subject: [PATCH 24/32] Introduce JSONDataEncoder for Gearman --- .../lib/gearman_encoder.py | 71 +++++++++++++++++++ .../test_gearman_encoder.py | 29 ++++++++ 2 files changed, 100 insertions(+) create mode 100644 src/archivematicaCommon/lib/gearman_encoder.py create mode 100644 tests/archivematicaCommon/test_gearman_encoder.py diff --git a/src/archivematicaCommon/lib/gearman_encoder.py b/src/archivematicaCommon/lib/gearman_encoder.py new file mode 100644 index 0000000000..eff0d8a621 --- /dev/null +++ b/src/archivematicaCommon/lib/gearman_encoder.py @@ -0,0 +1,71 @@ +import json +import uuid +from datetime import datetime + +import gearman +from django.utils.timezone import make_aware + + +class JSONDecoder(json.JSONDecoder): + def __init__(self, *args, **kwargs): + kwargs["object_hook"] = self.object_hook + super().__init__(*args, **kwargs) + + def object_hook(self, d: dict): + object_type = d.get("__type__") + if object_type == "datetime": + return make_aware( + datetime( + d["year"], + d["month"], + d["day"], + d["hour"], + d["minute"], + d["second"], + d["microsecond"], + ) + ) + elif object_type == "UUID": + return uuid.UUID(hex=d["value"]) + return d + + +class JSONEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, datetime): + return { + "__type__": "datetime", + "year": obj.year, + "month": obj.month, + "day": obj.day, + "hour": obj.hour, + "minute": obj.minute, + "second": obj.second, + "microsecond": obj.microsecond, + } + elif isinstance(obj, uuid.UUID): + return { + "__type__": "UUID", + "value": obj.hex, + } + else: + return super().default(obj) + + +class JSONDataEncoder(gearman.DataEncoder): + """Custom data encoder class for the `gearman` library (JSON). + + This class enables serialization and deserialization of data using JSON + format, supporting UUID and datetime data types. + """ + + @classmethod + def encode(cls, encodable_object): + # Object of type bytes is not JSON serializable. + if isinstance(encodable_object, bytes): + encodable_object = encodable_object.decode("utf-8") + return json.dumps(encodable_object, cls=JSONEncoder, separators=(",", ":")) + + @classmethod + def decode(cls, decodable_string): + return json.loads(decodable_string, cls=JSONDecoder) diff --git a/tests/archivematicaCommon/test_gearman_encoder.py b/tests/archivematicaCommon/test_gearman_encoder.py new file mode 100644 index 0000000000..7960a08d74 --- /dev/null +++ b/tests/archivematicaCommon/test_gearman_encoder.py @@ -0,0 +1,29 @@ +from datetime import datetime +from uuid import UUID + +from django.utils.timezone import make_aware +from gearman_encoder import JSONDataEncoder + + +def test_json_data_encoder(): + encoder = JSONDataEncoder + object_py = { + "id": UUID("ccf0368e-ec97-4375-a408-5c4d95c0c671"), + "data": [1, 2, 3], + "when": make_aware(datetime(2019, 6, 18, 1, 1, 1, 123)), + } + object_js = ( + "{" + + '"id":{"__type__":"UUID","value":"ccf0368eec974375a4085c4d95c0c671"},' + + '"data":[1,2,3],' + + '"when":{"__type__":"datetime","year":2019,"month":6,"day":18,"hour":1,"minute":1,"second":1,"microsecond":123}}' + ) + + assert encoder.encode(object_py) == object_js + assert encoder.decode(object_js) == object_py + + +def test_json_data_encoder_with_bytes(): + encoder = JSONDataEncoder + + assert encoder.encode(b"bytes") == '"bytes"' From ab6302757d1a46ac317c8bf616a9ed72f6e06562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Garc=C3=ADa=20Crespo?= Date: Wed, 10 Apr 2024 10:18:55 +0000 Subject: [PATCH 25/32] Use JSONDataEncoder in MCP task backend --- src/MCPClient/lib/client/gearman.py | 14 ++------------ .../lib/server/tasks/backends/gearman_backend.py | 14 +++++++------- tests/MCPServer/test_gearman.py | 10 ++++------ 3 files changed, 13 insertions(+), 25 deletions(-) diff --git a/src/MCPClient/lib/client/gearman.py b/src/MCPClient/lib/client/gearman.py index 1774448e49..f45ed9bbe1 100644 --- a/src/MCPClient/lib/client/gearman.py +++ b/src/MCPClient/lib/client/gearman.py @@ -1,7 +1,6 @@ import functools import logging import multiprocessing -import pickle import gearman from client import metrics @@ -11,23 +10,14 @@ from client.utils import replace_task_arguments from client.worker import run_task from django.conf import settings +from gearman_encoder import JSONDataEncoder logger = logging.getLogger("archivematica.mcp.client.gearman") -class PickleDataEncoder(gearman.DataEncoder): - @classmethod - def encode(cls, encodable_object): - return pickle.dumps(encodable_object, protocol=0) - - @classmethod - def decode(cls, decodable_bytes): - return pickle.loads(decodable_bytes) - - class MCPGearmanWorker(gearman.GearmanWorker): - data_encoder = PickleDataEncoder + data_encoder = JSONDataEncoder def __init__( self, hosts, client_scripts, shutdown_event=None, max_jobs_to_process=None diff --git a/src/MCPServer/lib/server/tasks/backends/gearman_backend.py b/src/MCPServer/lib/server/tasks/backends/gearman_backend.py index cde9083ee5..f297272eb0 100644 --- a/src/MCPServer/lib/server/tasks/backends/gearman_backend.py +++ b/src/MCPServer/lib/server/tasks/backends/gearman_backend.py @@ -3,7 +3,6 @@ and returns results. """ import logging -import pickle import uuid from django.conf import settings @@ -11,6 +10,7 @@ from gearman.constants import JOB_COMPLETE from gearman.constants import JOB_FAILED from gearman.constants import JOB_UNKNOWN +from gearman_encoder import JSONDataEncoder from server import metrics from server.tasks.backends.base import TaskBackend from server.tasks.task import Task @@ -25,6 +25,8 @@ class MCPGearmanClient(GearmanClient): more efficiently. """ + data_encoder = JSONDataEncoder + def wait_until_any_job_completed(self, job_requests, poll_timeout=None): """ Go into a select loop until any of our jobs have completed or failed. @@ -55,8 +57,8 @@ def continue_while_no_job_completed(any_activity): class GearmanTaskBackend(TaskBackend): """Submits tasks to MCPClient via Gearman. - Tasks are batched into BATCH_SIZE groups (default 128), pickled and sent to - MCPClient. This adds some complexity but saves a lot of overhead. + Tasks are batched into BATCH_SIZE groups (default 128), serialized and sent + to MCPClient. This adds some complexity but saves a lot of overhead. """ # The number of files we'll pack into each MCP Client job. Chosen somewhat @@ -204,11 +206,9 @@ def submit(self, client, job, max_retries=0): task_uuid = str(task.uuid) data["tasks"][task_uuid] = self.serialize_task(task) - pickled_data = pickle.dumps(data, protocol=0) - self.pending = client.submit_job( task=job.name.encode(), - data=pickled_data, + data=data, unique=str(self.uuid).encode(), wait_until_complete=False, background=False, @@ -222,7 +222,7 @@ def result(self): elif not self.pending.result: raise ValueError("Unexpected empty result from Gearman job") - job_result = pickle.loads(self.pending.result) + job_result = self.pending.result try: return job_result["task_results"] diff --git a/tests/MCPServer/test_gearman.py b/tests/MCPServer/test_gearman.py index a43204cc61..63195b75ed 100644 --- a/tests/MCPServer/test_gearman.py +++ b/tests/MCPServer/test_gearman.py @@ -1,5 +1,4 @@ import math -import pickle import uuid import gearman @@ -45,7 +44,7 @@ def format_gearman_request(tasks): "wants_output": task.wants_output, } - return pickle.dumps(request, protocol=0) + return request def format_gearman_response(task_results): @@ -55,7 +54,7 @@ def format_gearman_response(task_results): task_uuid = str(task_uuid) response["task_results"][task_uuid] = task_data - return pickle.dumps(response, protocol=0) + return response def test_gearman_task_submission(simple_job, simple_task, mocker): @@ -72,8 +71,7 @@ def test_gearman_task_submission(simple_job, simple_task, mocker): submit_job_kwargs = mock_client.return_value.submit_job.call_args[1] assert submit_job_kwargs["task"] == simple_job.name.encode() - # Comparing pickled strings is fragile, so compare the python version - assert pickle.loads(submit_job_kwargs["data"]) == pickle.loads(task_data) + assert submit_job_kwargs["data"] == task_data try: uuid.UUID(submit_job_kwargs["unique"].decode()) except ValueError: @@ -146,7 +144,7 @@ def test_gearman_task_result_error(simple_job, simple_task, mocker): def mock_jobs_completed(*args): job_request.state = gearman.JOB_FAILED - job_request.exception = pickle.dumps(Exception("Error!"), protocol=0) + job_request.exception = Exception("Error!") return [job_request] From 95153d7f4c5e30b162a7918265035d14059d02b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Garc=C3=ADa=20Crespo?= Date: Thu, 11 Apr 2024 05:20:08 +0000 Subject: [PATCH 26/32] Use JSONDataEncoder in RPC server --- src/MCPServer/lib/server/rpc_server.py | 11 +++++---- src/dashboard/src/contrib/mcp/client.py | 25 ++++++++++++-------- tests/dashboard/components/mcp/test_views.py | 8 ++----- tests/dashboard/test_api.py | 22 +++++++++++------ 4 files changed, 38 insertions(+), 28 deletions(-) diff --git a/src/MCPServer/lib/server/rpc_server.py b/src/MCPServer/lib/server/rpc_server.py index 785f02cb26..70e0cd4868 100644 --- a/src/MCPServer/lib/server/rpc_server.py +++ b/src/MCPServer/lib/server/rpc_server.py @@ -10,7 +10,6 @@ import configparser import inspect import logging -import pickle import re import time from collections import OrderedDict @@ -22,6 +21,7 @@ from django.conf import settings as django_settings from django.db import connection from gearman import GearmanWorker +from gearman_encoder import JSONDataEncoder from lxml import etree from main.models import Job from main.models import SIP @@ -95,6 +95,8 @@ class RPCServer(GearmanWorker): """ + data_encoder = JSONDataEncoder + # Regular expression to match handler configurations in method docstrings. ability_regex = re.compile( r"(?P.*)(?P\[config\].*)$", re.DOTALL | re.MULTILINE @@ -168,10 +170,9 @@ def _wrap_handler_method(self, name, handler, **opts): def wrap(worker, job): args = [worker, job] if opts["expect_payload"]: - payload = pickle.loads(job.data) - if not isinstance(payload, dict): + if not isinstance(job.data, dict): raise UnexpectedPayloadError("Payload is not a dictionary") - args.append(payload) + args.append(job.data) try: resp = handler(*args) except Exception as err: @@ -186,7 +187,7 @@ def wrap(worker, job): if opts["raise_exc"]: raise # So GearmanWorker knows that it failed. resp = {"error": True, "handler": name, "message": str(err)} - return pickle.dumps(resp, protocol=0) + return resp return wrap diff --git a/src/dashboard/src/contrib/mcp/client.py b/src/dashboard/src/contrib/mcp/client.py index be7a68c14b..e199d724b4 100644 --- a/src/dashboard/src/contrib/mcp/client.py +++ b/src/dashboard/src/contrib/mcp/client.py @@ -15,11 +15,11 @@ # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . import logging -import pickle import gearman from django.conf import settings from django.utils.translation import get_language +from gearman_encoder import JSONDataEncoder from main.models import Job @@ -61,7 +61,7 @@ class TimeoutError(RPCGearmanClientError): """Deadline exceeded. >> response = client.submit_job( - "doSomething", cPickle.dumps(data, protocol=0), + "doSomething", data, background=False, wait_until_complete=True, poll_timeout=INFLIGHT_POLL_TIMEOUT) if response.state == gearman.JOB_CREATED: @@ -84,6 +84,10 @@ def __init__(self, message=None): super().__init__(message) +class GearmanClient(gearman.GearmanClient): + data_encoder = JSONDataEncoder + + INFLIGHT_POLL_TIMEOUT = 30.0 @@ -107,10 +111,10 @@ def _rpc_sync_call(self, ability, data=None, timeout=INFLIGHT_POLL_TIMEOUT): data = b"" elif "user_id" not in data: data["user_id"] = self.user.id - client = gearman.GearmanClient([self.server]) + client = GearmanClient([self.server]) response = client.submit_job( ability.encode(), - pickle.dumps(data, protocol=0), + data, background=False, wait_until_complete=True, poll_timeout=timeout, @@ -120,20 +124,20 @@ def _rpc_sync_call(self, ability, data=None, timeout=INFLIGHT_POLL_TIMEOUT): raise TimeoutError(timeout) elif response.state != gearman.JOB_COMPLETE: raise RPCError(f"{ability} failed (check the logs)") - payload = pickle.loads(response.result) + payload = response.result if isinstance(payload, dict) and payload.get("error", False): raise RPCServerError(payload) return payload def execute(self, uuid, choice): - gm_client = gearman.GearmanClient([self.server]) + gm_client = GearmanClient([self.server]) data = {} data["jobUUID"] = uuid data["chain"] = choice # Since `execute` is not using `_rpc_sync_call` yet, the user ID needs # to be added manually here. data["user_id"] = self.user.id - gm_client.submit_job(b"approveJob", pickle.dumps(data, protocol=0)) + gm_client.submit_job(b"approveJob", data) gm_client.shutdown() return @@ -153,12 +157,13 @@ def execute_unit(self, unit_id, choice, mscl_id=None): self.execute(item.pk, choice) def list(self): - gm_client = gearman.GearmanClient([self.server]) + gm_client = GearmanClient([self.server]) completed_job_request = gm_client.submit_job( - b"getJobsAwaitingApproval", pickle.dumps({}, protocol=0) + b"getJobsAwaitingApproval", + {}, ) if completed_job_request.state == gearman.JOB_COMPLETE: - return pickle.loads(completed_job_request.result) + return completed_job_request.result elif completed_job_request.state == gearman.JOB_FAILED: raise RPCError("getJobsAwaitingApproval failed (check MCPServer logs)") diff --git a/tests/dashboard/components/mcp/test_views.py b/tests/dashboard/components/mcp/test_views.py index 65e1558624..fd37949cab 100644 --- a/tests/dashboard/components/mcp/test_views.py +++ b/tests/dashboard/components/mcp/test_views.py @@ -1,5 +1,3 @@ -import pickle as pickle - from components.mcp import views from django.urls import reverse from externals import xmltodict @@ -45,14 +43,12 @@ def test_list(mocker, rf, admin_user): "contrib.mcp.client.gearman.JOB_COMPLETE", ) mocker.patch( - "gearman.GearmanClient", + "contrib.mcp.client.GearmanClient", return_value=mocker.Mock( **{ "submit_job.return_value": mocker.Mock( state=job_complete, - result=pickle.dumps( - MCPSERVER_JOBS_AWAITING_APPROVAL_RESULT.encode() - ), + result=MCPSERVER_JOBS_AWAITING_APPROVAL_RESULT, ) } ), diff --git a/tests/dashboard/test_api.py b/tests/dashboard/test_api.py index 74a4ef5d58..7455f63878 100644 --- a/tests/dashboard/test_api.py +++ b/tests/dashboard/test_api.py @@ -910,14 +910,18 @@ def test_start_transfer_api_decodes_paths(mocker, admin_client): def test_reingest_approve(mocker, admin_client): - mocker.patch("contrib.mcp.client.pickle") job_complete = mocker.patch( "contrib.mcp.client.gearman.JOB_COMPLETE", ) mocker.patch( - "gearman.GearmanClient", + "contrib.mcp.client.GearmanClient", return_value=mocker.Mock( - **{"submit_job.return_value": mocker.Mock(state=job_complete)} + **{ + "submit_job.return_value": mocker.Mock( + state=job_complete, + result=None, + ) + } ), ) helpers.set_setting("dashboard_uuid", "test-uuid") @@ -1075,7 +1079,7 @@ def test_approve_transfer_failures(post_data, expected_error, admin_client, mock helpers.set_setting("dashboard_uuid", "test-uuid") # Simulate an unhandled error when calling Gearman. - mocker.patch("contrib.mcp.client.MCPClient", side_effect=Exception()) + mocker.patch("contrib.mcp.client.GearmanClient", side_effect=Exception()) response = admin_client.post(reverse("api:approve_transfer"), post_data) @@ -1090,14 +1094,18 @@ def test_approve_transfer(admin_client, mocker): # Simulate a dashboard <-> Gearman <-> MCPServer interaction. # The MCPServer approveTransferByPath RPC method returns a UUID. transfer_uuid = uuid.uuid4() - mocker.patch("contrib.mcp.client.pickle.loads", side_effect=[transfer_uuid]) job_complete = mocker.patch( "contrib.mcp.client.gearman.JOB_COMPLETE", ) mocker.patch( - "gearman.GearmanClient", + "contrib.mcp.client.GearmanClient", return_value=mocker.Mock( - **{"submit_job.return_value": mocker.Mock(state=job_complete)} + **{ + "submit_job.return_value": mocker.Mock( + state=job_complete, + result=transfer_uuid, + ) + } ), ) From 96e9d3ac138bfe91cfa215e6ef8a5d0092176898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Garc=C3=ADa=20Crespo?= Date: Thu, 11 Apr 2024 08:41:46 +0000 Subject: [PATCH 27/32] Remove use of pickle in Access.target --- .../lib/clientScripts/upload_qubit.py | 19 ++++------ src/dashboard/src/components/ingest/views.py | 8 ++--- .../migrations/0083_unpickle_access_target.py | 35 +++++++++++++++++++ tests/MCPClient/test_upload_qubit.py | 7 ++-- tests/dashboard/test_ingest.py | 17 +++++---- 5 files changed, 54 insertions(+), 32 deletions(-) create mode 100644 src/dashboard/src/main/migrations/0083_unpickle_access_target.py diff --git a/src/MCPClient/lib/clientScripts/upload_qubit.py b/src/MCPClient/lib/clientScripts/upload_qubit.py index ffb675c3d2..2b0715d0a7 100755 --- a/src/MCPClient/lib/clientScripts/upload_qubit.py +++ b/src/MCPClient/lib/clientScripts/upload_qubit.py @@ -18,7 +18,6 @@ import getpass import optparse import os -import pickle import re import subprocess import sys @@ -111,18 +110,12 @@ def start(job, data): # Look for access system ID transfers = models.Transfer.objects.filter(file__sip_id=data.uuid).distinct() if transfers.count() == 1: - # Django converts to text since Access.target is a TextField. - access.target = pickle.dumps( - {"target": transfers[0].access_system_id}, protocol=0 - ).decode() + access.target = transfers[0].access_system_id access.save() - # The target columns contents a serialized Python dictionary - # - target is the permalink string - try: - target = pickle.loads(access.target.encode()) - log("Target: %s" % (target["target"])) - except Exception: + target = access.target + log("Target: %s" % target) + if not target: return error(job, "No target was selected") # Rsync if data.rsync_target option was passed to this script @@ -225,13 +218,13 @@ def start(job, data): deposit_url = "{}/{}sword/deposit/{}".format( data.url, atom_url_prefix, - target["target"], + target, ) # Auth and request! log("About to deposit to: %s" % data.url) access.statuscode = 13 - access.resource = "{}/{}".format(data.url, target["target"]) + access.resource = f"{data.url}/{target}" access.save() auth = requests.auth.HTTPBasicAuth(data.email, data.password) diff --git a/src/dashboard/src/components/ingest/views.py b/src/dashboard/src/components/ingest/views.py index e28a31ee49..4627558c49 100644 --- a/src/dashboard/src/components/ingest/views.py +++ b/src/dashboard/src/components/ingest/views.py @@ -17,7 +17,6 @@ import json import logging import os -import pickle import re import shutil import uuid @@ -323,19 +322,16 @@ def ingest_upload(request, uuid): access = models.Access.objects.get(sipuuid=uuid) except Exception: access = models.Access(sipuuid=uuid) - access.target = pickle.dumps( - {"target": request.POST["target"]}, protocol=0 - ).decode() + access.target = request.POST["target"] access.save() response = {"ready": True} return helpers.json_response(response) elif request.method == "GET": try: access = models.Access.objects.get(sipuuid=uuid) - data = pickle.loads(access.target.encode()) except Exception: raise Http404 - return helpers.json_response(data) + return helpers.json_response({"target": access.target}) return HttpResponseNotAllowed(["GET", "POST"]) diff --git a/src/dashboard/src/main/migrations/0083_unpickle_access_target.py b/src/dashboard/src/main/migrations/0083_unpickle_access_target.py new file mode 100644 index 0000000000..d0342fdc7a --- /dev/null +++ b/src/dashboard/src/main/migrations/0083_unpickle_access_target.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.11 on 2024-04-11 08:19 +"""Unpickle target in the access table. +""" +import pickle + +from django.db import migrations + + +def data_migration_up(apps, _): + """Unpickle Access.target.""" + access = apps.get_model("main", "Access") + for item in access.objects.all(): + if item.target: + try: + item.target = pickle.loads(item.target.encode()).get("target", "") + item.save() + except Exception: + pass + + +def data_migration_down(apps, _): + """Pickle Access.target.""" + access = apps.get_model("main", "Access") + for item in access.objects.all(): + if item.target: + item.target = pickle.dumps({"target": item.target}, protocol=0).decode() + item.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0082_use_uuidfield"), + ] + + operations = [migrations.RunPython(data_migration_up, data_migration_down)] diff --git a/tests/MCPClient/test_upload_qubit.py b/tests/MCPClient/test_upload_qubit.py index b1259c796e..01cf0f3aad 100644 --- a/tests/MCPClient/test_upload_qubit.py +++ b/tests/MCPClient/test_upload_qubit.py @@ -1,5 +1,4 @@ import os -import pickle as pickle import uuid import pytest @@ -43,7 +42,7 @@ def job(sip, tmp_path): def access(db, sip): return models.Access.objects.create( sipuuid=sip.uuid, - target=pickle.dumps({"target": "atom-description-id"}, protocol=0).decode(), + target="atom-description-id", ) @@ -73,7 +72,7 @@ def test_start_synchronously(db, mocker, mcp_job, sip, job, access): assert access.statuscode == 14 assert access.resource == f"{opts.url}/atom-description-id" assert access.status == "Deposited synchronously" - assert pickle.loads(access.target.encode()) == {"target": "atom-description-id"} + assert access.target == "atom-description-id" def test_first_run(db, mocker, mcp_job, job, transfer, sip): @@ -102,4 +101,4 @@ def test_first_run(db, mocker, mcp_job, job, transfer, sip): assert access.statuscode == 14 assert access.resource == f"{opts.url}/atom-description-id" assert access.status == "Deposited synchronously" - assert pickle.loads(access.target.encode()) == {"target": "atom-description-id"} + assert access.target == "atom-description-id" diff --git a/tests/dashboard/test_ingest.py b/tests/dashboard/test_ingest.py index bd5fd28bcd..b572b059dc 100644 --- a/tests/dashboard/test_ingest.py +++ b/tests/dashboard/test_ingest.py @@ -1,7 +1,6 @@ import json import os import pathlib -import pickle import uuid from unittest import mock @@ -91,10 +90,10 @@ def test_add_metadata_files_view(self): def test_ingest_upload_get(self): sip_uuid = "4060ee97-9c3f-4822-afaf-ebdf838284c3" - access_target = {"target": "description-slug"} + access_target = "description-slug" Access.objects.create( sipuuid=sip_uuid, - target=pickle.dumps(access_target, protocol=0).decode(), + target=access_target, ) response = self.client.get( @@ -102,26 +101,26 @@ def test_ingest_upload_get(self): ) assert response.status_code == 200 - assert json.loads(response.content) == access_target + assert json.loads(response.content)["target"] == access_target def test_ingest_upload_post(self): sip_uuid = "4060ee97-9c3f-4822-afaf-ebdf838284c3" - access_target = {"target": "description-slug"} + access_target = "description-slug" # Check there is no Access object associated with the SIP yet. assert Access.objects.filter(sipuuid=sip_uuid).count() == 0 response = self.client.post( reverse("ingest:ingest_upload", args=[sip_uuid]), - data=access_target, + data={"target": access_target}, ) assert response.status_code == 200 assert json.loads(response.content) == {"ready": True} # An Access object was created for the SIP with the right target. - assert Access.objects.filter(sipuuid=sip_uuid).count() == 1 - access = Access.objects.get(sipuuid=sip_uuid) - assert pickle.loads(access.target.encode()) == access_target + assert ( + Access.objects.filter(sipuuid=sip_uuid, target=access_target).count() == 1 + ) def _assert_file_node_properties_match_record(file_node, record): From 5326b78c7f3781862748b408f9985d7d6f7f29c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Garc=C3=ADa=20Crespo?= Date: Tue, 23 Apr 2024 07:50:35 +0000 Subject: [PATCH 28/32] Use orjson in JSONDataEncoder orjson is significantly faster and comes with native serialization of some types like datetime or UUID. --- requirements-dev.txt | 2 + requirements.in | 1 + requirements.txt | 2 + .../server/tasks/backends/gearman_backend.py | 5 ++ .../lib/gearman_encoder.py | 62 ++----------------- .../test_gearman_encoder.py | 30 +++------ 6 files changed, 23 insertions(+), 79 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index b5be786399..e66d0bd816 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -153,6 +153,8 @@ olefile==0.47 # opf-fido opf-fido @ git+https://github.com/artefactual-labs/fido.git@564ceb8018a8650fe931cf20e6780ee008e60fca # via -r requirements.txt +orjson==3.10.1 + # via -r requirements.txt packaging==24.0 # via # -r requirements.txt diff --git a/requirements.in b/requirements.in index 04f05f4fd0..7b5744aafe 100644 --- a/requirements.in +++ b/requirements.in @@ -21,6 +21,7 @@ jsonschema lazy-paged-sequence lxml metsrw +orjson git+https://github.com/artefactual-labs/fido.git@564ceb8018a8650fe931cf20e6780ee008e60fca#opf-fido prometheus_client python-dateutil diff --git a/requirements.txt b/requirements.txt index 0b6aa5f0ec..92884aa7a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -98,6 +98,8 @@ olefile==0.47 # via opf-fido opf-fido @ git+https://github.com/artefactual-labs/fido.git@564ceb8018a8650fe931cf20e6780ee008e60fca # via -r requirements.in +orjson==3.10.1 + # via -r requirements.in packaging==24.0 # via gunicorn prometheus-client==0.20.0 diff --git a/src/MCPServer/lib/server/tasks/backends/gearman_backend.py b/src/MCPServer/lib/server/tasks/backends/gearman_backend.py index f297272eb0..8ef0b4a181 100644 --- a/src/MCPServer/lib/server/tasks/backends/gearman_backend.py +++ b/src/MCPServer/lib/server/tasks/backends/gearman_backend.py @@ -2,6 +2,7 @@ Gearman task backend. Submits `Task` objects to gearman for processing, and returns results. """ +import datetime import logging import uuid @@ -250,6 +251,10 @@ def update_task_results(self): task.stdout = task_result.get("stdout", "") task.stderr = task_result.get("stderr", "") task.finished_timestamp = task_result.get("finishedTimestamp") + if task.finished_timestamp: + task.finished_timestamp = datetime.datetime.fromisoformat( + task.finished_timestamp + ) task.write_output() task.done = True diff --git a/src/archivematicaCommon/lib/gearman_encoder.py b/src/archivematicaCommon/lib/gearman_encoder.py index eff0d8a621..3cf4b5382c 100644 --- a/src/archivematicaCommon/lib/gearman_encoder.py +++ b/src/archivematicaCommon/lib/gearman_encoder.py @@ -1,71 +1,17 @@ -import json -import uuid -from datetime import datetime - import gearman -from django.utils.timezone import make_aware - - -class JSONDecoder(json.JSONDecoder): - def __init__(self, *args, **kwargs): - kwargs["object_hook"] = self.object_hook - super().__init__(*args, **kwargs) - - def object_hook(self, d: dict): - object_type = d.get("__type__") - if object_type == "datetime": - return make_aware( - datetime( - d["year"], - d["month"], - d["day"], - d["hour"], - d["minute"], - d["second"], - d["microsecond"], - ) - ) - elif object_type == "UUID": - return uuid.UUID(hex=d["value"]) - return d - - -class JSONEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, datetime): - return { - "__type__": "datetime", - "year": obj.year, - "month": obj.month, - "day": obj.day, - "hour": obj.hour, - "minute": obj.minute, - "second": obj.second, - "microsecond": obj.microsecond, - } - elif isinstance(obj, uuid.UUID): - return { - "__type__": "UUID", - "value": obj.hex, - } - else: - return super().default(obj) +import orjson class JSONDataEncoder(gearman.DataEncoder): - """Custom data encoder class for the `gearman` library (JSON). - - This class enables serialization and deserialization of data using JSON - format, supporting UUID and datetime data types. - """ + """Custom data encoder class for the `gearman` library (JSON).""" @classmethod def encode(cls, encodable_object): # Object of type bytes is not JSON serializable. if isinstance(encodable_object, bytes): encodable_object = encodable_object.decode("utf-8") - return json.dumps(encodable_object, cls=JSONEncoder, separators=(",", ":")) + return orjson.dumps(encodable_object) @classmethod def decode(cls, decodable_string): - return json.loads(decodable_string, cls=JSONDecoder) + return orjson.loads(decodable_string) diff --git a/tests/archivematicaCommon/test_gearman_encoder.py b/tests/archivematicaCommon/test_gearman_encoder.py index 7960a08d74..6a758ff226 100644 --- a/tests/archivematicaCommon/test_gearman_encoder.py +++ b/tests/archivematicaCommon/test_gearman_encoder.py @@ -1,29 +1,17 @@ from datetime import datetime -from uuid import UUID from django.utils.timezone import make_aware from gearman_encoder import JSONDataEncoder -def test_json_data_encoder(): - encoder = JSONDataEncoder - object_py = { - "id": UUID("ccf0368e-ec97-4375-a408-5c4d95c0c671"), - "data": [1, 2, 3], - "when": make_aware(datetime(2019, 6, 18, 1, 1, 1, 123)), - } - object_js = ( - "{" - + '"id":{"__type__":"UUID","value":"ccf0368eec974375a4085c4d95c0c671"},' - + '"data":[1,2,3],' - + '"when":{"__type__":"datetime","year":2019,"month":6,"day":18,"hour":1,"minute":1,"second":1,"microsecond":123}}' +def test_encoder(): + assert JSONDataEncoder.encode(b"bytes") == b'"bytes"' + assert JSONDataEncoder.encode([1, 2, 3]) == b"[1,2,3]" + assert ( + JSONDataEncoder.encode( + {"date": make_aware(datetime(2019, 6, 18, 1, 1, 1, 123))} + ) + == b'{"date":"2019-06-18T01:01:01.000123+00:00"}' ) - assert encoder.encode(object_py) == object_js - assert encoder.decode(object_js) == object_py - - -def test_json_data_encoder_with_bytes(): - encoder = JSONDataEncoder - - assert encoder.encode(b"bytes") == '"bytes"' + assert JSONDataEncoder.decode("[1,2,3]") == [1, 2, 3] From caa3a56c837ae37995bf252b50b89c086274f9e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Garc=C3=ADa=20Crespo?= Date: Sat, 27 Apr 2024 05:06:15 +0000 Subject: [PATCH 29/32] Use JSONDataEncoder to encode task createdDate This commit modifies the MCPServer to encode the `createdDate` of tasks using `datetime` which is natively supported by our JSONDataEncoder. The conversion to the METS-compliant format now occurs during the decoding process on the client side. This change ensures consistent use of RFC 3339 format across all communications. --- src/MCPClient/lib/client/gearman.py | 10 +++++++++- .../lib/server/tasks/backends/gearman_backend.py | 2 +- tests/MCPServer/test_gearman.py | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/MCPClient/lib/client/gearman.py b/src/MCPClient/lib/client/gearman.py index f45ed9bbe1..d36053a1f5 100644 --- a/src/MCPClient/lib/client/gearman.py +++ b/src/MCPClient/lib/client/gearman.py @@ -1,6 +1,7 @@ import functools import logging import multiprocessing +from datetime import datetime import gearman from client import metrics @@ -66,9 +67,16 @@ def _prepare_jobs(task_name, gearman_job): jobs = [] for task_uuid, task_data in tasks.items(): + # We use ISO 8601 in our wire format but replace the `T` separator + # with a space for METS compatibility in client scripts. + created_date = datetime.fromisoformat(task_data["createdDate"]).isoformat( + " " + ) arguments = str(task_data["arguments"]) arguments = replace_task_arguments( - arguments, task_uuid, task_data.get("createdDate") + arguments, + task_uuid, + created_date, ) arguments = parse_command_line(arguments) diff --git a/src/MCPServer/lib/server/tasks/backends/gearman_backend.py b/src/MCPServer/lib/server/tasks/backends/gearman_backend.py index 8ef0b4a181..f55b89d8c9 100644 --- a/src/MCPServer/lib/server/tasks/backends/gearman_backend.py +++ b/src/MCPServer/lib/server/tasks/backends/gearman_backend.py @@ -190,7 +190,7 @@ def failed(self): def serialize_task(self, task): return { "uuid": str(task.uuid), - "createdDate": task.start_timestamp.isoformat(" "), + "createdDate": task.start_timestamp, "arguments": task.arguments, "wants_output": task.wants_output, } diff --git a/tests/MCPServer/test_gearman.py b/tests/MCPServer/test_gearman.py index 63195b75ed..ca8fab26f8 100644 --- a/tests/MCPServer/test_gearman.py +++ b/tests/MCPServer/test_gearman.py @@ -39,7 +39,7 @@ def format_gearman_request(tasks): task_uuid = str(task.uuid) request["tasks"][task_uuid] = { "uuid": task_uuid, - "createdDate": task.start_timestamp.isoformat(" "), + "createdDate": task.start_timestamp, "arguments": task.arguments, "wants_output": task.wants_output, } From b49270eccc48fc6b1f2cdbac172d70d85f1c1c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Garc=C3=ADa=20Crespo?= Date: Sat, 25 May 2024 05:47:49 +0000 Subject: [PATCH 30/32] Add SYS_NICE capability to MySQL container MySQL uses mbind to bind processes and threads to CPUs which is not allowed by the default configuration of Docker's seccomp profile. This causes MySQL to log the following error repeatedly: mbind: Operation not permitted Adding the `SYS_NICE` capability to the container forces Docker to adjust the default seccomp profile to include the necessary permissions. For more details, visit https://github.com/docker-library/mysql/issues/303. It looks like MySQL fixed the problem in [v8.0.13] but Percona doesn't seem to have received this update. [v8.0.13]: https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-13.html --- hack/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hack/docker-compose.yml b/hack/docker-compose.yml index 96f92ce3af..9e4be90de0 100644 --- a/hack/docker-compose.yml +++ b/hack/docker-compose.yml @@ -39,6 +39,8 @@ services: - "mysql_data:/var/lib/mysql" ports: - "127.0.0.1:62001:3306" + cap_add: + - "SYS_NICE" elasticsearch: image: "docker.elastic.co/elasticsearch/elasticsearch:6.8.23" From 20d633e0b8e015bcb9dedc184946b953c5c685ba Mon Sep 17 00:00:00 2001 From: Douglas Cerna Date: Tue, 28 May 2024 10:13:09 -0600 Subject: [PATCH 31/32] Use Ruff for Python linting and formatting This replaces the black, pyupgrade, reorder-python-imports and flake8 hooks in pre-commit with ruff. It also upgrades the django-upgrade and markdownlint-cli hooks. --- .pre-commit-config.yaml | 30 ++------- ruff.toml | 21 ++++++ src/MCPClient/lib/client/gearman.py | 6 +- src/MCPClient/lib/client/job.py | 2 +- src/MCPClient/lib/client/mcp.py | 2 +- src/MCPClient/lib/client/metrics.py | 32 ++++----- src/MCPClient/lib/client/pool.py | 7 +- src/MCPClient/lib/client/worker.py | 4 +- .../archivematicaCreateMETSMetadataXML.py | 9 +-- .../archivematicaCreateMETSReingest.py | 4 +- .../archivematicaCreateMETSRights.py | 2 +- .../archivematicaCreateMETSTrim.py | 17 ++--- .../clientScripts/archivematica_clamscan.py | 22 +++---- .../lib/clientScripts/assign_file_uuids.py | 18 ++--- .../assign_uuids_to_directories.py | 11 ++-- .../bag_with_empty_directories.py | 4 +- src/MCPClient/lib/clientScripts/bind_pid.py | 8 +-- src/MCPClient/lib/clientScripts/bind_pids.py | 26 ++++---- .../lib/clientScripts/change_object_names.py | 14 ++-- .../lib/clientScripts/change_sip_name.py | 6 +- .../lib/clientScripts/characterize_file.py | 35 ++++------ .../check_for_access_directory.py | 5 +- .../lib/clientScripts/compress_aip.py | 40 ++++------- .../convert_dataverse_structure.py | 12 ++-- .../copy_transfer_submission_documentation.py | 6 +- .../copy_transfers_metadata_and_logs.py | 6 +- .../lib/clientScripts/create_aic_mets.py | 11 ++-- .../clientScripts/create_mets_dataverse_v2.py | 5 +- .../lib/clientScripts/create_mets_v2.py | 53 +++++++-------- .../create_sip_from_transfer_objects.py | 14 ++-- ...eate_sips_from_trim_transfer_containers.py | 3 +- .../clientScripts/create_transfer_metadata.py | 3 +- .../lib/clientScripts/create_transfer_mets.py | 17 ++--- .../lib/clientScripts/email_fail_report.py | 7 +- .../lib/clientScripts/extract_contents.py | 15 +++-- .../extract_maildir_attachments.py | 20 +++--- .../clientScripts/extract_zipped_transfer.py | 6 +- .../lib/clientScripts/failed_sip_cleanup.py | 5 +- .../clientScripts/failed_transfer_cleanup.py | 9 +-- src/MCPClient/lib/clientScripts/fits.py | 1 - .../get_aip_storage_locations.py | 5 +- .../lib/clientScripts/has_packages.py | 5 +- .../identify_dspace_mets_files.py | 2 +- .../lib/clientScripts/identify_file_format.py | 23 +++---- .../load_premis_events_from_xml.py | 14 +--- ...ization_create_metadata_and_restructure.py | 27 ++++---- ..._normalization_move_access_files_to_dip.py | 13 ++-- ...ual_normalization_remove_mn_directories.py | 9 +-- src/MCPClient/lib/clientScripts/move_sip.py | 6 +- .../lib/clientScripts/move_to_backlog.py | 22 ++++--- .../lib/clientScripts/move_transfer.py | 6 +- src/MCPClient/lib/clientScripts/normalize.py | 22 +++---- .../lib/clientScripts/normalize_report.py | 2 +- .../lib/clientScripts/parse_dataverse_mets.py | 16 ++--- .../lib/clientScripts/parse_mets_to_db.py | 8 +-- .../lib/clientScripts/pid_declaration.py | 14 ++-- .../lib/clientScripts/policy_check.py | 66 ++++++++----------- .../lib/clientScripts/post_store_aip_hook.py | 15 ++--- .../clientScripts/remove_unneeded_files.py | 8 +-- .../restructure_bag_aip_to_sip.py | 2 +- .../restructure_for_compliance.py | 18 +++-- .../restructure_for_compliance_sip.py | 6 +- .../retry_normalize_remove_normalized.py | 4 +- .../set_maildir_file_grp_use_and_file_ids.py | 16 ++--- .../lib/clientScripts/set_transfer_type.py | 6 +- src/MCPClient/lib/clientScripts/store_aip.py | 21 +++--- .../store_file_modification_dates.py | 6 +- src/MCPClient/lib/clientScripts/transcoder.py | 11 +--- .../lib/clientScripts/transcribe_file.py | 21 +++--- .../trim_create_rights_entries.py | 18 +++-- .../trim_restructure_for_compliance.py | 5 +- .../clientScripts/trim_verify_checksums.py | 4 +- .../lib/clientScripts/trim_verify_manifest.py | 2 +- .../clientScripts/update_size_and_checksum.py | 13 ++-- .../lib/clientScripts/upload_archivesspace.py | 6 +- .../lib/clientScripts/upload_qubit.py | 12 +--- .../lib/clientScripts/validate_file.py | 45 +++++-------- src/MCPClient/lib/clientScripts/verify_aip.py | 62 +++++++---------- .../verify_and_restructure_transfer_bag.py | 10 +-- .../lib/clientScripts/verify_checksum.py | 27 +++----- .../lib/clientScripts/verify_mets.py | 1 + .../clientScripts/verify_sip_compliance.py | 1 - src/MCPClient/lib/settings/common.py | 6 +- src/MCPClient/lib/settings/test.py | 1 + src/MCPServer/lib/server/jobs/__init__.py | 1 + src/MCPServer/lib/server/jobs/base.py | 2 +- src/MCPServer/lib/server/jobs/chain.py | 2 +- src/MCPServer/lib/server/jobs/client.py | 4 +- src/MCPServer/lib/server/jobs/decisions.py | 3 +- src/MCPServer/lib/server/jobs/local.py | 2 + src/MCPServer/lib/server/mcp.py | 14 ++-- src/MCPServer/lib/server/metrics.py | 2 +- src/MCPServer/lib/server/packages.py | 15 ++--- src/MCPServer/lib/server/processing_config.py | 3 +- src/MCPServer/lib/server/queues.py | 3 +- src/MCPServer/lib/server/rpc_server.py | 7 +- src/MCPServer/lib/server/shared_dirs.py | 5 +- src/MCPServer/lib/server/tasks/__init__.py | 3 +- .../lib/server/tasks/backends/__init__.py | 2 +- .../server/tasks/backends/gearman_backend.py | 7 +- src/MCPServer/lib/server/tasks/task.py | 6 +- src/MCPServer/lib/server/translation.py | 2 +- src/MCPServer/lib/server/utils.py | 1 + src/MCPServer/lib/server/watch_dirs.py | 4 +- src/MCPServer/lib/server/workflow.py | 7 +- .../lib/server/workflow_abilities.py | 2 +- src/MCPServer/lib/settings/test.py | 1 + src/archivematicaCommon/lib/appconfig.py | 1 + .../lib/archivematicaFunctions.py | 17 +++-- src/archivematicaCommon/lib/bindpid.py | 10 +-- .../lib/custom_handlers.py | 4 +- .../lib/databaseFunctions.py | 2 +- src/archivematicaCommon/lib/dbconns.py | 2 +- src/archivematicaCommon/lib/dicts.py | 1 - .../lib/elasticSearchFunctions.py | 16 ++--- .../externals/extractMaildirAttachments.py | 12 ++-- .../externals/fiwalk_plugins/pronom_ident.py | 1 + .../lib/externals/maildirToMbox.py | 1 + .../lib/externals/xmltodict.py | 2 +- src/archivematicaCommon/lib/fileOperations.py | 22 ++----- .../lib/identifier_functions.py | 1 + src/archivematicaCommon/lib/processing.py | 1 - src/archivematicaCommon/lib/storageService.py | 19 ++---- src/archivematicaCommon/lib/xml2obj.py | 26 +++----- .../frontend/tools/extract_features.py | 1 - .../install/dashboard.gunicorn-config.py | 1 - src/dashboard/src/components/access/views.py | 14 ++-- src/dashboard/src/components/accounts/urls.py | 1 - .../administration/forms_dip_upload.py | 1 - .../src/components/administration/urls.py | 1 - .../src/components/administration/views.py | 1 - .../administration/views_dip_upload.py | 1 - .../administration/views_processing.py | 1 - .../src/components/api/validators.py | 1 + src/dashboard/src/components/api/views.py | 4 +- .../src/components/archival_storage/urls.py | 1 - .../src/components/archival_storage/views.py | 8 +-- src/dashboard/src/components/backlog/views.py | 4 +- src/dashboard/src/components/file/views.py | 1 - .../src/components/filesystem_ajax/helpers.py | 1 - .../src/components/filesystem_ajax/views.py | 1 - src/dashboard/src/components/helpers.py | 11 ++-- src/dashboard/src/components/rights/load.py | 3 +- .../src/components/transfer/views.py | 6 +- src/dashboard/src/contrib/mcp/client.py | 1 - .../management/commands/import_pronom_ids.py | 23 ++----- .../fpr/migrations-data/mc_validate_cmd.py | 1 - .../0007_embedded_default_thumbnail.py | 1 - .../migrations/0011_mediaconch_validation.py | 1 - .../0012_mediaconch_policy_checks.py | 1 - ...malize_jp2_with_ffmpeg_for_preservation.py | 1 - .../migrations/0026_fits_nailgun_compat.py | 1 - .../0028_idcommand_unique_enabled.py | 1 + .../0029_update_inkscape_svg_command.py | 2 +- .../migrations/0030_update_jhove_command.py | 1 + .../0031_update_normalization_rules.py | 1 + .../migrations/0035_python3_compatibility.py | 1 + src/dashboard/src/fpr/models.py | 2 +- .../src/fpr/templatetags/revisions_link.py | 1 - src/dashboard/src/installer/forms.py | 1 - src/dashboard/src/installer/middleware.py | 1 - src/dashboard/src/installer/steps.py | 1 - .../purge_transient_processing_data.py | 5 +- .../rebuild_aip_index_from_storage_service.py | 5 +- ...uild_elasticsearch_aip_index_from_files.py | 1 + .../commands/rebuild_transfer_backlog.py | 23 ++----- .../commands/reindex_from_remote_cluster.py | 5 +- .../commands/resolve_pending_jobs.py | 1 + .../commands/update_elasticsearch_mappings.py | 1 + .../main/migrations/0026_agent_m2m_event.py | 4 +- .../migrations/0029_backlog_removal_event.py | 1 - .../0037_mediaconch_policy_checks.py | 6 +- .../migrations/0038_fix_workflow_store_dip.py | 1 + .../src/main/migrations/0041_bind_pids.py | 1 + .../0043_remove_create_pointer_file_ms.py | 1 + .../0045_archivesspace_config_dict.py | 1 - .../migrations/0050_remove_magic_links.py | 1 + .../0051_remove_verify_premis_checksums.py | 1 + ..._correct_extract_packages_fallback_link.py | 1 + ...0055_normalize_thumbnail_mode_selection.py | 1 + .../migrations/0057_7zip_no_compression.py | 1 + .../0058_fix_unit_variable_pull_link.py | 1 + .../migrations/0059_siparrange_longblob.py | 2 +- .../migrations/0060_delete_orphan_mscl.py | 1 + .../0061_create_dataverse_transfer_type.py | 1 + .../migrations/0062_atom_upload_exit_code.py | 2 +- .../main/migrations/0064_version_number.py | 1 + .../main/migrations/0065_version_number.py | 1 + .../migrations/0066_archivesspace_base_url.py | 1 + .../main/migrations/0068_version_number.py | 1 + .../src/main/migrations/0069_remove_atk.py | 1 + .../0071_index_unitvariables_runsql.py | 1 + .../migrations/0073_index_files_runsql.py | 1 + .../main/migrations/0074_version_number.py | 1 + .../main/migrations/0076_version_number.py | 1 + .../src/main/migrations/0077_uuid_fields.py | 1 + .../main/migrations/0078_username_check.py | 1 + .../main/migrations/0079_version_number.py | 5 +- .../migrations/0083_unpickle_access_target.py | 4 +- src/dashboard/src/main/models.py | 26 ++------ src/dashboard/src/main/urls.py | 1 - src/dashboard/src/middleware/common.py | 1 - src/dashboard/src/settings/components/auth.py | 1 + .../src/settings/components/cas_auth.py | 6 +- .../src/settings/components/ldap_auth.py | 6 +- src/dashboard/src/settings/local.py | 1 + src/dashboard/src/settings/production.py | 1 + src/dashboard/src/settings/test.py | 7 +- src/dashboard/src/urls.py | 1 - src/dashboard/src/wsgi.py | 1 - tests/MCPClient/test_antivirus.py | 3 +- tests/MCPClient/test_antivirus_clamdscan.py | 5 +- tests/MCPClient/test_antivirus_clamscan.py | 1 + ...test_archivematicaCreateMETSMetadataCSV.py | 1 + ...test_archivematicaCreateMETSMetadataXML.py | 12 ++-- .../test_check_for_access_directory.py | 2 +- tests/MCPClient/test_create_aip_mets.py | 17 ++--- tests/MCPClient/test_create_dataverse_mets.py | 27 +++----- tests/MCPClient/test_create_mets_v2.py | 2 +- tests/MCPClient/test_create_transfer_mets.py | 5 +- .../test_ensure_no_mutable_globals.py | 1 + tests/MCPClient/test_filename_change.py | 2 +- tests/MCPClient/test_json_conversion.py | 1 - .../test_load_premis_events_from_xml.py | 8 +-- tests/MCPClient/test_parse_dataverse.py | 1 + tests/MCPClient/test_parse_external_mets.py | 1 - tests/MCPClient/test_pid_components.py | 3 +- tests/MCPClient/test_reingest_mets.py | 4 +- ...t_restructure_dip_for_content_dm_upload.py | 1 - tests/MCPClient/test_upload_archivesspace.py | 1 + tests/MCPClient/test_validate_file.py | 2 +- tests/MCPClient/test_verify_checksum.py | 19 ++---- tests/MCPServer/test_integration.py | 5 +- tests/MCPServer/test_package.py | 16 ++--- tests/MCPServer/test_processing_config.py | 7 +- tests/MCPServer/test_rpc_server.py | 1 - tests/MCPServer/test_translation.py | 2 +- tests/MCPServer/test_workflow.py | 5 +- tests/MCPServer/test_workflow_abilities.py | 1 - tests/archivematicaCommon/test_appconfig.py | 7 +- tests/archivematicaCommon/test_bindpid.py | 1 - .../test_database_functions.py | 1 - .../test_elasticsearch_functions.py | 10 +-- .../test_storage_service.py | 5 +- .../administration/test_processing_config.py | 2 +- .../components/administration/test_usage.py | 1 - .../archival_storage/test_archival_storage.py | 9 +-- tests/dashboard/components/mcp/test_views.py | 1 - tests/dashboard/fpr/test_forms.py | 5 +- tests/dashboard/test_api.py | 4 +- tests/dashboard/test_filesystem_ajax.py | 4 +- tests/dashboard/test_helpers.py | 8 +-- tests/dashboard/test_migrations.py | 5 +- tests/dashboard/test_rights.py | 3 +- tox.ini | 19 +----- 255 files changed, 762 insertions(+), 1085 deletions(-) create mode 100644 ruff.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f2771e889d..e67280ecd9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,35 +9,19 @@ repos: src/MCPServer/lib/assets/.*\.json| src/(MCPClient/MCPServer|dashboard)/osdeps/.*\.json ) -- repo: https://github.com/asottile/pyupgrade - rev: v3.15.2 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.5 hooks: - - id: pyupgrade - args: [--py38-plus] -- repo: https://github.com/asottile/reorder_python_imports - rev: v3.12.0 - hooks: - - id: reorder-python-imports - args: [--py38-plus] + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format - repo: https://github.com/adamchainz/django-upgrade - rev: "1.16.0" + rev: "1.17.0" hooks: - id: django-upgrade args: [--target-version, "4.2"] -- repo: https://github.com/psf/black - rev: "23.12.1" - hooks: - - id: black - args: [--safe, --quiet] -- repo: https://github.com/pycqa/flake8 - rev: "7.0.0" - hooks: - - id: flake8 - additional_dependencies: - - flake8-bugbear==24.2.6 - - flake8-comprehensions==3.14.0 - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.39.0 + rev: v0.41.0 hooks: - id: markdownlint exclude: | diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000000..c81f8b4f5c --- /dev/null +++ b/ruff.toml @@ -0,0 +1,21 @@ +[lint] +# Rule reference: https://docs.astral.sh/ruff/rules/ +select = [ + "B", + "C4", + "E", + "F", + "I", + "UP", + "W", +] +ignore = [ + "B018", + "B904", + "E402", + "E501", + "UP031", +] + +[lint.isort] +force-single-line = true diff --git a/src/MCPClient/lib/client/gearman.py b/src/MCPClient/lib/client/gearman.py index d36053a1f5..9ae950beae 100644 --- a/src/MCPClient/lib/client/gearman.py +++ b/src/MCPClient/lib/client/gearman.py @@ -4,15 +4,15 @@ from datetime import datetime import gearman +from django.conf import settings +from gearman_encoder import JSONDataEncoder + from client import metrics from client.job import Job from client.loader import load_job_modules from client.utils import parse_command_line from client.utils import replace_task_arguments from client.worker import run_task -from django.conf import settings -from gearman_encoder import JSONDataEncoder - logger = logging.getLogger("archivematica.mcp.client.gearman") diff --git a/src/MCPClient/lib/client/job.py b/src/MCPClient/lib/client/job.py index 143f5fb313..e04349fad9 100644 --- a/src/MCPClient/lib/client/job.py +++ b/src/MCPClient/lib/client/job.py @@ -4,6 +4,7 @@ batches by clientScript modules and populated with an exit code, standard out and standard error information. """ + import logging import sys import traceback @@ -13,7 +14,6 @@ from django.utils import timezone from main.models import Task - logger = logging.getLogger("archivematica.mcp.client.job") diff --git a/src/MCPClient/lib/client/mcp.py b/src/MCPClient/lib/client/mcp.py index f54ae8d324..60ff5add87 100644 --- a/src/MCPClient/lib/client/mcp.py +++ b/src/MCPClient/lib/client/mcp.py @@ -1,6 +1,7 @@ """ Main MCPClient entrypoint. """ + import logging import os import pathlib @@ -9,7 +10,6 @@ from client import metrics from client.pool import WorkerPool - logger = logging.getLogger("archivematica.mcp.client") diff --git a/src/MCPClient/lib/client/metrics.py b/src/MCPClient/lib/client/metrics.py index 2fa6177288..8c85968234 100644 --- a/src/MCPClient/lib/client/metrics.py +++ b/src/MCPClient/lib/client/metrics.py @@ -1,6 +1,7 @@ """ Exposes various metrics via Prometheus. """ + import configparser import datetime import functools @@ -9,28 +10,23 @@ django.setup() +from common_metrics import PACKAGE_FILE_COUNT_BUCKETS +from common_metrics import PACKAGE_SIZE_BUCKETS +from common_metrics import PROCESSING_TIME_BUCKETS +from common_metrics import TASK_DURATION_BUCKETS from django.conf import settings from django.db.models import Sum from django.utils import timezone -from prometheus_client import ( - CollectorRegistry, - Counter, - Gauge, - Histogram, - multiprocess, - start_http_server, -) - from fpr.models import FormatVersion -from main.models import File, FileFormatVersion, Transfer - -from common_metrics import ( - PACKAGE_FILE_COUNT_BUCKETS, - PACKAGE_SIZE_BUCKETS, - PROCESSING_TIME_BUCKETS, - TASK_DURATION_BUCKETS, -) - +from main.models import File +from main.models import FileFormatVersion +from main.models import Transfer +from prometheus_client import CollectorRegistry +from prometheus_client import Counter +from prometheus_client import Gauge +from prometheus_client import Histogram +from prometheus_client import multiprocess +from prometheus_client import start_http_server job_counter = Counter( "mcpclient_job_total", diff --git a/src/MCPClient/lib/client/pool.py b/src/MCPClient/lib/client/pool.py index bf2fd36005..2a40ab8594 100644 --- a/src/MCPClient/lib/client/pool.py +++ b/src/MCPClient/lib/client/pool.py @@ -12,6 +12,7 @@ Workers log events into a shared queue while the pool runs a background thread (log listener) that listens to the queue and writes the events safely. """ + import logging import multiprocessing import threading @@ -21,13 +22,13 @@ django.setup() -from django.conf import settings from django import db +from django.conf import settings -from client import loader, metrics +from client import loader +from client import metrics from client.gearman import MCPGearmanWorker - logger = logging.getLogger("archivematica.mcp.client.worker") diff --git a/src/MCPClient/lib/client/worker.py b/src/MCPClient/lib/client/worker.py index 19da7e39c7..0a33912e0c 100644 --- a/src/MCPClient/lib/client/worker.py +++ b/src/MCPClient/lib/client/worker.py @@ -1,9 +1,9 @@ import logging -from client import metrics -from client.job import Job from dbconns import auto_close_old_connections +from client import metrics +from client.job import Job logger = logging.getLogger("archivematica.mcp.client.worker") diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py index d8dd762c1e..bc045589d2 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . """Management of XML metadata files.""" + import csv from pathlib import Path from urllib.parse import urlparse @@ -132,16 +133,12 @@ def _get_xml_metadata_mapping(sip_path, reingest=False): for row in reader: if not all(k in row and row[k] for k in ["filename", "type"]): errors.append( - "A row in {} is missing the filename and/or type".format( - source_metadata_path - ) + f"A row in {source_metadata_path} is missing the filename and/or type" ) continue if row["type"] == "CUSTOM": errors.append( - "A row in {} is using CUSTOM, a reserved type".format( - source_metadata_path - ) + f"A row in {source_metadata_path} is using CUSTOM, a reserved type" ) continue if row["filename"] not in mapping: diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py index 843e0b5cce..1f6d8b58d5 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py @@ -27,9 +27,7 @@ def _create_premis_object(premis_object_type): { etree.QName(ns.xsiNS, "schemaLocation"): "%s %s" % (ns.premisNS, "https://www.loc.gov/standards/premis/premis.xsd"), - "{http://www.w3.org/2001/XMLSchema-instance}type": "premis:{}".format( - premis_object_type - ), + "{http://www.w3.org/2001/XMLSchema-instance}type": f"premis:{premis_object_type}", "version": "3.0", }, nsmap={"premis": ns.premisNS}, diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRights.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRights.py index cab039e02d..5f00d381a8 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRights.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRights.py @@ -345,7 +345,7 @@ def getrightsGranted(job, statement, parent, state): restriction = "Undefined" for restriction in granted.restrictions.all(): restriction = restriction.restriction - if not restriction.lower() in ["disallow", "conditional", "allow"]: + if restriction.lower() not in ["disallow", "conditional", "allow"]: job.pyprint( "The value of element restriction must be: 'Allow', 'Disallow', or 'Conditional':", restriction, diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py index ef908132ac..0f6f0436bd 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py @@ -96,10 +96,9 @@ def getTrimDmdSec(job, baseDirectoryPath, sipUUID): if maxDateMod is None or dateMod > maxDateMod: maxDateMod = dateMod - etree.SubElement(dublincore, ns.dctermsBNS + "date").text = "{}/{}".format( - minDateMod, - maxDateMod, - ) + etree.SubElement( + dublincore, ns.dctermsBNS + "date" + ).text = f"{minDateMod}/{maxDateMod}" return ret @@ -167,8 +166,9 @@ def getTrimFileAmdSec(job, baseDirectoryPath, sipUUID, fileUUID): label = os.path.basename(f.currentlocation.decode()) attrib = { "LABEL": label, - ns.xlinkBNS - + "href": f.currentlocation.decode().replace("%SIPDirectory%", "", 1), + ns.xlinkBNS + "href": f.currentlocation.decode().replace( + "%SIPDirectory%", "", 1 + ), "MDTYPE": "OTHER", "OTHERMDTYPE": "CUSTOM", "LOCTYPE": "OTHER", @@ -189,8 +189,9 @@ def getTrimAmdSec(job, baseDirectoryPath, sipUUID): for f in files: attrib = { "LABEL": "ContainerMetadata.xml", - ns.xlinkBNS - + "href": f.currentlocation.decode().replace("%SIPDirectory%", "", 1), + ns.xlinkBNS + "href": f.currentlocation.decode().replace( + "%SIPDirectory%", "", 1 + ), "MDTYPE": "OTHER", "OTHERMDTYPE": "CUSTOM", "LOCTYPE": "OTHER", diff --git a/src/MCPClient/lib/clientScripts/archivematica_clamscan.py b/src/MCPClient/lib/clientScripts/archivematica_clamscan.py index aceaa88c32..f3af3a2c91 100755 --- a/src/MCPClient/lib/clientScripts/archivematica_clamscan.py +++ b/src/MCPClient/lib/clientScripts/archivematica_clamscan.py @@ -27,19 +27,17 @@ import django django.setup() -from django.db import transaction -from django.conf import settings as mcpclient_settings - -from clamd import ( - ClamdUnixSocket, - ClamdNetworkSocket, - BufferTooLongError, - ConnectionError, -) +from clamd import BufferTooLongError +from clamd import ClamdNetworkSocket +from clamd import ClamdUnixSocket +from clamd import ConnectionError from custom_handlers import get_script_logger from databaseFunctions import insertIntoEvents -from main.models import Event, File +from django.conf import settings as mcpclient_settings from django.core.exceptions import ValidationError +from django.db import transaction +from main.models import Event +from main.models import File logger = get_script_logger("archivematica.mcp.client.clamscan") @@ -233,9 +231,7 @@ def queue_event(file_uuid, date, scanner, passed, queue): event_detail = "" if scanner is not None: - event_detail = 'program="{}"; version="{}"; virusDefinitions="{}"'.format( - scanner.program(), scanner.version(), scanner.virus_definitions() - ) + event_detail = f'program="{scanner.program()}"; version="{scanner.version()}"; virusDefinitions="{scanner.virus_definitions()}"' outcome = "Pass" if passed else "Fail" logger.info("Recording new event for file %s (outcome: %s)", file_uuid, outcome) diff --git a/src/MCPClient/lib/clientScripts/assign_file_uuids.py b/src/MCPClient/lib/clientScripts/assign_file_uuids.py index 8c5393cc6c..0ec7997bc4 100755 --- a/src/MCPClient/lib/clientScripts/assign_file_uuids.py +++ b/src/MCPClient/lib/clientScripts/assign_file_uuids.py @@ -27,6 +27,7 @@ path to the SIP directory, and the subdirectory being targeted if any. """ + import argparse import os import uuid @@ -36,16 +37,17 @@ django.setup() # dashboard -from main.models import File, Transfer +import metsrw +import namespaces as ns # archivematicaCommon -from archivematicaFunctions import chunk_iterable, find_mets_file +from archivematicaFunctions import chunk_iterable +from archivematicaFunctions import find_mets_file from custom_handlers import get_script_logger -from fileOperations import addFileToTransfer from fileOperations import addFileToSIP - -import metsrw -import namespaces as ns +from fileOperations import addFileToTransfer +from main.models import File +from main.models import Transfer logger = get_script_logger("archivematica.mcp.client.assignFileUUID") @@ -266,9 +268,7 @@ def call(jobs): job.print_error(f"METS file not found: {err}") if mets_file: job.print_output( - "Reading METS file {} for reingested file information.".format( - mets_file - ) + f"Reading METS file {mets_file} for reingested file information." ) kwargs["mets"] = metsrw.METSDocument.fromfile(mets_file) diff --git a/src/MCPClient/lib/clientScripts/assign_uuids_to_directories.py b/src/MCPClient/lib/clientScripts/assign_uuids_to_directories.py index 3b1801fb31..c1a2b049e9 100755 --- a/src/MCPClient/lib/clientScripts/assign_uuids_to_directories.py +++ b/src/MCPClient/lib/clientScripts/assign_uuids_to_directories.py @@ -26,6 +26,7 @@ without doing anything. """ + import argparse import os from functools import wraps @@ -34,14 +35,16 @@ django.setup() # dashboard -from main.models import Transfer, Directory +from archivematicaFunctions import format_subdir_path +from archivematicaFunctions import get_dir_uuids +from archivematicaFunctions import str2bool # archivematicaCommon from custom_handlers import get_script_logger -from archivematicaFunctions import get_dir_uuids, format_subdir_path, str2bool - -from django.db import transaction from django.core.exceptions import ValidationError +from django.db import transaction +from main.models import Directory +from main.models import Transfer logger = get_script_logger("archivematica.mcp.client.assignUUIDsToDirectories") diff --git a/src/MCPClient/lib/clientScripts/bag_with_empty_directories.py b/src/MCPClient/lib/clientScripts/bag_with_empty_directories.py index 09d2282ed2..483d1b5b53 100755 --- a/src/MCPClient/lib/clientScripts/bag_with_empty_directories.py +++ b/src/MCPClient/lib/clientScripts/bag_with_empty_directories.py @@ -23,12 +23,10 @@ import django django.setup() -from django.conf import settings as mcpclient_settings - # archivematicaCommon from archivematicaFunctions import get_setting - from bagit import make_bag +from django.conf import settings as mcpclient_settings def get_sip_directories(job, sip_dir): diff --git a/src/MCPClient/lib/clientScripts/bind_pid.py b/src/MCPClient/lib/clientScripts/bind_pid.py index b33cbcdc11..da7ee6eb31 100755 --- a/src/MCPClient/lib/clientScripts/bind_pid.py +++ b/src/MCPClient/lib/clientScripts/bind_pid.py @@ -38,6 +38,7 @@ The sole command-line argument is the File's UUID. If the --bind-pids option is something other than 'Yes', the script will exit without doing anything. """ + import argparse from functools import wraps @@ -46,13 +47,12 @@ django.setup() # dashboard -from main.models import DashboardSetting, File - # archivematicaCommon import bindpid -from custom_handlers import get_script_logger from archivematicaFunctions import str2bool - +from custom_handlers import get_script_logger +from main.models import DashboardSetting +from main.models import File logger = get_script_logger("archivematica.mcp.client.bind_pid") diff --git a/src/MCPClient/lib/clientScripts/bind_pids.py b/src/MCPClient/lib/clientScripts/bind_pids.py index 12bbb8f867..a8ebf9cf58 100755 --- a/src/MCPClient/lib/clientScripts/bind_pids.py +++ b/src/MCPClient/lib/clientScripts/bind_pids.py @@ -37,6 +37,7 @@ shared directory where SIPs are stored. If the --bind-pids option is something other than 'Yes', the script will continue to the next job without doing anything. """ + import argparse import os import sys @@ -46,24 +47,23 @@ import django django.setup() -from django.db import transaction -from lxml import etree -from django.core.exceptions import ValidationError - -# dashboard -from main.models import DashboardSetting, Directory, SIP +import namespaces as ns # archivematicaCommon from archivematicaFunctions import str2bool -from bindpid import ( - bind_pid, - BindPIDException, - _validate_handle_server_config, - _validate_entity_type_required_params, -) +from bindpid import BindPIDException +from bindpid import _validate_entity_type_required_params +from bindpid import _validate_handle_server_config +from bindpid import bind_pid from custom_handlers import get_script_logger -import namespaces as ns +from django.core.exceptions import ValidationError +from django.db import transaction +from lxml import etree +from main.models import SIP +# dashboard +from main.models import DashboardSetting +from main.models import Directory logger = get_script_logger("archivematica.mcp.client.bind_pids") diff --git a/src/MCPClient/lib/clientScripts/change_object_names.py b/src/MCPClient/lib/clientScripts/change_object_names.py index 1614054c38..17a77e37f5 100755 --- a/src/MCPClient/lib/clientScripts/change_object_names.py +++ b/src/MCPClient/lib/clientScripts/change_object_names.py @@ -22,14 +22,18 @@ import django django.setup() -from django.db import transaction - -# dashboard -from main.models import Event, File, Directory, Transfer, SIP +import change_names # archivematicaCommon from custom_handlers import get_script_logger -import change_names +from django.db import transaction +from main.models import SIP +from main.models import Directory + +# dashboard +from main.models import Event +from main.models import File +from main.models import Transfer logger = get_script_logger("archivematica.mcp.client.changeObjectNames") diff --git a/src/MCPClient/lib/clientScripts/change_sip_name.py b/src/MCPClient/lib/clientScripts/change_sip_name.py index 0eb9eecdc9..17373abea4 100755 --- a/src/MCPClient/lib/clientScripts/change_sip_name.py +++ b/src/MCPClient/lib/clientScripts/change_sip_name.py @@ -21,12 +21,12 @@ import django django.setup() +from change_names import change_path from django.db import transaction # dashboard -from main.models import SIP, Transfer - -from change_names import change_path +from main.models import SIP +from main.models import Transfer def call(jobs): diff --git a/src/MCPClient/lib/clientScripts/characterize_file.py b/src/MCPClient/lib/clientScripts/characterize_file.py index 18aeeb77b4..3fe46fabad 100755 --- a/src/MCPClient/lib/clientScripts/characterize_file.py +++ b/src/MCPClient/lib/clientScripts/characterize_file.py @@ -12,20 +12,21 @@ from lxml import etree django.setup() -from django.db import transaction +from databaseFunctions import insertIntoFPCommandOutput +from dicts import ReplacementDict +from dicts import replace_string_values from django.conf import settings as mcpclient_settings - -# dashboard -from main.models import FPCommandOutput -from fpr.models import FPRule, FormatVersion +from django.core.exceptions import ValidationError +from django.db import transaction # archivematicaCommon from executeOrRunSubProcess import executeOrRun -from databaseFunctions import insertIntoFPCommandOutput -from dicts import replace_string_values, ReplacementDict - +from fpr.models import FormatVersion +from fpr.models import FPRule from lib import setup_dicts -from django.core.exceptions import ValidationError + +# dashboard +from main.models import FPCommandOutput def concurrent_instances(): @@ -81,9 +82,7 @@ def main(job, file_path, file_uuid, sip_uuid): if exitstatus != 0: job.write_error( - "Command {} failed with exit status {}; stderr:".format( - rule.command.description, exitstatus - ) + f"Command {rule.command.description} failed with exit status {exitstatus}; stderr:" ) failed = True continue @@ -100,22 +99,16 @@ def main(job, file_path, file_uuid, sip_uuid): etree.fromstring(stdout.encode("utf8")) insertIntoFPCommandOutput(file_uuid, stdout, rule.uuid) job.write_output( - 'Saved XML output for command "{}" ({})'.format( - rule.command.description, rule.command.uuid - ) + f'Saved XML output for command "{rule.command.description}" ({rule.command.uuid})' ) except etree.XMLSyntaxError: failed = True job.write_error( - 'XML output for command "{}" ({}) was not valid XML; not saving to database'.format( - rule.command.description, rule.command.uuid - ) + f'XML output for command "{rule.command.description}" ({rule.command.uuid}) was not valid XML; not saving to database' ) else: job.write_error( - 'Tool output for command "{}" ({}) is not XML; not saving to database'.format( - rule.command.description, rule.command.uuid - ) + f'Tool output for command "{rule.command.description}" ({rule.command.uuid}) is not XML; not saving to database' ) if failed: diff --git a/src/MCPClient/lib/clientScripts/check_for_access_directory.py b/src/MCPClient/lib/clientScripts/check_for_access_directory.py index d7c833cead..8571db1596 100755 --- a/src/MCPClient/lib/clientScripts/check_for_access_directory.py +++ b/src/MCPClient/lib/clientScripts/check_for_access_directory.py @@ -24,12 +24,11 @@ django.setup() # dashboard -from main.models import File - +from fileOperations import rename # archivematicaCommon from fileOperations import updateFileLocation -from fileOperations import rename +from main.models import File def main( diff --git a/src/MCPClient/lib/clientScripts/compress_aip.py b/src/MCPClient/lib/clientScripts/compress_aip.py index 4978b0784f..10100c7c2b 100755 --- a/src/MCPClient/lib/clientScripts/compress_aip.py +++ b/src/MCPClient/lib/clientScripts/compress_aip.py @@ -8,11 +8,10 @@ django.setup() # dashboard -from main.models import SIP - # archivematicaCommon import databaseFunctions from executeOrRunSubProcess import executeOrRun +from main.models import SIP def update_unit(sip_uuid, compressed_location): @@ -59,49 +58,32 @@ def compress_aip( return 0, {"sip_uuid": sip_uuid, "location": uncompressed_location} job.pyprint( - "Compressing {} with {}, algorithm {}, level {}".format( - uncompressed_location, program, compression_algorithm, compression_level - ) + f"Compressing {uncompressed_location} with {program}, algorithm {compression_algorithm}, level {compression_level}" ) if program == "7z": compressed_location = uncompressed_location + ".7z" - command = '/usr/bin/7z a -bd -t7z -y -m0={algorithm} -mx={level} -mta=on -mtc=on -mtm=on -mmt=on "{compressed_location}" "{uncompressed_location}"'.format( - algorithm=compression_algorithm, - level=compression_level, - uncompressed_location=uncompressed_location, - compressed_location=compressed_location, - ) + command = f'/usr/bin/7z a -bd -t7z -y -m0={compression_algorithm} -mx={compression_level} -mta=on -mtc=on -mtm=on -mmt=on "{compressed_location}" "{uncompressed_location}"' tool_info_command = ( r'echo program="7z"\; ' - r'algorithm="{}"\; ' - 'version="`7z | grep Version`"'.format(compression_algorithm) + rf'algorithm="{compression_algorithm}"\; ' + 'version="`7z | grep Version`"' ) elif program == "pbzip2": compressed_location = uncompressed_location + ".tar.bz2" - command = '/bin/tar -c --directory "{sip_directory}" "{archive_path}" | /usr/bin/pbzip2 --compress -{level} > "{compressed_location}"'.format( - sip_directory=sip_directory, - archive_path=archive_path, - level=compression_level, - compressed_location=compressed_location, - ) + command = f'/bin/tar -c --directory "{sip_directory}" "{archive_path}" | /usr/bin/pbzip2 --compress -{compression_level} > "{compressed_location}"' tool_info_command = ( r'echo program="pbzip2"\; ' - r'algorithm="{}"\; ' - 'version="$((pbzip2 -V) 2>&1)"'.format(compression_algorithm) + rf'algorithm="{compression_algorithm}"\; ' + 'version="$((pbzip2 -V) 2>&1)"' ) elif program == "gzip": compressed_location = uncompressed_location + ".tar.gz" - command = '/bin/tar -c --directory "{sip_directory}" "{archive_path}" | /bin/gzip -{level} > "{compressed_location}"'.format( - sip_directory=sip_directory, - archive_path=archive_path, - level=compression_level, - compressed_location=compressed_location, - ) + command = f'/bin/tar -c --directory "{sip_directory}" "{archive_path}" | /bin/gzip -{compression_level} > "{compressed_location}"' tool_info_command = ( r'echo program="gzip"\; ' - r'algorithm="{}"\; ' - 'version="$((gzip -V) 2>&1)"'.format(compression_algorithm) + rf'algorithm="{compression_algorithm}"\; ' + 'version="$((gzip -V) 2>&1)"' ) else: msg = f"Program {program} not recognized, exiting script prematurely." diff --git a/src/MCPClient/lib/clientScripts/convert_dataverse_structure.py b/src/MCPClient/lib/clientScripts/convert_dataverse_structure.py index 8a817b4b80..c9ae12de44 100644 --- a/src/MCPClient/lib/clientScripts/convert_dataverse_structure.py +++ b/src/MCPClient/lib/clientScripts/convert_dataverse_structure.py @@ -21,6 +21,7 @@ https://wiki.archivematica.org/Dataverse """ + import json import os import sys @@ -33,9 +34,8 @@ django.setup() -from custom_handlers import get_script_logger import metsrw - +from custom_handlers import get_script_logger logger = get_script_logger("archivematica.mcp.client.convert_dataverse_struct") @@ -200,9 +200,7 @@ def display_checksum_for_user(job, fname, checksum_value, checksum_type="MD5"): this script is doing in the Dataverse workflow. """ job.pyprint( - "Checksum for '{}' retrieved from dataset.json: {} ({})".format( - fname, checksum_value, checksum_type - ) + f"Checksum for '{fname}' retrieved from dataset.json: {checksum_value} ({checksum_type})" ) @@ -580,9 +578,7 @@ def init_convert_dataverse(job): return convert_dataverse_to_mets(job, unit_path=transfer_dir) except IndexError: convert_dv_msg = ( - "Problem with the supplied arguments to the function len: {}".format( - len(job.args) - ) + f"Problem with the supplied arguments to the function len: {len(job.args)}" ) logger.error(convert_dv_msg) raise ConvertDataverseError(convert_dv_msg) diff --git a/src/MCPClient/lib/clientScripts/copy_transfer_submission_documentation.py b/src/MCPClient/lib/clientScripts/copy_transfer_submission_documentation.py index f5f2399304..b5e8bcdc23 100755 --- a/src/MCPClient/lib/clientScripts/copy_transfer_submission_documentation.py +++ b/src/MCPClient/lib/clientScripts/copy_transfer_submission_documentation.py @@ -23,10 +23,10 @@ django.setup() -from bag import is_bag -from main.models import File, SIP - from archivematicaFunctions import find_transfer_path_from_ingest +from bag import is_bag +from main.models import SIP +from main.models import File def call(jobs): diff --git a/src/MCPClient/lib/clientScripts/copy_transfers_metadata_and_logs.py b/src/MCPClient/lib/clientScripts/copy_transfers_metadata_and_logs.py index 5ba0874ff9..a642f31f06 100755 --- a/src/MCPClient/lib/clientScripts/copy_transfers_metadata_and_logs.py +++ b/src/MCPClient/lib/clientScripts/copy_transfers_metadata_and_logs.py @@ -24,10 +24,10 @@ django.setup() -from bag import is_bag -from main.models import File, SIP - from archivematicaFunctions import find_transfer_path_from_ingest +from bag import is_bag +from main.models import SIP +from main.models import File def main( diff --git a/src/MCPClient/lib/clientScripts/create_aic_mets.py b/src/MCPClient/lib/clientScripts/create_aic_mets.py index 90efa402e7..ce54c67bc4 100755 --- a/src/MCPClient/lib/clientScripts/create_aic_mets.py +++ b/src/MCPClient/lib/clientScripts/create_aic_mets.py @@ -12,9 +12,6 @@ django.setup() # dashboard -from django.utils import timezone -from main.models import UnitVariable - import create_mets_v2 # archivematicaCommon @@ -22,6 +19,8 @@ import fileOperations import namespaces as ns import storageService as storage_service +from django.utils import timezone +from main.models import UnitVariable def get_aip_info(aic_dir, job): @@ -97,9 +96,9 @@ def create_mets_file(aic, aips, job): TYPE="logical", # structMap ), ) - mets.attrib[ - "{{{ns}}}schemaLocation".format(ns=nsmap["xsi"]) - ] = "http://www.loc.gov/METS/ http://www.loc.gov/standards/mets/version1121/mets.xsd" + mets.attrib["{{{ns}}}schemaLocation".format(ns=nsmap["xsi"])] = ( + "http://www.loc.gov/METS/ http://www.loc.gov/standards/mets/version1121/mets.xsd" + ) # Add Dublin Core info xml_data = mets.find("mets:dmdSec/mets:mdWrap/mets:xmlData", namespaces=ns.NSMAP) diff --git a/src/MCPClient/lib/clientScripts/create_mets_dataverse_v2.py b/src/MCPClient/lib/clientScripts/create_mets_dataverse_v2.py index a2cf7ecc4c..741f4f3604 100644 --- a/src/MCPClient/lib/clientScripts/create_mets_dataverse_v2.py +++ b/src/MCPClient/lib/clientScripts/create_mets_dataverse_v2.py @@ -16,15 +16,14 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -"""Maps Dataverse specific elements into the AIP METS file generated on ingest. -""" +"""Maps Dataverse specific elements into the AIP METS file generated on ingest.""" + import sys import archivematicaFunctions import metsrw from custom_handlers import get_script_logger - # Create a module level logger. logger = get_script_logger("archivematica.mcp.client.createMETSDataverse") diff --git a/src/MCPClient/lib/clientScripts/create_mets_v2.py b/src/MCPClient/lib/clientScripts/create_mets_v2.py index 7049349ec1..d6460e2022 100755 --- a/src/MCPClient/lib/clientScripts/create_mets_v2.py +++ b/src/MCPClient/lib/clientScripts/create_mets_v2.py @@ -35,44 +35,39 @@ django.setup() # dashboard -from django.conf import settings as mcpclient_settings -from main.models import ( - Agent, - Derivation, - Directory, - DublinCore, - Event, - File, - FileID, - FPCommandOutput, - SIP, - SIPArrange, -) - +import archivematicaCreateMETSMetadataXML import archivematicaCreateMETSReingest +import namespaces as ns from archivematicaCreateMETSMetadataCSV import parseMetadata -import archivematicaCreateMETSMetadataXML from archivematicaCreateMETSRights import archivematicaGetRights from archivematicaCreateMETSRightsDspaceMDRef import ( archivematicaCreateMETSRightsDspaceMDRef, ) -from archivematicaCreateMETSTrim import getTrimDmdSec -from archivematicaCreateMETSTrim import getTrimFileDmdSec from archivematicaCreateMETSTrim import getTrimAmdSec +from archivematicaCreateMETSTrim import getTrimDmdSec from archivematicaCreateMETSTrim import getTrimFileAmdSec +from archivematicaCreateMETSTrim import getTrimFileDmdSec # archivematicaCommon from archivematicaFunctions import normalizeNonDcElementName -from create_mets_dataverse_v2 import ( - create_dataverse_sip_dmdsec, - create_dataverse_tabfile_dmdsec, -) -from custom_handlers import get_script_logger -import namespaces as ns +from bagit import Bag +from bagit import BagError from change_names import change_name - -from bagit import Bag, BagError +from create_mets_dataverse_v2 import create_dataverse_sip_dmdsec +from create_mets_dataverse_v2 import create_dataverse_tabfile_dmdsec +from custom_handlers import get_script_logger +from django.conf import settings as mcpclient_settings from django.core.exceptions import ValidationError +from main.models import SIP +from main.models import Agent +from main.models import Derivation +from main.models import Directory +from main.models import DublinCore +from main.models import Event +from main.models import File +from main.models import FileID +from main.models import FPCommandOutput +from main.models import SIPArrange SIP_DIR_VAR = r"%SIPDirectory%" @@ -1203,7 +1198,7 @@ def createFileSec( except (Derivation.DoesNotExist, ValidationError): job.pyprint( "Fatal error: unable to locate a Derivation object" - " where the derived file is {}".format(f.uuid) + f" where the derived file is {f.uuid}" ) raise GROUPID = f"Group-{d.source_file_id}" @@ -1484,7 +1479,7 @@ def create_object_metadata(job, struct_map, baseDirectoryPath, state): bag_tag = etree.SubElement(bag_metadata, key) except ValueError: job.pyprint( - "Skipping bag key {}; not a valid" " XML tag name".format(key), + f"Skipping bag key {key}; not a valid" " XML tag name", file=sys.stderr, ) continue @@ -1663,9 +1658,7 @@ def add_normative_structmap_div( parent_el = path_to_el[parent_path] except KeyError: logger.info( - "Unable to find parent path {} of item {} in path_to_el\n{}".format( - parent_path, fsitem.path, pprint.pformat(path_to_el) - ) + f"Unable to find parent path {parent_path} of item {fsitem.path} in path_to_el\n{pprint.pformat(path_to_el)}" ) raise el = etree.SubElement( diff --git a/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py b/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py index a9f769d495..0bf46612a2 100755 --- a/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py +++ b/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py @@ -23,14 +23,18 @@ import django django.setup() -from django.db import transaction - -# dashboard -from main.models import File, Directory, SIP, Transfer, UnitVariable, Agent - # archivematicaCommon import archivematicaFunctions from django.core.exceptions import ValidationError +from django.db import transaction +from main.models import SIP +from main.models import Agent +from main.models import Directory + +# dashboard +from main.models import File +from main.models import Transfer +from main.models import UnitVariable def call(jobs): diff --git a/src/MCPClient/lib/clientScripts/create_sips_from_trim_transfer_containers.py b/src/MCPClient/lib/clientScripts/create_sips_from_trim_transfer_containers.py index 564fb501b2..95c6d51800 100755 --- a/src/MCPClient/lib/clientScripts/create_sips_from_trim_transfer_containers.py +++ b/src/MCPClient/lib/clientScripts/create_sips_from_trim_transfer_containers.py @@ -25,11 +25,10 @@ django.setup() # dashboard -from main.models import File - # archivematicaCommon import archivematicaFunctions import databaseFunctions +from main.models import File def call(jobs): diff --git a/src/MCPClient/lib/clientScripts/create_transfer_metadata.py b/src/MCPClient/lib/clientScripts/create_transfer_metadata.py index 230184a774..801867ebc4 100755 --- a/src/MCPClient/lib/clientScripts/create_transfer_metadata.py +++ b/src/MCPClient/lib/clientScripts/create_transfer_metadata.py @@ -6,9 +6,8 @@ django.setup() # dashboard -from main.models import Transfer - from client import metrics +from main.models import Transfer def fetch_set(sip_uuid): diff --git a/src/MCPClient/lib/clientScripts/create_transfer_mets.py b/src/MCPClient/lib/clientScripts/create_transfer_mets.py index 61be662130..e13ee6d1f4 100755 --- a/src/MCPClient/lib/clientScripts/create_transfer_mets.py +++ b/src/MCPClient/lib/clientScripts/create_transfer_mets.py @@ -34,16 +34,13 @@ from countryCodes import getCodeForCountry # dashboard -from main.models import ( - Agent, - Derivation, - Directory, - File, - FPCommandOutput, - RightsStatement, - Transfer, -) - +from main.models import Agent +from main.models import Derivation +from main.models import Directory +from main.models import File +from main.models import FPCommandOutput +from main.models import RightsStatement +from main.models import Transfer PREMIS_META = metsrw.plugins.premisrw.PREMIS_3_0_META FILE_PREMIS_META = PREMIS_META.copy() diff --git a/src/MCPClient/lib/clientScripts/email_fail_report.py b/src/MCPClient/lib/clientScripts/email_fail_report.py index 8d8872b371..4edc21da46 100755 --- a/src/MCPClient/lib/clientScripts/email_fail_report.py +++ b/src/MCPClient/lib/clientScripts/email_fail_report.py @@ -29,7 +29,6 @@ from main.models import Job from main.models import Report - django.setup() logger = get_script_logger("archivematica.mcp.client.emailFailReport") @@ -234,11 +233,7 @@ def call(jobs): ) job.set_status(1) continue - subject = "Archivematica Fail Report for {}: {}-{}".format( - args.unit_type, - args.unit_name, - args.unit_uuid, - ) + subject = f"Archivematica Fail Report for {args.unit_type}: {args.unit_name}-{args.unit_uuid}" # Generate report in HTML and send it by email content = get_content_for( diff --git a/src/MCPClient/lib/clientScripts/extract_contents.py b/src/MCPClient/lib/clientScripts/extract_contents.py index b32b108e21..2bd1cf2930 100755 --- a/src/MCPClient/lib/clientScripts/extract_contents.py +++ b/src/MCPClient/lib/clientScripts/extract_contents.py @@ -8,18 +8,23 @@ django.setup() # dashboard -from fpr.models import FPCommand -from main.models import Directory, FileFormatVersion, File, Transfer +from archivematicaFunctions import format_subdir_path +from archivematicaFunctions import get_dir_uuids # archivematicaCommon from custom_handlers import get_script_logger -from executeOrRunSubProcess import executeOrRun from databaseFunctions import fileWasRemoved -from fileOperations import addFileToTransfer, updateSizeAndChecksum -from archivematicaFunctions import get_dir_uuids, format_subdir_path +from executeOrRunSubProcess import executeOrRun +from fileOperations import addFileToTransfer +from fileOperations import updateSizeAndChecksum +from fpr.models import FPCommand # clientScripts from has_packages import already_extracted +from main.models import Directory +from main.models import File +from main.models import FileFormatVersion +from main.models import Transfer logger = get_script_logger("archivematica.mcp.client.extractContents") diff --git a/src/MCPClient/lib/clientScripts/extract_maildir_attachments.py b/src/MCPClient/lib/clientScripts/extract_maildir_attachments.py index 2b0e70ccbb..3d9d9bf9df 100755 --- a/src/MCPClient/lib/clientScripts/extract_maildir_attachments.py +++ b/src/MCPClient/lib/clientScripts/extract_maildir_attachments.py @@ -25,15 +25,16 @@ from lxml import etree django.setup() -from django.db import transaction from django.core.exceptions import ValidationError - -# dashboard -from main.models import File +from django.db import transaction # archivematicaCommon from externals.extractMaildirAttachments import parse -from fileOperations import addFileToTransfer, updateSizeAndChecksum +from fileOperations import addFileToTransfer +from fileOperations import updateSizeAndChecksum + +# dashboard +from main.models import File class State: @@ -105,9 +106,7 @@ def addKeyFileToNormalizeMaildirOffOf( content = """#This file is used in the archivematica system to represent a maildir dirctory, for normalization and permission purposes. [archivematicaMaildir] path = %s - """ % ( - relativePathToRepresent - ) + """ % (relativePathToRepresent) f = open(outFile, "w") f.write(content) f.close() @@ -210,10 +209,7 @@ def handle_job(job): ) job.pyprint("\tAttachment path:", filePath) writeFile(filePath, attachment) - eventDetail = "Unpacked from: {{{}}}{}".format( - sourceFileUUID, - sourceFilePath, - ) + eventDetail = f"Unpacked from: {{{sourceFileUUID}}}{sourceFilePath}" addFile( filePath, transferDir, diff --git a/src/MCPClient/lib/clientScripts/extract_zipped_transfer.py b/src/MCPClient/lib/clientScripts/extract_zipped_transfer.py index 9d968036e6..9e8b057512 100755 --- a/src/MCPClient/lib/clientScripts/extract_zipped_transfer.py +++ b/src/MCPClient/lib/clientScripts/extract_zipped_transfer.py @@ -25,13 +25,13 @@ django.setup() from django.db import transaction -# dashboard -from main.models import Transfer - # archivematicaCommon from executeOrRunSubProcess import executeOrRun from fileOperations import get_extract_dir_name +# dashboard +from main.models import Transfer + def extract(job, target, destinationDirectory): filename, file_extension = os.path.splitext(target) diff --git a/src/MCPClient/lib/clientScripts/failed_sip_cleanup.py b/src/MCPClient/lib/clientScripts/failed_sip_cleanup.py index 71e5e3087b..61da95459e 100755 --- a/src/MCPClient/lib/clientScripts/failed_sip_cleanup.py +++ b/src/MCPClient/lib/clientScripts/failed_sip_cleanup.py @@ -6,13 +6,10 @@ django.setup() # dashboard -from main import models - # archivematicaCommon import storageService as storage_service - from client import metrics - +from main import models REJECTED = "reject" FAILED = "fail" diff --git a/src/MCPClient/lib/clientScripts/failed_transfer_cleanup.py b/src/MCPClient/lib/clientScripts/failed_transfer_cleanup.py index 90afa303d2..a3e58afd3b 100755 --- a/src/MCPClient/lib/clientScripts/failed_transfer_cleanup.py +++ b/src/MCPClient/lib/clientScripts/failed_transfer_cleanup.py @@ -5,15 +5,12 @@ import django django.setup() -from django.db import transaction - # archivematicaCommon import storageService as storage_service - -from main.models import File, Transfer - from client import metrics - +from django.db import transaction +from main.models import File +from main.models import Transfer REJECTED = "reject" FAILED = "fail" diff --git a/src/MCPClient/lib/clientScripts/fits.py b/src/MCPClient/lib/clientScripts/fits.py index 2a93bd1723..02c54ed296 100755 --- a/src/MCPClient/lib/clientScripts/fits.py +++ b/src/MCPClient/lib/clientScripts/fits.py @@ -32,7 +32,6 @@ # dashboard from main.models import FPCommandOutput - logger = get_script_logger("archivematica.mcp.client.FITS") formats = [] diff --git a/src/MCPClient/lib/clientScripts/get_aip_storage_locations.py b/src/MCPClient/lib/clientScripts/get_aip_storage_locations.py index c44c1f28bf..ced0a10dd2 100755 --- a/src/MCPClient/lib/clientScripts/get_aip_storage_locations.py +++ b/src/MCPClient/lib/clientScripts/get_aip_storage_locations.py @@ -9,7 +9,6 @@ # archivematicaCommon import storageService as storage_service - from custom_handlers import get_script_logger logger = get_script_logger("archivematica.mcp.client.get_aip_storage_locations") @@ -19,9 +18,7 @@ def get_aip_storage_locations(purpose, job): """Return a dict of AIP Storage Locations and their descriptions.""" storage_directories = storage_service.get_location(purpose=purpose) logger.debug( - "Storage Directories: {}".format( - json.dumps(storage_directories, indent=4, sort_keys=True) - ) + f"Storage Directories: {json.dumps(storage_directories, indent=4, sort_keys=True)}" ) choices = {} for storage_dir in storage_directories: diff --git a/src/MCPClient/lib/clientScripts/has_packages.py b/src/MCPClient/lib/clientScripts/has_packages.py index 67709383bf..954452e63c 100755 --- a/src/MCPClient/lib/clientScripts/has_packages.py +++ b/src/MCPClient/lib/clientScripts/has_packages.py @@ -4,7 +4,10 @@ django.setup() # dashboard from fpr.models import FPRule -from main.models import FileFormatVersion, Transfer, File, Event +from main.models import Event +from main.models import File +from main.models import FileFormatVersion +from main.models import Transfer def is_extractable(f): diff --git a/src/MCPClient/lib/clientScripts/identify_dspace_mets_files.py b/src/MCPClient/lib/clientScripts/identify_dspace_mets_files.py index f2f2a6a496..96b198be92 100755 --- a/src/MCPClient/lib/clientScripts/identify_dspace_mets_files.py +++ b/src/MCPClient/lib/clientScripts/identify_dspace_mets_files.py @@ -19,8 +19,8 @@ django.setup() # dashboard -from main.models import File from django.db import transaction +from main.models import File def call(jobs): diff --git a/src/MCPClient/lib/clientScripts/identify_file_format.py b/src/MCPClient/lib/clientScripts/identify_file_format.py index 3257839eac..4ca12a6164 100755 --- a/src/MCPClient/lib/clientScripts/identify_file_format.py +++ b/src/MCPClient/lib/clientScripts/identify_file_format.py @@ -7,14 +7,19 @@ django.setup() # dashboard -from fpr.models import IDCommand, IDRule, FormatVersion -from main.models import FileFormatVersion, File, FileID, UnitVariable +from databaseFunctions import insertIntoEvents from django.db import transaction from django.utils import timezone # archivematicaCommon from executeOrRunSubProcess import executeOrRun -from databaseFunctions import insertIntoEvents +from fpr.models import FormatVersion +from fpr.models import IDCommand +from fpr.models import IDRule +from main.models import File +from main.models import FileFormatVersion +from main.models import FileID +from main.models import UnitVariable def concurrent_instances(): @@ -46,8 +51,8 @@ def _save_id_preference(file_, value): def write_identification_event(file_uuid, command, format=None, success=True): - event_detail_text = 'program="{}"; version="{}"'.format( - command.tool.description, command.tool.version + event_detail_text = ( + f'program="{command.tool.description}"; version="{command.tool.version}"' ) if success: event_outcome_text = "Positive" @@ -164,17 +169,13 @@ def main(job, enabled, file_path, file_uuid, disable_reidentify): version = rule.format except IDRule.DoesNotExist: job.print_error( - 'Error: No FPR identification rule for tool output "{}" found'.format( - output - ) + f'Error: No FPR identification rule for tool output "{output}" found' ) write_identification_event(file_uuid, command, success=False) return 255 except IDRule.MultipleObjectsReturned: job.print_error( - 'Error: Multiple FPR identification rules for tool output "{}" found'.format( - output - ) + f'Error: Multiple FPR identification rules for tool output "{output}" found' ) write_identification_event(file_uuid, command, success=False) return 255 diff --git a/src/MCPClient/lib/clientScripts/load_premis_events_from_xml.py b/src/MCPClient/lib/clientScripts/load_premis_events_from_xml.py index e81e168ee5..8c0b192e73 100644 --- a/src/MCPClient/lib/clientScripts/load_premis_events_from_xml.py +++ b/src/MCPClient/lib/clientScripts/load_premis_events_from_xml.py @@ -56,9 +56,7 @@ def log_invalid_xsd(path, exception, printfn=print): def log_invalid_events_xml(path, exception, printfn=print): printfn( - "Could not validate the events XML file {} using the PREMIS 3 XML schema".format( - path - ), + f"Could not validate the events XML file {path} using the PREMIS 3 XML schema", str(exception), file=sys.stderr, ) @@ -216,9 +214,7 @@ def log_event_id_change(event_id, new_event_id, printfn=print): def log_event_successfully_created(db_event, printfn=print): printfn( - "Imported PREMIS {} event and assigned identifier {}".format( - db_event.event_type, db_event.event_id - ) + f"Imported PREMIS {db_event.event_type} event and assigned identifier {db_event.event_id}" ) @@ -310,11 +306,7 @@ def get_elements(tree, selector, element_factory, printfn=print): def format_identifier_details(identifier, element_type): identifier_type = identifier[0] identifier_value = identifier[1] - return "premis:{element_type}IdentifierType='{identifier_type}' and premis:{element_type}IdentifierValue='{identifier_value}'".format( - identifier_type=identifier_type, - identifier_value=identifier_value, - element_type=element_type, - ) + return f"premis:{element_type}IdentifierType='{identifier_type}' and premis:{element_type}IdentifierValue='{identifier_value}'" def format_identifier(identifier, element_type): diff --git a/src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py b/src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py index f96da55893..b8f5f4b0a4 100755 --- a/src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py +++ b/src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py @@ -10,6 +10,7 @@ :param fileUUID: UUID of the preservation file. :param filePath: Path on disk of the preservation file. """ + import os import uuid @@ -17,15 +18,15 @@ from django.db import transaction django.setup() -from django.db.models import Q - -# dashboard -from main.models import Event, File - # archivematicaCommon import databaseFunctions import fileOperations from django.core.exceptions import ValidationError +from django.db.models import Q + +# dashboard +from main.models import Event +from main.models import File def main(job): @@ -93,11 +94,7 @@ def main(job): ) return 3 else: - job.print_error( - "Could not find {preservation_file} in {filename}".format( - preservation_file=preservation_file, filename=csv_path - ) - ) + job.print_error(f"Could not find {preservation_file} in {csv_path}") return 2 # If we found the original file, retrieve it from the DB original_file = File.objects.get( @@ -157,7 +154,7 @@ def main(job): e.save() job.print_output( "Updated the eventOutcomeDetailNote of an existing normalization" - " Event for file {}. Not creating a Derivation object".format(fileUUID) + f" Event for file {fileUUID}. Not creating a Derivation object" ) except (Event.DoesNotExist, ValidationError): # No normalization event was created in normalize.py - probably manually @@ -173,9 +170,7 @@ def main(job): eventOutcomeDetailNote=dstR, ) job.print_output( - "Created a manual normalization Event for file {}.".format( - original_file.uuid - ) + f"Created a manual normalization Event for file {original_file.uuid}." ) # Add linking information between files @@ -186,8 +181,8 @@ def main(job): relatedEventUUID=derivationEventUUID, ) job.print_output( - "Created a Derivation for original file {}, derived file {}, and" - " event {}".format(original_file.uuid, fileUUID, derivationEventUUID) + f"Created a Derivation for original file {original_file.uuid}, derived file {fileUUID}, and" + f" event {derivationEventUUID}" ) return 0 diff --git a/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py b/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py index 99a355fb56..cbecf6fd78 100755 --- a/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py +++ b/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py @@ -22,14 +22,13 @@ django.setup() # dashboard -from main.models import File +# --sipUUID "%SIPUUID%" --sipDirectory "%SIPDirectory%" --filePath "%relativeLocation%" +from optparse import OptionParser # archivematicaCommon import fileOperations - -# --sipUUID "%SIPUUID%" --sipDirectory "%SIPDirectory%" --filePath "%relativeLocation%" -from optparse import OptionParser from django.core.exceptions import ValidationError +from main.models import File def main(job): @@ -96,11 +95,7 @@ def main(job): ) return 3 else: - job.print_error( - "Could not find {access_file} in {filename}".format( - access_file=access_file, filename=csv_path - ) - ) + job.print_error(f"Could not find {access_file} in {csv_path}") return 2 # If we found the original file, retrieve it from the DB kwargs = { diff --git a/src/MCPClient/lib/clientScripts/manual_normalization_remove_mn_directories.py b/src/MCPClient/lib/clientScripts/manual_normalization_remove_mn_directories.py index f58bd7182a..27a6f39a40 100755 --- a/src/MCPClient/lib/clientScripts/manual_normalization_remove_mn_directories.py +++ b/src/MCPClient/lib/clientScripts/manual_normalization_remove_mn_directories.py @@ -21,14 +21,13 @@ import django django.setup() +# archivematicaCommon +import databaseFunctions from django.db import transaction # dashboard from main.models import File -# archivematicaCommon -import databaseFunctions - def recursivelyRemoveEmptyDirectories(job, dir): error_count = 0 @@ -81,9 +80,7 @@ def call(jobs): os.rmdir(manual_normalization_dir) except OSError as e: job.pyprint( - "{} could not be deleted: {}".format( - manual_normalization_dir, e.args - ), + f"{manual_normalization_dir} could not be deleted: {e.args}", file=sys.stderr, ) errorCount += 1 diff --git a/src/MCPClient/lib/clientScripts/move_sip.py b/src/MCPClient/lib/clientScripts/move_sip.py index 2d123311dc..2229716c50 100755 --- a/src/MCPClient/lib/clientScripts/move_sip.py +++ b/src/MCPClient/lib/clientScripts/move_sip.py @@ -24,12 +24,12 @@ django.setup() from django.db import transaction -# dashboard -from main.models import SIP - # archivematicaCommon from fileOperations import rename +# dashboard +from main.models import SIP + def updateDB(dst, sip_uuid): SIP.objects.filter(uuid=sip_uuid).update(currentpath=dst) diff --git a/src/MCPClient/lib/clientScripts/move_to_backlog.py b/src/MCPClient/lib/clientScripts/move_to_backlog.py index 49c1342f80..c9af819664 100755 --- a/src/MCPClient/lib/clientScripts/move_to_backlog.py +++ b/src/MCPClient/lib/clientScripts/move_to_backlog.py @@ -7,6 +7,7 @@ PREMIS events are created after this job runs as part of the workflow. """ + import multiprocessing import os import pprint @@ -16,20 +17,21 @@ import django django.setup() -from django.conf import settings as mcpclient_settings -from django.db import transaction -from django.db.models import Q - -from archivematicaFunctions import get_bag_size, get_setting -from custom_handlers import get_script_logger -from databaseFunctions import insertIntoEvents import elasticSearchFunctions -from main.models import Agent, File, UnitVariable +import metsrw import storageService as storage_service - +from archivematicaFunctions import get_bag_size +from archivematicaFunctions import get_setting from bagit import make_bag -import metsrw +from custom_handlers import get_script_logger +from databaseFunctions import insertIntoEvents +from django.conf import settings as mcpclient_settings from django.core.exceptions import ValidationError +from django.db import transaction +from django.db.models import Q +from main.models import Agent +from main.models import File +from main.models import UnitVariable logger = get_script_logger("archivematica.mcp.client.move_to_backlog") diff --git a/src/MCPClient/lib/clientScripts/move_transfer.py b/src/MCPClient/lib/clientScripts/move_transfer.py index 89eca78c16..709800313c 100755 --- a/src/MCPClient/lib/clientScripts/move_transfer.py +++ b/src/MCPClient/lib/clientScripts/move_transfer.py @@ -22,12 +22,12 @@ django.setup() from django.db import transaction -# dashboard -from main.models import Transfer - # archivematicaCommon from fileOperations import rename +# dashboard +from main.models import Transfer + def updateDB(dst, transferUUID): Transfer.objects.filter(uuid=transferUUID).update(currentlocation=dst) diff --git a/src/MCPClient/lib/clientScripts/normalize.py b/src/MCPClient/lib/clientScripts/normalize.py index 7b32373a19..de3d21d171 100755 --- a/src/MCPClient/lib/clientScripts/normalize.py +++ b/src/MCPClient/lib/clientScripts/normalize.py @@ -14,18 +14,20 @@ django.setup() # dashboard -from fpr.models import FPRule -from main.models import Derivation, FileFormatVersion, File, FileID -from django.db import transaction - # archivematicaCommon import databaseFunctions import fileOperations from dicts import ReplacementDict - from django.conf import settings as mcpclient_settings -from .lib import setup_dicts from django.core.exceptions import ValidationError +from django.db import transaction +from fpr.models import FPRule +from main.models import Derivation +from main.models import File +from main.models import FileFormatVersion +from main.models import FileID + +from .lib import setup_dicts # Return codes SUCCESS = 0 @@ -170,8 +172,8 @@ def check_manual_normalization(job, opts): # FIXME: SQL uses removedtime=0. Cannot get Django to express this job.print_output( "Checking for a manually normalized file by trying to get the" - " unique file that matches SIP UUID {} and whose currentlocation" - " value starts with this path: {}.".format(opts.sip_uuid, path) + f" unique file that matches SIP UUID {opts.sip_uuid} and whose currentlocation" + f" value starts with this path: {path}." ) matches = File.objects.filter( # removedtime = 0 sip=opts.sip_uuid, currentlocation__startswith=path @@ -221,9 +223,7 @@ def once_normalized(job, command, opts, replacement_dict): command.exit_code = -2 derivation_event_uuid = str(uuid.uuid4()) - event_detail_output = 'ArchivematicaFPRCommandID="{}"'.format( - command.fpcommand.uuid - ) + event_detail_output = f'ArchivematicaFPRCommandID="{command.fpcommand.uuid}"' if command.event_detail_command is not None: event_detail_output += f"; {command.event_detail_command.std_out}" for ef in transcoded_files: diff --git a/src/MCPClient/lib/clientScripts/normalize_report.py b/src/MCPClient/lib/clientScripts/normalize_report.py index b12b02f098..331c814c13 100755 --- a/src/MCPClient/lib/clientScripts/normalize_report.py +++ b/src/MCPClient/lib/clientScripts/normalize_report.py @@ -13,10 +13,10 @@ from django.db import transaction from django.template import Context from django.template import Template +from main.models import SIP from main.models import File from main.models import Job from main.models import Report -from main.models import SIP from main.models import Task django.setup() diff --git a/src/MCPClient/lib/clientScripts/parse_dataverse_mets.py b/src/MCPClient/lib/clientScripts/parse_dataverse_mets.py index 32f1a35923..e2c5b0e9e3 100644 --- a/src/MCPClient/lib/clientScripts/parse_dataverse_mets.py +++ b/src/MCPClient/lib/clientScripts/parse_dataverse_mets.py @@ -3,6 +3,7 @@ validate against the objects expected to be part of the SIP generated during transfer. """ + import json import os import uuid @@ -12,15 +13,16 @@ # databaseFunctions requires Django to be set up django.setup() -from django.utils import timezone +import databaseFunctions +import metsrw # archivematicaCommon from archivematicaFunctions import get_file_checksum from custom_handlers import get_script_logger -import databaseFunctions -from main.models import Agent, File -import metsrw from django.core.exceptions import ValidationError +from django.utils import timezone +from main.models import Agent +from main.models import File logger = get_script_logger("archivematica.mcp.client.parse_dataverse_mets") transfer_objects_directory = "%transferDirectory%objects" @@ -203,9 +205,7 @@ def create_db_entries(job, mapping, dataverse_agent_id): "Added derivation from", original_uuid, "to", file_entry.uuid ) except django.db.IntegrityError: - err_log = "Database integrity error, entry: {} for file {}".format( - file_entry.currentlocation, file_entry.originallocation - ) + err_log = f"Database integrity error, entry: {file_entry.currentlocation} for file {file_entry.originallocation}" raise ParseDataverseError(err_log) @@ -256,7 +256,7 @@ def verify_checksum( checksumtype = checksumtype.lower() generated_checksum = get_file_checksum(path, checksumtype) - event_detail = 'program="python"; ' 'module="hashlib.{}()"'.format(checksumtype) + event_detail = 'program="python"; ' f'module="hashlib.{checksumtype}()"' if checksum != generated_checksum: job.pyprint("Checksum failed") event_outcome = "Fail" diff --git a/src/MCPClient/lib/clientScripts/parse_mets_to_db.py b/src/MCPClient/lib/clientScripts/parse_mets_to_db.py index 61407cf5af..162d391946 100755 --- a/src/MCPClient/lib/clientScripts/parse_mets_to_db.py +++ b/src/MCPClient/lib/clientScripts/parse_mets_to_db.py @@ -10,13 +10,13 @@ django.setup() # dashboard -from main import models -from fpr import models as fpr_models +import databaseFunctions +import fileOperations # archivematicaCommon import namespaces as ns -import fileOperations -import databaseFunctions +from fpr import models as fpr_models +from main import models MD_TYPE_SIP_ID = "3e48343d-e2d2-4956-aaa3-b54d26eb9761" diff --git a/src/MCPClient/lib/clientScripts/pid_declaration.py b/src/MCPClient/lib/clientScripts/pid_declaration.py index 3fabcf3fa4..ae4b2a9b03 100644 --- a/src/MCPClient/lib/clientScripts/pid_declaration.py +++ b/src/MCPClient/lib/clientScripts/pid_declaration.py @@ -6,6 +6,7 @@ with the objects in the transfer, so be translated to PREMIS objects in the AIP METS. """ + import json import os import sys @@ -15,10 +16,11 @@ django.setup() -from main.models import Directory, File, SIP - from change_names import change_name from django.core.exceptions import ValidationError +from main.models import SIP +from main.models import Directory +from main.models import File class DeclarePIDsException(Exception): @@ -112,9 +114,7 @@ def _validate_identifier(self, mdl, id_): identifier = id_.get("identifier", None) if identifier_type is None: self.job.pyprint( - "None value returned for identifier type: {} on object: {}".format( - id_, mdl - ), + f"None value returned for identifier type: {id_} on object: {mdl}", file=sys.stderr, ) return (False,) @@ -175,9 +175,7 @@ def parse_and_attach_identifiers(self, unit_uuid, json_data): except (Directory.DoesNotExist, ValidationError): pass self.job.pyprint( - "{} identifiers added for {} objects in the package".format( - self.actual_count, self.identifier_count - ) + f"{self.actual_count} identifiers added for {self.identifier_count} objects in the package" ) def _retrieve_identifiers_path(self, unit_uuid, sip_directory): diff --git a/src/MCPClient/lib/clientScripts/policy_check.py b/src/MCPClient/lib/clientScripts/policy_check.py index f924694ff0..7e1414b730 100755 --- a/src/MCPClient/lib/clientScripts/policy_check.py +++ b/src/MCPClient/lib/clientScripts/policy_check.py @@ -10,6 +10,7 @@ [FILE_PATH] [FILE_UUID] [SIP_UUID] [SHARED_PATH] [FILE_TYPE] """ + import json import os @@ -17,16 +18,19 @@ from custom_handlers import get_script_logger django.setup() +import databaseFunctions +from dicts import replace_string_values from django.conf import settings as mcpclient_settings +from django.core.exceptions import ValidationError from django.db import transaction -from fpr.models import FPRule, FormatVersion -from main.models import Derivation, File, SIP, Transfer - from executeOrRunSubProcess import executeOrRun -import databaseFunctions -from dicts import replace_string_values +from fpr.models import FormatVersion +from fpr.models import FPRule from lib import setup_dicts -from django.core.exceptions import ValidationError +from main.models import SIP +from main.models import Derivation +from main.models import File +from main.models import Transfer # Note that linkTaskManagerFiles.py will take the highest exit code it has seen # from all tasks and will use that as the exit code of the job as a whole. @@ -84,7 +88,7 @@ def check(self): except (File.DoesNotExist, ValidationError): self.job.pyprint( "Not performing a policy check because there is no file" - " with UUID {}.".format(self.file_uuid) + f" with UUID {self.file_uuid}." ) return NOT_APPLICABLE_CODE if not self._we_check_this_type_of_file(): @@ -119,8 +123,8 @@ def _we_check_this_type_of_file(self): elif self.file_type == "preservation": if (not self._file_is_derivative()) or self._file_is_for_access(): self.job.pyprint( - "File {uuid} is not a preservation derivative; not" - " performing a policy check.".format(uuid=self.file_uuid) + f"File {self.file_uuid} is not a preservation derivative; not" + " performing a policy check." ) return False return True @@ -128,8 +132,8 @@ def _we_check_this_type_of_file(self): if self._file_is_derivative(for_access=True) and self._file_is_for_access(): return True self.job.pyprint( - "File {uuid} is not an access derivative; not performing" - " a policy check.".format(uuid=self.file_uuid) + f"File {self.file_uuid} is not an access derivative; not performing" + " a policy check." ) if not self._file_is_derivative(for_access=True): self.job.pyprint(f"File {self.file_uuid} is not a derivative.") @@ -196,11 +200,7 @@ def _get_manually_normalized_access_derivative_file_uuid(self): (see ``self._get_rules()``). """ manually_normalized_file_name = os.path.basename(self.file_path)[37:] - manually_normalized_file_path = ( - "%transferDirectory%objects/manualNormalization/access/{}".format( - manually_normalized_file_name - ) - ) + manually_normalized_file_path = f"%transferDirectory%objects/manualNormalization/access/{manually_normalized_file_name}" try: return File.objects.get( originallocation=manually_normalized_file_path.encode(), @@ -255,21 +255,17 @@ def _execute_rule_command(self, rule): self._save_to_logs_dir(output) if exitstatus == 0: self.job.pyprint( - "Command {} completed with output {}".format( - rule.command.description, stdout - ) + f"Command {rule.command.description} completed with output {stdout}" ) else: self.job.print_error( - "Command {} failed with exit status {}; stderr:".format( - rule.command.description, exitstatus - ), + f"Command {rule.command.description} failed with exit status {exitstatus}; stderr:", stderr, ) return "failed" event_detail = ( - 'program="{tool.description}";' - ' version="{tool.version}"'.format(tool=rule.command.tool) + f'program="{rule.command.tool.description}";' + f' version="{rule.command.tool.version}"' ) if output.get("eventOutcomeInformation") != "pass": self.job.print_error( @@ -283,9 +279,7 @@ def _execute_rule_command(self, rule): ) result = "failed" self.job.pyprint( - "Creating policy checking event for {} ({})".format( - self.file_path, self.file_uuid - ) + f"Creating policy checking event for {self.file_path} ({self.file_uuid})" ) # Manually-normalized access derivatives have no file UUID so we can't # create a validation event for them. TODO/QUESTION: should we use the @@ -380,10 +374,8 @@ def sip_logs_dir(self): ValidationError, ): self.job.print_error( - "Warning: unable to retrieve {unit_type} model corresponding" - " to {unit_type} UUID {sip_uuid}".format( - unit_type=unit_type, sip_uuid=self.sip_uuid - ) + f"Warning: unable to retrieve {unit_type} model corresponding" + f" to {unit_type} UUID {self.sip_uuid}" ) return None else: @@ -400,10 +392,8 @@ def sip_logs_dir(self): self._sip_logs_dir = logs_dir return logs_dir self.job.print_error( - "Warning: unable to find a logs/ directory in the {unit_type}" - " with UUID {sip_uuid}".format( - unit_type=unit_type, sip_uuid=self.sip_uuid - ) + f"Warning: unable to find a logs/ directory in the {unit_type}" + f" with UUID {self.sip_uuid}" ) return None @@ -419,8 +409,8 @@ def sip_subm_doc_dir(self): except (SIP.DoesNotExist, SIP.MultipleObjectsReturned, ValidationError): self.job.print_error( "Warning: unable to retrieve SIP model corresponding to SIP" - " UUID {} (when attempting to get the path to" - " metadata/submissionDocumentation/".format(self.sip_uuid) + f" UUID {self.sip_uuid} (when attempting to get the path to" + " metadata/submissionDocumentation/" ) return None else: @@ -433,7 +423,7 @@ def sip_subm_doc_dir(self): return subm_doc_dir self.job.print_error( "Warning: unable to find a metadata/submissionDocumentation/" - " directory in the SIP with UUID {}".format(self.sip_uuid) + f" directory in the SIP with UUID {self.sip_uuid}" ) return None diff --git a/src/MCPClient/lib/clientScripts/post_store_aip_hook.py b/src/MCPClient/lib/clientScripts/post_store_aip_hook.py index 4002c601b2..d5a10c1572 100755 --- a/src/MCPClient/lib/clientScripts/post_store_aip_hook.py +++ b/src/MCPClient/lib/clientScripts/post_store_aip_hook.py @@ -8,19 +8,18 @@ import requests django.setup() -from django.conf import settings as mcpclient_settings -from django.db import transaction - -# dashboard -from main import models - -# archivematicaCommon -from custom_handlers import get_script_logger import elasticSearchFunctions import storageService as storage_service from archivematicaFunctions import find_transfer_path_from_ingest +# archivematicaCommon +from custom_handlers import get_script_logger +from django.conf import settings as mcpclient_settings from django.core.exceptions import ValidationError +from django.db import transaction + +# dashboard +from main import models logger = get_script_logger("archivematica.mcp.client.post_store_aip_hook") diff --git a/src/MCPClient/lib/clientScripts/remove_unneeded_files.py b/src/MCPClient/lib/clientScripts/remove_unneeded_files.py index 53777bb42b..b554a1f25f 100755 --- a/src/MCPClient/lib/clientScripts/remove_unneeded_files.py +++ b/src/MCPClient/lib/clientScripts/remove_unneeded_files.py @@ -6,6 +6,7 @@ the file and its UUID. There is a default list of file names that are deleted; however, this can be overridden in MCPClient/clientConfig.conf s """ + import os import shutil @@ -17,7 +18,6 @@ django.setup() # archivematicaCommon from databaseFunctions import fileWasRemoved - from django.conf import settings as mcpclient_settings @@ -25,11 +25,7 @@ def remove_file(job, target_file, file_uuid): removableFiles = {e.strip() for e in mcpclient_settings.REMOVABLE_FILES.split(",")} basename = os.path.basename(target_file) if basename in removableFiles: - job.print_output( - "Removing {filename} (UUID: {uuid})".format( - uuid=file_uuid, filename=basename - ) - ) + job.print_output(f"Removing {basename} (UUID: {file_uuid})") try: os.remove(target_file) except OSError: diff --git a/src/MCPClient/lib/clientScripts/restructure_bag_aip_to_sip.py b/src/MCPClient/lib/clientScripts/restructure_bag_aip_to_sip.py index ad93a196f1..3caba7ef17 100755 --- a/src/MCPClient/lib/clientScripts/restructure_bag_aip_to_sip.py +++ b/src/MCPClient/lib/clientScripts/restructure_bag_aip_to_sip.py @@ -92,7 +92,7 @@ def call(jobs): else: logger.info( "Unable to reconstruct empty directories: no METS file" - " could be found in {}".format(sip_path) + f" could be found in {sip_path}" ) archivematicaFunctions.create_structured_directory( diff --git a/src/MCPClient/lib/clientScripts/restructure_for_compliance.py b/src/MCPClient/lib/clientScripts/restructure_for_compliance.py index f4a475226c..502393ce1a 100755 --- a/src/MCPClient/lib/clientScripts/restructure_for_compliance.py +++ b/src/MCPClient/lib/clientScripts/restructure_for_compliance.py @@ -22,20 +22,18 @@ import django django.setup() -from django.db import transaction -from main.models import Transfer, SIP +import bag +from archivematicaFunctions import OPTIONAL_FILES +from archivematicaFunctions import REQUIRED_DIRECTORIES # archivematicaCommon -from archivematicaFunctions import ( - create_structured_directory, - reconstruct_empty_directories, - REQUIRED_DIRECTORIES, - OPTIONAL_FILES, -) - +from archivematicaFunctions import create_structured_directory +from archivematicaFunctions import reconstruct_empty_directories from custom_handlers import get_script_logger -import bag from django.core.exceptions import ValidationError +from django.db import transaction +from main.models import SIP +from main.models import Transfer logger = get_script_logger("archivematica.mcp.client.restructureForCompliance") diff --git a/src/MCPClient/lib/clientScripts/restructure_for_compliance_sip.py b/src/MCPClient/lib/clientScripts/restructure_for_compliance_sip.py index 98410b9912..913a7da75e 100755 --- a/src/MCPClient/lib/clientScripts/restructure_for_compliance_sip.py +++ b/src/MCPClient/lib/clientScripts/restructure_for_compliance_sip.py @@ -8,12 +8,12 @@ # fileOperations requires Django to be set up django.setup() -from django.db import transaction - # archivematicaCommon import archivematicaFunctions -from archivematicaFunctions import REQUIRED_DIRECTORIES, OPTIONAL_FILES import fileOperations +from archivematicaFunctions import OPTIONAL_FILES +from archivematicaFunctions import REQUIRED_DIRECTORIES +from django.db import transaction def restructureForComplianceFileUUIDsAssigned( diff --git a/src/MCPClient/lib/clientScripts/retry_normalize_remove_normalized.py b/src/MCPClient/lib/clientScripts/retry_normalize_remove_normalized.py index d6b8fd1637..9519c8b069 100755 --- a/src/MCPClient/lib/clientScripts/retry_normalize_remove_normalized.py +++ b/src/MCPClient/lib/clientScripts/retry_normalize_remove_normalized.py @@ -27,7 +27,9 @@ # dashboard from django.db import transaction from django.utils import timezone -from main.models import Derivation, Event, File +from main.models import Derivation +from main.models import Event +from main.models import File def removeDIP(job, SIPDirectory, SIPUUID): diff --git a/src/MCPClient/lib/clientScripts/set_maildir_file_grp_use_and_file_ids.py b/src/MCPClient/lib/clientScripts/set_maildir_file_grp_use_and_file_ids.py index c909c26e59..118a4e948a 100755 --- a/src/MCPClient/lib/clientScripts/set_maildir_file_grp_use_and_file_ids.py +++ b/src/MCPClient/lib/clientScripts/set_maildir_file_grp_use_and_file_ids.py @@ -27,8 +27,9 @@ from django.db import transaction # dashboard -from main.models import File, FileID, FileFormatVersion - +from main.models import File +from main.models import FileFormatVersion +from main.models import FileID logger = get_script_logger("archivematica.mcp.client.setMaildirFileGrpUseAndFileIDs") @@ -40,19 +41,16 @@ def get_files(sip_uuid, current_location, removed_time=0): def insert_file_format_version(file_uuid, description): - sql = """ - INSERT INTO {file_format_version} (fileUUID, fileID) + sql = f""" + INSERT INTO {FileFormatVersion._meta.db_table} (fileUUID, fileID) VALUES (%s, ( SELECT pk - FROM {file_id} + FROM {FileID._meta.db_table} WHERE enabled = TRUE AND description = %s )); - """.format( - file_format_version=FileFormatVersion._meta.db_table, - file_id=FileID._meta.db_table, - ) + """ with connection.cursor() as cursor: return cursor.execute(sql, [file_uuid, description]) diff --git a/src/MCPClient/lib/clientScripts/set_transfer_type.py b/src/MCPClient/lib/clientScripts/set_transfer_type.py index 497b0842ab..162425497d 100755 --- a/src/MCPClient/lib/clientScripts/set_transfer_type.py +++ b/src/MCPClient/lib/clientScripts/set_transfer_type.py @@ -19,14 +19,12 @@ django.setup() # dashboard -from main.models import Transfer +from client import metrics # archivematicaCommon from custom_handlers import get_script_logger from django.db import transaction - -from client import metrics - +from main.models import Transfer logger = get_script_logger("archivematica.mcp.client.setTransferType") diff --git a/src/MCPClient/lib/clientScripts/store_aip.py b/src/MCPClient/lib/clientScripts/store_aip.py index df014b5cf1..a0ba4d9d60 100755 --- a/src/MCPClient/lib/clientScripts/store_aip.py +++ b/src/MCPClient/lib/clientScripts/store_aip.py @@ -25,18 +25,19 @@ # storageService requires Django to be set up django.setup() -from django.db import transaction -from metsrw.plugins import premisrw - -from main.models import UnitVariable, Event, Agent, DublinCore - -# archivematicaCommon -from custom_handlers import get_script_logger import storageService as storage_service from archivematicaFunctions import escape - from client import metrics + +# archivematicaCommon +from custom_handlers import get_script_logger from django.core.exceptions import ValidationError +from django.db import transaction +from main.models import Agent +from main.models import DublinCore +from main.models import Event +from main.models import UnitVariable +from metsrw.plugins import premisrw logger = get_script_logger("archivematica.mcp.client.storeAIP") @@ -155,9 +156,7 @@ def store_aip(job, aip_destination_uri, aip_path, sip_uuid, sip_name, sip_type): variablevalue=uuid, ) job.pyprint( - "Noting DIP UUID {} related to AIP so relationship can be created when AIP is stored.".format( - uuid - ) + f"Noting DIP UUID {uuid} related to AIP so relationship can be created when AIP is stored." ) else: uuid = sip_uuid diff --git a/src/MCPClient/lib/clientScripts/store_file_modification_dates.py b/src/MCPClient/lib/clientScripts/store_file_modification_dates.py index e24a88a5b1..581b44b2af 100755 --- a/src/MCPClient/lib/clientScripts/store_file_modification_dates.py +++ b/src/MCPClient/lib/clientScripts/store_file_modification_dates.py @@ -21,15 +21,13 @@ import django django.setup() +# archivematicaCommon +from custom_handlers import get_script_logger from django.db import transaction # dashboard from main import models -# archivematicaCommon -from custom_handlers import get_script_logger - - logger = get_script_logger("archivematica.mcp.client.storeFileModificationDates") diff --git a/src/MCPClient/lib/clientScripts/transcoder.py b/src/MCPClient/lib/clientScripts/transcoder.py index 487ad06865..32c61e742a 100755 --- a/src/MCPClient/lib/clientScripts/transcoder.py +++ b/src/MCPClient/lib/clientScripts/transcoder.py @@ -64,12 +64,7 @@ def __init__(self, job, command, replacement_dict, on_success=None, opts=None): ) def __str__(self): - return "[COMMAND] {}\n\tExecuting: {}\n\tCommand: {}\n\tOutput location: {}\n".format( - self.fpcommand, - self.command, - self.verification_command, - self.output_location, - ) + return f"[COMMAND] {self.fpcommand}\n\tExecuting: {self.command}\n\tCommand: {self.verification_command}\n\tOutput location: {self.output_location}\n" def execute(self, skip_on_success=False): """Execute the the command, and associated verification and event detail commands. @@ -135,8 +130,8 @@ def __init__(self, job, fprule, command, replacement_dict, opts, on_success): ) def __str__(self): - return "[Command Linker] FPRule: {fprule} Command: {co}".format( - fprule=self.fprule.uuid, co=self.commandObject + return ( + f"[Command Linker] FPRule: {self.fprule.uuid} Command: {self.commandObject}" ) def execute(self): diff --git a/src/MCPClient/lib/clientScripts/transcribe_file.py b/src/MCPClient/lib/clientScripts/transcribe_file.py index 0f33d69013..2bd9b78a09 100755 --- a/src/MCPClient/lib/clientScripts/transcribe_file.py +++ b/src/MCPClient/lib/clientScripts/transcribe_file.py @@ -8,19 +8,20 @@ django.setup() # dashboard -from django.utils import timezone -from main.models import Derivation, File, FileFormatVersion -from fpr.models import FPRule - -# archivematicaCommon -from dicts import ReplacementDict -from executeOrRunSubProcess import executeOrRun import databaseFunctions import fileOperations +# archivematicaCommon +from dicts import ReplacementDict from django.conf import settings as mcpclient_settings -from lib import setup_dicts from django.core.exceptions import ValidationError +from django.utils import timezone +from executeOrRunSubProcess import executeOrRun +from fpr.models import FPRule +from lib import setup_dicts +from main.models import Derivation +from main.models import File +from main.models import FileFormatVersion def concurrent_instances(): @@ -124,9 +125,7 @@ def main(job, task_uuid, file_uuid): if not rules: job.print_error( - "No rules found for file {} and its derivatives; not transcribing".format( - file_uuid - ) + f"No rules found for file {file_uuid} and its derivatives; not transcribing" ) return 0 else: diff --git a/src/MCPClient/lib/clientScripts/trim_create_rights_entries.py b/src/MCPClient/lib/clientScripts/trim_create_rights_entries.py index e42223083f..360956bf07 100755 --- a/src/MCPClient/lib/clientScripts/trim_create_rights_entries.py +++ b/src/MCPClient/lib/clientScripts/trim_create_rights_entries.py @@ -30,19 +30,17 @@ django.setup() from django.db import transaction -# dashboard -from main.models import ( - RightsStatement, - RightsStatementOtherRightsInformation, - RightsStatementOtherRightsDocumentationIdentifier, - RightsStatementRightsGranted, - RightsStatementRightsGrantedNote, - RightsStatementRightsGrantedRestriction, -) - # archivematicaCommon from fileOperations import getFileUUIDLike +# dashboard +from main.models import RightsStatement +from main.models import RightsStatementOtherRightsDocumentationIdentifier +from main.models import RightsStatementOtherRightsInformation +from main.models import RightsStatementRightsGranted +from main.models import RightsStatementRightsGrantedNote +from main.models import RightsStatementRightsGrantedRestriction + while False: import time diff --git a/src/MCPClient/lib/clientScripts/trim_restructure_for_compliance.py b/src/MCPClient/lib/clientScripts/trim_restructure_for_compliance.py index 0b8f7fbf27..0a0ee000c9 100755 --- a/src/MCPClient/lib/clientScripts/trim_restructure_for_compliance.py +++ b/src/MCPClient/lib/clientScripts/trim_restructure_for_compliance.py @@ -22,12 +22,11 @@ # fileOperations requires Django to be set up django.setup() -from django.db import transaction - # archivematicaCommon import archivematicaFunctions -from archivematicaFunctions import REQUIRED_DIRECTORIES import fileOperations +from archivematicaFunctions import REQUIRED_DIRECTORIES +from django.db import transaction def restructureTRIMForComplianceFileUUIDsAssigned( diff --git a/src/MCPClient/lib/clientScripts/trim_verify_checksums.py b/src/MCPClient/lib/clientScripts/trim_verify_checksums.py index 71bf441bbf..02f27a3053 100755 --- a/src/MCPClient/lib/clientScripts/trim_verify_checksums.py +++ b/src/MCPClient/lib/clientScripts/trim_verify_checksums.py @@ -25,12 +25,12 @@ # fileOperations, databaseFunctions requires Django to be set up django.setup() -from django.db import transaction +import databaseFunctions # archivematicaCommon from archivematicaFunctions import get_file_checksum +from django.db import transaction from fileOperations import getFileUUIDLike -import databaseFunctions def call(jobs): diff --git a/src/MCPClient/lib/clientScripts/trim_verify_manifest.py b/src/MCPClient/lib/clientScripts/trim_verify_manifest.py index 1853fdd674..09bbebdde6 100755 --- a/src/MCPClient/lib/clientScripts/trim_verify_manifest.py +++ b/src/MCPClient/lib/clientScripts/trim_verify_manifest.py @@ -25,11 +25,11 @@ # fileOperations, databaseFunctions requires Django to be set up django.setup() +import databaseFunctions from django.db import transaction # archivematicaCommon from fileOperations import getFileUUIDLike -import databaseFunctions def call(jobs): diff --git a/src/MCPClient/lib/clientScripts/update_size_and_checksum.py b/src/MCPClient/lib/clientScripts/update_size_and_checksum.py index 51544957f9..a8edfcef96 100755 --- a/src/MCPClient/lib/clientScripts/update_size_and_checksum.py +++ b/src/MCPClient/lib/clientScripts/update_size_and_checksum.py @@ -26,16 +26,15 @@ django.setup() -from main.models import File, FileFormatVersion - +import metsrw +import parse_mets_to_db from archivematicaFunctions import find_mets_file from custom_handlers import get_script_logger from databaseFunctions import insertIntoDerivations -from fileOperations import get_size_and_checksum, updateSizeAndChecksum - -import metsrw - -import parse_mets_to_db +from fileOperations import get_size_and_checksum +from fileOperations import updateSizeAndChecksum +from main.models import File +from main.models import FileFormatVersion logger = get_script_logger("archivematica.mcp.client.updateSizeAndChecksum") diff --git a/src/MCPClient/lib/clientScripts/upload_archivesspace.py b/src/MCPClient/lib/clientScripts/upload_archivesspace.py index 5723e9ad56..d6328f69fe 100755 --- a/src/MCPClient/lib/clientScripts/upload_archivesspace.py +++ b/src/MCPClient/lib/clientScripts/upload_archivesspace.py @@ -16,8 +16,8 @@ # initialize Django (required for Django 1.7) django.setup() -from django.db import transaction from django.core.exceptions import ValidationError +from django.db import transaction logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -234,9 +234,7 @@ def upload_to_archivesspace( ) except ArchivesSpaceError as error: logger.error( - "Could not upload {} to ArchivesSpace record {}. Error: {}".format( - file_name, as_resource, str(error) - ) + f"Could not upload {file_name} to ArchivesSpace record {as_resource}. Error: {str(error)}" ) all_files_paired_successfully = False diff --git a/src/MCPClient/lib/clientScripts/upload_qubit.py b/src/MCPClient/lib/clientScripts/upload_qubit.py index 2b0715d0a7..84bc84fb2e 100755 --- a/src/MCPClient/lib/clientScripts/upload_qubit.py +++ b/src/MCPClient/lib/clientScripts/upload_qubit.py @@ -32,13 +32,11 @@ # externals django.setup() -from django.conf import settings as mcpclient_settings -from django.db import transaction - # dashboard import main.models as models - +from django.conf import settings as mcpclient_settings from django.core.exceptions import ValidationError +from django.db import transaction # moved after django.setup() logger = get_script_logger("archivematica.upload.qubit") @@ -215,11 +213,7 @@ def start(job, data): # Build URL (expected sth like http://localhost/ica-atom/index.php) atom_url_prefix = ";" if data.version == 1 else "" - deposit_url = "{}/{}sword/deposit/{}".format( - data.url, - atom_url_prefix, - target, - ) + deposit_url = f"{data.url}/{atom_url_prefix}sword/deposit/{target}" # Auth and request! log("About to deposit to: %s" % data.url) diff --git a/src/MCPClient/lib/clientScripts/validate_file.py b/src/MCPClient/lib/clientScripts/validate_file.py index 13898fadea..5214554ec6 100755 --- a/src/MCPClient/lib/clientScripts/validate_file.py +++ b/src/MCPClient/lib/clientScripts/validate_file.py @@ -14,6 +14,7 @@ [FILE_PATH] [FILE_UUID] [SIP_UUID] [SHARED_PATH] [FILE_TYPE] """ + import ast import os import sys @@ -24,17 +25,17 @@ from django.db import transaction django.setup() -from fpr.models import FPRule, FormatVersion -from main.models import Derivation, File, SIP - -from custom_handlers import get_script_logger import databaseFunctions -from executeOrRunSubProcess import executeOrRun +from custom_handlers import get_script_logger from dicts import replace_string_values - from django.conf import settings as mcpclient_settings +from executeOrRunSubProcess import executeOrRun +from fpr.models import FormatVersion +from fpr.models import FPRule from lib import setup_dicts - +from main.models import SIP +from main.models import Derivation +from main.models import File SUCCESS_CODE = 0 FAIL_CODE = 1 @@ -87,9 +88,7 @@ def validate(self): """ if self.file_type in DERIVATIVE_TYPES and not self._file_is_derivative(): self.job.print_output( - "File {uuid} {not_derivative_msg}; not validating.".format( - uuid=self.file_uuid, not_derivative_msg=self._not_derivative_msg() - ) + f"File {self.file_uuid} {self._not_derivative_msg()}; not validating." ) return NOT_DERIVATIVE_CODE rules = self._get_rules() @@ -143,12 +142,8 @@ def _execute_rule_command(self, rule): ) if exitstatus != 0: self.job.print_error( - "Command {description} failed with exit status {status};" - " stderr: {stderr}".format( - description=rule.command.description, - status=exitstatus, - stderr=stderr, - ) + f"Command {rule.command.description} failed with exit status {exitstatus};" + f" stderr: {stderr}" ) return "failed" # Parse output and generate an Event @@ -156,8 +151,8 @@ def _execute_rule_command(self, rule): # insecure practice; should be JSON. output = ast.literal_eval(stdout) event_detail = ( - 'program="{tool.description}";' - ' version="{tool.version}"'.format(tool=rule.command.tool) + f'program="{rule.command.tool.description}";' + f' version="{rule.command.tool.version}"' ) # If the FPR command has not errored but the actual validation # determined that the file is not valid, then we want to both create a @@ -176,19 +171,15 @@ def _execute_rule_command(self, rule): ) else: self.job.pyprint( - "Command {cmd_description} indicated failure with this" - " output:\n\n{output}".format( - cmd_description=rule.command.description, output=pformat(stdout) - ), + f"Command {rule.command.description} indicated failure with this" + f" output:\n\n{pformat(stdout)}", file=sys.stderr, ) result = "failed" if self.file_type == "preservation": self._save_stdout_to_logs_dir(output) self.job.print_output( - "Creating {purpose} event for {file_path} ({file_uuid})".format( - purpose=self.purpose, file_path=self.file_path, file_uuid=self.file_uuid - ) + f"Creating {self.purpose} event for {self.file_path} ({self.file_uuid})" ) databaseFunctions.insertIntoEvents( fileUUID=self.file_uuid, @@ -268,7 +259,7 @@ def sip_logs_dir(self): except (SIP.DoesNotExist, SIP.MultipleObjectsReturned, ValidationError): self.job.print_error( "Warning: unable to retrieve SIP model corresponding to SIP" - " UUID {}".format(self.sip_uuid) + f" UUID {self.sip_uuid}" ) return None else: @@ -281,7 +272,7 @@ def sip_logs_dir(self): return logs_dir self.job.print_error( "Warning: unable to find a logs/ directory in the SIP" - " with UUID {}".format(self.sip_uuid) + f" with UUID {self.sip_uuid}" ) return None diff --git a/src/MCPClient/lib/clientScripts/verify_aip.py b/src/MCPClient/lib/clientScripts/verify_aip.py index ce589a8e69..594aa383ab 100755 --- a/src/MCPClient/lib/clientScripts/verify_aip.py +++ b/src/MCPClient/lib/clientScripts/verify_aip.py @@ -7,17 +7,17 @@ import django django.setup() -from django.conf import settings as mcpclient_settings -from django.db import transaction +import databaseFunctions # archivematicaCommon from archivematicaFunctions import get_setting -import databaseFunctions +from bagit import Bag +from bagit import BagError +from django.conf import settings as mcpclient_settings +from django.db import transaction from executeOrRunSubProcess import executeOrRun - -from main.models import File, SIP - -from bagit import Bag, BagError +from main.models import SIP +from main.models import File class VerifyChecksumsError(Exception): @@ -50,9 +50,7 @@ def write_premis_event( databaseFunctions.insertIntoEvents( fileUUID=sip_uuid, eventType="fixity check", - eventDetail='program="python, bag"; module="hashlib.{}()"'.format( - checksum_type - ), + eventDetail=f'program="python, bag"; module="hashlib.{checksum_type}()"', eventOutcome=event_outcome, eventOutcomeDetailNote=event_outcome_detail_note, ) @@ -68,13 +66,9 @@ def assert_checksum_types_match(job, file_, sip_uuid, settings_checksum_type): """ if file_.checksumtype != settings_checksum_type: event_outcome_detail_note = ( - "The checksum type of file {file_uuid} is" - " {file_checksum_type}; given the current application settings, we" - " expect it to {settings_checksum_type}".format( - file_uuid=file_.uuid, - file_checksum_type=file_.checksumtype, - settings_checksum_type=settings_checksum_type, - ) + f"The checksum type of file {file_.uuid} is" + f" {file_.checksumtype}; given the current application settings, we" + f" expect it to {settings_checksum_type}" ) raise VerifyChecksumsError( write_premis_event( @@ -95,13 +89,9 @@ def get_expected_checksum( if is_reingest: return None event_outcome_detail_note = ( - "Unable to find expected path {expected_path} for file" - " {file_uuid} in the following mapping from file paths to" - " checksums: {mapping}:".format( - expected_path=file_path, - file_uuid=file_.uuid, - mapping=pformat(bag.entries), - ) + f"Unable to find expected path {file_path} for file" + f" {file_.uuid} in the following mapping from file paths to" + f" checksums: {pformat(bag.entries)}:" ) raise VerifyChecksumsError( write_premis_event( @@ -114,13 +104,9 @@ def assert_checksums_match(job, file_, sip_uuid, checksum_type, expected_checksu """Raise an exception if checksums do not match.""" if file_.checksum != expected_checksum: event_outcome_detail_note = ( - "The checksum {db_checksum} for file {file_uuid} from the" + f"The checksum {file_.checksum} for file {file_.uuid} from the" " database did not match the corresponding checksum from the" - " Bag manifest file {manifest_checksum}".format( - file_uuid=file_.uuid, - db_checksum=file_.checksum, - manifest_checksum=expected_checksum, - ) + f" Bag manifest file {expected_checksum}" ) raise VerifyChecksumsError( write_premis_event( @@ -170,18 +156,14 @@ def verify_checksums(job, bag, sip_uuid): job.print_error(repr(err)) raise event_outcome_detail_note = ( - "All checksums (count={verification_count}) generated at start of" - " transfer match those generated by BagIt (bag).".format( - verification_count=verification_count - ) + f"All checksums (count={verification_count}) generated at start of" + " transfer match those generated by BagIt (bag)." ) if verification_skipped_because_reingest: event_outcome_detail_note += ( - " Note that checksum verification was skipped for {skipped_count}" + f" Note that checksum verification was skipped for {verification_skipped_because_reingest}" " file(s) because this AIP is being re-ingested and the re-ingest" - " payload did not include said file(s).".format( - skipped_count=verification_skipped_because_reingest - ) + " payload did not include said file(s)." ) write_premis_event(job, sip_uuid, checksum_type, "Pass", event_outcome_detail_note) job.pyprint(event_outcome_detail_note) @@ -245,9 +227,9 @@ def verify_aip(job): shutil.rmtree(extract_dir) except OSError as err: job.pyprint( - "Failed to remove temporary directory at {extract_dir} which" + f"Failed to remove temporary directory at {extract_dir} which" " contains the AIP extracted for verification." - " Error:\n{err}".format(extract_dir=extract_dir, err=err), + f" Error:\n{err}", file=sys.stderr, ) diff --git a/src/MCPClient/lib/clientScripts/verify_and_restructure_transfer_bag.py b/src/MCPClient/lib/clientScripts/verify_and_restructure_transfer_bag.py index f6c2e071ad..8e37cda6fa 100755 --- a/src/MCPClient/lib/clientScripts/verify_and_restructure_transfer_bag.py +++ b/src/MCPClient/lib/clientScripts/verify_and_restructure_transfer_bag.py @@ -23,14 +23,14 @@ django.setup() # dashboard -from main.models import File - -# archivematicaCommon -from archivematicaFunctions import REQUIRED_DIRECTORIES, OPTIONAL_FILES import bag import fileOperations -from databaseFunctions import insertIntoEvents +from archivematicaFunctions import OPTIONAL_FILES +# archivematicaCommon +from archivematicaFunctions import REQUIRED_DIRECTORIES +from databaseFunctions import insertIntoEvents +from main.models import File from move_or_merge import move_or_merge diff --git a/src/MCPClient/lib/clientScripts/verify_checksum.py b/src/MCPClient/lib/clientScripts/verify_checksum.py index 20b9f562a3..8e4efed629 100755 --- a/src/MCPClient/lib/clientScripts/verify_checksum.py +++ b/src/MCPClient/lib/clientScripts/verify_checksum.py @@ -26,6 +26,7 @@ * SHA256 * SHA512 """ + import datetime import os import subprocess @@ -36,10 +37,10 @@ from django.db import transaction django.setup() -from main.models import Event, File, Transfer - from custom_handlers import get_script_logger - +from main.models import Event +from main.models import File +from main.models import Transfer logger = get_script_logger("archivematica.mcp.client.verify_checksum") @@ -111,8 +112,8 @@ def count_and_compare_lines(self, objects_dir): if lines == objects: return True self.job.pyprint( - "{}: Comparison failed with {} checksum lines and {} " - "transfer files".format(self.get_ext(self.hashfile), lines, objects), + f"{self.get_ext(self.hashfile)}: Comparison failed with {lines} checksum lines and {objects} " + "transfer files", file=sys.stderr, ) return False @@ -127,9 +128,7 @@ def compare_hashes(self, transfer_dir): return 0 except subprocess.CalledProcessError as err: if self.EXIT_NON_ZERO in str(err): - warn = "{}: comparison exited with status: {}. Please check the formatting of the checksums or integrity of the files.".format( - self.get_ext(self.hashfile), err.returncode - ) + warn = f"{self.get_ext(self.hashfile)}: comparison exited with status: {err.returncode}. Please check the formatting of the checksums or integrity of the files." self.job.pyprint(warn, file=sys.stderr) for line in self._decode(err.output): if not line: @@ -209,9 +208,7 @@ def get_file_queryset(transfer_uuid): **{"removedtime__isnull": True, "transfer_id": transfer_uuid} ) if not file_objs_queryset.exists(): - err_str = "Unable to find the transfer objects for the SIP: '{}' in the database".format( - transfer_uuid - ) + err_str = f"Unable to find the transfer objects for the SIP: '{transfer_uuid}' in the database" raise PREMISFailure(err_str) return file_objs_queryset @@ -266,16 +263,12 @@ def run_hashsum_commands(job): hashsum = Hashsum(hashfilepath, job) except NoHashCommandAvailable: job.pyprint( - "Nothing to do for {}. No command available.".format( - Hashsum.get_ext(hashfilepath) - ) + f"Nothing to do for {Hashsum.get_ext(hashfilepath)}. No command available." ) continue if hashsum: job.pyprint( - "Comparing transfer checksums with the supplied {} file".format( - Hashsum.get_ext(hashfilepath) - ), + f"Comparing transfer checksums with the supplied {Hashsum.get_ext(hashfilepath)} file", file=sys.stderr, ) result = hashsum.compare_hashes(transfer_dir=transfer_dir) diff --git a/src/MCPClient/lib/clientScripts/verify_mets.py b/src/MCPClient/lib/clientScripts/verify_mets.py index 50ba00eca6..ef61eae2dc 100755 --- a/src/MCPClient/lib/clientScripts/verify_mets.py +++ b/src/MCPClient/lib/clientScripts/verify_mets.py @@ -5,6 +5,7 @@ far is to verify the validity of custom structmaps included with transfers and supplied on ingest after appraisal. """ + import os from lxml import etree diff --git a/src/MCPClient/lib/clientScripts/verify_sip_compliance.py b/src/MCPClient/lib/clientScripts/verify_sip_compliance.py index b61663811a..eb9dd43df9 100755 --- a/src/MCPClient/lib/clientScripts/verify_sip_compliance.py +++ b/src/MCPClient/lib/clientScripts/verify_sip_compliance.py @@ -20,7 +20,6 @@ from client import metrics - REQUIRED_DIRECTORIES = ( "objects", "logs", diff --git a/src/MCPClient/lib/settings/common.py b/src/MCPClient/lib/settings/common.py index 57ae4505ea..ed7e716cc7 100644 --- a/src/MCPClient/lib/settings/common.py +++ b/src/MCPClient/lib/settings/common.py @@ -415,8 +415,6 @@ def workers(config, section): XML_VALIDATION_FAIL_ON_ERROR, bool ): raise ImproperlyConfigured( - "The metadata XML validation settings file {} does not contain " - "the right settings: an XML_VALIDATION dictionary and an XML_VALIDATION_FAIL_ON_ERROR boolean".format( - METADATA_XML_VALIDATION_SETTINGS_FILE - ) + f"The metadata XML validation settings file {METADATA_XML_VALIDATION_SETTINGS_FILE} does not contain " + "the right settings: an XML_VALIDATION dictionary and an XML_VALIDATION_FAIL_ON_ERROR boolean" ) diff --git a/src/MCPClient/lib/settings/test.py b/src/MCPClient/lib/settings/test.py index d749fa5699..96ad923c20 100644 --- a/src/MCPClient/lib/settings/test.py +++ b/src/MCPClient/lib/settings/test.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . """Test settings and globals.""" + from .common import * DATABASES = { diff --git a/src/MCPServer/lib/server/jobs/__init__.py b/src/MCPServer/lib/server/jobs/__init__.py index c08befcef3..6e515bf25c 100644 --- a/src/MCPServer/lib/server/jobs/__init__.py +++ b/src/MCPServer/lib/server/jobs/__init__.py @@ -12,6 +12,7 @@ * `DecisionJob`, handling workflow decision points * `LocalJob`, handling work done directly on MCPServer """ + from server.jobs.base import Job from server.jobs.chain import JobChain from server.jobs.client import ClientScriptJob diff --git a/src/MCPServer/lib/server/jobs/base.py b/src/MCPServer/lib/server/jobs/base.py index f19703ebb4..2c7249571c 100644 --- a/src/MCPServer/lib/server/jobs/base.py +++ b/src/MCPServer/lib/server/jobs/base.py @@ -1,6 +1,7 @@ """ A base class for other Job types to inherit from. """ + import abc import logging import uuid @@ -9,7 +10,6 @@ from django.utils import timezone from main import models - logger = logging.getLogger("archivematica.mcp.server.jobs") diff --git a/src/MCPServer/lib/server/jobs/chain.py b/src/MCPServer/lib/server/jobs/chain.py index 81a927d8b7..fd5ac0c741 100644 --- a/src/MCPServer/lib/server/jobs/chain.py +++ b/src/MCPServer/lib/server/jobs/chain.py @@ -8,6 +8,7 @@ This module knows about all `Job` subclasses, and will instantiate the correct one by looking at the workflow. """ + import logging from server.jobs.client import DirectoryClientScriptJob @@ -19,7 +20,6 @@ from server.jobs.local import GetUnitVarLinkJob from server.jobs.local import SetUnitVarLinkJob - logger = logging.getLogger("archivematica.mcp.server.jobs.chain") diff --git a/src/MCPServer/lib/server/jobs/client.py b/src/MCPServer/lib/server/jobs/client.py index 02fb9d2a08..6ea2f72d95 100644 --- a/src/MCPServer/lib/server/jobs/client.py +++ b/src/MCPServer/lib/server/jobs/client.py @@ -1,6 +1,7 @@ """ Jobs remotely executed by on MCP client. """ + import abc import ast import json @@ -10,10 +11,11 @@ from django.conf import settings from django.core.exceptions import ValidationError from main import models + from server import metrics from server.jobs.base import Job -from server.tasks import get_task_backend from server.tasks import Task +from server.tasks import get_task_backend logger = logging.getLogger("archivematica.mcp.server.jobs.client") diff --git a/src/MCPServer/lib/server/jobs/decisions.py b/src/MCPServer/lib/server/jobs/decisions.py index 0e6533f61b..d17e728eb2 100644 --- a/src/MCPServer/lib/server/jobs/decisions.py +++ b/src/MCPServer/lib/server/jobs/decisions.py @@ -1,6 +1,7 @@ """ Jobs relating to user decisions. """ + import abc import logging import threading @@ -8,13 +9,13 @@ from dbconns import auto_close_old_connections from main import models + from server.jobs.base import Job from server.processing_config import load_preconfigured_choice from server.processing_config import load_processing_xml from server.translation import TranslationLabel from server.workflow_abilities import choice_is_available - logger = logging.getLogger("archivematica.mcp.server.jobs.decisions") diff --git a/src/MCPServer/lib/server/jobs/local.py b/src/MCPServer/lib/server/jobs/local.py index cdcd849a1b..c9d869c461 100644 --- a/src/MCPServer/lib/server/jobs/local.py +++ b/src/MCPServer/lib/server/jobs/local.py @@ -1,12 +1,14 @@ """ Jobs executed locally in MCP server. """ + import abc import logging from dbconns import auto_close_old_connections from django.core.exceptions import ValidationError from main import models + from server.jobs.base import Job logger = logging.getLogger("archivematica.mcp.server.jobs.local") diff --git a/src/MCPServer/lib/server/mcp.py b/src/MCPServer/lib/server/mcp.py index d573f0c939..5c8808165a 100644 --- a/src/MCPServer/lib/server/mcp.py +++ b/src/MCPServer/lib/server/mcp.py @@ -19,6 +19,7 @@ watched dirs as set in the workflow. 10. The `PackageQueue.work` processing loop is started on the main thread. """ + import concurrent.futures import functools import getpass @@ -33,15 +34,20 @@ from django.conf import settings -from server import metrics, rpc_server, shared_dirs -from server.jobs import Job, JobChain -from server.packages import Package, DIP, Transfer, SIP +from server import metrics +from server import rpc_server +from server import shared_dirs +from server.jobs import Job +from server.jobs import JobChain +from server.packages import DIP +from server.packages import SIP +from server.packages import Package +from server.packages import Transfer from server.queues import PackageQueue from server.tasks import Task from server.watch_dirs import watch_directories from server.workflow import load_workflow - logger = logging.getLogger("archivematica.mcp.server") diff --git a/src/MCPServer/lib/server/metrics.py b/src/MCPServer/lib/server/metrics.py index ae1bd90499..e328f1229e 100644 --- a/src/MCPServer/lib/server/metrics.py +++ b/src/MCPServer/lib/server/metrics.py @@ -1,6 +1,7 @@ """ Exposes various metrics via Prometheus. """ + import functools import os @@ -13,7 +14,6 @@ from prometheus_client import start_http_server from version import get_full_version - gearman_active_jobs_gauge = Gauge( "mcpserver_gearman_active_jobs", "Number of gearman jobs currently being processed" ) diff --git a/src/MCPServer/lib/server/packages.py b/src/MCPServer/lib/server/packages.py index d33d15b99c..80f4d3a090 100644 --- a/src/MCPServer/lib/server/packages.py +++ b/src/MCPServer/lib/server/packages.py @@ -1,4 +1,5 @@ """Package management.""" + import abc import ast import collections @@ -15,6 +16,7 @@ from django.core.exceptions import ValidationError from django.utils import timezone from main import models + from server.jobs import JobChain from server.processing_config import processing_configuration_file_exists from server.utils import uuid_from_path @@ -487,12 +489,7 @@ def __init__(self, path, sep=":"): self.path = parts[2] def __repr__(self): - return "{} (uuid={!r}, sep={!r}, path={!r})".format( - self.__class__, - self.uuid, - self.sep, - self.path, - ) + return f"{self.__class__} (uuid={self.uuid!r}, sep={self.sep!r}, path={self.path!r})" def parts(self): return self.uuid, self.path @@ -541,11 +538,7 @@ def __init__(self, current_path, uuid): self.uuid = uuid def __repr__(self): - return '{class_name}("{current_path}", {uuid})'.format( - class_name=self.__class__.__name__, - uuid=self.uuid, - current_path=self.current_path, - ) + return f'{self.__class__.__name__}("{self.current_path}", {self.uuid})' @classmethod @auto_close_old_connections() diff --git a/src/MCPServer/lib/server/processing_config.py b/src/MCPServer/lib/server/processing_config.py index 5ad8137849..b896b7e7ca 100644 --- a/src/MCPServer/lib/server/processing_config.py +++ b/src/MCPServer/lib/server/processing_config.py @@ -4,6 +4,7 @@ ability to establish predefined choices via the user interface, and handles processing config file operations. """ + import abc import logging import os @@ -11,8 +12,8 @@ import storageService as storage_service from django.conf import settings from lxml import etree -from server.workflow_abilities import choice_is_available +from server.workflow_abilities import choice_is_available logger = logging.getLogger("archivematica.mcp.server.processing_config") diff --git a/src/MCPServer/lib/server/queues.py b/src/MCPServer/lib/server/queues.py index 9afcf92264..a68d692098 100644 --- a/src/MCPServer/lib/server/queues.py +++ b/src/MCPServer/lib/server/queues.py @@ -1,18 +1,19 @@ """ The PackageQueue class handles job queueing, as it relates to packages. """ + import functools import logging import queue as Queue import threading from django.conf import settings + from server import metrics from server.jobs import DecisionJob from server.packages import DIP from server.packages import SIP - logger = logging.getLogger("archivematica.mcp.server.queues") diff --git a/src/MCPServer/lib/server/rpc_server.py b/src/MCPServer/lib/server/rpc_server.py index 70e0cd4868..714921acd4 100644 --- a/src/MCPServer/lib/server/rpc_server.py +++ b/src/MCPServer/lib/server/rpc_server.py @@ -6,6 +6,7 @@ need it, but it needs to be tested further. The main thing to check is whether the client is ready to handle application-level exceptions. """ + import calendar import configparser import inspect @@ -23,14 +24,14 @@ from gearman import GearmanWorker from gearman_encoder import JSONDataEncoder from lxml import etree -from main.models import Job from main.models import SIP +from main.models import Job from main.models import Transfer + from server.packages import create_package from server.packages import get_approve_transfer_chain_id from server.processing_config import get_processing_fields - logger = logging.getLogger("archivematica.mcp.server.rpc_server") @@ -309,7 +310,7 @@ def _approve_partial_reingest_handler(self, worker, job, payload): if not job: # No job to be found. raise NotFoundError( 'There is no "Reingest AIP" job awaiting a' - " decision for SIP {}".format(sip_uuid) + f" decision for SIP {sip_uuid}" ) not_found = NotFoundError("Could not find choice for approve AIP reingest") try: diff --git a/src/MCPServer/lib/server/shared_dirs.py b/src/MCPServer/lib/server/shared_dirs.py index 3213232d07..ffbb365203 100644 --- a/src/MCPServer/lib/server/shared_dirs.py +++ b/src/MCPServer/lib/server/shared_dirs.py @@ -1,5 +1,5 @@ -"""Shared directory setup. -""" +"""Shared directory setup.""" + import logging import os @@ -7,7 +7,6 @@ from processing import BUILTIN_CONFIGS from processing import install_builtin_config - logger = logging.getLogger("archivematica.mcp.server.shareddirs") diff --git a/src/MCPServer/lib/server/tasks/__init__.py b/src/MCPServer/lib/server/tasks/__init__.py index 5dc28b56c0..857e7bc711 100644 --- a/src/MCPServer/lib/server/tasks/__init__.py +++ b/src/MCPServer/lib/server/tasks/__init__.py @@ -1,7 +1,6 @@ from server.tasks.backends import GearmanTaskBackend -from server.tasks.backends import get_task_backend from server.tasks.backends import TaskBackend +from server.tasks.backends import get_task_backend from server.tasks.task import Task - __all__ = ("GearmanTaskBackend", "Task", "TaskBackend", "get_task_backend") diff --git a/src/MCPServer/lib/server/tasks/backends/__init__.py b/src/MCPServer/lib/server/tasks/backends/__init__.py index a2b98cac20..1d013a132f 100644 --- a/src/MCPServer/lib/server/tasks/backends/__init__.py +++ b/src/MCPServer/lib/server/tasks/backends/__init__.py @@ -1,12 +1,12 @@ """ Handle offloading of Task objects to MCP Client for processing. """ + import threading from server.tasks.backends.base import TaskBackend from server.tasks.backends.gearman_backend import GearmanTaskBackend - backend_local = threading.local() diff --git a/src/MCPServer/lib/server/tasks/backends/gearman_backend.py b/src/MCPServer/lib/server/tasks/backends/gearman_backend.py index f55b89d8c9..1518a3e4a7 100644 --- a/src/MCPServer/lib/server/tasks/backends/gearman_backend.py +++ b/src/MCPServer/lib/server/tasks/backends/gearman_backend.py @@ -2,6 +2,7 @@ Gearman task backend. Submits `Task` objects to gearman for processing, and returns results. """ + import datetime import logging import uuid @@ -12,11 +13,11 @@ from gearman.constants import JOB_FAILED from gearman.constants import JOB_UNKNOWN from gearman_encoder import JSONDataEncoder + from server import metrics from server.tasks.backends.base import TaskBackend from server.tasks.task import Task - logger = logging.getLogger("archivematica.mcp.server.jobs.tasks") @@ -229,9 +230,7 @@ def result(self): return job_result["task_results"] except KeyError: raise ValueError( - "Expected a map containing 'task_results', but got: {!r}".format( - job_result - ) + f"Expected a map containing 'task_results', but got: {job_result!r}" ) def update_task_results(self): diff --git a/src/MCPServer/lib/server/tasks/task.py b/src/MCPServer/lib/server/tasks/task.py index 9c860468a6..c455c6566b 100644 --- a/src/MCPServer/lib/server/tasks/task.py +++ b/src/MCPServer/lib/server/tasks/task.py @@ -4,6 +4,7 @@ Tasks are passed to MCPClient for processing. """ + import logging import os import uuid @@ -12,7 +13,6 @@ from django.utils import timezone from main import models - logger = logging.getLogger("archivematica.mcp.server.tasks") @@ -44,9 +44,7 @@ def __init__( self.finished_timestamp = None def __repr__(self): - return "Task(uuid={}, arguments={}, start_timestamp={}, done={})".format( - self.uuid, self.arguments, self.start_timestamp, self.done - ) + return f"Task(uuid={self.uuid}, arguments={self.arguments}, start_timestamp={self.start_timestamp}, done={self.done})" @classmethod @auto_close_old_connections() diff --git a/src/MCPServer/lib/server/translation.py b/src/MCPServer/lib/server/translation.py index a8ad337814..8761076f0e 100644 --- a/src/MCPServer/lib/server/translation.py +++ b/src/MCPServer/lib/server/translation.py @@ -1,8 +1,8 @@ """ i18n handling. """ -import pprint +import pprint FALLBACK_LANG = "en" UNKNOWN_TRANSLATION_LABEL = "" diff --git a/src/MCPServer/lib/server/utils.py b/src/MCPServer/lib/server/utils.py index d8fe91f3dd..a18659fc21 100644 --- a/src/MCPServer/lib/server/utils.py +++ b/src/MCPServer/lib/server/utils.py @@ -1,6 +1,7 @@ """ Utility functions. """ + import uuid diff --git a/src/MCPServer/lib/server/watch_dirs.py b/src/MCPServer/lib/server/watch_dirs.py index 9ff80586b0..76901c49b4 100644 --- a/src/MCPServer/lib/server/watch_dirs.py +++ b/src/MCPServer/lib/server/watch_dirs.py @@ -7,6 +7,7 @@ workflows, as many chains start the next chain by moving a transfer or SIP t the appropriate watched directory. """ + import logging import os import sys @@ -14,9 +15,8 @@ import warnings from django.conf import settings -from inotify_simple import flags from inotify_simple import INotify - +from inotify_simple import flags IS_LINUX = sys.platform.startswith("linux") WATCHED_BASE_DIR = os.path.abspath(settings.WATCH_DIRECTORY) diff --git a/src/MCPServer/lib/server/workflow.py b/src/MCPServer/lib/server/workflow.py index 1ec5d34eab..0e697081d1 100644 --- a/src/MCPServer/lib/server/workflow.py +++ b/src/MCPServer/lib/server/workflow.py @@ -13,6 +13,7 @@ classes ``Chain``, ``Link`` and ``WatchedDir``. They have different method sets. """ + import json import os @@ -20,11 +21,11 @@ from jsonschema import FormatChecker from jsonschema import validate from jsonschema.exceptions import ValidationError + from server.jobs import Job from server.translation import FALLBACK_LANG from server.translation import TranslationLabel - _LATEST_SCHEMA = "workflow-schema-v1.json" ASSETS_DIR = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(os.path.join(__file__)))), "assets" @@ -57,9 +58,7 @@ def __init__(self, parsed_obj): self._decode_wdirs() def __str__(self): - return "Chains {}, links {}, watched directories: {}".format( - len(self.chains), len(self.links), len(self.wdirs) - ) + return f"Chains {len(self.chains)}, links {len(self.links)}, watched directories: {len(self.wdirs)}" def _decode_chains(self): self.chains = {} diff --git a/src/MCPServer/lib/server/workflow_abilities.py b/src/MCPServer/lib/server/workflow_abilities.py index 72c1d3b8e9..8d346d13eb 100644 --- a/src/MCPServer/lib/server/workflow_abilities.py +++ b/src/MCPServer/lib/server/workflow_abilities.py @@ -2,8 +2,8 @@ dependencies across Archivematica components (i.e., dashboard, MCPServer, and MCPClient). """ -from django.conf import settings +from django.conf import settings ABILITIES = ( # The (elastic)search Transfers indexes must be enabled for the transfers diff --git a/src/MCPServer/lib/settings/test.py b/src/MCPServer/lib/settings/test.py index d749fa5699..96ad923c20 100644 --- a/src/MCPServer/lib/settings/test.py +++ b/src/MCPServer/lib/settings/test.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . """Test settings and globals.""" + from .common import * DATABASES = { diff --git a/src/archivematicaCommon/lib/appconfig.py b/src/archivematicaCommon/lib/appconfig.py index 1e95f8949a..c08de919e1 100644 --- a/src/archivematicaCommon/lib/appconfig.py +++ b/src/archivematicaCommon/lib/appconfig.py @@ -13,6 +13,7 @@ where a specific parsing process can be defined. Those callbacks must accept the current appconfig Config object and the section. """ + import configparser from django.core.exceptions import ImproperlyConfigured diff --git a/src/archivematicaCommon/lib/archivematicaFunctions.py b/src/archivematicaCommon/lib/archivematicaFunctions.py index 755f61d472..00cf727691 100644 --- a/src/archivematicaCommon/lib/archivematicaFunctions.py +++ b/src/archivematicaCommon/lib/archivematicaFunctions.py @@ -17,6 +17,7 @@ """archivematicaFunctions provides various helper functions across the different Archivematica modules. """ + import base64 import collections import errno @@ -376,23 +377,21 @@ def reconstruct_empty_directories(mets_file_path, objects_path, logger=None): if logger: logger.info( "Unable to construct empty directories, either because" - " there is no METS file at {} or because there is no" - " objects/ directory at {}".format(mets_file_path, objects_path) + f" there is no METS file at {mets_file_path} or because there is no" + f" objects/ directory at {objects_path}" ) return doc = etree.parse(mets_file_path, etree.XMLParser(remove_blank_text=True)) logical_struct_map_el = doc.find( - 'mets:structMap[@TYPE="logical"][@LABEL="{}"]'.format( - NORMATIVE_STRUCTMAP_LABEL - ), + f'mets:structMap[@TYPE="logical"][@LABEL="{NORMATIVE_STRUCTMAP_LABEL}"]', NSMAP, ) if logical_struct_map_el is None: if logger: logger.info( - "Unable to locate a logical structMap labelled {}." + f"Unable to locate a logical structMap labelled {NORMATIVE_STRUCTMAP_LABEL}." " Aborting attempt to reconstruct empty" - " directories.".format(NORMATIVE_STRUCTMAP_LABEL) + " directories." ) return root_div_el = logical_struct_map_el.find( @@ -401,9 +400,9 @@ def reconstruct_empty_directories(mets_file_path, objects_path, logger=None): if root_div_el is None: if logger: logger.info( - "Unable to locate a logical structMap labelled {}." + f"Unable to locate a logical structMap labelled {NORMATIVE_STRUCTMAP_LABEL}." " Aborting attempt to reconstruct empty" - " directories.".format(NORMATIVE_STRUCTMAP_LABEL) + " directories." ) return paths = div_el_to_dir_paths(root_div_el, include=False) diff --git a/src/archivematicaCommon/lib/bindpid.py b/src/archivematicaCommon/lib/bindpid.py index 3041bf981d..08c15e8f88 100755 --- a/src/archivematicaCommon/lib/bindpid.py +++ b/src/archivematicaCommon/lib/bindpid.py @@ -130,18 +130,18 @@ """ + import argparse import configparser import os - try: from jinja2 import Template except ImportError: - from django.template import Context, Template + from django.template import Context + from django.template import Template import requests - # Parameters required when requesting the binding of a handle PID. REQ_PARAMS = ( "naming_authority", @@ -237,8 +237,8 @@ def _validate_entity_type_required_params(argdict): for param in et_validation["required"]: if not argdict.get(param): raise BindPIDException( - "To request a PID for a {}, you must also supply a value" - " for {}".format(entity_type, param) + f"To request a PID for a {entity_type}, you must also supply a value" + f" for {param}" ) diff --git a/src/archivematicaCommon/lib/custom_handlers.py b/src/archivematicaCommon/lib/custom_handlers.py index 999d980c9a..8abb452725 100644 --- a/src/archivematicaCommon/lib/custom_handlers.py +++ b/src/archivematicaCommon/lib/custom_handlers.py @@ -29,9 +29,7 @@ def emit(self, record): STANDARD_FORMAT = ( "%(levelname)-8s %(asctime)s %(name)s.%(funcName)s:%(lineno)d %(message)s" ) -SCRIPT_FILE_FORMAT = "{}: %(levelname)-8s %(asctime)s %(name)s:%(funcName)s:%(lineno)d: %(message)s".format( - os.path.basename(sys.argv[0]) -) +SCRIPT_FILE_FORMAT = f"{os.path.basename(sys.argv[0])}: %(levelname)-8s %(asctime)s %(name)s:%(funcName)s:%(lineno)d: %(message)s" def get_script_logger( diff --git a/src/archivematicaCommon/lib/databaseFunctions.py b/src/archivematicaCommon/lib/databaseFunctions.py index db1a84c187..de31eb5067 100644 --- a/src/archivematicaCommon/lib/databaseFunctions.py +++ b/src/archivematicaCommon/lib/databaseFunctions.py @@ -19,12 +19,12 @@ import uuid from django.utils import timezone +from main.models import SIP from main.models import Agent from main.models import Derivation from main.models import Event from main.models import File from main.models import FPCommandOutput -from main.models import SIP LOGGER = logging.getLogger("archivematica.common") diff --git a/src/archivematicaCommon/lib/dbconns.py b/src/archivematicaCommon/lib/dbconns.py index cfecac8c7f..8a4cbe2f92 100644 --- a/src/archivematicaCommon/lib/dbconns.py +++ b/src/archivematicaCommon/lib/dbconns.py @@ -16,6 +16,7 @@ all SQL queries and allow us to check that all logged queries occur within the wrapper. Note though, this will result in _very_ verbose logs. """ + import logging import threading import traceback @@ -24,7 +25,6 @@ from django.conf import settings from django.db import close_old_connections - logger = logging.getLogger("archivematica.mcp.server.db") thread_locals = threading.local() diff --git a/src/archivematicaCommon/lib/dicts.py b/src/archivematicaCommon/lib/dicts.py index e5b4122d4b..bc71930b9b 100644 --- a/src/archivematicaCommon/lib/dicts.py +++ b/src/archivematicaCommon/lib/dicts.py @@ -20,7 +20,6 @@ from main import models - config = {} diff --git a/src/archivematicaCommon/lib/elasticSearchFunctions.py b/src/archivematicaCommon/lib/elasticSearchFunctions.py index e67b441098..232f2e4968 100644 --- a/src/archivematicaCommon/lib/elasticSearchFunctions.py +++ b/src/archivematicaCommon/lib/elasticSearchFunctions.py @@ -618,9 +618,7 @@ def _generator(): file_id = file_.attrib.get("ID", None) file_pointer_division = ns.xml_find_premis( mets, - "mets:structMap[@TYPE='physical']//mets:fptr[@FILEID='{}']/..".format( - file_id - ), + f"mets:structMap[@TYPE='physical']//mets:fptr[@FILEID='{file_id}']/..", ) if file_pointer_division is not None: descriptive_metadata = _get_file_metadata(file_pointer_division, mets) @@ -637,9 +635,7 @@ def _generator(): for dmd_section_id_item in dmd_section_id.split(): dmd_section_info = ns.xml_find_premis( mets, - "mets:dmdSec[@ID='{}']/mets:mdWrap[@MDTYPE='DC']/mets:xmlData".format( - dmd_section_id_item - ), + f"mets:dmdSec[@ID='{dmd_section_id_item}']/mets:mdWrap[@MDTYPE='DC']/mets:xmlData", ) if dmd_section_info is not None: xml = etree.tostring(dmd_section_info, encoding="utf8") @@ -1288,9 +1284,7 @@ def get_file_tags(client, uuid): raise EmptySearchResultError(f"No matches found for file with UUID {uuid}") if count > 1: raise TooManyResultsError( - "{} matches found for file with UUID {}; unable to fetch a single result".format( - count, uuid - ) + f"{count} matches found for file with UUID {uuid}; unable to fetch a single result" ) result = results["hits"]["hits"][0] @@ -1317,9 +1311,7 @@ def set_file_tags(client, uuid, tags): raise EmptySearchResultError(f"No matches found for file with UUID {uuid}") if count > 1: raise TooManyResultsError( - "{} matches found for file with UUID {}; unable to fetch a single result".format( - count, uuid - ) + f"{count} matches found for file with UUID {uuid}; unable to fetch a single result" ) body = {"doc": {"tags": tags}} diff --git a/src/archivematicaCommon/lib/externals/extractMaildirAttachments.py b/src/archivematicaCommon/lib/externals/extractMaildirAttachments.py index 81d659ff18..bf08474bef 100644 --- a/src/archivematicaCommon/lib/externals/extractMaildirAttachments.py +++ b/src/archivematicaCommon/lib/externals/extractMaildirAttachments.py @@ -52,9 +52,7 @@ def parse_attachment(message_part, state, attachments=None): name, encoding = email.header.decode_header(filename)[0] if encoding: print( - "\t{filename} encoded with {encoding}, converting to unicode".format( - filename=filename, encoding=encoding - ) + f"\t{filename} encoded with {encoding}, converting to unicode" ) filename = name.decode(encoding) else: # filename not in Content-Disposition @@ -119,8 +117,12 @@ def parse2(msgobj, state, attachments=None): attachments.append(attachment) return { "subject": subject, - "from": email.utils.parseaddr(msgobj.get("From"))[1], # 名前は除いてメールアドレスのみ抽出 - "to": email.utils.parseaddr(msgobj.get("To"))[1], # 名前は除いてメールアドレスのみ抽出 + "from": email.utils.parseaddr(msgobj.get("From"))[ + 1 + ], # 名前は除いてメールアドレスのみ抽出 + "to": email.utils.parseaddr(msgobj.get("To"))[ + 1 + ], # 名前は除いてメールアドレスのみ抽出 "attachments": attachments, "msgobj": msgobj, } diff --git a/src/archivematicaCommon/lib/externals/fiwalk_plugins/pronom_ident.py b/src/archivematicaCommon/lib/externals/fiwalk_plugins/pronom_ident.py index 588a542d85..b9dd56fce8 100755 --- a/src/archivematicaCommon/lib/externals/fiwalk_plugins/pronom_ident.py +++ b/src/archivematicaCommon/lib/externals/fiwalk_plugins/pronom_ident.py @@ -2,6 +2,7 @@ """ pronom-ident.py - Identify a bitstream against PRONOM; uses fido """ + # https://github.com/anarchivist/fiwalk-dgi/blob/master/python/pronom_ident.py # Author anarchivist import os diff --git a/src/archivematicaCommon/lib/externals/maildirToMbox.py b/src/archivematicaCommon/lib/externals/maildirToMbox.py index dc3a5fedb3..3f05348358 100755 --- a/src/archivematicaCommon/lib/externals/maildirToMbox.py +++ b/src/archivematicaCommon/lib/externals/maildirToMbox.py @@ -31,6 +31,7 @@ [mbox_filename] will be newly created, as well as a [mbox_filename].sbd the directory. """ + import email import mailbox import os diff --git a/src/archivematicaCommon/lib/externals/xmltodict.py b/src/archivematicaCommon/lib/externals/xmltodict.py index 0212a03f39..59121355eb 100755 --- a/src/archivematicaCommon/lib/externals/xmltodict.py +++ b/src/archivematicaCommon/lib/externals/xmltodict.py @@ -142,8 +142,8 @@ def parse(xml_input, *args, **kwargs): if __name__ == "__main__": - import sys import marshal + import sys (item_depth,) = sys.argv[1:] item_depth = int(item_depth) diff --git a/src/archivematicaCommon/lib/fileOperations.py b/src/archivematicaCommon/lib/fileOperations.py index 822c1623a6..a396094ec9 100644 --- a/src/archivematicaCommon/lib/fileOperations.py +++ b/src/archivematicaCommon/lib/fileOperations.py @@ -353,17 +353,13 @@ def findFileInNormalizationCSV( ) except File.MultipleObjectsReturned: printfn( - "More than one result found for {} file ({}) in DB.".format( - commandClassification, target_file - ), + f"More than one result found for {commandClassification} file ({target_file}) in DB.", file=sys.stderr, ) raise FindFileInNormalizatonCSVError(2) except File.DoesNotExist: printfn( - "{} file ({}) not found in DB.".format( - commandClassification, target_file - ), + f"{commandClassification} file ({target_file}) not found in DB.", file=sys.stderr, ) raise FindFileInNormalizatonCSVError(2) @@ -380,29 +376,21 @@ def findFileInNormalizationCSV( continue original, access, preservation = row if commandClassification == "access" and access == target_file: - printfn( - "Found access file ({}) for original ({})".format( - access, original - ) - ) + printfn(f"Found access file ({access}) for original ({original})") return original if ( commandClassification == "preservation" and preservation == target_file ): printfn( - "Found preservation file ({}) for original ({})".format( - preservation, original - ) + f"Found preservation file ({preservation}) for original ({original})" ) return original else: return None except csv.Error: printfn( - "Error reading {filename} on line {linenum}".format( - filename=csv_path, linenum=reader.line_num - ), + f"Error reading {csv_path} on line {reader.line_num}", file=sys.stderr, ) raise FindFileInNormalizatonCSVError(2) diff --git a/src/archivematicaCommon/lib/identifier_functions.py b/src/archivematicaCommon/lib/identifier_functions.py index dac768f109..549996a7ee 100644 --- a/src/archivematicaCommon/lib/identifier_functions.py +++ b/src/archivematicaCommon/lib/identifier_functions.py @@ -3,6 +3,7 @@ See also src/MCPClient/lib/clientScripts/index_aip.py where these are used. """ + from lxml import etree diff --git a/src/archivematicaCommon/lib/processing.py b/src/archivematicaCommon/lib/processing.py index 7e4bd21a54..03453de9da 100644 --- a/src/archivematicaCommon/lib/processing.py +++ b/src/archivematicaCommon/lib/processing.py @@ -2,7 +2,6 @@ from django.conf import settings as django_settings - DEFAULT_PROCESSING_CONFIG = """ diff --git a/src/archivematicaCommon/lib/storageService.py b/src/archivematicaCommon/lib/storageService.py index b712ed48df..c50d3a0946 100644 --- a/src/archivematicaCommon/lib/storageService.py +++ b/src/archivematicaCommon/lib/storageService.py @@ -463,9 +463,7 @@ def download_file_url(file_uuid): """ storage_service_url = _storage_service_url() params = _storage_api_params() - download_url = "{base_url}file/{uuid}/download/?{params}".format( - base_url=storage_service_url, uuid=file_uuid, params=params - ) + download_url = f"{storage_service_url}file/{file_uuid}/download/?{params}" return download_url @@ -475,12 +473,7 @@ def extract_file_url(file_uuid, relative_path): """ storage_service_url = _storage_service_url() api_params = _storage_api_params() - download_url = "{base_url}file/{uuid}/extract_file/?relative_path_to_file={path}&{params}".format( - base_url=storage_service_url, - uuid=file_uuid, - path=relative_path, - params=api_params, - ) + download_url = f"{storage_service_url}file/{file_uuid}/extract_file/?relative_path_to_file={relative_path}&{api_params}" return download_url @@ -503,9 +496,7 @@ def pointer_file_url(file_uuid): """ storage_service_url = _storage_service_url() params = _storage_api_params() - download_url = "{base_url}file/{uuid}/pointer_file/?{params}".format( - base_url=storage_service_url, uuid=file_uuid, params=params - ) + download_url = f"{storage_service_url}file/{file_uuid}/pointer_file/?{params}" return download_url @@ -676,8 +667,6 @@ def retrieve_storage_location_description(aip_location_slug, logger=None): location_description = response.get(KEY_PATH, "") if logger: logger.info( - "Storage location retrieved: {} ({})".format( - location_description, aip_location_slug - ) + f"Storage location retrieved: {location_description} ({aip_location_slug})" ) return location_description diff --git a/src/archivematicaCommon/lib/xml2obj.py b/src/archivematicaCommon/lib/xml2obj.py index 595d99cbc2..97a8577af0 100644 --- a/src/archivematicaCommon/lib/xml2obj.py +++ b/src/archivematicaCommon/lib/xml2obj.py @@ -20,29 +20,21 @@ def mets_file(src): if amd.mets_techMD is None: continue - file_uuid = ( - amd.mets_techMD.mets_mdWrap.mets_xmlData.premis_object.premis_objectIdentifier.premis_objectIdentifierValue - ) + file_uuid = amd.mets_techMD.mets_mdWrap.mets_xmlData.premis_object.premis_objectIdentifier.premis_objectIdentifierValue if amd.mets_rightsMD: for rights in amd.mets_rightsMD: # print "found rightsMD: {}".format(rights['ID']) - if ( - rights.mets_mdWrap.mets_xmlData.premis_rightsStatement.premis_rightsGranted - ): - for ( - granted - ) in ( - rights.mets_mdWrap.mets_xmlData.premis_rightsStatement.premis_rightsGranted - ): + if rights.mets_mdWrap.mets_xmlData.premis_rightsStatement.premis_rightsGranted: + for granted in rights.mets_mdWrap.mets_xmlData.premis_rightsStatement.premis_rightsGranted: act = granted.premis_act # note = granted.premis_rightsGrantedNote # print "found {} has {} with {}".format(file_uuid, act, note) - mets[file_uuid]["premis"][act][ - "restriction" - ] = granted.premis_restriction - mets[file_uuid]["premis"][act][ - "rightsGrantedNote" - ] = granted.premis_rightsGrantedNote + mets[file_uuid]["premis"][act]["restriction"] = ( + granted.premis_restriction + ) + mets[file_uuid]["premis"][act]["rightsGrantedNote"] = ( + granted.premis_rightsGrantedNote + ) return mets diff --git a/src/dashboard/frontend/tools/extract_features.py b/src/dashboard/frontend/tools/extract_features.py index 3c23c1963e..6f5504c898 100644 --- a/src/dashboard/frontend/tools/extract_features.py +++ b/src/dashboard/frontend/tools/extract_features.py @@ -4,7 +4,6 @@ import os import sys - FILES = ["pii", "ccn"] HEADERS = ["offset", "content", "context"] diff --git a/src/dashboard/install/dashboard.gunicorn-config.py b/src/dashboard/install/dashboard.gunicorn-config.py index b55a7b7ca9..fe71f394cd 100644 --- a/src/dashboard/install/dashboard.gunicorn-config.py +++ b/src/dashboard/install/dashboard.gunicorn-config.py @@ -4,7 +4,6 @@ import shutil import tempfile - # http://docs.gunicorn.org/en/stable/settings.html#user user = os.environ.get("AM_GUNICORN_USER", "archivematica") diff --git a/src/dashboard/src/components/access/views.py b/src/dashboard/src/components/access/views.py index e624f3ea09..5ac3b35045 100644 --- a/src/dashboard/src/components/access/views.py +++ b/src/dashboard/src/components/access/views.py @@ -13,9 +13,9 @@ from components.ingest.views_as import get_as_system_client from django.shortcuts import redirect from django.utils import timezone +from main.models import SIP from main.models import ArchivesSpaceDigitalObject from main.models import DublinCore -from main.models import SIP from main.models import SIPArrange from main.models import SIPArrangeAccessMapping @@ -75,9 +75,7 @@ def wrapper(request, record_id=""): except SIPArrangeAccessMapping.DoesNotExist: response = { "success": False, - "message": "No SIP Arrange mapping exists for record {}".format( - record_id - ), + "message": f"No SIP Arrange mapping exists for record {record_id}", } return helpers.json_response(response, status_code=404) @@ -288,9 +286,7 @@ def digital_object_components(client, request, record_id=""): except ArchivesSpaceDigitalObject.DoesNotExist: response = { "success": False, - "message": "No digital object component exists with the specified ID: {}".format( - component_id - ), + "message": f"No digital object component exists with the specified ID: {component_id}", } return helpers.json_response(response, status_code=404) else: @@ -426,9 +422,7 @@ def access_arrange_start_sip(client, request, mapping): except SIPArrange.DoesNotExist: response = { "success": False, - "message": "No SIP Arrange object exists for record {}".format( - mapping.identifier - ), + "message": f"No SIP Arrange object exists for record {mapping.identifier}", } return helpers.json_response(response, status_code=404) diff --git a/src/dashboard/src/components/accounts/urls.py b/src/dashboard/src/components/accounts/urls.py index 4926a22d54..343fc20f11 100644 --- a/src/dashboard/src/components/accounts/urls.py +++ b/src/dashboard/src/components/accounts/urls.py @@ -19,7 +19,6 @@ from django.conf import settings from django.urls import path - app_name = "accounts" urlpatterns = [ path("", views.list, name="accounts_index"), diff --git a/src/dashboard/src/components/administration/forms_dip_upload.py b/src/dashboard/src/components/administration/forms_dip_upload.py index 4b62fac4de..208ea29090 100644 --- a/src/dashboard/src/components/administration/forms_dip_upload.py +++ b/src/dashboard/src/components/administration/forms_dip_upload.py @@ -1,7 +1,6 @@ from django import forms from django.utils.translation import gettext_lazy as _ - PREMIS_CHOICES = [("yes", _("Yes")), ("no", _("No")), ("premis", _("Base on PREMIS"))] EAD_ACTUATE_CHOICES = [ diff --git a/src/dashboard/src/components/administration/urls.py b/src/dashboard/src/components/administration/urls.py index b88d334af3..9c444278d2 100644 --- a/src/dashboard/src/components/administration/urls.py +++ b/src/dashboard/src/components/administration/urls.py @@ -24,7 +24,6 @@ from django.urls import re_path from django.views.generic import TemplateView - app_name = "administration" urlpatterns = [ path("", views.administration, name="administration_index"), diff --git a/src/dashboard/src/components/administration/views.py b/src/dashboard/src/components/administration/views.py index 082a6b07d9..4346140895 100644 --- a/src/dashboard/src/components/administration/views.py +++ b/src/dashboard/src/components/administration/views.py @@ -48,7 +48,6 @@ from version import get_full_version from version import get_preservation_system_identifier - logger = logging.getLogger("archivematica.dashboard") diff --git a/src/dashboard/src/components/administration/views_dip_upload.py b/src/dashboard/src/components/administration/views_dip_upload.py index a750151f3a..6596a54ca8 100644 --- a/src/dashboard/src/components/administration/views_dip_upload.py +++ b/src/dashboard/src/components/administration/views_dip_upload.py @@ -5,7 +5,6 @@ from django.utils.translation import gettext as _ from main.models import DashboardSetting - _AS_DICTNAME = "upload-archivesspace_v0.0" _ATOM_DICTNAME = "upload-qubit_v0.0" diff --git a/src/dashboard/src/components/administration/views_processing.py b/src/dashboard/src/components/administration/views_processing.py index 1c588bc336..abd23e539d 100644 --- a/src/dashboard/src/components/administration/views_processing.py +++ b/src/dashboard/src/components/administration/views_processing.py @@ -29,7 +29,6 @@ from .forms import ProcessingConfigurationForm - logger = logging.getLogger("archivematica.dashboard") diff --git a/src/dashboard/src/components/api/validators.py b/src/dashboard/src/components/api/validators.py index bbe9bb47e8..39e321e840 100644 --- a/src/dashboard/src/components/api/validators.py +++ b/src/dashboard/src/components/api/validators.py @@ -11,6 +11,7 @@ New validators must be added to the `` _VALIDATORS`` registry. """ + import collections import csv from io import StringIO diff --git a/src/dashboard/src/components/api/views.py b/src/dashboard/src/components/api/views.py index 4bc90e9b3a..6e6f7b707d 100644 --- a/src/dashboard/src/components/api/views.py +++ b/src/dashboard/src/components/api/views.py @@ -258,9 +258,7 @@ def status(request, unit_uuid, unit_type): response["type"] = "SIP" if unit is None: - response["message"] = "Cannot fetch {} with UUID {}".format( - unit_type, unit_uuid - ) + response["message"] = f"Cannot fetch {unit_type} with UUID {unit_uuid}" response["error"] = True return django.http.HttpResponseBadRequest( # 400 json.dumps(response), content_type="application/json" diff --git a/src/dashboard/src/components/archival_storage/urls.py b/src/dashboard/src/components/archival_storage/urls.py index b2bb936b41..41c95c808a 100644 --- a/src/dashboard/src/components/archival_storage/urls.py +++ b/src/dashboard/src/components/archival_storage/urls.py @@ -19,7 +19,6 @@ from django.urls import path from django.urls import re_path - app_name = "archival_storage" urlpatterns = [ re_path( diff --git a/src/dashboard/src/components/archival_storage/views.py b/src/dashboard/src/components/archival_storage/views.py index 63c67f598b..8a41e6d2b6 100644 --- a/src/dashboard/src/components/archival_storage/views.py +++ b/src/dashboard/src/components/archival_storage/views.py @@ -84,9 +84,7 @@ def sync_es_aip_status_with_storage_service(uuid, es_status): if api_results in AMCLIENT_ERROR_CODES: logger.warning( - "Package {} not found in Storage Service. AMClient error code: {}".format( - uuid, api_results - ) + f"Package {uuid} not found in Storage Service. AMClient error code: {api_results}" ) return keep_in_results @@ -649,9 +647,7 @@ def aips_pending_deletion(): except Exception as e: # TODO this should be messages.warning, but we need 'request' here logger.warning( - "Error retrieving AIPs pending deletion: is the storage server running? Error: {}".format( - e - ) + f"Error retrieving AIPs pending deletion: is the storage server running? Error: {e}" ) else: for aip in aips: diff --git a/src/dashboard/src/components/backlog/views.py b/src/dashboard/src/components/backlog/views.py index 5d6c4bb8c3..0b2ecdf2fe 100644 --- a/src/dashboard/src/components/backlog/views.py +++ b/src/dashboard/src/components/backlog/views.py @@ -62,9 +62,7 @@ def sync_es_transfer_status_with_storage_service(uuid, pending_deletion): if api_results in AMCLIENT_ERROR_CODES: logger.warning( - "Package {} not found in Storage Service. AMClient error code: {}".format( - uuid, api_results - ) + f"Package {uuid} not found in Storage Service. AMClient error code: {api_results}" ) return keep_in_results diff --git a/src/dashboard/src/components/file/views.py b/src/dashboard/src/components/file/views.py index 3460a1dcef..7eadd86dfc 100644 --- a/src/dashboard/src/components/file/views.py +++ b/src/dashboard/src/components/file/views.py @@ -10,7 +10,6 @@ from django.views.generic import View from main import models - logger = logging.getLogger("archivematica.dashboard") diff --git a/src/dashboard/src/components/filesystem_ajax/helpers.py b/src/dashboard/src/components/filesystem_ajax/helpers.py index ff09a79a9d..6a6d33a708 100644 --- a/src/dashboard/src/components/filesystem_ajax/helpers.py +++ b/src/dashboard/src/components/filesystem_ajax/helpers.py @@ -20,7 +20,6 @@ from archivematicaFunctions import b64encode_string from components import helpers - logger = logging.getLogger("archivematica.dashboard") diff --git a/src/dashboard/src/components/filesystem_ajax/views.py b/src/dashboard/src/components/filesystem_ajax/views.py index acc17c36ff..20acde5de0 100644 --- a/src/dashboard/src/components/filesystem_ajax/views.py +++ b/src/dashboard/src/components/filesystem_ajax/views.py @@ -37,7 +37,6 @@ from django.utils.translation import ngettext from main import models - logger = logging.getLogger("archivematica.dashboard") SHARED_DIRECTORY_ROOT = django_settings.SHARED_DIRECTORY diff --git a/src/dashboard/src/components/helpers.py b/src/dashboard/src/components/helpers.py index 898fc014fa..8404bbd740 100644 --- a/src/dashboard/src/components/helpers.py +++ b/src/dashboard/src/components/helpers.py @@ -41,7 +41,6 @@ from main import models from tastypie.models import ApiKey - logger = logging.getLogger("archivematica.dashboard") @@ -416,14 +415,12 @@ def completed_units_efficient(unit_type="transfer", include_failed=True): table_name = "SIPs" pk_name = "sipUUID" q = ( - "SELECT u.{pk_name}, j.jobType, j.microserviceGroup" - " FROM {table_name} as u" + f"SELECT u.{pk_name}, j.jobType, j.microserviceGroup" + f" FROM {table_name} as u" " INNER JOIN Jobs as j" - " ON j.SIPUUID = u.{pk_name}" + f" ON j.SIPUUID = u.{pk_name}" " WHERE u.hidden = 0" - " ORDER BY u.{pk_name}, j.createdTime DESC, j.createdTimeDec DESC;".format( - table_name=table_name, pk_name=pk_name - ) + f" ORDER BY u.{pk_name}, j.createdTime DESC, j.createdTimeDec DESC;" ) completed = set() current_uuid = None diff --git a/src/dashboard/src/components/rights/load.py b/src/dashboard/src/components/rights/load.py index 9d5340e68d..9265f635fa 100644 --- a/src/dashboard/src/components/rights/load.py +++ b/src/dashboard/src/components/rights/load.py @@ -6,7 +6,9 @@ This module does not use the ``__`` notation for accessors offered by ``premisrw``. """ + from components.helpers import get_metadata_type_id_by_description +from main.models import SIP from main.models import File from main.models import RightsStatement from main.models import RightsStatementCopyright @@ -14,7 +16,6 @@ from main.models import RightsStatementOtherRightsInformation from main.models import RightsStatementRightsGranted from main.models import RightsStatementStatuteInformation -from main.models import SIP from main.models import Transfer diff --git a/src/dashboard/src/components/transfer/views.py b/src/dashboard/src/components/transfer/views.py index e0b15a8ddc..2909858438 100644 --- a/src/dashboard/src/components/transfer/views.py +++ b/src/dashboard/src/components/transfer/views.py @@ -111,9 +111,9 @@ def component(request, uuid): field_value.fieldvalue = request.POST.get(field.fieldname, "") field_value.save() fields_saved = True - values[ - (field.fieldname) - ] = field_value.fieldvalue # override initially loaded value, if any + values[(field.fieldname)] = ( + field_value.fieldvalue + ) # override initially loaded value, if any if fields_saved: messages.append("Metadata saved.") diff --git a/src/dashboard/src/contrib/mcp/client.py b/src/dashboard/src/contrib/mcp/client.py index e199d724b4..f27d924358 100644 --- a/src/dashboard/src/contrib/mcp/client.py +++ b/src/dashboard/src/contrib/mcp/client.py @@ -22,7 +22,6 @@ from gearman_encoder import JSONDataEncoder from main.models import Job - LOGGER = logging.getLogger("archivematica.dashboard.mcp.client") diff --git a/src/dashboard/src/fpr/management/commands/import_pronom_ids.py b/src/dashboard/src/fpr/management/commands/import_pronom_ids.py index 00c78659c1..94f6cf24f6 100644 --- a/src/dashboard/src/fpr/management/commands/import_pronom_ids.py +++ b/src/dashboard/src/fpr/management/commands/import_pronom_ids.py @@ -11,7 +11,6 @@ from fpr.models import IDRule from lxml import etree - # Introduced in fpr/migrations/0035_python3_compatibility.py FILE_BY_EXTENSION_CMD_UUID = "8546b624-7894-4201-8df6-f239d5e0d5ba" @@ -137,11 +136,7 @@ def main(pronom_xml, output_format=SQL_OUTPUT, output_file=sys.stdout): group=unknown_format_group, uuid=str(uuid.uuid4()), ) - migration = ''' Format.objects.create(description="""{}""", group_id="{}", uuid="{}")'''.format( - new_format.format_name, - unknown_format_group.uuid, - parent_format.uuid, - ) + migration = f''' Format.objects.create(description="""{new_format.format_name}""", group_id="{unknown_format_group.uuid}", uuid="{parent_format.uuid}")''' sql = save_object(parent_format) choose_output(output_format, output_file, sql, migration) archivematica_formats[new_format.format_name] = parent_format @@ -154,13 +149,7 @@ def main(pronom_xml, output_format=SQL_OUTPUT, output_file=sys.stdout): uuid=str(uuid.uuid4()), ) sql = save_object(format_version) - migration = ''' FormatVersion.objects.create(format_id="{}", pronom_id="{}", description="""{}""", version="{}", uuid="{}")'''.format( - parent_format.uuid, - new_format.puid, - new_format.version_name, - new_format.version, - format_version.uuid, - ) + migration = f''' FormatVersion.objects.create(format_id="{parent_format.uuid}", pronom_id="{new_format.puid}", description="""{new_format.version_name}""", version="{new_format.version}", uuid="{format_version.uuid}")''' choose_output(output_format, output_file, sql, migration) # If an extension is listed, set up a new IDRule so that @@ -173,9 +162,7 @@ def main(pronom_xml, output_format=SQL_OUTPUT, output_file=sys.stdout): ).delete() if deleted_count: sql = connection.queries[-1]["sql"] - migration = """ IDRule.objects.filter(command_output="{}").delete()""".format( - new_format.extension - ) + migration = f""" IDRule.objects.filter(command_output="{new_format.extension}").delete()""" choose_output(output_format, output_file, sql, migration) else: rule = IDRule( @@ -184,8 +171,6 @@ def main(pronom_xml, output_format=SQL_OUTPUT, output_file=sys.stdout): command_output=new_format.extension, ) sql = save_object(rule) - migration = """ IDRule.objects.create(format_id="{}", command_id="{}", command_output="{}")""".format( - format_version.uuid, file_by_extension.uuid, new_format.extension - ) + migration = f""" IDRule.objects.create(format_id="{format_version.uuid}", command_id="{file_by_extension.uuid}", command_output="{new_format.extension}")""" choose_output(output_format, output_file, sql, migration) print(file=output_file) diff --git a/src/dashboard/src/fpr/migrations-data/mc_validate_cmd.py b/src/dashboard/src/fpr/migrations-data/mc_validate_cmd.py index 9163125969..673332612a 100644 --- a/src/dashboard/src/fpr/migrations-data/mc_validate_cmd.py +++ b/src/dashboard/src/fpr/migrations-data/mc_validate_cmd.py @@ -6,7 +6,6 @@ from lxml import etree - SUCCESS_CODE = 0 ERROR_CODE = 1 NS = "{https://mediaarea.net/mediaconch}" diff --git a/src/dashboard/src/fpr/migrations/0007_embedded_default_thumbnail.py b/src/dashboard/src/fpr/migrations/0007_embedded_default_thumbnail.py index fcb1011dcd..a764188ff5 100644 --- a/src/dashboard/src/fpr/migrations/0007_embedded_default_thumbnail.py +++ b/src/dashboard/src/fpr/migrations/0007_embedded_default_thumbnail.py @@ -2,7 +2,6 @@ from django.db import migrations - REPLACE_COMMAND = """ import argparse import sys diff --git a/src/dashboard/src/fpr/migrations/0011_mediaconch_validation.py b/src/dashboard/src/fpr/migrations/0011_mediaconch_validation.py index ade6ef0a35..806df59e93 100644 --- a/src/dashboard/src/fpr/migrations/0011_mediaconch_validation.py +++ b/src/dashboard/src/fpr/migrations/0011_mediaconch_validation.py @@ -2,7 +2,6 @@ from django.db import migrations - HERE = os.path.dirname(os.path.abspath(__file__)) MIGR_DATA = os.path.join(os.path.dirname(HERE), "migrations-data") VALIDATE_CMD_FNM = os.path.join(MIGR_DATA, "mc_validate_cmd.py") diff --git a/src/dashboard/src/fpr/migrations/0012_mediaconch_policy_checks.py b/src/dashboard/src/fpr/migrations/0012_mediaconch_policy_checks.py index 4c24942557..54c3ce66e0 100644 --- a/src/dashboard/src/fpr/migrations/0012_mediaconch_policy_checks.py +++ b/src/dashboard/src/fpr/migrations/0012_mediaconch_policy_checks.py @@ -1,7 +1,6 @@ from django.db import migrations from django.db import models - # All MediaConch policy check commands in the FPR should follow the template in # the following constant. They depend on the ammcpc module / command-line tool # https://github.com/artefactual-labs/ammcpc being installed, which is a simple diff --git a/src/dashboard/src/fpr/migrations/0021_normalize_jp2_with_ffmpeg_for_preservation.py b/src/dashboard/src/fpr/migrations/0021_normalize_jp2_with_ffmpeg_for_preservation.py index 82b5d5a76f..bebac25659 100644 --- a/src/dashboard/src/fpr/migrations/0021_normalize_jp2_with_ffmpeg_for_preservation.py +++ b/src/dashboard/src/fpr/migrations/0021_normalize_jp2_with_ffmpeg_for_preservation.py @@ -1,6 +1,5 @@ from django.db import migrations - NEW_FFMPEG_CMD_UUID = "66ba8388-2cd2-44a7-9d30-5d194d47c75b" OLD_CONVERT_RULE_UUID = "76ece049-6812-402e-b231-8cf1445a04b4" NEW_FFMPEG_RULE_UUID = "b98667be-2614-410b-9079-41942dbc3b32" diff --git a/src/dashboard/src/fpr/migrations/0026_fits_nailgun_compat.py b/src/dashboard/src/fpr/migrations/0026_fits_nailgun_compat.py index a06877f8ed..b7726bcb23 100644 --- a/src/dashboard/src/fpr/migrations/0026_fits_nailgun_compat.py +++ b/src/dashboard/src/fpr/migrations/0026_fits_nailgun_compat.py @@ -1,6 +1,5 @@ from django.db import migrations - OLD_FITS_CMD_UUID = "0b0d301f-b586-45d5-ba35-d76da57ce32a" NEW_FITS_CMD_UUID = "1a0394dc-528b-4a03-8c2b-1c158f1fb313" OLD_FITS_RULE_UUID = "296a131d-ce70-45ae-b800-6c71b3ea46f0" diff --git a/src/dashboard/src/fpr/migrations/0028_idcommand_unique_enabled.py b/src/dashboard/src/fpr/migrations/0028_idcommand_unique_enabled.py index 28cd352e97..d31ebfab1c 100644 --- a/src/dashboard/src/fpr/migrations/0028_idcommand_unique_enabled.py +++ b/src/dashboard/src/fpr/migrations/0028_idcommand_unique_enabled.py @@ -3,6 +3,7 @@ We've changed file format identification commands so only one object can be enabled. This migration makes Siegfried the new default. """ + from django.db import migrations diff --git a/src/dashboard/src/fpr/migrations/0029_update_inkscape_svg_command.py b/src/dashboard/src/fpr/migrations/0029_update_inkscape_svg_command.py index 03a43b0041..35dafab41c 100644 --- a/src/dashboard/src/fpr/migrations/0029_update_inkscape_svg_command.py +++ b/src/dashboard/src/fpr/migrations/0029_update_inkscape_svg_command.py @@ -1,6 +1,6 @@ """Stop using sudo in the `Transcoding to plain svg with inkscape` command.""" -from django.db import migrations +from django.db import migrations INKSCAPE_TOOL_UUID = "34afccc3-2cfd-4780-ba56-06221f1bfe18" diff --git a/src/dashboard/src/fpr/migrations/0030_update_jhove_command.py b/src/dashboard/src/fpr/migrations/0030_update_jhove_command.py index bb78277e27..1f36283bac 100644 --- a/src/dashboard/src/fpr/migrations/0030_update_jhove_command.py +++ b/src/dashboard/src/fpr/migrations/0030_update_jhove_command.py @@ -1,4 +1,5 @@ """Allow JHOVE bytestream to convey partial pass message instead of fail.""" + from django.db import migrations JHOVE_TOOL_UUID = "085d8690-93b7-4d31-84f7-2c5f4cbf6735" diff --git a/src/dashboard/src/fpr/migrations/0031_update_normalization_rules.py b/src/dashboard/src/fpr/migrations/0031_update_normalization_rules.py index 7b4d7f2bcc..ef4ea45a6a 100644 --- a/src/dashboard/src/fpr/migrations/0031_update_normalization_rules.py +++ b/src/dashboard/src/fpr/migrations/0031_update_normalization_rules.py @@ -1,5 +1,6 @@ """Remove normalization for preservation rules for all video files and for some common image formats.""" + from django.db import migrations NORMALIZE_TO_MKV_FOR_PRES_COMMANDS = ( diff --git a/src/dashboard/src/fpr/migrations/0035_python3_compatibility.py b/src/dashboard/src/fpr/migrations/0035_python3_compatibility.py index bcbbba14bb..a4c89416c5 100644 --- a/src/dashboard/src/fpr/migrations/0035_python3_compatibility.py +++ b/src/dashboard/src/fpr/migrations/0035_python3_compatibility.py @@ -1,4 +1,5 @@ """Update commands and rules for Python 3 compatibility.""" + from django.db import migrations OLD_JHOVE_CMD_UUID = "11036e14-78d9-4449-8360-e2da394279ad" diff --git a/src/dashboard/src/fpr/models.py b/src/dashboard/src/fpr/models.py index 1f02fc2e1a..e504394f14 100644 --- a/src/dashboard/src/fpr/models.py +++ b/src/dashboard/src/fpr/models.py @@ -4,6 +4,7 @@ Describes the data model for the FPR """ + import logging import uuid as uuid @@ -15,7 +16,6 @@ from django.utils.translation import gettext_lazy as _ from main.models import UUIDField - logger = logging.getLogger(__name__) # ############################## API V2 MODELS ############################### diff --git a/src/dashboard/src/fpr/templatetags/revisions_link.py b/src/dashboard/src/fpr/templatetags/revisions_link.py index a56df095e6..e70b627bc3 100644 --- a/src/dashboard/src/fpr/templatetags/revisions_link.py +++ b/src/dashboard/src/fpr/templatetags/revisions_link.py @@ -2,7 +2,6 @@ from django import template from django.urls import reverse - register = template.Library() diff --git a/src/dashboard/src/installer/forms.py b/src/dashboard/src/installer/forms.py index 6419f69054..48e7091343 100644 --- a/src/dashboard/src/installer/forms.py +++ b/src/dashboard/src/installer/forms.py @@ -22,7 +22,6 @@ from django.utils.translation import gettext_lazy as _ from main.models import DashboardSetting - site_url_field = forms.CharField( label=_("Site URL"), help_text=_("This is the public URL of your Archivematica dashboard."), diff --git a/src/dashboard/src/installer/middleware.py b/src/dashboard/src/installer/middleware.py index b825360d17..6fb7fe0fe5 100644 --- a/src/dashboard/src/installer/middleware.py +++ b/src/dashboard/src/installer/middleware.py @@ -22,7 +22,6 @@ from django.urls import reverse from django.utils.deprecation import MiddlewareMixin - EXEMPT_URLS = None diff --git a/src/dashboard/src/installer/steps.py b/src/dashboard/src/installer/steps.py index 0c3582f852..52128cf01f 100644 --- a/src/dashboard/src/installer/steps.py +++ b/src/dashboard/src/installer/steps.py @@ -26,7 +26,6 @@ from main.models import User from tastypie.models import ApiKey - logger = logging.getLogger("archivematica.dashboard") diff --git a/src/dashboard/src/main/management/commands/purge_transient_processing_data.py b/src/dashboard/src/main/management/commands/purge_transient_processing_data.py index 02eb71edee..4e1be20e3a 100644 --- a/src/dashboard/src/main/management/commands/purge_transient_processing_data.py +++ b/src/dashboard/src/main/management/commands/purge_transient_processing_data.py @@ -30,6 +30,7 @@ generally not recommended because it can affect SIPs being created from recently purged transfers. """ + import logging import traceback @@ -152,9 +153,7 @@ def handle(self, *args, **options): package_id = transfer.pk if not options["quiet"]: self.warning( - "» Transfer {} with status {}".format( - package_id, transfer.status_str - ) + f"» Transfer {package_id} with status {transfer.status_str}" ) if options["dry_run"]: continue diff --git a/src/dashboard/src/main/management/commands/rebuild_aip_index_from_storage_service.py b/src/dashboard/src/main/management/commands/rebuild_aip_index_from_storage_service.py index 30a39dc563..f7c06aecdb 100644 --- a/src/dashboard/src/main/management/commands/rebuild_aip_index_from_storage_service.py +++ b/src/dashboard/src/main/management/commands/rebuild_aip_index_from_storage_service.py @@ -32,6 +32,7 @@ Execution example: ./manage.py rebuild_aip_index_from_storage_service --delete-all """ + import logging import os import shutil @@ -180,9 +181,7 @@ def handle(self, *args, **options): "AIP/AIC" if aip_indexed_count == 1 else "AIPs/AICs" ) self.success( - "Indexing complete. Successfully indexed {count} {term}.".format( - count=aip_indexed_count, term=pluralized_aips_aics_term - ) + f"Indexing complete. Successfully indexed {aip_indexed_count} {pluralized_aips_aics_term}." ) def process_package( diff --git a/src/dashboard/src/main/management/commands/rebuild_elasticsearch_aip_index_from_files.py b/src/dashboard/src/main/management/commands/rebuild_elasticsearch_aip_index_from_files.py index 5fbeaef54f..01352d3b19 100644 --- a/src/dashboard/src/main/management/commands/rebuild_elasticsearch_aip_index_from_files.py +++ b/src/dashboard/src/main/management/commands/rebuild_elasticsearch_aip_index_from_files.py @@ -32,6 +32,7 @@ (this is useful when the system /tmp directory used by default is on a volume with limited space). """ + import os import re import shutil diff --git a/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py b/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py index fc04f3797c..0d65a65e7d 100644 --- a/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py +++ b/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py @@ -35,6 +35,7 @@ ``--delete``: before re-reindexing a transfer, will delete any data found in Elasticsearch with a matching UUID. In contrast with the ``--delete-all``, it does not delete the entire index. """ + import logging import multiprocessing import os @@ -58,8 +59,8 @@ from fileOperations import addFileToTransfer from fileOperations import extract_package from fpr.models import FormatVersion -from main.management.commands import boolean_input from main.management.commands import DashboardCommand +from main.management.commands import boolean_input from main.models import Agent from main.models import FileFormatVersion from main.models import FileID @@ -320,9 +321,7 @@ def populate_data_from_storage_service( ) except storageService.Error: self.error( - "Transfer {} not indexed. Unable to download from Storage Service.".format( - transfer_uuid - ) + f"Transfer {transfer_uuid} not indexed. Unable to download from Storage Service." ) continue # Transfers are downloaded as .tar files, so we extract files @@ -331,9 +330,7 @@ def populate_data_from_storage_service( extract_package(local_package, temp_backlog_dir) except CalledProcessError as err: self.error( - "Transfer {} not indexed. File extraction from tar failed: {}.".format( - transfer_uuid, err - ) + f"Transfer {transfer_uuid} not indexed. File extraction from tar failed: {err}." ) continue local_package_without_extension = am.package_name_from_path(local_package) @@ -342,9 +339,7 @@ def populate_data_from_storage_service( if entry.is_dir() and entry.name == local_package_without_extension: transfer_path = entry.path self.info( - "Importing transfer {} from temporarily downloaded copy.".format( - transfer_uuid - ) + f"Importing transfer {transfer_uuid} from temporarily downloaded copy." ) _import_self_describing_transfer( self, @@ -360,9 +355,7 @@ def populate_data_from_storage_service( processed += 1 else: self.error( - "Transfer {} not indexed. Unable to find files extracted from tar.".format( - transfer_uuid - ) + f"Transfer {transfer_uuid} not indexed. Unable to find files extracted from tar." ) self.success(f"{processed} transfers indexed!") @@ -550,9 +543,7 @@ def _import_self_describing_transfer( _import_file_from_fsentry(cmd, fsentry, transfer_uuid) except Exception as err: cmd.warning( - "There was an error processing file {} (transfer {}): {}".format( - fsentry.file_uuid, transfer_uuid, err - ) + f"There was an error processing file {fsentry.file_uuid} (transfer {transfer_uuid}): {err}" ) if django_settings.DEBUG: traceback.print_exc() diff --git a/src/dashboard/src/main/management/commands/reindex_from_remote_cluster.py b/src/dashboard/src/main/management/commands/reindex_from_remote_cluster.py index baa1399802..8cf3523988 100644 --- a/src/dashboard/src/main/management/commands/reindex_from_remote_cluster.py +++ b/src/dashboard/src/main/management/commands/reindex_from_remote_cluster.py @@ -11,6 +11,7 @@ https://192.168.168.196:9200 \ -u test -p 1234 """ + import json import sys @@ -88,8 +89,8 @@ def handle(self, *args, **options): elasticSearchFunctions.create_indexes_if_needed(es_client, indexes) except Exception as e: self.error( - "The Elasticsearch indexes could not be recreated in {}. " - "Error: {}".format(settings.ELASTICSEARCH_SERVER, e) + f"The Elasticsearch indexes could not be recreated in {settings.ELASTICSEARCH_SERVER}. " + f"Error: {e}" ) sys.exit(1) diff --git a/src/dashboard/src/main/management/commands/resolve_pending_jobs.py b/src/dashboard/src/main/management/commands/resolve_pending_jobs.py index e3cafd0257..95492b5654 100644 --- a/src/dashboard/src/main/management/commands/resolve_pending_jobs.py +++ b/src/dashboard/src/main/management/commands/resolve_pending_jobs.py @@ -12,6 +12,7 @@ interactive command-line interface in the future it will rely on public APIs. This is an alternative to the old mcp-rpc-cli command. """ + from contrib.mcp.client import MCPClient from django.contrib.auth import get_user_model from django.core.management.base import CommandError diff --git a/src/dashboard/src/main/management/commands/update_elasticsearch_mappings.py b/src/dashboard/src/main/management/commands/update_elasticsearch_mappings.py index 6d9de83b3a..ce75324deb 100644 --- a/src/dashboard/src/main/management/commands/update_elasticsearch_mappings.py +++ b/src/dashboard/src/main/management/commands/update_elasticsearch_mappings.py @@ -9,6 +9,7 @@ ./manage.py update_elasticsearch_mappings """ + import sys import elasticSearchFunctions as es diff --git a/src/dashboard/src/main/migrations/0026_agent_m2m_event.py b/src/dashboard/src/main/migrations/0026_agent_m2m_event.py index 85e62618eb..b05b675373 100644 --- a/src/dashboard/src/main/migrations/0026_agent_m2m_event.py +++ b/src/dashboard/src/main/migrations/0026_agent_m2m_event.py @@ -14,9 +14,7 @@ def data_migration(apps, schema_editor): agent = Agent.objects.create( identifiertype="Archivematica user pk", identifiervalue=str(u.id), - name='username="{u.username}", first_name="{u.first_name}", last_name="{u.last_name}"'.format( - u=u - ), + name=f'username="{u.username}", first_name="{u.first_name}", last_name="{u.last_name}"', agenttype="Archivematica user", ) UserProfile.objects.create(user=u, agent=agent) diff --git a/src/dashboard/src/main/migrations/0029_backlog_removal_event.py b/src/dashboard/src/main/migrations/0029_backlog_removal_event.py index 96a46625e0..043cea70c6 100644 --- a/src/dashboard/src/main/migrations/0029_backlog_removal_event.py +++ b/src/dashboard/src/main/migrations/0029_backlog_removal_event.py @@ -1,6 +1,5 @@ from django.db import migrations - BCKLG_RMVL_EVT_UUID = "463e5d1c-d680-47fa-a27a-7efd4f702355" diff --git a/src/dashboard/src/main/migrations/0037_mediaconch_policy_checks.py b/src/dashboard/src/main/migrations/0037_mediaconch_policy_checks.py index 5a0210e30d..c0e42b53a2 100644 --- a/src/dashboard/src/main/migrations/0037_mediaconch_policy_checks.py +++ b/src/dashboard/src/main/migrations/0037_mediaconch_policy_checks.py @@ -247,7 +247,7 @@ def data_migration(apps, schema_editor): chain_yes_prsrvtn_drvtv_plcy_chck_msc = MicroServiceChain.objects.create( id=chain_yes_prsrvtn_drvtv_plcy_chck_msc_pk, startinglink=prsrvtn_drvtv_policy_check_cl, - description="Yes" # Yes, policy checks on preservation derivatives + description="Yes", # Yes, policy checks on preservation derivatives # please!' ) @@ -264,7 +264,7 @@ def data_migration(apps, schema_editor): chain_no_prsrvtn_drvtv_plcy_chck_msc = MicroServiceChain.objects.create( id=chain_no_prsrvtn_drvtv_plcy_chck_msc_pk, startinglink=ccss_drvtv_policy_check_choice_cl, - description="No" # No policy checks on preservation derivatives + description="No", # No policy checks on preservation derivatives # please!' ) @@ -316,7 +316,7 @@ def data_migration(apps, schema_editor): execute="policyCheck_v0.0", arguments=( '"%relativeLocation%" "%fileUUID%" "%SIPUUID%"' ' "%sharedPath%" "original"' - ) + ), # filter_subdir='objects/' <- not needed during transfer, I believe ... ) diff --git a/src/dashboard/src/main/migrations/0038_fix_workflow_store_dip.py b/src/dashboard/src/main/migrations/0038_fix_workflow_store_dip.py index 911965d900..7f06190779 100644 --- a/src/dashboard/src/main/migrations/0038_fix_workflow_store_dip.py +++ b/src/dashboard/src/main/migrations/0038_fix_workflow_store_dip.py @@ -1,4 +1,5 @@ """Migration to change workflow related to DIP upload and storage.""" + from django.db import migrations diff --git a/src/dashboard/src/main/migrations/0041_bind_pids.py b/src/dashboard/src/main/migrations/0041_bind_pids.py index d43ee96c8a..fa5ffddc4f 100644 --- a/src/dashboard/src/main/migrations/0041_bind_pids.py +++ b/src/dashboard/src/main/migrations/0041_bind_pids.py @@ -1,4 +1,5 @@ """Migration for PID Binding: persistent identifier creation & resolution.""" + from django.db import migrations from django.db import models diff --git a/src/dashboard/src/main/migrations/0043_remove_create_pointer_file_ms.py b/src/dashboard/src/main/migrations/0043_remove_create_pointer_file_ms.py index e34dc5751f..e59d635d45 100644 --- a/src/dashboard/src/main/migrations/0043_remove_create_pointer_file_ms.py +++ b/src/dashboard/src/main/migrations/0043_remove_create_pointer_file_ms.py @@ -1,4 +1,5 @@ """Migration to remove the "Create AIP Pointer File" micro-service.""" + from django.db import migrations diff --git a/src/dashboard/src/main/migrations/0045_archivesspace_config_dict.py b/src/dashboard/src/main/migrations/0045_archivesspace_config_dict.py index 3ac9adbd86..0cefebc18a 100644 --- a/src/dashboard/src/main/migrations/0045_archivesspace_config_dict.py +++ b/src/dashboard/src/main/migrations/0045_archivesspace_config_dict.py @@ -1,6 +1,5 @@ from django.db import migrations - _AS_DICTNAME = "upload-archivesspace_v0.0" diff --git a/src/dashboard/src/main/migrations/0050_remove_magic_links.py b/src/dashboard/src/main/migrations/0050_remove_magic_links.py index 8ebccccd21..1dac6e0947 100644 --- a/src/dashboard/src/main/migrations/0050_remove_magic_links.py +++ b/src/dashboard/src/main/migrations/0050_remove_magic_links.py @@ -1,4 +1,5 @@ """Migration to replace magic links with unit variables.""" + from collections import namedtuple from django.db import migrations diff --git a/src/dashboard/src/main/migrations/0051_remove_verify_premis_checksums.py b/src/dashboard/src/main/migrations/0051_remove_verify_premis_checksums.py index f163ab05e8..8fcfcf5b09 100644 --- a/src/dashboard/src/main/migrations/0051_remove_verify_premis_checksums.py +++ b/src/dashboard/src/main/migrations/0051_remove_verify_premis_checksums.py @@ -2,6 +2,7 @@ micro-service which is redundant. See https://github.com/artefactual/archivematica/issues/918. """ + from django.db import migrations diff --git a/src/dashboard/src/main/migrations/0052_correct_extract_packages_fallback_link.py b/src/dashboard/src/main/migrations/0052_correct_extract_packages_fallback_link.py index cc8e61918a..c2ce5d8e32 100644 --- a/src/dashboard/src/main/migrations/0052_correct_extract_packages_fallback_link.py +++ b/src/dashboard/src/main/migrations/0052_correct_extract_packages_fallback_link.py @@ -2,6 +2,7 @@ fails that the workflow doesn't continue to trudge along to completion where there is a likelihood of other errors. """ + from django.db import migrations diff --git a/src/dashboard/src/main/migrations/0055_normalize_thumbnail_mode_selection.py b/src/dashboard/src/main/migrations/0055_normalize_thumbnail_mode_selection.py index 19cc5696e3..c31fb66c07 100644 --- a/src/dashboard/src/main/migrations/0055_normalize_thumbnail_mode_selection.py +++ b/src/dashboard/src/main/migrations/0055_normalize_thumbnail_mode_selection.py @@ -3,6 +3,7 @@ entirely, or to only generate thumbnails if there is an active FPRule for the given file type. """ + from dateutil.parser import parse as parse_date from django.db import migrations from django.db.models import F diff --git a/src/dashboard/src/main/migrations/0057_7zip_no_compression.py b/src/dashboard/src/main/migrations/0057_7zip_no_compression.py index 02ea7aa92e..a869223c4e 100644 --- a/src/dashboard/src/main/migrations/0057_7zip_no_compression.py +++ b/src/dashboard/src/main/migrations/0057_7zip_no_compression.py @@ -1,6 +1,7 @@ """Migration to add a 7-zip method to the list of Archivematica AIP 7-zip methods that does not use compression. 7-zip copy mode. """ + from django.db import migrations choice_no_compression_uuid = "e3906da0-41fb-4cb8-a598-6c73981b63e9" diff --git a/src/dashboard/src/main/migrations/0058_fix_unit_variable_pull_link.py b/src/dashboard/src/main/migrations/0058_fix_unit_variable_pull_link.py index ae0ad62006..708d674ce0 100644 --- a/src/dashboard/src/main/migrations/0058_fix_unit_variable_pull_link.py +++ b/src/dashboard/src/main/migrations/0058_fix_unit_variable_pull_link.py @@ -1,6 +1,7 @@ """Migration to update "Resume after normalization file identification tool selected." chain link to exit with the successful status. """ + from django.db import migrations from main.models import Job diff --git a/src/dashboard/src/main/migrations/0059_siparrange_longblob.py b/src/dashboard/src/main/migrations/0059_siparrange_longblob.py index ef8d038d5e..e4eca2719a 100644 --- a/src/dashboard/src/main/migrations/0059_siparrange_longblob.py +++ b/src/dashboard/src/main/migrations/0059_siparrange_longblob.py @@ -1,9 +1,9 @@ import uuid import main.models +from django.db import OperationalError from django.db import migrations from django.db import models -from django.db import OperationalError def drop_original_path_unique_key(apps, schema_editor): diff --git a/src/dashboard/src/main/migrations/0060_delete_orphan_mscl.py b/src/dashboard/src/main/migrations/0060_delete_orphan_mscl.py index c540230ff5..f204327743 100644 --- a/src/dashboard/src/main/migrations/0060_delete_orphan_mscl.py +++ b/src/dashboard/src/main/migrations/0060_delete_orphan_mscl.py @@ -1,4 +1,5 @@ """Migration for the deletion of an orphan chain link in the workflow.""" + from django.db import migrations diff --git a/src/dashboard/src/main/migrations/0061_create_dataverse_transfer_type.py b/src/dashboard/src/main/migrations/0061_create_dataverse_transfer_type.py index 8176986434..288f2908e9 100644 --- a/src/dashboard/src/main/migrations/0061_create_dataverse_transfer_type.py +++ b/src/dashboard/src/main/migrations/0061_create_dataverse_transfer_type.py @@ -15,6 +15,7 @@ Once the tasks have completed, the workflow is picked up from where it would normally for a standard transfer type. """ + from django.db import migrations from main.models import Job diff --git a/src/dashboard/src/main/migrations/0062_atom_upload_exit_code.py b/src/dashboard/src/main/migrations/0062_atom_upload_exit_code.py index e4cb47fe8a..a8dc839e02 100644 --- a/src/dashboard/src/main/migrations/0062_atom_upload_exit_code.py +++ b/src/dashboard/src/main/migrations/0062_atom_upload_exit_code.py @@ -2,10 +2,10 @@ DIP upload`` link is marked as completed successfully when the exit code is ``0``. """ + from django.db import migrations from main.models import Job - ATOM_DIP_UPLOAD_CONFIG_EXIT_CODE_ZERO = "c9e90d83-533f-44c3-8220-083a6eb91751" diff --git a/src/dashboard/src/main/migrations/0064_version_number.py b/src/dashboard/src/main/migrations/0064_version_number.py index 2c2196be4a..7f58007ee5 100644 --- a/src/dashboard/src/main/migrations/0064_version_number.py +++ b/src/dashboard/src/main/migrations/0064_version_number.py @@ -1,4 +1,5 @@ """Migrate the Archivematica agent version string to Archivematica-1.8.""" + from django.db import migrations diff --git a/src/dashboard/src/main/migrations/0065_version_number.py b/src/dashboard/src/main/migrations/0065_version_number.py index 21fc85551a..b5833fe1f6 100644 --- a/src/dashboard/src/main/migrations/0065_version_number.py +++ b/src/dashboard/src/main/migrations/0065_version_number.py @@ -1,4 +1,5 @@ """Migrate the Archivematica agent version string to Archivematica-1.9.""" + from django.db import migrations diff --git a/src/dashboard/src/main/migrations/0066_archivesspace_base_url.py b/src/dashboard/src/main/migrations/0066_archivesspace_base_url.py index 7f69bbdde6..404c5959bc 100644 --- a/src/dashboard/src/main/migrations/0066_archivesspace_base_url.py +++ b/src/dashboard/src/main/migrations/0066_archivesspace_base_url.py @@ -2,6 +2,7 @@ ``host`` and `port`` are replaced by ``base_url``. """ + import sys from urllib.parse import urlparse diff --git a/src/dashboard/src/main/migrations/0068_version_number.py b/src/dashboard/src/main/migrations/0068_version_number.py index 6bc724c50e..c689f88037 100644 --- a/src/dashboard/src/main/migrations/0068_version_number.py +++ b/src/dashboard/src/main/migrations/0068_version_number.py @@ -1,4 +1,5 @@ """Migrate the Archivematica agent version string to Archivematica-1.9.""" + from django.db import migrations diff --git a/src/dashboard/src/main/migrations/0069_remove_atk.py b/src/dashboard/src/main/migrations/0069_remove_atk.py index 6b87121f79..d2c6dd7ef6 100644 --- a/src/dashboard/src/main/migrations/0069_remove_atk.py +++ b/src/dashboard/src/main/migrations/0069_remove_atk.py @@ -1,4 +1,5 @@ """Remove Archivists' Toolkit data and model.""" + from django.db import migrations from django.db import models diff --git a/src/dashboard/src/main/migrations/0071_index_unitvariables_runsql.py b/src/dashboard/src/main/migrations/0071_index_unitvariables_runsql.py index 5fc1fdd7b5..a228f30f40 100644 --- a/src/dashboard/src/main/migrations/0071_index_unitvariables_runsql.py +++ b/src/dashboard/src/main/migrations/0071_index_unitvariables_runsql.py @@ -1,4 +1,5 @@ """A MySQL specific index for UnitVariables.""" + from django.db import migrations diff --git a/src/dashboard/src/main/migrations/0073_index_files_runsql.py b/src/dashboard/src/main/migrations/0073_index_files_runsql.py index a136196260..5fa2c9eaf6 100644 --- a/src/dashboard/src/main/migrations/0073_index_files_runsql.py +++ b/src/dashboard/src/main/migrations/0073_index_files_runsql.py @@ -1,4 +1,5 @@ """MySQL specific indexes for Files.""" + from django.db import migrations diff --git a/src/dashboard/src/main/migrations/0074_version_number.py b/src/dashboard/src/main/migrations/0074_version_number.py index e281f968da..593646bf5e 100644 --- a/src/dashboard/src/main/migrations/0074_version_number.py +++ b/src/dashboard/src/main/migrations/0074_version_number.py @@ -1,4 +1,5 @@ """Migrate the Archivematica agent version string to Archivematica-1.11.""" + from django.db import migrations diff --git a/src/dashboard/src/main/migrations/0076_version_number.py b/src/dashboard/src/main/migrations/0076_version_number.py index 3705f41f73..194480245b 100644 --- a/src/dashboard/src/main/migrations/0076_version_number.py +++ b/src/dashboard/src/main/migrations/0076_version_number.py @@ -1,4 +1,5 @@ """Migrate the Archivematica agent version string to Archivematica-1.12.""" + from django.db import migrations diff --git a/src/dashboard/src/main/migrations/0077_uuid_fields.py b/src/dashboard/src/main/migrations/0077_uuid_fields.py index bff0589593..d36cb0ecae 100644 --- a/src/dashboard/src/main/migrations/0077_uuid_fields.py +++ b/src/dashboard/src/main/migrations/0077_uuid_fields.py @@ -1,4 +1,5 @@ """Migration generated by Django 1.11.29.""" + import uuid import main.models diff --git a/src/dashboard/src/main/migrations/0078_username_check.py b/src/dashboard/src/main/migrations/0078_username_check.py index 23067aac6b..e5b51ee4c5 100644 --- a/src/dashboard/src/main/migrations/0078_username_check.py +++ b/src/dashboard/src/main/migrations/0078_username_check.py @@ -17,6 +17,7 @@ The error will not be raised when ``--no-input`` is used, which may be useful for users that want to deal with the situation later or ignore it entirely. """ + import sys from django.db import migrations diff --git a/src/dashboard/src/main/migrations/0079_version_number.py b/src/dashboard/src/main/migrations/0079_version_number.py index 96076f5f3d..4ed3e73d9d 100644 --- a/src/dashboard/src/main/migrations/0079_version_number.py +++ b/src/dashboard/src/main/migrations/0079_version_number.py @@ -1,7 +1,6 @@ -"""Migrate the Archivematica agent version string to Archivematica-1.13. -""" -from django.db import migrations +"""Migrate the Archivematica agent version string to Archivematica-1.13.""" +from django.db import migrations DOWN_VERSION = "Archivematica-1.12" UP_VERSION = "Archivematica-1.13" diff --git a/src/dashboard/src/main/migrations/0083_unpickle_access_target.py b/src/dashboard/src/main/migrations/0083_unpickle_access_target.py index d0342fdc7a..70604a4625 100644 --- a/src/dashboard/src/main/migrations/0083_unpickle_access_target.py +++ b/src/dashboard/src/main/migrations/0083_unpickle_access_target.py @@ -1,6 +1,6 @@ # Generated by Django 4.2.11 on 2024-04-11 08:19 -"""Unpickle target in the access table. -""" +"""Unpickle target in the access table.""" + import pickle from django.db import migrations diff --git a/src/dashboard/src/main/models.py b/src/dashboard/src/main/models.py index 98a38105e5..08fab1c8c2 100644 --- a/src/dashboard/src/main/models.py +++ b/src/dashboard/src/main/models.py @@ -94,11 +94,7 @@ def create_user_agent(sender, instance, **kwargs): defaults={ "identifiertype": "Archivematica user pk", "identifiervalue": str(instance.id), - "name": 'username="{}", first_name="{}", last_name="{}"'.format( - instance.username, - instance.first_name, - instance.last_name, - ), + "name": f'username="{instance.username}", first_name="{instance.first_name}", last_name="{instance.last_name}"', "agenttype": "Archivematica user", }, ) @@ -668,9 +664,7 @@ class SIPArrangeAccessMapping(models.Model): identifier = models.CharField(max_length=255) def __str__(self): - return "arrange_path={s.arrange_path}, system={s.system}, identifier={s.identifier}".format( - s=self - ) + return f"arrange_path={self.arrange_path}, system={self.system}, identifier={self.identifier}" class Identifier(models.Model): @@ -691,7 +685,7 @@ class Identifier(models.Model): ) def __str__(self): - return "Identifier {i.value} of type {i.type}".format(i=self) + return f"Identifier {self.value} of type {self.type}" class Meta: db_table = "Identifiers" @@ -1068,11 +1062,7 @@ class Agent(models.Model): objects = AgentManager() def __str__(self): - return ( - "{a.agenttype}; {a.identifiertype}: {a.identifiervalue}; {a.name}".format( - a=self - ) - ) + return f"{self.agenttype}; {self.identifiertype}: {self.identifiervalue}; {self.name}" class Meta: db_table = "Agents" @@ -1671,9 +1661,7 @@ class ArchivesSpaceDIPObjectResourcePairing(models.Model): resourceid = models.CharField(max_length=150, db_column="resourceId") def __str__(self): - return "ArchivesSpace Pairing".format( - s=self - ) + return f"ArchivesSpace Pairing" class Meta: db_table = "ArchivesSpaceDIPObjectResourcePairing" @@ -1809,9 +1797,7 @@ class FPCommandOutput(models.Model): # Table name is main_fpcommandoutput def __str__(self): - return ". """Development settings and globals.""" + import os from .base import * diff --git a/src/dashboard/src/settings/production.py b/src/dashboard/src/settings/production.py index 3211b61e19..0cf723049a 100644 --- a/src/dashboard/src/settings/production.py +++ b/src/dashboard/src/settings/production.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . """Production settings and globals.""" + from .base import * diff --git a/src/dashboard/src/settings/test.py b/src/dashboard/src/settings/test.py index 27c0b69f7e..f7b33fe070 100644 --- a/src/dashboard/src/settings/test.py +++ b/src/dashboard/src/settings/test.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . """Test settings and globals.""" + import ldap from django_auth_ldap.config import LDAPSearch @@ -53,9 +54,9 @@ } # Disable whitenoise -STORAGES["staticfiles"][ - "BACKEND" -] = "django.contrib.staticfiles.storage.StaticFilesStorage" +STORAGES["staticfiles"]["BACKEND"] = ( + "django.contrib.staticfiles.storage.StaticFilesStorage" +) if MIDDLEWARE[0] == "whitenoise.middleware.WhiteNoiseMiddleware": del MIDDLEWARE[0] diff --git a/src/dashboard/src/urls.py b/src/dashboard/src/urls.py index 4f78e2d8a7..fc7dba8228 100644 --- a/src/dashboard/src/urls.py +++ b/src/dashboard/src/urls.py @@ -19,7 +19,6 @@ from django.urls import path from django.urls import re_path - urlpatterns = [ path("mcp/", include("components.mcp.urls")), path("installer/", include("installer.urls")), diff --git a/src/dashboard/src/wsgi.py b/src/dashboard/src/wsgi.py index 1bdaa1c858..1e6b3bba7c 100644 --- a/src/dashboard/src/wsgi.py +++ b/src/dashboard/src/wsgi.py @@ -5,7 +5,6 @@ django.setup() import elasticSearchFunctions - application = get_wsgi_application() # Set up Elasticsearch client diff --git a/tests/MCPClient/test_antivirus.py b/tests/MCPClient/test_antivirus.py index f3eb84c31b..b1d48ee1fd 100644 --- a/tests/MCPClient/test_antivirus.py +++ b/tests/MCPClient/test_antivirus.py @@ -1,6 +1,7 @@ """Tests for the archivematica_clamscan.py client script.""" -from collections import namedtuple + from collections import OrderedDict +from collections import namedtuple import archivematica_clamscan import pytest diff --git a/tests/MCPClient/test_antivirus_clamdscan.py b/tests/MCPClient/test_antivirus_clamdscan.py index 23eb741a1f..f9f1a2b2b1 100644 --- a/tests/MCPClient/test_antivirus_clamdscan.py +++ b/tests/MCPClient/test_antivirus_clamdscan.py @@ -1,4 +1,5 @@ """Tests for the archivematica_clamscan.py client script.""" + import errno from collections import namedtuple @@ -124,14 +125,14 @@ def patch(scanner, ret=OKAY_RET, excepts=False): # Testing a generic IOError that is not a broken pipe error that we're # expecting to be able to manage from clamdscan. - patch(scanner, ret=OKAY_RET, excepts=IOError("Testing a generic IO Error")) + patch(scanner, ret=OKAY_RET, excepts=OSError("Testing a generic IO Error")) passed, state, details = scanner.scan("/file") assert passed is False assert state is None assert details is None # Broken pipe is a known error from the clamd library. - brokenpipe_error = IOError("Testing a broken pipe error") + brokenpipe_error = OSError("Testing a broken pipe error") brokenpipe_error.errno = errno.EPIPE patch(scanner, ret=OKAY_RET, excepts=brokenpipe_error) passed, state, details = scanner.scan("/file") diff --git a/tests/MCPClient/test_antivirus_clamscan.py b/tests/MCPClient/test_antivirus_clamscan.py index 827ab26335..f783bbe904 100644 --- a/tests/MCPClient/test_antivirus_clamscan.py +++ b/tests/MCPClient/test_antivirus_clamscan.py @@ -1,4 +1,5 @@ """Tests for the archivematica_clamscan.py client script.""" + import subprocess import archivematica_clamscan diff --git a/tests/MCPClient/test_archivematicaCreateMETSMetadataCSV.py b/tests/MCPClient/test_archivematicaCreateMETSMetadataCSV.py index 60517b2df6..228e9511f0 100755 --- a/tests/MCPClient/test_archivematicaCreateMETSMetadataCSV.py +++ b/tests/MCPClient/test_archivematicaCreateMETSMetadataCSV.py @@ -2,6 +2,7 @@ """ Tests for CSV metadata management on the METS creation process """ + from archivematicaCreateMETSMetadataCSV import parseMetadataCSV content_when_value_has_empty_strings = """ diff --git a/tests/MCPClient/test_archivematicaCreateMETSMetadataXML.py b/tests/MCPClient/test_archivematicaCreateMETSMetadataXML.py index 43275c0912..b8b9dff7fa 100644 --- a/tests/MCPClient/test_archivematicaCreateMETSMetadataXML.py +++ b/tests/MCPClient/test_archivematicaCreateMETSMetadataXML.py @@ -4,6 +4,7 @@ archivematicaCreateMETSMetadataXML.process_xml_metadata() """ + from pathlib import Path from uuid import uuid4 @@ -13,9 +14,8 @@ from archivematicaCreateMETSMetadataXML import process_xml_metadata from importlib_metadata import version from lxml.etree import parse -from main.models import File from main.models import SIP - +from main.models import File METADATA_DIR = Path("objects") / "metadata" TRANSFER_METADATA_DIR = METADATA_DIR / "transfers" / "transfer_a" @@ -277,9 +277,7 @@ def test_validation( schema_uri = str(make_schema_file(schema)) settings.METADATA_XML_VALIDATION_ENABLED = True xml_validation = {"foo": schema_uri} - source_metadata_csv_contents = "filename,metadata,type\nobjects,{},mdtype".format( - xml_file - ) + source_metadata_csv_contents = f"filename,metadata,type\nobjects,{xml_file},mdtype" metadata_csv_path = sip.currentpath / TRANSFER_SOURCE_METADATA_CSV metadata_csv_path.write_text(source_metadata_csv_contents) metadata_file_rel_path = TRANSFER_METADATA_DIR / xml_file @@ -407,9 +405,7 @@ def test_source_metadata_errors(settings, make_mock_mets, sip): mock_mets, sip.currentpath, sip.uuid, "sip_type", xml_validation ) assert ( - "More than one entry in {} for path objects and type mdtype".format( - metadata_csv_path - ) + f"More than one entry in {metadata_csv_path} for path objects and type mdtype" in errors[0] ) diff --git a/tests/MCPClient/test_check_for_access_directory.py b/tests/MCPClient/test_check_for_access_directory.py index 87d734f0e5..7d44a01655 100644 --- a/tests/MCPClient/test_check_for_access_directory.py +++ b/tests/MCPClient/test_check_for_access_directory.py @@ -2,9 +2,9 @@ import pytest from client.job import Job from django.utils import timezone +from main.models import SIP from main.models import Event from main.models import File -from main.models import SIP @pytest.mark.django_db diff --git a/tests/MCPClient/test_create_aip_mets.py b/tests/MCPClient/test_create_aip_mets.py index 7f4410574a..a35dded304 100644 --- a/tests/MCPClient/test_create_aip_mets.py +++ b/tests/MCPClient/test_create_aip_mets.py @@ -863,8 +863,8 @@ def test_create_file_sec(self): assert ( self.state.fileNameToFileID is not None ), "fileNameToFileID mapping hasn't been generated" - assert self.count_dir_objects(self.objects_dir) == len( - self.state.fileNameToFileID + assert ( + self.count_dir_objects(self.objects_dir) == len(self.state.fileNameToFileID) ), "State hasn't been generated for all objects on disk, duplicate names may not be counted for" def test_get_included_structmap_invalid_mets(self): @@ -884,9 +884,7 @@ def test_get_included_structmap_invalid_mets(self): try: self.validate_mets(self.mets_xsd_path, broken_structmap_path) except etree.DocumentInvalid: - assert ( - True - ), "Expecting a validation error so that we know validation is working correctly" + assert True, "Expecting a validation error so that we know validation is working correctly" def test_get_included_structmap_valid_mets(self): """Test the valid output of custom structmaps in create_mets_v2.""" @@ -964,8 +962,9 @@ def test_get_included_structmap_valid_mets(self): self.validate_mets(self.mets_xsd_path, custom_structmap) assert custom_structmap.tag == f"{{{ns.metsNS}}}structMap" if not res.structmap_id: - assert custom_structmap.attrib["ID"].lower() == "structmap_{}".format( - self.state.globalStructMapCounter + assert ( + custom_structmap.attrib["ID"].lower() + == f"structmap_{self.state.globalStructMapCounter}" ), "structmap id is incorrect" else: assert ( @@ -1017,9 +1016,7 @@ def test_get_included_structmap_incomplete_mets(self): ) assert ( custom_structmap == [] - ), "Return from include_custom_structmap should be an empty array: {}".format( - custom_structmap - ) + ), f"Return from include_custom_structmap should be an empty array: {custom_structmap}" assert ( self.state.error_accumulator.error_count == 1 ), "error counter should be incremented on error" diff --git a/tests/MCPClient/test_create_dataverse_mets.py b/tests/MCPClient/test_create_dataverse_mets.py index 49cbc4a72e..f19327091e 100644 --- a/tests/MCPClient/test_create_dataverse_mets.py +++ b/tests/MCPClient/test_create_dataverse_mets.py @@ -1,5 +1,6 @@ #!/usr/bin/env python """Unit tests against the Convert Dataverse Structure MCP Client script.""" + import os from collections import namedtuple @@ -203,9 +204,7 @@ def test_generated_mets_(self, fixture, tmpdir): assert ( len(mets.all_files()) == fixture.all_file_count - ), "File count incorrect: '{}', expected '{}'".format( - len(mets.all_files()), fixture.all_file_count - ) + ), f"File count incorrect: '{len(mets.all_files())}', expected '{fixture.all_file_count}'" dir_counter = 0 item_counter = 0 @@ -217,14 +216,10 @@ def test_generated_mets_(self, fixture, tmpdir): assert ( dir_counter == fixture.dir_count - ), "Directory count incorrect: '{}', expected: '{}'".format( - dir_counter, fixture.dir_count - ) + ), f"Directory count incorrect: '{dir_counter}', expected: '{fixture.dir_count}'" assert ( item_counter == fixture.item_count - ), "Item count incorrect: '{}', expected: '{}'".format( - item_counter, fixture.item_count - ) + ), f"Item count incorrect: '{item_counter}', expected: '{fixture.item_count}'" @pytest.mark.parametrize( "fixture", [dv_1, dv_2, dv_3, dv_4, dv_5, dv_6, dv_7, dv_8] @@ -252,7 +247,7 @@ def test_mets_content(self, fixture, tmpdir): codebook = mets_root.findall(".//ddi:codebook", namespace) assert len(codebook) == 1, ( "Incorrect number of codebook entries " - "discovered: '{}' expected 1.".format(len(codebook)) + f"discovered: '{len(codebook)}' expected 1." ) # Test that we have a single title instance and the title matches what @@ -261,9 +256,7 @@ def test_mets_content(self, fixture, tmpdir): assert len(title) == 1 assert ( title[0].text == fixture.ddi_title - ), "DDI title: '{}' is not what was expected: '{}'".format( - title[0].text, fixture.ddi_title - ) + ), f"DDI title: '{title[0].text}' is not what was expected: '{fixture.ddi_title}'" # Test that we have a single PID and that it matches what is expected. pid = mets_root.findall(".//ddi:IDNo", namespace) @@ -273,14 +266,10 @@ def test_mets_content(self, fixture, tmpdir): pid_agency = pid[0].get("agency") assert ( pid_agency == fixture.pid_type - ), "PID type: '{}' is not what was expected: '{}'".format( - pid_agency, fixture.pid_type - ) + ), f"PID type: '{pid_agency}' is not what was expected: '{fixture.pid_type}'" assert ( pid[0].text == fixture.pid_value - ), "PID value: '{}' is not what was expected: '{}".format( - pid[0].text, fixture.pid_value - ) + ), f"PID value: '{pid[0].text}' is not what was expected: '{fixture.pid_value}" # Title is used in three other locations in the METS. Make sure that # they are found as well. diff --git a/tests/MCPClient/test_create_mets_v2.py b/tests/MCPClient/test_create_mets_v2.py index 933a8c9776..058ebbf2d3 100644 --- a/tests/MCPClient/test_create_mets_v2.py +++ b/tests/MCPClient/test_create_mets_v2.py @@ -6,12 +6,12 @@ from create_mets_v2 import createDMDIDsFromCSVMetadata from create_mets_v2 import main from lxml import etree +from main.models import SIP from main.models import DublinCore from main.models import Event from main.models import File from main.models import MetadataAppliesToType from main.models import RightsStatement -from main.models import SIP from main.models import SIPArrange from namespaces import NSMAP diff --git a/tests/MCPClient/test_create_transfer_mets.py b/tests/MCPClient/test_create_transfer_mets.py index b0e03b1060..549d737cd1 100644 --- a/tests/MCPClient/test_create_transfer_mets.py +++ b/tests/MCPClient/test_create_transfer_mets.py @@ -23,7 +23,6 @@ from metsrw.plugins.premisrw import PREMIS_3_0_NAMESPACES from version import get_preservation_system_identifier - PREMIS_NAMESPACES = PREMIS_3_0_NAMESPACES @@ -779,9 +778,7 @@ def test_transfer_mets_includes_other_rights( other_rights_note_db_obj = ( other_rights_db_obj.rightsstatementotherrightsinformationnote_set.first() ) - doc_identifier_db_obj = ( - other_rights_db_obj.rightsstatementotherrightsdocumentationidentifier_set.first() - ) + doc_identifier_db_obj = other_rights_db_obj.rightsstatementotherrightsdocumentationidentifier_set.first() other_rights_info = mets_xml.find( ".//premis:otherRightsInformation", namespaces=PREMIS_NAMESPACES diff --git a/tests/MCPClient/test_ensure_no_mutable_globals.py b/tests/MCPClient/test_ensure_no_mutable_globals.py index 8d3dab3ed1..aa3124af1d 100755 --- a/tests/MCPClient/test_ensure_no_mutable_globals.py +++ b/tests/MCPClient/test_ensure_no_mutable_globals.py @@ -11,6 +11,7 @@ access them. If the dict is non-empty, it is printed and an exit code of 1 is returned. Otherwise a happy message is printed and 0 is returned. """ + import importlib import logging import pathlib diff --git a/tests/MCPClient/test_filename_change.py b/tests/MCPClient/test_filename_change.py index 00508b1ebd..c7eb92c570 100644 --- a/tests/MCPClient/test_filename_change.py +++ b/tests/MCPClient/test_filename_change.py @@ -7,10 +7,10 @@ import pytest from client.job import Job from django.test import TestCase +from main.models import SIP from main.models import Directory from main.models import Event from main.models import File -from main.models import SIP from main.models import Transfer from main.models import User from pytest_django.asserts import assertQuerysetEqual diff --git a/tests/MCPClient/test_json_conversion.py b/tests/MCPClient/test_json_conversion.py index 9be07ac441..580ad55705 100644 --- a/tests/MCPClient/test_json_conversion.py +++ b/tests/MCPClient/test_json_conversion.py @@ -4,7 +4,6 @@ import json_metadata_to_csv from client.job import Job - JSON = '[{"dc.title": "This is a test item", "filename": "objects/test.txt"}]' CSV = "filename,dc.title\nobjects/test.txt,This is a test item\n" diff --git a/tests/MCPClient/test_load_premis_events_from_xml.py b/tests/MCPClient/test_load_premis_events_from_xml.py index 9216d87bbe..9e44b0050b 100644 --- a/tests/MCPClient/test_load_premis_events_from_xml.py +++ b/tests/MCPClient/test_load_premis_events_from_xml.py @@ -120,9 +120,7 @@ def test_get_premis_schema_with_invalid_schema(mocker, invalid_xsd_path): ) printfn.assert_called_once_with( f"Could not parse the PREMIS XML schema {invalid_xsd_path.as_posix()}", - "The XML document '{}' is not a schema document.".format( - invalid_xsd_path.as_posix() - ), + f"The XML document '{invalid_xsd_path.as_posix()}' is not a schema document.", file=sys.stderr, ) assert result is None @@ -725,9 +723,7 @@ def test_ensure_event_id_is_uuid_with_existent_event(mocker, existent_event_id): ) assert result != existent_event_id printfn.assert_called_once_with( - "Changed event identifier from {} to f11ea76b-1921-4152-b1b4-a93dbbfeaa11".format( - existent_event_id - ) + f"Changed event identifier from {existent_event_id} to f11ea76b-1921-4152-b1b4-a93dbbfeaa11" ) diff --git a/tests/MCPClient/test_parse_dataverse.py b/tests/MCPClient/test_parse_dataverse.py index b5c0343ffe..bdc0deebfd 100644 --- a/tests/MCPClient/test_parse_dataverse.py +++ b/tests/MCPClient/test_parse_dataverse.py @@ -1,5 +1,6 @@ #!/usr/bin/env python """Tests for the parse Dataverse functionality in Archivematica.""" + import os import metsrw diff --git a/tests/MCPClient/test_parse_external_mets.py b/tests/MCPClient/test_parse_external_mets.py index 4cb78d183c..eab609e8b3 100644 --- a/tests/MCPClient/test_parse_external_mets.py +++ b/tests/MCPClient/test_parse_external_mets.py @@ -6,7 +6,6 @@ from client.job import Job from main import models - THIS_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/tests/MCPClient/test_pid_components.py b/tests/MCPClient/test_pid_components.py index 45dedff144..5f14874268 100644 --- a/tests/MCPClient/test_pid_components.py +++ b/tests/MCPClient/test_pid_components.py @@ -5,6 +5,7 @@ The tests in this module cover both the two bind_pid(s) microservice jobs but also limited unit testing in create_mets_v2 (AIP METS generation). """ + import os from itertools import chain @@ -14,10 +15,10 @@ import namespaces as ns import pytest from client.job import Job +from main.models import SIP from main.models import DashboardSetting from main.models import Directory from main.models import File -from main.models import SIP from main.models import Transfer from pid_declaration import DeclarePIDs from pid_declaration import DeclarePIDsException diff --git a/tests/MCPClient/test_reingest_mets.py b/tests/MCPClient/test_reingest_mets.py index 5f76dfbb79..c8b7512d8a 100644 --- a/tests/MCPClient/test_reingest_mets.py +++ b/tests/MCPClient/test_reingest_mets.py @@ -525,8 +525,8 @@ def test__update_premis_object(self): for i in range(1, 4): path = ( - './/mets:techMD[@ID="techMD_{}"]' - "/mets:mdWrap/mets:xmlData/premis:object".format(i) + f'.//mets:techMD[@ID="techMD_{i}"]' + "/mets:mdWrap/mets:xmlData/premis:object" ) premis_object = root.find(path, namespaces=nsmap_for_premis2()) # This is what we're trying to avoid: PREMIS as the default ns. diff --git a/tests/MCPClient/test_restructure_dip_for_content_dm_upload.py b/tests/MCPClient/test_restructure_dip_for_content_dm_upload.py index 9bfb6ab7e8..27b4275ad6 100644 --- a/tests/MCPClient/test_restructure_dip_for_content_dm_upload.py +++ b/tests/MCPClient/test_restructure_dip_for_content_dm_upload.py @@ -5,7 +5,6 @@ import restructure_dip_for_content_dm_upload from client.job import Job - THIS_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/tests/MCPClient/test_upload_archivesspace.py b/tests/MCPClient/test_upload_archivesspace.py index e3a2e1ea87..2b47ce90aa 100644 --- a/tests/MCPClient/test_upload_archivesspace.py +++ b/tests/MCPClient/test_upload_archivesspace.py @@ -1,4 +1,5 @@ """Tests for the upload_archivesspace.py client script.""" + import uuid import pytest diff --git a/tests/MCPClient/test_validate_file.py b/tests/MCPClient/test_validate_file.py index cf33c88f47..1460a26bc9 100644 --- a/tests/MCPClient/test_validate_file.py +++ b/tests/MCPClient/test_validate_file.py @@ -6,10 +6,10 @@ from fpr.models import FPCommand from fpr.models import FPRule from fpr.models import FPTool +from main.models import SIP from main.models import Event from main.models import File from main.models import FileFormatVersion -from main.models import SIP from validate_file import main diff --git a/tests/MCPClient/test_verify_checksum.py b/tests/MCPClient/test_verify_checksum.py index 124c94bf1b..98ca588b82 100644 --- a/tests/MCPClient/test_verify_checksum.py +++ b/tests/MCPClient/test_verify_checksum.py @@ -21,6 +21,7 @@ mapped consistently to something that can be understood by users when debugging their preservation workflow. """ + import subprocess import pytest @@ -29,10 +30,10 @@ from main.models import File from main.models import Transfer from main.models import User -from verify_checksum import get_file_queryset from verify_checksum import Hashsum from verify_checksum import NoHashCommandAvailable from verify_checksum import PREMISFailure +from verify_checksum import get_file_queryset from verify_checksum import write_premis_event_per_file @@ -338,14 +339,10 @@ def test_write_premis_event_to_db(self, transfer): ), f"Length of the event objects is not '1', it is: {len(events)}" assert ( events[0].event_outcome == event_outcome - ), "Event outcome retrieved from the database is incorrect: {}".format( - events[0].event_outcome - ) + ), f"Event outcome retrieved from the database is incorrect: {events[0].event_outcome}" assert ( detail in events[0].event_detail - ), "Event detail retrieved from the database is incorrect: {}".format( - events[0].event_detail - ) + ), f"Event detail retrieved from the database is incorrect: {events[0].event_detail}" # Ensure that UUID creation has happened as anticipated. Will raise # a TypeError if otherwise. (events[0].event_id) @@ -372,9 +369,7 @@ def test_write_premis_event_to_db(self, transfer): assert set(agenttypes) == set(agent_types), "agent types don't match" assert ( _agent_count == number_of_expected_agents - ), "Number of agents is incorrect: {} expected: {}".format( - _agent_count, number_of_expected_agents - ) + ), f"Number of agents is incorrect: {_agent_count} expected: {number_of_expected_agents}" # Collect the different checksum algorithms written to ensure they # were all written independently in the function. for event in events: @@ -394,7 +389,5 @@ def test_get_file_obj_queryset(self, transfer): with pytest.raises(PREMISFailure): get_file_queryset(invalid_package_uuid) pytest.fail( - "Unable to find the transfer objects for the SIP: '{}' in the database".format( - invalid_package_uuid - ) + f"Unable to find the transfer objects for the SIP: '{invalid_package_uuid}' in the database" ) diff --git a/tests/MCPServer/test_integration.py b/tests/MCPServer/test_integration.py index 3487681bb6..b5ea3e5b5c 100644 --- a/tests/MCPServer/test_integration.py +++ b/tests/MCPServer/test_integration.py @@ -22,7 +22,6 @@ from server.tasks import TaskBackend from server.workflow import load as load_workflow - FIXTURES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures") INTEGRATION_TEST_PATH = os.path.join(FIXTURES_DIR, "workflow-integration-test.json") DEFAULT_STORAGE_LOCATION = "/api/v2/location/default/" @@ -140,9 +139,7 @@ def test_workflow_integration( assert isinstance(job, DirectoryClientScriptJob) assert job.exit_code == 0 - assert task.arguments == '"{}" "{}"'.format( - settings.PROCESSING_DIRECTORY, transfer.uuid - ) + assert task.arguments == f'"{settings.PROCESSING_DIRECTORY}" "{transfer.uuid}"' # Next job in chain should be queued assert package_queue.job_queue.qsize() == 1 diff --git a/tests/MCPServer/test_package.py b/tests/MCPServer/test_package.py index fda8289791..fad43d7e62 100644 --- a/tests/MCPServer/test_package.py +++ b/tests/MCPServer/test_package.py @@ -5,14 +5,14 @@ import pytest from django.core.exceptions import ValidationError from main import models +from server.packages import DIP +from server.packages import SIP +from server.packages import Package +from server.packages import Transfer from server.packages import _determine_transfer_paths from server.packages import _move_to_internal_shared_dir from server.packages import _pad_destination_filepath_if_it_already_exists from server.packages import create_package -from server.packages import DIP -from server.packages import Package -from server.packages import SIP -from server.packages import Transfer from server.queues import PackageQueue from server.workflow import Workflow @@ -147,9 +147,7 @@ def test_package_get_or_create_from_db_by_path_updates_model( model.objects.get(**{"uuid": package_id, loc_attribute: path_dst}) except (models.Transfer.DoesNotExist, ValidationError): pytest.fail( - "Method {}.get_or_create_from_db_by_path didn't update {} model".format( - package_class.__name__, model.__name__ - ) + f"Method {package_class.__name__}.get_or_create_from_db_by_path didn't update {model.__name__} model" ) @@ -240,9 +238,7 @@ def test_reload_file_list(tmp_path): for _file_count, file_info in enumerate(transfer.files(None, "/objects"), 1): if file_info.get("%fileUUID%") == "None": raise AssertionError( - "Non-database entries returned from package.files(): {}".format( - file_info - ) + f"Non-database entries returned from package.files(): {file_info}" ) assert _file_count == 3 diff --git a/tests/MCPServer/test_processing_config.py b/tests/MCPServer/test_processing_config.py index 726d6fde91..83e355614f 100644 --- a/tests/MCPServer/test_processing_config.py +++ b/tests/MCPServer/test_processing_config.py @@ -3,15 +3,14 @@ import pytest from server.processing_config import ChainChoicesField -from server.processing_config import get_processing_fields -from server.processing_config import processing_configuration_file_exists -from server.processing_config import processing_fields from server.processing_config import ReplaceDictField from server.processing_config import SharedChainChoicesField from server.processing_config import StorageLocationField +from server.processing_config import get_processing_fields +from server.processing_config import processing_configuration_file_exists +from server.processing_config import processing_fields from server.workflow import load - ASSETS_DIR = ( pathlib.Path(__file__).parent.parent.parent / "src" / "MCPServer" / "lib" / "assets" ) diff --git a/tests/MCPServer/test_rpc_server.py b/tests/MCPServer/test_rpc_server.py index 1ef1909a3a..333611940b 100644 --- a/tests/MCPServer/test_rpc_server.py +++ b/tests/MCPServer/test_rpc_server.py @@ -8,7 +8,6 @@ from server import rpc_server from server import workflow - ASSETS_DIR = ( pathlib.Path(__file__).parent.parent.parent / "src" / "MCPServer" / "lib" / "assets" ) diff --git a/tests/MCPServer/test_translation.py b/tests/MCPServer/test_translation.py index fb412bfe85..0ff10e46fd 100644 --- a/tests/MCPServer/test_translation.py +++ b/tests/MCPServer/test_translation.py @@ -1,5 +1,5 @@ -from server.translation import TranslationLabel from server.translation import UNKNOWN_TRANSLATION_LABEL +from server.translation import TranslationLabel def test_translation_label(mocker): diff --git a/tests/MCPServer/test_workflow.py b/tests/MCPServer/test_workflow.py index 2a1033dc40..23b3021875 100644 --- a/tests/MCPServer/test_workflow.py +++ b/tests/MCPServer/test_workflow.py @@ -78,8 +78,9 @@ def test_load_valid_document(path): assert isinstance(first_wdir.chain, workflow.BaseLink) # Workflow __str__ method - assert str(wf) == "Chains {}, links {}, watched directories: {}".format( - len(chains), len(links), len(wdirs) + assert ( + str(wf) + == f"Chains {len(chains)}, links {len(links)}, watched directories: {len(wdirs)}" ) # Test normalization of job statuses. diff --git a/tests/MCPServer/test_workflow_abilities.py b/tests/MCPServer/test_workflow_abilities.py index 3f43e39de3..b5c5bab3c9 100644 --- a/tests/MCPServer/test_workflow_abilities.py +++ b/tests/MCPServer/test_workflow_abilities.py @@ -5,7 +5,6 @@ from server import workflow from server.workflow_abilities import choice_is_available - ASSETS_DIR = ( pathlib.Path(__file__).parent.parent.parent / "src" / "MCPServer" / "lib" / "assets" ) diff --git a/tests/archivematicaCommon/test_appconfig.py b/tests/archivematicaCommon/test_appconfig.py index feec0dce4b..3b99461030 100644 --- a/tests/archivematicaCommon/test_appconfig.py +++ b/tests/archivematicaCommon/test_appconfig.py @@ -6,7 +6,6 @@ from appconfig import process_search_enabled from django.core.exceptions import ImproperlyConfigured - CONFIG_MAPPING = { "search_enabled": { "section": "Dashboard", @@ -33,11 +32,7 @@ ) def test_mapping_list_config_file(option, value, expect): config = Config(env_prefix="ARCHIVEMATICA_DASHBOARD", attrs=CONFIG_MAPPING) - config.read_defaults( - StringIO( - "[Dashboard]\n" "{option} = {value}".format(option=option, value=value) - ) - ) + config.read_defaults(StringIO("[Dashboard]\n" f"{option} = {value}")) if isinstance(expect, list): assert sorted(config.get("search_enabled")) == sorted(expect) else: diff --git a/tests/archivematicaCommon/test_bindpid.py b/tests/archivematicaCommon/test_bindpid.py index 22edc8984e..9bc10e43cd 100644 --- a/tests/archivematicaCommon/test_bindpid.py +++ b/tests/archivematicaCommon/test_bindpid.py @@ -2,7 +2,6 @@ import pytest from django.test import TestCase - VALID_ARG_DICT = { "entity_type": "file", "resolve_url_template_file": "https://access.my.org/access/{{ naming_authority }}/{{ pid }}", diff --git a/tests/archivematicaCommon/test_database_functions.py b/tests/archivematicaCommon/test_database_functions.py index e26a2c3569..2d36ba6d2e 100644 --- a/tests/archivematicaCommon/test_database_functions.py +++ b/tests/archivematicaCommon/test_database_functions.py @@ -7,7 +7,6 @@ from main.models import Event from main.models import File - AGENTS_FIXTURE = pathlib.Path(__file__).parent / "fixtures" / "agents.json" TEST_DATABASE_FUNCTIONS_FIXTURE = ( pathlib.Path(__file__).parent / "fixtures" / "test_database_functions.json" diff --git a/tests/archivematicaCommon/test_elasticsearch_functions.py b/tests/archivematicaCommon/test_elasticsearch_functions.py index b63033f8d7..786fe168f5 100644 --- a/tests/archivematicaCommon/test_elasticsearch_functions.py +++ b/tests/archivematicaCommon/test_elasticsearch_functions.py @@ -9,10 +9,10 @@ from components import helpers from django.utils.timezone import make_aware from lxml import etree +from main.models import SIP from main.models import Directory from main.models import File from main.models import Identifier -from main.models import SIP from main.models import Transfer THIS_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -424,17 +424,13 @@ def _bulk(client, actions, stats_only=False, *args, **kwargs): "rights.csv", ) for filename in files_in_metadata_directory: - path = "objects/metadata/transfers/DemoTransfer-{}/{}".format( - mets_object_id, filename - ) + path = f"objects/metadata/transfers/DemoTransfer-{mets_object_id}/{filename}" assert indexed_data[path] is None # Neither will the generated files during the transfer process generated_files = ("dc.json", "directory_tree.txt") for filename in generated_files: - path = "objects/metadata/transfers/DemoTransfer-{}/{}".format( - mets_object_id, filename - ) + path = f"objects/metadata/transfers/DemoTransfer-{mets_object_id}/{filename}" assert indexed_data[path] is None diff --git a/tests/archivematicaCommon/test_storage_service.py b/tests/archivematicaCommon/test_storage_service.py index 2d07b692e0..96df376017 100644 --- a/tests/archivematicaCommon/test_storage_service.py +++ b/tests/archivematicaCommon/test_storage_service.py @@ -2,6 +2,7 @@ Tests for the Archivematica Common Storage Service helpers. """ + import json import pytest @@ -44,9 +45,7 @@ def test_location_desc_from_slug(status_code, content_type, expected_result, moc return_value=mock_response(status_code, content_type, expected_result), ) res = location_description_from_slug("mock_uri") - assert res == expected_result, "Unexpected result for status test: {}".format( - status_code - ) + assert res == expected_result, f"Unexpected result for status test: {status_code}" @pytest.mark.parametrize( diff --git a/tests/dashboard/components/administration/test_processing_config.py b/tests/dashboard/components/administration/test_processing_config.py index 19730c093f..8a41f7d5e8 100644 --- a/tests/dashboard/components/administration/test_processing_config.py +++ b/tests/dashboard/components/administration/test_processing_config.py @@ -68,7 +68,7 @@ def test_edit_new_config(self, mock_conf_fields): ) @mock.patch( "components.administration.forms.ProcessingConfigurationForm.load_config", - side_effect=IOError(), + side_effect=OSError(), ) def test_edit_not_found_config(self, mock_load_config, mock_conf_fields): response = self.client.get("/administration/processing/edit/not_found_config/") diff --git a/tests/dashboard/components/administration/test_usage.py b/tests/dashboard/components/administration/test_usage.py index 593403288f..1123acfa3c 100644 --- a/tests/dashboard/components/administration/test_usage.py +++ b/tests/dashboard/components/administration/test_usage.py @@ -6,7 +6,6 @@ from django.conf import settings from django.test import TestCase - TEST_USER_FIXTURE = ( pathlib.Path(__file__).parent.parent.parent / "fixtures" / "test_user.json" ) diff --git a/tests/dashboard/components/archival_storage/test_archival_storage.py b/tests/dashboard/components/archival_storage/test_archival_storage.py index 1bb2534d12..b5f8d85b7b 100644 --- a/tests/dashboard/components/archival_storage/test_archival_storage.py +++ b/tests/dashboard/components/archival_storage/test_archival_storage.py @@ -35,7 +35,6 @@ from elasticsearch import Elasticsearch from main.models import DashboardSetting - THIS_DIR = os.path.dirname(os.path.abspath(__file__)) CONTENT_DISPOSITION = "Content-Disposition" CONTENT_TYPE = "Content-Type" @@ -148,9 +147,7 @@ def test_get_mets_known_mets(mocker, amsetup, admin_client, mets_hdr): def test_get_pointer_unknown_pointer(mocker, amsetup, admin_client): sip_uuid = "33333333-3333-3333-3333-333333333331" pointer_url = ( - "http://archivematica-storage-service:8000/api/v2/file/{}/pointer_file".format( - sip_uuid - ) + f"http://archivematica-storage-service:8000/api/v2/file/{sip_uuid}/pointer_file" ) mocker.patch("elasticSearchFunctions.get_client") mocker.patch("storageService.pointer_file_url", return_value=pointer_url) @@ -172,9 +169,7 @@ def test_get_pointer_unknown_pointer(mocker, amsetup, admin_client): def test_get_pointer_known_pointer(mocker, amsetup, admin_client, mets_hdr): sip_uuid = "44444444-4444-4444-4444-444444444444" pointer_url = ( - "http://archivematica-storage-service:8000/api/v2/file/{}/pointer_file".format( - sip_uuid - ) + f"http://archivematica-storage-service:8000/api/v2/file/{sip_uuid}/pointer_file" ) pointer_file = f"pointer.{sip_uuid}.xml" content_disposition = f'attachment; filename="{pointer_file}"' diff --git a/tests/dashboard/components/mcp/test_views.py b/tests/dashboard/components/mcp/test_views.py index fd37949cab..097c42f657 100644 --- a/tests/dashboard/components/mcp/test_views.py +++ b/tests/dashboard/components/mcp/test_views.py @@ -3,7 +3,6 @@ from externals import xmltodict from lxml import etree - FILE_FORMAT_IDENTIFICATION_CHOICES = """ b8d5dcca-b60e-40fd-b160-32447abbf7f8 diff --git a/tests/dashboard/fpr/test_forms.py b/tests/dashboard/fpr/test_forms.py index de818f849c..e37d230fe9 100644 --- a/tests/dashboard/fpr/test_forms.py +++ b/tests/dashboard/fpr/test_forms.py @@ -36,10 +36,7 @@ def test_FPRuleForm(self): self.assertFalse(form.is_valid()) self.assertEqual( form.non_field_errors(), - [ - "An identical FP rule already exists." - " See rule {}.".format(fprule.uuid) - ], + ["An identical FP rule already exists." f" See rule {fprule.uuid}."], ) @staticmethod diff --git a/tests/dashboard/test_api.py b/tests/dashboard/test_api.py index 7455f63878..93f1538678 100644 --- a/tests/dashboard/test_api.py +++ b/tests/dashboard/test_api.py @@ -11,15 +11,15 @@ from django.urls import reverse from django.utils.timezone import make_aware from lxml import etree +from main.models import PACKAGE_STATUS_COMPLETED_SUCCESSFULLY +from main.models import SIP from main.models import DashboardSetting from main.models import DublinCore from main.models import File from main.models import Job from main.models import LevelOfDescription from main.models import MetadataAppliesToType -from main.models import PACKAGE_STATUS_COMPLETED_SUCCESSFULLY from main.models import RightsStatement -from main.models import SIP from main.models import SIPArrange from main.models import Task from main.models import Transfer diff --git a/tests/dashboard/test_filesystem_ajax.py b/tests/dashboard/test_filesystem_ajax.py index f02da71225..2f482d9ce1 100644 --- a/tests/dashboard/test_filesystem_ajax.py +++ b/tests/dashboard/test_filesystem_ajax.py @@ -537,9 +537,7 @@ def test_copy_metadata_files( # Verify the copier helper was called with the right parameters _copy_from_transfer_sources_mock.assert_called_once_with( ["locationuuid:/some/path"], - "more/path/metadataReminder/mysip-{}/{}".format( - sip_uuid, expected_metadata_dir - ), + f"more/path/metadataReminder/mysip-{sip_uuid}/{expected_metadata_dir}", ) diff --git a/tests/dashboard/test_helpers.py b/tests/dashboard/test_helpers.py index 3e8ff40baa..2633404dcc 100644 --- a/tests/dashboard/test_helpers.py +++ b/tests/dashboard/test_helpers.py @@ -33,9 +33,7 @@ def mets_hdr(): def setup_ptr_info(sip_uuid): pointer_url = ( - "http://archivematica-storage-service:8000/api/v2/file/{}/pointer_file".format( - sip_uuid - ) + f"http://archivematica-storage-service:8000/api/v2/file/{sip_uuid}/pointer_file" ) pointer_file = f"pointer.{sip_uuid}.xml" content_disposition = f'attachment; filename="{pointer_file}"' @@ -109,9 +107,7 @@ def test_stream_mets_from_storage_success(mocker, mets_hdr, tmpdir): transfer_name="mets_transfer", sip_uuid=sip_uuid ) assert response.get(CONTENT_TYPE) == CONTENT_XML - assert response.get(CONTENT_DISPOSITION) == "attachment; filename={};".format( - mets_file - ) + assert response.get(CONTENT_DISPOSITION) == f"attachment; filename={mets_file};" response_text = get_streaming_response(response.streaming_content) assert response_text == mets_hdr diff --git a/tests/dashboard/test_migrations.py b/tests/dashboard/test_migrations.py index abda71c096..201d857ae5 100644 --- a/tests/dashboard/test_migrations.py +++ b/tests/dashboard/test_migrations.py @@ -2,7 +2,6 @@ import pytest - # Import package that starts with a number. mod = importlib.import_module("main.migrations.0066_archivesspace_base_url") @@ -43,6 +42,4 @@ def test_0066_get_host_and_port(base_url, host_and_port): """Test _get_host_and_port.""" assert host_and_port == mod._get_host_and_port( base_url - ), "Failed with arg {}".format( - base_url, - ) + ), f"Failed with arg {base_url}" diff --git a/tests/dashboard/test_rights.py b/tests/dashboard/test_rights.py index 9d2a4c440b..452f2a9e3b 100644 --- a/tests/dashboard/test_rights.py +++ b/tests/dashboard/test_rights.py @@ -2,13 +2,12 @@ import pytest from components.rights import load +from main.models import SIP from main.models import File from main.models import MetadataAppliesToType -from main.models import SIP from main.models import Transfer from metsrw.plugins import premisrw - RIGHTS_STATEMENT_IDENTIFIER = ( "rights_statement_identifier", ("rights_statement_identifier_type", "UUID"), diff --git a/tox.ini b/tox.ini index bd1d74fc97..69dac05f1a 100644 --- a/tox.ini +++ b/tox.ini @@ -52,7 +52,7 @@ setenv = migrations-dashboard: PYTHONPATH = {env:DASHBOARD_PYTHONPATH} migrations-storage-service: PYTHONPATH = {env:STORAGE_SERVICE_PYTHONPATH} # Setting HOME prevents Python's pwd module to ask for a real uid inside - # the container, and using {temp_dir} allows caching the pre-commit and flake8 + # the container, and using {temp_dir} allows caching the pre-commit # dependencies in the tox host linting: HOME = {temp_dir}/user changedir = @@ -77,20 +77,3 @@ commands = {[testenv:migrations-dashboard]commands} basepython = python3 deps = pre-commit commands = pre-commit run --all-files --show-diff-on-failure - -[flake8] -exclude = .tox, .git, __pycache__, .cache, build, dist, *.pyc, *.egg-info, .eggs -# Error codes: -# - https://flake8.pycqa.org/en/latest/user/error-codes.html -# - https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes -# - https://github.com/PyCQA/flake8-bugbear#list-of-warnings -# -# E203: whitespace before ‘,’, ‘;’, or ‘:’ -# E402: module level import not at top of file -# E501: line too long -# W503: line break before binary operator -ignore = - E203, - E402, - E501, - W503 From 6ea792c5ae2fb2f4d1ae2d9ed178fb67771c1c76 Mon Sep 17 00:00:00 2001 From: "Douglas Cerna (Soy Douglas)" Date: Thu, 30 May 2024 22:32:47 +0200 Subject: [PATCH 32/32] Allow triggering the test CI workflows manually --- .github/workflows/acceptance-test.yml | 16 +++++++++++++++- .github/workflows/test.yml | 25 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/.github/workflows/acceptance-test.yml b/.github/workflows/acceptance-test.yml index 6b8e175a3c..cf6e886b9e 100644 --- a/.github/workflows/acceptance-test.yml +++ b/.github/workflows/acceptance-test.yml @@ -1,11 +1,18 @@ --- name: "Acceptance Test" on: + workflow_dispatch: + inputs: + am_version: + description: "Archivematica ref (branch, tag or SHA to checkout)" + default: "qa/1.x" + required: true + type: "string" pull_request: types: [labeled] jobs: test: - if: github.event.label.name == 'AMAUAT' + if: "${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.label.name == 'AMAUAT') }}" name: "Test ${{ matrix.tag }} on ${{ matrix.browser }}" runs-on: "ubuntu-22.04" strategy: @@ -30,9 +37,16 @@ jobs: browser: Firefox steps: - name: "Check out repository" + if: "${{ github.event_name != 'workflow_dispatch' }}" uses: "actions/checkout@v4" with: submodules: true + - name: "Check out repository (manually triggered)" + if: "${{ github.event_name == 'workflow_dispatch' }}" + uses: "actions/checkout@v4" + with: + ref: "${{ inputs.am_version || 'qa/1.x' }}" + submodules: true - name: "Create external volumes" run: | make -C hack/ create-volumes diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d54048c6a8..6947225555 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,13 @@ --- name: "Test" on: + workflow_dispatch: + inputs: + am_version: + description: "Archivematica ref (branch, tag or SHA to checkout)" + default: "qa/1.x" + required: true + type: "string" pull_request: push: branches: @@ -38,7 +45,13 @@ jobs: python-version: "3.9" steps: - name: "Check out repository" + if: "${{ github.event_name != 'workflow_dispatch' }}" uses: "actions/checkout@v4" + - name: "Check out repository (manually triggered)" + if: "${{ github.event_name == 'workflow_dispatch' }}" + uses: "actions/checkout@v4" + with: + ref: "${{ inputs.am_version || 'qa/1.x' }}" - name: "Check out the archivematica-storage-service submodule" run: | git submodule update --init hack/submodules/archivematica-storage-service/ @@ -97,7 +110,13 @@ jobs: working-directory: "./src/dashboard/frontend/" steps: - name: "Check out repository" + if: "${{ github.event_name != 'workflow_dispatch' }}" + uses: "actions/checkout@v4" + - name: "Check out repository (manually triggered)" + if: "${{ github.event_name == 'workflow_dispatch' }}" uses: "actions/checkout@v4" + with: + ref: "${{ inputs.am_version || 'qa/1.x' }}" - name: "Set up Node JS" uses: "actions/setup-node@v4" with: @@ -116,7 +135,13 @@ jobs: runs-on: "ubuntu-22.04" steps: - name: "Check out repository" + if: "${{ github.event_name != 'workflow_dispatch' }}" + uses: "actions/checkout@v4" + - name: "Check out repository (manually triggered)" + if: "${{ github.event_name == 'workflow_dispatch' }}" uses: "actions/checkout@v4" + with: + ref: "${{ inputs.am_version || 'qa/1.x' }}" - name: "Set up Python 3.9" uses: "actions/setup-python@v5" with: