diff --git a/.github/workflows/acceptance-test.yml b/.github/workflows/acceptance-test.yml index 76a78f78c7..60a7b9b99a 100644 --- a/.github/workflows/acceptance-test.yml +++ b/.github/workflows/acceptance-test.yml @@ -30,7 +30,7 @@ jobs: browser: Firefox steps: - name: "Check out repository" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" with: submodules: true - name: "Create external volumes" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 20bd632f16..538f258d59 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,8 +13,17 @@ jobs: strategy: fail-fast: false matrix: - rule: ["mcp-server", "mcp-client", "dashboard", "archivematica-common"] - python-version: ["3.8", "3.9", "3.10"] + rule: [ + "mcp-server", + "mcp-client", + "dashboard", + "archivematica-common", + ] + python-version: [ + "3.8", + "3.9", + "3.10", + ] include: - rule: "storage-service" python-version: "3.9" @@ -22,26 +31,14 @@ jobs: python-version: "3.9" steps: - name: "Check out repository" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Check out the archivematica-storage-service submodule" run: | git submodule update --init hack/submodules/archivematica-storage-service/ - name: "Set up buildx" - uses: "docker/setup-buildx-action@v2" - id: buildx - with: - install: true - - name: "Set up docker cache" - uses: "actions/cache@v3" - with: - path: /tmp/.docker-cache-old - key: ${{ runner.os }}-docker-${{ matrix.python-version }}-${{ matrix.rule }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-docker-${{ matrix.python-version }}-${{ matrix.rule }}- - ${{ runner.os }}-docker-${{ matrix.python-version }}- - ${{ runner.os }}-docker- + uses: "docker/setup-buildx-action@v3" - name: "Build archivematica-tests image" - uses: "docker/build-push-action@v4" + uses: "docker/build-push-action@v5" with: context: . file: ./hack/Dockerfile @@ -51,13 +48,14 @@ jobs: tags: archivematica-tests:latest push: false load: true - cache-from: type=local,src=/tmp/.docker-cache-old - cache-to: type=local,dest=/tmp/.docker-cache-new,mode=max + cache-from: type=gha + cache-to: type=gha,mode=max - name: "Create external volumes" run: | make -C hack/ create-volumes - name: "Run make rule" run: | + docker buildx install make -C hack/ test-${{ matrix.rule }} env: TOXARGS: -vv @@ -71,10 +69,6 @@ jobs: fail_ci_if_error: false verbose: true name: ${{ matrix.rule }} - - name: "Set newest docker cache" - run: | - rm -rf /tmp/.docker-cache-old - mv /tmp/.docker-cache-new /tmp/.docker-cache-old frontend: name: "Test frontend" runs-on: "ubuntu-22.04" @@ -85,16 +79,12 @@ jobs: - name: "Check out repository" uses: "actions/checkout@v3" - name: "Set up Node JS" - uses: "actions/setup-node@v3" - with: - node-version: "8" - - name: "Cache npm packages" - uses: "actions/cache@v3" + uses: "actions/setup-node@v4" with: - path: "~/.npm" - key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}" - restore-keys: | - ${{ runner.os }}-node- + node-version: "14" + cache: "npm" + cache-dependency-path: | + **/package-lock.json - name: "Install frontend dependencies" run: | npm install diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9c2bbb3c74..c482932253 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: pretty-format-json args: [--no-ensure-ascii, --autofix] @@ -10,28 +10,29 @@ repos: src/(MCPClient/MCPServer|dashboard)/osdeps/.*\.json ) - repo: https://github.com/asottile/pyupgrade - rev: v3.10.1 + rev: v3.15.0 hooks: - id: pyupgrade args: [--py38-plus] - repo: https://github.com/asottile/reorder_python_imports - rev: v3.10.0 + rev: v3.12.0 hooks: - id: reorder-python-imports args: [--py38-plus] - repo: https://github.com/adamchainz/django-upgrade - rev: "1.14.1" + rev: "1.15.0" hooks: - id: django-upgrade args: [--target-version, "3.2"] - repo: https://github.com/psf/black - rev: "23.7.0" + rev: "23.10.1" hooks: - id: black args: [--safe, --quiet] - language_version: python3 - repo: https://github.com/pycqa/flake8 rev: "6.1.0" hooks: - id: flake8 - language_version: python3 + additional_dependencies: + - flake8-bugbear==23.9.16 + - flake8-comprehensions==3.14.0 diff --git a/hack/Makefile b/hack/Makefile index f9e0f962cb..0d10f13f38 100644 --- a/hack/Makefile +++ b/hack/Makefile @@ -218,6 +218,26 @@ upgrade-requirements-ss: ## Run pip-upgrade for Storage Service --entrypoint bash archivematica-storage-service \ -c "make pip-upgrade" +define amauat_make + docker compose \ + -f docker-compose.yml \ + -f docker-compose.acceptance-tests.yml \ + run \ + --workdir /src \ + -e XDG_CACHE_HOME=/tmp/pip-cache \ + --rm \ + --no-deps \ + --user $(CALLER_UID):$(CALLER_GID) \ + --entrypoint bash archivematica-acceptance-tests \ + -c "make $(1)" +endef + +compile-requirements-amauat: test-at-build ## Run pip-compile for AMAUATs + $(call amauat_make,pip-compile) + +upgrade-requirements-amauat: test-at-build ## Run pip-upgrade for AMAUATs + $(call amauat_make,pip-upgrade) + db: ## Connect to the MySQL server using the CLI. docker compose exec mysql mysql -hlocalhost -uroot -p12345 diff --git a/hack/docker-compose.acceptance-tests.yml b/hack/docker-compose.acceptance-tests.yml index 81233e58b4..a868bd7f2a 100644 --- a/hack/docker-compose.acceptance-tests.yml +++ b/hack/docker-compose.acceptance-tests.yml @@ -13,6 +13,7 @@ services: security_opt: - "seccomp:unconfined" volumes: + - "./submodules/archivematica-acceptance-tests/:/src/" - "/dev/shm:/dev/shm" - "./submodules/archivematica-sampledata:/home/archivematica/archivematica-sampledata:ro" - "archivematica_pipeline_data:/var/archivematica/sharedDirectory:ro" diff --git a/hack/submodules/archivematica-acceptance-tests b/hack/submodules/archivematica-acceptance-tests index d0d4a2ab08..19e8f4c465 160000 --- a/hack/submodules/archivematica-acceptance-tests +++ b/hack/submodules/archivematica-acceptance-tests @@ -1 +1 @@ -Subproject commit d0d4a2ab08cd7432946f6bebfe8ec4fbd6d3a85d +Subproject commit 19e8f4c46577043f73ccb562dbf0b0d85e42e0d5 diff --git a/hack/submodules/archivematica-storage-service b/hack/submodules/archivematica-storage-service index b19fa4b6c8..b0c5f93e93 160000 --- a/hack/submodules/archivematica-storage-service +++ b/hack/submodules/archivematica-storage-service @@ -1 +1 @@ -Subproject commit b19fa4b6c83e60bdaf1a93c7d29859b1c356d45e +Subproject commit b0c5f93e938e0502fd5cb31bd9c9efbf1c618fa2 diff --git a/requirements-dev.in b/requirements-dev.in index ed13bd27b3..629905d132 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -8,4 +8,3 @@ pytest-django pytest-mock pytest-randomly tox -vcrpy diff --git a/requirements-dev.txt b/requirements-dev.txt index 9f8d0a0aab..7ee68b394c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,11 +4,11 @@ # # pip-compile --allow-unsafe --output-file=requirements-dev.txt requirements-dev.in # -agentarchives==0.8.0 +agentarchives==0.9.0 # via -r requirements.txt -amclient==1.2.3 +amclient==1.3.0 # via -r requirements.txt -ammcpc==0.1.3 +ammcpc==0.2.0 # via -r requirements.txt asgiref==3.7.2 # via @@ -27,7 +27,7 @@ build==1.0.3 # via # -r requirements.txt # pip-tools -cachetools==5.3.1 +cachetools==5.3.2 # via tox certifi==2023.7.22 # via @@ -39,7 +39,7 @@ cffi==1.16.0 # cryptography chardet==5.2.0 # via tox -charset-normalizer==3.3.0 +charset-normalizer==3.3.2 # via # -r requirements.txt # requests @@ -55,7 +55,7 @@ coverage[toml]==7.3.2 # via # -r requirements-dev.in # pytest-cov -cryptography==41.0.4 +cryptography==41.0.5 # via # -r requirements.txt # josepy @@ -63,7 +63,7 @@ cryptography==41.0.4 # pyopenssl distlib==0.3.7 # via virtualenv -django==3.2.22 +django==3.2.23 # via # -r requirements.txt # django-auth-ldap @@ -74,7 +74,7 @@ django-auth-ldap==4.6.0 # via -r requirements.txt django-autoslug==1.9.9 # via -r requirements.txt -django-cas-ng==5.0.0 +django-cas-ng==5.0.1 # via -r requirements.txt django-csp==3.7 # via -r requirements.txt @@ -90,7 +90,7 @@ elasticsearch==6.8.2 # via -r requirements.txt exceptiongroup==1.1.3 # via pytest -filelock==3.12.4 +filelock==3.13.1 # via # tox # virtualenv @@ -100,7 +100,7 @@ gearman3 @ git+https://github.com/artefactual-labs/python-gearman.git@b68efc868c # via -r requirements.txt gevent==23.9.1 # via -r requirements.txt -greenlet==3.0.0 +greenlet==3.0.1 # via # -r requirements.txt # gevent @@ -110,13 +110,12 @@ idna==3.4 # via # -r requirements.txt # requests - # yarl importlib-metadata==6.8.0 # via # -r requirements.txt # build # pytest-randomly -importlib-resources==6.1.0 +importlib-resources==6.1.1 # via # -r requirements.txt # opf-fido @@ -124,11 +123,11 @@ iniconfig==2.0.0 # via pytest inotify-simple==1.3.5 # via -r requirements.txt -josepy==1.13.0 +josepy==1.14.0 # via # -r requirements.txt # mozilla-django-oidc -jsonschema==4.19.1 +jsonschema==4.19.2 # via -r requirements.txt jsonschema-specifications==2023.7.1 # via @@ -142,14 +141,12 @@ lxml==4.9.3 # ammcpc # metsrw # python-cas -metsrw==0.4.0 +metsrw==0.5.0 # via -r requirements.txt mockldap @ git+https://github.com/artefactual-labs/mockldap@v0.3.1 # via -r requirements-dev.in mozilla-django-oidc==3.0.0 # via -r requirements.txt -multidict==6.0.4 - # via yarl mysqlclient==2.2.0 # via # -r requirements.txt @@ -178,7 +175,7 @@ pluggy==1.3.0 # via # pytest # tox -prometheus-client==0.17.1 +prometheus-client==0.18.0 # via # -r requirements.txt # django-prometheus @@ -195,7 +192,7 @@ pycparser==2.21 # via # -r requirements.txt # cffi -pyopenssl==23.2.0 +pyopenssl==23.3.0 # via # -r requirements.txt # josepy @@ -205,7 +202,7 @@ pyproject-hooks==1.0.0 # via # -r requirements.txt # build -pytest==7.4.2 +pytest==7.4.3 # via # -r requirements-dev.in # pytest-cov @@ -214,9 +211,9 @@ pytest==7.4.2 # pytest-randomly pytest-cov==4.1.0 # via -r requirements-dev.in -pytest-django==4.5.2 +pytest-django==4.7.0 # via -r requirements-dev.in -pytest-mock==3.11.1 +pytest-mock==3.12.0 # via -r requirements-dev.in pytest-randomly==3.15.0 # via -r requirements-dev.in @@ -241,8 +238,6 @@ pytz==2023.3.post1 # via # -r requirements.txt # django -pyyaml==6.0.1 - # via vcrpy referencing==0.30.2 # via # -r requirements.txt @@ -256,7 +251,7 @@ requests==2.31.0 # mozilla-django-oidc # opf-fido # python-cas -rpds-py==0.10.6 +rpds-py==0.12.0 # via # -r requirements.txt # jsonschema @@ -264,7 +259,6 @@ rpds-py==0.10.6 six==1.16.0 # via # -r requirements.txt - # amclient # opf-fido # python-cas # python-dateutil @@ -290,27 +284,20 @@ typing-extensions==4.8.0 # asgiref unidecode==1.3.7 # via -r requirements.txt -urllib3==1.26.17 +urllib3==2.1.0 # via # -r requirements.txt # amclient # elasticsearch # requests - # vcrpy -vcrpy==5.1.0 - # via -r requirements-dev.in -virtualenv==20.24.5 +virtualenv==20.24.6 # via tox -wheel==0.41.2 +wheel==0.41.3 # via # -r requirements.txt # pip-tools whitenoise==6.6.0 # via -r requirements.txt -wrapt==1.15.0 - # via vcrpy -yarl==1.9.2 - # via vcrpy zipp==3.17.0 # via # -r requirements.txt @@ -326,14 +313,13 @@ zope-interface==6.1 # gevent # The following packages are considered to be unsafe in a requirements file: -pip==23.2.1 +pip==23.3.1 # via # -r requirements.txt # pip-tools setuptools==68.2.2 # via # -r requirements.txt - # josepy # pip-tools # zope-event # zope-interface diff --git a/requirements.txt b/requirements.txt index d7cdf23a0e..1693252f77 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,11 +4,11 @@ # # pip-compile --allow-unsafe --output-file=requirements.txt requirements.in # -agentarchives==0.8.0 +agentarchives==0.9.0 # via -r requirements.in -amclient==1.2.3 +amclient==1.3.0 # via -r requirements.in -ammcpc==0.1.3 +ammcpc==0.2.0 # via -r requirements.in asgiref==3.7.2 # via django @@ -26,18 +26,18 @@ certifi==2023.7.22 # via requests cffi==1.16.0 # via cryptography -charset-normalizer==3.3.0 +charset-normalizer==3.3.2 # via requests clamd==1.0.2 # via -r requirements.in click==8.1.7 # via pip-tools -cryptography==41.0.4 +cryptography==41.0.5 # via # josepy # mozilla-django-oidc # pyopenssl -django==3.2.22 +django==3.2.23 # via # -r requirements.in # django-auth-ldap @@ -48,7 +48,7 @@ django-auth-ldap==4.6.0 # via -r requirements.in django-autoslug==1.9.9 # via -r requirements.in -django-cas-ng==5.0.0 +django-cas-ng==5.0.1 # via -r requirements.in django-csp==3.7 # via -r requirements.in @@ -66,7 +66,7 @@ gearman3 @ git+https://github.com/artefactual-labs/python-gearman.git@b68efc868c # via -r requirements.in gevent==23.9.1 # via -r requirements.in -greenlet==3.0.0 +greenlet==3.0.1 # via gevent gunicorn==21.2.0 # via -r requirements.in @@ -76,13 +76,13 @@ importlib-metadata==6.8.0 # via # -r requirements.in # build -importlib-resources==6.1.0 +importlib-resources==6.1.1 # via opf-fido inotify-simple==1.3.5 # via -r requirements.in -josepy==1.13.0 +josepy==1.14.0 # via mozilla-django-oidc -jsonschema==4.19.1 +jsonschema==4.19.2 # via -r requirements.in jsonschema-specifications==2023.7.1 # via jsonschema @@ -94,7 +94,7 @@ lxml==4.9.3 # ammcpc # metsrw # python-cas -metsrw==0.4.0 +metsrw==0.5.0 # via -r requirements.in mozilla-django-oidc==3.0.0 # via -r requirements.in @@ -110,7 +110,7 @@ packaging==23.2 # gunicorn pip-tools==7.3.0 # via -r requirements.in -prometheus-client==0.17.1 +prometheus-client==0.18.0 # via # -r requirements.in # django-prometheus @@ -122,7 +122,7 @@ pyasn1-modules==0.3.0 # via python-ldap pycparser==2.21 # via cffi -pyopenssl==23.2.0 +pyopenssl==23.3.0 # via josepy pyproject-hooks==1.0.0 # via build @@ -152,13 +152,12 @@ requests==2.31.0 # mozilla-django-oidc # opf-fido # python-cas -rpds-py==0.10.6 +rpds-py==0.12.0 # via # jsonschema # referencing six==1.16.0 # via - # amclient # opf-fido # python-cas # python-dateutil @@ -173,12 +172,12 @@ typing-extensions==4.8.0 # via asgiref unidecode==1.3.7 # via -r requirements.in -urllib3==1.26.17 +urllib3==2.1.0 # via # amclient # elasticsearch # requests -wheel==0.41.2 +wheel==0.41.3 # via pip-tools whitenoise==6.6.0 # via -r requirements.in @@ -192,13 +191,12 @@ zope-interface==6.1 # via gevent # The following packages are considered to be unsafe in a requirements file: -pip==23.2.1 +pip==23.3.1 # via # -r requirements.in # pip-tools setuptools==68.2.2 # via - # josepy # pip-tools # zope-event # zope-interface diff --git a/src/MCPClient/lib/archivematicaClient.py b/src/MCPClient/lib/archivematicaClient.py index d54c001139..fc1eb8382e 100755 --- a/src/MCPClient/lib/archivematicaClient.py +++ b/src/MCPClient/lib/archivematicaClient.py @@ -46,9 +46,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClient -# @author Joseph Perry import configparser import logging import os diff --git a/src/MCPClient/lib/archivematicaClientModules b/src/MCPClient/lib/archivematicaClientModules index d955a7bbef..a432bf8088 100644 --- a/src/MCPClient/lib/archivematicaClientModules +++ b/src/MCPClient/lib/archivematicaClientModules @@ -15,10 +15,6 @@ # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage Ingest -# @author Joseph Perry -# @version svn: $Id$ #!!! this is a list of supported commands/tasks, not modules #the version numbers are versions of the protocol, not versions of the programs themselves. diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataCSV.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataCSV.py index ff9ccfcb28..57fb1130e0 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataCSV.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataCSV.py @@ -16,10 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry -# @version svn: $Id$ # /src/dashboard/src/main/models.py import collections import csv diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRights.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRights.py index 26fd5409d7..cab039e02d 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRights.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRights.py @@ -16,9 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry # /src/dashboard/src/main/models.py import sys import uuid diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRightsDspaceMDRef.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRightsDspaceMDRef.py index 84a9665ad5..229bad2b33 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRightsDspaceMDRef.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRightsDspaceMDRef.py @@ -16,9 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import sys diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py index aceecb7e30..ef908132ac 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py @@ -16,10 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry -# @version svn: $Id$ import os import sys from datetime import datetime diff --git a/src/MCPClient/lib/clientScripts/archivematica_clamscan.py b/src/MCPClient/lib/clientScripts/archivematica_clamscan.py index 0802f32046..aceaa88c32 100755 --- a/src/MCPClient/lib/clientScripts/archivematica_clamscan.py +++ b/src/MCPClient/lib/clientScripts/archivematica_clamscan.py @@ -122,10 +122,9 @@ def scan(self, path): state, details = result[result_key] except Exception as err: passed = ClamdScanner.clamd_exception_handler(err) - finally: - if state == "OK": - passed = True - return passed, state, details + if state == "OK": + passed = True + return passed, state, details @staticmethod def clamd_exception_handler(err): @@ -297,7 +296,7 @@ def get_size(file_uuid, path): # Our fallback. try: return os.path.getsize(path) - except: + except Exception: return None @@ -349,7 +348,7 @@ def scan_file(event_queue, file_uuid, path, date, task_uuid): else: passed, state, details = None, None, None - except: + except Exception: logger.error("Unexpected error scanning file %s", path, exc_info=True) return 1 else: diff --git a/src/MCPClient/lib/clientScripts/bag_with_empty_directories.py b/src/MCPClient/lib/clientScripts/bag_with_empty_directories.py index dccb7973b7..09d2282ed2 100755 --- a/src/MCPClient/lib/clientScripts/bag_with_empty_directories.py +++ b/src/MCPClient/lib/clientScripts/bag_with_empty_directories.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import argparse import multiprocessing import os @@ -56,7 +53,7 @@ def create_directories(base_dir, dir_list): directory = os.path.join(base_dir, directory) try: os.makedirs(directory) - except os.error: + except OSError: pass diff --git a/src/MCPClient/lib/clientScripts/change_names.py b/src/MCPClient/lib/clientScripts/change_names.py index 96655da16f..3ed60176f8 100755 --- a/src/MCPClient/lib/clientScripts/change_names.py +++ b/src/MCPClient/lib/clientScripts/change_names.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage MCPClient -# @author Joseph Perry import os import re import shutil diff --git a/src/MCPClient/lib/clientScripts/change_object_names.py b/src/MCPClient/lib/clientScripts/change_object_names.py index 7d69cc67f2..1614054c38 100755 --- a/src/MCPClient/lib/clientScripts/change_object_names.py +++ b/src/MCPClient/lib/clientScripts/change_object_names.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import unicodedata import uuid diff --git a/src/MCPClient/lib/clientScripts/change_sip_name.py b/src/MCPClient/lib/clientScripts/change_sip_name.py index 8c957f0804..0eb9eecdc9 100755 --- a/src/MCPClient/lib/clientScripts/change_sip_name.py +++ b/src/MCPClient/lib/clientScripts/change_sip_name.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import sys diff --git a/src/MCPClient/lib/clientScripts/check_for_access_directory.py b/src/MCPClient/lib/clientScripts/check_for_access_directory.py index 27f926dc2c..d7c833cead 100755 --- a/src/MCPClient/lib/clientScripts/check_for_access_directory.py +++ b/src/MCPClient/lib/clientScripts/check_for_access_directory.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import sys from optparse import OptionParser @@ -169,7 +166,7 @@ def call(jobs): os.mkdir(DIPDirectory) if not os.path.isdir(os.path.join(DIPDirectory, "objects")): os.mkdir(os.path.join(DIPDirectory, "objects")) - except: + except Exception: job.pyprint("error creating DIP directory") exitCode = main( diff --git a/src/MCPClient/lib/clientScripts/check_for_service_directory.py b/src/MCPClient/lib/clientScripts/check_for_service_directory.py index 8fc4a16b5a..a88dfe712d 100755 --- a/src/MCPClient/lib/clientScripts/check_for_service_directory.py +++ b/src/MCPClient/lib/clientScripts/check_for_service_directory.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import re from optparse import OptionParser @@ -35,7 +32,7 @@ def something(job, SIPDirectory, serviceDirectory, objectsDirectory, SIPUUID, da exitCode = 0 job.pyprint(SIPDirectory) # For every file, & directory Try to find the matching file & directory in the objects directory - for path, dirs, files in os.walk(serviceDirectory): + for path, _, files in os.walk(serviceDirectory): for file in files: servicePreExtension = "_me" originalPreExtension = "_m" @@ -82,7 +79,7 @@ def regular(SIPDirectory, objectsDirectory, SIPUUID, date): if not searchForRegularExpressions: return - for path, dirs, files in os.walk(objectsDirectory): + for path, _, files in os.walk(objectsDirectory): for file in files: m = re.search(r"_me\.[a-zA-Z0-9]{2,4}$", file) if m is not None: diff --git a/src/MCPClient/lib/clientScripts/check_for_submission_documentation.py b/src/MCPClient/lib/clientScripts/check_for_submission_documentation.py index 98a56fdd12..7df5ecb9fe 100755 --- a/src/MCPClient/lib/clientScripts/check_for_submission_documentation.py +++ b/src/MCPClient/lib/clientScripts/check_for_submission_documentation.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import sys diff --git a/src/MCPClient/lib/clientScripts/check_transfer_directory_for_objects.py b/src/MCPClient/lib/clientScripts/check_transfer_directory_for_objects.py index 49cc2fe6e9..9d95aad567 100755 --- a/src/MCPClient/lib/clientScripts/check_transfer_directory_for_objects.py +++ b/src/MCPClient/lib/clientScripts/check_transfer_directory_for_objects.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os diff --git a/src/MCPClient/lib/clientScripts/copy_thumbnails_to_dip_directory.py b/src/MCPClient/lib/clientScripts/copy_thumbnails_to_dip_directory.py index 601c36165c..6e0396d2a1 100755 --- a/src/MCPClient/lib/clientScripts/copy_thumbnails_to_dip_directory.py +++ b/src/MCPClient/lib/clientScripts/copy_thumbnails_to_dip_directory.py @@ -15,8 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript import os import shutil diff --git a/src/MCPClient/lib/clientScripts/copy_transfer_submission_documentation.py b/src/MCPClient/lib/clientScripts/copy_transfer_submission_documentation.py index 085437f4f5..f5f2399304 100755 --- a/src/MCPClient/lib/clientScripts/copy_transfer_submission_documentation.py +++ b/src/MCPClient/lib/clientScripts/copy_transfer_submission_documentation.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import re import shutil diff --git a/src/MCPClient/lib/clientScripts/copy_transfers_metadata_and_logs.py b/src/MCPClient/lib/clientScripts/copy_transfers_metadata_and_logs.py index badc8cb841..5ba0874ff9 100755 --- a/src/MCPClient/lib/clientScripts/copy_transfers_metadata_and_logs.py +++ b/src/MCPClient/lib/clientScripts/copy_transfers_metadata_and_logs.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import shutil import traceback diff --git a/src/MCPClient/lib/clientScripts/create_event.py b/src/MCPClient/lib/clientScripts/create_event.py index 65b78f357d..b026757a05 100755 --- a/src/MCPClient/lib/clientScripts/create_event.py +++ b/src/MCPClient/lib/clientScripts/create_event.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import uuid from optparse import OptionParser diff --git a/src/MCPClient/lib/clientScripts/create_mets_v2.py b/src/MCPClient/lib/clientScripts/create_mets_v2.py index 48865fc03d..59ffa8f003 100755 --- a/src/MCPClient/lib/clientScripts/create_mets_v2.py +++ b/src/MCPClient/lib/clientScripts/create_mets_v2.py @@ -16,9 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import collections import copy import os @@ -1004,7 +1001,7 @@ def createFileSec( dspaceMetsDMDID = None try: directoryContents = sorted(os.listdir(directoryPath)) - except os.error: + except OSError: # Directory doesn't exist job.pyprint(directoryPath, "doesn't exist", file=sys.stderr) return diff --git a/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py b/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py index 2f6d58389f..a9f769d495 100755 --- a/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py +++ b/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import shutil import sys diff --git a/src/MCPClient/lib/clientScripts/create_sips_from_trim_transfer_containers.py b/src/MCPClient/lib/clientScripts/create_sips_from_trim_transfer_containers.py index 4f7b85faa0..564fb501b2 100755 --- a/src/MCPClient/lib/clientScripts/create_sips_from_trim_transfer_containers.py +++ b/src/MCPClient/lib/clientScripts/create_sips_from_trim_transfer_containers.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import shutil import sys diff --git a/src/MCPClient/lib/clientScripts/create_transfer_mets.py b/src/MCPClient/lib/clientScripts/create_transfer_mets.py index 9574da9202..61be662130 100755 --- a/src/MCPClient/lib/clientScripts/create_transfer_mets.py +++ b/src/MCPClient/lib/clientScripts/create_transfer_mets.py @@ -16,8 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript import argparse import logging import os @@ -258,11 +256,11 @@ def load_rights_data_from_db(self): ) for rights in transfer_rights: - for path, fsentry in self.file_index.items(): + for _, fsentry in self.file_index.items(): premis_rights = rights_to_premis(rights, fsentry.file_uuid) fsentry.add_premis_rights(premis_rights) - for path, fsentry in self.file_index.items(): + for _, fsentry in self.file_index.items(): file_rights = self.rights_queryset.filter( metadataappliestoidentifier=fsentry.file_uuid, metadataappliestotype_id=self.FILE_RIGHTS_LOOKUP_UUID, diff --git a/src/MCPClient/lib/clientScripts/determine_aip_version_key_exit_code.py b/src/MCPClient/lib/clientScripts/determine_aip_version_key_exit_code.py index 594b3f45df..c53d25e39c 100755 --- a/src/MCPClient/lib/clientScripts/determine_aip_version_key_exit_code.py +++ b/src/MCPClient/lib/clientScripts/determine_aip_version_key_exit_code.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import sys diff --git a/src/MCPClient/lib/clientScripts/email_fail_report.py b/src/MCPClient/lib/clientScripts/email_fail_report.py index e95a7dfb89..8d8872b371 100755 --- a/src/MCPClient/lib/clientScripts/email_fail_report.py +++ b/src/MCPClient/lib/clientScripts/email_fail_report.py @@ -57,7 +57,7 @@ def send_email(subject, to, content): recipient_list=to, html_message=content, ) - except: + except Exception: logger.exception("Report email was not delivered") raise else: @@ -171,7 +171,7 @@ def get_content_for(unit_type, unit_name, unit_uuid, html=True): else: root.append(t1) etree.SubElement(root, "p") - except: + except Exception: pass html2code = get_unit_job_log_html(unit_uuid) @@ -255,7 +255,7 @@ def call(jobs): # Generate report in plain text and store it in the database with transaction.atomic(): - for arg in reports_to_store: + for args in reports_to_store: content = get_content_for( args.unit_type, args.unit_name, args.unit_uuid, html=False ) diff --git a/src/MCPClient/lib/clientScripts/extract_contents.py b/src/MCPClient/lib/clientScripts/extract_contents.py index ba67acc8c3..b32b108e21 100755 --- a/src/MCPClient/lib/clientScripts/extract_contents.py +++ b/src/MCPClient/lib/clientScripts/extract_contents.py @@ -131,7 +131,7 @@ def main(job, transfer_uuid, sip_directory, date, task_uuid, delete=False): try: format_id = FileFormatVersion.objects.get(file_uuid=file_.uuid) # Can't do anything if the file wasn't identified in the previous step - except: + except Exception: job.pyprint( "Not extracting contents from", os.path.basename(file_.currentlocation.decode()), diff --git a/src/MCPClient/lib/clientScripts/extract_maildir_attachments.py b/src/MCPClient/lib/clientScripts/extract_maildir_attachments.py index 0c7bedce69..2b0e70ccbb 100755 --- a/src/MCPClient/lib/clientScripts/extract_maildir_attachments.py +++ b/src/MCPClient/lib/clientScripts/extract_maildir_attachments.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import mailbox import os import sys @@ -49,7 +46,7 @@ def __init__(self): def writeFile(filePath, fileContents): try: os.makedirs(os.path.dirname(filePath)) - except: + except Exception: pass FILE = open(filePath, "w") FILE.writelines(fileContents) @@ -62,9 +59,11 @@ def addFile( transferUUID, date, eventDetail="", - fileUUID=uuid.uuid4().__str__(), + fileUUID=None, ): - taskUUID = uuid.uuid4().__str__() + if fileUUID is None: + fileUUID = str(uuid.uuid4()) + taskUUID = str(uuid.uuid4()) filePathRelativeToSIP = filePath.replace(transferPath, "%transferDirectory%", 1) addFileToTransfer( filePathRelativeToSIP, @@ -75,7 +74,7 @@ def addFile( sourceType="unpacking", eventDetail=eventDetail, ) - updateSizeAndChecksum(fileUUID, filePath, date, uuid.uuid4.__str__()) + updateSizeAndChecksum(fileUUID, filePath, date, str(uuid.uuid4())) def getFileUUIDofSourceFile(transferUUID, sourceFilePath): @@ -96,8 +95,10 @@ def addKeyFileToNormalizeMaildirOffOf( transferUUID, date, eventDetail="", - fileUUID=uuid.uuid4().__str__(), + fileUUID=None, ): + if fileUUID is None: + fileUUID = str(uuid.uuid4()) basename = os.path.basename(mirrorDir) dirname = os.path.dirname(mirrorDir) outFile = os.path.join(dirname, basename + ".archivematicaMaildir") @@ -132,7 +133,7 @@ def handle_job(job): mirrorDir = os.path.join(transferDir, "objects", "attachments") try: os.makedirs(mirrorDir) - except os.error: + except OSError: pass # print "Extracting attachments from: " + maildir root = etree.Element("ArchivematicaMaildirAttachmentExtractionRecord") @@ -251,7 +252,7 @@ def handle_job(job): mirrorDir = os.path.join(transferDir, "objects/attachments", maildirsub) try: os.makedirs(mirrorDir) - except: + except Exception: pass eventDetail = "added for normalization purposes" fileUUID = uuid.uuid4().__str__() diff --git a/src/MCPClient/lib/clientScripts/extract_zipped_transfer.py b/src/MCPClient/lib/clientScripts/extract_zipped_transfer.py index 1a7d7b4dc2..9d968036e6 100755 --- a/src/MCPClient/lib/clientScripts/extract_zipped_transfer.py +++ b/src/MCPClient/lib/clientScripts/extract_zipped_transfer.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import argparse import os import shutil diff --git a/src/MCPClient/lib/clientScripts/fits.py b/src/MCPClient/lib/clientScripts/fits.py index 9271a0b237..2a93bd1723 100755 --- a/src/MCPClient/lib/clientScripts/fits.py +++ b/src/MCPClient/lib/clientScripts/fits.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import tempfile @@ -91,7 +88,7 @@ def main(target, xml_file, date, event_uuid, file_uuid, file_grpuse): try: tree = etree.parse(temp_file) - except: + except Exception: logger.exception("Failed to read Fits's XML.") return 2 diff --git a/src/MCPClient/lib/clientScripts/identify_dspace_files.py b/src/MCPClient/lib/clientScripts/identify_dspace_files.py index 1c07ba3d95..17e18dda28 100755 --- a/src/MCPClient/lib/clientScripts/identify_dspace_files.py +++ b/src/MCPClient/lib/clientScripts/identify_dspace_files.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import django diff --git a/src/MCPClient/lib/clientScripts/identify_dspace_mets_files.py b/src/MCPClient/lib/clientScripts/identify_dspace_mets_files.py index 923cdcd884..f2f2a6a496 100755 --- a/src/MCPClient/lib/clientScripts/identify_dspace_mets_files.py +++ b/src/MCPClient/lib/clientScripts/identify_dspace_mets_files.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import django django.setup() diff --git a/src/MCPClient/lib/clientScripts/is_maildir_aip.py b/src/MCPClient/lib/clientScripts/is_maildir_aip.py index 2e091059a8..c94613a3ea 100755 --- a/src/MCPClient/lib/clientScripts/is_maildir_aip.py +++ b/src/MCPClient/lib/clientScripts/is_maildir_aip.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import mailbox import os @@ -34,7 +31,7 @@ def isMaildir(path): for maildirsub2 in os.listdir(maildir): maildirsub = os.path.join(maildir, maildirsub2) mailbox.Maildir(maildirsub, None) - except: + except Exception: return False return True diff --git a/src/MCPClient/lib/clientScripts/json_metadata_to_csv.py b/src/MCPClient/lib/clientScripts/json_metadata_to_csv.py index 6d25b4b28d..2ff69a1731 100755 --- a/src/MCPClient/lib/clientScripts/json_metadata_to_csv.py +++ b/src/MCPClient/lib/clientScripts/json_metadata_to_csv.py @@ -30,7 +30,7 @@ def fetch_keys(objects): # Column order is important so the output is consistent. # "filename" and "parts" must be column 0. # (They are mutually exclusive.) - keys = sorted(list(keys)) + keys = sorted(keys) if "filename" in keys: keys.remove("filename") keys.insert(0, "filename") diff --git a/src/MCPClient/lib/clientScripts/load_labels_from_csv.py b/src/MCPClient/lib/clientScripts/load_labels_from_csv.py index 34df80eb95..6284eb299e 100755 --- a/src/MCPClient/lib/clientScripts/load_labels_from_csv.py +++ b/src/MCPClient/lib/clientScripts/load_labels_from_csv.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import csv import os diff --git a/src/MCPClient/lib/clientScripts/manual_normalization_check_for_manual_normalization_directory.py b/src/MCPClient/lib/clientScripts/manual_normalization_check_for_manual_normalization_directory.py index 304db727c7..acb5ac730f 100755 --- a/src/MCPClient/lib/clientScripts/manual_normalization_check_for_manual_normalization_directory.py +++ b/src/MCPClient/lib/clientScripts/manual_normalization_check_for_manual_normalization_directory.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os diff --git a/src/MCPClient/lib/clientScripts/manual_normalization_identify_files_included.py b/src/MCPClient/lib/clientScripts/manual_normalization_identify_files_included.py index d89c389bec..2e75037372 100755 --- a/src/MCPClient/lib/clientScripts/manual_normalization_identify_files_included.py +++ b/src/MCPClient/lib/clientScripts/manual_normalization_identify_files_included.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry # fileOperations requires Django to be set up import django from django.db import transaction diff --git a/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py b/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py index 750991ee05..99a355fb56 100755 --- a/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py +++ b/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import django @@ -144,7 +141,7 @@ def main(job): try: if not os.path.isdir(dstDir): os.makedirs(dstDir) - except: + except Exception: pass # Rename the file or directory src to dst. If dst is a directory, OSError will be raised. On Unix, if dst exists and is a file, it will be replaced silently if the user has permission. The operation may fail on some Unix flavors if src and dst are on different filesystems. diff --git a/src/MCPClient/lib/clientScripts/manual_normalization_remove_mn_directories.py b/src/MCPClient/lib/clientScripts/manual_normalization_remove_mn_directories.py index 08ca391b7e..f58bd7182a 100755 --- a/src/MCPClient/lib/clientScripts/manual_normalization_remove_mn_directories.py +++ b/src/MCPClient/lib/clientScripts/manual_normalization_remove_mn_directories.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import sys @@ -35,7 +32,7 @@ def recursivelyRemoveEmptyDirectories(job, dir): error_count = 0 - for root, dirs, files in os.walk(dir, topdown=False): + for root, dirs, _ in os.walk(dir, topdown=False): for directory in dirs: try: os.rmdir(os.path.join(root, directory)) diff --git a/src/MCPClient/lib/clientScripts/move_sip.py b/src/MCPClient/lib/clientScripts/move_sip.py index 827d76c599..2d123311dc 100755 --- a/src/MCPClient/lib/clientScripts/move_sip.py +++ b/src/MCPClient/lib/clientScripts/move_sip.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import shutil import sys diff --git a/src/MCPClient/lib/clientScripts/move_transfer.py b/src/MCPClient/lib/clientScripts/move_transfer.py index 70ac4d4dc0..89eca78c16 100755 --- a/src/MCPClient/lib/clientScripts/move_transfer.py +++ b/src/MCPClient/lib/clientScripts/move_transfer.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import django diff --git a/src/MCPClient/lib/clientScripts/normalize_report.py b/src/MCPClient/lib/clientScripts/normalize_report.py index 3eac0d1da1..b12b02f098 100755 --- a/src/MCPClient/lib/clientScripts/normalize_report.py +++ b/src/MCPClient/lib/clientScripts/normalize_report.py @@ -184,7 +184,7 @@ def report(uuid): recipient_list=recipient_list, html_message=html_message, ) - except: + except Exception: logger.exception("Report email was not delivered") return 1 else: diff --git a/src/MCPClient/lib/clientScripts/remove_directories.py b/src/MCPClient/lib/clientScripts/remove_directories.py index d3a88dbdcc..c5df1e58d7 100755 --- a/src/MCPClient/lib/clientScripts/remove_directories.py +++ b/src/MCPClient/lib/clientScripts/remove_directories.py @@ -15,8 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript import os import shutil diff --git a/src/MCPClient/lib/clientScripts/remove_files_without_premis_metadata.py b/src/MCPClient/lib/clientScripts/remove_files_without_premis_metadata.py index 2af792d302..9e782e6d1b 100755 --- a/src/MCPClient/lib/clientScripts/remove_files_without_premis_metadata.py +++ b/src/MCPClient/lib/clientScripts/remove_files_without_premis_metadata.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import argparse import os diff --git a/src/MCPClient/lib/clientScripts/remove_hidden_files_and_directories.py b/src/MCPClient/lib/clientScripts/remove_hidden_files_and_directories.py index 3747881618..1f4123f342 100755 --- a/src/MCPClient/lib/clientScripts/remove_hidden_files_and_directories.py +++ b/src/MCPClient/lib/clientScripts/remove_hidden_files_and_directories.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import shutil import sys diff --git a/src/MCPClient/lib/clientScripts/restructure_bag_aip_to_sip.py b/src/MCPClient/lib/clientScripts/restructure_bag_aip_to_sip.py index aecdb86531..ad93a196f1 100755 --- a/src/MCPClient/lib/clientScripts/restructure_bag_aip_to_sip.py +++ b/src/MCPClient/lib/clientScripts/restructure_bag_aip_to_sip.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import shutil diff --git a/src/MCPClient/lib/clientScripts/restructure_dip_for_content_dm_upload.py b/src/MCPClient/lib/clientScripts/restructure_dip_for_content_dm_upload.py index f6561f6360..70cbac9263 100755 --- a/src/MCPClient/lib/clientScripts/restructure_dip_for_content_dm_upload.py +++ b/src/MCPClient/lib/clientScripts/restructure_dip_for_content_dm_upload.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Mark Jordan import argparse import collections import csv @@ -85,9 +82,9 @@ def getItemCountType(structMap): """ divs_with_dmdsecs = structMap.findall(".//mets:div[@DMDID]", namespaces=ns.NSMAP) # If any are TYPE Directory, then it is compound - if any([e.get("TYPE") == "Directory" for e in divs_with_dmdsecs]): + if any(e.get("TYPE") == "Directory" for e in divs_with_dmdsecs): # If all are TYPE Directory then it is bulk - if all([e.get("TYPE") == "Directory" for e in divs_with_dmdsecs]): + if all(e.get("TYPE") == "Directory" for e in divs_with_dmdsecs): return "compound-dirs" else: return "compound-files" diff --git a/src/MCPClient/lib/clientScripts/restructure_for_compliance.py b/src/MCPClient/lib/clientScripts/restructure_for_compliance.py index bf3fc8769d..f4a475226c 100755 --- a/src/MCPClient/lib/clientScripts/restructure_for_compliance.py +++ b/src/MCPClient/lib/clientScripts/restructure_for_compliance.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import re import shutil diff --git a/src/MCPClient/lib/clientScripts/restructure_for_compliance_maildir.py b/src/MCPClient/lib/clientScripts/restructure_for_compliance_maildir.py index fb03d8c1a1..cf35d22241 100755 --- a/src/MCPClient/lib/clientScripts/restructure_for_compliance_maildir.py +++ b/src/MCPClient/lib/clientScripts/restructure_for_compliance_maildir.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import shutil diff --git a/src/MCPClient/lib/clientScripts/retry_normalize_remove_normalized.py b/src/MCPClient/lib/clientScripts/retry_normalize_remove_normalized.py index 09bf351da8..d6b8fd1637 100755 --- a/src/MCPClient/lib/clientScripts/retry_normalize_remove_normalized.py +++ b/src/MCPClient/lib/clientScripts/retry_normalize_remove_normalized.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import shutil import sys @@ -38,7 +35,7 @@ def removeDIP(job, SIPDirectory, SIPUUID): DIP = os.path.join(SIPDirectory, "DIP") if os.path.isdir(DIP): shutil.rmtree(DIP) - except (os.error, shutil.Error): + except (OSError, shutil.Error): job.pyprint("Error deleting DIP", file=sys.stderr) job.print_output(traceback.format_exc()) @@ -48,7 +45,7 @@ def removeThumbnails(job, SIPDirectory, SIPUUID): thumbnails = os.path.join(SIPDirectory, "thumbnails") if os.path.isdir(thumbnails): shutil.rmtree(thumbnails) - except (os.error, shutil.Error): + except (OSError, shutil.Error): job.pyprint("Error deleting thumbnails", file=sys.stderr) job.print_output(traceback.format_exc()) diff --git a/src/MCPClient/lib/clientScripts/rights_from_csv.py b/src/MCPClient/lib/clientScripts/rights_from_csv.py index 85a9468300..074f17ce80 100755 --- a/src/MCPClient/lib/clientScripts/rights_from_csv.py +++ b/src/MCPClient/lib/clientScripts/rights_from_csv.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Mike Cantelon import csv import os diff --git a/src/MCPClient/lib/clientScripts/save_dublin_core.py b/src/MCPClient/lib/clientScripts/save_dublin_core.py index 8b7ca5e930..450ca8b081 100755 --- a/src/MCPClient/lib/clientScripts/save_dublin_core.py +++ b/src/MCPClient/lib/clientScripts/save_dublin_core.py @@ -34,7 +34,9 @@ def main(job, transfer_uuid, target_path): jsonified = {} try: dc = models.DublinCore.objects.get(metadataappliestoidentifier=transfer_uuid) - except: # There may not be any DC metadata for this transfer, and that's fine + except ( + Exception + ): # There may not be any DC metadata for this transfer, and that's fine job.pyprint("No DC metadata found; skipping", file=sys.stderr) return 0 for field in FIELDS: diff --git a/src/MCPClient/lib/clientScripts/set_maildir_file_grp_use_and_file_ids.py b/src/MCPClient/lib/clientScripts/set_maildir_file_grp_use_and_file_ids.py index d0121f8597..c909c26e59 100755 --- a/src/MCPClient/lib/clientScripts/set_maildir_file_grp_use_and_file_ids.py +++ b/src/MCPClient/lib/clientScripts/set_maildir_file_grp_use_and_file_ids.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import django @@ -67,7 +64,7 @@ def set_maildir_files(sip_uuid, sip_path): maildir_path, sip_uuid, ) - for root, dirs, files in os.walk(maildir_path): + for root, _, files in os.walk(maildir_path): for item in files: file_relative_path = os.path.join(root, item).replace( sip_path, "%SIPDirectory%", 1 @@ -87,7 +84,7 @@ def set_archivematica_maildir_files(sip_uuid, sip_path): attachments_path, sip_uuid, ) - for root, dirs, files in os.walk(attachments_path): + for root, _, files in os.walk(attachments_path): for item in files: if not item.endswith(".archivematicaMaildir"): continue diff --git a/src/MCPClient/lib/clientScripts/set_transfer_type.py b/src/MCPClient/lib/clientScripts/set_transfer_type.py index 8c5a4e8886..b011e73a99 100755 --- a/src/MCPClient/lib/clientScripts/set_transfer_type.py +++ b/src/MCPClient/lib/clientScripts/set_transfer_type.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import django django.setup() diff --git a/src/MCPClient/lib/clientScripts/store_aip.py b/src/MCPClient/lib/clientScripts/store_aip.py index 4ed2a80c8b..ad8dee9863 100755 --- a/src/MCPClient/lib/clientScripts/store_aip.py +++ b/src/MCPClient/lib/clientScripts/store_aip.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import argparse import os import shutil diff --git a/src/MCPClient/lib/clientScripts/transcoder.py b/src/MCPClient/lib/clientScripts/transcoder.py index 3ed67a184d..487ad06865 100755 --- a/src/MCPClient/lib/clientScripts/transcoder.py +++ b/src/MCPClient/lib/clientScripts/transcoder.py @@ -15,8 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClient # archivematicaCommon from django.db.models import F from executeOrRunSubProcess import executeOrRun diff --git a/src/MCPClient/lib/clientScripts/trim_create_rights_entries.py b/src/MCPClient/lib/clientScripts/trim_create_rights_entries.py index 365120a713..e42223083f 100755 --- a/src/MCPClient/lib/clientScripts/trim_create_rights_entries.py +++ b/src/MCPClient/lib/clientScripts/trim_create_rights_entries.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import string import sys @@ -112,7 +109,7 @@ def call(jobs): try: tree = etree.parse(xmlFilePath) root = tree.getroot() - except: + except Exception: job.pyprint( "Error parsing: ", xmlFilePath.replace(transferPath, "%transferDirectory%", 1), @@ -125,7 +122,7 @@ def call(jobs): "Container/RetentionSchedule" ).text DateClosed = root.find("Container/DateClosed").text - except: + except Exception: job.pyprint( "Error retrieving values from: ", xmlFilePath.replace(transferPath, "%transferDirectory%", 1), diff --git a/src/MCPClient/lib/clientScripts/trim_restructure_for_compliance.py b/src/MCPClient/lib/clientScripts/trim_restructure_for_compliance.py index 2518a76e91..0b8f7fbf27 100755 --- a/src/MCPClient/lib/clientScripts/trim_restructure_for_compliance.py +++ b/src/MCPClient/lib/clientScripts/trim_restructure_for_compliance.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import django @@ -110,7 +107,7 @@ def restructureTRIMForComplianceFileUUIDsAssigned( files = fileOperations.getFileUUIDLike( dst, unitPath, unitIdentifier, unitIdentifierType, unitPathReplaceWith ) - for key, value in files.items(): + for value in files.values(): fileUUID = value fileOperations.updateFileGrpUse(fileUUID, "TRIM metadata") diff --git a/src/MCPClient/lib/clientScripts/trim_verify_checksums.py b/src/MCPClient/lib/clientScripts/trim_verify_checksums.py index 05bf10806c..71bf441bbf 100755 --- a/src/MCPClient/lib/clientScripts/trim_verify_checksums.py +++ b/src/MCPClient/lib/clientScripts/trim_verify_checksums.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import sys import uuid @@ -70,7 +67,7 @@ def call(jobs): root = tree.getroot() xmlMD5 = root.find("Document/MD5").text - except: + except Exception: job.pyprint("Error parsing: ", xmlFilePath, file=sys.stderr) exitCode += 1 continue @@ -91,14 +88,14 @@ def call(jobs): "transfer", "%transferDirectory%", ) - for path, fileUUID in fileID.items(): + for fileUUID in fileID.values(): eventDetail = 'program="python"; module="hashlib.md5()"' eventOutcome = "Pass" eventOutcomeDetailNote = "{} {}".format( xmlFile.__str__(), "verified", ) - eventIdentifierUUID = uuid.uuid4().__str__() + eventIdentifierUUID = str(uuid.uuid4()) databaseFunctions.insertIntoEvents( fileUUID=fileUUID, diff --git a/src/MCPClient/lib/clientScripts/trim_verify_manifest.py b/src/MCPClient/lib/clientScripts/trim_verify_manifest.py index 582220d4ac..1853fdd674 100755 --- a/src/MCPClient/lib/clientScripts/trim_verify_manifest.py +++ b/src/MCPClient/lib/clientScripts/trim_verify_manifest.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import re import sys @@ -121,11 +118,11 @@ def call(jobs): file=sys.stderr, ) exitCode += 1 - for paths, fileUUID in fileID.items(): + for fileUUID in fileID.values(): eventDetail = 'program="archivematica"; module="trimVerifyManifest"' eventOutcome = "Pass" eventOutcomeDetailNote = "Verified file exists" - eventIdentifierUUID = uuid.uuid4().__str__() + eventIdentifierUUID = str(uuid.uuid4()) databaseFunctions.insertIntoEvents( fileUUID=fileUUID, eventIdentifierUUID=eventIdentifierUUID, @@ -161,11 +158,11 @@ def call(jobs): file=sys.stderr, ) exitCode += 1 - for paths, fileUUID in fileID.items(): + for fileUUID in fileID.values(): eventDetail = 'program="archivematica"; module="trimVerifyManifest"' eventOutcome = "Pass" eventOutcomeDetailNote = "Verified file exists, but with implicit extension case" - eventIdentifierUUID = uuid.uuid4().__str__() + eventIdentifierUUID = str(uuid.uuid4()) databaseFunctions.insertIntoEvents( fileUUID=fileUUID, eventIdentifierUUID=eventIdentifierUUID, diff --git a/src/MCPClient/lib/clientScripts/upload_archivesspace.py b/src/MCPClient/lib/clientScripts/upload_archivesspace.py index 5054c5ef72..5723e9ad56 100755 --- a/src/MCPClient/lib/clientScripts/upload_archivesspace.py +++ b/src/MCPClient/lib/clientScripts/upload_archivesspace.py @@ -26,7 +26,7 @@ def recursive_file_gen(mydir): - for root, dirs, files in os.walk(mydir): + for root, _, files in os.walk(mydir): for file in files: yield os.path.join(root, file) diff --git a/src/MCPClient/lib/clientScripts/verify_and_restructure_transfer_bag.py b/src/MCPClient/lib/clientScripts/verify_and_restructure_transfer_bag.py index 203b7b6879..f6c2e071ad 100755 --- a/src/MCPClient/lib/clientScripts/verify_and_restructure_transfer_bag.py +++ b/src/MCPClient/lib/clientScripts/verify_and_restructure_transfer_bag.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import sys diff --git a/src/MCPClient/lib/clientScripts/verify_checksum.py b/src/MCPClient/lib/clientScripts/verify_checksum.py index 46ecced726..20b9f562a3 100755 --- a/src/MCPClient/lib/clientScripts/verify_checksum.py +++ b/src/MCPClient/lib/clientScripts/verify_checksum.py @@ -191,12 +191,9 @@ def get_ext(path): @staticmethod def _count_lines(path): """Count the number of lines in a checksum file.""" - count = 0 with open(path) as hashfile: - for count, _ in enumerate(hashfile): - pass - # Negate zero-based count. - return count + 1 + count = sum(1 for _ in hashfile) + return count @staticmethod def _count_files(path): diff --git a/src/MCPClient/lib/clientScripts/verify_checksums_in_file_sec_of_dspace_mets_files.py b/src/MCPClient/lib/clientScripts/verify_checksums_in_file_sec_of_dspace_mets_files.py index 02476fd7d2..fdca386cd8 100755 --- a/src/MCPClient/lib/clientScripts/verify_checksums_in_file_sec_of_dspace_mets_files.py +++ b/src/MCPClient/lib/clientScripts/verify_checksums_in_file_sec_of_dspace_mets_files.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import hashlib import os import sys diff --git a/src/MCPClient/lib/clientScripts/verify_sip_compliance.py b/src/MCPClient/lib/clientScripts/verify_sip_compliance.py index cc6fa69aaa..b68d15953a 100755 --- a/src/MCPClient/lib/clientScripts/verify_sip_compliance.py +++ b/src/MCPClient/lib/clientScripts/verify_sip_compliance.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import sys @@ -36,7 +33,7 @@ def checkDirectory(job, directory, ret=0): try: - for directory, subDirectories, files in os.walk(directory): + for _, _, files in os.walk(directory): for file in files: os.path.join(directory, file) except Exception as inst: diff --git a/src/MCPClient/lib/clientScripts/verify_transfer_compliance.py b/src/MCPClient/lib/clientScripts/verify_transfer_compliance.py index 60e9adb206..137aa6513c 100755 --- a/src/MCPClient/lib/clientScripts/verify_transfer_compliance.py +++ b/src/MCPClient/lib/clientScripts/verify_transfer_compliance.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaClientScript -# @author Joseph Perry import os import sys diff --git a/src/MCPClient/lib/fork_runner.py b/src/MCPClient/lib/fork_runner.py index 0d078ab2c9..c9311683e1 100755 --- a/src/MCPClient/lib/fork_runner.py +++ b/src/MCPClient/lib/fork_runner.py @@ -31,11 +31,13 @@ THIS_SCRIPT = "fork_runner.py" -def call(module_name, jobs, task_count=multiprocessing.cpu_count()): +def call(module_name, jobs, task_count=None): """ Split `jobs` into `task_count` groups and fork a subprocess to run `module_name`.call() for each of them. """ + if task_count is None: + task_count = multiprocessing.cpu_count() jobs_by_uuid = {} for job in jobs: jobs_by_uuid[job.UUID] = job diff --git a/src/MCPClient/tests/fixtures/test_no_files_in_db.yaml b/src/MCPClient/tests/fixtures/test_no_files_in_db.yaml deleted file mode 100644 index 1aa296a81b..0000000000 --- a/src/MCPClient/tests/fixtures/test_no_files_in_db.yaml +++ /dev/null @@ -1,25 +0,0 @@ -interactions: -- request: - body: password=admin - headers: - Accept: ['*/*'] - Accept-Encoding: ['gzip, deflate'] - Connection: [keep-alive] - Content-Length: ['14'] - Content-Type: [application/x-www-form-urlencoded] - User-Agent: [python-requests/2.7.0 CPython/2.7.6 Linux/3.13.0-43-generic] - method: POST - uri: http://localhost:8089/users/admin/login - response: - body: {string: !!python/unicode '{"session":"88373637ab6bd52646d959ad310c1f281fb4ba02073e64c3f4da50b43d67b24a","user":{"lock_version":1159,"username":"admin","name":"Administrator","is_system_user":true,"create_time":"2014-12-05T20:32:17Z","system_mtime":"2015-07-09T23:18:47Z","user_mtime":"2015-07-09T23:18:47Z","jsonmodel_type":"user","groups":[],"is_admin":false,"uri":"/users/1","agent_record":{"ref":"/agents/people/1"},"permissions":{"/repositories/2":["view_repository","update_accession_record","update_resource_record","update_digital_object_record"],"_archivesspace":[]}}} - - '} - headers: - cache-control: ['private, must-revalidate, max-age=0'] - content-length: ['551'] - content-type: [application/json] - date: ['Thu, 09 Jul 2015 23:18:47 GMT'] - server: [Jetty(8.1.5.v20120716)] - x-content-type-options: [nosniff] - status: {code: 200, message: OK} -version: 1 diff --git a/src/MCPClient/tests/fixtures/test_parse_archivesspace_ids.yaml b/src/MCPClient/tests/fixtures/test_parse_archivesspace_ids.yaml deleted file mode 100644 index e4081bdef6..0000000000 --- a/src/MCPClient/tests/fixtures/test_parse_archivesspace_ids.yaml +++ /dev/null @@ -1,48 +0,0 @@ -interactions: -- request: - body: password=admin - headers: - Accept: ['*/*'] - Accept-Encoding: ['gzip, deflate'] - Connection: [keep-alive] - Content-Length: ['14'] - Content-Type: [application/x-www-form-urlencoded] - User-Agent: [python-requests/2.7.0 CPython/2.7.6 Linux/3.13.0-43-generic] - method: POST - uri: http://localhost:8089/users/admin/login - response: - body: {string: !!python/unicode '{"session":"4a108561f24f7850cb136cd765405fd563853b39b626e7cf3bfc4a99ef2bab0c","user":{"lock_version":898,"username":"admin","name":"Administrator","is_system_user":true,"create_time":"2014-12-05T20:32:17Z","system_mtime":"2015-07-08T21:38:45Z","user_mtime":"2015-07-08T21:38:45Z","jsonmodel_type":"user","groups":[],"is_admin":false,"uri":"/users/1","agent_record":{"ref":"/agents/people/1"},"permissions":{"/repositories/2":["view_repository","update_accession_record","update_resource_record","update_digital_object_record"],"_archivesspace":[]}}} - - '} - headers: - cache-control: ['private, must-revalidate, max-age=0'] - content-length: ['550'] - content-type: [application/json] - date: ['Wed, 08 Jul 2015 21:38:45 GMT'] - server: [Jetty(8.1.5.v20120716)] - x-content-type-options: [nosniff] - status: {code: 200, message: OK} -- request: - body: null - headers: - Accept: ['*/*'] - Accept-Encoding: ['gzip, deflate'] - Connection: [keep-alive] - User-Agent: [python-requests/2.7.0 CPython/2.7.6 Linux/3.13.0-43-generic] - X-ArchivesSpace-Session: [!!python/unicode '4a108561f24f7850cb136cd765405fd563853b39b626e7cf3bfc4a99ef2bab0c'] - method: GET - uri: http://localhost:8089/repositories/2/find_by_id/archival_objects?resolve%5B%5D=archival_objects&ref_id%5B%5D=a118514fab1b2ee6a7e9ad259e1de355 - response: - body: {string: !!python/unicode '{"archival_objects":[{"ref":"/repositories/2/archival_objects/752250","_resolved":{"lock_version":0,"position":0,"publish":true,"ref_id":"a118514fab1b2ee6a7e9ad259e1de355","component_id":"test111","title":"Test - AO","display_string":"Test AO","restrictions_apply":false,"created_by":"admin","last_modified_by":"admin","create_time":"2015-09-22T18:35:41Z","system_mtime":"2015-09-22T18:35:41Z","user_mtime":"2015-09-22T18:35:41Z","suppressed":false,"level":"file","jsonmodel_type":"archival_object","external_ids":[],"subjects":[],"linked_events":[],"extents":[],"dates":[],"external_documents":[],"rights_statements":[],"linked_agents":[],"instances":[],"notes":[],"uri":"/repositories/2/archival_objects/752250","repository":{"ref":"/repositories/2"},"resource":{"ref":"/repositories/2/resources/11319"},"has_unpublished_ancestor":false}}]} - - '} - headers: - cache-control: ['private, must-revalidate, max-age=0'] - content-length: ['841'] - content-type: [application/json] - date: ['Wed, 08 Jul 2015 21:38:45 GMT'] - server: [Jetty(8.1.5.v20120716)] - x-content-type-options: [nosniff] - status: {code: 200, message: OK} -version: 1 diff --git a/src/MCPClient/tests/fixtures/vcr_cassettes/test_bind_pid_to_files.yaml b/src/MCPClient/tests/fixtures/vcr_cassettes/test_bind_pid_to_files.yaml deleted file mode 100644 index 9cd598ce6c..0000000000 --- a/src/MCPClient/tests/fixtures/vcr_cassettes/test_bind_pid_to_files.yaml +++ /dev/null @@ -1,65 +0,0 @@ -interactions: -- request: - body: !!python/object/new:django.utils.safestring.SafeText - - !!python/unicode "\r\n \r\n - \ \r\n \r\n 12345\r\n - \ \r\n 12345/06da9555-dc5b-425c-b967-6f30a740f1c3\r\n - \ \r\n \r\n - \ \r\n \r\n \r\n - \ \r\n \r\n - \ \r\n \r\n - \ \r\n \r\n - \ \r\n \r\n " - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Authorization: - - !!python/unicode bearer 84214c59-8694-48d5-89b5-d40a88cd7768 - Connection: - - keep-alive - Content-Length: - - '1711' - Content-Type: - - !!python/unicode text/xml - User-Agent: - - python-requests/2.21.0 - method: POST - uri: https://pid.socialhistoryservices.org/secure - response: - body: - string: !!python/unicode 12345/06DA9555-DC5B-425C-B967-6F30A740F1C3 - headers: - accept: - - text/xml, text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2 - connection: - - keep-alive - content-length: - - '881' - content-type: - - text/xml;charset=utf-8 - date: - - Thu, 02 May 2019 12:44:02 GMT - server: - - nginx - soapaction: - - '""' - status: - code: 200 - message: OK -version: 1 diff --git a/src/MCPClient/tests/fixtures/vcr_cassettes/test_bind_pids_to_sip_and_dirs.yaml b/src/MCPClient/tests/fixtures/vcr_cassettes/test_bind_pids_to_sip_and_dirs.yaml deleted file mode 100644 index e6c55fccf2..0000000000 --- a/src/MCPClient/tests/fixtures/vcr_cassettes/test_bind_pids_to_sip_and_dirs.yaml +++ /dev/null @@ -1,183 +0,0 @@ -interactions: -- request: - body: !!python/object/new:django.utils.safestring.SafeText - - !!python/unicode "\r\n \r\n - \ \r\n \r\n 12345\r\n - \ \r\n 12345/cb5ebaf5-beda-40b4-8d0c-fefbd546b8de\r\n - \ \r\n \r\n - \ \r\n \r\n \r\n - \ \r\n \r\n - \ \r\n \r\n " - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Authorization: - - !!python/unicode bearer 84214c59-8694-48d5-89b5-d40a88cd7768 - Connection: - - keep-alive - Content-Length: - - '1098' - Content-Type: - - !!python/unicode text/xml - User-Agent: - - python-requests/2.21.0 - method: POST - uri: https://pid.socialhistoryservices.org/secure - response: - body: - string: !!python/unicode 12345/CB5EBAF5-BEDA-40B4-8D0C-FEFBD546B8DE - headers: - accept: - - text/xml, text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2 - connection: - - keep-alive - content-length: - - '594' - content-type: - - text/xml;charset=utf-8 - date: - - Thu, 02 May 2019 13:08:30 GMT - server: - - nginx - soapaction: - - '""' - status: - code: 200 - message: OK -- request: - body: !!python/object/new:django.utils.safestring.SafeText - - !!python/unicode "\r\n \r\n - \ \r\n \r\n 12345\r\n - \ \r\n 12345/966755bd-0ae3-4f85-b4ec-b359fefeff33\r\n - \ \r\n \r\n - \ \r\n \r\n \r\n - \ \r\n \r\n - \ \r\n \r\n - \ \r\n \r\n - \ \r\n \r\n " - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Authorization: - - !!python/unicode bearer 84214c59-8694-48d5-89b5-d40a88cd7768 - Connection: - - keep-alive - Content-Length: - - '1711' - Content-Type: - - !!python/unicode text/xml - User-Agent: - - python-requests/2.21.0 - method: POST - uri: https://pid.socialhistoryservices.org/secure - response: - body: - string: !!python/unicode 12345/966755BD-0AE3-4F85-B4EC-B359FEFEFF33 - headers: - accept: - - text/xml, text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2 - connection: - - keep-alive - content-length: - - '881' - content-type: - - text/xml;charset=utf-8 - date: - - Thu, 02 May 2019 13:08:30 GMT - server: - - nginx - soapaction: - - '""' - status: - code: 200 - message: OK -- request: - body: !!python/object/new:django.utils.safestring.SafeText - - !!python/unicode "\r\n \r\n - \ \r\n \r\n 12345\r\n - \ \r\n 12345/d298dd3f-c5d1-4445-99fe-09123fba8b30\r\n - \ \r\n \r\n - \ \r\n \r\n \r\n - \ \r\n \r\n - \ \r\n \r\n - \ \r\n \r\n - \ \r\n \r\n " - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Authorization: - - !!python/unicode bearer 84214c59-8694-48d5-89b5-d40a88cd7768 - Connection: - - keep-alive - Content-Length: - - '1711' - Content-Type: - - !!python/unicode text/xml - User-Agent: - - python-requests/2.21.0 - method: POST - uri: https://pid.socialhistoryservices.org/secure - response: - body: - string: !!python/unicode 12345/D298DD3F-C5D1-4445-99FE-09123FBA8B30 - headers: - accept: - - text/xml, text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2 - connection: - - keep-alive - content-length: - - '881' - content-type: - - text/xml;charset=utf-8 - date: - - Thu, 02 May 2019 13:08:30 GMT - server: - - nginx - soapaction: - - '""' - status: - code: 200 - message: OK -version: 1 diff --git a/src/MCPClient/tests/fixtures/vcr_cassettes/test_dspace_handle_to_archivesspace.yaml b/src/MCPClient/tests/fixtures/vcr_cassettes/test_dspace_handle_to_archivesspace.yaml deleted file mode 100644 index 05b0436c23..0000000000 --- a/src/MCPClient/tests/fixtures/vcr_cassettes/test_dspace_handle_to_archivesspace.yaml +++ /dev/null @@ -1,117 +0,0 @@ -interactions: -- request: - body: null - headers: - Accept-Encoding: ['gzip, deflate'] - Authorization: ['ApiKey test:326e86562d76b1db4a2479eccfd76d35a1d2fb5e'] - Connection: [keep-alive] - User-Agent: [python-requests/2.7.0 CPython/2.7.6 Linux/3.13.0-91-generic] - accept: [application/json] - content-type: [application/json] - method: GET - uri: http://localhost:8000/api/v2/file/?uuid=4060ee97-9c3f-4822-afaf-ebdf838284c3&offset=0 - response: - body: {string: !!python/unicode '{"meta": {"limit": 20, "next": null, "offset": - 0, "previous": null, "total_count": 1}, "objects": [{"current_full_path": - "http:/dspace5x.archivematica.org:8080/swordv2/collection/123456789/2/http:/dspace5x.archivematica.org:8080/swordv2/statement/92.atom", - "current_location": "/api/v2/location/758cf517-45f7-4c78-8074-32c686c653cb/", - "current_path": "http://dspace5x.archivematica.org:8080/swordv2/statement/92.atom", - "misc_attributes": {"handle": "123456789/41"}, "origin_pipeline": "/api/v2/pipeline/32836b5b-950e-4409-a823-830d2dd01026/", - "package_type": "AIP", "related_packages": [], "resource_uri": "/api/v2/file/4060ee97-9c3f-4822-afaf-ebdf838284c3/", - "size": 21244662, "status": "UPLOADED", "uuid": "4060ee97-9c3f-4822-afaf-ebdf838284c3"}]}'} - headers: - cache-control: [no-cache] - content-type: [application/json] - date: ['Tue, 20 Sep 2016 23:11:32 GMT'] - server: [WSGIServer/0.1 Python/2.7.6] - vary: ['Accept, Cookie'] - x-frame-options: [SAMEORIGIN] - status: {code: 200, message: OK} -- request: - body: null - headers: - Accept: ['*/*'] - Accept-Encoding: ['gzip, deflate'] - Connection: [keep-alive] - Content-Length: ['0'] - User-Agent: [python-requests/2.7.0 CPython/2.7.6 Linux/3.13.0-91-generic] - method: POST - uri: http://localhost:8089/users/admin/login?password=admin - response: - body: {string: !!python/unicode '{"session":"57823397705778ab52fea1cbb1a690c46218dfb0277ed021f496849dc328d37b","user":{"lock_version":6298,"username":"admin","name":"Administrator","is_system_user":true,"create_time":"2015-09-22T18:46:11Z","system_mtime":"2016-09-20T23:11:32Z","user_mtime":"2016-09-20T23:11:32Z","jsonmodel_type":"user","groups":[],"is_admin":true,"uri":"/users/1","agent_record":{"ref":"/agents/people/1"},"permissions":{"/repositories/1":["update_location_record","delete_vocabulary_record","update_subject_record","delete_subject_record","update_agent_record","delete_agent_record","update_vocabulary_record","merge_subject_record","merge_agent_record","system_config","administer_system","manage_users","become_user","view_all_records","create_repository","delete_repository","transfer_repository","index_system","manage_repository","update_accession_record","update_resource_record","update_digital_object_record","update_event_record","delete_event_record","suppress_archival_record","transfer_archival_record","delete_archival_record","view_suppressed","view_repository","update_classification_record","delete_classification_record","mediate_edits","import_records","cancel_importer_job","manage_subject_record","manage_agent_record","manage_vocabulary_record","merge_agents_and_subjects","merge_archival_record","manage_rde_templates"],"_archivesspace":["system_config","administer_system","manage_users","become_user","view_all_records","create_repository","delete_repository","transfer_repository","index_system","manage_repository","update_accession_record","update_resource_record","update_digital_object_record","update_event_record","delete_event_record","suppress_archival_record","transfer_archival_record","delete_archival_record","view_suppressed","view_repository","update_classification_record","delete_classification_record","mediate_edits","import_records","cancel_importer_job","manage_subject_record","manage_agent_record","manage_vocabulary_record","merge_agents_and_subjects","merge_archival_record","manage_rde_templates","update_location_record","delete_vocabulary_record","update_subject_record","delete_subject_record","update_agent_record","delete_agent_record","update_vocabulary_record","merge_subject_record","merge_agent_record"]}}} - -'} - headers: - cache-control: ['private, must-revalidate, max-age=0'] - content-length: ['2253'] - content-type: [application/json] - date: ['Tue, 20 Sep 2016 23:11:32 GMT'] - server: [Jetty(8.1.5.v20120716)] - x-content-type-options: [nosniff] - status: {code: 200, message: OK} -- request: - body: null - headers: - Accept: ['*/*'] - Accept-Encoding: ['gzip, deflate'] - Connection: [keep-alive] - User-Agent: [python-requests/2.7.0 CPython/2.7.6 Linux/3.13.0-91-generic] - X-ArchivesSpace-Session: [!!python/unicode 57823397705778ab52fea1cbb1a690c46218dfb0277ed021f496849dc328d37b] - method: GET - uri: http://localhost:8089/repositories/2/digital_objects/211 - response: - body: {string: !!python/unicode '{"lock_version":4,"digital_object_id":"ac256851-ba50-4c99-a488-25fa442ed242","title":"misty-test-1","publish":false,"restrictions":false,"created_by":"admin","last_modified_by":"admin","create_time":"2016-09-20T18:22:56Z","system_mtime":"2016-09-20T18:48:06Z","user_mtime":"2016-09-20T18:48:06Z","suppressed":false,"digital_object_type":"text","jsonmodel_type":"digital_object","external_ids":[],"subjects":[],"linked_events":[],"extents":[],"dates":[],"external_documents":[],"rights_statements":[],"linked_agents":[],"file_versions":[{"lock_version":0,"file_uri":"123456789/41","created_by":"admin","last_modified_by":"admin","create_time":"2016-09-20T18:48:06Z","system_mtime":"2016-09-20T18:48:06Z","user_mtime":"2016-09-20T18:48:06Z","use_statement":"text-data","xlink_actuate_attribute":"none","xlink_show_attribute":"embed","jsonmodel_type":"file_version","identifier":"289"},{"lock_version":0,"file_uri":"123456789/41","created_by":"admin","last_modified_by":"admin","create_time":"2016-09-20T18:48:06Z","system_mtime":"2016-09-20T18:48:06Z","user_mtime":"2016-09-20T18:48:06Z","use_statement":"text-data","xlink_actuate_attribute":"none","xlink_show_attribute":"embed","jsonmodel_type":"file_version","identifier":"290"}],"notes":[],"linked_instances":[{"ref":"/repositories/2/archival_objects/8887"}],"uri":"/repositories/2/digital_objects/211","repository":{"ref":"/repositories/2"},"tree":{"ref":"/repositories/2/digital_objects/211/tree"}} - -'} - headers: - cache-control: ['private, must-revalidate, max-age=0'] - content-length: ['1453'] - content-type: [application/json] - date: ['Tue, 20 Sep 2016 23:11:32 GMT'] - server: [Jetty(8.1.5.v20120716)] - x-content-type-options: [nosniff] - status: {code: 200, message: OK} -- request: - body: !!python/unicode '{"lock_version": 4, "system_mtime": "2016-09-20T18:48:06Z", - "extents": [], "jsonmodel_type": "digital_object", "create_time": "2016-09-20T18:22:56Z", - "publish": false, "repository": {"ref": "/repositories/2"}, "linked_instances": - [{"ref": "/repositories/2/archival_objects/8887"}], "title": "misty-test-1", - "created_by": "admin", "subjects": [], "external_documents": [], "linked_agents": - [], "digital_object_type": "text", "file_versions": [{"lock_version": 0, "system_mtime": - "2016-09-20T18:48:06Z", "xlink_show_attribute": "embed", "jsonmodel_type": "file_version", - "file_uri": "123456789/41", "user_mtime": "2016-09-20T18:48:06Z", "last_modified_by": - "admin", "create_time": "2016-09-20T18:48:06Z", "xlink_actuate_attribute": "none", - "created_by": "admin", "use_statement": "text-data", "identifier": "289"}, {"lock_version": - 0, "system_mtime": "2016-09-20T18:48:06Z", "xlink_show_attribute": "embed", - "jsonmodel_type": "file_version", "file_uri": "123456789/41", "user_mtime": - "2016-09-20T18:48:06Z", "last_modified_by": "admin", "create_time": "2016-09-20T18:48:06Z", - "xlink_actuate_attribute": "none", "created_by": "admin", "use_statement": "text-data", - "identifier": "290"}, {"file_uri": "123456789/41", "xlink_actuate_attribute": - "none", "xlink_show_attribute": "embed", "use_statement": "text-data"}], "user_mtime": - "2016-09-20T18:48:06Z", "rights_statements": [], "linked_events": [], "external_ids": - [], "suppressed": false, "restrictions": false, "dates": [], "notes": [], "tree": - {"ref": "/repositories/2/digital_objects/211/tree"}, "uri": "/repositories/2/digital_objects/211", - "last_modified_by": "admin", "digital_object_id": "ac256851-ba50-4c99-a488-25fa442ed242"}' - headers: - Accept: ['*/*'] - Accept-Encoding: ['gzip, deflate'] - Connection: [keep-alive] - Content-Length: ['1683'] - Content-Type: [application/json] - User-Agent: [python-requests/2.7.0 CPython/2.7.6 Linux/3.13.0-91-generic] - X-ArchivesSpace-Session: [!!python/unicode 57823397705778ab52fea1cbb1a690c46218dfb0277ed021f496849dc328d37b] - method: POST - uri: http://localhost:8089/repositories/2/digital_objects/211 - response: - body: {string: !!python/unicode '{"status":"Updated","id":211,"lock_version":5,"stale":null,"uri":"/repositories/2/digital_objects/211","warnings":[]} - -'} - headers: - cache-control: ['private, must-revalidate, max-age=0'] - content-length: ['118'] - content-type: [application/json] - date: ['Tue, 20 Sep 2016 23:11:33 GMT'] - server: [Jetty(8.1.5.v20120716)] - x-content-type-options: [nosniff] - status: {code: 200, message: OK} -version: 1 diff --git a/src/MCPClient/tests/fixtures/vcr_cassettes/test_no_dspace.yaml b/src/MCPClient/tests/fixtures/vcr_cassettes/test_no_dspace.yaml deleted file mode 100644 index 29d944dcbf..0000000000 --- a/src/MCPClient/tests/fixtures/vcr_cassettes/test_no_dspace.yaml +++ /dev/null @@ -1,30 +0,0 @@ -interactions: -- request: - body: null - headers: - Accept-Encoding: ['gzip, deflate'] - Authorization: ['ApiKey test:326e86562d76b1db4a2479eccfd76d35a1d2fb5e'] - Connection: [keep-alive] - User-Agent: [python-requests/2.7.0 CPython/2.7.6 Linux/3.13.0-91-generic] - accept: [application/json] - content-type: [application/json] - method: GET - uri: http://localhost:8000/api/v2/file/?uuid=4060ee97-9c3f-4822-afaf-ebdf838284c3&offset=0 - response: - body: {string: !!python/unicode '{"meta": {"limit": 20, "next": null, "offset": - 0, "previous": null, "total_count": 1}, "objects": [{"current_full_path": - "http:/dspace5x.archivematica.org:8080/swordv2/collection/123456789/2/http:/dspace5x.archivematica.org:8080/swordv2/statement/92.atom", - "current_location": "/api/v2/location/758cf517-45f7-4c78-8074-32c686c653cb/", - "current_path": "http://dspace5x.archivematica.org:8080/swordv2/statement/92.atom", - "misc_attributes": {}, "origin_pipeline": "/api/v2/pipeline/32836b5b-950e-4409-a823-830d2dd01026/", - "package_type": "AIP", "related_packages": [], "resource_uri": "/api/v2/file/4060ee97-9c3f-4822-afaf-ebdf838284c3/", - "size": 21244662, "status": "UPLOADED", "uuid": "4060ee97-9c3f-4822-afaf-ebdf838284c3"}]}'} - headers: - cache-control: [no-cache] - content-type: [application/json] - date: ['Tue, 20 Sep 2016 23:11:32 GMT'] - server: [WSGIServer/0.1 Python/2.7.6] - vary: ['Accept, Cookie'] - x-frame-options: [SAMEORIGIN] - status: {code: 200, message: OK} -version: 1 diff --git a/src/MCPClient/tests/test_archivematicaCreateMETSMetadataXML.py b/src/MCPClient/tests/test_archivematicaCreateMETSMetadataXML.py index 1a02fa83a5..43275c0912 100644 --- a/src/MCPClient/tests/test_archivematicaCreateMETSMetadataXML.py +++ b/src/MCPClient/tests/test_archivematicaCreateMETSMetadataXML.py @@ -102,7 +102,9 @@ def _make_mock_fsentry(**kwargs): @pytest.fixture def make_mock_mets(mocker, make_mock_fsentry): - def _make_mock_mets(metadata_file_uuids=[]): + def _make_mock_mets(metadata_file_uuids=None): + if metadata_file_uuids is None: + metadata_file_uuids = [] sip = make_mock_fsentry(label="sip", type="Directory") objects = make_mock_fsentry(label="objects", type="Directory") directory = make_mock_fsentry(label="directory", type="Directory") diff --git a/src/MCPClient/tests/test_assign_uuids_to_directories.py b/src/MCPClient/tests/test_assign_uuids_to_directories.py index 500ff985d5..47ae9b7fa7 100755 --- a/src/MCPClient/tests/test_assign_uuids_to_directories.py +++ b/src/MCPClient/tests/test_assign_uuids_to_directories.py @@ -36,11 +36,11 @@ def test_main(mocker, transfer, transfer_dir): include_dirs = True # Verify there are no directories. - models.Directory.objects.count() == 0 + assert models.Directory.objects.count() == 0 result = main(job, str(transfer_dir), transfer.uuid, include_dirs) assert result == 0 # Verify two directories were created. - models.Directory.objects.filter(transfer=transfer).count() == 2 + assert models.Directory.objects.filter(transfer=transfer).count() == 2 diff --git a/src/MCPClient/tests/test_create_aip_mets.py b/src/MCPClient/tests/test_create_aip_mets.py index f454b8a356..cacaff65be 100644 --- a/src/MCPClient/tests/test_create_aip_mets.py +++ b/src/MCPClient/tests/test_create_aip_mets.py @@ -826,7 +826,7 @@ def generate_aip_mets_v2_state(self): ) self.state = create_mets_v2.MetsState() self.state.globalStructMapCounter = random.choice( - [x for x in range(arbitrary_max_structmaps)] + list(range(arbitrary_max_structmaps)) ) self.structmap_div_element = create_mets_v2.createFileSec( job=Job("stub", "stub", []), diff --git a/src/MCPClient/tests/test_dip_generation_helper.py b/src/MCPClient/tests/test_dip_generation_helper.py index f0ba76ac84..98daa2651d 100644 --- a/src/MCPClient/tests/test_dip_generation_helper.py +++ b/src/MCPClient/tests/test_dip_generation_helper.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import os +from unittest import mock -import vcr from django.test import TestCase THIS_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -31,7 +31,6 @@ def test_empty_csv(self): assert rc == 1 assert ArchivesSpaceDIPObjectResourcePairing.objects.all().exists() is False - @vcr.use_cassette(os.path.join(THIS_DIR, "fixtures", "test_no_files_in_db.yaml")) def test_no_files_in_db(self): """It should do nothing if no files are found in the DB.""" sip_path = os.path.join(THIS_DIR, "fixtures", "metadata_csv_sip", "") @@ -41,10 +40,17 @@ def test_no_files_in_db(self): assert rc == 0 assert ArchivesSpaceDIPObjectResourcePairing.objects.all().exists() is False - @vcr.use_cassette( - os.path.join(THIS_DIR, "fixtures", "test_parse_archivesspace_ids.yaml") + @mock.patch( + "dip_generation_helper.create_archivesspace_client", + return_value=mock.Mock( + **{ + "find_by_id.return_value": [ + {"id": "/repositories/2/archival_objects/752250"} + ] + } + ), ) - def test_parse_to_db(self): + def test_parse_to_db(self, create_archivesspace_client): """ It should create an entry in ArchivesSpaceDIPObjectResourcePairing for each file in archivesspaceids.csv It should match the reference ID to a resource ID. diff --git a/src/MCPClient/tests/test_load_premis_events_from_xml.py b/src/MCPClient/tests/test_load_premis_events_from_xml.py index 9178cb328f..7589c2156d 100644 --- a/src/MCPClient/tests/test_load_premis_events_from_xml.py +++ b/src/MCPClient/tests/test_load_premis_events_from_xml.py @@ -223,7 +223,7 @@ def test_get_premis_element_children_identifiers(mocker): "params", [ {"original_name": "name", "expected_original_name": "name"}, - {"original_name": tuple(), "expected_original_name": ""}, + {"original_name": (), "expected_original_name": ""}, ], ids=["original_name_as_string", "original_name_as_empty_string"], ) diff --git a/src/MCPClient/tests/test_pid_components.py b/src/MCPClient/tests/test_pid_components.py index 70e05682aa..2f4697fc85 100644 --- a/src/MCPClient/tests/test_pid_components.py +++ b/src/MCPClient/tests/test_pid_components.py @@ -8,30 +8,22 @@ import os from itertools import chain +import bind_pid +import bind_pids +import create_mets_v2 +import namespaces as ns import pytest -import vcr from job import Job from main.models import DashboardSetting from main.models import Directory from main.models import File from main.models import SIP from main.models import Transfer - +from pid_declaration import DeclarePIDs +from pid_declaration import DeclarePIDsException THIS_DIR = os.path.dirname(os.path.abspath(__file__)) -vcr_cassettes = vcr.VCR( - cassette_library_dir=os.path.join(THIS_DIR, "fixtures", "vcr_cassettes"), - path_transformer=vcr.VCR.ensure_suffix(".yaml"), -) - -import bind_pid -import bind_pids -import create_mets_v2 -import namespaces as ns -from pid_declaration import DeclarePIDs, DeclarePIDsException - - PACKAGE_UUID = "cb5ebaf5-beda-40b4-8d0c-fefbd546b8de" INCOMPLETE_CONFIG_MSG = "A value for parameter" BOUND_URI = "http://195.169.88.240:8017/12345/" @@ -257,6 +249,11 @@ def data(db, sip, settings, transfer, files, directories): return +@pytest.fixture +def pid_web_service(mocker): + mocker.patch("requests.post", return_value=mocker.Mock(status_code=200)) + + @pytest.mark.django_db def test_bind_pids_no_config(data, caplog, job): """Test the output of the code without any args. @@ -275,7 +272,7 @@ def test_bind_pids_no_config(data, caplog, job): @pytest.mark.django_db -def test_bind_pids(data, mocker, job): +def test_bind_pids(data, mocker, job, pid_web_service): """Test the bind_pids function end-to-end and ensure that the result is that which is anticipated. @@ -286,10 +283,10 @@ def test_bind_pids(data, mocker, job): # test here using the package UUID, the function's fallback position. mocker.patch.object(bind_pids, "_get_unique_acc_no", return_value=PACKAGE_UUID) mocker.patch.object(bind_pids, "_validate_handle_server_config", return_value=None) - with vcr_cassettes.use_cassette("test_bind_pids_to_sip_and_dirs.yaml") as cassette: - # Primary entry-point for the bind_pids microservice job. - bind_pids.main(job, PACKAGE_UUID, "") - assert cassette.all_played + + # Primary entry-point for the bind_pids microservice job. + bind_pids.main(job, PACKAGE_UUID, "") + sip_mdl = SIP.objects.filter(uuid=PACKAGE_UUID).first() assert len(sip_mdl.identifiers.all()) == len( BOUND_IDENTIFIER_TYPES @@ -350,7 +347,7 @@ def test_bind_pid_no_config(data, caplog, job): @pytest.mark.django_db -def test_bind_pid(data, job): +def test_bind_pid(data, job, pid_web_service): """Test the bind_pid function end-to-end and ensure that the result is that which is anticipated. @@ -369,9 +366,7 @@ def test_bind_pid(data, job): package_files ), "Number of files returned from package is incorrect" for file_ in files: - with vcr_cassettes.use_cassette("test_bind_pid_to_files.yaml") as cassette: - bind_pid.main(job, file_.pk) - assert cassette.all_played + bind_pid.main(job, file_.pk) for file_mdl in files: bound = {idfr.type: idfr.value for idfr in file_mdl.identifiers.all()} assert ( @@ -421,7 +416,7 @@ def test_bind_pid_no_settings(data, caplog, job): @pytest.mark.django_db -def test_pid_declaration(data, mocker, job): +def test_pid_declaration(data, mocker, job, pid_web_service): """Test that the overall functionality of the PID declaration functions work as expected. """ @@ -459,13 +454,10 @@ def test_pid_declaration(data, mocker, job): assert example_uri in value, "Example URI type not preserved" if key == PID_ULID: assert len(example_ulid) == len(value) - # Use the previous PID binding vcr cassettes to ensure declared PIDs can - # co-exist with bound ones. mocker.patch.object(bind_pids, "_get_unique_acc_no", return_value=PACKAGE_UUID) mocker.patch.object(bind_pids, "_validate_handle_server_config", return_value=None) - with vcr_cassettes.use_cassette("test_bind_pids_to_sip_and_dirs.yaml") as cassette: - # Primary entry-point for the bind_pids microservice job. - bind_pids.main(job, PACKAGE_UUID, "") + # Primary entry-point for the bind_pids microservice job. + bind_pids.main(job, PACKAGE_UUID, "") for mdl in chain((sip_mdl,), dir_mdl): dir_dmd_sec = create_mets_v2.getDirDmdSec(mdl, "") id_type = dir_dmd_sec.xpath( @@ -486,9 +478,7 @@ def test_pid_declaration(data, mocker, job): if key == PID_ULID: assert len(example_ulid) == len(value) for file_ in files: - with vcr_cassettes.use_cassette("test_bind_pid_to_files.yaml") as cassette: - bind_pid.main(job, file_.pk) - assert cassette.all_played + bind_pid.main(job, file_.pk) for file_mdl in files: file_level_premis = create_mets_v2.create_premis_object(file_mdl.pk) id_type = file_level_premis.xpath( diff --git a/src/MCPClient/tests/test_post_store_aip_hook.py b/src/MCPClient/tests/test_post_store_aip_hook.py index 1bdc44d3f6..a118c8d7ea 100644 --- a/src/MCPClient/tests/test_post_store_aip_hook.py +++ b/src/MCPClient/tests/test_post_store_aip_hook.py @@ -1,7 +1,7 @@ import os +from unittest import mock import pytest -import vcr from django.test import TestCase THIS_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -10,11 +10,6 @@ from main import models -my_vcr = vcr.VCR( - cassette_library_dir=os.path.join(THIS_DIR, "fixtures", "vcr_cassettes"), - path_transformer=vcr.VCR.ensure_suffix(".yaml"), -) - class TestDSpaceToArchivesSpace(TestCase): """Test sending the DSpace handle to ArchivesSpace.""" @@ -38,23 +33,53 @@ def test_no_archivesspace(self): ) assert rc == 1 - def test_no_dspace(self): + @mock.patch( + "storageService.get_file_info", + return_value=[{"misc_attributes": {}}], + ) + def test_no_dspace(self, get_file_info): """It should abort if no DSpace handle found.""" - with my_vcr.use_cassette("test_no_dspace.yaml") as c: - rc = post_store_aip_hook.dspace_handle_to_archivesspace( - Job("stub", "stub", []), self.sip_uuid - ) - assert rc == 1 - assert c.all_played - - def test_dspace_handle_to_archivesspace(self): + rc = post_store_aip_hook.dspace_handle_to_archivesspace( + Job("stub", "stub", []), self.sip_uuid + ) + assert rc == 1 + + @mock.patch( + "storageService.get_file_info", + return_value=[{"misc_attributes": {"handle": "123456789/41"}}], + ) + @mock.patch( + "requests.post", + side_effect=[ + mock.Mock(**{"json.return_value": {"session": "session-id"}}), + mock.Mock(status_code=200), + ], + ) + @mock.patch( + "requests.get", + return_value=mock.Mock( + **{ + "json.return_value": { + "file_versions": [ + { + "file_uri": "123456789/41", + "use_statement": "text-data", + "xlink_actuate_attribute": "none", + "xlink_show_attribute": "embed", + }, + ], + } + } + ), + ) + def test_dspace_handle_to_archivesspace( + self, requests_get, requests_post, get_file_info + ): """It should send the DSpace handle to ArchivesSpace.""" - with my_vcr.use_cassette("test_dspace_handle_to_archivesspace.yaml") as c: - rc = post_store_aip_hook.dspace_handle_to_archivesspace( - Job("stub", "stub", []), self.sip_uuid - ) - assert rc == 0 - assert c.all_played + rc = post_store_aip_hook.dspace_handle_to_archivesspace( + Job("stub", "stub", []), self.sip_uuid + ) + assert rc == 0 @pytest.fixture diff --git a/src/MCPClient/tests/test_reingest_mets.py b/src/MCPClient/tests/test_reingest_mets.py index 379ef3ecc8..c836670fce 100644 --- a/src/MCPClient/tests/test_reingest_mets.py +++ b/src/MCPClient/tests/test_reingest_mets.py @@ -1981,8 +1981,11 @@ def test_new_dmdsecs_for_directories(self): dmdsec = ( mets.get_file(label="Landing_zone", type="Directory").dmdsecs[0].serialize() ) - dmdsec.findtext(".//dc:title", namespaces=NSMAP) == "The landing zone" - dmdsec.findtext(".//dc:description", namespaces=NSMAP) == "A zone for landing" + assert dmdsec.findtext(".//dc:title", namespaces=NSMAP) == "The landing zone" + assert ( + dmdsec.findtext(".//dc:description", namespaces=NSMAP) + == "A zone for landing" + ) def test_update_existing(self): """ diff --git a/src/MCPClient/tests/test_verify_checksum.py b/src/MCPClient/tests/test_verify_checksum.py index f39cda9f5f..5afdbe4ffe 100644 --- a/src/MCPClient/tests/test_verify_checksum.py +++ b/src/MCPClient/tests/test_verify_checksum.py @@ -355,7 +355,7 @@ def test_write_premis_event_to_db(self, transfer): idtypes = [] agentnames = [] agenttypes = [] - for agent_count, agent in enumerate(event.agents.all(), 1): + for _agent_count, agent in enumerate(event.agents.all(), 1): idvalues.append(agent.identifiervalue) idtypes.append(agent.identifiertype) agentnames.append(agent.name) @@ -371,9 +371,9 @@ def test_write_premis_event_to_db(self, transfer): ), "agent name values returned don't match" assert set(agenttypes) == set(agent_types), "agent types don't match" assert ( - agent_count == number_of_expected_agents + _agent_count == number_of_expected_agents ), "Number of agents is incorrect: {} expected: {}".format( - agent_count, number_of_expected_agents + _agent_count, number_of_expected_agents ) # Collect the different checksum algorithms written to ensure they # were all written independently in the function. diff --git a/src/MCPServer/lib/assets/workflow-schema-v1.json b/src/MCPServer/lib/assets/workflow-schema-v1.json index 25415dbedc..62a44cdbd9 100644 --- a/src/MCPServer/lib/assets/workflow-schema-v1.json +++ b/src/MCPServer/lib/assets/workflow-schema-v1.json @@ -67,50 +67,25 @@ "additionalProperties": { "properties": { "job_status": { - "oneOf": [ - { - "$ref": "#/definitions/job_status" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/job_status" }, "link_id": { - "oneOf": [ - { - "$ref": "#/definitions/uuid" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/uuid" } }, + "required": [ + "job_status" + ], "type": "object" }, "minProperties": 0, "type": "object" }, "fallback_job_status": { - "oneOf": [ - { - "$ref": "#/definitions/job_status" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/job_status" }, "fallback_link_id": { - "oneOf": [ - { - "$ref": "#/definitions/uuid" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/uuid" }, "group": { "$ref": "#/definitions/translations" @@ -120,7 +95,6 @@ "config", "exit_codes", "fallback_job_status", - "fallback_link_id", "description", "group" ], @@ -138,11 +112,9 @@ "type": "string" }, "chain_choices": { - "items": [ - { - "$ref": "#/definitions/uuid" - } - ], + "items": { + "$ref": "#/definitions/uuid" + }, "minItems": 1, "type": "array" } @@ -166,55 +138,16 @@ "type": "string" }, "arguments": { - "type": [ - "string", - "null" - ] + "type": "string" }, "execute": { "type": "string" - }, - "filter_file_end": { - "type": [ - "string", - "null" - ] - }, - "filter_file_start": { - "type": [ - "string", - "null" - ] - }, - "filter_subdir": { - "type": [ - "string", - "null" - ] - }, - "stderr_file": { - "type": [ - "string", - "null" - ] - }, - "stdout_file": { - "type": [ - "string", - "null" - ] } }, "required": [ "@manager", "@model", - "execute", - "arguments", - "filter_file_start", - "filter_file_end", - "filter_subdir", - "stdout_file", - "stderr_file" + "execute" ], "type": "object" }, @@ -229,56 +162,14 @@ "pattern": "StandardTaskConfig", "type": "string" }, - "arguments": { - "type": [ - "string", - "null" - ] - }, "execute": { "type": "string" - }, - "filter_file_end": { - "type": [ - "string", - "null" - ] - }, - "filter_file_start": { - "type": [ - "string", - "null" - ] - }, - "filter_subdir": { - "type": [ - "string", - "null" - ] - }, - "stderr_file": { - "type": [ - "string", - "null" - ] - }, - "stdout_file": { - "type": [ - "string", - "null" - ] } }, "required": [ "@manager", "@model", - "execute", - "arguments", - "filter_file_start", - "filter_file_end", - "filter_subdir", - "stdout_file", - "stderr_file" + "execute" ], "type": "object" }, @@ -294,14 +185,7 @@ "type": "string" }, "chain_id": { - "oneOf": [ - { - "$ref": "#/definitions/uuid" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/uuid" }, "variable": { "type": "string" @@ -310,7 +194,6 @@ "required": [ "@manager", "@model", - "chain_id", "variable" ], "type": "object" @@ -327,31 +210,29 @@ "type": "string" }, "replacements": { - "items": [ - { - "additionalProperties": false, - "properties": { - "description": { - "$ref": "#/definitions/translations" - }, - "id": { - "$ref": "#/definitions/uuid" - }, - "items": { - "additionalProperties": { - "type": "string" - }, - "type": "object" - } + "items": { + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/definitions/translations" }, - "required": [ - "id", - "description", - "items" - ], - "type": "object" - } - ], + "id": { + "$ref": "#/definitions/uuid" + }, + "items": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "required": [ + "id", + "description", + "items" + ], + "type": "object" + }, "minItems": 0, "type": "array" } @@ -375,31 +256,19 @@ "type": "string" }, "chain_id": { - "oneOf": [ - { - "$ref": "#/definitions/uuid" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/uuid" }, "variable": { "type": "string" }, "variable_value": { - "type": [ - "string", - "null" - ] + "type": "string" } }, "required": [ "@manager", "@model", - "chain_id", - "variable", - "variable_value" + "variable" ], "type": "object" }, @@ -415,55 +284,31 @@ "type": "string" }, "arguments": { - "type": [ - "string", - "null" - ] + "type": "string" }, "execute": { "type": "string" }, "filter_file_end": { - "type": [ - "string", - "null" - ] + "type": "string" }, "filter_file_start": { - "type": [ - "string", - "null" - ] + "type": "string" }, "filter_subdir": { - "type": [ - "string", - "null" - ] + "type": "string" }, "stderr_file": { - "type": [ - "string", - "null" - ] + "type": "string" }, "stdout_file": { - "type": [ - "string", - "null" - ] + "type": "string" } }, "required": [ "@manager", "@model", - "execute", - "arguments", - "filter_file_start", - "filter_file_end", - "filter_subdir", - "stdout_file", - "stderr_file" + "execute" ], "type": "object" }, @@ -479,55 +324,31 @@ "type": "string" }, "arguments": { - "type": [ - "string", - "null" - ] + "type": "string" }, "execute": { "type": "string" }, "filter_file_end": { - "type": [ - "string", - "null" - ] + "type": "string" }, "filter_file_start": { - "type": [ - "string", - "null" - ] + "type": "string" }, "filter_subdir": { - "type": [ - "string", - "null" - ] + "type": "string" }, "stderr_file": { - "type": [ - "string", - "null" - ] + "type": "string" }, "stdout_file": { - "type": [ - "string", - "null" - ] + "type": "string" } }, "required": [ "@manager", "@model", - "execute", - "arguments", - "filter_file_start", - "filter_file_end", - "filter_subdir", - "stdout_file", - "stderr_file" + "execute" ], "type": "object" }, diff --git a/src/MCPServer/lib/assets/workflow.json b/src/MCPServer/lib/assets/workflow.json index f0ce8546ac..8af6a54e32 100644 --- a/src/MCPServer/lib/assets/workflow.json +++ b/src/MCPServer/lib/assets/workflow.json @@ -969,12 +969,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%watchDirectoryPath%workFlowDecisions/compressionAIPDecisions/.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to compressionAIPDecisions directory", @@ -984,8 +979,7 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", @@ -1004,8 +998,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%SIPObjectsDirectory%attachments/\" \"%SIPUUID%\" \"%date%\" \"%taskUUID%\" \"transferDirectory\" \"transfer_id\" \"%SIPDirectory%\"", "execute": "changeObjectNames_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects/attachments", "stderr_file": "%SIPLogsDirectory%filenameChanges.log", "stdout_file": "%SIPLogsDirectory%filenameChanges.log" @@ -1131,7 +1123,6 @@ } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Prepare AIP", "es": "Preparar AIP", @@ -1237,7 +1228,6 @@ } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Prepare AIP", "es": "Preparar AIP", @@ -1251,12 +1241,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPObjectsDirectory%\"", - "execute": "checkTransferDirectoryForObjects_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "checkTransferDirectoryForObjects_v0.0" }, "description": { "en": "Check transfer directory for objects", @@ -1290,11 +1275,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%SIPUUID%\"", "execute": "characterizeFile_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/metadata/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/metadata/" }, "description": { "en": "Characterize and extract metadata on metadata files", @@ -1326,12 +1307,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"Standard\"", - "execute": "archivematicaSetTransferType_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "archivematicaSetTransferType_v0.0" }, "description": { "en": "Set transfer type: Standard", @@ -1417,11 +1393,7 @@ "@model": "StandardTaskConfig", "arguments": "thumbnail \"%fileUUID%\" \"%relativeLocation%\" \"%SIPDirectory%\" \"%SIPUUID%\" \"%taskUUID%\" \"original\" --thumbnail_mode \"%ThumbnailMode%\"", "execute": "normalize_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/" }, "description": { "en": "Normalize for thumbnails", @@ -1461,11 +1433,7 @@ "@model": "StandardTaskConfig", "arguments": "thumbnail \"%fileUUID%\" \"%relativeLocation%\" \"%SIPDirectory%\" \"%SIPUUID%\" \"%taskUUID%\" \"original\" --thumbnail_mode \"%ThumbnailMode%\"", "execute": "normalize_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/" }, "description": { "en": "Normalize for thumbnails", @@ -1504,12 +1472,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPName%-%SIPUUID%\"", - "execute": "copySubmissionDocs_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copySubmissionDocs_v0.0" }, "description": { "en": "Copy submission documentation", @@ -1542,11 +1505,7 @@ "@model": "StandardTaskConfig", "arguments": "thumbnail \"%fileUUID%\" \"%relativeLocation%\" \"%SIPDirectory%\" \"%SIPUUID%\" \"%taskUUID%\" \"original\" --thumbnail_mode \"%ThumbnailMode%\"", "execute": "normalize_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/" }, "description": { "en": "Normalize for thumbnails", @@ -1585,12 +1544,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"Dataverse\"", - "execute": "archivematicaSetTransferType_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "archivematicaSetTransferType_v0.0" }, "description": { "en": "Set transfer type: Dataverse", @@ -1618,12 +1572,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "-m 770 \"%SIPDirectory%thumbnails/\"", - "execute": "createDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createDirectory_v0.0" }, "description": { "en": "Create thumbnails directory", @@ -1655,12 +1604,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\"", - "execute": "restructureForCompliance_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "restructureForCompliance_v0.0" }, "description": { "en": "Attempt restructure for compliance", @@ -1707,7 +1651,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Approve transfer", "es": "Aprobar transferencia", @@ -1723,12 +1666,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%sharedMicroServiceTasksConfigs/processingMCPConfigs/%processingConfiguration%ProcessingMCP.xml\" \"%SIPDirectory%processingMCP.xml\" -n", - "execute": "copy_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copy_v0.0" }, "description": { "en": "Include default Transfer processingMCP.xml", @@ -1756,12 +1694,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%rejectedDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to the rejected directory", @@ -1772,12 +1705,10 @@ "end": true, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Reject AIP", "es": "Rechazar AIP", @@ -1793,12 +1724,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to processing directory", @@ -1828,8 +1754,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%IDCommand%\" \"%relativeLocation%\" \"%fileUUID%\"", "execute": "identifyFileFormat_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects/attachments", "stderr_file": "%SIPLogsDirectory%fileFormatIdentification.log", "stdout_file": "%SIPLogsDirectory%fileFormatIdentification.log" @@ -1864,12 +1788,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to processing directory", @@ -1899,12 +1818,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPName%\" \"%SIPDirectory%\"", - "execute": "trimRestructureForCompliance_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "trimRestructureForCompliance_v0.0" }, "description": { "en": "Restructure TRIM for compliance", @@ -1932,12 +1846,7 @@ "@manager": "linkTaskManagerFiles", "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%SIPUUID%\" \"%sharedPath%\" \"preservation\"", - "execute": "policyCheck_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "policyCheck_v0.0" }, "description": { "en": "Policy checks for preservation derivatives", @@ -1972,11 +1881,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%SIPDirectory%\" \"%fileUUID%\"", "execute": "examineContents_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects" }, "description": { "en": "Examine contents", @@ -2011,11 +1916,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%SIPLogsDirectory%fileMeta/%fileUUID%.xml\" \"%date%\" \"%taskUUID%\" \"%fileUUID%\" \"%fileGrpUse%\"", "execute": "FITS_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/manualNormalization/preservation", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/manualNormalization/preservation" }, "description": { "en": "Run FITS on manual normalized preservation files", @@ -2045,8 +1946,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%objects/submissionDocumentation/\" \"%SIPUUID%\" \"%date%\" \"%taskUUID%\" \"SIPDirectory\" \"sip_id\" \"%SIPDirectory%\"", "execute": "changeObjectNames_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects/submissionDocumentation", "stderr_file": "%SIPLogsDirectory%filenameChanges.log", "stdout_file": "%SIPLogsDirectory%filenameChanges.log" @@ -2078,12 +1977,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%processingMCP.xml\" \"%SIPDirectory%DIP/processingMCP.xml\"", - "execute": "copy_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copy_v0.0" }, "description": { "en": "Copy preconfigured choice XML to DIP directory", @@ -2115,12 +2009,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%\"", - "execute": "createAIC_METS_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createAIC_METS_v1.0" }, "description": { "en": "Create AIC METS file", @@ -2165,7 +2054,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Policy checks for derivatives", "fr": "Vérifications de politique sur les copies", @@ -2194,7 +2082,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Approve transfer", "es": "Aprobar transferencia", @@ -2210,12 +2097,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%%SIPName%-%SIPUUID%\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Rename with transfer UUID", @@ -2305,11 +2187,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%fileUUID%\"", "execute": "manualNormalizationIdentifyFilesIncluded_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/manualNormalization/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/manualNormalization/" }, "description": { "en": "Identify manually normalized files", @@ -2341,12 +2219,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPObjectsDirectory%\" \"%SIPName%\" \"%SIPUUID%\" \"%processingDirectory%\" \"%sharedPath%watchedDirectories/system/autoProcessSIP/\" \"%sharedPath%\"", - "execute": "createSIPsfromTRIMTransferContainers_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createSIPsfromTRIMTransferContainers_v0.0" }, "description": { "en": "Create SIPs from TRIM transfer containers", @@ -2359,8 +2232,7 @@ "end": true, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", @@ -2380,12 +2252,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%metadata/submissionDocumentation\" \"%SIPDirectory%objects/submissionDocumentation\"", - "execute": "moveOrMerge_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveOrMerge_v0.0" }, "description": { "en": "Move submission documentation into objects directory", @@ -2416,11 +2283,7 @@ "@model": "StandardTaskConfig", "arguments": "thumbnail \"%fileUUID%\" \"%relativeLocation%\" \"%SIPDirectory%\" \"%SIPUUID%\" \"%taskUUID%\" \"original\" --thumbnail_mode \"%ThumbnailMode%\"", "execute": "normalize_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/" }, "description": { "en": "Normalize for thumbnails", @@ -2459,12 +2322,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--amdSec --baseDirectoryPath \"%SIPDirectory%\" --sipUUID \"%SIPUUID%\" --xmlFile \"%SIPDirectory%METS.%SIPUUID%.xml\" --sipType \"%SIPType%\" --createNormativeStructmap", - "execute": "createMETS_v2.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createMETS_v2.0" }, "description": { "en": "Generate METS.xml document", @@ -2529,12 +2387,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"reject\" \"%SIPUUID%\"", - "execute": "failedSIPCleanup_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "failedSIPCleanup_v1.0" }, "description": { "en": "Cleanup rejected SIP", @@ -2568,12 +2421,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%metadata/rights.csv\"", - "execute": "rightsFromCSV_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "rightsFromCSV_v0.0" }, "description": { "en": "Load rights", @@ -2604,12 +2452,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%metadata/file_labels.csv\"", - "execute": "loadLabelsFromCSV_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "loadLabelsFromCSV_v0.0" }, "description": { "en": "Load labels from metadata/file_labels.csv", @@ -2640,12 +2483,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"TRIM\"", - "execute": "archivematicaSetTransferType_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "archivematicaSetTransferType_v0.0" }, "description": { "en": "Set transfer type: TRIM", @@ -2675,11 +2513,8 @@ "@model": "StandardTaskConfig", "arguments": "\"%fileUUID%\" \"%relativeLocation%\" \"%date%\" \"%taskUUID%\"", "execute": "archivematicaClamscan_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects/submissionDocumentation", - "stderr_file": "%SIPLogsDirectory%clamAVScan.txt", - "stdout_file": null + "stderr_file": "%SIPLogsDirectory%clamAVScan.txt" }, "description": { "en": "Scan for viruses in submission documentation", @@ -2709,11 +2544,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%fileUUID%\" \"%relativeLocation%\" \"%date%\" \"%taskUUID%\"", "execute": "archivematicaClamscan_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": "%SIPLogsDirectory%clamAVScan.txt", - "stdout_file": null + "stderr_file": "%SIPLogsDirectory%clamAVScan.txt" }, "description": { "en": "Scan for viruses in directories", @@ -2746,10 +2577,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%transferDirectory%\" \"%date%\" \"%taskUUID%\" \"%DeletePackage%\"", "execute": "extractContents_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, "stdout_file": "%SIPLogsDirectory%extractContents.log" }, "description": { @@ -2783,12 +2610,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--SIPDirectory \"%SIPDirectory%\" --serviceDirectory \"objects/service/\" --objectsDirectory \"objects/\" --SIPUUID \"%SIPUUID%\" --date \"%date%\"", - "execute": "checkForServiceDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "checkForServiceDirectory_v0.0" }, "description": { "en": "Check for Service directory", @@ -2830,7 +2652,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Extract packages", "es": "Extraer paquetes", @@ -2847,11 +2668,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%IDCommand%\" \"%relativeLocation%\" \"%fileUUID%\"", "execute": "identifyFileFormat_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/submissionDocumentation", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/submissionDocumentation" }, "description": { "en": "Identify file format", @@ -2882,8 +2699,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "16415d2f-5642-496d-a46d-00028ef6eb0a", - "variable": "loadOptionsToCreateSIP", - "variable_value": null + "variable": "loadOptionsToCreateSIP" }, "description": { "en": "Set TRIM options to create SIPs", @@ -2911,12 +2727,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%AIPsStore%\" \"%SIPDirectory%%AIPFilename%\" \"%SIPUUID%\" \"%SIPName%\" \"%SIPType%\"", - "execute": "storeAIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "storeAIP_v0.0" }, "description": { "en": "Store the AIP", @@ -2947,12 +2758,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", - "execute": "verifySIPCompliance_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "verifySIPCompliance_v0.0" }, "description": { "en": "Verify SIP compliance", @@ -2984,12 +2790,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%sharedMicroServiceTasksConfigs/processingMCPConfigs/%processingConfiguration%ProcessingMCP.xml\" \"%SIPDirectory%processingMCP.xml\" -n", - "execute": "copy_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copy_v0.0" }, "description": { "en": "Include default Transfer processingMCP.xml", @@ -3017,8 +2818,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "9ec31d55-f053-4695-b86d-8c2a8abdb0fc", - "variable": "linkToConvertDataverseStructure", - "variable_value": null + "variable": "linkToConvertDataverseStructure" }, "description": { "en": "Set convert Dataverse structure flag" @@ -3044,12 +2844,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPName%\" \"%SIPDirectory%\" \"%date%\"", - "execute": "trimCreateRightsEntries_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "trimCreateRightsEntries_v0.0" }, "description": { "en": "Create rights to flag closed AIPS.", @@ -3078,11 +2873,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%fileUUID%\" \"%relativeLocation%\" \"%date%\" \"%taskUUID%\"", "execute": "archivematicaClamscan_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": "%SIPLogsDirectory%clamAVScan.txt", - "stdout_file": null + "stderr_file": "%SIPLogsDirectory%clamAVScan.txt" }, "description": { "en": "Scan for viruses in attachments", @@ -3115,8 +2906,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%IDCommand%\" \"%relativeLocation%\" \"%fileUUID%\"", "execute": "identifyFileFormat_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects/attachments", "stderr_file": "%SIPLogsDirectory%fileFormatIdentification.log", "stdout_file": "%SIPLogsDirectory%fileFormatIdentification.log" @@ -3175,12 +2964,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%%SIPName%-%SIPUUID%\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Rename with transfer UUID", @@ -3210,8 +2994,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%IDCommand%\" \"%relativeLocation%\" \"%fileUUID%\" --disable-reidentify", "execute": "identifyFileFormat_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects", "stderr_file": "%SIPLogsDirectory%fileFormatIdentification.log", "stdout_file": "%SIPLogsDirectory%fileFormatIdentification.log" @@ -3247,8 +3029,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%SIPObjectsDirectory%\" \"%SIPUUID%\" \"%date%\" \"%taskUUID%\" \"transferDirectory\" \"transfer_id\" \"%SIPDirectory%\"", "execute": "changeObjectNames_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects", "stderr_file": "%SIPLogsDirectory%filenameChanges.log", "stdout_file": "%SIPLogsDirectory%filenameChanges.log" @@ -3281,12 +3061,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"Dspace\"", - "execute": "archivematicaSetTransferType_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "archivematicaSetTransferType_v0.0" }, "description": { "en": "Set transfer type: DSpace", @@ -3316,12 +3091,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%\" --transferUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --filterSubdir \"objects\"", - "execute": "updateSizeAndChecksum_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "updateSizeAndChecksum_v0.0" }, "description": { "en": "Assign checksums and file sizes to objects", @@ -3356,11 +3126,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%SIPUUID%\" \"%sharedPath%\" \"access\"", "execute": "validateFile_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "DIP/objects/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "DIP/objects/" }, "description": { "en": "Validate access derivatives", @@ -3396,12 +3162,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to processing directory", @@ -3432,12 +3193,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%\" --transferUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --filterSubdir \"objects\"", - "execute": "updateSizeAndChecksum_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "updateSizeAndChecksum_v0.0" }, "description": { "en": "Assign checksums and file sizes to objects", @@ -3472,11 +3228,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%taskUUID%\" \"%fileUUID%\"", "execute": "transcribeFile_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects" }, "description": { "en": "Transcribe", @@ -3528,12 +3280,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%\" --sipUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --filterSubdir \"objects/submissionDocumentation\"", - "execute": "updateSizeAndChecksum_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "updateSizeAndChecksum_v0.0" }, "description": { "en": "Assign checksums and file sizes to submissionDocumentation", @@ -3579,7 +3326,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Store AIP", "fr": "Stocker l’AIP", @@ -3595,8 +3341,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%IDCommand%\" \"%relativeLocation%\" \"%fileUUID%\" --disable-reidentify", "execute": "identifyFileFormat_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects", "stderr_file": "%SIPLogsDirectory%fileFormatIdentification.log", "stdout_file": "%SIPLogsDirectory%fileFormatIdentification.log" @@ -3631,12 +3375,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%watchDirectoryPath%uploadedDIPs/.\" -R", - "execute": "copy_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copy_v0.0" }, "description": { "en": "Move to the uploadedDIPs directory", @@ -3651,7 +3390,6 @@ } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Upload DIP", "es": "Subir DIP", @@ -3667,8 +3405,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "5c459c1a-f998-404d-a0dd-808709510b72", - "variable": "linkAfterAutoRestructureForCompliance", - "variable_value": null + "variable": "linkAfterAutoRestructureForCompliance" }, "description": { "en": "Designate to process as a standard transfer", @@ -3701,7 +3438,6 @@ "arguments": "\"%relativeLocation%\" \"%date%\" \"%taskUUID%\"", "execute": "verifyChecksumsInFileSecOfDspaceMETSFiles_v0.0", "filter_file_end": "mets.xml", - "filter_file_start": null, "filter_subdir": "objects", "stderr_file": "%SIPLogsDirectory%verifyChecksumsInFileSecOfDSpaceMETSFiles.log", "stdout_file": "%SIPLogsDirectory%verifyChecksumsInFileSecOfDSpaceMETSFiles.log" @@ -3734,11 +3470,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%SIPUUID%\"", "execute": "characterizeFile_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects" }, "description": { "en": "Characterize and extract metadata", @@ -3772,12 +3504,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--sipUUID \"%SIPUUID%\" --basePath \"%SIPDirectory%\" --xmlFile \"%SIPDirectory%\"metadata/submissionDocumentation/METS.xml --basePathString \"transferDirectory\"", - "execute": "createTransferMETS_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createTransferMETS_v1.0" }, "description": { "en": "Generate METS.xml document", @@ -3811,12 +3538,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "-m 770 \"%SIPDirectory%thumbnails/\"", - "execute": "createDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createDirectory_v0.0" }, "description": { "en": "Create thumbnails directory", @@ -3848,12 +3570,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", - "execute": "restructureBagAIPToSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "restructureBagAIPToSIP_v0.0" }, "description": { "en": "Restructure from bag AIP to SIP directory format", @@ -3883,12 +3600,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\" \"%processingDirectory%\" \"%sharedPath%\" --bag", - "execute": "extractZippedTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "extractZippedTransfer_v0.0" }, "description": { "en": "Extract zipped bag transfer", @@ -3920,12 +3632,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%rejectedDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to the rejected directory", @@ -3936,12 +3643,10 @@ "end": true, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Reject transfer", "es": "Rechazar transferencia", @@ -3957,12 +3662,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%\"", - "execute": "parseMETStoDB_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "parseMETStoDB_v1.0" }, "description": { "en": "Populate database with reingested AIP data", @@ -3993,11 +3693,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%SIPUUID%\"", "execute": "characterizeFile_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/submissionDocumentation", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/submissionDocumentation" }, "description": { "en": "Characterize and extract metadata on submission documentation", @@ -4029,12 +3725,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%rejectedDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to the rejected directory", @@ -4045,12 +3736,10 @@ "end": true, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Reject SIP", "es": "Rechazar SIP", @@ -4066,8 +3755,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "09b85517-e5f5-415b-a950-1a60ee285242", - "variable": "normalizationThumbnailProcessing", - "variable_value": null + "variable": "normalizationThumbnailProcessing" }, "description": { "en": "Set normalize path", @@ -4081,7 +3769,6 @@ } }, "fallback_job_status": "Completed successfully", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -4096,12 +3783,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "-m 770 \"%SIPDirectory%thumbnails/\"", - "execute": "createDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createDirectory_v0.0" }, "description": { "en": "Create thumbnails directory", @@ -4133,8 +3815,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "fba1fd92-150a-4969-84fb-f2c6097855cf", - "variable": "linkToParseDataverseMETS", - "variable_value": null + "variable": "linkToParseDataverseMETS" }, "description": { "en": "Set parse Dataverse METS flag" @@ -4160,12 +3841,7 @@ "@manager": "linkTaskManagerFiles", "@model": "StandardTaskConfig", "arguments": "--eventType \"removal from backlog\" --fileUUID \"%fileUUID%\" --eventIdentifierUUID \"%taskUUID%\" --eventDateTime \"%jobCreatedDate%\"", - "execute": "createEvent_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createEvent_v0.0" }, "description": { "en": "Create removal from backlog PREMIS events", @@ -4197,12 +3873,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%\" --transferUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --filterSubdir \"objects\"", - "execute": "updateSizeAndChecksum_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "updateSizeAndChecksum_v0.0" }, "description": { "en": "Assign checksums and file sizes to objects", @@ -4236,12 +3907,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%failed/.\" \"%SIPUUID%\" \"%sharedPath%\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to the failed directory", @@ -4252,12 +3918,10 @@ "end": true, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Failed transfer", "es": "Transferencia fallida", @@ -4273,12 +3937,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%thumbnails\" \"%SIPDirectory%DIP\"", - "execute": "copyThumbnailsToDIPDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copyThumbnailsToDIPDirectory_v0.0" }, "description": { "en": "Copy thumbnails to DIP directory", @@ -4319,7 +3978,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Scan for viruses", "fr": "Lancer l’antivirus", @@ -4334,12 +3992,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPObjectsDirectory%\" \"%SIPName%\" \"%SIPUUID%\" \"%processingDirectory%\" \"%sharedPath%watchedDirectories/system/autoProcessSIP/\" \"%sharedPath%\"", - "execute": "createSIPfromTransferObjects_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createSIPfromTransferObjects_v0.0" }, "description": { "en": "Create SIP from transfer objects", @@ -4353,12 +4006,10 @@ "end": true, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Create SIP from Transfer", "es": "Crear SIP a partir de la transferencia", @@ -4374,12 +4025,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--uuid \"%SIPUUID%\"", - "execute": "normalizeReport_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "normalizeReport_v0.0" }, "description": { "en": "Normalization report", @@ -4411,12 +4057,7 @@ "@manager": "linkTaskManagerFiles", "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%SIPUUID%\" \"%sharedPath%\" \"access\"", - "execute": "policyCheck_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "policyCheck_v0.0" }, "description": { "en": "Policy checks for access derivatives", @@ -4450,12 +4091,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--transferUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\"", - "execute": "assignFileUUIDs_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "assignFileUUIDs_v0.0" }, "description": { "en": "Assign file UUIDs to objects", @@ -4489,12 +4125,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%%SIPName%-%SIPUUID%\" \"%SIPDirectory%\" \"%SIPUUID%\"", - "execute": "bagit_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "bagit_v0.0" }, "description": { "en": "Prepare AIP", @@ -4524,12 +4155,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%%AIPFilename%\"", - "execute": "verifyAIP_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "verifyAIP_v1.0" }, "description": { "en": "Verify AIP", @@ -4562,12 +4188,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "-m 770 \"%SIPDirectory%thumbnails/\"", - "execute": "createDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createDirectory_v0.0" }, "description": { "en": "Create thumbnails directory", @@ -4599,12 +4220,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to processing directory", @@ -4634,12 +4250,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", - "execute": "verifyTransferCompliance_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "verifyTransferCompliance_v0.0" }, "description": { "en": "Verify transfer compliance", @@ -4669,12 +4280,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPObjectsDirectory%metadata/OCRfiles\" \"%SIPDirectory%DIP\"", - "execute": "copyRecursive_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copyRecursive_v0.0" }, "description": { "en": "Copy OCR data to DIP directory", @@ -4707,11 +4313,7 @@ "@model": "StandardTaskConfig", "arguments": "preservation \"%fileUUID%\" \"%relativeLocation%\" \"%SIPDirectory%\" \"%SIPUUID%\" \"%taskUUID%\" \"original\"", "execute": "normalize_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/" }, "description": { "en": "Normalize for preservation", @@ -4751,8 +4353,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "0e41c244-6c3e-46b9-a554-65e66e5c9324", - "variable": "fileIDcommand-transfer", - "variable_value": null + "variable": "fileIDcommand-transfer" }, "description": { "en": "Set files to identify", @@ -4796,7 +4397,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Approve transfer", "es": "Aprobar transferencia", @@ -4812,12 +4412,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", - "execute": "verifyTransferCompliance_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "verifyTransferCompliance_v0.0" }, "description": { "en": "Verify transfer compliance", @@ -4847,12 +4442,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%sharedMicroServiceTasksConfigs/processingMCPConfigs/%processingConfiguration%ProcessingMCP.xml\" \"%SIPDirectory%processingMCP.xml\" -n", - "execute": "copy_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copy_v0.0" }, "description": { "en": "Include default Transfer processingMCP.xml", @@ -4883,12 +4473,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "-m 770 \"%SIPDirectory%DIP/\" \"%SIPDirectory%DIP/objects/\"", - "execute": "createDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createDirectory_v0.0" }, "description": { "en": "Create DIP directory", @@ -4921,12 +4506,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%metadata/submissionDocumentation\"", - "execute": "checkForSubmissionDocumenation_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "checkForSubmissionDocumenation_v0.0" }, "description": { "en": "Check for submission documentation", @@ -4957,12 +4537,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPName%\" \"%SIPDirectory%\" \"%SIPType%\" \"%AIPsStore%\" ", - "execute": "indexAIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "indexAIP_v0.0" }, "description": { "en": "Index AIP", @@ -5049,7 +4624,6 @@ } }, "fallback_job_status": "Completed successfully", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -5064,12 +4638,7 @@ "@manager": "linkTaskManagerGetMicroserviceGeneratedListInStdOut", "@model": "StandardTaskConfig", "arguments": "", - "execute": "getAipStorageLocations_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "getAipStorageLocations_v0.0" }, "description": { "en": "Retrieve AIP Storage Locations", @@ -5100,8 +4669,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "8ce130d4-3f7e-46ec-868a-505cf9033d96", - "variable": "normalizationThumbnailProcessing", - "variable_value": null + "variable": "normalizationThumbnailProcessing" }, "description": { "en": "Set normalize path", @@ -5115,7 +4683,6 @@ } }, "fallback_job_status": "Completed successfully", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -5131,9 +4698,6 @@ "@model": "StandardTaskConfig", "arguments": "--sipUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --use \"submissionDocumentation\" --filterSubdir \"objects/submissionDocumentation\"", "execute": "assignFileUUIDs_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, "stderr_file": "%SIPLogsDirectory%FileUUIDsError.log", "stdout_file": "%SIPLogsDirectory%FileUUIDs.log" }, @@ -5167,12 +4731,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPObjectsDirectory%\" -o \"%SIPDirectory%metadata/directory_tree.txt\"", - "execute": "createDirectoryTree_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createDirectoryTree_v0.0" }, "description": { "en": "Save directory tree", @@ -5203,12 +4762,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", - "execute": "removeHiddenFilesAndDirectories_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "removeHiddenFilesAndDirectories_v0.0" }, "description": { "en": "Remove hidden files and directories", @@ -5238,9 +4792,6 @@ "@model": "StandardTaskConfig", "arguments": "--transferUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --filterSubdir \"objects\"", "execute": "assignFileUUIDs_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, "stderr_file": "%SIPLogsDirectory%FileUUIDsError.log", "stdout_file": "%SIPLogsDirectory%FileUUIDs.log" }, @@ -5276,12 +4827,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%clientAssetsDirectory%README/README.html\" \"%SIPDirectory%README.html\"", - "execute": "copy_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copy_v0.0" }, "description": { "en": "Add README file", @@ -5311,12 +4857,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--baseDirectoryPath \"%SIPDirectory%\" --sipUUID \"%SIPUUID%\" --xmlFile \"%SIPDirectory%METS.%SIPUUID%.xml\"", - "execute": "createMETS_v2.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createMETS_v2.0" }, "description": { "en": "Generate METS.xml document", @@ -5349,12 +4890,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\" --include-dirs \"%AssignUUIDsToDirectories%\"", - "execute": "assignUUIDsToDirectories_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "assignUUIDsToDirectories_v0.0" }, "description": { "en": "Assign UUIDs to directories", @@ -5386,12 +4922,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\" \"%processingDirectory%\" \"%sharedPath%\"", - "execute": "extractZippedTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "extractZippedTransfer_v0.0" }, "description": { "en": "Extract zipped transfer", @@ -5423,12 +4954,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%watchedDirectories/workFlowDecisions/metadataReminder/.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to metadata reminder", @@ -5438,8 +4964,7 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", @@ -5459,12 +4984,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%watchedDirectories/workFlowDecisions/createTree/.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to generate transfer tree", @@ -5474,8 +4994,7 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", @@ -5494,12 +5013,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "-m 770 \"%SIPDirectory%thumbnails/\"", - "execute": "createDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createDirectory_v0.0" }, "description": { "en": "Create thumbnails directory", @@ -5545,7 +5059,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Generate transfer structure report", "fr": "Générer un rapport de structure de transfert ", @@ -5560,12 +5073,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--sipUUID \"%SIPUUID%\" --sipPath \"%SIPDirectory%\"", - "execute": "dipGenerationHelper", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "dipGenerationHelper" }, "description": { "en": "Pre-processing for DIP generation", @@ -5596,8 +5104,6 @@ "@model": "StandardTaskConfig", "arguments": "--fileUUID \"%fileUUID%\" --inputFile \"%relativeLocation%\" --sipDirectory \"%SIPDirectory%\"", "execute": "removeFilesWithoutPresmisMetadata_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects/", "stderr_file": "%SIPLogsDirectory%removedFilesWithNoPremisMetadata.log", "stdout_file": "%SIPLogsDirectory%removedFilesWithNoPremisMetadata.log" @@ -5630,11 +5136,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%SIPUUID%\" \"%sharedPath%\" \"preservation\"", "execute": "validateFile_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/" }, "description": { "en": "Validate preservation derivatives", @@ -5671,11 +5173,7 @@ "@model": "StandardTaskConfig", "arguments": "\"access\" \"%fileUUID%\" \"%relativeLocation%\" \"%SIPDirectory%\" \"%SIPUUID%\" \"%taskUUID%\" \"service\"", "execute": "normalize_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/service", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/service" }, "description": { "en": "Normalize service files for access", @@ -5730,7 +5228,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Failed transfer compliance", "fr": "Échec de la mise en conformité du transfert", @@ -5744,12 +5241,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to processing directory", @@ -5778,8 +5270,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "8ba83807-2832-4e41-843c-2e55ad10ea0b", - "variable": "reNormalize", - "variable_value": null + "variable": "reNormalize" }, "description": { "en": "Set remove preservation and access normalized files to renormalize link.", @@ -5809,12 +5300,7 @@ "@manager": "linkTaskManagerFiles", "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\"", - "execute": "removeUnneededFiles_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "removeUnneededFiles_v0.0" }, "description": { "en": "Remove unneeded files", @@ -5843,12 +5329,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%SIPUUID%\"", - "execute": "verifychecksum_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "verifychecksum_v0.0" }, "description": { "en": "Verify metadata directory checksums", @@ -5892,7 +5373,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Upload DIP", "es": "Subir DIP", @@ -5908,12 +5388,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to processing directory", @@ -5943,12 +5418,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "775 \"%SIPDirectory%%AIPFilename%\"", - "execute": "setFilePermission_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "setFilePermission_v0.0" }, "description": { "en": "Set bag file permissions", @@ -5978,12 +5448,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%\"", - "execute": "determineAIPVersionKeyExitCode_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "determineAIPVersionKeyExitCode_v0.0" }, "description": { "en": "Determine processing path for this AIP version", @@ -6014,12 +5479,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%DIP\" \"%sharedPath%watchedDirectories/uploadDIP/%SIPDirectoryBasename%\"", - "execute": "move_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "move_v0.0" }, "description": { "en": "Generate DIP", @@ -6051,12 +5511,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%watchedDirectories/activeTransfers/Dspace/.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Failed compliance. See output in dashboard. Transfer moved back to activeTransfers.", @@ -6067,7 +5522,6 @@ "end": true, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Include default Transfer processingMCP.xml", "no": "Inkluder standardoverførings processingMCP.xml", @@ -6080,12 +5534,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--unitType \"%unitType%\" --unitIdentifier \"%SIPUUID%\" --unitName \"%SIPName%\"", - "execute": "emailFailReport_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "emailFailReport_v0.0" }, "description": { "en": "Email fail report", @@ -6118,8 +5567,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "0a6558cf-cf5f-4646-977e-7d6b4fde47e8", - "variable": "normalizationThumbnailProcessing", - "variable_value": null + "variable": "normalizationThumbnailProcessing" }, "description": { "en": "Set normalize path", @@ -6133,7 +5581,6 @@ } }, "fallback_job_status": "Completed successfully", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -6148,12 +5595,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to processing directory", @@ -6183,12 +5625,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPLogsDirectory%\" \"%SIPObjectsDirectory%\" \"%SIPDirectory%thumbnails/\"", - "execute": "removeDirectories_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "removeDirectories_v0.0" }, "description": { "en": "Remove bagged files", @@ -6248,12 +5685,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\" --include-dirs \"%AssignUUIDsToDirectories%\"", - "execute": "assignUUIDsToDirectories_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "assignUUIDsToDirectories_v0.0" }, "description": { "en": "Assign UUIDs to directories", @@ -6289,12 +5721,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--url=\"%url%\" --email=\"%email%\" --password=\"%password%\" --uuid=\"%SIPUUID%\" --debug=\"%debug%\" --version=\"%version%\" --rsync-command=\"%rsync_command%\" --rsync-target=\"%rsync_target%\"", - "execute": "upload-qubit_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "upload-qubit_v0.0" }, "description": { "en": "Upload DIP", @@ -6328,12 +5755,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%DIPsStore%\" \"%watchDirectoryPath%uploadedDIPs/%SIPName%-%SIPUUID%\" \"%SIPUUID%\" \"%SIPName%\" \"DIP\"", - "execute": "storeAIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "storeAIP_v0.0" }, "description": { "en": "Store DIP", @@ -6347,8 +5769,7 @@ "end": true, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", @@ -6369,9 +5790,6 @@ "@model": "StandardTaskConfig", "arguments": "--transferUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --filterSubdir \"objects\"", "execute": "assignFileUUIDs_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, "stderr_file": "%SIPLogsDirectory%FileUUIDsError.log", "stdout_file": "%SIPLogsDirectory%FileUUIDs.log" }, @@ -6407,12 +5825,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%\"", - "execute": "parseExternalMETS", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "parseExternalMETS" }, "description": { "en": "Parse external METS", @@ -6442,12 +5855,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--SIPDirectory \"%SIPDirectory%\" --accessDirectory \"objects/access/\" --objectsDirectory \"objects/\" --DIPDirectory \"DIP\" --SIPUUID \"%SIPUUID%\" --date \"%date%\"", - "execute": "checkForAccessDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "checkForAccessDirectory_v0.0" }, "description": { "en": "Check for Access directory", @@ -6484,12 +5892,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to processing directory", @@ -6519,12 +5922,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "-m 770 \"%SIPDirectory%thumbnails/\"", - "execute": "createDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createDirectory_v0.0" }, "description": { "en": "Create thumbnails directory", @@ -6556,12 +5954,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%%SIPName%-%SIPUUID%\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Rename with transfer UUID", @@ -6590,12 +5983,7 @@ "@manager": "linkTaskManagerFiles", "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%SIPUUID%\" \"%sharedPath%\" \"original\"", - "execute": "policyCheck_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "policyCheck_v0.0" }, "description": { "en": "Policy checks for originals", @@ -6629,7 +6017,6 @@ "config": { "@manager": "linkTaskManagerUnitVariableLinkPull", "@model": "TaskConfigUnitVariableLinkPull", - "chain_id": null, "variable": "normalizationThumbnailProcessing" }, "description": { @@ -6639,7 +6026,6 @@ }, "exit_codes": {}, "fallback_job_status": "Completed successfully", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -6654,12 +6040,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to processing directory", @@ -6689,12 +6070,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to processing directory", @@ -6724,12 +6100,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to processing directory", @@ -6772,7 +6143,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Validation", "es": "Validación", @@ -6787,8 +6157,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "cb8e5706-e73f-472f-ad9b-d1236af8095f", - "variable": "resumeAfterNormalizationFileIdentificationToolSelected", - "variable_value": null + "variable": "resumeAfterNormalizationFileIdentificationToolSelected" }, "description": { "en": "Grant normalization options for no pre-existing DIP", @@ -6818,12 +6187,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%%SIPName%-%SIPUUID%\" \"%SIPLogsDirectory%\" \"%SIPObjectsDirectory%\" \"%SIPDirectory%thumbnails/\"", - "execute": "removeDirectories_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "removeDirectories_v0.0" }, "description": { "en": "Remove bagged files", @@ -6867,7 +6231,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -6882,12 +6245,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", - "execute": "manualNormalizationRemoveMNDirectories_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "manualNormalizationRemoveMNDirectories_v0.0" }, "description": { "en": "Remove empty manual normalization directories", @@ -6917,10 +6275,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%sharedPath%\"", "execute": "bindPIDs_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, "stdout_file": "%SIPLogsDirectory%handles.log" }, "description": { @@ -6952,11 +6306,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPName%\" \"%SIPDirectory%\"", "execute": "manualNormalizationCheckForManualNormalizationDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/manualNormalization/preservation", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/manualNormalization/preservation" }, "description": { "en": "Check for manual normalized files", @@ -7045,12 +6395,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "-m 770 \"%SIPDirectory%DIP/\" \"%SIPDirectory%DIP/objects/\"", - "execute": "createDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createDirectory_v0.0" }, "description": { "en": "Create DIP directory", @@ -7083,12 +6428,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "-m 770 \"%SIPDirectory%DIP/\" \"%SIPDirectory%DIP/objects/\"", - "execute": "createDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createDirectory_v0.0" }, "description": { "en": "Create DIP directory", @@ -7121,12 +6461,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%watchedDirectories/storeAIP/.\" \"%SIPUUID%\" \"%sharedPath%\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to the store AIP approval directory", @@ -7136,12 +6471,10 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Store AIP", "fr": "Stocker l’AIP", @@ -7156,12 +6489,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%SIPUUID%\"", - "execute": "verifychecksum_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "verifychecksum_v0.0" }, "description": { "en": "Verify metadata directory checksums", @@ -7191,12 +6519,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%%SIPName%-%SIPUUID%\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Rename with transfer UUID", @@ -7226,12 +6549,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\"", - "execute": "restructureForComplianceSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "restructureForComplianceSIP_v0.0" }, "description": { "en": "Attempt restructure for compliance", @@ -7265,11 +6583,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%fileUUID%\" \"%relativeLocation%\" \"%date%\" \"%taskUUID%\"", "execute": "archivematicaClamscan_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/" }, "description": { "en": "Scan for viruses on extracted files", @@ -7300,12 +6614,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--unitType \"%unitType%\" --unitIdentifier \"%SIPUUID%\" --unitName \"%SIPName%\"", - "execute": "emailFailReport_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "emailFailReport_v0.0" }, "description": { "en": "Email fail report", @@ -7338,12 +6647,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%clientAssetsDirectory%README/TRANSFER_README.html\" \"%SIPDirectory%README.html\"", - "execute": "copy_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copy_v0.0" }, "description": { "en": "Add README file", @@ -7384,7 +6688,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Scan for viruses", "fr": "Lancer l’antivirus", @@ -7427,7 +6730,6 @@ "config": { "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", - "chain_id": null, "variable": "normalize_v1.0", "variable_value": "{'filterSubDir':'objects/attachments'}" }, @@ -7459,12 +6761,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%failed/.\" \"%SIPUUID%\" \"%sharedPath%\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to the failed directory", @@ -7475,12 +6772,10 @@ "end": true, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Failed SIP", "es": "SIP fallido", @@ -7506,7 +6801,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Transcribe SIP contents", "fr": "Transcrire le contenu du SIP", @@ -7519,7 +6813,6 @@ "config": { "@manager": "linkTaskManagerUnitVariableLinkPull", "@model": "TaskConfigUnitVariableLinkPull", - "chain_id": null, "variable": "resumeAfterNormalizationFileIdentificationToolSelected" }, "description": { @@ -7530,7 +6823,6 @@ }, "exit_codes": {}, "fallback_job_status": "Completed successfully", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -7545,12 +6837,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%\"", - "execute": "setMaildirFileGrpUseAndFileIDs_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "setMaildirFileGrpUseAndFileIDs_v0.0" }, "description": { "en": "Set file group use and fileIDs for maildir AIP", @@ -7589,7 +6876,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Process metadata directory", "es": "Procesar directorio de metadatos", @@ -7604,11 +6890,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%\"", "execute": "declarepids_v0.0", - "filter_file_end": "", - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "filter_file_end": "" }, "description": { "en": "Load persistent identifiers from external file", @@ -7635,12 +6917,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%clientAssetsDirectory%mets/mets.xsd\"", - "execute": "archivematicaVerifyMets_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "archivematicaVerifyMets_v0.0" }, "description": { "en": "Verify mets_structmap.xml compliance", @@ -7671,10 +6948,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%fileUUID%\"", "execute": "bindPID_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, "stdout_file": "%SIPLogsDirectory%handles.log" }, "description": { @@ -7769,12 +7042,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPName%\" \"%SIPDirectory%\" \"%date%\"", - "execute": "trimVerifyChecksums_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "trimVerifyChecksums_v0.0" }, "description": { "en": "Verify TRIM checksums", @@ -7802,12 +7070,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\"", - "execute": "restructureForCompliance_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "restructureForCompliance_v0.0" }, "description": { "en": "Attempt restructure for compliance", @@ -7839,12 +7102,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%watchedDirectories/activeTransfers/standardTransfer/.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move transfer back to activeTransfers directory.", @@ -7856,12 +7114,10 @@ "end": true, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Failed transfer compliance", "fr": "Échec de la mise en conformité du transfert", @@ -7875,12 +7131,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--SIPDirectory \"%SIPDirectory%\" --SIPUUID \"%SIPUUID%\" --preservation --thumbnails --access", - "execute": "retryNormalizeRemoveNormalized_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "retryNormalizeRemoveNormalized_v0.0" }, "description": { "en": "Remove preservation and access normalized files to renormalize.", @@ -7911,11 +7162,8 @@ "@model": "StandardTaskConfig", "arguments": "\"%fileUUID%\" \"%relativeLocation%\" \"%date%\" \"%taskUUID%\"", "execute": "archivematicaClamscan_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects/metadata", - "stderr_file": "%SIPLogsDirectory%clamAVScan.txt", - "stdout_file": null + "stderr_file": "%SIPLogsDirectory%clamAVScan.txt" }, "description": { "en": "Scan for viruses in metadata", @@ -7946,8 +7194,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "e950cd98-574b-4e57-9ef8-c2231e1ce451", - "variable": "normalizationThumbnailProcessing", - "variable_value": null + "variable": "normalizationThumbnailProcessing" }, "description": { "en": "Set normalize path", @@ -7961,7 +7208,6 @@ } }, "fallback_job_status": "Completed successfully", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -7976,12 +7222,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%metadata/metadata.json\"", - "execute": "jsonMetadataToCSV_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "jsonMetadataToCSV_v0.0" }, "description": { "en": "Process transfer JSON metadata", @@ -8025,7 +7266,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Policy checks for derivatives", "fr": "Vérifications de politique sur les copies", @@ -8040,11 +7280,7 @@ "@model": "StandardTaskConfig", "arguments": "thumbnail \"%fileUUID%\" \"%relativeLocation%\" \"%SIPDirectory%\" \"%SIPUUID%\" \"%taskUUID%\" \"original\" --thumbnail_mode \"%ThumbnailMode%\"", "execute": "normalize_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/" }, "description": { "en": "Normalize for thumbnails", @@ -8084,11 +7320,7 @@ "@model": "StandardTaskConfig", "arguments": "preservation \"%fileUUID%\" \"%relativeLocation%\" \"%SIPDirectory%\" \"%SIPUUID%\" \"%taskUUID%\" \"original\"", "execute": "normalize_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/" }, "description": { "en": "Normalize for preservation", @@ -8156,12 +7388,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--SIPDirectory \"%SIPDirectory%\" --SIPUUID \"%SIPUUID%\" --preservation --thumbnails", - "execute": "retryNormalizeRemoveNormalized_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "retryNormalizeRemoveNormalized_v0.0" }, "description": { "en": "Remove preservation normalized files to renormalize.", @@ -8193,10 +7420,7 @@ "arguments": "\"%fileUUID%\"", "execute": "identifyDspaceMETSFiles_v0.0", "filter_file_end": "mets.xml", - "filter_file_start": null, - "filter_subdir": "objects", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects" }, "description": { "en": "Identify DSpace mets files", @@ -8226,12 +7450,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to processing directory", @@ -8263,11 +7482,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPName%\" \"%SIPDirectory%\" \"%fileUUID%\" \"%relativeLocation%\" \"%date%\"", "execute": "manualNormalizationCreateMetadataAndRestructure_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/manualNormalization/preservation", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/manualNormalization/preservation" }, "description": { "en": "Relate manual normalized preservation files to the original files", @@ -8296,12 +7511,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "-d \"%SIPDirectory%%AIPFilename%\"", - "execute": "test_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "test_v0.0" }, "description": { "en": "Check if AIP is a file or directory", @@ -8353,7 +7563,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Upload DIP", "es": "Subir DIP", @@ -8399,10 +7608,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%transferDirectory%\" \"%date%\" \"%taskUUID%\" \"%DeletePackage%\"", "execute": "extractContents_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, "stdout_file": "%SIPLogsDirectory%extractContents.log" }, "description": { @@ -8436,12 +7641,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\" --include-dirs \"%AssignUUIDsToDirectories%\"", - "execute": "assignUUIDsToDirectories_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "assignUUIDsToDirectories_v0.0" }, "description": { "en": "Assign UUIDs to directories", @@ -8470,12 +7670,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%relativeLocation%metadata/dc.json\"", - "execute": "loadDublinCore_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "loadDublinCore_v0.0" }, "description": { "en": "Load Dublin Core metadata from disk", @@ -8516,7 +7711,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Process submission documentation", "fr": "Traiter la documentation de soumission", @@ -8545,7 +7739,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Approve transfer", "es": "Aprobar transferencia", @@ -8561,8 +7754,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "8de9fe10-932f-4151-88b0-b50cf271e156", - "variable": "reNormalize", - "variable_value": null + "variable": "reNormalize" }, "description": { "en": "Set remove preservation normalized files to renormalize link.", @@ -8592,12 +7784,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "-m 770 \"%SIPDirectory%thumbnails/\"", - "execute": "createDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createDirectory_v0.0" }, "description": { "en": "Create thumbnails directory", @@ -8628,12 +7815,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--sipUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --use \"preservation\" --filterSubdir \"objects/manualNormalization/preservation\"", - "execute": "assignFileUUIDs_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "assignFileUUIDs_v0.0" }, "description": { "en": "Assign UUIDs to manual normalized preservation files", @@ -8664,12 +7846,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", - "execute": "convertDataverseStructure_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "convertDataverseStructure_v0.0" }, "description": { "en": "Convert Dataverse structure", @@ -8728,11 +7905,7 @@ "@model": "StandardTaskConfig", "arguments": "--sipUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --filePath \"%relativeLocation%\"", "execute": "manualNormalizationMoveAccessFilesToDIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/manualNormalization/access", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/manualNormalization/access" }, "description": { "en": "Move access files to DIP", @@ -8762,12 +7935,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%watchedDirectories/workFlowDecisions/selectFormatIDToolIngest/.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to select file ID tool", @@ -8777,8 +7945,7 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", @@ -8810,7 +7977,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Bind PIDs", "fr": "Lier les PIDs", @@ -8825,11 +7991,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%SIPUUID%\" \"%date%\" \"%sharedPath%\" \"%unitType%\"", "execute": "changeSIPName_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": "%SIPLogsDirectory%SIPnameCleanup.log", - "stdout_file": null + "stderr_file": "%SIPLogsDirectory%SIPnameCleanup.log" }, "description": { "en": "Change transfer name", @@ -8860,11 +8022,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%SIPUUID%\" \"%date%\" \"%sharedPath%\" \"%unitType%\"", "execute": "changeSIPName_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": "%SIPLogsDirectory%SIPnameCleanup.log", - "stdout_file": null + "stderr_file": "%SIPLogsDirectory%SIPnameCleanup.log" }, "description": { "en": "Change SIP name", @@ -8895,11 +8053,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%SIPUUID%\"", "execute": "validateFile_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects" }, "description": { "en": "Validate formats", @@ -8932,11 +8086,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%SIPUUID%\" \"%sharedPath%\" \"access\"", "execute": "validateFile_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "DIP/objects/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "DIP/objects/" }, "description": { "en": "Validate access derivatives", @@ -8975,7 +8125,6 @@ "config": { "@manager": "linkTaskManagerUnitVariableLinkPull", "@model": "TaskConfigUnitVariableLinkPull", - "chain_id": null, "variable": "linkAfterAutoRestructureForCompliance" }, "description": { @@ -9000,12 +8149,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\"", - "execute": "restructureForCompliance_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "restructureForCompliance_v0.0" }, "description": { "en": "Attempt restructure for compliance", @@ -9038,8 +8182,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%IDCommand%\" \"%relativeLocation%\" \"%fileUUID%\" --disable-reidentify", "execute": "identifyFileFormat_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects", "stderr_file": "%SIPLogsDirectory%fileFormatIdentification.log", "stdout_file": "%SIPLogsDirectory%fileFormatIdentification.log" @@ -9090,7 +8232,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Approve SIP creation", "es": "Aprobar creación de SIP", @@ -9106,12 +8247,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%\" \"%jobCreatedDate%\"", - "execute": "moveToBacklog_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveToBacklog_v1.0" }, "description": { "en": "Move transfer to backlog", @@ -9123,8 +8259,7 @@ "end": true, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", @@ -9159,7 +8294,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Examine contents", "es": "Examinar contenidos", @@ -9175,12 +8309,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%METS.%SIPUUID%.xml\" \"%SIPDirectory%DIP/METS.%SIPUUID%.xml\"", - "execute": "copy_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copy_v0.0" }, "description": { "en": "Copy METS to DIP directory", @@ -9212,12 +8341,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"reject\" \"%SIPUUID%\" \"%SIPDirectory%\"", - "execute": "failedTransferCleanup", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "failedTransferCleanup" }, "description": { "en": "Cleanup rejected transfer", @@ -9262,8 +8386,7 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", @@ -9283,12 +8406,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\" --include-dirs \"%AssignUUIDsToDirectories%\"", - "execute": "assignUUIDsToDirectories_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "assignUUIDsToDirectories_v0.0" }, "description": { "en": "Assign UUIDs to directories", @@ -9317,12 +8435,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%metadata/metadata.json\"", - "execute": "jsonMetadataToCSV_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "jsonMetadataToCSV_v0.0" }, "description": { "en": "Process JSON metadata", @@ -9351,7 +8464,6 @@ "config": { "@manager": "linkTaskManagerUnitVariableLinkPull", "@model": "TaskConfigUnitVariableLinkPull", - "chain_id": null, "variable": "reNormalize" }, "description": { @@ -9364,12 +8476,10 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -9384,12 +8494,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to processing directory", @@ -9420,8 +8525,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%objects/metadata/\" \"%SIPUUID%\" \"%date%\" \"%taskUUID%\" \"SIPDirectory\" \"sip_id\" \"%SIPDirectory%\"", "execute": "changeObjectNames_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects/metadata", "stderr_file": "%SIPLogsDirectory%filenameChanges.log", "stdout_file": "%SIPLogsDirectory%filenameChanges.log" @@ -9454,11 +8557,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%IDCommand%\" \"%relativeLocation%\" \"%fileUUID%\"", "execute": "identifyFileFormat_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/metadata/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/metadata/" }, "description": { "en": "Identify file format of metadata files", @@ -9490,11 +8589,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\" \"%date%\"", "execute": "extractMaildirAttachments_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects" }, "description": { "en": "Extract attachments", @@ -9526,12 +8621,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"fail\" \"%SIPUUID%\"", - "execute": "failedSIPCleanup_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "failedSIPCleanup_v1.0" }, "description": { "en": "Cleanup failed SIP", @@ -9564,13 +8654,7 @@ "config": { "@manager": "linkTaskManagerGetUserChoiceFromMicroserviceGeneratedList", "@model": "StandardTaskConfig", - "arguments": null, - "execute": "%AIPsStore%", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "%AIPsStore%" }, "description": { "en": "Store AIP location", @@ -9587,7 +8671,6 @@ } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Store AIP", "fr": "Stocker l’AIP", @@ -9602,8 +8685,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "180ae3d0-aa6c-4ed4-ab94-d0a2121e7f21", - "variable": "normalizationThumbnailProcessing", - "variable_value": null + "variable": "normalizationThumbnailProcessing" }, "description": { "en": "Set normalize path", @@ -9617,7 +8699,6 @@ } }, "fallback_job_status": "Completed successfully", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -9632,12 +8713,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", - "execute": "isMaildirAIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "isMaildirAIP_v0.0" }, "description": { "en": "Check if SIP is from Maildir Transfer", @@ -9672,12 +8748,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to processing directory", @@ -9706,12 +8777,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%\" --sipUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --filterSubdir \"objects/metadata\"", - "execute": "updateSizeAndChecksum_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "updateSizeAndChecksum_v0.0" }, "description": { "en": "Assign checksums and file sizes to metadata", @@ -9743,12 +8809,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\"", - "execute": "postStoreAIPHook_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "postStoreAIPHook_v1.0" }, "description": { "en": "Clean up after storing AIP", @@ -9780,12 +8841,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\"", - "execute": "hasPackages_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "hasPackages_v0.0" }, "description": { "en": "Determine if transfer contains packages", @@ -9806,7 +8862,6 @@ } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Extract packages", "es": "Extraer paquetes", @@ -9837,7 +8892,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Approve transfer", "es": "Aprobar transferencia", @@ -9869,7 +8923,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Create SIP from Transfer", "es": "Crear SIP a partir de la transferencia", @@ -9885,12 +8938,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%watchedDirectories/system/autoRestructureForCompliance/.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Failed compliance.", @@ -9901,12 +8949,10 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Failed transfer compliance", "fr": "Échec de la mise en conformité du transfert", @@ -9921,11 +8967,7 @@ "@model": "StandardTaskConfig", "arguments": "access \"%fileUUID%\" \"%relativeLocation%\" \"%SIPDirectory%\" \"%SIPUUID%\" \"%taskUUID%\" \"original\"", "execute": "normalize_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/" }, "description": { "en": "Normalize for access", @@ -9966,11 +9008,7 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%SIPLogsDirectory%fileMeta/%fileUUID%.xml\" \"%date%\" \"%taskUUID%\" \"%fileUUID%\" \"%fileGrpUse%\"", "execute": "FITS_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/attachments", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/attachments" }, "description": { "en": "Characterize and extract metadata for attachments", @@ -10004,12 +9042,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\"", - "execute": "hasPackages_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "hasPackages_v0.0" }, "description": { "en": "Determine if transfer still contains packages", @@ -10110,12 +9143,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "fileToFolder_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "fileToFolder_v1.0" }, "description": { "en": "Check if file or folder", @@ -10148,8 +9176,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\"", "execute": "removeUnneededFiles_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects", "stderr_file": "%SIPLogsDirectory%removeUnneededFiles.log", "stdout_file": "%SIPLogsDirectory%removeUnneededFiles.log" @@ -10182,7 +9208,6 @@ "config": { "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", - "chain_id": null, "variable": "identifyFileFormat_v0.0", "variable_value": "{'filterSubDir':'objects/attachments'}" }, @@ -10214,12 +9239,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to processing directory", @@ -10249,12 +9269,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%watchedDirectories/approveNormalization/.\" \"%SIPUUID%\" \"%sharedPath%\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to approve normalization directory", @@ -10264,12 +9279,10 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -10296,8 +9309,7 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", @@ -10316,12 +9328,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPObjectsDirectory%\" \"%SIPUUID%\" \"%date%\" \"%taskUUID%\" \"transferDirectory\" \"transfer_id\" \"%SIPDirectory%\"", - "execute": "changeObjectNames_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "changeObjectNames_v0.0" }, "description": { "en": "Change extracted objects' file and directory names", @@ -10352,8 +9359,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "f6fdd1a7-f0c5-4631-b5d3-19421155bd7a", - "variable": "normalizationThumbnailProcessing", - "variable_value": null + "variable": "normalizationThumbnailProcessing" }, "description": { "en": "Set normalize path", @@ -10367,7 +9373,6 @@ } }, "fallback_job_status": "Completed successfully", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -10382,12 +9387,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%\" --transferUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\"", - "execute": "updateSizeAndChecksum_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "updateSizeAndChecksum_v0.0" }, "description": { "en": "Assign checksums and file sizes to objects", @@ -10422,8 +9422,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%SIPObjectsDirectory%attachments/\" \"%SIPUUID%\" \"%date%\" \"%taskUUID%\" \"transferDirectory\" \"transfer_id\" \"%SIPDirectory%\"", "execute": "changeObjectNames_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects/attachments", "stderr_file": "%SIPLogsDirectory%filenameChanges.log", "stdout_file": "%SIPLogsDirectory%filenameChanges.log" @@ -10456,12 +9454,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPName%\" \"%SIPDirectory%\" \"%date%\"", - "execute": "trimVerifyManifest_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "trimVerifyManifest_v0.0" }, "description": { "en": "Verify TRIM manifest", @@ -10509,7 +9502,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -10524,12 +9516,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%watchedDirectories/workFlowDecisions/extractPackagesChoice/.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to extract packages", @@ -10540,8 +9527,7 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", @@ -10561,12 +9547,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--amdSec --baseDirectoryPath \"%SIPDirectory%\" --sipUUID \"%SIPUUID%\" --xmlFile \"%SIPDirectory%METS.%SIPUUID%.xml\" --sipType \"%SIPType%\"", - "execute": "createMETS_v2.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createMETS_v2.0" }, "description": { "en": "Generate METS.xml document", @@ -10598,13 +9579,7 @@ "config": { "@manager": "linkTaskManagerGetUserChoiceFromMicroserviceGeneratedList", "@model": "StandardTaskConfig", - "arguments": null, - "execute": "%DIPsStore%", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "%DIPsStore%" }, "description": { "en": "Store DIP location", @@ -10637,12 +9612,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to processing directory", @@ -10672,12 +9642,7 @@ "@manager": "linkTaskManagerGetMicroserviceGeneratedListInStdOut", "@model": "StandardTaskConfig", "arguments": "DS", - "execute": "getAipStorageLocations_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "getAipStorageLocations_v0.0" }, "description": { "en": "Retrieve DIP Storage Locations", @@ -10709,12 +9674,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%clientAssetsDirectory%mets/mets.xsd\"", - "execute": "archivematicaVerifyMets_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "archivematicaVerifyMets_v0.0" }, "description": { "en": "Verify mets_structmap.xml compliance", @@ -10755,7 +9715,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Generate AIP METS", "fr": "Générer un AIP METS", @@ -10772,10 +9731,7 @@ "arguments": "\"%relativeLocation%\" \"%SIPDirectory%\" \"%SIPUUID%\"", "execute": "identifyDspaceFiles_v0.0", "filter_file_end": "mets.xml", - "filter_file_start": null, - "filter_subdir": "objects", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects" }, "description": { "en": "Identify DSpace text files", @@ -10805,12 +9761,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%watchedDirectories/workFlowDecisions/selectFormatIDToolTransfer/.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to select file ID tool", @@ -10820,8 +9771,7 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", @@ -10840,12 +9790,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%\" --transferUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\"", - "execute": "updateSizeAndChecksum_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "updateSizeAndChecksum_v0.0" }, "description": { "en": "Assign checksums and file sizes to objects", @@ -10876,12 +9821,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%watchedDirectories/SIPCreation/completedTransfers/.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to SIP creation directory for completed transfers", @@ -10891,12 +9831,10 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Complete transfer", "es": "Completar transferencia", @@ -10912,12 +9850,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%sharedMicroServiceTasksConfigs/processingMCPConfigs/defaultProcessingMCP.xml\" \"%SIPDirectory%processingMCP.xml\" -n", - "execute": "copy_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copy_v0.0" }, "description": { "en": "Include default SIP processingMCP.xml", @@ -10946,12 +9879,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to processing directory", @@ -10980,12 +9908,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%clientAssetsDirectory%premis/premis.xsd\" \"%SIPDirectory%metadata/premis.xml\"", - "execute": "loadPREMISEventsFromXML_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "loadPREMISEventsFromXML_v1.0" }, "description": { "en": "Load PREMIS events from metadata/premis.xml" @@ -11013,12 +9936,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%AIPCompressionAlgorithm%\" \"%AIPCompressionLevel%\" \"%SIPDirectory%\" \"%SIPName%\" \"%SIPUUID%\"", - "execute": "compressAIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "compressAIP_v0.0" }, "description": { "en": "Compress AIP", @@ -11050,12 +9968,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", - "execute": "removeDirectories_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "removeDirectories_v0.0" }, "description": { "en": "Remove the processing directory", @@ -11066,12 +9979,10 @@ "end": true, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Store AIP", "fr": "Stocker l’AIP", @@ -11143,8 +10054,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "092b47db-6f77-4072-aed3-eb248ab69e9c", - "variable": "normalizationThumbnailProcessing", - "variable_value": null + "variable": "normalizationThumbnailProcessing" }, "description": { "en": "Set normalize path", @@ -11158,7 +10068,6 @@ } }, "fallback_job_status": "Completed successfully", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -11173,12 +10082,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"Maildir\"", - "execute": "archivematicaSetTransferType_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "archivematicaSetTransferType_v0.0" }, "description": { "en": "Set transfer type: Maildir", @@ -11208,12 +10112,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%watchedDirectories/workFlowDecisions/examineContentsChoice/.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to examine contents", @@ -11223,8 +10122,7 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", @@ -11245,8 +10143,6 @@ "@model": "StandardTaskConfig", "arguments": "--fileUUID \"%fileUUID%\" --inputFile \"%relativeLocation%\" --sipDirectory \"%SIPDirectory%\"", "execute": "removeFilesWithoutPresmisMetadata_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects/", "stderr_file": "%SIPLogsDirectory%removedFilesWithNoPremisMetadata.log", "stdout_file": "%SIPLogsDirectory%removedFilesWithNoPremisMetadata.log" @@ -11279,12 +10175,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--sipUUID \"%SIPUUID%\" --xmlFile \"%SIPDirectory%\"metadata/transfer_metadata.xml", - "execute": "createTransferMetadata_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createTransferMetadata_v0.0" }, "description": { "en": "Create transfer metadata XML", @@ -11301,7 +10192,6 @@ } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Complete transfer", "es": "Completar transferencia", @@ -11318,8 +10208,6 @@ "@model": "StandardTaskConfig", "arguments": "--fileUUID \"%fileUUID%\" --inputFile \"%relativeLocation%\" --sipDirectory \"%SIPDirectory%\"", "execute": "removeFilesWithoutPresmisMetadata_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects/", "stderr_file": "%SIPLogsDirectory%removedFilesWithNoPremisMetadata.log", "stdout_file": "%SIPLogsDirectory%removedFilesWithNoPremisMetadata.log" @@ -11353,9 +10241,6 @@ "@model": "StandardTaskConfig", "arguments": "--transferUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --filterSubdir \"objects\"", "execute": "assignFileUUIDs_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, "stderr_file": "%SIPLogsDirectory%FileUUIDsError.log", "stdout_file": "%SIPLogsDirectory%FileUUIDs.log" }, @@ -11392,9 +10277,6 @@ "@model": "StandardTaskConfig", "arguments": "--sipUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --use \"metadata\" --filterSubdir \"objects/metadata\" --disable-update-filegrpuse", "execute": "assignFileUUIDs_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, "stderr_file": "%SIPLogsDirectory%FileUUIDsError.log", "stdout_file": "%SIPLogsDirectory%FileUUIDs.log" }, @@ -11444,7 +10326,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -11474,7 +10355,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Extract packages", "es": "Extraer paquetes", @@ -11490,12 +10370,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%sharedMicroServiceTasksConfigs/processingMCPConfigs/defaultProcessingMCP.xml\" \"%SIPDirectory%processingMCP.xml\" -n", - "execute": "copy_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copy_v0.0" }, "description": { "en": "Include default SIP processingMCP.xml", @@ -11581,8 +10456,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\"", "execute": "removeUnneededFiles_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects", "stderr_file": "%SIPLogsDirectory%removeUnneededFiles.log", "stdout_file": "%SIPLogsDirectory%removeUnneededFiles.log" @@ -11616,8 +10489,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "7509e7dc-1e1b-4dce-8d21-e130515fce73", - "variable": "resumeAfterNormalizationFileIdentificationToolSelected", - "variable_value": null + "variable": "resumeAfterNormalizationFileIdentificationToolSelected" }, "description": { "en": "Grant normalization options for pre-existing DIP", @@ -11647,12 +10519,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%sharedMicroServiceTasksConfigs/processingMCPConfigs/%processingConfiguration%ProcessingMCP.xml\" \"%SIPDirectory%processingMCP.xml\" -n", - "execute": "copy_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copy_v0.0" }, "description": { "en": "Include default Transfer processingMCP.xml", @@ -11680,12 +10547,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%%SIPName%-%SIPUUID%\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Rename SIP directory with SIP UUID", @@ -11715,12 +10577,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%metadata\" \"%SIPDirectory%objects/metadata\"", - "execute": "moveOrMerge_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveOrMerge_v0.0" }, "description": { "en": "Move metadata to objects directory", @@ -11750,12 +10607,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", - "execute": "isMaildirAIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "isMaildirAIP_v0.0" }, "description": { "en": "Is maildir AIP", @@ -11790,12 +10642,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--transferUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\"", - "execute": "assignFileUUIDs_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "assignFileUUIDs_v0.0" }, "description": { "en": "Assign file UUIDs to objects", @@ -11826,12 +10673,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%relativeLocation%metadata/dc.json\"", - "execute": "saveDublinCore_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "saveDublinCore_v0.0" }, "description": { "en": "Serialize Dublin Core metadata to disk", @@ -11862,12 +10704,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%\" --sipUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --filterSubdir \"objects/manualNormalization/preservation\"", - "execute": "updateSizeAndChecksum_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "updateSizeAndChecksum_v0.0" }, "description": { "en": "Assign checksums to manual normalized preservation files", @@ -11899,12 +10736,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"fail\" \"%SIPUUID%\" \"%SIPDirectory%\"", - "execute": "failedTransferCleanup", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "failedTransferCleanup" }, "description": { "en": "Cleanup failed Transfer", @@ -11939,11 +10771,7 @@ "@model": "StandardTaskConfig", "arguments": "thumbnail \"%fileUUID%\" \"%relativeLocation%\" \"%SIPDirectory%\" \"%SIPUUID%\" \"%taskUUID%\" \"service\" --thumbnail_mode \"%ThumbnailMode%\"", "execute": "normalize_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/service", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/service" }, "description": { "en": "Normalize service files for thumbnails", @@ -11982,12 +10810,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\"", - "execute": "restructureForCompliance_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "restructureForCompliance_v0.0" }, "description": { "en": "Attempt restructure for compliance", @@ -12043,12 +10866,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--sipDirectory \"%SIPDirectory%\" --sipUUID \"%SIPUUID%\" --sharedPath \"%sharedPath%\"", - "execute": "copyTransfersMetadataAndLogs_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copyTransfersMetadataAndLogs_v0.0" }, "description": { "en": "Copy transfers metadata and logs", @@ -12093,7 +10911,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Add final metadata", "es": "Añadir metadatos finales", @@ -12109,12 +10926,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%%SIPName%-%SIPUUID%\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Rename with transfer UUID", @@ -12145,11 +10957,7 @@ "@model": "StandardTaskConfig", "arguments": "\"access\" \"%fileUUID%\" \"%relativeLocation%\" \"%SIPDirectory%\" \"%SIPUUID%\" \"%taskUUID%\" \"original\"", "execute": "normalize_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/" }, "description": { "en": "Normalize for access", @@ -12189,12 +10997,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Move to processing directory", @@ -12223,12 +11026,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%watchedDirectories/SIPCreation/SIPsUnderConstruction/.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveSIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveSIP_v0.0" }, "description": { "en": "Failed compliance. See output in dashboard. SIP moved back to SIPsUnderConstruction", @@ -12239,12 +11037,10 @@ "end": true, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Failed compliance", "fr": "Échec de la mise en conformité", @@ -12314,12 +11110,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\"", - "execute": "verifyAndRestructureTransferBag_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "verifyAndRestructureTransferBag_v0.0" }, "description": { "en": "Verify bag, and restructure for compliance", @@ -12350,12 +11141,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--uuid=\"%SIPUUID%\" --dipDir \"%SIPDirectory%\"", - "execute": "restructureDIPForContentDMUpload_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "restructureDIPForContentDMUpload_v0.0" }, "description": { "en": "Restructure DIP for CONTENTdm upload", @@ -12432,7 +11218,6 @@ } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Extract packages", "es": "Extraer paquetes", @@ -12448,12 +11233,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%SIPUUID%\"", - "execute": "verifychecksum_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "verifychecksum_v0.0" }, "description": { "en": "Verify metadata directory checksums", @@ -12483,12 +11263,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "-d \"%SIPDirectory%DIP\"", - "execute": "test_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "test_v0.0" }, "description": { "en": "Check if DIP should be generated", @@ -12523,8 +11298,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "2fd123ea-196f-4c9c-95c0-117aa65ed9c6", - "variable": "postExtractSpecializedProcessing", - "variable_value": null + "variable": "postExtractSpecializedProcessing" }, "description": { "en": "Set specialized processing link", @@ -12553,12 +11327,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"reject\" \"%SIPUUID%\"", - "execute": "failedSIPCleanup_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "failedSIPCleanup_v1.0" }, "description": { "en": "Cleanup rejected SIP", @@ -12592,12 +11361,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%relativeLocation%metadata/dc.json\"", - "execute": "saveDublinCore_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "saveDublinCore_v0.0" }, "description": { "en": "Serialize Dublin Core metadata to disk", @@ -12628,12 +11392,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "-m 770 \"%SIPDirectory%thumbnails/\"", - "execute": "createDirectory_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "createDirectory_v0.0" }, "description": { "en": "Create thumbnails directory", @@ -12666,8 +11425,6 @@ "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\"", "execute": "removeUnneededFiles_v0.0", - "filter_file_end": null, - "filter_file_start": null, "filter_subdir": "objects", "stderr_file": "%SIPLogsDirectory%removeUnneededFiles.log", "stdout_file": "%SIPLogsDirectory%removeUnneededFiles.log" @@ -12698,12 +11455,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%metadata/submissionDocumentation\" \"%sharedPath%\"", - "execute": "copyTransferSubmissionDocumentation_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copyTransferSubmissionDocumentation_v0.0" }, "description": { "en": "Copy transfer submission documentation", @@ -12750,7 +11502,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Approve transfer", "es": "Aprobar transferencia", @@ -12767,11 +11518,7 @@ "@model": "StandardTaskConfig", "arguments": "thumbnail \"%fileUUID%\" \"%relativeLocation%\" \"%SIPDirectory%\" \"%SIPUUID%\" \"%taskUUID%\" \"original\" --thumbnail_mode \"%ThumbnailMode%\"", "execute": "normalize_v1.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": "objects/", - "stderr_file": null, - "stdout_file": null + "filter_subdir": "objects/" }, "description": { "en": "Normalize for thumbnails", @@ -12810,12 +11557,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to processing directory", @@ -12843,12 +11585,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%sharedMicroServiceTasksConfigs/processingMCPConfigs/%processingConfiguration%ProcessingMCP.xml\" \"%SIPDirectory%processingMCP.xml\" -n", - "execute": "copy_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copy_v0.0" }, "description": { "en": "Include default Transfer processingMCP.xml", @@ -12876,12 +11613,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%rejectedDirectory%\" \"%watchDirectoryPath%uploadedDIPs/\"", - "execute": "handleUnstoredDIP_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "handleUnstoredDIP_v0.0" }, "description": { "en": "Handle unstored DIP", @@ -12892,12 +11624,10 @@ "end": true, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Upload DIP", "es": "Subir DIP", @@ -12913,12 +11643,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%sharedPath%\"", - "execute": "storeFileModificationDates_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "storeFileModificationDates_v0.0" }, "description": { "en": "Store file modification dates", @@ -12950,12 +11675,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\" --include-dirs \"%AssignUUIDsToDirectories%\"", - "execute": "assignUUIDsToDirectories_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "assignUUIDsToDirectories_v0.0" }, "description": { "en": "Assign UUIDs to directories", @@ -12984,12 +11704,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPUUID%\"", - "execute": "parseDataverse_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "parseDataverse_v0.0" }, "description": { "en": "Parse Dataverse METS XML", @@ -13014,12 +11729,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", - "execute": "moveTransfer_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "moveTransfer_v0.0" }, "description": { "en": "Move to processing directory", @@ -13048,12 +11758,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", - "execute": "restructureForComplianceMaildir_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "restructureForComplianceMaildir_v0.0" }, "description": { "en": "Attempt restructure for compliance", @@ -13085,12 +11790,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "\"%sharedPath%sharedMicroServiceTasksConfigs/processingMCPConfigs/defaultProcessingMCP.xml\" \"%SIPDirectory%processingMCP.xml\" -n", - "execute": "copy_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "copy_v0.0" }, "description": { "en": "Include default SIP processingMCP.xml", @@ -13120,12 +11820,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "arguments": "--base-url \"%base_url%\" --user \"%user%\" --passwd \"%passwd%\" --dip_location \"%SIPDirectory%\" --dip_name \"%SIPName%\" --dip_uuid \"%SIPUUID%\" --restrictions \"%restrictions%\" --object_type \"%object_type%\" --xlink_actuate \"%xlink_actuate%\" --xlink_show \"%xlink_show%\" --use_statement \"%use_statement%\" --uri_prefix \"%uri_prefix%\" --access_conditions \"%access_conditions%\" --use_conditions \"%use_conditions%\" --inherit_notes \"%inherit_notes%\"", - "execute": "upload-archivesspace_v0.0", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null + "execute": "upload-archivesspace_v0.0" }, "description": { "en": "Upload to ArchivesSpace", diff --git a/src/MCPServer/lib/server/jobs/decisions.py b/src/MCPServer/lib/server/jobs/decisions.py index da9bdc5d7a..2f6767fa95 100644 --- a/src/MCPServer/lib/server/jobs/decisions.py +++ b/src/MCPServer/lib/server/jobs/decisions.py @@ -133,7 +133,7 @@ def get_preconfigured_choice(self): self.package.current_path, self.link.id ) if desired_choice and self.job_chain.generated_choices: - for key, data in self.job_chain.generated_choices.items(): + for _, data in self.job_chain.generated_choices.items(): if data["uri"] == desired_choice: return data["uri"] diff --git a/src/MCPServer/lib/server/jobs/local.py b/src/MCPServer/lib/server/jobs/local.py index 99c47a8719..84200ccb91 100644 --- a/src/MCPServer/lib/server/jobs/local.py +++ b/src/MCPServer/lib/server/jobs/local.py @@ -41,7 +41,7 @@ def run(self, *args, **kwargs): variable=self.link.config["variable"], ) except (models.UnitVariable.DoesNotExist, ValidationError): - link_id = self.link.config["chain_id"] + link_id = self.link.config.get("chain_id") else: link_id = unitvar.microservicechainlink @@ -69,8 +69,8 @@ def run(self, *args, **kwargs): self.package.set_variable( self.link.config["variable"], - self.link.config["variable_value"], - self.link.config["chain_id"], + self.link.config.get("variable_value"), + self.link.config.get("chain_id"), ) self.mark_complete() diff --git a/src/MCPServer/lib/server/packages.py b/src/MCPServer/lib/server/packages.py index 13224cd4d0..cad8587b7c 100644 --- a/src/MCPServer/lib/server/packages.py +++ b/src/MCPServer/lib/server/packages.py @@ -681,7 +681,7 @@ def files( files_returned_already.add(file_obj_mapped.get("%inputFile%")) yield file_obj_mapped - for basedir, subdirs, files in os.walk(start_path): + for basedir, _, files in os.walk(start_path): for file_name in files: if ( filter_filename_start @@ -712,7 +712,7 @@ def set_variable(self, key, value, chain_link_id): unittype=self.UNIT_VARIABLE_TYPE, unituuid=self.uuid, variable=key, - defaults=dict(variablevalue=value, microservicechainlink=chain_link_id), + defaults={"variablevalue": value, "microservicechainlink": chain_link_id}, ) if created: message = "New UnitVariable %s created for %s: %s (MSCL: %s)" diff --git a/src/MCPServer/lib/server/rpc_server.py b/src/MCPServer/lib/server/rpc_server.py index f9e6fb9bca..5c5cde9204 100644 --- a/src/MCPServer/lib/server/rpc_server.py +++ b/src/MCPServer/lib/server/rpc_server.py @@ -218,7 +218,7 @@ def _job_awaiting_approval_handler(self, worker, job): raise_exc = True """ ret = etree.Element("choicesAvailableForUnits") - for uuid, choice in self.package_queue.jobs_awaiting_decisions().items(): + for choice in self.package_queue.jobs_awaiting_decisions().values(): unit_choices = etree.SubElement(ret, "choicesAvailableForUnit") etree.SubElement(unit_choices, "UUID").text = str(choice.uuid) unit = etree.SubElement(unit_choices, "unit") diff --git a/src/MCPServer/lib/server/watch_dirs.py b/src/MCPServer/lib/server/watch_dirs.py index bb6daa75ff..9ff80586b0 100644 --- a/src/MCPServer/lib/server/watch_dirs.py +++ b/src/MCPServer/lib/server/watch_dirs.py @@ -74,6 +74,7 @@ def watch_directories_inotify( warnings.warn( "inotify may not work as a watched directory method on non-linux systems.", RuntimeWarning, + stacklevel=2, ) inotify = INotify() diff --git a/src/MCPServer/lib/server/workflow.py b/src/MCPServer/lib/server/workflow.py index 40dffa24df..1ec5d34eab 100644 --- a/src/MCPServer/lib/server/workflow.py +++ b/src/MCPServer/lib/server/workflow.py @@ -177,11 +177,15 @@ def is_terminal(self): return self._src.get("end", False) def get_next_link(self, code): + """Return the next link based on the exit code. + + Raises KeyError which should be handled by the caller. + """ code = str(code) try: - link_id = self._src["exit_codes"][code]["link_id"] + link_id = self._src["exit_codes"][code].get("link_id") except KeyError: - link_id = self._src["fallback_link_id"] + link_id = self._src.get("fallback_link_id") return self._workflow.get_link(link_id) def get_status_id(self, code): diff --git a/src/MCPServer/tests/fixtures/workflow-integration-test.json b/src/MCPServer/tests/fixtures/workflow-integration-test.json index eb6689043d..83a5768520 100644 --- a/src/MCPServer/tests/fixtures/workflow-integration-test.json +++ b/src/MCPServer/tests/fixtures/workflow-integration-test.json @@ -25,12 +25,7 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "execute": "link1_command", - "arguments": "\"%processingDirectory%\" \"%SIPUUID%\"", - "filter_file_start": null, - "filter_file_end": null, - "filter_subdir": null, - "stdout_file": null, - "stderr_file": null + "arguments": "\"%processingDirectory%\" \"%SIPUUID%\"" }, "description": { "en": "Starting link" @@ -42,7 +37,6 @@ } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Group 1" } @@ -52,12 +46,7 @@ "@manager": "linkTaskManagerFiles", "@model": "StandardTaskConfig", "execute": "link2_command", - "arguments": "\"%fileUUID%\"", - "filter_file_start": null, - "filter_file_end": null, - "filter_subdir": null, - "stdout_file": null, - "stderr_file": null + "arguments": "\"%fileUUID%\"" }, "description": { "en": "Second link" @@ -69,7 +58,6 @@ } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Group 1" } @@ -79,12 +67,7 @@ "@manager": "linkTaskManagerGetMicroserviceGeneratedListInStdOut", "@model": "StandardTaskConfig", "execute": "link3_command", - "arguments": "{\"default\": {\"description\": \"Default Location\", \"uri\": \"/api/v2/location/default/\"}}", - "filter_file_start": null, - "filter_file_end": null, - "filter_subdir": null, - "stdout_file": null, - "stderr_file": null + "arguments": "{\"default\": {\"description\": \"Default Location\", \"uri\": \"/api/v2/location/default/\"}}" }, "description": { "en": "Third link" @@ -96,7 +79,6 @@ } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Group 1" } @@ -105,13 +87,7 @@ "config": { "@manager": "linkTaskManagerGetUserChoiceFromMicroserviceGeneratedList", "@model": "StandardTaskConfig", - "execute": "%AIPsStore%", - "arguments": "", - "filter_file_start": null, - "filter_file_end": null, - "filter_subdir": null, - "stdout_file": null, - "stderr_file": null + "execute": "%AIPsStore%" }, "description": { "en": "Fourth link" @@ -123,7 +99,6 @@ } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Group 1" } @@ -142,7 +117,6 @@ }, "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Group 1" } @@ -182,7 +156,6 @@ } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Group 1" } @@ -192,8 +165,7 @@ "@manager": "linkTaskManagerSetUnitVariable", "@model": "TaskConfigSetUnitVariable", "chain_id": "f8e4c1ee-3e43-4caa-a664-f6b6bd8f156e", - "variable": "test_unit_variable", - "variable_value": null + "variable": "test_unit_variable" }, "description": { "en": "Chain 2 second link" @@ -205,7 +177,6 @@ } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Group 1" } @@ -214,7 +185,6 @@ "config": { "@manager": "linkTaskManagerUnitVariableLinkPull", "@model": "TaskConfigUnitVariableLinkPull", - "chain_id": null, "variable": "test_unit_variable" }, "description": { @@ -222,12 +192,10 @@ }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Group 1" } @@ -237,28 +205,21 @@ "@manager": "linkTaskManagerDirectories", "@model": "StandardTaskConfig", "execute": "final_command", - "arguments": "\"%processingDirectory%\" \"%SIPDirectory%\" \"%SIPName%-%SIPUUID%\"", - "filter_file_start": null, - "filter_file_end": null, - "filter_subdir": null, - "stdout_file": null, - "stderr_file": null + "arguments": "\"%processingDirectory%\" \"%SIPDirectory%\" \"%SIPName%-%SIPUUID%\"" }, "description": { "en": "Last link" }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Group 1" } } }, "watched_directories": [] -} \ No newline at end of file +} diff --git a/src/MCPServer/tests/fixtures/workflow-sample.json b/src/MCPServer/tests/fixtures/workflow-sample.json index f35ab0d8f5..034c639ea0 100644 --- a/src/MCPServer/tests/fixtures/workflow-sample.json +++ b/src/MCPServer/tests/fixtures/workflow-sample.json @@ -13,24 +13,17 @@ "@manager": "linkTaskManagerFiles", "@model": "StandardTaskConfig", "execute": "command", - "arguments": "--debug", - "filter_file_start": null, - "filter_file_end": null, - "filter_subdir": null, - "stdout_file": null, - "stderr_file": null + "arguments": "--debug" }, "description": { "en": "Go!" }, "exit_codes": { "0": { - "job_status": "Completed successfully", - "link_id": null + "job_status": "Completed successfully" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, "group": { "en": "Group" } diff --git a/src/MCPServer/tests/test_gearman.py b/src/MCPServer/tests/test_gearman.py index 91f2b3ceef..26de4fbb4b 100644 --- a/src/MCPServer/tests/test_gearman.py +++ b/src/MCPServer/tests/test_gearman.py @@ -193,7 +193,7 @@ def test_gearman_multiple_batches( backend = GearmanTaskBackend() job_requests = [] - for i in range(3): + for _ in range(3): mock_gearman_job = mocker.Mock() job_request = gearman.job.GearmanJobRequest( mock_gearman_job, background=True, max_attempts=0 diff --git a/src/MCPServer/tests/test_package.py b/src/MCPServer/tests/test_package.py index b0d7fa4116..068db7b078 100644 --- a/src/MCPServer/tests/test_package.py +++ b/src/MCPServer/tests/test_package.py @@ -185,7 +185,7 @@ def test_reload_file_list(tmp_path): # One file will be returned from the database with a UUID, another from # the filesystem without a UUID. - for file_count, file_info in enumerate(transfer.files(None, None, "/objects"), 1): + for _file_count, file_info in enumerate(transfer.files(None, None, "/objects"), 1): assert "%fileUUID%" in file_info assert "%fileGrpUse%" in file_info assert "%relativeLocation%" in file_info @@ -205,7 +205,7 @@ def test_reload_file_list(tmp_path): } models.File.objects.create(**kwargs) assert ( - file_count == 2 + _file_count == 2 ), "Database and file objects were not returned by the generator" assert models.File.objects.filter(transfer_id=str(transfer_uuid)).count() == 2 @@ -215,7 +215,7 @@ def test_reload_file_list(tmp_path): sub_dir.mkdir() new_file = sub_dir / "another_new_file.txt" new_file.touch() - for file_count, file_info in enumerate(transfer.files(None, None, "/objects"), 1): + for _file_count, file_info in enumerate(transfer.files(None, None, "/objects"), 1): if file_info.get("%fileUUID%") != "None": continue file_path = Path( @@ -231,20 +231,20 @@ def test_reload_file_list(tmp_path): } models.File.objects.create(**kwargs) assert ( - file_count == 3 + _file_count == 3 ), "Database and file objects were not returned by the generator" assert models.File.objects.filter(transfer_id=str(transfer_uuid)).count() == 3 # Now the database is updated, we will still have the same file count, but # all objects will be returned from the database and we will have uuids. - for file_count, file_info in enumerate(transfer.files(None, None, "/objects"), 1): + for _file_count, file_info in enumerate(transfer.files(None, None, "/objects"), 1): if file_info.get("%fileUUID%") == "None": - assert ( - False - ), "Non-database entries returned from package.files(): {}".format( - file_info + raise AssertionError( + "Non-database entries returned from package.files(): {}".format( + file_info + ) ) - assert file_count == 3 + assert _file_count == 3 # Finally, let's just see if the scan works for a slightly larger no. # files, i.e. a number with an increment slightly larger than one. @@ -253,11 +253,11 @@ def test_reload_file_list(tmp_path): new_file = objects_path / file_ new_file.touch() new_count = 0 - for file_count, file_info in enumerate(transfer.files(None, None, "/objects"), 1): + for _file_count, file_info in enumerate(transfer.files(None, None, "/objects"), 1): if file_info.get("%fileUUID%") == "None": new_count += 1 assert new_count == 5 - assert file_count == 8 + assert _file_count == 8 # Clean up state and ensure test doesn't interfere with other transfers # expected to be in the database, e.g. in test_queues.py. diff --git a/src/MCPServer/tests/test_queues.py b/src/MCPServer/tests/test_queues.py index 6af32c3d27..debce6b4b5 100644 --- a/src/MCPServer/tests/test_queues.py +++ b/src/MCPServer/tests/test_queues.py @@ -93,11 +93,8 @@ def workflow_link(request): "execute": "testLink_v0", }, "description": {"en": "A Test link"}, - "exit_codes": { - "0": {"job_status": "Completed successfully", "link_id": None} - }, + "exit_codes": {"0": {"job_status": "Completed successfully"}}, "fallback_job_status": "Failed", - "fallback_link_id": None, "group": {"en": "Testing"}, }, object(), diff --git a/src/archivematicaCommon/lib/archivematicaFunctions.py b/src/archivematicaCommon/lib/archivematicaFunctions.py index be41e27735..755f61d472 100644 --- a/src/archivematicaCommon/lib/archivematicaFunctions.py +++ b/src/archivematicaCommon/lib/archivematicaFunctions.py @@ -14,9 +14,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaCommon -# @author Joseph Perry """archivematicaFunctions provides various helper functions across the different Archivematica modules. """ @@ -527,4 +524,4 @@ def chunk_iterable(iterable, chunk_size=10, fillvalue=None): [('A', 'B', 'C'), ('D', 'E', 'F'), ('G', 'x', 'x')] """ args = [iter(iterable)] * chunk_size - return zip_longest(fillvalue=fillvalue, *args) + return zip_longest(*args, fillvalue=fillvalue) diff --git a/src/archivematicaCommon/lib/bindpid.py b/src/archivematicaCommon/lib/bindpid.py index ae0ac305c0..01172a5e0c 100755 --- a/src/archivematicaCommon/lib/bindpid.py +++ b/src/archivematicaCommon/lib/bindpid.py @@ -316,12 +316,12 @@ def _render_request_body(argsdict, resolve_url, qualified_resolve_urls): """ return _render_template( argsdict["pid_request_body_template"], - dict( - naming_authority=argsdict["naming_authority"], - pid=argsdict["desired_pid"], - base_resolve_url=resolve_url, - qualified_resolve_urls=qualified_resolve_urls, - ), + { + "naming_authority": argsdict["naming_authority"], + "pid": argsdict["desired_pid"], + "base_resolve_url": resolve_url, + "qualified_resolve_urls": qualified_resolve_urls, + }, ).encode("utf8") diff --git a/src/archivematicaCommon/lib/countryCodes.py b/src/archivematicaCommon/lib/countryCodes.py index 606c08e173..8c2223fcad 100644 --- a/src/archivematicaCommon/lib/countryCodes.py +++ b/src/archivematicaCommon/lib/countryCodes.py @@ -14,9 +14,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaCommon -# @author Joseph Perry # http://www.iso.org/iso/country_codes.htm # Country Codes - ISO 3166 # Country Name;ISO 3166-1-alpha-2 code diff --git a/src/archivematicaCommon/lib/databaseFunctions.py b/src/archivematicaCommon/lib/databaseFunctions.py index 0fe9866544..d5a96e3455 100644 --- a/src/archivematicaCommon/lib/databaseFunctions.py +++ b/src/archivematicaCommon/lib/databaseFunctions.py @@ -14,9 +14,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaCommon -# @author Joseph Perry import logging import random import string diff --git a/src/archivematicaCommon/lib/dicts.py b/src/archivematicaCommon/lib/dicts.py index b1bbe063e8..e5b4122d4b 100644 --- a/src/archivematicaCommon/lib/dicts.py +++ b/src/archivematicaCommon/lib/dicts.py @@ -14,9 +14,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage MCPServer -# @author Joseph Perry import ast import os import re @@ -93,7 +90,7 @@ def frommodel(type_="file", sip=None, file_=None, expand_path=True): # sip can be a SIP or Transfer try: sip = models.SIP.objects.get(uuid=sip) - except: + except Exception: sip = models.Transfer.objects.get(uuid=sip) shared_path = config["shared_directory"] @@ -103,7 +100,7 @@ def frommodel(type_="file", sip=None, file_=None, expand_path=True): if file_ and not sip: try: sip = file_.sip - except: + except Exception: sip = file_.transfer rd = ReplacementDict() @@ -138,7 +135,7 @@ def frommodel(type_="file", sip=None, file_=None, expand_path=True): rd["%fileUUID%"] = str(file_.uuid) try: base_location = file_.sip.currentpath - except: + except Exception: base_location = file_.transfer.currentlocation if expand_path and sipdir is not None: diff --git a/src/archivematicaCommon/lib/elasticSearchFunctions.py b/src/archivematicaCommon/lib/elasticSearchFunctions.py index 49d33da7d8..e67b441098 100644 --- a/src/archivematicaCommon/lib/elasticSearchFunctions.py +++ b/src/archivematicaCommon/lib/elasticSearchFunctions.py @@ -14,9 +14,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaCommon -# @author Mike Cantelon import calendar import copy import datetime @@ -105,7 +102,7 @@ class TooManyResultsError(ElasticsearchError): DEPTH_LIMIT = 1000 -def setup(hosts, timeout=DEFAULT_TIMEOUT, enabled=[AIPS_INDEX, TRANSFERS_INDEX]): +def setup(hosts, timeout=DEFAULT_TIMEOUT, enabled=(AIPS_INDEX, TRANSFERS_INDEX)): """Initialize and share the Elasticsearch client. Share it as the attribute _es_client in the current module. An additional @@ -175,7 +172,7 @@ def _wait_for_cluster_yellow_status(client, wait_between_tries=10, max_tries=10) try: health = client.cluster.health() - except: + except Exception: print("ERROR: failed health check.") health["status"] = None @@ -804,9 +801,8 @@ def _index_transfer_files( # The BagIt is created when the package is sent to backlog hence # the locations in the database do not reflect the BagIt paths. # Strip the "data/" part when looking up the file entry. - currentlocation = "%transferDirectory%" + os.path.relpath( - filepath, path - ).lstrip("data/") + stripped_path = re.sub(r"^data/", "", os.path.relpath(filepath, path)) + currentlocation = "%transferDirectory%" + stripped_path try: f = File.objects.get( currentlocation=currentlocation.encode(), transfer_id=uuid @@ -824,7 +820,7 @@ def _index_transfer_files( bulk_extractor_reports = [] # Get file path info - relative_path = filepath.replace(path, transfer_name + "/") + stripped_path = filepath.replace(path, transfer_name + "/") file_extension = os.path.splitext(filepath)[1][1:].lower() filename = os.path.basename(filepath) # Size in megabytes @@ -832,12 +828,12 @@ def _index_transfer_files( create_time = os.stat(filepath).st_ctime if filename not in ignore_files: - printfn(f"Indexing {relative_path} (UUID: {file_uuid})") + printfn(f"Indexing {stripped_path} (UUID: {file_uuid})") # TODO: Index Backlog Location UUID? indexData = { "filename": filename, - "relative_path": relative_path, + "relative_path": stripped_path, "fileuuid": file_uuid, "sipuuid": uuid, "accessionid": accession_id, @@ -859,7 +855,7 @@ def _index_transfer_files( files_indexed = files_indexed + 1 else: - printfn(f"Skipping indexing {relative_path}") + printfn(f"Skipping indexing {stripped_path}") return files_indexed diff --git a/src/archivematicaCommon/lib/email_settings.py b/src/archivematicaCommon/lib/email_settings.py index 5420d0c96e..2e64f489e1 100644 --- a/src/archivematicaCommon/lib/email_settings.py +++ b/src/archivematicaCommon/lib/email_settings.py @@ -59,27 +59,27 @@ def get_settings(config): This should be invoked from a Django settings module and the result merged into the globals() dict. """ - settings = dict( + settings = { # Which backend to use? - EMAIL_BACKEND=config.get("email_backend"), + "EMAIL_BACKEND": config.get("email_backend"), # File Backend # See https://docs.djangoproject.com/en/dev/topics/email/#file-backend - EMAIL_FILE_PATH=config.get("email_file_path"), + "EMAIL_FILE_PATH": config.get("email_file_path"), # SMTP Backend # See https://docs.djangoproject.com/en/dev/topics/email/#smtp-backend - EMAIL_HOST=config.get("email_host"), - EMAIL_HOST_PASSWORD=config.get("email_host_password"), - EMAIL_HOST_USER=config.get("email_host_user"), - EMAIL_PORT=config.get("email_port"), - EMAIL_SSL_CERTFILE=config.get("email_ssl_certfile"), - EMAIL_SSL_KEYFILE=config.get("email_ssl_keyfile"), - EMAIL_USE_SSL=config.get("email_use_ssl"), - EMAIL_USE_TLS=config.get("email_use_tls"), + "EMAIL_HOST": config.get("email_host"), + "EMAIL_HOST_PASSWORD": config.get("email_host_password"), + "EMAIL_HOST_USER": config.get("email_host_user"), + "EMAIL_PORT": config.get("email_port"), + "EMAIL_SSL_CERTFILE": config.get("email_ssl_certfile"), + "EMAIL_SSL_KEYFILE": config.get("email_ssl_keyfile"), + "EMAIL_USE_SSL": config.get("email_use_ssl"), + "EMAIL_USE_TLS": config.get("email_use_tls"), # General settings, not backend-specific - DEFAULT_FROM_EMAIL=config.get("default_from_email"), - EMAIL_SUBJECT_PREFIX=config.get("email_subject_prefix"), - EMAIL_TIMEOUT=config.get("email_timeout", None), - ) + "DEFAULT_FROM_EMAIL": config.get("default_from_email"), + "EMAIL_SUBJECT_PREFIX": config.get("email_subject_prefix"), + "EMAIL_TIMEOUT": config.get("email_timeout", None), + } settings["SERVER_EMAIL"] = config.get("server_email", settings["EMAIL_HOST_USER"]) return settings diff --git a/src/archivematicaCommon/lib/executeOrRunSubProcess.py b/src/archivematicaCommon/lib/executeOrRunSubProcess.py index 7acb561688..7af0070c73 100644 --- a/src/archivematicaCommon/lib/executeOrRunSubProcess.py +++ b/src/archivematicaCommon/lib/executeOrRunSubProcess.py @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaCommon -# @author Joseph Perry import io import os import shlex @@ -25,15 +22,14 @@ import sys import uuid -# https://stackoverflow.com/a/36321030 -try: - file_types = (file, io.IOBase) -except NameError: - file_types = (io.IOBase,) - def launchSubProcess( - command, stdIn="", printing=True, arguments=[], env_updates={}, capture_output=False + command, + stdIn="", + printing=True, + arguments=None, + env_updates=None, + capture_output=False, ): """ Launches a subprocess using ``command``, where ``command`` is either: @@ -64,6 +60,10 @@ def launchSubProcess( returned IF the subprocess has failed, i.e., returned a non-zero exit code. """ + if arguments is None: + arguments = [] + if env_updates is None: + env_updates = {} stdError = "" stdOut = "" @@ -88,7 +88,7 @@ def launchSubProcess( elif isinstance(stdIn, bytes): stdin_pipe = subprocess.PIPE communicate_input = stdIn - elif isinstance(stdIn, file_types): + elif isinstance(stdIn, io.IOBase): stdin_pipe = stdIn communicate_input = None else: @@ -139,8 +139,12 @@ def launchSubProcess( def createAndRunScript( - text, stdIn="", printing=True, arguments=[], env_updates={}, capture_output=True + text, stdIn="", printing=True, arguments=None, env_updates=None, capture_output=True ): + if arguments is None: + arguments = [] + if env_updates is None: + env_updates = {} # Output the text to a /tmp/ file scriptPath = "/tmp/" + uuid.uuid4().__str__() FILE = os.open(scriptPath, os.O_WRONLY | os.O_CREAT, 0o770) @@ -169,8 +173,8 @@ def executeOrRun( text, stdIn="", printing=True, - arguments=[], - env_updates={}, + arguments=None, + env_updates=None, capture_output=True, ): """ @@ -203,6 +207,10 @@ def executeOrRun( capture_output: Whether or not to capture output for the executed process. Default is `True`. """ + if arguments is None: + arguments = [] + if env_updates is None: + env_updates = {} if type == "command": return launchSubProcess( text, diff --git a/src/archivematicaCommon/lib/fileOperations.py b/src/archivematicaCommon/lib/fileOperations.py index 96566233a1..822c1623a6 100644 --- a/src/archivematicaCommon/lib/fileOperations.py +++ b/src/archivematicaCommon/lib/fileOperations.py @@ -14,9 +14,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage archivematicaCommon -# @author Joseph Perry import csv import os import shutil @@ -252,7 +249,7 @@ def updateFileLocation( eventType="", eventDateTime="", eventDetail="", - eventIdentifierUUID=uuid.uuid4().__str__(), + eventIdentifierUUID=None, fileUUID="None", sipUUID=None, transferUUID=None, @@ -265,6 +262,8 @@ def updateFileLocation( If the file uuid is not provided, will use the SIP uuid and the old path to find the file uuid. To suppress creation of an event, pass the createEvent keyword argument (for example, if the file moved due to the renaming of a parent directory and not the file itself). """ + if eventIdentifierUUID is None: + eventIdentifierUUID = str(uuid.uuid4()) if not fileUUID or fileUUID == "None": kwargs = {"removedtime__isnull": True, "currentlocation": src.encode()} diff --git a/src/archivematicaCommon/tests/fixtures/test_delete_aip.yaml b/src/archivematicaCommon/tests/fixtures/test_delete_aip.yaml deleted file mode 100644 index 39fdb2bc0b..0000000000 --- a/src/archivematicaCommon/tests/fixtures/test_delete_aip.yaml +++ /dev/null @@ -1,41 +0,0 @@ -interactions: -- request: - body: !!python/unicode '{"query":{"term":{"uuid":"b34521a3-1c63-43dd-b901-584416f36c91"}}}' - headers: - connection: [keep-alive] - content-type: [application/json] - method: GET - uri: http://elasticsearch:9200/aips/_search?_source=uuid - response: - body: {string: !!python/unicode '{"took":2,"timed_out":false,"_shards":{"total":5,"successful":5,"skipped":0,"failed":0},"hits":{"total":1,"max_score":0.2876821,"hits":[{"_index":"aips","_type":"_doc","_id":"lBsZBWgBn49OAVhMXeO8","_score":0.2876821,"_source":{"uuid":"b34521a3-1c63-43dd-b901-584416f36c91"}}]}}'} - headers: - content-length: ['277'] - content-type: [application/json; charset=UTF-8] - status: {code: 200, message: OK} -- request: - body: !!python/unicode '{"query":{"term":{"uuid":"b34521a3-1c63-43dd-b901-584416f36c91"}}}' - headers: - connection: [keep-alive] - content-type: [application/json] - method: POST - uri: http://elasticsearch:9200/aips/_delete_by_query - response: - body: {string: !!python/unicode '{"took":8,"timed_out":false,"total":1,"deleted":1,"batches":1,"version_conflicts":0,"noops":0,"retries":{"bulk":0,"search":0},"throttled_millis":0,"requests_per_second":-1.0,"throttled_until_millis":0,"failures":[]}'} - headers: - content-length: ['215'] - content-type: [application/json; charset=UTF-8] - status: {code: 200, message: OK} -- request: - body: !!python/unicode '{"query":{"term":{"uuid":"b34521a3-1c63-43dd-b901-584416f36c91"}}}' - headers: - connection: [keep-alive] - content-type: [application/json] - method: GET - uri: http://elasticsearch:9200/aips/_search?_source=uuid - response: - body: {string: !!python/unicode '{"took":0,"timed_out":false,"_shards":{"total":5,"successful":5,"skipped":0,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'} - headers: - content-length: ['134'] - content-type: [application/json; charset=UTF-8] - status: {code: 200, message: OK} -version: 1 diff --git a/src/archivematicaCommon/tests/fixtures/test_delete_aip_files.yaml b/src/archivematicaCommon/tests/fixtures/test_delete_aip_files.yaml deleted file mode 100644 index 38f27fe248..0000000000 --- a/src/archivematicaCommon/tests/fixtures/test_delete_aip_files.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: !!python/unicode '{"query":{"term":{"AIPUUID":"b34521a3-1c63-43dd-b901-584416f36c91"}}}' - headers: - connection: [keep-alive] - content-type: [application/json] - method: GET - uri: http://elasticsearch:9200/aipfiles/_search - response: - body: {string: !!python/unicode '{"took":1,"timed_out":false,"_shards":{"total":5,"successful":5,"skipped":0,"failed":0},"hits":{"total":2,"max_score":0.2876821,"hits":[{"_index":"aipfiles","_type":"_doc","_id":"lRsZBWgBn49OAVhMXuMC","_score":0.2876821,"_source":{"origin":"1a14043f-68ef-4bfe-a129-e2e4cdbe391b","METS":{"dmdSec":{"ns0:xmlData_dict_list":[{"@xmlns:ns1":"http://www.loc.gov/premis/v3","@xmlns:ns0":"http://www.loc.gov/METS/","@xmlns:xsi":"http://www.w3.org/2001/XMLSchema-instance","ns1:object_dict_list":[{"ns1:originalName":"20181231153024-b34521a3-1c63-43dd-b901-584416f36c91","@version":"3.0","@xsi:type":"premis:intellectualEntity","ns1:objectIdentifier_dict_list":[{"ns1:objectIdentifierType":"UUID","ns1:objectIdentifierValue":"b34521a3-1c63-43dd-b901-584416f36c91"}],"@xsi:schemaLocation":"http://www.loc.gov/premis/v3 - http://www.loc.gov/standards/premis/v3/premis.xsd"}]}]},"amdSec":{"ns0:amdSec_dict_list":[{"ns0:techMD_dict_list":[{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:OBJECT","ns0:xmlData_dict_list":[{"ns1:object_dict_list":[{"@version":"2.2","@xsi:type":"premis:file","ns1:objectCharacteristics_dict_list":[{"ns1:compositionLevel":"0","ns1:creatingApplication_dict_list":[{"ns1:dateCreatedByApplication":"2018-12-11"}],"ns1:size":"1361321","ns1:fixity_dict_list":[{"ns1:messageDigest":"a469c730e705d757d66f53f38bb4455e89d5691a3d87fc7bc069b91fa2a50d46","ns1:messageDigestAlgorithm":"sha256"}],"ns1:format_dict_list":[{"ns1:formatDesignation_dict_list":[{"ns1:formatName":"JPEG","ns1:formatVersion":"1.01"}],"ns1:formatRegistry_dict_list":[{"ns1:formatRegistryKey":"fmt/43","ns1:formatRegistryName":"PRONOM"}]}],"ns1:objectCharacteristicsExtension":null}],"ns1:originalName":"%transferDirectory%objects/Landing - zone.jpg","ns1:relationship_dict_list":[{"ns1:relatedObjectIdentification_dict_list":[{"ns1:relatedObjectIdentifierType":"UUID","ns1:relatedObjectIdentifierValue":"ba0f9dab-6fd8-4e39-bac8-fe0492bc0a84"}],"ns1:relationshipType":"derivation","ns1:relationshipSubType":"is - source of","ns1:relatedEventIdentification_dict_list":[{"ns1:relatedEventIdentifierValue":"488dc3f8-89d8-41bb-8dd4-6569b28981a9","ns1:relatedEventIdentifierType":"UUID"}]}],"ns1:objectIdentifier_dict_list":[{"ns1:objectIdentifierType":"UUID","ns1:objectIdentifierValue":"80624596-0c7d-436e-a5a0-5ca37c8ce417"}],"@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd"}]}]}],"@ID":"techMD_2"}],"@xmlns:ns1":"info:lc/xmlns/premis-v2","@xmlns:ns0":"http://www.loc.gov/METS/","@xmlns:xsi":"http://www.w3.org/2001/XMLSchema-instance","@ID":"amdSec_2","ns0:digiprovMD_dict_list":[{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:EVENT","ns0:xmlData_dict_list":[{"ns1:event_dict_list":[{"ns1:eventOutcomeInformation_dict_list":[{"ns1:eventOutcome":null,"ns1:eventOutcomeDetail_dict_list":[{"ns1:eventOutcomeDetailNote":null}]}],"ns1:eventIdentifier_dict_list":[{"ns1:eventIdentifierType":"UUID","ns1:eventIdentifierValue":"22822e01-4438-4e78-822e-a8a53e7412e9"}],"ns1:linkingAgentIdentifier_dict_list":[{"ns1:linkingAgentIdentifierValue":"Archivematica-1.9","ns1:linkingAgentIdentifierType":"preservation - system"},{"ns1:linkingAgentIdentifierValue":"test","ns1:linkingAgentIdentifierType":"repository - code"}],"ns1:eventDateTime":"2018-12-31T14:30:34.773872+00:00","ns1:eventDetail":null,"ns1:eventType":"ingestion","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd"}]}]}],"@ID":"digiprovMD_6"},{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:EVENT","ns0:xmlData_dict_list":[{"ns1:event_dict_list":[{"ns1:eventOutcomeInformation_dict_list":[{"ns1:eventOutcome":null,"ns1:eventOutcomeDetail_dict_list":[{"ns1:eventOutcomeDetailNote":"a469c730e705d757d66f53f38bb4455e89d5691a3d87fc7bc069b91fa2a50d46"}]}],"ns1:eventIdentifier_dict_list":[{"ns1:eventIdentifierType":"UUID","ns1:eventIdentifierValue":"23ba11bf-f51f-4e76-8101-40ced150bb12"}],"ns1:linkingAgentIdentifier_dict_list":[{"ns1:linkingAgentIdentifierValue":"Archivematica-1.9","ns1:linkingAgentIdentifierType":"preservation - system"},{"ns1:linkingAgentIdentifierValue":"test","ns1:linkingAgentIdentifierType":"repository - code"}],"ns1:eventDateTime":"2018-12-31T14:30:35.144602+00:00","ns1:eventDetail":"program=\"python\"; - module=\"hashlib.sha256()\"","ns1:eventType":"message digest calculation","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd"}]}]}],"@ID":"digiprovMD_7"},{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:EVENT","ns0:xmlData_dict_list":[{"ns1:event_dict_list":[{"ns1:eventOutcomeInformation_dict_list":[{"ns1:eventOutcome":"Pass","ns1:eventOutcomeDetail_dict_list":[{"ns1:eventOutcomeDetailNote":null}]}],"ns1:eventIdentifier_dict_list":[{"ns1:eventIdentifierType":"UUID","ns1:eventIdentifierValue":"f3e14ac8-fd56-4fd2-b201-b1cb9fb45bee"}],"ns1:linkingAgentIdentifier_dict_list":[{"ns1:linkingAgentIdentifierValue":"Archivematica-1.9","ns1:linkingAgentIdentifierType":"preservation - system"},{"ns1:linkingAgentIdentifierValue":"test","ns1:linkingAgentIdentifierType":"repository - code"}],"ns1:eventDateTime":"2018-12-31T14:30:39.496393+00:00","ns1:eventDetail":"program=\"ClamAV - (clamd)\"; version=\"ClamAV 0.99.2\"; virusDefinitions=\"25255/Mon Dec 31 - 06:25:07 2018\"","ns1:eventType":"virus check","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd"}]}]}],"@ID":"digiprovMD_8"},{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:EVENT","ns0:xmlData_dict_list":[{"ns1:event_dict_list":[{"ns1:eventOutcomeInformation_dict_list":[{"ns1:eventOutcome":null,"ns1:eventOutcomeDetail_dict_list":[{"ns1:eventOutcomeDetailNote":"Original - name=\"%transferDirectory%objects/Landing zone.jpg\"; cleaned up name=\"%transferDirectory%objects/Landing_zone.jpg\""}]}],"ns1:eventIdentifier_dict_list":[{"ns1:eventIdentifierType":"UUID","ns1:eventIdentifierValue":"e914cd93-a32e-4e7c-a994-ab13059c2ac4"}],"ns1:linkingAgentIdentifier_dict_list":[{"ns1:linkingAgentIdentifierValue":"Archivematica-1.9","ns1:linkingAgentIdentifierType":"preservation - system"},{"ns1:linkingAgentIdentifierValue":"test","ns1:linkingAgentIdentifierType":"repository - code"}],"ns1:eventDateTime":"2018-12-31T14:30:41.226291+00:00","ns1:eventDetail":"prohibited - characters removed:program=\"sanitize_names\"; version=\"1.10.d0ccb7d7661cf35c769dcc0846d8f087998af713\"","ns1:eventType":"name - cleanup","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd"}]}]}],"@ID":"digiprovMD_9"},{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:EVENT","ns0:xmlData_dict_list":[{"ns1:event_dict_list":[{"ns1:eventOutcomeInformation_dict_list":[{"ns1:eventOutcome":"Positive","ns1:eventOutcomeDetail_dict_list":[{"ns1:eventOutcomeDetailNote":"fmt/43"}]}],"ns1:eventIdentifier_dict_list":[{"ns1:eventIdentifierType":"UUID","ns1:eventIdentifierValue":"1ae9453a-595f-40df-b0a8-f443af2c976a"}],"ns1:linkingAgentIdentifier_dict_list":[{"ns1:linkingAgentIdentifierValue":"Archivematica-1.9","ns1:linkingAgentIdentifierType":"preservation - system"},{"ns1:linkingAgentIdentifierValue":"test","ns1:linkingAgentIdentifierType":"repository - code"}],"ns1:eventDateTime":"2018-12-31T14:30:43.662024+00:00","ns1:eventDetail":"program=\"Siegfried\"; - version=\"1.7.10\"","ns1:eventType":"format identification","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd"}]}]}],"@ID":"digiprovMD_10"},{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:EVENT","ns0:xmlData_dict_list":[{"ns1:event_dict_list":[{"ns1:eventOutcomeInformation_dict_list":[{"ns1:eventOutcome":"pass","ns1:eventOutcomeDetail_dict_list":[{"ns1:eventOutcomeDetailNote":"format=\"JPEG\"; - version=\"1.01\"; result=\"Well-Formed and valid\""}]}],"ns1:eventIdentifier_dict_list":[{"ns1:eventIdentifierType":"UUID","ns1:eventIdentifierValue":"efa6d51a-d3e9-4dad-a612-82596c032795"}],"ns1:linkingAgentIdentifier_dict_list":[{"ns1:linkingAgentIdentifierValue":"Archivematica-1.9","ns1:linkingAgentIdentifierType":"preservation - system"},{"ns1:linkingAgentIdentifierValue":"test","ns1:linkingAgentIdentifierType":"repository - code"}],"ns1:eventDateTime":"2018-12-31T14:30:48.839005+00:00","ns1:eventDetail":"program=\"JHOVE\"; - version=\"1.6\"","ns1:eventType":"validation","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd"}]}]}],"@ID":"digiprovMD_11"},{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:EVENT","ns0:xmlData_dict_list":[{"ns1:event_dict_list":[{"ns1:eventOutcomeInformation_dict_list":[{"ns1:eventOutcome":null,"ns1:eventOutcomeDetail_dict_list":[{"ns1:eventOutcomeDetailNote":"%SIPDirectory%objects/Landing_zone-ba0f9dab-6fd8-4e39-bac8-fe0492bc0a84.tif"}]}],"ns1:eventIdentifier_dict_list":[{"ns1:eventIdentifierType":"UUID","ns1:eventIdentifierValue":"488dc3f8-89d8-41bb-8dd4-6569b28981a9"}],"ns1:linkingAgentIdentifier_dict_list":[{"ns1:linkingAgentIdentifierValue":"Archivematica-1.9","ns1:linkingAgentIdentifierType":"preservation - system"},{"ns1:linkingAgentIdentifierValue":"test","ns1:linkingAgentIdentifierType":"repository - code"}],"ns1:eventDateTime":"2018-12-31T14:31:00.435014+00:00","ns1:eventDetail":"ArchivematicaFPRCommandID=\"a34ddc9b-c922-4bb6-8037-bbe713332175\"; - program=\"convert\"; version=\"Version: ImageMagick 6.9.7-4 Q16 x86_64 20170114 - http://www.imagemagick.org\"","ns1:eventType":"normalization","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd"}]}]}],"@ID":"digiprovMD_12"},{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:AGENT","ns0:xmlData_dict_list":[{"ns1:agent_dict_list":[{"ns1:agentIdentifier_dict_list":[{"ns1:agentIdentifierType":"preservation - system","ns1:agentIdentifierValue":"Archivematica-1.9"}],"ns1:agentType":"software","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd","ns1:agentName":"Archivematica"}]}]}],"@ID":"digiprovMD_13"},{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:AGENT","ns0:xmlData_dict_list":[{"ns1:agent_dict_list":[{"ns1:agentIdentifier_dict_list":[{"ns1:agentIdentifierType":"repository - code","ns1:agentIdentifierValue":"test"}],"ns1:agentType":"organization","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd","ns1:agentName":"test"}]}]}],"@ID":"digiprovMD_14"}]}]}},"FILEUUID":"80624596-0c7d-436e-a5a0-5ca37c8ce417","sipName":"20181231153024","filePath":"objects/Landing_zone.jpg","indexedAt":1546273775.097954,"identifiers":[],"AIPUUID":"b34521a3-1c63-43dd-b901-584416f36c91","isPartOf":null,"archivematicaVersion":"1.9","fileExtension":"jpg","transferMetadata":[],"AICID":null}},{"_index":"aipfiles","_type":"_doc","_id":"lhsZBWgBn49OAVhMXuMh","_score":0.2876821,"_source":{"origin":"1a14043f-68ef-4bfe-a129-e2e4cdbe391b","METS":{"dmdSec":{"ns0:xmlData_dict_list":[{"@xmlns:ns1":"http://www.loc.gov/premis/v3","@xmlns:ns0":"http://www.loc.gov/METS/","@xmlns:xsi":"http://www.w3.org/2001/XMLSchema-instance","ns1:object_dict_list":[{"ns1:originalName":"20181231153024-b34521a3-1c63-43dd-b901-584416f36c91","@version":"3.0","@xsi:type":"premis:intellectualEntity","ns1:objectIdentifier_dict_list":[{"ns1:objectIdentifierType":"UUID","ns1:objectIdentifierValue":"b34521a3-1c63-43dd-b901-584416f36c91"}],"@xsi:schemaLocation":"http://www.loc.gov/premis/v3 - http://www.loc.gov/standards/premis/v3/premis.xsd"}]}]},"amdSec":{"ns0:amdSec_dict_list":[{"ns0:techMD_dict_list":[{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:OBJECT","ns0:xmlData_dict_list":[{"ns1:object_dict_list":[{"@version":"2.2","@xsi:type":"premis:file","ns1:objectCharacteristics_dict_list":[{"ns1:compositionLevel":"0","ns1:creatingApplication_dict_list":[{"ns1:dateCreatedByApplication":"2018-12-11"}],"ns1:size":"4261301","ns1:fixity_dict_list":[{"ns1:messageDigest":"91a5ddca3637590c2ddb50da5feb73ff0b8a98cd09a98afb79adc2cf70bc6220","ns1:messageDigestAlgorithm":"sha256"}],"ns1:format_dict_list":[{"ns1:formatDesignation_dict_list":[{"ns1:formatName":"Truevision - TGA Bitmap","ns1:formatVersion":"2.0"}],"ns1:formatRegistry_dict_list":[{"ns1:formatRegistryKey":"fmt/402","ns1:formatRegistryName":"PRONOM"}]}],"ns1:objectCharacteristicsExtension":null}],"ns1:originalName":"%transferDirectory%objects/MARBLES.TGA","ns1:relationship_dict_list":[{"ns1:relatedObjectIdentification_dict_list":[{"ns1:relatedObjectIdentifierType":"UUID","ns1:relatedObjectIdentifierValue":"c8979006-38b4-4ac2-a630-6f8b6f4b9c37"}],"ns1:relationshipType":"derivation","ns1:relationshipSubType":"is - source of","ns1:relatedEventIdentification_dict_list":[{"ns1:relatedEventIdentifierValue":"1317ed36-c367-4703-824f-6ae4c2c31340","ns1:relatedEventIdentifierType":"UUID"}]}],"ns1:objectIdentifier_dict_list":[{"ns1:objectIdentifierType":"UUID","ns1:objectIdentifierValue":"97e39f9d-1577-42cd-bc40-80b055747081"}],"@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd"}]}]}],"@ID":"techMD_4"}],"@xmlns:ns1":"info:lc/xmlns/premis-v2","@xmlns:ns0":"http://www.loc.gov/METS/","@xmlns:xsi":"http://www.w3.org/2001/XMLSchema-instance","@ID":"amdSec_4","ns0:digiprovMD_dict_list":[{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:EVENT","ns0:xmlData_dict_list":[{"ns1:event_dict_list":[{"ns1:eventOutcomeInformation_dict_list":[{"ns1:eventOutcome":null,"ns1:eventOutcomeDetail_dict_list":[{"ns1:eventOutcomeDetailNote":null}]}],"ns1:eventIdentifier_dict_list":[{"ns1:eventIdentifierType":"UUID","ns1:eventIdentifierValue":"48868c6e-af8c-44c0-847a-1eba98bf78f8"}],"ns1:linkingAgentIdentifier_dict_list":[{"ns1:linkingAgentIdentifierValue":"Archivematica-1.9","ns1:linkingAgentIdentifierType":"preservation - system"},{"ns1:linkingAgentIdentifierValue":"test","ns1:linkingAgentIdentifierType":"repository - code"}],"ns1:eventDateTime":"2018-12-31T14:30:34.787958+00:00","ns1:eventDetail":null,"ns1:eventType":"ingestion","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd"}]}]}],"@ID":"digiprovMD_20"},{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:EVENT","ns0:xmlData_dict_list":[{"ns1:event_dict_list":[{"ns1:eventOutcomeInformation_dict_list":[{"ns1:eventOutcome":null,"ns1:eventOutcomeDetail_dict_list":[{"ns1:eventOutcomeDetailNote":"91a5ddca3637590c2ddb50da5feb73ff0b8a98cd09a98afb79adc2cf70bc6220"}]}],"ns1:eventIdentifier_dict_list":[{"ns1:eventIdentifierType":"UUID","ns1:eventIdentifierValue":"6d188185-d836-4f0d-9b9d-225b351c7dbe"}],"ns1:linkingAgentIdentifier_dict_list":[{"ns1:linkingAgentIdentifierValue":"Archivematica-1.9","ns1:linkingAgentIdentifierType":"preservation - system"},{"ns1:linkingAgentIdentifierValue":"test","ns1:linkingAgentIdentifierType":"repository - code"}],"ns1:eventDateTime":"2018-12-31T14:30:35.165978+00:00","ns1:eventDetail":"program=\"python\"; - module=\"hashlib.sha256()\"","ns1:eventType":"message digest calculation","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd"}]}]}],"@ID":"digiprovMD_21"},{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:EVENT","ns0:xmlData_dict_list":[{"ns1:event_dict_list":[{"ns1:eventOutcomeInformation_dict_list":[{"ns1:eventOutcome":"Pass","ns1:eventOutcomeDetail_dict_list":[{"ns1:eventOutcomeDetailNote":null}]}],"ns1:eventIdentifier_dict_list":[{"ns1:eventIdentifierType":"UUID","ns1:eventIdentifierValue":"69882dce-8d4a-4ba6-b25c-c27ee6d938a3"}],"ns1:linkingAgentIdentifier_dict_list":[{"ns1:linkingAgentIdentifierValue":"Archivematica-1.9","ns1:linkingAgentIdentifierType":"preservation - system"},{"ns1:linkingAgentIdentifierValue":"test","ns1:linkingAgentIdentifierType":"repository - code"}],"ns1:eventDateTime":"2018-12-31T14:30:39.502799+00:00","ns1:eventDetail":"program=\"ClamAV - (clamd)\"; version=\"ClamAV 0.99.2\"; virusDefinitions=\"25255/Mon Dec 31 - 06:25:07 2018\"","ns1:eventType":"virus check","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd"}]}]}],"@ID":"digiprovMD_22"},{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:EVENT","ns0:xmlData_dict_list":[{"ns1:event_dict_list":[{"ns1:eventOutcomeInformation_dict_list":[{"ns1:eventOutcome":"Positive","ns1:eventOutcomeDetail_dict_list":[{"ns1:eventOutcomeDetailNote":"fmt/402"}]}],"ns1:eventIdentifier_dict_list":[{"ns1:eventIdentifierType":"UUID","ns1:eventIdentifierValue":"9b3c3036-73b7-4db9-aaad-ea9ed785c0a4"}],"ns1:linkingAgentIdentifier_dict_list":[{"ns1:linkingAgentIdentifierValue":"Archivematica-1.9","ns1:linkingAgentIdentifierType":"preservation - system"},{"ns1:linkingAgentIdentifierValue":"test","ns1:linkingAgentIdentifierType":"repository - code"}],"ns1:eventDateTime":"2018-12-31T14:30:43.547201+00:00","ns1:eventDetail":"program=\"Siegfried\"; - version=\"1.7.10\"","ns1:eventType":"format identification","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd"}]}]}],"@ID":"digiprovMD_23"},{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:EVENT","ns0:xmlData_dict_list":[{"ns1:event_dict_list":[{"ns1:eventOutcomeInformation_dict_list":[{"ns1:eventOutcome":null,"ns1:eventOutcomeDetail_dict_list":[{"ns1:eventOutcomeDetailNote":"%SIPDirectory%objects/MARBLES-c8979006-38b4-4ac2-a630-6f8b6f4b9c37.tif"}]}],"ns1:eventIdentifier_dict_list":[{"ns1:eventIdentifierType":"UUID","ns1:eventIdentifierValue":"1317ed36-c367-4703-824f-6ae4c2c31340"}],"ns1:linkingAgentIdentifier_dict_list":[{"ns1:linkingAgentIdentifierValue":"Archivematica-1.9","ns1:linkingAgentIdentifierType":"preservation - system"},{"ns1:linkingAgentIdentifierValue":"test","ns1:linkingAgentIdentifierType":"repository - code"}],"ns1:eventDateTime":"2018-12-31T14:31:00.171076+00:00","ns1:eventDetail":"ArchivematicaFPRCommandID=\"a34ddc9b-c922-4bb6-8037-bbe713332175\"; - program=\"convert\"; version=\"Version: ImageMagick 6.9.7-4 Q16 x86_64 20170114 - http://www.imagemagick.org\"","ns1:eventType":"normalization","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd"}]}]}],"@ID":"digiprovMD_24"},{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:AGENT","ns0:xmlData_dict_list":[{"ns1:agent_dict_list":[{"ns1:agentIdentifier_dict_list":[{"ns1:agentIdentifierType":"preservation - system","ns1:agentIdentifierValue":"Archivematica-1.9"}],"ns1:agentType":"software","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd","ns1:agentName":"Archivematica"}]}]}],"@ID":"digiprovMD_25"},{"ns0:mdWrap_dict_list":[{"@MDTYPE":"PREMIS:AGENT","ns0:xmlData_dict_list":[{"ns1:agent_dict_list":[{"ns1:agentIdentifier_dict_list":[{"ns1:agentIdentifierType":"repository - code","ns1:agentIdentifierValue":"test"}],"ns1:agentType":"organization","@version":"2.2","@xsi:schemaLocation":"info:lc/xmlns/premis-v2 - http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd","ns1:agentName":"test"}]}]}],"@ID":"digiprovMD_26"}]}]}},"FILEUUID":"97e39f9d-1577-42cd-bc40-80b055747081","sipName":"20181231153024","filePath":"objects/MARBLES.TGA","indexedAt":1546273775.097954,"identifiers":[],"AIPUUID":"b34521a3-1c63-43dd-b901-584416f36c91","isPartOf":null,"archivematicaVersion":"1.9","fileExtension":"tga","transferMetadata":[],"AICID":null}}]}}'} - headers: - content-length: ['19387'] - content-type: [application/json; charset=UTF-8] - status: {code: 200, message: OK} -- request: - body: !!python/unicode '{"query":{"term":{"AIPUUID":"b34521a3-1c63-43dd-b901-584416f36c91"}}}' - headers: - connection: [keep-alive] - content-type: [application/json] - method: POST - uri: http://elasticsearch:9200/aipfiles/_delete_by_query - response: - body: {string: !!python/unicode '{"took":11,"timed_out":false,"total":2,"deleted":2,"batches":1,"version_conflicts":0,"noops":0,"retries":{"bulk":0,"search":0},"throttled_millis":0,"requests_per_second":-1.0,"throttled_until_millis":0,"failures":[]}'} - headers: - content-length: ['216'] - content-type: [application/json; charset=UTF-8] - status: {code: 200, message: OK} -- request: - body: !!python/unicode '{"query":{"term":{"AIPUUID":"b34521a3-1c63-43dd-b901-584416f36c91"}}}' - headers: - connection: [keep-alive] - content-type: [application/json] - method: GET - uri: http://elasticsearch:9200/aipfiles/_search - response: - body: {string: !!python/unicode '{"took":0,"timed_out":false,"_shards":{"total":5,"successful":5,"skipped":0,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'} - headers: - content-length: ['134'] - content-type: [application/json; charset=UTF-8] - status: {code: 200, message: OK} -version: 1 diff --git a/src/archivematicaCommon/tests/fixtures/test_elasticsearch_setup.yaml b/src/archivematicaCommon/tests/fixtures/test_elasticsearch_setup.yaml deleted file mode 100644 index 6f55f24ff1..0000000000 --- a/src/archivematicaCommon/tests/fixtures/test_elasticsearch_setup.yaml +++ /dev/null @@ -1,15 +0,0 @@ -interactions: -- request: - body: null - headers: - connection: [keep-alive] - content-type: [application/json] - method: HEAD - uri: http://elasticsearch:9200/aips,aipfiles,transfers,transferfiles - response: - body: {string: !!python/unicode ''} - headers: - content-length: ['25355'] - content-type: [application/json; charset=UTF-8] - status: {code: 200, message: OK} -version: 1 diff --git a/src/archivematicaCommon/tests/fixtures/test_get_tags_no_matches.yaml b/src/archivematicaCommon/tests/fixtures/test_get_tags_no_matches.yaml deleted file mode 100644 index 75b2cbceae..0000000000 --- a/src/archivematicaCommon/tests/fixtures/test_get_tags_no_matches.yaml +++ /dev/null @@ -1,15 +0,0 @@ -interactions: -- request: - body: !!python/unicode '{"query":{"term":{"fileuuid":"no_such_file"}}}' - headers: - connection: [keep-alive] - content-type: [application/json] - method: GET - uri: http://elasticsearch:9200/transferfiles/_search?_source=tags - response: - body: {string: !!python/unicode '{"took":1,"timed_out":false,"_shards":{"total":5,"successful":5,"skipped":0,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'} - headers: - content-length: ['134'] - content-type: [application/json; charset=UTF-8] - status: {code: 200, message: OK} -version: 1 diff --git a/src/archivematicaCommon/tests/fixtures/test_set_get_tags.yaml b/src/archivematicaCommon/tests/fixtures/test_set_get_tags.yaml deleted file mode 100644 index bb84c1fdfb..0000000000 --- a/src/archivematicaCommon/tests/fixtures/test_set_get_tags.yaml +++ /dev/null @@ -1,41 +0,0 @@ -interactions: -- request: - body: !!python/unicode '{"query":{"term":{"fileuuid":"268421a7-a986-4fa0-95c1-54176e508210"}}}' - headers: - connection: [keep-alive] - content-type: [application/json] - method: GET - uri: http://elasticsearch:9200/transferfiles/_search?size=10000 - response: - body: {string: !!python/unicode '{"took":1,"timed_out":false,"_shards":{"total":5,"successful":5,"skipped":0,"failed":0},"hits":{"total":1,"max_score":0.6931472,"hits":[{"_index":"transferfiles","_type":"_doc","_id":"mBsZBWgBn49OAVhMh-OV","_score":0.6931472,"_source":{"accessionid":"","status":"backlog","sipuuid":"17b168b6-cbba-4f43-8838-a53360238acb","tags":[],"file_extension":"jpg","relative_path":"test-17b168b6-cbba-4f43-8838-a53360238acb/objects/Landing_zone.jpg","bulk_extractor_reports":[],"origin":"1a14043f-68ef-4bfe-a129-e2e4cdbe391b","size":1.2982568740844727,"modification_date":"2018-12-11","created":1546273029.7313669,"format":[],"ingestdate":"2018-12-31","filename":"Landing_zone.jpg","fileuuid":"268421a7-a986-4fa0-95c1-54176e508210"}}]}}'} - headers: - content-length: ['725'] - content-type: [application/json; charset=UTF-8] - status: {code: 200, message: OK} -- request: - body: !!python/unicode '{"doc":{"tags":["test"]}}' - headers: - connection: [keep-alive] - content-type: [application/json] - method: POST - uri: http://elasticsearch:9200/transferfiles/_doc/mBsZBWgBn49OAVhMh-OV/_update - response: - body: {string: !!python/unicode '{"_index":"transferfiles","_type":"_doc","_id":"mBsZBWgBn49OAVhMh-OV","_version":2,"result":"updated","forced_refresh":true,"_shards":{"total":2,"successful":1,"failed":0},"_seq_no":2,"_primary_term":1}'} - headers: - content-length: ['202'] - content-type: [application/json; charset=UTF-8] - status: {code: 200, message: OK} -- request: - body: !!python/unicode '{"query":{"term":{"fileuuid":"268421a7-a986-4fa0-95c1-54176e508210"}}}' - headers: - connection: [keep-alive] - content-type: [application/json] - method: GET - uri: http://elasticsearch:9200/transferfiles/_search?_source=tags - response: - body: {string: !!python/unicode '{"took":2,"timed_out":false,"_shards":{"total":5,"successful":5,"skipped":0,"failed":0},"hits":{"total":1,"max_score":0.47000363,"hits":[{"_index":"transferfiles","_type":"_doc","_id":"mBsZBWgBn49OAVhMh-OV","_score":0.47000363,"_source":{"tags":["test"]}}]}}'} - headers: - content-length: ['258'] - content-type: [application/json; charset=UTF-8] - status: {code: 200, message: OK} -version: 1 diff --git a/src/archivematicaCommon/tests/fixtures/test_set_tags_no_matches.yaml b/src/archivematicaCommon/tests/fixtures/test_set_tags_no_matches.yaml deleted file mode 100644 index 95a225a1fd..0000000000 --- a/src/archivematicaCommon/tests/fixtures/test_set_tags_no_matches.yaml +++ /dev/null @@ -1,15 +0,0 @@ -interactions: -- request: - body: !!python/unicode '{"query":{"term":{"fileuuid":"no_such_file"}}}' - headers: - connection: [keep-alive] - content-type: [application/json] - method: GET - uri: http://elasticsearch:9200/transferfiles/_search?size=10000 - response: - body: {string: !!python/unicode '{"took":0,"timed_out":false,"_shards":{"total":5,"successful":5,"skipped":0,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}}'} - headers: - content-length: ['134'] - content-type: [application/json; charset=UTF-8] - status: {code: 200, message: OK} -version: 1 diff --git a/src/archivematicaCommon/tests/test_elasticsearch_functions.py b/src/archivematicaCommon/tests/test_elasticsearch_functions.py index c2b4d8af8a..31c976a87c 100644 --- a/src/archivematicaCommon/tests/test_elasticsearch_functions.py +++ b/src/archivematicaCommon/tests/test_elasticsearch_functions.py @@ -6,7 +6,6 @@ import elasticSearchFunctions import pytest -import vcr from lxml import etree from main.models import Directory from main.models import Identifier @@ -16,17 +15,57 @@ class TestElasticSearchFunctions(unittest.TestCase): - @vcr.use_cassette( - os.path.join(THIS_DIR, "fixtures", "test_elasticsearch_setup.yaml") - ) def setUp(self): - elasticSearchFunctions.setup("elasticsearch:9200") + with mock.patch("elasticsearch.transport.Transport.perform_request"): + elasticSearchFunctions.setup("elasticsearch:9200") self.client = elasticSearchFunctions.get_client() self.aip_uuid = "b34521a3-1c63-43dd-b901-584416f36c91" self.file_uuid = "268421a7-a986-4fa0-95c1-54176e508210" - @vcr.use_cassette(os.path.join(THIS_DIR, "fixtures", "test_delete_aip.yaml")) - def test_delete_aip(self): + @mock.patch( + "elasticsearch.transport.Transport.perform_request", + side_effect=[ + { + "took": 2, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": { + "total": 1, + "max_score": 0.2876821, + "hits": [ + { + "_index": "aips", + "_type": "_doc", + "_id": "lBsZBWgBn49OAVhMXeO8", + "_score": 0.2876821, + "_source": {"uuid": "b34521a3-1c63-43dd-b901-584416f36c91"}, + } + ], + }, + }, + { + "took": 8, + "timed_out": False, + "total": 1, + "deleted": 1, + "batches": 1, + "version_conflicts": 0, + "noops": 0, + "retries": {"bulk": 0, "search": 0}, + "throttled_millis": 0, + "requests_per_second": -1.0, + "throttled_until_millis": 0, + "failures": [], + }, + { + "took": 0, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": {"total": 0, "max_score": None, "hits": []}, + }, + ], + ) + def test_delete_aip(self, perform_request): # Verify AIP exists results = self.client.search( index="aips", @@ -45,8 +84,61 @@ def test_delete_aip(self): ) assert results["hits"]["total"] == 0 - @vcr.use_cassette(os.path.join(THIS_DIR, "fixtures", "test_delete_aip_files.yaml")) - def test_delete_aip_files(self): + @mock.patch( + "elasticsearch.transport.Transport.perform_request", + side_effect=[ + { + "took": 1, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": { + "total": 2, + "max_score": 0.2876821, + "hits": [ + { + "_index": "aipfiles", + "_type": "_doc", + "_id": "lRsZBWgBn49OAVhMXuMC", + "_score": 0.2876821, + "_source": { + "origin": "1a14043f-68ef-4bfe-a129-e2e4cdbe391b" + }, + }, + { + "_index": "aipfiles", + "_type": "_doc", + "_id": "lhsZBWgBn49OAVhMXuMh", + "_score": 0.2876821, + "_source": { + "origin": "1a14043f-68ef-4bfe-a129-e2e4cdbe391b" + }, + }, + ], + }, + }, + { + "took": 11, + "timed_out": False, + "total": 2, + "deleted": 2, + "batches": 1, + "version_conflicts": 0, + "noops": 0, + "retries": {"bulk": 0, "search": 0}, + "throttled_millis": 0, + "requests_per_second": -1.0, + "throttled_until_millis": 0, + "failures": [], + }, + { + "took": 0, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": {"total": 0, "max_score": None, "hits": []}, + }, + ], + ) + def test_delete_aip_files(self, perform_request): # Verify AIP files exist results = self.client.search( index="aipfiles", body={"query": {"term": {"AIPUUID": self.aip_uuid}}} @@ -60,26 +152,183 @@ def test_delete_aip_files(self): ) assert results["hits"]["total"] == 0 - @vcr.use_cassette(os.path.join(THIS_DIR, "fixtures", "test_set_get_tags.yaml")) - def test_set_get_tags(self): + assert perform_request.mock_calls == [ + mock.call( + "GET", + "/aipfiles/_search", + params={}, + body={ + "query": { + "term": {"AIPUUID": "b34521a3-1c63-43dd-b901-584416f36c91"} + } + }, + ), + mock.call( + "POST", + "/aipfiles/_delete_by_query", + params={}, + body={ + "query": { + "term": {"AIPUUID": "b34521a3-1c63-43dd-b901-584416f36c91"} + } + }, + ), + mock.call( + "GET", + "/aipfiles/_search", + params={}, + body={ + "query": { + "term": {"AIPUUID": "b34521a3-1c63-43dd-b901-584416f36c91"} + } + }, + ), + ] + + @mock.patch( + "elasticsearch.transport.Transport.perform_request", + side_effect=[ + { + "took": 1, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": { + "total": 1, + "max_score": 0.6931472, + "hits": [ + { + "_index": "transferfiles", + "_type": "_doc", + "_id": "mBsZBWgBn49OAVhMh-OV", + "_score": 0.6931472, + "_source": { + "accessionid": "", + "status": "backlog", + "sipuuid": "17b168b6-cbba-4f43-8838-a53360238acb", + "tags": [], + "file_extension": "jpg", + "relative_path": "test-17b168b6-cbba-4f43-8838-a53360238acb/objects/Landing_zone.jpg", + "bulk_extractor_reports": [], + "origin": "1a14043f-68ef-4bfe-a129-e2e4cdbe391b", + "size": 1.2982568740844727, + "modification_date": "2018-12-11", + "created": 1546273029.7313669, + "format": [], + "ingestdate": "2018-12-31", + "filename": "Landing_zone.jpg", + "fileuuid": "268421a7-a986-4fa0-95c1-54176e508210", + }, + } + ], + }, + }, + { + "_index": "transferfiles", + "_type": "_doc", + "_id": "mBsZBWgBn49OAVhMh-OV", + "_version": 2, + "result": "updated", + "forced_refresh": True, + "_shards": {"total": 2, "successful": 1, "failed": 0}, + "_seq_no": 2, + "_primary_term": 1, + }, + { + "took": 2, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": { + "total": 1, + "max_score": 0.47000363, + "hits": [ + { + "_index": "transferfiles", + "_type": "_doc", + "_id": "mBsZBWgBn49OAVhMh-OV", + "_score": 0.47000363, + "_source": {"tags": ["test"]}, + } + ], + }, + }, + ], + ) + def test_set_get_tags(self, perform_request): elasticSearchFunctions.set_file_tags(self.client, self.file_uuid, ["test"]) assert elasticSearchFunctions.get_file_tags(self.client, self.file_uuid) == [ "test" ] - @vcr.use_cassette( - os.path.join(THIS_DIR, "fixtures", "test_get_tags_no_matches.yaml") + assert perform_request.mock_calls == [ + mock.call( + "GET", + "/transferfiles/_search", + params={"size": "10000"}, + body={ + "query": { + "term": {"fileuuid": "268421a7-a986-4fa0-95c1-54176e508210"} + } + }, + ), + mock.call( + "POST", + "/transferfiles/_doc/mBsZBWgBn49OAVhMh-OV/_update", + params={}, + body={"doc": {"tags": ["test"]}}, + ), + mock.call( + "GET", + "/transferfiles/_search", + params={"_source": b"tags"}, + body={ + "query": { + "term": {"fileuuid": "268421a7-a986-4fa0-95c1-54176e508210"} + } + }, + ), + ] + + @mock.patch( + "elasticsearch.transport.Transport.perform_request", + side_effect=[ + { + "took": 1, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": {"total": 0, "max_score": None, "hits": []}, + } + ], ) - def test_list_tags_fails_when_file_cant_be_found(self): + def test_list_tags_fails_when_file_cant_be_found(self, perform_request): with pytest.raises(elasticSearchFunctions.EmptySearchResultError): elasticSearchFunctions.get_file_tags(self.client, "no_such_file") + perform_request.assert_called_once_with( + "GET", + "/transferfiles/_search", + params={"_source": b"tags"}, + body={"query": {"term": {"fileuuid": "no_such_file"}}}, + ) - @vcr.use_cassette( - os.path.join(THIS_DIR, "fixtures", "test_set_tags_no_matches.yaml") + @mock.patch( + "elasticsearch.transport.Transport.perform_request", + side_effect=[ + { + "took": 0, + "timed_out": False, + "_shards": {"total": 5, "successful": 5, "skipped": 0, "failed": 0}, + "hits": {"total": 0, "max_score": None, "hits": []}, + } + ], ) - def test_set_tags_fails_when_file_cant_be_found(self): + def test_set_tags_fails_when_file_cant_be_found(self, perform_request): with pytest.raises(elasticSearchFunctions.EmptySearchResultError): elasticSearchFunctions.set_file_tags(self.client, "no_such_file", []) + perform_request.assert_called_once_with( + "GET", + "/transferfiles/_search", + params={"size": "10000"}, + body={"query": {"term": {"fileuuid": "no_such_file"}}}, + ) @pytest.mark.django_db @mock.patch("elasticSearchFunctions.get_dashboard_uuid") @@ -196,8 +445,7 @@ def _bulk(client, actions, stats_only=False, *args, **kwargs): @mock.patch("elasticSearchFunctions.bulk") def test_index_mets_file_metadata_with_utf8(self, dummy_helpers_bulk): def _bulk(client, actions, stats_only=False, *args, **kwargs): - for action in actions: - pass + pass dummy_helpers_bulk.side_effect = _bulk mets_file_path = os.path.join( diff --git a/src/dashboard/frontend/package-lock.json b/src/dashboard/frontend/package-lock.json index 93420da33a..8a70739a07 100644 --- a/src/dashboard/frontend/package-lock.json +++ b/src/dashboard/frontend/package-lock.json @@ -4,6 +4,12 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@babel/parser": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.0.tgz", + "integrity": "sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==", + "dev": true + }, "accepts": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.3.tgz", @@ -64,6 +70,46 @@ "resolved": "https://registry.npmjs.org/angular-gettext/-/angular-gettext-2.3.10.tgz", "integrity": "sha1-9xJb8aJ7dnA3wgTeNS87FE25aXw=" }, + "angular-gettext-cli": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/angular-gettext-cli/-/angular-gettext-cli-1.2.0.tgz", + "integrity": "sha512-GBqJCP96dREebVLxpe45/QOpvg9CqmdcoNA8VuuaTO7APnW3Uc3CqlRydAeo/jxHcdnEPIssgb9wcBlAyO6lVA==", + "dev": true, + "requires": { + "angular-gettext-tools": "^2", + "glob": "^7.1", + "minimist": "^1.2" + }, + "dependencies": { + "minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true + } + } + }, + "angular-gettext-tools": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/angular-gettext-tools/-/angular-gettext-tools-2.5.3.tgz", + "integrity": "sha512-/SyBMhPwMSoasZRXfMkXsKxaAHkDHHaFr3jAMqPuh3wGNJ8s9LfHKOGzcM0FW2mGIotmF1CRoWKgUIHMCwUYcg==", + "dev": true, + "requires": { + "@babel/parser": "^7.4.3", + "binary-search": "^1.2.0", + "cheerio": "^0.22.0", + "lodash": "^4.17.5", + "pofile": "~1.0.0" + }, + "dependencies": { + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true + } + } + }, "angular-loader": { "version": "1.5.11", "resolved": "https://registry.npmjs.org/angular-loader/-/angular-loader-1.5.11.tgz", @@ -828,6 +874,12 @@ "integrity": "sha1-ZlBsFs5vTWkopbPNajPKQelB43s=", "dev": true }, + "binary-search": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/binary-search/-/binary-search-1.3.6.tgz", + "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==", + "dev": true + }, "blob": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/blob/-/blob-0.0.4.tgz", @@ -875,6 +927,12 @@ } } }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "dev": true + }, "brace-expansion": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz", @@ -1000,6 +1058,30 @@ "supports-color": "^2.0.0" } }, + "cheerio": { + "version": "0.22.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-0.22.0.tgz", + "integrity": "sha512-8/MzidM6G/TgRelkzDG13y3Y9LxBjCb+8yOEZ9+wwq5gVF2w2pV0wmHvjfT0RvuxGyR7UEuK36r+yYMbT4uKgA==", + "dev": true, + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.0", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash.assignin": "^4.0.9", + "lodash.bind": "^4.1.4", + "lodash.defaults": "^4.0.1", + "lodash.filter": "^4.4.0", + "lodash.flatten": "^4.2.0", + "lodash.foreach": "^4.3.0", + "lodash.map": "^4.4.0", + "lodash.merge": "^4.4.0", + "lodash.pick": "^4.2.1", + "lodash.reduce": "^4.4.0", + "lodash.reject": "^4.4.0", + "lodash.some": "^4.4.0" + } + }, "chokidar": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-1.7.0.tgz", @@ -1242,6 +1324,18 @@ "source-list-map": "^0.1.4" } }, + "css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha512-dUQOBoqdR7QwV90WysXPLXG5LO7nhYBgiWVfxF80DKPF8zx1t/pUd2FYy73emg3zrjtM6dzmYgbHKfV2rxiHQA==", + "dev": true, + "requires": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + } + }, "css-selector-tokenizer": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/css-selector-tokenizer/-/css-selector-tokenizer-0.5.4.tgz", @@ -1252,6 +1346,12 @@ "fastparse": "^1.1.1" } }, + "css-what": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", + "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==", + "dev": true + }, "cssesc": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-0.1.0.tgz", @@ -1387,6 +1487,16 @@ } } }, + "dom-serializer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", + "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "dev": true, + "requires": { + "domelementtype": "^1.3.0", + "entities": "^1.1.1" + } + }, "dom-walk": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/dom-walk/-/dom-walk-0.1.1.tgz", @@ -1399,6 +1509,31 @@ "integrity": "sha1-hnqksJP6oF8d4IwG9NeyH9+GmLw=", "dev": true }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==", + "dev": true + }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "dev": true, + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha512-gSu5Oi/I+3wDENBsOWBiRK1eoGxcywYSqg3rR960/+EfY0CF4EX1VPkgHOZ3WiS/Jg2DtliF6BhWcHlfpYUcGw==", + "dev": true, + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -1540,6 +1675,12 @@ } } }, + "entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==", + "dev": true + }, "errno": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.4.tgz", @@ -2858,6 +2999,48 @@ "integrity": "sha1-ZouTd26q5V696POtRkswekljYl4=", "dev": true }, + "htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "dev": true, + "requires": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "requires": { + "safe-buffer": "~5.2.0" + } + } + } + }, "http-errors": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz", @@ -3411,6 +3594,18 @@ "integrity": "sha1-+6HEUkwZ7ppfgTa0YJ8BfPTe1pI=", "dev": true }, + "lodash.assignin": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.assignin/-/lodash.assignin-4.2.0.tgz", + "integrity": "sha512-yX/rx6d/UTVh7sSVWVSIMjfnz95evAgDFdb1ZozC35I9mSFCkmzptOzevxjgbQUsc78NR44LVHWjsoMQXy9FDg==", + "dev": true + }, + "lodash.bind": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/lodash.bind/-/lodash.bind-4.2.1.tgz", + "integrity": "sha512-lxdsn7xxlCymgLYo1gGvVrfHmkjDiyqVv62FAeF2i5ta72BipE1SLxw8hPEPLhD4/247Ijw07UQH7Hq/chT5LA==", + "dev": true + }, "lodash.camelcase": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-3.0.1.tgz", @@ -3429,12 +3624,72 @@ "lodash._root": "^3.0.0" } }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "dev": true + }, + "lodash.filter": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.filter/-/lodash.filter-4.6.0.tgz", + "integrity": "sha512-pXYUy7PR8BCLwX5mgJ/aNtyOvuJTdZAo9EQFUvMIYugqmJxnrYaANvTbgndOzHSCSR0wnlBBfRXJL5SbWxo3FQ==", + "dev": true + }, + "lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==", + "dev": true + }, + "lodash.foreach": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.foreach/-/lodash.foreach-4.5.0.tgz", + "integrity": "sha512-aEXTF4d+m05rVOAUG3z4vZZ4xVexLKZGF0lIxuHZ1Hplpk/3B6Z1+/ICICYRLm7c41Z2xiejbkCkJoTlypoXhQ==", + "dev": true + }, + "lodash.map": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.map/-/lodash.map-4.6.0.tgz", + "integrity": "sha512-worNHGKLDetmcEYDvh2stPCrrQRkP20E4l0iIS7F8EvzMqBBi7ltvFN5m1HvTf1P7Jk1txKhvFcmYsCr8O2F1Q==", + "dev": true + }, "lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", "integrity": "sha1-vMbEmkKihA7Zl/Mj6tpezRguC/4=", "dev": true }, + "lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true + }, + "lodash.pick": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz", + "integrity": "sha512-hXt6Ul/5yWjfklSGvLQl8vM//l3FtyHZeuelpzK6mm99pNvN9yTDruNZPEJZD1oWrqo+izBmB7oUfWgcCX7s4Q==", + "dev": true + }, + "lodash.reduce": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.reduce/-/lodash.reduce-4.6.0.tgz", + "integrity": "sha512-6raRe2vxCYBhpBu+B+TtNGUzah+hQjVdu3E17wfusjyrXBka2nBS8OH/gjVZ5PvHOhWmIZTYri09Z6n/QfnNMw==", + "dev": true + }, + "lodash.reject": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.reject/-/lodash.reject-4.6.0.tgz", + "integrity": "sha512-qkTuvgEzYdyhiJBx42YPzPo71R1aEr0z79kAv7Ixg8wPFEjgRgJdUsGMG3Hf3OYSF/kHI79XhNlt+5Ar6OzwxQ==", + "dev": true + }, + "lodash.some": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.some/-/lodash.some-4.6.0.tgz", + "integrity": "sha512-j7MJE+TuT51q9ggt4fSgVqro163BEFjAt3u97IqU+JA2DkWl80nFTrowzLpZ/BnpN7rrl0JA/593NAdd8p/scQ==", + "dev": true + }, "lodash.uniq": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", @@ -3817,6 +4072,15 @@ "sort-keys": "^1.0.0" } }, + "nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "dev": true, + "requires": { + "boolbase": "~1.0.0" + } + }, "null-check": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/null-check/-/null-check-1.0.0.tgz", @@ -4001,6 +4265,12 @@ "find-up": "^1.0.0" } }, + "pofile": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pofile/-/pofile-1.0.11.tgz", + "integrity": "sha512-Vy9eH1dRD9wHjYt/QqXcTz+RnX/zg53xK+KljFSX30PvdDMb2z+c6uDUeblUGqqJgz3QFsdlA0IJvHziPmWtQg==", + "dev": true + }, "postcss": { "version": "5.2.17", "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.17.tgz", diff --git a/src/dashboard/src/components/accounts/views.py b/src/dashboard/src/components/accounts/views.py index b44f74fde6..7a06666715 100644 --- a/src/dashboard/src/components/accounts/views.py +++ b/src/dashboard/src/components/accounts/views.py @@ -191,5 +191,5 @@ def delete(request, id): raise Http404 user.delete() return redirect("accounts:accounts_index") - except: + except Exception: raise Http404 diff --git a/src/dashboard/src/components/administration/views.py b/src/dashboard/src/components/administration/views.py index 7a38b28260..082a6b07d9 100644 --- a/src/dashboard/src/components/administration/views.py +++ b/src/dashboard/src/components/administration/views.py @@ -181,7 +181,7 @@ def storage(request): """ try: response_locations = storage_service.get_location() - except: + except Exception: messages.warning( request, _( @@ -326,7 +326,7 @@ def _get_shared_dirs(calculate_usage=False): ) if calculate_usage: - for id, dir_spec in dirs.items(): + for dir_spec in dirs.values(): dir_spec["used"] = _usage_get_directory_used_bytes(dir_spec["path"]) return dirs @@ -561,7 +561,7 @@ def general(request): ) forms = (general_form, storage_form, checksum_form) - if all([form.is_valid() for form in forms]): + if all(form.is_valid() for form in forms): for item in forms: item.save() messages.info(request, _("Saved.")) diff --git a/src/dashboard/src/components/advanced_search.py b/src/dashboard/src/components/advanced_search.py index 5cd53666c7..f6e9c37425 100644 --- a/src/dashboard/src/components/advanced_search.py +++ b/src/dashboard/src/components/advanced_search.py @@ -46,19 +46,19 @@ def search_parameter_prep(request): fields = [""] else: # Make sure each query has field/ops set - for index, query in enumerate(queries): + for index, _ in enumerate(queries): # A blank query makes ES error if queries[index] == "": queries[index] = "*" try: fields[index] - except: + except Exception: fields.insert(index, "") try: ops[index] - except: + except Exception: ops.insert(index, "and") if ops[index] == "": @@ -66,7 +66,7 @@ def search_parameter_prep(request): try: types[index] - except: + except Exception: types.insert(index, "") # For "other" fields, the actual title of the subfield is located in a @@ -95,18 +95,20 @@ def extract_url_search_params_from_request(request): search_params = request.get_full_path().split("?")[1] end_of_search_params = search_params.index("&page") search_params = search_params[:end_of_search_params] - except: + except Exception: pass return search_params -def assemble_query(queries, ops, fields, types, filters=[]): +def assemble_query(queries, ops, fields, types, filters=None): + if filters is None: + filters = [] must_haves = [] should_haves = [] must_not_haves = [] index = 0 - for query in queries: + for _ in queries: if queries[index] != "": clause = _query_clause(index, queries, ops, fields, types) if clause: @@ -230,5 +232,5 @@ def _query_clause(index, queries, ops, fields, types): def indexed_count(es_client, index, query=None): try: return es_client.count(index=index, body=query)["count"] - except: + except Exception: return 0 diff --git a/src/dashboard/src/components/api/urls.py b/src/dashboard/src/components/api/urls.py index 22446b0a64..e66c65da13 100644 --- a/src/dashboard/src/components/api/urls.py +++ b/src/dashboard/src/components/api/urls.py @@ -21,8 +21,10 @@ app_name = "api" urlpatterns = [ - re_path(r"transfer/approve", views.approve_transfer), - re_path(r"transfer/unapproved", views.unapproved_transfers), + re_path(r"transfer/approve", views.approve_transfer, name="approve_transfer"), + re_path( + r"transfer/unapproved", views.unapproved_transfers, name="unapproved_transfers" + ), re_path( r"transfer/completed", views.completed_transfers, name="completed_transfers" ), @@ -35,13 +37,20 @@ re_path( r"transfer/start_transfer/", views.start_transfer_api, name="start_transfer" ), - re_path(r"transfer/reingest", views.reingest, {"target": "transfer"}), + re_path( + r"transfer/reingest", + views.reingest, + {"target": "transfer"}, + name="transfer_reingest", + ), re_path( r"ingest/status/(?P" + settings.UUID_REGEX + ")", views.status, {"unit_type": "unitSIP"}, ), - re_path(r"ingest/waiting", views.waiting_for_user_input), + re_path( + r"ingest/waiting", views.waiting_for_user_input, name="waiting_for_user_input" + ), re_path( r"^(?Ptransfer|ingest)/(?P" + settings.UUID_REGEX @@ -52,7 +61,12 @@ re_path( r"^ingest/reingest/approve", views.reingest_approve, name="reingest_approve" ), - re_path(r"^ingest/reingest", views.reingest, {"target": "ingest"}), + re_path( + r"^ingest/reingest", + views.reingest, + {"target": "ingest"}, + name="ingest_reingest", + ), re_path(r"^ingest/completed", views.completed_ingests, name="completed_ingests"), path("ingest/copy_metadata_files/", views.copy_metadata_files_api), path( diff --git a/src/dashboard/src/components/api/validators.py b/src/dashboard/src/components/api/validators.py index 57f0e0cc7a..bbe9bb47e8 100644 --- a/src/dashboard/src/components/api/validators.py +++ b/src/dashboard/src/components/api/validators.py @@ -155,7 +155,7 @@ def _check_field_pairs(row): :param row: list: metadata fields """ - for i, field in enumerate(row): + for field in row: if field == "Other Identifier Type": if not all( f in row for f in ["Other Identifier", "Other Identifier Type"] diff --git a/src/dashboard/src/components/api/views.py b/src/dashboard/src/components/api/views.py index 94b97ca89c..acee7b930b 100644 --- a/src/dashboard/src/components/api/views.py +++ b/src/dashboard/src/components/api/views.py @@ -302,7 +302,7 @@ def waiting_for_user_input(request): # TODO should this filter based on unit type into transfer vs SIP? jobs = models.Job.objects.filter(currentstep=models.Job.STATUS_AWAITING_DECISION) for job in jobs: - unit_uuid = job.sipuuid + unit_uuid = str(job.sipuuid) directory = os.path.basename(os.path.normpath(job.directory)) unit_name = directory.replace("-" + unit_uuid, "", 1) @@ -468,7 +468,11 @@ def unapproved_transfers(request): ) unapproved.append( - {"type": transfer_type, "directory": job_directory, "uuid": job.sipuuid} + { + "type": transfer_type, + "directory": job_directory, + "uuid": str(job.sipuuid), + } ) # get list of unapproved transfers diff --git a/src/dashboard/src/components/archival_storage/tests/test_archival_storage.py b/src/dashboard/src/components/archival_storage/tests/test_archival_storage.py index 91964ebf80..819834bc39 100644 --- a/src/dashboard/src/components/archival_storage/tests/test_archival_storage.py +++ b/src/dashboard/src/components/archival_storage/tests/test_archival_storage.py @@ -271,7 +271,7 @@ def test_search_as_csv(mocker, amsetup, admin_client, tmp_path): response.get(CONTENT_DISPOSITION) == 'attachment; filename="test-filename.csv"' ) - streamed_content = b"".join([content for content in response.streaming_content]) + streamed_content = b"".join(list(response.streaming_content)) csv_file = StringIO(streamed_content.decode("utf8")) assert csv_file.read() == ( diff --git a/src/dashboard/src/components/filesystem_ajax/helpers.py b/src/dashboard/src/components/filesystem_ajax/helpers.py index e009ece42a..ff09a79a9d 100644 --- a/src/dashboard/src/components/filesystem_ajax/helpers.py +++ b/src/dashboard/src/components/filesystem_ajax/helpers.py @@ -29,7 +29,9 @@ def sorted_directory_list(path): return sorted(entries, key=helpers.keynat) -def directory_to_dict(path, directory={}, entry=False): +def directory_to_dict(path, directory=None, entry=False): + if directory is None: + directory = {} # if starting traversal, set entry to directory root if entry is False: entry = directory @@ -77,7 +79,7 @@ def check_filepath_exists(filepath): try: filepath.index("..") error = "Illegal path." - except: + except Exception: pass return error diff --git a/src/dashboard/src/components/filesystem_ajax/views.py b/src/dashboard/src/components/filesystem_ajax/views.py index b169b0ba3c..acc17c36ff 100644 --- a/src/dashboard/src/components/filesystem_ajax/views.py +++ b/src/dashboard/src/components/filesystem_ajax/views.py @@ -438,7 +438,7 @@ def _create_arranged_sip(staging_sip_path, files, sip_uuid): ): try: os.mkdir(os.path.join(staging_abs_path, directory)) - except os.error as exception: + except OSError as exception: if exception.errno != errno.EEXIST: raise diff --git a/src/dashboard/src/components/helpers.py b/src/dashboard/src/components/helpers.py index 09b64af811..898fc014fa 100644 --- a/src/dashboard/src/components/helpers.py +++ b/src/dashboard/src/components/helpers.py @@ -178,7 +178,7 @@ def get_setting(setting, default=""): try: setting = models.DashboardSetting.objects.get(name=setting) return setting.value - except: + except Exception: return default @@ -193,7 +193,7 @@ def get_boolean_setting(setting, default=""): def set_setting(setting, value=""): try: setting_data = models.DashboardSetting.objects.get(name=setting) - except: + except Exception: setting_data = models.DashboardSetting.objects.create() setting_data.name = setting # ``DashboardSetting.value`` is a string-based field. The empty string is diff --git a/src/dashboard/src/components/ingest/pair_matcher.py b/src/dashboard/src/components/ingest/pair_matcher.py index 72f6cf0007..c6b25b8fc3 100644 --- a/src/dashboard/src/components/ingest/pair_matcher.py +++ b/src/dashboard/src/components/ingest/pair_matcher.py @@ -178,8 +178,10 @@ def match_dip_objects_to_resource_levels( parent_url, reset_url, uuid, - matches=[], + matches=None, ): + if matches is None: + matches = [] # load object relative paths object_path_json = json.JSONEncoder().encode( ingest_upload_atk_get_dip_object_paths(uuid) @@ -203,8 +205,10 @@ def match_dip_objects_to_resource_component_levels( parent_url, reset_url, uuid, - matches=[], + matches=None, ): + if matches is None: + matches = [] # load object relative paths object_path_json = json.JSONEncoder().encode( ingest_upload_atk_get_dip_object_paths(uuid) @@ -238,7 +242,9 @@ def remove_review_matches_prefixes(path): return remove_objects_prefix(remove_sip_dir_prefix(path)) -def review_matches(client, request, template, uuid, matches=[]): +def review_matches(client, request, template, uuid, matches=None): + if matches is None: + matches = [] object_paths = { file_.uuid: remove_review_matches_prefixes(file_.currentlocation.decode()) for file_ in models.File.objects.filter(sip=uuid) @@ -262,7 +268,7 @@ def ingest_upload_atk_get_dip_object_paths(uuid): dip_upload_dir = os.path.join(watch_dir, "uploadDIP") try: sip = models.SIP.objects.get(uuid=uuid) - except: + except Exception: raise Http404 directory = os.path.basename(os.path.dirname(sip.currentpath)) metsFilePath = os.path.join(dip_upload_dir, directory, "METS." + uuid + ".xml") diff --git a/src/dashboard/src/components/ingest/views.py b/src/dashboard/src/components/ingest/views.py index ffe2592dfe..40ca99f266 100644 --- a/src/dashboard/src/components/ingest/views.py +++ b/src/dashboard/src/components/ingest/views.py @@ -80,7 +80,7 @@ def _adjust_directories_draggability(nodes): def ingest_grid(request): try: storage_service.get_location(purpose="BL") - except: + except Exception: messages.warning( request, _( @@ -175,7 +175,7 @@ def ingest_metadata_edit(request, uuid, id=None): def ingest_metadata_add_files(request, sip_uuid): try: source_directories = storage_service.get_location(purpose="TS") - except: + except Exception: messages.warning( request, _( @@ -288,7 +288,7 @@ def ingest_metadata_delete(request, uuid, id): models.DublinCore.objects.get(pk=id).delete() messages.info(request, _("Deleted.")) return redirect("ingest:ingest_metadata_list", uuid) - except: + except Exception: raise Http404 @@ -322,7 +322,7 @@ def ingest_upload(request, uuid): if "target" in request.POST: try: access = models.Access.objects.get(sipuuid=uuid) - except: + except Exception: access = models.Access(sipuuid=uuid) access.target = pickle.dumps( {"target": request.POST["target"]}, protocol=0 @@ -334,7 +334,7 @@ def ingest_upload(request, uuid): try: access = models.Access.objects.get(sipuuid=uuid) data = pickle.loads(access.target.encode()) - except: + except Exception: raise Http404 return helpers.json_response(data) @@ -629,7 +629,7 @@ def transfer_backlog(request, ui): query = advanced_search.assemble_query( queries, ops, fields, types, filters=[backlog_filter] ) - except: + except Exception: logger.exception("Error accessing index.") return HttpResponse("Error accessing index.") @@ -638,7 +638,7 @@ def transfer_backlog(request, ui): results = elasticSearchFunctions.search_all_results( es_client, body=query, index="transferfiles" ) - except: + except Exception: logger.exception("Error accessing index.") return HttpResponse("Error accessing index.") @@ -693,7 +693,7 @@ def transfer_file_download(request, uuid): # get file basename try: file = models.File.objects.get(uuid=uuid) - except: + except Exception: raise Http404 shared_directory_path = django_settings.SHARED_DIRECTORY diff --git a/src/dashboard/src/components/ingest/views_NormalizationReport.py b/src/dashboard/src/components/ingest/views_NormalizationReport.py index 32363296ca..feb50ae54d 100755 --- a/src/dashboard/src/components/ingest/views_NormalizationReport.py +++ b/src/dashboard/src/components/ingest/views_NormalizationReport.py @@ -16,10 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @subpackage Dashboard -# @author Joseph Perry -# @author Justin Simpson from components import helpers from django.db import connection diff --git a/src/dashboard/src/components/rights/views.py b/src/dashboard/src/components/rights/views.py index c07261069b..6e28b0f9af 100644 --- a/src/dashboard/src/components/rights/views.py +++ b/src/dashboard/src/components/rights/views.py @@ -254,7 +254,7 @@ def rights_edit(request, uuid, id=None, section="ingest"): createdCopyright = models.RightsStatementCopyright.objects.get( rightsstatement=createdRights ) - except: + except Exception: createdCopyright = models.RightsStatementCopyright( rightsstatement=createdRights ) @@ -280,7 +280,7 @@ def rights_edit(request, uuid, id=None, section="ingest"): createdCopyright = models.RightsStatementCopyright.objects.get( rightsstatement=createdRights ) - except: + except Exception: createdCopyright = models.RightsStatementCopyright( rightsstatement=createdRights ) @@ -324,7 +324,7 @@ def rights_edit(request, uuid, id=None, section="ingest"): createdLicense = models.RightsStatementLicense.objects.get( rightsstatement=createdRights ) - except: + except Exception: createdLicense = models.RightsStatementLicense( rightsstatement=createdRights ) @@ -350,7 +350,7 @@ def rights_edit(request, uuid, id=None, section="ingest"): createdLicense = models.RightsStatementLicense.objects.get( rightsstatement=createdRights ) - except: + except Exception: createdLicense = models.RightsStatementLicense( rightsstatement=createdRights ) @@ -533,7 +533,7 @@ def rights_edit(request, uuid, id=None, section="ingest"): rightsstatement=createdRights ) ) - except: + except Exception: createdOther = models.RightsStatementOtherRightsInformation( rightsstatement=createdRights ) @@ -561,7 +561,7 @@ def rights_edit(request, uuid, id=None, section="ingest"): rightsstatement=createdRights ) ) - except: + except Exception: createdOther = models.RightsStatementOtherRightsInformation( rightsstatement=createdRights ) @@ -594,7 +594,7 @@ def rights_edit(request, uuid, id=None, section="ingest"): url = reverse("rights_%s:edit" % section, args=[uuid, createdRights.pk]) try: url = url + "?created=" + new_content_type_created - except: + except Exception: pass return redirect(url) else: @@ -751,7 +751,7 @@ def rights_holders_lookup(request, id): try: agent = models.RightsStatementLinkingAgentIdentifier.objects.get(pk=id) result = agent.linkingagentidentifiervalue + " [" + str(agent.id) + "]" - except: + except Exception: result = "" return HttpResponse(result) diff --git a/src/dashboard/src/components/transfer/views.py b/src/dashboard/src/components/transfer/views.py index d7dd2475ab..e0b15a8ddc 100644 --- a/src/dashboard/src/components/transfer/views.py +++ b/src/dashboard/src/components/transfer/views.py @@ -53,7 +53,7 @@ def grid(request): def transfer_source_locations(request): try: return helpers.json_response(storage_service.get_location(purpose="TS")) - except: + except Exception: message = _("Error retrieving source directories") logger.exception(message) response = {"message": message, "status": "Failure"} diff --git a/src/dashboard/src/fpr/migrations/0035_python3_compatibility.py b/src/dashboard/src/fpr/migrations/0035_python3_compatibility.py index eab9d87c7f..bcbbba14bb 100644 --- a/src/dashboard/src/fpr/migrations/0035_python3_compatibility.py +++ b/src/dashboard/src/fpr/migrations/0035_python3_compatibility.py @@ -535,7 +535,7 @@ def identify(file_): try: results = stdout.split('\n')[0].split(',') - except: + except Exception: raise FidoFailed(stdout, stderr, process.returncode) if process.returncode != 0 or results[-1] == '"fail"': diff --git a/src/dashboard/src/fpr/south_migrations/recreateData.sh b/src/dashboard/src/fpr/south_migrations/recreateData.sh index d87776f746..5f76356c60 100755 --- a/src/dashboard/src/fpr/south_migrations/recreateData.sh +++ b/src/dashboard/src/fpr/south_migrations/recreateData.sh @@ -17,8 +17,6 @@ # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -# @package Archivematica -# @author Joseph Perry database="$FPR_DB_NAME" origDir="`pwd`" diff --git a/src/dashboard/src/fpr/views.py b/src/dashboard/src/fpr/views.py index 9143a888e6..8c8050b474 100644 --- a/src/dashboard/src/fpr/views.py +++ b/src/dashboard/src/fpr/views.py @@ -730,7 +730,7 @@ def revision_list(request, entity_name, uuid): # restrict to models that are intended to have revisions try: - getattr(model, "replaces") + getattr(model, "replaces") # noqa: B009 # get specific revision's data and augment with detail URL revision = model.objects.get(uuid=uuid) diff --git a/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py b/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py index e2d6aa48d5..fc04f3797c 100644 --- a/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py +++ b/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py @@ -522,13 +522,11 @@ def _import_self_describing_transfer( """ transfer, created = Transfer.objects.get_or_create( uuid=transfer_uuid, - defaults=dict( - type="Standard", - diruuids=False, - currentlocation="%sharedPath%www/AIPsStore/transferBacklog/originals/" - + str(transfer_dir.name) - + "/", - ), + defaults={ + "type": "Standard", + "diruuids": False, + "currentlocation": f"%sharedPath%www/AIPsStore/transferBacklog/originals/{transfer_dir.name}/", + }, ) # The transfer did not exist, we need to populate everything else. diff --git a/src/dashboard/src/main/management/commands/resolve_pending_jobs.py b/src/dashboard/src/main/management/commands/resolve_pending_jobs.py index 3f7585a530..e3cafd0257 100644 --- a/src/dashboard/src/main/management/commands/resolve_pending_jobs.py +++ b/src/dashboard/src/main/management/commands/resolve_pending_jobs.py @@ -97,7 +97,7 @@ def loop(self, *args, **options): try: client.execute_unit(package_id, chain_id) break - except: + except Exception: self.error("There was a problem executing the selected choice") def admin_user(self): diff --git a/src/dashboard/src/main/migrations/0001_initial.py b/src/dashboard/src/main/migrations/0001_initial.py index df8c75f4fc..50863ea58c 100644 --- a/src/dashboard/src/main/migrations/0001_initial.py +++ b/src/dashboard/src/main/migrations/0001_initial.py @@ -4,6 +4,9 @@ from django.db import migrations from django.db import models +# HELP TEXT VALUES + +USE_ISO8601="Use ISO 8601 (YYYY-MM-DD)" class Migration(migrations.Migration): dependencies = [("fpr", "__first__")] @@ -154,7 +157,7 @@ class Migration(migrations.Migration): ( "date", models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD or YYYY-MM-DD/YYYY-MM-DD)", + help_text=USE_ISO8601, db_column="date", blank=True, ), @@ -786,7 +789,7 @@ class Migration(migrations.Migration): ( "copyrightstatusdeterminationdate", models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, verbose_name="Copyright determination date", db_column="copyrightStatusDeterminationDate", blank=True, @@ -795,7 +798,7 @@ class Migration(migrations.Migration): ( "copyrightapplicablestartdate", models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, verbose_name="Copyright start date", db_column="copyrightApplicableStartDate", blank=True, @@ -804,7 +807,7 @@ class Migration(migrations.Migration): ( "copyrightapplicableenddate", models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, verbose_name="Copyright end date", db_column="copyrightApplicableEndDate", blank=True, @@ -942,7 +945,7 @@ class Migration(migrations.Migration): ( "licenseapplicablestartdate", models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, verbose_name="License start date", db_column="licenseApplicableStartDate", blank=True, @@ -951,7 +954,7 @@ class Migration(migrations.Migration): ( "licenseapplicableenddate", models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, verbose_name="License end date", db_column="licenseApplicableEndDate", blank=True, @@ -1168,7 +1171,7 @@ class Migration(migrations.Migration): ( "otherrightsapplicablestartdate", models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, verbose_name="Other rights start date", db_column="otherRightsApplicableStartDate", blank=True, @@ -1177,7 +1180,7 @@ class Migration(migrations.Migration): ( "otherrightsapplicableenddate", models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, verbose_name="Other rights end date", db_column="otherRightsApplicableEndDate", blank=True, @@ -1247,7 +1250,7 @@ class Migration(migrations.Migration): ( "startdate", models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, verbose_name="Start", db_column="startDate", blank=True, @@ -1256,7 +1259,7 @@ class Migration(migrations.Migration): ( "enddate", models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, verbose_name="End", db_column="endDate", blank=True, @@ -1412,7 +1415,7 @@ class Migration(migrations.Migration): ( "statutedeterminationdate", models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, verbose_name="Statute determination date", db_column="statuteInformationDeterminationDate", blank=True, @@ -1421,7 +1424,7 @@ class Migration(migrations.Migration): ( "statuteapplicablestartdate", models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, verbose_name="Statute start date", db_column="statuteApplicableStartDate", blank=True, @@ -1430,7 +1433,7 @@ class Migration(migrations.Migration): ( "statuteapplicableenddate", models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, verbose_name="Statute end date", db_column="statuteApplicableEndDate", blank=True, diff --git a/src/dashboard/src/main/migrations/0004_rights.py b/src/dashboard/src/main/migrations/0004_rights.py index c680d8ce72..6834637e3b 100644 --- a/src/dashboard/src/main/migrations/0004_rights.py +++ b/src/dashboard/src/main/migrations/0004_rights.py @@ -1,6 +1,7 @@ from django.db import migrations from django.db import models +USE_ISO8601="Use ISO 8601 (YYYY-MM-DD)" class Migration(migrations.Migration): dependencies = [("main", "0003_archivesspacedipobjectresourcepairing_data")] @@ -37,7 +38,7 @@ class Migration(migrations.Migration): model_name="rightsstatementcopyright", name="copyrightapplicableenddate", field=models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, null=True, verbose_name="Copyright end date", db_column="copyrightApplicableEndDate", @@ -49,7 +50,7 @@ class Migration(migrations.Migration): model_name="rightsstatementcopyright", name="copyrightapplicablestartdate", field=models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, null=True, verbose_name="Copyright start date", db_column="copyrightApplicableStartDate", @@ -77,7 +78,7 @@ class Migration(migrations.Migration): model_name="rightsstatementcopyright", name="copyrightstatusdeterminationdate", field=models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, null=True, verbose_name="Copyright determination date", db_column="copyrightStatusDeterminationDate", @@ -126,7 +127,7 @@ class Migration(migrations.Migration): model_name="rightsstatementlicense", name="licenseapplicableenddate", field=models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, null=True, verbose_name="License end date", db_column="licenseApplicableEndDate", @@ -138,7 +139,7 @@ class Migration(migrations.Migration): model_name="rightsstatementlicense", name="licenseapplicablestartdate", field=models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, null=True, verbose_name="License start date", db_column="licenseApplicableStartDate", @@ -227,7 +228,7 @@ class Migration(migrations.Migration): model_name="rightsstatementotherrightsinformation", name="otherrightsapplicableenddate", field=models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, null=True, verbose_name="Other rights end date", db_column="otherRightsApplicableEndDate", @@ -239,7 +240,7 @@ class Migration(migrations.Migration): model_name="rightsstatementotherrightsinformation", name="otherrightsapplicablestartdate", field=models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, null=True, verbose_name="Other rights start date", db_column="otherRightsApplicableStartDate", @@ -275,7 +276,7 @@ class Migration(migrations.Migration): model_name="rightsstatementrightsgranted", name="enddate", field=models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, null=True, verbose_name="End", db_column="endDate", @@ -287,7 +288,7 @@ class Migration(migrations.Migration): model_name="rightsstatementrightsgranted", name="startdate", field=models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, null=True, verbose_name="Start", db_column="startDate", @@ -342,7 +343,7 @@ class Migration(migrations.Migration): model_name="rightsstatementstatuteinformation", name="statuteapplicableenddate", field=models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, null=True, verbose_name="Statute end date", db_column="statuteApplicableEndDate", @@ -354,7 +355,7 @@ class Migration(migrations.Migration): model_name="rightsstatementstatuteinformation", name="statuteapplicablestartdate", field=models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, null=True, verbose_name="Statute start date", db_column="statuteApplicableStartDate", @@ -374,7 +375,7 @@ class Migration(migrations.Migration): model_name="rightsstatementstatuteinformation", name="statutedeterminationdate", field=models.TextField( - help_text="Use ISO 8061 (YYYY-MM-DD)", + help_text=USE_ISO8601, null=True, verbose_name="Statute determination date", db_column="statuteInformationDeterminationDate", diff --git a/src/dashboard/src/main/models.py b/src/dashboard/src/main/models.py index 47b49abe0b..5b55a05960 100644 --- a/src/dashboard/src/main/models.py +++ b/src/dashboard/src/main/models.py @@ -51,6 +51,10 @@ # How many objects are created through bulk_create in a single database query BULK_CREATE_BATCH_SIZE = 2000 +# HELP TEXT VALUES + +USE_ISO8601="Use ISO 8601 (YYYY-MM-DD)" + # CUSTOM FIELDS @@ -131,12 +135,9 @@ def get_dict(self, scope): {u'foo': u'bar'} """ - return { - key: value - for (key, value) in self.get_queryset() - .filter(scope=scope) - .values_list("name", "value") - } + return dict( + self.get_queryset().filter(scope=scope).values_list("name", "value") + ) def set_dict(self, scope, items): """ @@ -258,7 +259,7 @@ class DublinCore(models.Model): publisher = models.TextField(db_column="publisher", blank=True) contributor = models.TextField(db_column="contributor", blank=True) date = models.TextField( - help_text=_("Use ISO 8601 (YYYY-MM-DD or YYYY-MM-DD/YYYY-MM-DD)"), + help_text=_(USE_ISO8601), db_column="date", blank=True, ) @@ -1196,21 +1197,21 @@ class RightsStatementCopyright(models.Model): blank=True, null=True, verbose_name=_("Copyright determination date"), - help_text=_("Use ISO 8061 (YYYY-MM-DD)"), + help_text=_(USE_ISO8601), ) copyrightapplicablestartdate = models.TextField( db_column="copyrightApplicableStartDate", blank=True, null=True, verbose_name=_("Copyright start date"), - help_text=_("Use ISO 8061 (YYYY-MM-DD)"), + help_text=_(USE_ISO8601), ) copyrightapplicableenddate = models.TextField( db_column="copyrightApplicableEndDate", blank=True, null=True, verbose_name=_("Copyright end date"), - help_text=_("Use ISO 8061 (YYYY-MM-DD)"), + help_text=_(USE_ISO8601), ) copyrightenddateopen = models.BooleanField( default=False, @@ -1280,14 +1281,14 @@ class RightsStatementLicense(models.Model): blank=True, null=True, verbose_name=_("License start date"), - help_text=_("Use ISO 8061 (YYYY-MM-DD)"), + help_text=_(USE_ISO8601), ) licenseapplicableenddate = models.TextField( db_column="licenseApplicableEndDate", blank=True, null=True, verbose_name=_("License end date"), - help_text=_("Use ISO 8061 (YYYY-MM-DD)"), + help_text=_(USE_ISO8601), ) licenseenddateopen = models.BooleanField( default=False, @@ -1353,14 +1354,14 @@ class RightsStatementRightsGranted(models.Model): startdate = models.TextField( db_column="startDate", verbose_name=_("Start"), - help_text=_("Use ISO 8061 (YYYY-MM-DD)"), + help_text=_(USE_ISO8601), blank=True, null=True, ) enddate = models.TextField( db_column="endDate", verbose_name=_("End"), - help_text=_("Use ISO 8061 (YYYY-MM-DD)"), + help_text=_(USE_ISO8601), blank=True, null=True, ) @@ -1422,7 +1423,7 @@ class RightsStatementStatuteInformation(models.Model): statutedeterminationdate = models.TextField( db_column="statuteInformationDeterminationDate", verbose_name=_("Statute determination date"), - help_text=_("Use ISO 8061 (YYYY-MM-DD)"), + help_text=_(USE_ISO8601), blank=True, null=True, ) @@ -1431,14 +1432,14 @@ class RightsStatementStatuteInformation(models.Model): blank=True, null=True, verbose_name=_("Statute start date"), - help_text=_("Use ISO 8061 (YYYY-MM-DD)"), + help_text=_(USE_ISO8601), ) statuteapplicableenddate = models.TextField( db_column="statuteApplicableEndDate", blank=True, null=True, verbose_name=_("Statute end date"), - help_text=_("Use ISO 8061 (YYYY-MM-DD)"), + help_text=_(USE_ISO8601), ) statuteenddateopen = models.BooleanField( default=False, @@ -1510,14 +1511,14 @@ class RightsStatementOtherRightsInformation(models.Model): blank=True, null=True, verbose_name=_("Other rights start date"), - help_text=_("Use ISO 8061 (YYYY-MM-DD)"), + help_text=_(USE_ISO8601), ) otherrightsapplicableenddate = models.TextField( db_column="otherRightsApplicableEndDate", blank=True, null=True, verbose_name=_("Other rights end date"), - help_text=_("Use ISO 8061 (YYYY-MM-DD)"), + help_text=_(USE_ISO8601), ) otherrightsenddateopen = models.BooleanField( default=False, diff --git a/src/dashboard/src/main/templatetags/breadcrumb.py b/src/dashboard/src/main/templatetags/breadcrumb.py index 51b68f3e7a..43213ac5bc 100644 --- a/src/dashboard/src/main/templatetags/breadcrumb.py +++ b/src/dashboard/src/main/templatetags/breadcrumb.py @@ -83,7 +83,7 @@ def render(self, context): try: val = self.vars[0] title = val.resolve(context) - except: + except Exception: title = "" else: @@ -115,7 +115,7 @@ def render(self, context): try: val = self.title title = val.resolve(context) - except: + except Exception: title = "" else: title = title.strip("'").strip('"') diff --git a/src/dashboard/tests/test_api.py b/src/dashboard/tests/test_api.py index 7650df79ff..7211276881 100644 --- a/src/dashboard/tests/test_api.py +++ b/src/dashboard/tests/test_api.py @@ -6,6 +6,7 @@ import archivematicaFunctions import pytest +import requests from components import helpers from components.api import views from django.core.management import call_command @@ -14,8 +15,12 @@ from django.urls import reverse from django.utils.timezone import make_aware from lxml import etree +from main.models import DashboardSetting +from main.models import DublinCore from main.models import Job from main.models import LevelOfDescription +from main.models import MetadataAppliesToType +from main.models import RightsStatement from main.models import SIP from main.models import SIPArrange from main.models import Task @@ -517,10 +522,8 @@ def test_list_processing_configs(self): assert len(processing_configs) == 2 expected_names = sorted(["default", "automated"]) assert all( - [ - actual == expected - for actual, expected in zip(processing_configs, expected_names) - ] + actual == expected + for actual, expected in zip(processing_configs, expected_names) ) def test_get_existing_processing_config(self): @@ -753,3 +756,415 @@ def test_path_metadata_post_resets_level_of_description(admin_client): list(SIPArrange.objects.values("arrange_path", "level_of_description")) == expected ) + + +@pytest.mark.django_db +def test_unapproved_transfers(admin_client): + helpers.set_setting("dashboard_uuid", "test-uuid") + + # Create a couple of jobs with one awaiting for decision, i.e. unapproved. + approve_transfer_uuid = uuid.uuid4() + Job.objects.create( + jobtype="Approve standard transfer", + currentstep=Job.STATUS_AWAITING_DECISION, + directory="%sharedPath%watchedDirectories/activeTransfers/standardTransfer/test-2/", + createdtime=make_aware(datetime.datetime(2023, 11, 14, 9, 20)), + unittype="unitTransfer", + sipuuid=approve_transfer_uuid, + ) + Job.objects.create( + jobtype="Store AIP", + currentstep=Job.STATUS_COMPLETED_SUCCESSFULLY, + directory="%sharedPath%watchedDirectories/storeAIP/test-1/", + createdtime=make_aware(datetime.datetime(2023, 11, 13, 0, 0)), + unittype="unitSIP", + sipuuid=uuid.UUID("520327f1-cd4e-47fe-9d8a-d9c02fded504"), + ) + + response = admin_client.get(reverse("api:unapproved_transfers")) + assert response.status_code == 200 + + # Verify the awaiting transfer is listed. + payload = json.loads(response.content.decode("utf8")) + assert payload == { + "message": "Fetched unapproved transfers successfully.", + "results": [ + { + "directory": "test-2", + "type": "standard", + "uuid": str(approve_transfer_uuid), + } + ], + } + + +@pytest.mark.parametrize( + "post_data,expected_error", + [ + ({}, "Please specify a transfer directory."), + ({"directory": "mytransfer", "type": ""}, "Please specify a transfer type."), + ({"directory": "mytransfer", "type": "bogus"}, "Invalid transfer type."), + ( + {"directory": "mytransfer", "type": "standard"}, + "Unable to start the transfer.", + ), + ], + ids=[ + "no_transfer_directory", + "no_transfer_type", + "invalid_transfer_type", + "mcpclient_error", + ], +) +def test_approve_transfer_failures(post_data, expected_error, admin_client, mocker): + helpers.set_setting("dashboard_uuid", "test-uuid") + + # Simulate an unhandled error when calling Gearman. + mocker.patch("contrib.mcp.client.MCPClient", side_effect=Exception()) + + response = admin_client.post(reverse("api:approve_transfer"), post_data) + + assert response.status_code == 500 + payload = json.loads(response.content.decode("utf8")) + assert payload == {"error": True, "message": expected_error} + + +def test_approve_transfer(admin_client, mocker): + helpers.set_setting("dashboard_uuid", "test-uuid") + + # Simulate a dashboard <-> Gearman <-> MCPServer interaction. + # The MCPServer approveTransferByPath RPC method returns a UUID. + transfer_uuid = uuid.uuid4() + mocker.patch("contrib.mcp.client.pickle.loads", side_effect=[transfer_uuid]) + job_complete = mocker.patch( + "contrib.mcp.client.gearman.JOB_COMPLETE", + ) + mocker.patch( + "gearman.GearmanClient", + return_value=mocker.Mock( + **{"submit_job.return_value": mocker.Mock(state=job_complete)} + ), + ) + + response = admin_client.post( + reverse("api:approve_transfer"), {"directory": "mytransfer", "type": "standard"} + ) + assert response.status_code == 200 + + payload = json.loads(response.content.decode("utf8")) + assert payload == {"message": "Approval successful.", "uuid": str(transfer_uuid)} + + +@pytest.mark.django_db +def test_waiting_for_user_input(admin_client): + helpers.set_setting("dashboard_uuid", "test-uuid") + + # Create a couple of jobs with one awaiting for decision. + approve_transfer_uuid = uuid.uuid4() + Job.objects.create( + jobtype="Approve standard transfer", + currentstep=Job.STATUS_AWAITING_DECISION, + directory="%sharedPath%watchedDirectories/activeTransfers/standardTransfer/test-2/", + createdtime=make_aware(datetime.datetime(2023, 11, 14, 9, 20)), + unittype="unitTransfer", + sipuuid=approve_transfer_uuid, + ) + Job.objects.create( + jobtype="Store AIP", + currentstep=Job.STATUS_COMPLETED_SUCCESSFULLY, + directory="%sharedPath%watchedDirectories/storeAIP/test-1/", + createdtime=make_aware(datetime.datetime(2023, 11, 13, 0, 0)), + unittype="unitSIP", + sipuuid=uuid.uuid4(), + ) + + response = admin_client.get(reverse("api:waiting_for_user_input")) + assert response.status_code == 200 + + # Verify the awaiting transfer is listed. + payload = json.loads(response.content.decode("utf8")) + assert payload == { + "message": "Fetched units successfully.", + "results": [ + { + "microservice": "Approve standard transfer", + "sip_directory": "test-2", + "sip_name": "test-2", + "sip_uuid": str(approve_transfer_uuid), + } + ], + } + + +def test_reingest_fails_with_missing_parameters(admin_client): + helpers.set_setting("dashboard_uuid", "test-uuid") + + response = admin_client.post( + reverse("api:transfer_reingest", kwargs={"target": "transfer"}), {} + ) + + assert response.status_code == 400 + payload = json.loads(response.content.decode("utf8")) + assert payload == {"error": True, "message": '"name" and "uuid" are required.'} + + +@pytest.fixture +def sip_path(tmp_path): + shared_dir = tmp_path / "dir" + shared_dir.mkdir() + + (shared_dir / "watchedDirectories" / "activeTransfers" / "standardTransfer").mkdir( + parents=True + ) + (shared_dir / "watchedDirectories" / "system" / "reingestAIP").mkdir(parents=True) + + tmp_dir = shared_dir / "tmp" + tmp_dir.mkdir() + + sip_dir = tmp_dir / f"mytransfer-{uuid.uuid4()}" + sip_dir.mkdir() + (sip_dir / "myfile.txt").write_text("my file") + + return sip_dir + + +@pytest.mark.django_db +def test_reingest_deletes_existing_models_related_to_sip( + sip_path, settings, admin_client +): + helpers.set_setting("dashboard_uuid", "test-uuid") + + # Set the SHARED_DIRECTORY setting based on the sip_path fixture. + shared_directory = sip_path.parent.parent + transfer_uuid = sip_path.name[-36:] + settings.SHARED_DIRECTORY = shared_directory.as_posix() + + # Create a Transfer and related models. + transfer = Transfer.objects.create(uuid=transfer_uuid) + job = Job.objects.create( + sipuuid=transfer.uuid, + createdtime=make_aware(datetime.datetime(2023, 11, 15, 8, 30)), + ) + Task.objects.create( + job=job, + createdtime=make_aware(datetime.datetime(2023, 11, 15, 8, 30)), + ) + SIP.objects.create(uuid=transfer.uuid) + metadata_applies_to_type = MetadataAppliesToType.objects.create() + RightsStatement.objects.create( + metadataappliestoidentifier=transfer.uuid, + metadataappliestotype=metadata_applies_to_type, + ) + DublinCore.objects.create( + metadataappliestoidentifier=transfer.uuid, + metadataappliestotype=metadata_applies_to_type, + ) + + response = admin_client.post( + reverse("api:transfer_reingest", kwargs={"target": "transfer"}), + {"name": f"mytransfer-{transfer_uuid}", "uuid": str(transfer_uuid)}, + ) + assert response.status_code == 200 + + # Verify the related models were deleted. + assert Job.objects.count() == 0 + assert Task.objects.count() == 0 + assert SIP.objects.count() == 0 + assert RightsStatement.objects.count() == 0 + assert DublinCore.objects.count() == 0 + + +@pytest.mark.django_db +def test_reingest_full(sip_path, settings, admin_client, mocker): + helpers.set_setting("dashboard_uuid", "test-uuid") + + # Fake UUID generation from the endpoint for a new Transfer. + transfer_uuid = uuid.uuid4() + mocker.patch("uuid.uuid4", return_value=transfer_uuid) + + # Set the SHARED_DIRECTORY setting based on the sip_path fixture. + shared_directory = sip_path.parent.parent + settings.SHARED_DIRECTORY = shared_directory.as_posix() + + # There are no existing Transfers initially. + assert Transfer.objects.count() == 0 + + response = admin_client.post( + reverse("api:transfer_reingest", kwargs={"target": "transfer"}), + {"name": sip_path.name, "uuid": sip_path.name[-36:]}, + ) + assert response.status_code == 200 + + # Verify the Transfer in the payload contains the fake UUID. + payload = json.loads(response.content.decode("utf8")) + assert payload == { + "message": "Approval successful.", + "reingest_uuid": str(transfer_uuid), + } + + # Verify a Transfer model was created. + assert ( + Transfer.objects.filter( + currentlocation=f"%sharedPath%/watchedDirectories/activeTransfers/standardTransfer/mytransfer-{transfer_uuid}/", + type=Transfer.ARCHIVEMATICA_AIP, + ).count() + == 1 + ) + + # Verify the original content was moved to the active transfers directory. + active_transfers_path = ( + shared_directory / "watchedDirectories" / "activeTransfers" / "standardTransfer" + ) + assert [e.name for e in active_transfers_path.iterdir()] == [ + f"mytransfer-{transfer_uuid}" + ] + assert ( + active_transfers_path / f"mytransfer-{transfer_uuid}" / "myfile.txt" + ).read_text() == "my file" + + +@pytest.mark.django_db +def test_reingest_full_fails_if_target_directory_already_exists( + sip_path, settings, admin_client, mocker +): + helpers.set_setting("dashboard_uuid", "test-uuid") + + # Fake UUID generation from the endpoint for a new Transfer. + transfer_uuid = uuid.uuid4() + mocker.patch("uuid.uuid4", return_value=transfer_uuid) + + # Set the SHARED_DIRECTORY setting based on the sip_path fixture. + shared_directory = sip_path.parent.parent + settings.SHARED_DIRECTORY = shared_directory.as_posix() + + # Create a directory with the same transfer name under active transfers. + active_transfers_path = ( + shared_directory / "watchedDirectories" / "activeTransfers" / "standardTransfer" + ) + (active_transfers_path / f"mytransfer-{transfer_uuid}").mkdir() + + response = admin_client.post( + reverse("api:transfer_reingest", kwargs={"target": "transfer"}), + {"name": sip_path.name, "uuid": sip_path.name[-36:]}, + ) + + assert response.status_code == 400 + payload = json.loads(response.content.decode("utf8")) + assert payload == { + "error": True, + "message": "There is already a transfer in standardTransfer with the same name.", + } + + +@pytest.mark.django_db +def test_reingest_partial(sip_path, settings, admin_client): + helpers.set_setting("dashboard_uuid", "test-uuid") + + # Set the SHARED_DIRECTORY setting based on the sip_path fixture. + shared_directory = sip_path.parent.parent + settings.SHARED_DIRECTORY = shared_directory.as_posix() + + # A partial reingest reuses the SIP UUID in the response. + reingest_uuid = sip_path.name[-36:] + + response = admin_client.post( + reverse("api:ingest_reingest", kwargs={"target": "ingest"}), + {"name": sip_path.name, "uuid": reingest_uuid}, + ) + assert response.status_code == 200 + + # Verify the payload contains the reingest UUID. + payload = json.loads(response.content.decode("utf8")) + assert payload == { + "message": "Approval successful.", + "reingest_uuid": reingest_uuid, + } + + # Verify the original content was moved to the reingest directory. + reingests_path = shared_directory / "watchedDirectories" / "system" / "reingestAIP" + assert [e.name for e in reingests_path.iterdir()] == [sip_path.name] + assert (reingests_path / sip_path.name / "myfile.txt").read_text() == "my file" + + +@pytest.mark.django_db +def test_fetch_levels_of_description_from_atom(admin_client, mocker): + helpers.set_setting("dashboard_uuid", "test-uuid") + + # Set up the AtoM settings used on the Administration tab. + DashboardSetting.objects.create( + name="upload-qubit_v0.0", + value=str( + { + "url": "http://example.com", + "email": "demo@example.com", + "password": "password", + } + ), + ) + + # Simulate interaction with AtoM. + lods = ["Series", "Subseries", "File"] + mocker.patch( + "requests.get", + side_effect=[ + mocker.Mock( + **{ + "status_code": 200, + "json.return_value": [{"name": lod} for lod in lods], + }, + spec=requests.Response, + ) + ], + ) + + # Add existing LODs before calling the endpoint. + LevelOfDescription.objects.create(name="One existing", sortorder=1) + LevelOfDescription.objects.create(name="Another existing", sortorder=1) + + response = admin_client.get(reverse("api:fetch_atom_lods")) + assert response.status_code == 200 + + # Verify the initial LODS were deleted and we only have the retrieved ones. + result = [] + for lod in json.loads(response.content.decode("utf8")): + # LODs are represented as [{"8263fd14-2488-49f7-ac9d-fcfd02b524f0": "Series"}, ...] + name = list(lod.values())[0] + result.append(name) + assert result == lods + assert set(LevelOfDescription.objects.values_list("name")) == { + (lod,) for lod in lods + } + + +@pytest.mark.django_db +def test_fetch_levels_of_description_from_atom_communication_failure( + admin_client, mocker +): + helpers.set_setting("dashboard_uuid", "test-uuid") + + # Set up the AtoM settings used on the Administration tab. + DashboardSetting.objects.create( + name="upload-qubit_v0.0", + value=str( + { + "url": "http://example.com", + "email": "demo@example.com", + "password": "password", + } + ), + ) + + # Simulate failing interaction with AtoM. + mocker.patch( + "requests.get", + side_effect=[mocker.Mock(status_code=503, spec=requests.Response)], + ) + + response = admin_client.get(reverse("api:fetch_atom_lods")) + + assert response.status_code == 500 + payload = json.loads(response.content.decode("utf8")) + assert payload == { + "success": False, + "error": "Unable to fetch levels of description from AtoM!", + } diff --git a/src/dashboard/tests/test_filesystem_ajax.py b/src/dashboard/tests/test_filesystem_ajax.py index 6d87dc5a2a..a6eb3ec035 100644 --- a/src/dashboard/tests/test_filesystem_ajax.py +++ b/src/dashboard/tests/test_filesystem_ajax.py @@ -691,4 +691,4 @@ def test_copy_within_arrange(mocker, admin_client): # Verify SIPArrange instances were created as expected. assert models.SIPArrange.objects.count() == 2 - assert set(list(models.SIPArrange.objects.values_list(*attrs))) == expected + assert set(models.SIPArrange.objects.values_list(*attrs)) == expected diff --git a/src/dashboard/tests/test_transfer.py b/src/dashboard/tests/test_transfer.py index 310e7449fc..a7fce448b8 100644 --- a/src/dashboard/tests/test_transfer.py +++ b/src/dashboard/tests/test_transfer.py @@ -85,9 +85,7 @@ def test_component_post(admin_client): assert "Metadata saved." in response.content.decode() assert set( - list( - TransferMetadataFieldValue.objects.filter( - set=transfer_uuid, field__fieldname__in=["media_format", "media_number"] - ).values_list("fieldvalue") - ) + TransferMetadataFieldValue.objects.filter( + set=transfer_uuid, field__fieldname__in=["media_format", "media_number"] + ).values_list("fieldvalue") ) == {('3.5" floppy',), ("123",)} diff --git a/tox.ini b/tox.ini index e8facf4f3b..0c0187c133 100644 --- a/tox.ini +++ b/tox.ini @@ -84,6 +84,17 @@ commands = pre-commit run --all-files --show-diff-on-failure [flake8] exclude = .tox, .git, __pycache__, .cache, build, dist, *.pyc, *.egg-info, .eggs -application-import-names = flake8 -select = C, E, F, W, B, B950 -ignore = E203, E402, E501, E722, W503, W605 +# Error codes: +# - https://flake8.pycqa.org/en/latest/user/error-codes.html +# - https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes +# - https://github.com/PyCQA/flake8-bugbear#list-of-warnings +# +# E203: whitespace before ‘,’, ‘;’, or ‘:’ +# E402: module level import not at top of file +# E501: line too long +# W503: line break before binary operator +ignore = + E203, + E402, + E501, + W503