From c19b805a6884722df372c53949dd7b3d619b26f0 Mon Sep 17 00:00:00 2001 From: Douglas Cerna Date: Thu, 14 Mar 2024 12:02:35 -0600 Subject: [PATCH] Update build process * Add pre-commit configuration * Add lint CI job * Add linkcheck to test CI job * Lint Markdown and YAML files * Fix linting errors --- .github/workflows/test.yml | 14 ++++++++++-- .pre-commit-config.yaml | 13 +++++++++++ .yamllint.yaml | 4 ++++ administrators.rst | 35 +++++++++++++++-------------- conf.py | 6 +++++ fixity.rst | 2 +- install.rst | 2 +- management.rst | 46 +++++++++++++++++++------------------- migration-sqlite-mysql.rst | 2 +- recovery.rst | 4 ++-- 10 files changed, 81 insertions(+), 47 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 .yamllint.yaml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 91a1321..c6a67c4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,9 +9,9 @@ jobs: runs-on: "ubuntu-22.04" steps: - name: "Check out repository" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Set up Python" - uses: "actions/setup-python@v4" + uses: "actions/setup-python@v5" with: python-version: "3.10" - name: "Get pip cache dir" @@ -32,3 +32,13 @@ jobs: - name: "Run build test" run: | make test + - name: "Linkcheck" + run: | + make linkcheck + lint: + name: "Linting" + runs-on: "ubuntu-22.04" + steps: + - uses: "actions/checkout@v4" + - uses: "actions/setup-python@v5" + - uses: "pre-commit/action@v3.0.1" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..40eee08 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +repos: +- repo: "https://github.com/sphinx-contrib/sphinx-lint" + rev: "v0.9.1" + hooks: + - id: "sphinx-lint" +- repo: "https://github.com/igorshubovych/markdownlint-cli" + rev: "v0.39.0" + hooks: + - id: "markdownlint" +- repo: "https://github.com/adrienverge/yamllint" + rev: "v1.34.0" + hooks: + - id: "yamllint" diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..a8c0c07 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,4 @@ +--- +rules: + line-length: "disable" + quoted-strings: "enable" diff --git a/administrators.rst b/administrators.rst index 4805a07..d7edc09 100644 --- a/administrators.rst +++ b/administrators.rst @@ -39,10 +39,11 @@ Storage Service can be used to configure multiple Archivematica pipelines. Spaces ------ -A space models a specific storage device. That device might be a locally- -accessible disk, a network share, or a remote system accessible via a protocol -like FEDORA, SWIFT, DuraCloud, or LOCKSS. The space provides the Storage Service -with configuration to read and/or write data stored within itself. +A space models a specific storage device. That device might be a +locally-accessible disk, a network share, or a remote system accessible via a +protocol like FEDORA, SWIFT, DuraCloud, or LOCKSS. The space provides the +Storage Service with configuration to read and/or write data stored within +itself. Packages are not stored directly inside a space; instead, packages are stored within locations, which are organized subdivisions of a space. @@ -462,7 +463,7 @@ Fedora via SWORD2 Fedora via SWORD2 is currently supported in the Storage Service as an Access Protocol to facilitate use of the -`Archidora plugin `_, +`Archidora plugin `_, which allows ingest of material from Islandora to Archivematica. This workflow is in beta testing as of Storage Service 0.9/Archivematica 1.5/Islandora 7.x-1.6. @@ -791,7 +792,7 @@ Fields: its locations:: botocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the ListObjects operation: Access Denied - ERROR 2024-03-04 12:56:10 django.request:log:log_response:241: Internal Server Error: /api/v2/location/34664de6-025f-40a2-87f5-8720ce51169d/browse + ERROR 2024-03-04 12:56:10 django.request:log:log_response:241: Internal Server Error: /api/v2/location/34664de6-025f-40a2-87f5-8720ce51169d/browse *Read permissions* @@ -804,7 +805,7 @@ Fields: package or extract a file in it:: botocore.exceptions.ClientError: An error occurred (403) when calling the HeadObject operation: Forbidden - ERROR 2024-03-04 12:59:36 django.request:log:log_response:241: Internal Server Error: /api/v2/file/bf96a036-2631-4fb6-bcdb-781d7690163e/download/ + ERROR 2024-03-04 12:59:36 django.request:log:log_response:241: Internal Server Error: /api/v2/file/bf96a036-2631-4fb6-bcdb-781d7690163e/download/ botocore.exceptions.ClientError: An error occurred (403) when calling the HeadObject operation: Forbidden ERROR 2024-03-04 12:59:19 django.request:log:log_response:241: Internal Server Error: /api/v2/file/bf96a036-2631-4fb6-bcdb-781d7690163e/extract_file/ @@ -817,7 +818,7 @@ Fields: to store the package:: locations.models.StorageException: An error occurred (AccessDenied) when calling the GetBucketLocation operation: Access Denied - ERROR 2024-03-04 13:03:04 django.request:log:log_response:241: Internal Server Error: /api/v2/file/ + ERROR 2024-03-04 13:03:04 django.request:log:log_response:241: Internal Server Error: /api/v2/file/ *Write permissions* @@ -829,7 +830,7 @@ Fields: to store the package:: botocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the PutObject operation: Access Denied - ERROR 2024-03-04 13:06:26 django.request:log:log_response:241: Internal Server Error: /api/v2/file/ + ERROR 2024-03-04 13:06:26 django.request:log:log_response:241: Internal Server Error: /api/v2/file/ `s3:DeleteObject`_ @@ -840,7 +841,7 @@ Fields: Missing this permission raises this exception when the Storage Service tries to delete a package:: - ERROR 2024-03-05 10:20:41 django.request:log:log_response:241: Internal Server Error: /packages/package_delete_request/ + ERROR 2024-03-05 10:20:41 django.request:log:log_response:241: Internal Server Error: /packages/package_delete_request/ Traceback (most recent call last): ... botocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the DeleteObject operation: Access Denied @@ -856,7 +857,7 @@ Fields: exception when the Storage Service tries to store the package:: botocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the CreateBucket operation: Access Denied - ERROR 2024-03-05 08:11:34 django.request:log:log_response:241: Internal Server Error: /api/v2/file/ + ERROR 2024-03-05 08:11:34 django.request:log:log_response:241: Internal Server Error: /api/v2/file/ In Amazon Web Services these permissions can be granted through IAM policies attached to the user or its groups. For example, a policy that grants these @@ -1427,17 +1428,17 @@ platform (Transifex). :ref:`Back to the top ` .. _`API documentation`: https://wiki.archivematica.org/Storage_Service_API#AIP_storage_callback_request -.. _`LOCKSS`: http://www.lockss.org/ -.. _`Dataverse api guide`: http://guides.dataverse.org/en/latest/api/search.html -.. _`DSpace 5 REST API documentation`: https://wiki.duraspace.org/display/DSDOC5x/REST+API -.. _`DSpace 6 REST API documentation`: https://wiki.duraspace.org/display/DSDOC6x/REST+API#RESTAPI-Index/Authentication +.. _`LOCKSS`: https://www.lockss.org/ +.. _`Dataverse api guide`: https://guides.dataverse.org/en/latest/api/search.html +.. _`DSpace 5 REST API documentation`: https://wiki.lyrasis.org/display/DSDOC5x/REST+API +.. _`DSpace 6 REST API documentation`: https://wiki.lyrasis.org/display/DSDOC6x/REST+API#RESTAPI-Index/Authentication .. _`SCOPE documentation`: https://github.com/CCA-Public/scope .. _`Swift documentation for large objects`: https://docs.openstack.org/swift/latest/overview_large_objects.html .. _`Boto3`: https://github.com/boto/boto3 -.. _`installation README.md`: https://github.com/artefactual/archivematica-storage-service/tree/stable/0.21.x/install#logging-configuration +.. _`installation README.md`: https://github.com/artefactual/archivematica-storage-service/blob/stable/0.21.x/install/README.md#logging-configuration .. _`logging configuration`: https://github.com/artefactual/archivematica-storage-service/blob/e68825db8819aceaa426a6066d612e810bf52ddd/install/storageService.logging.json .. _`Boto3 developers`: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/boto3.html#boto3.set_stream_logger -.. _`rclone`: http://rclone.org +.. _`rclone`: https://rclone.org .. _`rclone documentation on configuration via environment variables`: https://rclone.org/docs/#environment-variables .. _`s3:ListBucket`: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html .. _`s3:GetObject`: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html diff --git a/conf.py b/conf.py index e5fcb7d..03758d6 100644 --- a/conf.py +++ b/conf.py @@ -277,3 +277,9 @@ # How to display URL addresses: 'footnote', 'no', or 'inline'. # texinfo_show_urls = 'footnote' + +# -- Options for linkcheck ------------------------------------ + +linkcheck_ignore = [ + r"([(http:)(https:)]*(\/\/)[\S]*(.)*(github.com)([\s\S])*((.md)[#][\w-]*))", +] diff --git a/fixity.rst b/fixity.rst index 6949c6d..2a4b32a 100644 --- a/fixity.rst +++ b/fixity.rst @@ -87,7 +87,7 @@ changed. :ref:`Return to the top ` .. _Archivematica AIP: https://www.archivematica.org/en/docs/archivematica-1.9/user-manual/archival-storage/aip-structure/#bagit-documentation -.. _BagIt File Packaging Format: https://tools.ietf.org/html/rfc8493 +.. _BagIt File Packaging Format: https://datatracker.ietf.org/doc/html/rfc8493 .. _API endpoint: https://wiki.archivematica.org/Storage_Service_API#Check_fixity .. _Fixity: https://github.com/artefactual/fixity .. _Digital Preservation Handbook: https://www.dpconline.org/handbook/technical-solutions-and-tools/fixity-and-checksums diff --git a/install.rst b/install.rst index 38fa327..748424c 100644 --- a/install.rst +++ b/install.rst @@ -61,4 +61,4 @@ you use `the Docker Compose based development environment`_. :ref:`Back to the top ` .. _`deadsnakes ppa`: https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa -.. _`the Docker Compose based development environment`: https://github.com/artefactual/archivematica/tree/qa/1.x/hack#archivematica-development-on-docker-compose +.. _`the Docker Compose based development environment`: https://github.com/artefactual/archivematica/blob/qa/1.x/hack/README.md diff --git a/management.rst b/management.rst index 355167e..523c679 100644 --- a/management.rst +++ b/management.rst @@ -11,28 +11,27 @@ A full list of commands can be retrieved as follows: .. code:: bash - sudo -u archivematica bash -c " \ - set -a -e -x - source /etc/default/archivematica-storage-service || \ - source /etc/sysconfig/archivematica-storage-service \ - || (echo 'Environment file not found'; exit 1) - cd /usr/lib/archivematica/storage-service - /usr/share/archivematica/virtualenvs/archivematica-storage-service/bin/python manage.py help - "; + sudo -u archivematica bash -c " \ + set -a -e -x + source /etc/default/archivematica-storage-service || \ + source /etc/sysconfig/archivematica-storage-service \ + || (echo 'Environment file not found'; exit 1) + cd /usr/lib/archivematica/storage-service + /usr/share/archivematica/virtualenvs/archivematica-storage-service/bin/python manage.py help + "; If you want to see the help message of a specific command, try: .. code:: bash - sudo -u archivematica bash -c " \ - set -a -e -x - source /etc/default/archivematica-storage-service || \ - source /etc/sysconfig/archivematica-storage-service \ - || (echo 'Environment file not found'; exit 1) - cd /usr/lib/archivematica/storage-service - /usr/share/archivematica/virtualenvs/archivematica-storage-service/bin/python - manage.py help create_aip_replicas - "; + sudo -u archivematica bash -c " \ + set -a -e -x + source /etc/default/archivematica-storage-service || \ + source /etc/sysconfig/archivematica-storage-service \ + || (echo 'Environment file not found'; exit 1) + cd /usr/lib/archivematica/storage-service + /usr/share/archivematica/virtualenvs/archivematica-storage-service/bin/python manage.py help create_aip_replicas + "; We've looked up the command for :ref:`creating new AIP replicas ` above. @@ -56,12 +55,13 @@ Replicator locations. .. code:: bash sudo -u archivematica bash -c " \ - set -a -e -x - source /etc/default/archivematica-storage-service || \ - source /etc/sysconfig/archivematica-storage-service \ - || (echo 'Environment file not found'; exit 1) - cd /usr/lib/archivematica/storage-service - /usr/share/archivematica/virtualenvs/archivematica-storage-service/bin/python manage.py create_aip_replicas + set -a -e -x + source /etc/default/archivematica-storage-service || \ + source /etc/sysconfig/archivematica-storage-service \ + || (echo 'Environment file not found'; exit 1) + cd /usr/lib/archivematica/storage-service + /usr/share/archivematica/virtualenvs/archivematica-storage-service/bin/python manage.py create_aip_replicas + "; The command accepts the following parameters: diff --git a/migration-sqlite-mysql.rst b/migration-sqlite-mysql.rst index 119bf11..4b7e824 100644 --- a/migration-sqlite-mysql.rst +++ b/migration-sqlite-mysql.rst @@ -76,4 +76,4 @@ If you use Ansible, the Archivematica role can perform the migration automatically. Please follow the `instructions`_. .. _issue 952: https://github.com/archivematica/Issues/issues/952 -.. _instructions: https://github.com/artefactual-labs/ansible-archivematica-src#migration-to-mysql-in-storage-service +.. _instructions: https://github.com/artefactual-labs/ansible-archivematica-src/blob/stable/1.15.x/README.md#migration-to-mysql-in-storage-service diff --git a/recovery.rst b/recovery.rst index 0eef1d6..51bd31e 100644 --- a/recovery.rst +++ b/recovery.rst @@ -112,5 +112,5 @@ and password into the two fields below it. Click **Save** when you are done. :ref:`Back to the top ` -.. _Binder: http://binder.readthedocs.org/en/latest/ -.. _Binder recovery documentation: https://github.com/artefactual/binder#storage-service-aip-recovery-process-and-configuration +.. _Binder: https://binder.readthedocs.io/en/latest/ +.. _Binder recovery documentation: https://github.com/artefactual/binder/blob/qa/0.9.x/README.md#storage-service-aip-recovery-process-and-configuration