From e9e42a268d37cf452b05ce4a697351431af9dde0 Mon Sep 17 00:00:00 2001 From: "Douglas Cerna (Soy Douglas)" Date: Sun, 28 Apr 2024 00:29:49 +0200 Subject: [PATCH] Use podman-compose --- .github/workflows/dip-upload.yml | 60 +++++++++++++------------ tests/dip-upload/README.md | 58 ++++++++---------------- tests/dip-upload/archivematica-vars.yml | 11 ++--- tests/dip-upload/compose.yaml | 23 ++++++++++ tests/dip-upload/requirements.txt | 1 + 5 files changed, 78 insertions(+), 75 deletions(-) create mode 100644 tests/dip-upload/compose.yaml diff --git a/.github/workflows/dip-upload.yml b/.github/workflows/dip-upload.yml index 7b5d82d6..3e9b318b 100644 --- a/.github/workflows/dip-upload.yml +++ b/.github/workflows/dip-upload.yml @@ -20,13 +20,14 @@ on: jobs: test: name: "DIP upload test" - runs-on: "ubuntu-latest" + runs-on: "ubuntu-22.04" env: am_version: "${{ inputs.am_version }}" ss_version: "${{ inputs.ss_version }}" atom_version: "${{ inputs.atom_version }}" + python_version: "3.9" steps: - - name: "Check out code" + - name: "Check out the code" uses: "actions/checkout@v4" - name: "Upgrade crun (supports Ubuntu's systemd in the Dockerfile)" run: | @@ -36,33 +37,42 @@ jobs: - name: "Install Python" uses: "actions/setup-python@v5" with: - python-version: "3.9" + python-version: "${{ env.python_version }}" cache: "pip" cache-dependency-path: | tests/dip-upload/requirements.txt - - name: "Install ansible" + - name: "Cache the virtual environment" + id: "venv-cache" + uses: "actions/cache@v4" + with: + path: | + tests/dip-upload/.venv/ + key: "os-${{ runner.os }}-python_version-${{ env.python_version }}-hash-${{ hashFiles('tests/dip-upload/requirements.txt') }}" + - name: "Set up the virtual environment" + if: "steps.venv-cache.outputs.cache-hit == false" working-directory: "${{ github.workspace }}/tests/dip-upload" run: | python3 -m venv .venv - source .venv/bin/activate - python3 -m pip install -r requirements.txt - - name: "Generate a SSH key and copy it next to the Dockerfile" + .venv/bin/python -m pip install -r requirements.txt + - name: "Add virtual environment to PATH" + working-directory: "${{ github.workspace }}/tests/dip-upload" + run: + echo "$PWD/.venv/bin" >> $GITHUB_PATH + - name: "Generate an SSH key and copy it next to the Dockerfile" run: | mkdir $HOME/.ssh ssh-keygen -t rsa -f $HOME/.ssh/id_rsa -N "" cp $HOME/.ssh/id_rsa.pub ${{ github.workspace }}/tests/dip-upload/ssh_pub_key - - name: "Start the Archivematica VM" + - name: "Start the Compose environment" working-directory: "${{ github.workspace }}/tests/dip-upload" run: | - podman build -t vm-am --build-arg UBUNTU_VERSION=22.04 . - podman run --rm --detach --hostname archivematica --network=bridge -p "2222:22,8000:80,8001:8000" vm-am + podman-compose up --detach - name: "Install Archivematica" working-directory: "${{ github.workspace }}/tests/dip-upload" env: ANSIBLE_HOST_KEY_CHECKING: "False" ANSIBLE_REMOTE_PORT: 2222 run: | - source .venv/bin/activate ansible-galaxy install -f -p roles/ -r requirements.yml ansible-playbook -i localhost, archivematica.yml \ -u ubuntu \ @@ -71,24 +81,18 @@ jobs: - name: "Add the ubuntu user to the archivematica group" working-directory: "${{ github.workspace }}/tests/dip-upload" run: | - ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost 'sudo usermod -a -G archivematica ubuntu' + podman-compose exec --user root archivematica usermod -a -G archivematica ubuntu - name: "Get the archivematica SSH public key" id: archivematica_ssh_pub_key working-directory: "${{ github.workspace }}/tests/dip-upload" run: | - echo "key=$(ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost 'sudo cat /var/lib/archivematica/.ssh/id_rsa.pub')" >> $GITHUB_OUTPUT - - name: "Start the AtoM VM" - working-directory: "${{ github.workspace }}/tests/dip-upload" - run: | - podman build -t vm-atom --build-arg UBUNTU_VERSION=20.04 . - podman run --rm --detach --hostname atom --network=bridge -p "9222:22,9000:80,9001:8000" vm-atom + echo "key=$(podman-compose exec --user archivematica archivematica cat /var/lib/archivematica/.ssh/id_rsa.pub)" >> $GITHUB_OUTPUT - name: "Install AtoM" working-directory: "${{ github.workspace }}/tests/dip-upload" env: ANSIBLE_HOST_KEY_CHECKING: "False" ANSIBLE_REMOTE_PORT: 9222 run: | - source .venv/bin/activate ansible-playbook -i localhost, atom.yml \ -u ubuntu \ -e "atom_repository_version=${{ env.atom_version }} archivematica_ssh_pub_key='${{ steps.archivematica_ssh_pub_key.outputs.key }}'" \ @@ -111,21 +115,21 @@ jobs: - name: "Create a processing configuration for DIP upload" working-directory: "${{ github.workspace }}/tests/dip-upload" run: | - ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost "sudo -u archivematica cp /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/automatedProcessingMCP.xml /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml" + podman-compose exec --user archivematica archivematica cp /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/automatedProcessingMCP.xml /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml - name: "Update the DIP upload processing configuration" working-directory: "${{ github.workspace }}/tests/dip-upload" run: | # Change 'Normalize for preservation' to 'Normalize for preservation and access' - ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost "sudo -u archivematica sed --in-place 's|612e3609-ce9a-4df6-a9a3-63d634d2d934|b93cecd4-71f2-4e28-bc39-d32fd62c5a94|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml" + podman-compose exec --user archivematica archivematica sed --in-place 's|612e3609-ce9a-4df6-a9a3-63d634d2d934|b93cecd4-71f2-4e28-bc39-d32fd62c5a94|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml # Change 'Do not upload DIP' to 'Upload DIP to AtoM/Binder' - ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost "sudo -u archivematica sed --in-place 's|6eb8ebe7-fab3-4e4c-b9d7-14de17625baa|0fe9842f-9519-4067-a691-8a363132ae24|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml" - - name: "Import Atom sample data" + podman-compose exec --user archivematica archivematica sed --in-place 's|6eb8ebe7-fab3-4e4c-b9d7-14de17625baa|0fe9842f-9519-4067-a691-8a363132ae24|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml + - name: "Import the Atom sample data" working-directory: "${{ github.workspace }}/tests/dip-upload" run: | - ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony csv:import /usr/share/nginx/atom/lib/task/import/example/isad/example_information_objects_isad.csv" - ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony propel:build-nested-set" - ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony cc" - ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony search:populate" + podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony csv:import /usr/share/nginx/atom/lib/task/import/example/isad/example_information_objects_isad.csv + podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony propel:build-nested-set + podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony cc + podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony search:populate - name: "Start a transfer and upload the DIP to the sample archival description" run: | curl \ @@ -141,7 +145,7 @@ jobs: http://localhost:8000/api/v2beta/package - name: "Wait for the transfer to finish" run: | - sleep 180 + sleep 120 - name: "Verify a digital object was uploaded and attached to the sample archival description" run: | curl \ diff --git a/tests/dip-upload/README.md b/tests/dip-upload/README.md index 9e0b1cdb..b04a96cd 100644 --- a/tests/dip-upload/README.md +++ b/tests/dip-upload/README.md @@ -29,25 +29,21 @@ Install the playbook requirements: ansible-galaxy install -f -p roles/ -r requirements.yml ``` -## Provisioning the Archivematica container +## Starting the Compose environment -Copy your SSH public key as the `ssh_pub_key` file next to the `Dockerfile`: +Copy your SSH public key as the `ssh_pub_key` file next to the `Containerfile`: ```shell cp $HOME/.ssh/id_rsa.pub ssh_pub_key ``` -Build the Dockerfile passing the `UBUNTU_VERSION=22.04` build argument: +Start the Compose services: ```shell -podman build -t vm-am --build-arg UBUNTU_VERSION=22.04 . +podman-compose up --detach ``` -Start the container mapping the exposed ports: - -```shell -podman run --rm --detach --hostname archivematica --network=bridge -p "2222:22,8000:80,8001:8000" vm-am -``` +## Installing Archivematica Run the Archivematica installation playbook: @@ -63,35 +59,17 @@ Add the `ubuntu` user to the `archivematica` group so it can copy AIPs from the shared directory: ```shell -ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost 'sudo usermod -a -G archivematica ubuntu' +podman-compose exec --user root archivematica usermod -a -G archivematica ubuntu ``` Get the SSH public key of the `archivematica` user so we can use it when -provisioning the AtoM VM: - -```shell -AM_SSH_PUB_KEY=$(ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost 'sudo cat /var/lib/archivematica/.ssh/id_rsa.pub') -``` - -## Provisioning the AtoM container - -Copy your SSH public key as the `ssh_pub_key` file next to the `Dockerfile`: +installing AtoM: ```shell -cp $HOME/.ssh/id_rsa.pub ssh_pub_key -``` - -Build the Dockerfile passing the `UBUNTU_VERSION=20.04` build argument: - -```shell -podman build -t vm-atom --build-arg UBUNTU_VERSION=20.04 . +AM_SSH_PUB_KEY=$(podman-compose exec --user archivematica archivematica cat /var/lib/archivematica/.ssh/id_rsa.pub) ``` -Start the container mapping the exposed ports: - -```shell -podman run --rm --detach --hostname atom --network=bridge -p "9222:22,9000:80,9001:8000" vm-atom -``` +## Installing AtoM Run the AtoM installation playbook passing the `archivematica_ssh_pub_key` variable with the contents of `$AM_SSH_PUB_KEY`: @@ -132,25 +110,25 @@ curl --header "REST-API-Key: this_is_the_atom_dip_upload_api_key" http://localho Create a processing configuration for DIP upload: ```shell -ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost "sudo -u archivematica cp /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/automatedProcessingMCP.xml /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml" +podman-compose exec --user archivematica archivematica cp /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/automatedProcessingMCP.xml /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml ``` Update the DIP upload processing configuration: ```shell # Change 'Normalize for preservation' to 'Normalize for preservation and access' -ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost "sudo -u archivematica sed --in-place 's|612e3609-ce9a-4df6-a9a3-63d634d2d934|b93cecd4-71f2-4e28-bc39-d32fd62c5a94|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml" +podman-compose exec --user archivematica archivematica sed --in-place 's|612e3609-ce9a-4df6-a9a3-63d634d2d934|b93cecd4-71f2-4e28-bc39-d32fd62c5a94|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml # Change 'Do not upload DIP' to 'Upload DIP to AtoM/Binder' -ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost "sudo -u archivematica sed --in-place 's|6eb8ebe7-fab3-4e4c-b9d7-14de17625baa|0fe9842f-9519-4067-a691-8a363132ae24|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml" +podman-compose exec --user archivematica archivematica sed --in-place 's|6eb8ebe7-fab3-4e4c-b9d7-14de17625baa|0fe9842f-9519-4067-a691-8a363132ae24|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml ``` -Import Atom sample data: +Import the Atom sample data: ```shell -ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony csv:import /usr/share/nginx/atom/lib/task/import/example/isad/example_information_objects_isad.csv" -ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony propel:build-nested-set" -ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony cc" -ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony search:populate" +podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony csv:import /usr/share/nginx/atom/lib/task/import/example/isad/example_information_objects_isad.csv +podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony propel:build-nested-set +podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony cc +podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony search:populate ``` Start a transfer and upload the DIP to the sample archival description: @@ -172,7 +150,7 @@ curl \ Wait for the transfer to finish: ```shell -sleep 180 +sleep 120 ``` Verify a digital object was uploaded and attached to the sample archival description: diff --git a/tests/dip-upload/archivematica-vars.yml b/tests/dip-upload/archivematica-vars.yml index e0fb764b..62992b4f 100644 --- a/tests/dip-upload/archivematica-vars.yml +++ b/tests/dip-upload/archivematica-vars.yml @@ -9,12 +9,12 @@ archivematica_src_configure_am_user: "admin" archivematica_src_configure_am_password: "archivematica" archivematica_src_configure_am_email: "admin@example.com" archivematica_src_configure_am_api_key: "this_is_the_am_api_key" -archivematica_src_configure_am_site_url: "http://localhost" +archivematica_src_configure_am_site_url: "http://archivematica" archivematica_src_configure_ss_user: "admin" archivematica_src_configure_ss_password: "archivematica" archivematica_src_configure_ss_api_key: "this_is_the_ss_api_key" -archivematica_src_configure_ss_url: "http://localhost:8000" +archivematica_src_configure_ss_url: "http://archivematica:8000" archivematica_src_configure_ss_email: "admin@example.com" archivematica_src_am_db_password: "aaGKHyMls.20ki$" @@ -62,9 +62,6 @@ archivematica_src_ss_environment: archivematica_src_am_mcpclient_environment: ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_METADATA_XML_VALIDATION_ENABLED: "true" METADATA_XML_VALIDATION_SETTINGS_FILE: "/home/{{ ansible_user_id }}/archivematica-sampledata/xml-validation/xml_validation.py" - # This allows the rsync command from the Archivematica container to connect - # to the non-standard port in the AtoM container. - RSYNC_RSH: "ssh -p 9222" # Disable FITS @@ -88,8 +85,8 @@ archivematica_src_configure_fprule: # Instead, these custom settings are populated in the post tasks of the # Archivematica provisioning playbook. custom_archivematica_src_configure_dashboardsettings: - url: "http://host.containers.internal:9000" - rsync_target: "host.containers.internal:/home/archivematica/atom_sword_deposit" + url: "http://atom" + rsync_target: "atom:/home/archivematica/atom_sword_deposit" email: "dip_upload@example.com" password: "dip_upload@example.com" key: "this_is_the_atom_dip_upload_api_key" diff --git a/tests/dip-upload/compose.yaml b/tests/dip-upload/compose.yaml new file mode 100644 index 00000000..08e2c08d --- /dev/null +++ b/tests/dip-upload/compose.yaml @@ -0,0 +1,23 @@ +--- +name: dip-upload-test + +services: + + archivematica: + build: + args: + UBUNTU_VERSION: "22.04" + ports: + - "2222:22" + - "8000:80" + - "8001:8000" + + atom: + build: + args: + UBUNTU_VERSION: "20.04" + ports: + - "9222:22" + - "9000:80" + links: + - "archivematica" diff --git a/tests/dip-upload/requirements.txt b/tests/dip-upload/requirements.txt index 03a81f30..7b2ecd66 100644 --- a/tests/dip-upload/requirements.txt +++ b/tests/dip-upload/requirements.txt @@ -1,2 +1,3 @@ ansible==8.5.0 ansible-core==2.15.5 +git+https://github.com/containers/podman-compose.git@2681566580b4eaadfc5e6000ad19e49e56006e2b#egg=podman-compose