Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use podman-compose
Browse files Browse the repository at this point in the history
replaceafill committed Apr 28, 2024
1 parent 4cb580b commit e9e42a2
Showing 5 changed files with 78 additions and 75 deletions.
60 changes: 32 additions & 28 deletions .github/workflows/dip-upload.yml
Original file line number Diff line number Diff line change
@@ -20,13 +20,14 @@ on:
jobs:
test:
name: "DIP upload test"
runs-on: "ubuntu-latest"
runs-on: "ubuntu-22.04"
env:
am_version: "${{ inputs.am_version }}"
ss_version: "${{ inputs.ss_version }}"
atom_version: "${{ inputs.atom_version }}"
python_version: "3.9"
steps:
- name: "Check out code"
- name: "Check out the code"
uses: "actions/checkout@v4"
- name: "Upgrade crun (supports Ubuntu's systemd in the Dockerfile)"
run: |
@@ -36,33 +37,42 @@ jobs:
- name: "Install Python"
uses: "actions/setup-python@v5"
with:
python-version: "3.9"
python-version: "${{ env.python_version }}"
cache: "pip"
cache-dependency-path: |
tests/dip-upload/requirements.txt
- name: "Install ansible"
- name: "Cache the virtual environment"
id: "venv-cache"
uses: "actions/cache@v4"
with:
path: |
tests/dip-upload/.venv/
key: "os-${{ runner.os }}-python_version-${{ env.python_version }}-hash-${{ hashFiles('tests/dip-upload/requirements.txt') }}"
- name: "Set up the virtual environment"
if: "steps.venv-cache.outputs.cache-hit == false"
working-directory: "${{ github.workspace }}/tests/dip-upload"
run: |
python3 -m venv .venv
source .venv/bin/activate
python3 -m pip install -r requirements.txt
- name: "Generate a SSH key and copy it next to the Dockerfile"
.venv/bin/python -m pip install -r requirements.txt
- name: "Add virtual environment to PATH"
working-directory: "${{ github.workspace }}/tests/dip-upload"
run:
echo "$PWD/.venv/bin" >> $GITHUB_PATH
- name: "Generate an SSH key and copy it next to the Dockerfile"
run: |
mkdir $HOME/.ssh
ssh-keygen -t rsa -f $HOME/.ssh/id_rsa -N ""
cp $HOME/.ssh/id_rsa.pub ${{ github.workspace }}/tests/dip-upload/ssh_pub_key
- name: "Start the Archivematica VM"
- name: "Start the Compose environment"
working-directory: "${{ github.workspace }}/tests/dip-upload"
run: |
podman build -t vm-am --build-arg UBUNTU_VERSION=22.04 .
podman run --rm --detach --hostname archivematica --network=bridge -p "2222:22,8000:80,8001:8000" vm-am
podman-compose up --detach
- name: "Install Archivematica"
working-directory: "${{ github.workspace }}/tests/dip-upload"
env:
ANSIBLE_HOST_KEY_CHECKING: "False"
ANSIBLE_REMOTE_PORT: 2222
run: |
source .venv/bin/activate
ansible-galaxy install -f -p roles/ -r requirements.yml
ansible-playbook -i localhost, archivematica.yml \
-u ubuntu \
@@ -71,24 +81,18 @@ jobs:
- name: "Add the ubuntu user to the archivematica group"
working-directory: "${{ github.workspace }}/tests/dip-upload"
run: |
ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost 'sudo usermod -a -G archivematica ubuntu'
podman-compose exec --user root archivematica usermod -a -G archivematica ubuntu
- name: "Get the archivematica SSH public key"
id: archivematica_ssh_pub_key
working-directory: "${{ github.workspace }}/tests/dip-upload"
run: |
echo "key=$(ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost 'sudo cat /var/lib/archivematica/.ssh/id_rsa.pub')" >> $GITHUB_OUTPUT
- name: "Start the AtoM VM"
working-directory: "${{ github.workspace }}/tests/dip-upload"
run: |
podman build -t vm-atom --build-arg UBUNTU_VERSION=20.04 .
podman run --rm --detach --hostname atom --network=bridge -p "9222:22,9000:80,9001:8000" vm-atom
echo "key=$(podman-compose exec --user archivematica archivematica cat /var/lib/archivematica/.ssh/id_rsa.pub)" >> $GITHUB_OUTPUT
- name: "Install AtoM"
working-directory: "${{ github.workspace }}/tests/dip-upload"
env:
ANSIBLE_HOST_KEY_CHECKING: "False"
ANSIBLE_REMOTE_PORT: 9222
run: |
source .venv/bin/activate
ansible-playbook -i localhost, atom.yml \
-u ubuntu \
-e "atom_repository_version=${{ env.atom_version }} archivematica_ssh_pub_key='${{ steps.archivematica_ssh_pub_key.outputs.key }}'" \
@@ -111,21 +115,21 @@ jobs:
- name: "Create a processing configuration for DIP upload"
working-directory: "${{ github.workspace }}/tests/dip-upload"
run: |
ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost "sudo -u archivematica cp /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/automatedProcessingMCP.xml /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml"
podman-compose exec --user archivematica archivematica cp /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/automatedProcessingMCP.xml /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml
- name: "Update the DIP upload processing configuration"
working-directory: "${{ github.workspace }}/tests/dip-upload"
run: |
# Change 'Normalize for preservation' to 'Normalize for preservation and access'
ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost "sudo -u archivematica sed --in-place 's|612e3609-ce9a-4df6-a9a3-63d634d2d934|b93cecd4-71f2-4e28-bc39-d32fd62c5a94|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml"
podman-compose exec --user archivematica archivematica sed --in-place 's|612e3609-ce9a-4df6-a9a3-63d634d2d934|b93cecd4-71f2-4e28-bc39-d32fd62c5a94|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml
# Change 'Do not upload DIP' to 'Upload DIP to AtoM/Binder'
ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost "sudo -u archivematica sed --in-place 's|6eb8ebe7-fab3-4e4c-b9d7-14de17625baa|0fe9842f-9519-4067-a691-8a363132ae24|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml"
- name: "Import Atom sample data"
podman-compose exec --user archivematica archivematica sed --in-place 's|6eb8ebe7-fab3-4e4c-b9d7-14de17625baa|0fe9842f-9519-4067-a691-8a363132ae24|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml
- name: "Import the Atom sample data"
working-directory: "${{ github.workspace }}/tests/dip-upload"
run: |
ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony csv:import /usr/share/nginx/atom/lib/task/import/example/isad/example_information_objects_isad.csv"
ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony propel:build-nested-set"
ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony cc"
ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony search:populate"
podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony csv:import /usr/share/nginx/atom/lib/task/import/example/isad/example_information_objects_isad.csv
podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony propel:build-nested-set
podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony cc
podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony search:populate
- name: "Start a transfer and upload the DIP to the sample archival description"
run: |
curl \
@@ -141,7 +145,7 @@ jobs:
http://localhost:8000/api/v2beta/package
- name: "Wait for the transfer to finish"
run: |
sleep 180
sleep 120
- name: "Verify a digital object was uploaded and attached to the sample archival description"
run: |
curl \
58 changes: 18 additions & 40 deletions tests/dip-upload/README.md
Original file line number Diff line number Diff line change
@@ -29,25 +29,21 @@ Install the playbook requirements:
ansible-galaxy install -f -p roles/ -r requirements.yml
```

## Provisioning the Archivematica container
## Starting the Compose environment

Copy your SSH public key as the `ssh_pub_key` file next to the `Dockerfile`:
Copy your SSH public key as the `ssh_pub_key` file next to the `Containerfile`:

```shell
cp $HOME/.ssh/id_rsa.pub ssh_pub_key
```

Build the Dockerfile passing the `UBUNTU_VERSION=22.04` build argument:
Start the Compose services:

```shell
podman build -t vm-am --build-arg UBUNTU_VERSION=22.04 .
podman-compose up --detach
```

Start the container mapping the exposed ports:

```shell
podman run --rm --detach --hostname archivematica --network=bridge -p "2222:22,8000:80,8001:8000" vm-am
```
## Installing Archivematica

Run the Archivematica installation playbook:

@@ -63,35 +59,17 @@ Add the `ubuntu` user to the `archivematica` group so it can copy AIPs
from the shared directory:

```shell
ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost 'sudo usermod -a -G archivematica ubuntu'
podman-compose exec --user root archivematica usermod -a -G archivematica ubuntu
```

Get the SSH public key of the `archivematica` user so we can use it when
provisioning the AtoM VM:

```shell
AM_SSH_PUB_KEY=$(ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost 'sudo cat /var/lib/archivematica/.ssh/id_rsa.pub')
```

## Provisioning the AtoM container

Copy your SSH public key as the `ssh_pub_key` file next to the `Dockerfile`:
installing AtoM:

```shell
cp $HOME/.ssh/id_rsa.pub ssh_pub_key
```

Build the Dockerfile passing the `UBUNTU_VERSION=20.04` build argument:

```shell
podman build -t vm-atom --build-arg UBUNTU_VERSION=20.04 .
AM_SSH_PUB_KEY=$(podman-compose exec --user archivematica archivematica cat /var/lib/archivematica/.ssh/id_rsa.pub)
```

Start the container mapping the exposed ports:

```shell
podman run --rm --detach --hostname atom --network=bridge -p "9222:22,9000:80,9001:8000" vm-atom
```
## Installing AtoM

Run the AtoM installation playbook passing the `archivematica_ssh_pub_key`
variable with the contents of `$AM_SSH_PUB_KEY`:
@@ -132,25 +110,25 @@ curl --header "REST-API-Key: this_is_the_atom_dip_upload_api_key" http://localho
Create a processing configuration for DIP upload:

```shell
ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost "sudo -u archivematica cp /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/automatedProcessingMCP.xml /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml"
podman-compose exec --user archivematica archivematica cp /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/automatedProcessingMCP.xml /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml
```

Update the DIP upload processing configuration:

```shell
# Change 'Normalize for preservation' to 'Normalize for preservation and access'
ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost "sudo -u archivematica sed --in-place 's|612e3609-ce9a-4df6-a9a3-63d634d2d934|b93cecd4-71f2-4e28-bc39-d32fd62c5a94|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml"
podman-compose exec --user archivematica archivematica sed --in-place 's|612e3609-ce9a-4df6-a9a3-63d634d2d934|b93cecd4-71f2-4e28-bc39-d32fd62c5a94|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml
# Change 'Do not upload DIP' to 'Upload DIP to AtoM/Binder'
ssh -o StrictHostKeyChecking=no -p 2222 ubuntu@localhost "sudo -u archivematica sed --in-place 's|6eb8ebe7-fab3-4e4c-b9d7-14de17625baa|0fe9842f-9519-4067-a691-8a363132ae24|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml"
podman-compose exec --user archivematica archivematica sed --in-place 's|6eb8ebe7-fab3-4e4c-b9d7-14de17625baa|0fe9842f-9519-4067-a691-8a363132ae24|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml
```

Import Atom sample data:
Import the Atom sample data:

```shell
ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony csv:import /usr/share/nginx/atom/lib/task/import/example/isad/example_information_objects_isad.csv"
ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony propel:build-nested-set"
ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony cc"
ssh -o StrictHostKeyChecking=no -p 9222 ubuntu@localhost "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony search:populate"
podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony csv:import /usr/share/nginx/atom/lib/task/import/example/isad/example_information_objects_isad.csv
podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony propel:build-nested-set
podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony cc
podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony search:populate
```

Start a transfer and upload the DIP to the sample archival description:
@@ -172,7 +150,7 @@ curl \
Wait for the transfer to finish:

```shell
sleep 180
sleep 120
```

Verify a digital object was uploaded and attached to the sample archival description:
11 changes: 4 additions & 7 deletions tests/dip-upload/archivematica-vars.yml
Original file line number Diff line number Diff line change
@@ -9,12 +9,12 @@ archivematica_src_configure_am_user: "admin"
archivematica_src_configure_am_password: "archivematica"
archivematica_src_configure_am_email: "admin@example.com"
archivematica_src_configure_am_api_key: "this_is_the_am_api_key"
archivematica_src_configure_am_site_url: "http://localhost"
archivematica_src_configure_am_site_url: "http://archivematica"

archivematica_src_configure_ss_user: "admin"
archivematica_src_configure_ss_password: "archivematica"
archivematica_src_configure_ss_api_key: "this_is_the_ss_api_key"
archivematica_src_configure_ss_url: "http://localhost:8000"
archivematica_src_configure_ss_url: "http://archivematica:8000"
archivematica_src_configure_ss_email: "admin@example.com"

archivematica_src_am_db_password: "aaGKHyMls.20ki$"
@@ -62,9 +62,6 @@ archivematica_src_ss_environment:
archivematica_src_am_mcpclient_environment:
ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_METADATA_XML_VALIDATION_ENABLED: "true"
METADATA_XML_VALIDATION_SETTINGS_FILE: "/home/{{ ansible_user_id }}/archivematica-sampledata/xml-validation/xml_validation.py"
# This allows the rsync command from the Archivematica container to connect
# to the non-standard port in the AtoM container.
RSYNC_RSH: "ssh -p 9222"

# Disable FITS

@@ -88,8 +85,8 @@ archivematica_src_configure_fprule:
# Instead, these custom settings are populated in the post tasks of the
# Archivematica provisioning playbook.
custom_archivematica_src_configure_dashboardsettings:
url: "http://host.containers.internal:9000"
rsync_target: "host.containers.internal:/home/archivematica/atom_sword_deposit"
url: "http://atom"
rsync_target: "atom:/home/archivematica/atom_sword_deposit"
email: "dip_upload@example.com"
password: "dip_upload@example.com"
key: "this_is_the_atom_dip_upload_api_key"
23 changes: 23 additions & 0 deletions tests/dip-upload/compose.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
---
name: dip-upload-test

services:

archivematica:
build:
args:
UBUNTU_VERSION: "22.04"
ports:
- "2222:22"
- "8000:80"
- "8001:8000"

atom:
build:
args:
UBUNTU_VERSION: "20.04"
ports:
- "9222:22"
- "9000:80"
links:
- "archivematica"
1 change: 1 addition & 0 deletions tests/dip-upload/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
ansible==8.5.0
ansible-core==2.15.5
git+https://github.com/containers/podman-compose.git@2681566580b4eaadfc5e6000ad19e49e56006e2b#egg=podman-compose

0 comments on commit e9e42a2

Please sign in to comment.