Skip to content

Commit

Permalink
Add DIP upload test workflow
Browse files Browse the repository at this point in the history 
Co-authored-by: Miguel Angel Medinilla <[email protected]>
  • Loading branch information
@replaceafill @mamedin
replaceafill and mamedin committed Apr 25, 2024
1 parent 8cfbdc9 commit ee57ffd
Show file tree
Hide file tree
Showing 8 changed files with 895 additions and 0 deletions.
171 changes: 171 additions & 0 deletions .github/workflows/dip-upload.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
name: "DIP Upload Test"
on:
workflow_dispatch:
inputs:
am_version:
description: "Archivematica ref (branch, tag or SHA to checkout)"
default: "qa/1.x"
required: true
type: "string"
ss_version:
description: "Archivematica Storage Service ref (branch, tag or SHA to checkout)"
default: "qa/0.x"
required: true
type: "string"
atom_version:
description: "AtoM ref (branch, tag or SHA to checkout)"
default: "qa/2.x"
required: true
type: "string"
jobs:
test:
name: "DIP upload test"
runs-on: "ubuntu-latest"
env:
am_version: "${{ inputs.am_version }}"
ss_version: "${{ inputs.ss_version }}"
atom_version: "${{ inputs.atom_version }}"
am_vagrant_box_id: "ubuntu/jammy64"
atom_vagrant_box_id: "ubuntu/focal64"
steps:
- name: "Check out code"
uses: "actions/checkout@v4"
- name: "Create Vagrant boxes directory"
run: |
mkdir -p /home/runner/.vagrant.d/boxes
- name: "Cache Vagrant boxes"
uses: "actions/cache@v4"
with:
path: "/home/runner/.vagrant.d/boxes"
key: "${{ runner.os }}-boxes"
- name: "Install Python"
uses: "actions/setup-python@v5"
with:
python-version: "3.9"
- name: "Install Vagrant"
run: |
wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
sudo apt update && sudo apt install vagrant
- name: "Install VirtualBox"
run: |
wget -O- https://www.virtualbox.org/download/oracle_vbox_2016.asc | sudo gpg --yes --output /usr/share/keyrings/oracle-virtualbox-2016.gpg --dearmor
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/oracle-virtualbox-2016.gpg] https://download.virtualbox.org/virtualbox/debian jammy contrib" | sudo tee /etc/apt/sources.list.d/virtualbox.list
sudo apt update && sudo apt install virtualbox-7.0
- name: "Downgrade VirtualBox"
run: |
sudo apt-get purge virtualbox-7.0
wget -O /tmp/virtualbox-7.0_7.0.14-161095~Ubuntu~jammy_amd64.deb -L https://download.virtualbox.org/virtualbox/7.0.14/virtualbox-7.0_7.0.14-161095~Ubuntu~jammy_amd64.deb
sudo dpkg -i /tmp/virtualbox-7.0_7.0.14-161095~Ubuntu~jammy_amd64.deb
- name: "Install the vagrant-vbguest plugin"
run: |
vagrant plugin install vagrant-vbguest
- name: "Update the VirtualBox networks file"
run: |
sudo mkdir -p /etc/vbox/
echo "* 192.168.33.0/24" | sudo tee -a /etc/vbox/networks.conf
- name: "Install ansible"
working-directory: "${{ github.workspace }}/tests/dip-upload"
run: |
python3 -m venv .venv
source .venv/bin/activate
python3 -m pip install ansible==8.5.0 ansible-core==2.15.5
- name: "Start the Archivematica VM"
working-directory: "${{ github.workspace }}/tests/dip-upload"
env:
VAGRANT_BOX: "${{ env.am_vagrant_box_id }}"
run: |
vagrant up archivematica
- name: "Install Archivematica"
working-directory: "${{ github.workspace }}/tests/dip-upload"
env:
ANSIBLE_HOST_KEY_CHECKING: "False"
run: |
source .venv/bin/activate
ansible-galaxy install -f -p roles/ -r requirements.yml
ansible-playbook -i 192.168.33.2, archivematica.yml \
-u vagrant \
--private-key ${{ github.workspace }}/tests/dip-upload/.vagrant/machines/archivematica/virtualbox/private_key \
-e "archivematica_src_am_version=${{ env.am_version }} archivematica_src_ss_version=${{ env.ss_version }}" \
-v
- name: "Add the vagrant user to the archivematica group"
working-directory: "${{ github.workspace }}/tests/dip-upload"
run: |
vagrant ssh archivematica -c 'sudo usermod -a -G archivematica vagrant'
- name: "Get the archivematica SSH public key"
id: archivematica_ssh_pub_key
working-directory: "${{ github.workspace }}/tests/dip-upload"
run: |
echo "key=$(vagrant ssh archivematica -c 'sudo cat /var/lib/archivematica/.ssh/id_rsa.pub')" >> $GITHUB_OUTPUT
- name: "Start the AtoM VM"
working-directory: "${{ github.workspace }}/tests/dip-upload"
env:
VAGRANT_BOX: "${{ env.atom_vagrant_box_id }}"
run: |
vagrant up atom
- name: "Install AtoM"
working-directory: "${{ github.workspace }}/tests/dip-upload"
env:
ANSIBLE_HOST_KEY_CHECKING: "False"
run: |
source .venv/bin/activate
ansible-playbook -i 192.168.33.3, atom.yml \
-u vagrant \
--private-key ${{ github.workspace }}/tests/dip-upload/.vagrant/machines/atom/virtualbox/private_key \
-e "atom_repository_version=${{ env.atom_version }} archivematica_ssh_pub_key='${{ steps.archivematica_ssh_pub_key.outputs.key }}'" \
-v
- name: "Call an Archivematica API endpoint"
run: |
curl \
--header "Authorization: ApiKey admin:this_is_the_am_api_key" \
http://192.168.33.2/api/processing-configuration/
- name: "Call a Storage Service API endpoint"
run: |
curl \
--header "Authorization: ApiKey admin:this_is_the_ss_api_key" \
http://192.168.33.2:8000/api/v2/pipeline/
- name: "Call an AtoM API endpoint"
run: |
curl \
--header "REST-API-Key: this_is_the_atom_dip_upload_api_key" \
http://192.168.33.3/index.php/api/informationobjects
- name: "Create a processing configuration for DIP upload"
working-directory: "${{ github.workspace }}/tests/dip-upload"
run: |
vagrant ssh archivematica -c "sudo -u archivematica cp /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/{automated,dipupload}ProcessingMCP.xml"
- name: "Update the DIP upload processing configuration"
working-directory: "${{ github.workspace }}/tests/dip-upload"
run: |
# Change 'Normalize for preservation' to 'Normalize for preservation and access'
vagrant ssh archivematica -c "sudo -u archivematica sed --in-place 's|612e3609-ce9a-4df6-a9a3-63d634d2d934|b93cecd4-71f2-4e28-bc39-d32fd62c5a94|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml"
# Change 'Do not upload DIP' to 'Upload DIP to AtoM/Binder'
vagrant ssh archivematica -c "sudo -u archivematica sed --in-place 's|6eb8ebe7-fab3-4e4c-b9d7-14de17625baa|0fe9842f-9519-4067-a691-8a363132ae24|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml"
- name: "Import Atom sample data"
working-directory: "${{ github.workspace }}/tests/dip-upload"
run: |
vagrant ssh atom -c "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony csv:import /usr/share/nginx/atom/lib/task/import/example/isad/example_information_objects_isad.csv"
vagrant ssh atom -c "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony propel:build-nested-set"
vagrant ssh atom -c "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony cc"
vagrant ssh atom -c "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony search:populate"
- name: "Start a transfer and upload the DIP to the sample archival description"
run: |
curl \
--header "Authorization: ApiKey admin:this_is_the_am_api_key" \
--request POST \
--data "{ \
\"name\": \"dip-upload-test\", \
\"path\": \"$(echo -n '/home/vagrant/archivematica-sampledata/SampleTransfers/DemoTransferCSV' | base64 -w 0)\", \
\"type\": \"standard\", \
\"processing_config\": \"dipupload\", \
\"access_system_id\": \"example-item\" \
}" \
http://192.168.33.2/api/v2beta/package
- name: "Wait for the transfer to finish"
run: |
sleep 120
- name: "Verify a digital object was uploaded and attached to the sample archival description"
run: |
curl \
--header "REST-API-Key: this_is_the_atom_dip_upload_api_key" \
--silent \
http://192.168.33.3/index.php/api/informationobjects/beihai-guanxi-china-1988 | python3 -m json.tool | grep '"parent": "example-item"'
184 changes: 184 additions & 0 deletions tests/dip-upload/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
# DIP upload test

## Software requirements

- Vagrant 2.4.1 (with vagrant-vbguest plugin)
- VirtualBox 7.0
- Python 3
- curl

## Tested Vagrant boxes

This playbook has been tested with Vagrant 2.4.1 and VirtualBox 7.0.14 r161095
using any of the following Vagrant boxes and versions:

- Archivematica: ubuntu/jammy64 (v20240403.0.0)
- AtoM: ubuntu/focal64 (v20231003.0.0)

## Installing Ansible

Create a virtual environment and activate it:

```shell
python3 -m venv .venv
source .venv/bin/activate
```

Install `ansible` and `ansible-core` (these versions are compatible with
symbolic links which are used in the the artefactual-atom role):

```shell
python3 -m pip install ansible==8.5.0 ansible-core==2.15.5
```

Install the playbook requirements:

```shell
ansible-galaxy install -f -p roles/ -r requirements.yml
```

## Setting up VirtualBox

Install the `vagrant-vbguest` plugin:

```shell
vagrant plugin install vagrant-vbguest
```

Add the VMs IP network to the VirtualBox networks file:

```shell
sudo mkdir -p /etc/vbox/
echo "* 192.168.33.0/24" | sudo tee -a /etc/vbox/networks.conf
```

## Provisioning the Archivematica VM

Start the VM passing the `VAGRANT_BOX` environment variable with the ID of the
Ubuntu 22.04 Vagrant Cloud:

```shell
env VAGRANT_BOX=ubuntu/jammy64 vagrant up archivematica
```

Run the Archivematica installation playbook:

```shell
ansible-playbook -i 192.168.33.2, archivematica.yml \
-u vagrant \
--private-key $PWD/.vagrant/machines/archivematica/virtualbox/private_key \
-v
```

Add the `vagrant` user to the `archivematica` group so it can copy AIPs
from the shared directory:

```shell
vagrant ssh archivematica -c 'sudo usermod -a -G archivematica vagrant'
```

Get the SSH public key of the `archivematica` user so we can use it when
provisioning the AtoM VM:

```shell
AM_SSH_PUB_KEY=$(vagrant ssh archivematica -c 'sudo cat /var/lib/archivematica/.ssh/id_rsa.pub')
```

## Provisioning the AtoM VM

Start the VM passing the `VAGRANT_BOX` environment variable with the ID of the
Ubuntu 20.04 Vagrant Cloud:

```shell
env VAGRANT_BOX=ubuntu/focal64 vagrant up atom
```

Run the AtoM installation playbook passing the `archivematica_ssh_pub_key`
variable with the contents of `$AM_SSH_PUB_KEY`:

```shell
ansible-playbook -i 192.168.33.3, atom.yml \
-u vagrant \
--private-key $PWD/.vagrant/machines/atom/virtualbox/private_key \
-e "archivematica_ssh_pub_key='$AM_SSH_PUB_KEY'" \
-v
```

## Testing the Archivematica installation

Call an Archivematica API endpoint:

```shell
curl --header "Authorization: ApiKey admin:this_is_the_am_api_key" http://192.168.33.2/api/processing-configuration/
```

Call a Storage Service API endpoint:

```shell
curl --header "Authorization: ApiKey admin:this_is_the_ss_api_key" http://192.168.33.2:8000/api/v2/pipeline/
```

## Testing the AtoM installation

Call an AtoM API endpoint:

```shell
curl --header "REST-API-Key: this_is_the_atom_dip_upload_api_key" http://192.168.33.3/index.php/api/informationobjects
```

## Testing DIP upload

Create a processing configuration for DIP upload:

```shell
vagrant ssh archivematica -c "sudo -u archivematica cp /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/{automated,dipupload}ProcessingMCP.xml"
```

Update the DIP upload processing configuration:

```shell
# Change 'Normalize for preservation' to 'Normalize for preservation and access'
vagrant ssh archivematica -c "sudo -u archivematica sed --in-place 's|612e3609-ce9a-4df6-a9a3-63d634d2d934|b93cecd4-71f2-4e28-bc39-d32fd62c5a94|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml"
# Change 'Do not upload DIP' to 'Upload DIP to AtoM/Binder'
vagrant ssh archivematica -c "sudo -u archivematica sed --in-place 's|6eb8ebe7-fab3-4e4c-b9d7-14de17625baa|0fe9842f-9519-4067-a691-8a363132ae24|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml"
```

Import Atom sample data:

```shell
vagrant ssh atom -c "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony csv:import /usr/share/nginx/atom/lib/task/import/example/isad/example_information_objects_isad.csv"
vagrant ssh atom -c "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony propel:build-nested-set"
vagrant ssh atom -c "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony cc"
vagrant ssh atom -c "cd /usr/share/nginx/atom/ && sudo -u www-data php -d memory_limit=-1 symfony search:populate"
```

Start a transfer and upload the DIP to the sample archival description:

```shell
curl \
--header "Authorization: ApiKey admin:this_is_the_am_api_key" \
--request POST \
--data "{ \
\"name\": \"dip-upload-test\", \
\"path\": \"$(echo -n '/home/vagrant/archivematica-sampledata/SampleTransfers/DemoTransferCSV' | base64 -w 0)\", \
\"type\": \"standard\", \
\"processing_config\": \"dipupload\", \
\"access_system_id\": \"example-item\" \
}" \
http://192.168.33.2/api/v2beta/package
```

Wait for the transfer to finish:

```shell
sleep 120
```

Verify a digital object was uploaded and attached to the sample archival description:

```shell
curl \
--header "REST-API-Key: this_is_the_atom_dip_upload_api_key" \
--silent \
http://192.168.33.3/index.php/api/informationobjects/beihai-guanxi-china-1988 | python3 -m json.tool | grep '"parent": "example-item"'
```
Loading

0 comments on commit ee57ffd

Please sign in to comment.