DIP Upload Test #313
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "DIP Upload Test" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| am_version: | |
| description: "Archivematica ref (branch, tag or SHA to checkout)" | |
| default: "qa/1.x" | |
| required: true | |
| type: "string" | |
| ss_version: | |
| description: "Archivematica Storage Service ref (branch, tag or SHA to checkout)" | |
| default: "qa/0.x" | |
| required: true | |
| type: "string" | |
| atom_version: | |
| description: "AtoM ref (branch, tag or SHA to checkout)" | |
| default: "qa/2.x" | |
| required: true | |
| type: "string" | |
| schedule: | |
| - cron: "0 2 * * *" | |
| jobs: | |
| test: | |
| name: "DIP upload test" | |
| runs-on: "ubuntu-22.04" | |
| env: | |
| am_version: "${{ inputs.am_version || 'qa/1.x' }}" | |
| ss_version: "${{ inputs.ss_version || 'qa/0.x' }}" | |
| atom_version: "${{ inputs.atom_version || 'qa/2.x' }}" | |
| python_version: "3.10" | |
| steps: | |
| - name: "Check out the code" | |
| uses: "actions/checkout@v4" | |
| - name: "Upgrade crun (supports Ubuntu's systemd in the Dockerfile)" | |
| run: | | |
| wget https://github.com/containers/crun/releases/download/1.15/crun-1.15-linux-amd64 | |
| sudo install crun-1.15-linux-amd64 /usr/bin/crun | |
| rm crun-1.15-linux-amd64 | |
| - name: "Install Python" | |
| uses: "actions/setup-python@v5" | |
| with: | |
| python-version: "${{ env.python_version }}" | |
| cache: "pip" | |
| cache-dependency-path: | | |
| tests/dip-upload/requirements.txt | |
| - name: "Cache the virtual environment" | |
| id: "venv-cache" | |
| uses: "actions/cache@v4" | |
| with: | |
| path: | | |
| tests/dip-upload/.venv/ | |
| key: "os-${{ runner.os }}-python_version-${{ env.python_version }}-hash-${{ hashFiles('tests/dip-upload/requirements.txt') }}" | |
| - name: "Set up the virtual environment" | |
| if: "steps.venv-cache.outputs.cache-hit == false" | |
| working-directory: "${{ github.workspace }}/tests/dip-upload" | |
| run: | | |
| python3 -m venv .venv | |
| .venv/bin/python -m pip install -r requirements.txt | |
| - name: "Add virtual environment to PATH" | |
| working-directory: "${{ github.workspace }}/tests/dip-upload" | |
| run: | |
| echo "$PWD/.venv/bin" >> $GITHUB_PATH | |
| - name: "Generate an SSH key and copy it next to the Dockerfile" | |
| run: | | |
| mkdir $HOME/.ssh | |
| ssh-keygen -t rsa -f $HOME/.ssh/id_rsa -N "" | |
| cp $HOME/.ssh/id_rsa.pub ${{ github.workspace }}/tests/dip-upload/ssh_pub_key | |
| - name: "Start the Compose environment" | |
| working-directory: "${{ github.workspace }}/tests/dip-upload" | |
| run: | | |
| podman-compose up --detach | |
| - name: "Install Archivematica" | |
| working-directory: "${{ github.workspace }}/tests/dip-upload" | |
| env: | |
| ANSIBLE_HOST_KEY_CHECKING: "False" | |
| ANSIBLE_REMOTE_PORT: 2222 | |
| run: | | |
| ansible-galaxy install -f -p roles/ -r requirements.yml | |
| ansible-playbook -i localhost, archivematica.yml \ | |
| -u ubuntu \ | |
| -e "archivematica_src_am_version=${{ env.am_version }} archivematica_src_ss_version=${{ env.ss_version }}" \ | |
| -v | |
| - name: "Add the ubuntu user to the archivematica group" | |
| working-directory: "${{ github.workspace }}/tests/dip-upload" | |
| run: | | |
| podman-compose exec --user root archivematica usermod -a -G archivematica ubuntu | |
| - name: "Get the archivematica SSH public key" | |
| id: archivematica_ssh_pub_key | |
| working-directory: "${{ github.workspace }}/tests/dip-upload" | |
| run: | | |
| echo "key=$(podman-compose exec --user archivematica archivematica cat /var/lib/archivematica/.ssh/id_rsa.pub)" >> $GITHUB_OUTPUT | |
| - name: "Install AtoM" | |
| working-directory: "${{ github.workspace }}/tests/dip-upload" | |
| env: | |
| ANSIBLE_HOST_KEY_CHECKING: "False" | |
| ANSIBLE_REMOTE_PORT: 9222 | |
| run: | | |
| ansible-playbook -i localhost, atom.yml \ | |
| -u ubuntu \ | |
| -e "atom_repository_version=${{ env.atom_version }} archivematica_ssh_pub_key='${{ steps.archivematica_ssh_pub_key.outputs.key }}'" \ | |
| -v | |
| - name: "Call an Archivematica API endpoint" | |
| run: | | |
| test $( \ | |
| curl \ | |
| --silent \ | |
| --header 'Authorization: ApiKey admin:this_is_the_am_api_key' \ | |
| --header 'Content-Type: application/json' \ | |
| 'http://localhost:8000/api/processing-configuration/' \ | |
| | jq -r '.processing_configurations == ["automated", "default"]' \ | |
| ) == true | |
| - name: "Call a Storage Service API endpoint" | |
| run: | | |
| test $( \ | |
| curl \ | |
| --silent \ | |
| --header 'Authorization: ApiKey admin:this_is_the_ss_api_key' \ | |
| --header 'Content-Type: application/json' \ | |
| 'http://localhost:8001/api/v2/pipeline/' \ | |
| | jq -r '.meta.total_count == 1' \ | |
| ) == true | |
| - name: "Call an AtoM API endpoint" | |
| run: | | |
| test $( \ | |
| curl \ | |
| --silent \ | |
| --header 'REST-API-Key: this_is_the_atom_dip_upload_api_key' \ | |
| --header 'Content-Type: application/json' \ | |
| http://localhost:9000/index.php/api/informationobjects \ | |
| | jq -r '.results == []' \ | |
| ) == true | |
| - name: "Create a processing configuration for DIP upload" | |
| working-directory: "${{ github.workspace }}/tests/dip-upload" | |
| run: | | |
| podman-compose exec --user archivematica archivematica cp /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/automatedProcessingMCP.xml /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml | |
| - name: "Update the DIP upload processing configuration" | |
| working-directory: "${{ github.workspace }}/tests/dip-upload" | |
| run: | | |
| # Change 'Normalize for preservation' to 'Normalize for preservation and access' | |
| podman-compose exec --user archivematica archivematica sed --in-place 's|612e3609-ce9a-4df6-a9a3-63d634d2d934|b93cecd4-71f2-4e28-bc39-d32fd62c5a94|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml | |
| # Change 'Do not upload DIP' to 'Upload DIP to AtoM/Binder' | |
| podman-compose exec --user archivematica archivematica sed --in-place 's|6eb8ebe7-fab3-4e4c-b9d7-14de17625baa|0fe9842f-9519-4067-a691-8a363132ae24|g' /var/archivematica/sharedDirectory/sharedMicroServiceTasksConfigs/processingMCPConfigs/dipuploadProcessingMCP.xml | |
| - name: "Import the Atom sample data" | |
| working-directory: "${{ github.workspace }}/tests/dip-upload" | |
| run: | | |
| podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony csv:import /usr/share/nginx/atom/lib/task/import/example/isad/example_information_objects_isad.csv | |
| podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony propel:build-nested-set | |
| podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony cc | |
| podman-compose exec --user www-data --workdir /usr/share/nginx/atom/ atom php -d memory_limit=-1 symfony search:populate | |
| - name: "Start a transfer and upload the DIP to the sample archival description" | |
| run: | | |
| curl \ | |
| --header "Authorization: ApiKey admin:this_is_the_am_api_key" \ | |
| --request POST \ | |
| --data "{ \ | |
| \"name\": \"dip-upload-test\", \ | |
| \"path\": \"$(echo -n '/home/ubuntu/archivematica-sampledata/SampleTransfers/DemoTransferCSV' | base64 -w 0)\", \ | |
| \"type\": \"standard\", \ | |
| \"processing_config\": \"dipupload\", \ | |
| \"access_system_id\": \"example-item\" \ | |
| }" \ | |
| http://localhost:8000/api/v2beta/package | |
| - name: "Wait for the transfer to finish" | |
| run: | | |
| sleep 120 | |
| - name: "Display the contents of the DIP" | |
| working-directory: "${{ github.workspace }}/tests/dip-upload" | |
| run: | | |
| podman-compose exec --user archivematica archivematica bash -c "find /var/archivematica/sharedDirectory/www/DIPsStore/ -name 'dip-upload-test-*' | xargs tree" | |
| - name: "Verify all digital objects were uploaded and attached to the sample archival description" | |
| run: | | |
| json_data=$(cat <<EOF | |
| { | |
| "parent": "example-item", | |
| "title": "Beihai, Guanxi, China, 1988", | |
| "publication_status": "Draft", | |
| "level_of_description": "Item", | |
| "dates": [ | |
| { | |
| "date": "February 29,2016", | |
| "type": "Creation" | |
| } | |
| ], | |
| "creators": [ | |
| { | |
| "authorized_form_of_name": "NASA/GSFC/METI/ERSDAC/JAROS and U.S./Japan ASTER Science Team" | |
| } | |
| ], | |
| "repository": "Example Repository", | |
| "repository_inherited_from": "Example fonds", | |
| "scope_and_content": "Beihai is a city in the south of Guangxi, People's republic of China. It is a seaport on the north shore of the Gulf of Tonkin. Historically it has been an international trade port for many provinces to the north. From a prefecture-level population of about 500,000 in 1990, Beihai has grown to over 1,600,000 in 2013. Between the years 2006 and 2020, Beihai is predicted to be the world's fastest growing city. The Landsat image was acquired on 31 December 1988, the ASTER image on 31 December 2014. The images cover an area of 27 by 34 km, and are located at 21.5 degrees north, 109.1 degrees east.\n\nWith its 14 spectral bands from the visible to the thermal infrared wavelength region and its high spatial resolution of 15 to 90 meters (about 50 to 300 feet), ASTER images Earth to map and monitor the changing surface of our planet. ASTER is one of five Earth-observing instruments launched Dec. 18, 1999, on Terra. The instrument was built by Japan's Ministry of Economy, Trade and Industry. A joint U.S./Japan science team is responsible for validation and calibration of the instrument and data products.\n\nThe broad spectral coverage and high spectral resolution of ASTER provides scientists in numerous disciplines with critical information for surface mapping and monitoring of dynamic conditions and temporal change. Example applications are: monitoring glacial advances and retreats; monitoring potentially active volcanoes; identifying crop stress; determining cloud morphology and physical properties; wetlands evaluation; thermal pollution monitoring; coral reef degradation; surface temperature mapping of soils and geology; and measuring surface heat balance.\n\nThe U.S. science team is located at NASA's Jet Propulsion Laboratory, Pasadena, Calif. The Terra mission is part of NASA's Science Mission Directorate, Washington, D.C.\n\nMore information about ASTER is available at http://asterweb.jpl.nasa.gov/.", | |
| "conditions_governing_access": "Public domain", | |
| "existence_and_location_of_originals": "NASA Jet Propulsion Laboratory", | |
| "subject_access_points": [ | |
| "satellite imagery", | |
| "China", | |
| "Beihai" | |
| ], | |
| "place_access_points": [ | |
| "Beihai, China" | |
| ], | |
| "digital_object": { | |
| "media_type": "Image", | |
| "mime_type": "image/jpeg" | |
| } | |
| } | |
| EOF | |
| ) | |
| test $(\ | |
| curl \ | |
| --silent \ | |
| --header "REST-API-Key: this_is_the_atom_dip_upload_api_key" \ | |
| http://localhost:9000/index.php/api/informationobjects/beihai-guanxi-china-1988 | jq --argjson small "$json_data" 'contains($small)' | |
| ) == true | |
| json_data=$(cat <<EOF | |
| { | |
| "parent": "example-item", | |
| "reference_code": "F1-IT-1-aporee_36644_41997", | |
| "title": "14000 Caen, France - Bird in my garden", | |
| "publication_status": "Draft", | |
| "level_of_description": "Item", | |
| "dates": [ | |
| { | |
| "date": "2017-05-27", | |
| "type": "Creation" | |
| } | |
| ], | |
| "creators": [ | |
| { | |
| "authorized_form_of_name": "Nicolas Germain" | |
| } | |
| ], | |
| "repository": "Example Repository", | |
| "repository_inherited_from": "Example fonds", | |
| "scope_and_content": "Bird singing in my garden, Caen, France, Zoom H6", | |
| "conditions_governing_access": "Public domain", | |
| "existence_and_location_of_originals": "Internet Archive", | |
| "subject_access_points": [ | |
| "field recording", | |
| "phonography", | |
| "soundscapes", | |
| "sound art", | |
| "radio aporee" | |
| ], | |
| "place_access_points": [ | |
| "Caen, France" | |
| ], | |
| "digital_object": { | |
| "media_type": "Audio", | |
| "mime_type": "audio/mpeg" | |
| } | |
| } | |
| EOF | |
| ) | |
| test $( | |
| curl \ | |
| --silent \ | |
| --header "REST-API-Key: this_is_the_atom_dip_upload_api_key" \ | |
| http://localhost:9000/index.php/api/informationobjects/14000-caen-france-bird-in-my-garden | jq --argjson small "$json_data" 'contains($small)' | |
| ) == true | |
| json_data=$(cat <<EOF | |
| { | |
| "parent": "example-item", | |
| "title": "Marble Madness", | |
| "publication_status": "Draft", | |
| "level_of_description": "Item", | |
| "dates": [ | |
| { | |
| "date": "2019-05-01", | |
| "type": "Creation" | |
| } | |
| ], | |
| "creators": [ | |
| { | |
| "authorized_form_of_name": "Milton Bradley" | |
| } | |
| ], | |
| "repository": "Example Repository", | |
| "repository_inherited_from": "Example fonds", | |
| "scope_and_content": "An artistic close-up photograph of marbles arranged in a uniform pattern.", | |
| "conditions_governing_access": "Public domain", | |
| "subject_access_points": [ | |
| "Art", | |
| "Photography", | |
| "Children’s games" | |
| ], | |
| "digital_object": { | |
| "media_type": "Image", | |
| "mime_type": "image/jpeg" | |
| } | |
| } | |
| EOF | |
| ) | |
| test $( | |
| curl \ | |
| --silent \ | |
| --header "REST-API-Key: this_is_the_atom_dip_upload_api_key" \ | |
| http://localhost:9000/index.php/api/informationobjects/marble-madness | jq --argjson small "$json_data" 'contains($small)' | |
| ) == true | |
| json_data=$(cat <<EOF | |
| { | |
| "parent": "example-item", | |
| "title": "Flag of Montreal", | |
| "publication_status": "Draft", | |
| "level_of_description": "Item", | |
| "creators": [ | |
| { | |
| "authorized_form_of_name": "Public Domain" | |
| } | |
| ], | |
| "repository": "Example Repository", | |
| "repository_inherited_from": "Example fonds", | |
| "conditions_governing_access": "Public domain", | |
| "subject_access_points": [ | |
| "Art", | |
| "Heraldtry" | |
| ], | |
| "digital_object": { | |
| "media_type": "Image", | |
| "mime_type": "image/jpeg" | |
| } | |
| } | |
| EOF | |
| ) | |
| test $( | |
| curl \ | |
| --silent \ | |
| --header "REST-API-Key: this_is_the_atom_dip_upload_api_key" \ | |
| http://localhost:9000/index.php/api/informationobjects/flag-of-montreal | jq --argjson small "$json_data" 'contains($small)' | |
| ) == true | |
| json_data=$(cat <<EOF | |
| { | |
| "parent": "example-item", | |
| "title": "OCR image", | |
| "publication_status": "Draft", | |
| "level_of_description": "Item", | |
| "creators": [ | |
| { | |
| "authorized_form_of_name": "Tesseract" | |
| } | |
| ], | |
| "repository": "Example Repository", | |
| "repository_inherited_from": "Example fonds", | |
| "scope_and_content": "This image was retrieved from the Tesseract wiki (https://github.com/tesseract-ocr/tesseract/wiki/Command-Line-Usage) to test optical character recognition in Archivematica.", | |
| "languages_of_material": [ | |
| "German" | |
| ], | |
| "existence_and_location_of_originals": "Tesseract project", | |
| "digital_object": { | |
| "media_type": "Image", | |
| "mime_type": "image/jpeg" | |
| } | |
| } | |
| EOF | |
| ) | |
| test $( | |
| curl \ | |
| --silent \ | |
| --header "REST-API-Key: this_is_the_atom_dip_upload_api_key" \ | |
| http://localhost:9000/index.php/api/informationobjects/ocr-image | jq --argjson small "$json_data" 'contains($small)' | |
| ) == true | |
| json_data=$(cat <<EOF | |
| { | |
| "parent": "example-item", | |
| "title": "PII Test Data (Credit Card Numbers)", | |
| "publication_status": "Draft", | |
| "level_of_description": "Item", | |
| "creators": [ | |
| { | |
| "authorized_form_of_name": "Creator (ISAD 3.2.1)", | |
| "history": "Example fonds Creator history (ISAD 3.2.2). Note that this will be added to the related authority record for Creator (ISAD 3.2.1).", | |
| "inherited_from": "Example fonds" | |
| } | |
| ], | |
| "repository": "Example Repository", | |
| "repository_inherited_from": "Example fonds", | |
| "scope_and_content": "A collection of strings used to trigger the Bulk Extractor suite of tools.", | |
| "subject_access_points": [ | |
| "Infosec" | |
| ], | |
| "digital_object": { | |
| "media_type": "Text", | |
| "mime_type": "text/plain" | |
| } | |
| } | |
| EOF | |
| ) | |
| test $( | |
| curl \ | |
| --silent \ | |
| --header "REST-API-Key: this_is_the_atom_dip_upload_api_key" \ | |
| http://localhost:9000/index.php/api/informationobjects/pii-test-data-credit-card-numbers | jq --argjson small "$json_data" 'contains($small)' | |
| ) == true | |
| json_data=$(cat <<EOF | |
| { | |
| "parent": "example-item", | |
| "title": "Morning view from lookout over Queenstown towards the Remarkables in spring", | |
| "publication_status": "Draft", | |
| "level_of_description": "Item", | |
| "dates": [ | |
| { | |
| "date": "7 October 2014", | |
| "type": "Creation" | |
| } | |
| ], | |
| "creators": [ | |
| { | |
| "authorized_form_of_name": "Pseudopanax at English Wikipedia" | |
| } | |
| ], | |
| "repository": "Example Repository", | |
| "repository_inherited_from": "Example fonds", | |
| "conditions_governing_access": "Public domain", | |
| "existence_and_location_of_originals": "Wikimedia Commons", | |
| "subject_access_points": [ | |
| "The Remarkables", | |
| "Lake Wakatipu" | |
| ], | |
| "place_access_points": [ | |
| "Lake Wakatipu" | |
| ], | |
| "digital_object": { | |
| "media_type": "Image", | |
| "mime_type": "image/jpeg" | |
| } | |
| } | |
| EOF | |
| ) | |
| test $( | |
| curl \ | |
| --silent \ | |
| --header "REST-API-Key: this_is_the_atom_dip_upload_api_key" \ | |
| http://localhost:9000/index.php/api/informationobjects/morning-view-from-lookout-over-queenstown-towards-the-remarkables-in-spring | jq --argjson small "$json_data" 'contains($small)' | |
| ) == true |