diff --git a/.gitignore b/.gitignore index 3fe0482c..b9d91c20 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,5 @@ dpul_collections-*.tar # In case you use Node.js/npm, you want to ignore these. npm-debug.log /assets/node_modules/ - +figgy-fixture-container/fixture-exports/*.binary +figgy-fixture-container/fixture-exports/*.sql diff --git a/.lando.yml b/.lando.yml index 2d5701a2..d71e8bfb 100644 --- a/.lando.yml +++ b/.lando.yml @@ -3,3 +3,14 @@ services: database: type: postgres:15 portforward: 5434 + figgy_database: + type: compose + app_mount: false + services: + image: "ghcr.io/pulibrary/dpul-collections:figgy-fixtures" + environment: + POSTGRES_PASSWORD: "postgres" + ports: + - "5435:5432" + # This comes from the base image ENTRYPOINT + CMD + command: "docker-entrypoint.sh postgres" diff --git a/README.md b/README.md index db46567d..5a792adf 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,13 @@ Run Docker Image: `docker run -t -p 4000:4000 -e DATABASE_URL='ecto://postgres:@ 1. Connect to VPN 1. `BRANCH= ./bin/deploy staging` +## Figgy Fixtures + +We copy fixtures from Figgy's production database into a Docker container so that we can easily use it for testing indexing. To rebuild that container: + +`brew install lastpass-cli` +`cd figgy-fixture-container && ./build-and-push.sh` + ## Learn more * Official website: https://www.phoenixframework.org/ diff --git a/figgy-fixture-container/Dockerfile b/figgy-fixture-container/Dockerfile new file mode 100644 index 00000000..de86800f --- /dev/null +++ b/figgy-fixture-container/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:15 +ADD fixture-exports /tmp/fixture-exports +ADD import-container-fixtures.sh /docker-entrypoint-initdb.d/import-container-fixtures.sh diff --git a/figgy-fixture-container/build-and-push.sh b/figgy-fixture-container/build-and-push.sh new file mode 100755 index 00000000..901f9964 --- /dev/null +++ b/figgy-fixture-container/build-and-push.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +./create-fixture-exports.sh +echo "Enter lastpass username:" +read USERNAME +lpass login $USERNAME +export GHCR_TOKEN=$(lpass show "Shared-ITIMS-Passwords/dpul_collections_fixture_container_github_token" --notes) +echo $GHCR_TOKEN | docker login ghcr.io -u pulbot --password-stdin +docker buildx create --name multiarch --driver docker-container --use || true +docker buildx build --push --platform linux/arm64,linux/amd64 -t ghcr.io/pulibrary/dpul-collections:figgy-fixtures . diff --git a/figgy-fixture-container/create-fixture-exports.sh b/figgy-fixture-container/create-fixture-exports.sh new file mode 100755 index 00000000..a25a7445 --- /dev/null +++ b/figgy-fixture-container/create-fixture-exports.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Women Life Freedom Movement: Iran 2022 +PROJECT_ID=2961c153-54ab-4c6a-b5cd-aa992f4c349b +# First box in Women Life Freedom Movement +BOX_ID=82624edb-c360-4d8a-b202-f103ee639e8e + +# Export the project +ssh deploy@figgy-web-prod1.princeton.edu "cd /opt/figgy/current && PGPASSWORD=\$FIGGY_DB_RO_PASSWORD psql -d \$FIGGY_DB -U \$FIGGY_DB_RO_USERNAME -h \$FIGGY_DB_HOST -c \"\\copy (select * from orm_resources WHERE id = '$PROJECT_ID') TO '/tmp/project-export.binary' BINARY\"" + +# Export the project members recursively +# This will get all the Boxes, Folders, and FileSets +MEMBERS_QUERY=$(cat <<-END + WITH RECURSIVE deep_members AS ( + select member.* + FROM orm_resources a, + jsonb_array_elements(a.metadata->'member_ids') AS b(member) + JOIN orm_resources member ON (b.member->>'id')::UUID = member.id + WHERE a.id = '${PROJECT_ID}' + UNION + SELECT mem.* + FROM deep_members f, + jsonb_array_elements(f.metadata->'member_ids') AS g(member) + JOIN orm_resources mem ON (g.member->>'id')::UUID = mem.id + WHERE f.metadata @> '{\"member_ids\": [{}]}' + ) + select * from deep_members +END +) + +ssh deploy@figgy-web-prod1.princeton.edu "cd /opt/figgy/current && PGPASSWORD=\$FIGGY_DB_RO_PASSWORD psql -d \$FIGGY_DB -U \$FIGGY_DB_RO_USERNAME -h \$FIGGY_DB_HOST -c \"\\copy ($MEMBERS_QUERY) TO '/tmp/project-members-export.binary' BINARY\"" + +# Just export all the ephemera vocabularies and terms, so we have them. +VOCABULARY_QUERY=$(cat <<-END + select * from orm_resources WHERE internal_resource = 'EphemeraVocabulary' OR internal_resource = 'EphemeraTerm' +END +) + +ssh deploy@figgy-web-prod1.princeton.edu "cd /opt/figgy/current && PGPASSWORD=\$FIGGY_DB_RO_PASSWORD psql -d \$FIGGY_DB -U \$FIGGY_DB_RO_USERNAME -h \$FIGGY_DB_HOST -c \"\\copy ($VOCABULARY_QUERY) TO '/tmp/project-vocabulary-export.binary' BINARY\"" + +# Get deletion markers for any Folders deleted from the project's box. +DELETION_MARKERS_QUERY=$(cat <<-END +select * FROM orm_resources WHERE internal_resource='DeletionMarker' AND metadata @> '{\"parent_id\": [{\"id\": \"$BOX_ID\"}]}' +END +) + +ssh deploy@figgy-web-prod1.princeton.edu "cd /opt/figgy/current && PGPASSWORD=\$FIGGY_DB_RO_PASSWORD psql -d \$FIGGY_DB -U \$FIGGY_DB_RO_USERNAME -h \$FIGGY_DB_HOST -c \"\\copy ($DELETION_MARKERS_QUERY) TO '/tmp/project-dm-export.binary' BINARY\"" + +# Get a few resources we don't want to index in Phase 1-3 +IGNORABLE_RESOURCES_QUERY=$(cat <<-END +select * from orm_resources WHERE internal_resource = 'Event' LIMIT 10 +END +) +IGNORABLE_RESOURCES_QUERY_2=$(cat <<-END +select * from orm_resources WHERE internal_resource = 'ScannedResource' LIMIT 10 +END +) + +ssh deploy@figgy-web-prod1.princeton.edu "cd /opt/figgy/current && PGPASSWORD=\$FIGGY_DB_RO_PASSWORD psql -d \$FIGGY_DB -U \$FIGGY_DB_RO_USERNAME -h \$FIGGY_DB_HOST -c \"\\copy ($IGNORABLE_RESOURCES_QUERY) TO '/tmp/project-ignore-export.binary' BINARY\"" +ssh deploy@figgy-web-prod1.princeton.edu "cd /opt/figgy/current && PGPASSWORD=\$FIGGY_DB_RO_PASSWORD psql -d \$FIGGY_DB -U \$FIGGY_DB_RO_USERNAME -h \$FIGGY_DB_HOST -c \"\\copy ($IGNORABLE_RESOURCES_QUERY_2) TO '/tmp/project-ignore2-export.binary' BINARY\"" + +# Get the DB schema +ssh deploy@figgy-web-prod1.princeton.edu "cd /opt/figgy/current && PGPASSWORD=\$FIGGY_DB_RO_PASSWORD pg_dump -Fc -U \$FIGGY_DB_RO_USERNAME -h \$FIGGY_DB_HOST -f /tmp/db-schema.sql --schema-only \$FIGGY_DB" + +scp deploy@figgy-web-prod1.princeton.edu:/tmp/\{db-schema.sql,*-export.binary\} fixture-exports/ diff --git a/figgy-fixture-container/fixture-exports/.keep b/figgy-fixture-container/fixture-exports/.keep new file mode 100644 index 00000000..e69de29b diff --git a/figgy-fixture-container/import-container-fixtures.sh b/figgy-fixture-container/import-container-fixtures.sh new file mode 100755 index 00000000..ce48b2fb --- /dev/null +++ b/figgy-fixture-container/import-container-fixtures.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +pg_restore -d "$POSTGRES_DB" --no-owner --no-privileges --role=$POSTGRES_USER /tmp/fixture-exports/db-schema.sql + +psql -d "$POSTGRES_DB" -U $POSTGRES_USER -c "\copy orm_resources FROM /tmp/fixture-exports/project-export.binary BINARY" +psql -d "$POSTGRES_DB" -U $POSTGRES_USER -c "\copy orm_resources FROM /tmp/fixture-exports/project-members-export.binary BINARY" +psql -d "$POSTGRES_DB" -U $POSTGRES_USER -c "\copy orm_resources FROM /tmp/fixture-exports/project-vocabulary-export.binary BINARY" +psql -d "$POSTGRES_DB" -U $POSTGRES_USER -c "\copy orm_resources FROM /tmp/fixture-exports/project-dm-export.binary BINARY" +psql -d "$POSTGRES_DB" -U $POSTGRES_USER -c "\copy orm_resources FROM /tmp/fixture-exports/project-ignore-export.binary BINARY" +psql -d "$POSTGRES_DB" -U $POSTGRES_USER -c "\copy orm_resources FROM /tmp/fixture-exports/project-ignore2-export.binary BINARY"