Build a fixture container of Figgy data for indexing.

.gitignore

@@ -34,4 +34,5 @@ dpul_collections-*.tar
 # In case you use Node.js/npm, you want to ignore these.
 npm-debug.log
 /assets/node_modules/
-
+figgy-fixture-container/fixture-exports/*.binary
+figgy-fixture-container/fixture-exports/*.sql

.lando.yml

@@ -3,3 +3,14 @@ services:
   database:
     type: postgres:15
     portforward: 5434
+  figgy_database:
+    type: compose
+    app_mount: false
+    services:
+      image: "ghcr.io/pulibrary/dpul-collections:figgy-fixtures"
+      environment:
+        POSTGRES_PASSWORD: "postgres"
+      ports:
+        - "5435:5432"
+      # This comes from the base image ENTRYPOINT + CMD
+      command: "docker-entrypoint.sh postgres"

README.md

@@ -24,6 +24,13 @@ Run Docker Image: `docker run -t -p 4000:4000 -e DATABASE_URL='ecto://postgres:@
 1. Connect to VPN
 1. `BRANCH=<branch> ./bin/deploy staging`
 
+## Figgy Fixtures
+
+We copy fixtures from Figgy's production database into a Docker container so that we can easily use it for testing indexing. To rebuild that container:
+
+`brew install lastpass-cli`
+`cd figgy-fixture-container && ./build-and-push.sh`
+
 ## Learn more
 
   * Official website: https://www.phoenixframework.org/

figgy-fixture-container/Dockerfile

@@ -0,0 +1,3 @@
+FROM postgres:15
+ADD fixture-exports /tmp/fixture-exports
+ADD import-container-fixtures.sh /docker-entrypoint-initdb.d/import-container-fixtures.sh

figgy-fixture-container/build-and-push.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+./create-fixture-exports.sh
+echo "Enter lastpass username:" 
+read USERNAME
+lpass login $USERNAME
+export GHCR_TOKEN=$(lpass show "Shared-ITIMS-Passwords/dpul_collections_fixture_container_github_token" --notes)
+echo $GHCR_TOKEN | docker login ghcr.io -u pulbot --password-stdin
+docker buildx create --name multiarch --driver docker-container --use || true
+docker buildx build --push --platform linux/arm64,linux/amd64 -t ghcr.io/pulibrary/dpul-collections:figgy-fixtures .

figgy-fixture-container/create-fixture-exports.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Women Life Freedom Movement: Iran 2022
+PROJECT_ID=2961c153-54ab-4c6a-b5cd-aa992f4c349b
+# First box in Women Life Freedom Movement
+BOX_ID=82624edb-c360-4d8a-b202-f103ee639e8e
+
+# Export the project
+ssh [email protected] "cd /opt/figgy/current && PGPASSWORD=\$FIGGY_DB_RO_PASSWORD psql -d \$FIGGY_DB -U \$FIGGY_DB_RO_USERNAME -h \$FIGGY_DB_HOST -c \"\\copy (select * from orm_resources WHERE id = '$PROJECT_ID') TO '/tmp/project-export.binary' BINARY\""
+
+# Export the project members recursively
+# This will get all the Boxes, Folders, and FileSets
+MEMBERS_QUERY=$(cat <<-END
+  WITH RECURSIVE deep_members AS (
+    select member.*
+    FROM orm_resources a,
+    jsonb_array_elements(a.metadata->'member_ids') AS b(member)
+    JOIN orm_resources member ON (b.member->>'id')::UUID = member.id
+    WHERE a.id = '${PROJECT_ID}'
+    UNION
+    SELECT mem.*
+    FROM deep_members f,
+    jsonb_array_elements(f.metadata->'member_ids') AS g(member)
+    JOIN orm_resources mem ON (g.member->>'id')::UUID = mem.id
+    WHERE f.metadata @> '{\"member_ids\": [{}]}'
+  )
+  select * from deep_members
+END
+)
+
+ssh [email protected] "cd /opt/figgy/current && PGPASSWORD=\$FIGGY_DB_RO_PASSWORD psql -d \$FIGGY_DB -U \$FIGGY_DB_RO_USERNAME -h \$FIGGY_DB_HOST -c \"\\copy ($MEMBERS_QUERY) TO '/tmp/project-members-export.binary' BINARY\""
+
+# Just export all the ephemera vocabularies and terms, so we have them.
+VOCABULARY_QUERY=$(cat <<-END
+  select * from orm_resources WHERE internal_resource = 'EphemeraVocabulary' OR internal_resource = 'EphemeraTerm'
+END
+)
+
+ssh [email protected] "cd /opt/figgy/current && PGPASSWORD=\$FIGGY_DB_RO_PASSWORD psql -d \$FIGGY_DB -U \$FIGGY_DB_RO_USERNAME -h \$FIGGY_DB_HOST -c \"\\copy ($VOCABULARY_QUERY) TO '/tmp/project-vocabulary-export.binary' BINARY\""
+
+# Get deletion markers for any Folders deleted from the project's box.
+DELETION_MARKERS_QUERY=$(cat <<-END
+select * FROM orm_resources WHERE internal_resource='DeletionMarker' AND metadata @> '{\"parent_id\": [{\"id\": \"$BOX_ID\"}]}'
+END
+)
+
+ssh [email protected] "cd /opt/figgy/current && PGPASSWORD=\$FIGGY_DB_RO_PASSWORD psql -d \$FIGGY_DB -U \$FIGGY_DB_RO_USERNAME -h \$FIGGY_DB_HOST -c \"\\copy ($DELETION_MARKERS_QUERY) TO '/tmp/project-dm-export.binary' BINARY\""
+
+# Get a few resources we don't want to index in Phase 1-3
+IGNORABLE_RESOURCES_QUERY=$(cat <<-END
+select * from orm_resources WHERE internal_resource = 'Event' LIMIT 10
+END
+)
+IGNORABLE_RESOURCES_QUERY_2=$(cat <<-END
+select * from orm_resources WHERE internal_resource = 'ScannedResource' LIMIT 10
+END
+)
+
+ssh [email protected] "cd /opt/figgy/current && PGPASSWORD=\$FIGGY_DB_RO_PASSWORD psql -d \$FIGGY_DB -U \$FIGGY_DB_RO_USERNAME -h \$FIGGY_DB_HOST -c \"\\copy ($IGNORABLE_RESOURCES_QUERY) TO '/tmp/project-ignore-export.binary' BINARY\""
+ssh [email protected] "cd /opt/figgy/current && PGPASSWORD=\$FIGGY_DB_RO_PASSWORD psql -d \$FIGGY_DB -U \$FIGGY_DB_RO_USERNAME -h \$FIGGY_DB_HOST -c \"\\copy ($IGNORABLE_RESOURCES_QUERY_2) TO '/tmp/project-ignore2-export.binary' BINARY\""
+
+# Get the DB schema
+ssh [email protected] "cd /opt/figgy/current && PGPASSWORD=\$FIGGY_DB_RO_PASSWORD pg_dump -Fc -U \$FIGGY_DB_RO_USERNAME -h \$FIGGY_DB_HOST -f /tmp/db-schema.sql --schema-only \$FIGGY_DB"
+
+scp [email protected]:/tmp/\{db-schema.sql,*-export.binary\} fixture-exports/

figgy-fixture-container/import-container-fixtures.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+pg_restore -d "$POSTGRES_DB" --no-owner --no-privileges --role=$POSTGRES_USER /tmp/fixture-exports/db-schema.sql
+
+psql -d "$POSTGRES_DB" -U $POSTGRES_USER -c "\copy orm_resources FROM /tmp/fixture-exports/project-export.binary BINARY"
+psql -d "$POSTGRES_DB" -U $POSTGRES_USER -c "\copy orm_resources FROM /tmp/fixture-exports/project-members-export.binary BINARY"
+psql -d "$POSTGRES_DB" -U $POSTGRES_USER -c "\copy orm_resources FROM /tmp/fixture-exports/project-vocabulary-export.binary BINARY"
+psql -d "$POSTGRES_DB" -U $POSTGRES_USER -c "\copy orm_resources FROM /tmp/fixture-exports/project-dm-export.binary BINARY"
+psql -d "$POSTGRES_DB" -U $POSTGRES_USER -c "\copy orm_resources FROM /tmp/fixture-exports/project-ignore-export.binary BINARY"
+psql -d "$POSTGRES_DB" -U $POSTGRES_USER -c "\copy orm_resources FROM /tmp/fixture-exports/project-ignore2-export.binary BINARY"