[devOps] reusing erase for import, using instances

dataquest-dev · Nov 7, 2023 · 397be81 · 397be81
1 parent d89d199
commit 397be81
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 18 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -7,6 +7,7 @@ on:
 jobs:
   deploy:
     runs-on: dspace-dep-1
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v3
         with:
@@ -40,15 +41,21 @@ jobs:
           export ENVFILE=$(pwd)/.env.dev-5
           ./start.sh dspace-5
 
+  deploy-import:
+    runs-on: dspace-dep-1
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+
       - name: deploy dspace-import on dev-5
         working-directory: build-scripts/run/
         run: |
           ENVFILE=/opt/dspace-envs/.env.dspace.imported.dev-5 ./start.sh dspace-5-import
-          sleep 120
-          echo "done with start.sh, going to request import"
-          curl -H "Accept: application/vnd.github.everest-preview+json" \
-          -H "Authorization: token ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }}" \
-          --request POST \
-          https://api.github.com/repos/dataquest-dev/\
-          dspace-angular/actions/workflows/import.yml/dispatches \
-          --data "{\"ref\":\"refs/heads/dtq-dev\"}"
+          /bin/bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://dev-5.pc:88/server/api)" != "200" ]]; do sleep 5; done'
+
+  post-deploy-import:
+    needs: deploy-import
+    uses: dataquest-dev/dspace-angular/.github/workflows/import.yml@dtq-dev
+    secrets: inherit
+    with:
+      INSTANCE: '8'
diff --git a/.github/workflows/erase_db.yml b/.github/workflows/erase_db.yml
@@ -1,6 +1,11 @@
 name: Erase database
 
 on:
+  workflow_call:
+    inputs:
+      INSTANCE:
+        required: true
+        type: string
   workflow_dispatch:
     inputs:
       INSTANCE:
@@ -13,24 +18,24 @@ on:
 
 
 jobs:
-  restart_db:
+  recreate_db:
+    runs-on: dspace-dep-1
+    timeout-minutes: 5
     env:
       INSTANCE: ${{ inputs.INSTANCE }}
-    runs-on: dspace-dep-1
     steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: false
+
       - name: stop and remove containers
         run: |
           docker stop dspacedb$INSTANCE dspace$INSTANCE dspace-angular$INSTANCE
           docker rm dspacedb$INSTANCE dspace$INSTANCE dspace-angular$INSTANCE
+
       - name: remove volumes
         if: '!cancelled()'
         run: |
           docker volume rm dspace-${{env.INSTANCE}}_pgdata dspace-${{env.INSTANCE}}_assetstore
+
       - name: redeploy
         if: '!cancelled()'
         run: |
           curl -H "Accept: application/vnd.github.everest-preview+json" -H "Authorization: token ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }}" --request POST https://api.github.com/repos/dataquest-dev/dspace-angular/actions/workflows/deploy.yml/dispatches --data "{\"ref\":\"refs/heads/dtq-dev\"}"
-
diff --git a/.github/workflows/import.yml b/.github/workflows/import.yml
@@ -1,32 +1,60 @@
 name: Import DSpace v5
 
 on:
+  workflow_call:
+    inputs:
+      INSTANCE:
+        required: true
+        type: string
   workflow_dispatch:
+    inputs:
+      INSTANCE:
+        required: true
+        default: '8'
+        type: choice
+        options:
+          - '8'
 
 jobs:
-  deploy:
+  pre-import-cleanup:
+    uses: dataquest-dev/dspace-angular/.github/workflows/erase_db.yml@dtq-dev
+    secrets: inherit
+    with:
+      INSTANCE: ${{ inputs.INSTANCE }}
+
+  import:
     runs-on: dspace-dep-1
+    needs: pre-import-cleanup
     steps:
+
       - uses: actions/checkout@v4
         with:
           repository: dataquest-dev/dspace-python-api
           path: dspace-python-api
           ref: 'refactor_jm'
 
+      - name: info
+        run: |
+          docker ps -a
+
       - name: deploy dspace-import on dev-5
         id: import
         working-directory: dspace-python-api/scripts
         run: |
           docker stop dspace-import-db5 || true
           DB5PORT=15432
-          cid=$(docker run -d --name dspace-import-db5 -v $pwd:/dq/scripts -v $(pwd)/../input/dump:/dq/dump \
+          cid=$(docker run -d --rm --name dspace-import-db5 -v $pwd:/dq/scripts -v $(pwd)/../input/dump:/dq/dump \
             -p 127.0.0.1:$DB5PORT:5432 -e POSTGRES_DB=empty -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=dspace postgres /bin/bash -c "cd /dq/scripts && ./init.dspacedb5.sh")
           echo "cid=$cid" >> $GITHUB_OUTPUT
           sleep 10
           cd ../src
           # cleanup resume
           rm __temp/resume/*.json || true
-          echo python repo_import.py --resume=false
+          python repo_import.py --resume=false \
+            --config=backend.endpoint=http://dev-5.pc:88/server/api/ \
+            --config=db_dspace_7.port=5438 \
+            --config=db_dspace_5.port=$DB5PORT \
+            --config=db_utilities_5.port=$DB5PORT
 
       - name: cleanup
         run: |

diff --git a/build-scripts/run/start.sh b/build-scripts/run/start.sh
@@ -12,7 +12,7 @@ docker pull $DSPACE_UI_IMAGE
 
 pushd ../..
 docker compose --env-file $ENVFILE -f docker/docker-compose.yml -f docker/docker-compose-rest.yml pull
-docker compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/docker-compose.yml -f docker/docker-compose-rest.yml up -d --no-build
+docker compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/docker-compose.yml -f docker/docker-compose-rest.yml up -d --no-build --remove-orphans
 popd
 
 # Create admin user