Internal/multideploy support (#238)

* all done.. but will it work? * fix -p * added multiinstance support * moved to standard port 80 * fixed .env for deployment on dev-5 * use correct version for cli docker * added DSPACE_VER * remove automatic deploy * update erasing database * space issues * renames and redo deploy * redeploy always * set correct instance * correct message * further separation, better comments * get rid of space at end attempt 0 * erase trailing space attempt 1 * erase trailing space attempt 2 * erase trailing space attempt 3 * erase trailing space attempt 4
dataquest-dev · Jun 19, 2024 · 6f91d24 · 6f91d24
1 parent 965b387
commit 6f91d24
Show file tree

Hide file tree

Showing 6 changed files with 83 additions and 61 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -2,13 +2,6 @@
 name: Deploy DSpace
 
 on:
-  workflow_run:
-    workflows: ["Docker images", "Erase database"]
-    types:
-      - completed
-    branches:
-      - dtq-dev
-      - dtq-dev-present
   workflow_dispatch:
 
 jobs:
@@ -25,12 +18,23 @@ jobs:
 
           touch .env.dev-5 || true
           echo DSPACE_REST_HOST=dev-5.pc > .env.dev-5
-          echo REST_URL=http://dev-5.pc:8080/server >> .env.dev-5
-          echo UI_URL=http://dev-5.pc >> .env.dev-5
+          echo REST_URL=http://dev-5.pc:8082/server >> .env.dev-5
+          echo UI_URL=http://dev-5.pc:82 >> .env.dev-5
           echo DSPACE_REST_IMAGE=dataquest/dspace:dspace-7_x >> .env.dev-5
           echo DOCKER_OWNER=dataquest >> .env.dev-5
           ## echo NODE_ENV="development" >> .env.dev-5
           echo DSPACE_UI_IMAGE="dataquest/dspace-angular:dspace-7_x" >> .env.dev-5
+          echo INSTANCE=2 >> .env.dev-5
+          echo DSPACE_VER="dspace-7_x" >> .env.dev-5
+          
+          # echo S3_STORAGE=1 >> .env.dev-5
+          # echo S3_ENABLED=true >> .env.dev-5
+          # echo S3_RELATIVE_PATH=true >> .env.dev-5
+          # echo S3_BUCKET=dev-5 >> .env.dev-5
+          # echo S3_SUBFOLDER=seven-five >> .env.dev-5
+          # echo S3_ACCESS=${{ secrets.AMAZON_ACCESS }} >> .env.dev-5
+          # echo S3_SECRET=${{ secrets.AMAZON_SECRET }} >> .env.dev-5
+          # echo S3_REGION_NAME=us-east-1 >> .env.dev-5
 
           export ENVFILE=$(pwd)/.env.dev-5
-          ./start.sh
+          ./start.sh dspace-2
diff --git a/.github/workflows/erase_db.yml b/.github/workflows/erase_db.yml
@@ -4,28 +4,24 @@ on:
   workflow_dispatch:
 
 jobs:
-  deploy:
+  restart_db:
+    env: 
+      INSTANCE: 2
     runs-on: dspace-dep-1
     steps:
       - uses: actions/checkout@v3
         with:
           submodules: false
-      - name: deploy
+      - name: stop and remove containers
         run: |
-          cd $GITHUB_WORKSPACE/build-scripts/run/
-          pwd
-
-          touch .env.dev-5 || true
-          echo DSPACE_REST_HOST=dev-5.pc > .env.dev-5
-          echo REST_URL=http://dev-5.pc:8080/server >> .env.dev-5
-          echo UI_URL=http://dev-5.pc >> .env.dev-5
-          echo DSPACE_REST_IMAGE=dataquest/dspace:dspace-7_x >> .env.dev-5
-          echo DOCKER_OWNER=dataquest >> .env.dev-5
-          ## echo NODE_ENV="development" >> .env.dev-5
-          echo DSPACE_UI_IMAGE="dataquest/dspace-angular:dspace-7_x" >> .env.dev-5
-
-          export ENVFILE=$(pwd)/.env.dev-5
-          docker stop dspacedb dspace dspace-angular
-          docker rm dspacedb dspace dspace-angular
-          docker volume rm dq-d7_pgdata 
+          docker stop dspacedb$INSTANCE dspace$INSTANCE dspace-angular$INSTANCE
+          docker rm dspacedb$INSTANCE dspace$INSTANCE dspace-angular$INSTANCE
+      - name: remove volumes
+        if: '!cancelled()'
+        run: |
+          docker volume rm dspace-${{env.INSTANCE}}_pgdata dspace-${{env.INSTANCE}}_assetstore
+      - name: redeploy
+        if: '!cancelled()'
+        run: |
+          curl -H "Accept: application/vnd.github.everest-preview+json" -H "Authorization: token ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }}" --request POST https://api.github.com/repos/dataquest-dev/dspace-angular/actions/workflows/deploy.yml/dispatches --data "{\"ref\":\"refs/heads/internal/multideploy-support\"}"
           
diff --git a/build-scripts/run/start.sh b/build-scripts/run/start.sh
@@ -11,15 +11,15 @@ docker pull $DSPACE_UI_IMAGE
 
 pushd ../..
 docker-compose --env-file $ENVFILE -f docker/docker-compose.yml -f docker/docker-compose-rest.yml pull
-docker-compose --env-file $ENVFILE -p dq-d7 -f docker/docker-compose.yml -f docker/docker-compose-rest.yml up -d --no-build
+docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/docker-compose.yml -f docker/docker-compose-rest.yml up -d --no-build
 popd
 
 # Create admin user
 # set DOCKER_OWNER to match our image (see cli.yml)
 pushd ../..
-docker-compose --env-file $ENVFILE -p dq-d7 -f docker/matomo-w-db.yml pull
-docker-compose --env-file $ENVFILE -p dq-d7 -f docker/matomo-w-db.yml up -d --no-build
-docker-compose --env-file $ENVFILE -p dq-d7 -f docker/cli.yml run --rm dspace-cli create-administrator -e [email protected] -f admin -l user -p admin -c en -o dataquest
-docker-compose --env-file $ENVFILE -p dq-d7 -f docker/cli.yml run --rm dspace-cli user --add -m [email protected] -g meno -s priezvisko -l en -p user -o dataquest
-docker-compose --env-file $ENVFILE -p dq-d7 -f docker/cli.yml run --rm dspace-cli version
+docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/matomo-w-db.yml pull
+docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/matomo-w-db.yml up -d --no-build
+docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/cli.yml run --rm dspace-cli create-administrator -e [email protected] -f admin -l user -p admin -c en -o dataquest
+docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/cli.yml run --rm dspace-cli user --add -m [email protected] -g meno -s priezvisko -l en -p user -o dataquest
+docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/cli.yml run --rm dspace-cli version
 popd
diff --git a/docker/cli.yml b/docker/cli.yml
@@ -16,7 +16,7 @@ version: "3.7"
 
 services:
   dspace-cli:
-    image: "${DOCKER_OWNER:-dspace}/dspace-cli:${DSPACE_VER:-dspace-7_x}"
+    image: "${DOCKER_OWNER:-dataquest}/dspace-cli:${DSPACE_VER:-dspace-7_x}"
     container_name: dspace-cli
     environment:
       # Below syntax may look odd, but it is how to override dspace.cfg settings via env variables.
@@ -26,9 +26,21 @@ services:
       # dspace.dir
       dspace__P__dir: /dspace
       # db.url: Ensure we are using the 'dspacedb' image for our database
-      db__P__url: 'jdbc:postgresql://dspacedb:5432/dspace'
+      db__P__url: 'jdbc:postgresql://dspacedb:543${INSTANCE}/dspace'
       # solr.server: Ensure we are using the 'dspacesolr' image for Solr
-      solr__P__server: http://dspacesolr:8983/solr
+
+      solr__P__server: http://dspacesolr:898${INSTANCE}/solr
+
+      # S3
+      assetstore__P__index__P__primary: ${S3_STORAGE:-0}
+      assetstore__P__s3__P__enabled: ${S3_ENABLED:-false}
+      assetstore__P__s3__P__useRelativePath: ${S3_RELATIVE_PATH:-false}
+      assetstore__P__s3__P__bucketName: ${S3_BUCKET:-bucket-for-dspace}
+      assetstore__P__s3__P__subfolder: ${S3_SUBFOLDER:-}
+      assetstore__P__s3__P__awsAccessKey: ${S3_ACCESS:-}
+      assetstore__P__s3__P__awsSecretKey: ${S3_SECRET:-}
+      assetstore__P__s3__P__awsRegionName: ${S3_REGION_NAME:-}   
+
     volumes:
     - "assetstore:/dspace/assetstore"
     entrypoint: /dspace/bin/dspace

diff --git a/docker/docker-compose-rest.yml b/docker/docker-compose-rest.yml
@@ -17,30 +17,39 @@ services:
   # DSpace (backend) webapp container
   dspace:
     restart: unless-stopped
-    container_name: dspace
+    container_name: dspace${INSTANCE}
     environment:
       # Below syntax may look odd, but it is how to override dspace.cfg settings via env variables.
       # See https://github.com/DSpace/DSpace/blob/main/dspace/config/config-definition.xml
       # __P__ => "." (e.g. dspace__P__dir => dspace.dir)
       # __D__ => "-" (e.g. google__D__metadata => google-metadata)
       # dspace.dir, dspace.server.url, dspace.ui.url and dspace.name
       dspace__P__dir: /dspace
-      dspace__P__server__P__url: ${REST_URL:-http://localhost:8080/server}
-      dspace__P__ui__P__url: ${UI_URL:-http://localhost:4000}
+      dspace__P__server__P__url: ${REST_URL:-http://127.0.0.1:8080/server}
+      dspace__P__ui__P__url: ${UI_URL:-http://127.0.0.1:4000}
       dspace__P__name: 'DSpace Started with Docker Compose'
       # db.url: Ensure we are using the 'dspacedb' image for our database
-      db__P__url: 'jdbc:postgresql://dspacedb:5432/dspace'
+      db__P__url: 'jdbc:postgresql://dspacedb:543${INSTANCE}/dspace'
       # solr.server: Ensure we are using the 'dspacesolr' image for Solr
-      solr__P__server: http://dspacesolr:8983/solr
-    image: ${DSPACE_REST_IMAGE:-dataquest/dspace:dspace-7_x-test}
+      solr__P__server: http://dspacesolr:898${INSTANCE}/solr
+      #S3 config
+      assetstore__P__index__P__primary: ${S3_STORAGE:-0}
+      assetstore__P__s3__P__enabled: ${S3_ENABLED:-false}
+      assetstore__P__s3__P__useRelativePath: ${S3_RELATIVE_PATH:-false}
+      assetstore__P__s3__P__bucketName: ${S3_BUCKET:-bucket-for-dspace}
+      assetstore__P__s3__P__subfolder: ${S3_SUBFOLDER:-}
+      assetstore__P__s3__P__awsAccessKey: ${S3_ACCESS:-}
+      assetstore__P__s3__P__awsSecretKey: ${S3_SECRET:-}
+      assetstore__P__s3__P__awsRegionName: ${S3_REGION_NAME:-}
+    image: ${DSPACE_REST_IMAGE:-dataquest/dspace:dtq-dev-7.5}
     depends_on:
     - dspacedb
     networks:
       dspacenet:
     ports:
-    - published: 8080
+    - published: 808${INSTANCE}
       target: 8080
-    - published: 8000
+    - published: 800${INSTANCE}
       target: 8000
     stdin_open: true
     tty: true
@@ -56,29 +65,30 @@ services:
     - /bin/bash
     - '-c'
     - |
-      while (!</dev/tcp/dspacedb/5432) > /dev/null 2>&1; do sleep 1; done;
-      /dspace/bin/dspace database migrate
+      while (!</dev/tcp/dspacedb/543${INSTANCE}) > /dev/null 2>&1; do sleep 1; done;
+      /dspace/bin/dspace database migrate force
       custom_run.sh
   # DSpace database container
   dspacedb:
     restart: unless-stopped
-    container_name: dspacedb
+    container_name: dspacedb${INSTANCE}
     environment:
       PGDATA: /pgdata
     image: dspace/dspace-postgres-pgcrypto
     networks:
       dspacenet:
     ports:
-    - published: 5432
-      target: 5432
+    - published: 543${INSTANCE}
+      target: 543${INSTANCE}
     stdin_open: true
     tty: true
     volumes:
     - pgdata:/pgdata
+    command: -p 543${INSTANCE}
   # DSpace Solr container
   dspacesolr:
     restart: unless-stopped
-    container_name: dspacesolr
+    container_name: dspacesolr${INSTANCE}
     # Uses official Solr image at https://hub.docker.com/_/solr/
     image: solr:8.11-slim
     # Needs main 'dspace' container to start first to guarantee access to solr_configs
@@ -87,8 +97,8 @@ services:
     networks:
       dspacenet:
     ports:
-    - published: 8983
-      target: 8983
+    - published: 898${INSTANCE}
+      target: 898${INSTANCE}
     stdin_open: true
     tty: true
     working_dir: /var/solr/data
@@ -113,7 +123,7 @@ services:
       cp -r -u /opt/solr/server/solr/configsets/dspace/search/* search
       precreate-core statistics /opt/solr/server/solr/configsets/dspace/statistics
       cp -r -u /opt/solr/server/solr/configsets/dspace/statistics/* statistics
-      exec solr -f -m 4g
+      exec solr -p 898${INSTANCE} -f -m 4g
 volumes:
   assetstore:
   pgdata:

diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -15,15 +15,15 @@ networks:
 services:
   dspace-angular:
     restart: unless-stopped
-    container_name: dspace-angular
+    container_name: dspace-angular${INSTANCE}
     environment:
       DSPACE_UI_SSL: 'false'
       DSPACE_UI_HOST: dspace-angular
-      DSPACE_UI_PORT: '4000'
+      DSPACE_UI_PORT: 400${INSTANCE}
       DSPACE_UI_NAMESPACE: /
       DSPACE_REST_SSL: 'false'
       DSPACE_REST_HOST: ${DSPACE_REST_HOST:-localhost}
-      DSPACE_REST_PORT: 8080
+      DSPACE_REST_PORT: 808${INSTANCE}
       DSPACE_REST_NAMESPACE: /server
     image: ${DSPACE_UI_IMAGE:-dataquest/dspace-angular:dspace-7_x}
     build:
@@ -32,9 +32,9 @@ services:
     networks:
       dspacenet:
     ports:
-    - published: 4000
-      target: 4000
-    - published: 9876
-      target: 9876
+    - published: 400${INSTANCE}
+      target: 400${INSTANCE}
+    - published: 987${INSTANCE}
+      target: 987${INSTANCE}
     stdin_open: true
     tty: true