From 6f91d24ecc27b195b04642532ca003ecfcc36272 Mon Sep 17 00:00:00 2001 From: MajoBerger <88670521+MajoBerger@users.noreply.github.com> Date: Thu, 29 Jun 2023 08:07:02 +0200 Subject: [PATCH] Internal/multideploy support (#238) * all done.. but will it work? * fix -p * added multiinstance support * moved to standard port 80 * fixed .env for deployment on dev-5 * use correct version for cli docker * added DSPACE_VER * remove automatic deploy * update erasing database * space issues * renames and redo deploy * redeploy always * set correct instance * correct message * further separation, better comments * get rid of space at end attempt 0 * erase trailing space attempt 1 * erase trailing space attempt 2 * erase trailing space attempt 3 * erase trailing space attempt 4 --- .github/workflows/deploy.yml | 24 +++++++++++-------- .github/workflows/erase_db.yml | 32 +++++++++++-------------- build-scripts/run/start.sh | 12 +++++----- docker/cli.yml | 18 +++++++++++--- docker/docker-compose-rest.yml | 44 +++++++++++++++++++++------------- docker/docker-compose.yml | 14 +++++------ 6 files changed, 83 insertions(+), 61 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7c1eae926c5..5775ed1a15a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,13 +2,6 @@ name: Deploy DSpace on: - workflow_run: - workflows: ["Docker images", "Erase database"] - types: - - completed - branches: - - dtq-dev - - dtq-dev-present workflow_dispatch: jobs: @@ -25,12 +18,23 @@ jobs: touch .env.dev-5 || true echo DSPACE_REST_HOST=dev-5.pc > .env.dev-5 - echo REST_URL=http://dev-5.pc:8080/server >> .env.dev-5 - echo UI_URL=http://dev-5.pc >> .env.dev-5 + echo REST_URL=http://dev-5.pc:8082/server >> .env.dev-5 + echo UI_URL=http://dev-5.pc:82 >> .env.dev-5 echo DSPACE_REST_IMAGE=dataquest/dspace:dspace-7_x >> .env.dev-5 echo DOCKER_OWNER=dataquest >> .env.dev-5 ## echo NODE_ENV="development" >> .env.dev-5 echo DSPACE_UI_IMAGE="dataquest/dspace-angular:dspace-7_x" >> .env.dev-5 + echo INSTANCE=2 >> .env.dev-5 + echo DSPACE_VER="dspace-7_x" >> .env.dev-5 + + # echo S3_STORAGE=1 >> .env.dev-5 + # echo S3_ENABLED=true >> .env.dev-5 + # echo S3_RELATIVE_PATH=true >> .env.dev-5 + # echo S3_BUCKET=dev-5 >> .env.dev-5 + # echo S3_SUBFOLDER=seven-five >> .env.dev-5 + # echo S3_ACCESS=${{ secrets.AMAZON_ACCESS }} >> .env.dev-5 + # echo S3_SECRET=${{ secrets.AMAZON_SECRET }} >> .env.dev-5 + # echo S3_REGION_NAME=us-east-1 >> .env.dev-5 export ENVFILE=$(pwd)/.env.dev-5 - ./start.sh + ./start.sh dspace-2 diff --git a/.github/workflows/erase_db.yml b/.github/workflows/erase_db.yml index ed7aa0bc777..ea0211d0a59 100644 --- a/.github/workflows/erase_db.yml +++ b/.github/workflows/erase_db.yml @@ -4,28 +4,24 @@ on: workflow_dispatch: jobs: - deploy: + restart_db: + env: + INSTANCE: 2 runs-on: dspace-dep-1 steps: - uses: actions/checkout@v3 with: submodules: false - - name: deploy + - name: stop and remove containers run: | - cd $GITHUB_WORKSPACE/build-scripts/run/ - pwd - - touch .env.dev-5 || true - echo DSPACE_REST_HOST=dev-5.pc > .env.dev-5 - echo REST_URL=http://dev-5.pc:8080/server >> .env.dev-5 - echo UI_URL=http://dev-5.pc >> .env.dev-5 - echo DSPACE_REST_IMAGE=dataquest/dspace:dspace-7_x >> .env.dev-5 - echo DOCKER_OWNER=dataquest >> .env.dev-5 - ## echo NODE_ENV="development" >> .env.dev-5 - echo DSPACE_UI_IMAGE="dataquest/dspace-angular:dspace-7_x" >> .env.dev-5 - - export ENVFILE=$(pwd)/.env.dev-5 - docker stop dspacedb dspace dspace-angular - docker rm dspacedb dspace dspace-angular - docker volume rm dq-d7_pgdata + docker stop dspacedb$INSTANCE dspace$INSTANCE dspace-angular$INSTANCE + docker rm dspacedb$INSTANCE dspace$INSTANCE dspace-angular$INSTANCE + - name: remove volumes + if: '!cancelled()' + run: | + docker volume rm dspace-${{env.INSTANCE}}_pgdata dspace-${{env.INSTANCE}}_assetstore + - name: redeploy + if: '!cancelled()' + run: | + curl -H "Accept: application/vnd.github.everest-preview+json" -H "Authorization: token ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }}" --request POST https://api.github.com/repos/dataquest-dev/dspace-angular/actions/workflows/deploy.yml/dispatches --data "{\"ref\":\"refs/heads/internal/multideploy-support\"}" diff --git a/build-scripts/run/start.sh b/build-scripts/run/start.sh index 0aa268964e4..1e23583e0bb 100755 --- a/build-scripts/run/start.sh +++ b/build-scripts/run/start.sh @@ -11,15 +11,15 @@ docker pull $DSPACE_UI_IMAGE pushd ../.. docker-compose --env-file $ENVFILE -f docker/docker-compose.yml -f docker/docker-compose-rest.yml pull -docker-compose --env-file $ENVFILE -p dq-d7 -f docker/docker-compose.yml -f docker/docker-compose-rest.yml up -d --no-build +docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/docker-compose.yml -f docker/docker-compose-rest.yml up -d --no-build popd # Create admin user # set DOCKER_OWNER to match our image (see cli.yml) pushd ../.. -docker-compose --env-file $ENVFILE -p dq-d7 -f docker/matomo-w-db.yml pull -docker-compose --env-file $ENVFILE -p dq-d7 -f docker/matomo-w-db.yml up -d --no-build -docker-compose --env-file $ENVFILE -p dq-d7 -f docker/cli.yml run --rm dspace-cli create-administrator -e test@test.edu -f admin -l user -p admin -c en -o dataquest -docker-compose --env-file $ENVFILE -p dq-d7 -f docker/cli.yml run --rm dspace-cli user --add -m user@test.edu -g meno -s priezvisko -l en -p user -o dataquest -docker-compose --env-file $ENVFILE -p dq-d7 -f docker/cli.yml run --rm dspace-cli version +docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/matomo-w-db.yml pull +docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/matomo-w-db.yml up -d --no-build +docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/cli.yml run --rm dspace-cli create-administrator -e test@test.edu -f admin -l user -p admin -c en -o dataquest +docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/cli.yml run --rm dspace-cli user --add -m user@test.edu -g meno -s priezvisko -l en -p user -o dataquest +docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/cli.yml run --rm dspace-cli version popd diff --git a/docker/cli.yml b/docker/cli.yml index 54b83d45036..d568b50b65a 100644 --- a/docker/cli.yml +++ b/docker/cli.yml @@ -16,7 +16,7 @@ version: "3.7" services: dspace-cli: - image: "${DOCKER_OWNER:-dspace}/dspace-cli:${DSPACE_VER:-dspace-7_x}" + image: "${DOCKER_OWNER:-dataquest}/dspace-cli:${DSPACE_VER:-dspace-7_x}" container_name: dspace-cli environment: # Below syntax may look odd, but it is how to override dspace.cfg settings via env variables. @@ -26,9 +26,21 @@ services: # dspace.dir dspace__P__dir: /dspace # db.url: Ensure we are using the 'dspacedb' image for our database - db__P__url: 'jdbc:postgresql://dspacedb:5432/dspace' + db__P__url: 'jdbc:postgresql://dspacedb:543${INSTANCE}/dspace' # solr.server: Ensure we are using the 'dspacesolr' image for Solr - solr__P__server: http://dspacesolr:8983/solr + + solr__P__server: http://dspacesolr:898${INSTANCE}/solr + + # S3 + assetstore__P__index__P__primary: ${S3_STORAGE:-0} + assetstore__P__s3__P__enabled: ${S3_ENABLED:-false} + assetstore__P__s3__P__useRelativePath: ${S3_RELATIVE_PATH:-false} + assetstore__P__s3__P__bucketName: ${S3_BUCKET:-bucket-for-dspace} + assetstore__P__s3__P__subfolder: ${S3_SUBFOLDER:-} + assetstore__P__s3__P__awsAccessKey: ${S3_ACCESS:-} + assetstore__P__s3__P__awsSecretKey: ${S3_SECRET:-} + assetstore__P__s3__P__awsRegionName: ${S3_REGION_NAME:-} + volumes: - "assetstore:/dspace/assetstore" entrypoint: /dspace/bin/dspace diff --git a/docker/docker-compose-rest.yml b/docker/docker-compose-rest.yml index ceea4c78a46..b7f8aada460 100644 --- a/docker/docker-compose-rest.yml +++ b/docker/docker-compose-rest.yml @@ -17,7 +17,7 @@ services: # DSpace (backend) webapp container dspace: restart: unless-stopped - container_name: dspace + container_name: dspace${INSTANCE} environment: # Below syntax may look odd, but it is how to override dspace.cfg settings via env variables. # See https://github.com/DSpace/DSpace/blob/main/dspace/config/config-definition.xml @@ -25,22 +25,31 @@ services: # __D__ => "-" (e.g. google__D__metadata => google-metadata) # dspace.dir, dspace.server.url, dspace.ui.url and dspace.name dspace__P__dir: /dspace - dspace__P__server__P__url: ${REST_URL:-http://localhost:8080/server} - dspace__P__ui__P__url: ${UI_URL:-http://localhost:4000} + dspace__P__server__P__url: ${REST_URL:-http://127.0.0.1:8080/server} + dspace__P__ui__P__url: ${UI_URL:-http://127.0.0.1:4000} dspace__P__name: 'DSpace Started with Docker Compose' # db.url: Ensure we are using the 'dspacedb' image for our database - db__P__url: 'jdbc:postgresql://dspacedb:5432/dspace' + db__P__url: 'jdbc:postgresql://dspacedb:543${INSTANCE}/dspace' # solr.server: Ensure we are using the 'dspacesolr' image for Solr - solr__P__server: http://dspacesolr:8983/solr - image: ${DSPACE_REST_IMAGE:-dataquest/dspace:dspace-7_x-test} + solr__P__server: http://dspacesolr:898${INSTANCE}/solr + #S3 config + assetstore__P__index__P__primary: ${S3_STORAGE:-0} + assetstore__P__s3__P__enabled: ${S3_ENABLED:-false} + assetstore__P__s3__P__useRelativePath: ${S3_RELATIVE_PATH:-false} + assetstore__P__s3__P__bucketName: ${S3_BUCKET:-bucket-for-dspace} + assetstore__P__s3__P__subfolder: ${S3_SUBFOLDER:-} + assetstore__P__s3__P__awsAccessKey: ${S3_ACCESS:-} + assetstore__P__s3__P__awsSecretKey: ${S3_SECRET:-} + assetstore__P__s3__P__awsRegionName: ${S3_REGION_NAME:-} + image: ${DSPACE_REST_IMAGE:-dataquest/dspace:dtq-dev-7.5} depends_on: - dspacedb networks: dspacenet: ports: - - published: 8080 + - published: 808${INSTANCE} target: 8080 - - published: 8000 + - published: 800${INSTANCE} target: 8000 stdin_open: true tty: true @@ -56,29 +65,30 @@ services: - /bin/bash - '-c' - | - while (! /dev/null 2>&1; do sleep 1; done; - /dspace/bin/dspace database migrate + while (! /dev/null 2>&1; do sleep 1; done; + /dspace/bin/dspace database migrate force custom_run.sh # DSpace database container dspacedb: restart: unless-stopped - container_name: dspacedb + container_name: dspacedb${INSTANCE} environment: PGDATA: /pgdata image: dspace/dspace-postgres-pgcrypto networks: dspacenet: ports: - - published: 5432 - target: 5432 + - published: 543${INSTANCE} + target: 543${INSTANCE} stdin_open: true tty: true volumes: - pgdata:/pgdata + command: -p 543${INSTANCE} # DSpace Solr container dspacesolr: restart: unless-stopped - container_name: dspacesolr + container_name: dspacesolr${INSTANCE} # Uses official Solr image at https://hub.docker.com/_/solr/ image: solr:8.11-slim # Needs main 'dspace' container to start first to guarantee access to solr_configs @@ -87,8 +97,8 @@ services: networks: dspacenet: ports: - - published: 8983 - target: 8983 + - published: 898${INSTANCE} + target: 898${INSTANCE} stdin_open: true tty: true working_dir: /var/solr/data @@ -113,7 +123,7 @@ services: cp -r -u /opt/solr/server/solr/configsets/dspace/search/* search precreate-core statistics /opt/solr/server/solr/configsets/dspace/statistics cp -r -u /opt/solr/server/solr/configsets/dspace/statistics/* statistics - exec solr -f -m 4g + exec solr -p 898${INSTANCE} -f -m 4g volumes: assetstore: pgdata: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b223d9276b0..f81f0ad4be7 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -15,15 +15,15 @@ networks: services: dspace-angular: restart: unless-stopped - container_name: dspace-angular + container_name: dspace-angular${INSTANCE} environment: DSPACE_UI_SSL: 'false' DSPACE_UI_HOST: dspace-angular - DSPACE_UI_PORT: '4000' + DSPACE_UI_PORT: 400${INSTANCE} DSPACE_UI_NAMESPACE: / DSPACE_REST_SSL: 'false' DSPACE_REST_HOST: ${DSPACE_REST_HOST:-localhost} - DSPACE_REST_PORT: 8080 + DSPACE_REST_PORT: 808${INSTANCE} DSPACE_REST_NAMESPACE: /server image: ${DSPACE_UI_IMAGE:-dataquest/dspace-angular:dspace-7_x} build: @@ -32,9 +32,9 @@ services: networks: dspacenet: ports: - - published: 4000 - target: 4000 - - published: 9876 - target: 9876 + - published: 400${INSTANCE} + target: 400${INSTANCE} + - published: 987${INSTANCE} + target: 987${INSTANCE} stdin_open: true tty: true