diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ac3aeac41a33..59cf1935fb64 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,31 +92,31 @@ jobs: path: 'dspace/target/site/jacoco-aggregate/jacoco.xml' retention-days: 14 - # Codecov upload is a separate job in order to allow us to restart this separate from the entire build/test - # job above. This is necessary because Codecov uploads seem to randomly fail at times. - # See https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954 - codecov: - # Must run after 'tests' job above - needs: tests - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 + # # Codecov upload is a separate job in order to allow us to restart this separate from the entire build/test + # # job above. This is necessary because Codecov uploads seem to randomly fail at times. + # # See https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954 + # codecov: + # # Must run after 'tests' job above + # needs: tests + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v3 - # Download artifacts from previous 'tests' job - - name: Download coverage artifacts - uses: actions/download-artifact@v3 + # # Download artifacts from previous 'tests' job + # - name: Download coverage artifacts + # uses: actions/download-artifact@v3 - # Now attempt upload to Codecov using its action. - # NOTE: We use a retry action to retry the Codecov upload if it fails the first time. - # - # Retry action: https://github.com/marketplace/actions/retry-action - # Codecov action: https://github.com/codecov/codecov-action - - name: Upload coverage to Codecov.io - uses: Wandalen/wretry.action@v1.0.36 - with: - action: codecov/codecov-action@v3 - # Try upload 5 times max - attempt_limit: 5 - # Run again in 30 seconds - attempt_delay: 30000 + # # Now attempt upload to Codecov using its action. + # # NOTE: We use a retry action to retry the Codecov upload if it fails the first time. + # # + # # Retry action: https://github.com/marketplace/actions/retry-action + # # Codecov action: https://github.com/codecov/codecov-action + # - name: Upload coverage to Codecov.io + # uses: Wandalen/wretry.action@v1.0.36 + # with: + # action: codecov/codecov-action@v3 + # # Try upload 5 times max + # attempt_limit: 5 + # # Run again in 30 seconds + # attempt_delay: 30000 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 371bfe00b023..6984b85d942a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,13 +1,14 @@ # DSpace Docker image build for hub.docker.com name: Docker images -# Run this Build for all pushes to customer/uk branch +# Run this Build for all pushes to dtq-dev branch # Also run for PRs to ensure PR doesn't break Docker build process on: push: branches: - dtq-dev - customer/* + pull_request: workflow_dispatch: @@ -15,229 +16,22 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: - docker: - # Ensure this job never runs on forked repos. It's only executed for our repo - if: github.repository == 'dataquest-dev/dspace' - runs-on: ubuntu-latest - env: - # Define tags to use for Docker images based on Git tags/branches (for docker/metadata-action) - # For a new commit on default branch (main), use the literal tag 'dspace-7_x' on Docker image. - # For a new commit on other branches, use the branch name as the tag for Docker image. - # For a new tag, copy that tag name as the tag for Docker image. - IMAGE_TAGS: | - type=raw,value=dspace-7_x,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} - type=ref,event=branch,enable=${{ !endsWith(github.ref, github.event.repository.default_branch) }} - type=ref,event=tag - # Define default tag "flavor" for docker/metadata-action per - # https://github.com/docker/metadata-action#flavor-input - # We turn off 'latest' tag by default. - TAGS_FLAVOR: | - latest=false - # Architectures / Platforms for which we will build Docker images - # If this is a PR, we ONLY build for AMD64. For PRs we only do a sanity check test to ensure Docker builds work. - # If this is NOT a PR (e.g. a tag or merge commit), also build for ARM64. NOTE: The ARM64 build takes MUCH - # longer (around 45mins or so) which is why we only run it when pushing a new Docker image. - PLATFORMS: linux/amd64${{ github.event_name != 'pull_request' && ', linux/arm64' || '' }} - - steps: - # https://github.com/actions/checkout - - name: Checkout codebase - uses: actions/checkout@v3 - - - name: Add version - run: python scripts/sourceversion.py > dspace/config/VERSION_D.txt - - # https://github.com/docker/setup-buildx-action - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v2 - - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU emulation to build for multiple architectures - uses: docker/setup-qemu-action@v2 - - # https://github.com/docker/login-action - - name: Login to DockerHub - # Only login if not a PR, as PRs only trigger a Docker build and not a push - if: github.event_name != 'pull_request' - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - - #################################################### - # Build/Push the 'dataquest/dspace-dependencies' image - #################################################### - # https://github.com/docker/metadata-action - # Get Metadata for docker_build_deps step below - - name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-dependencies' image - id: meta_build_deps - uses: docker/metadata-action@v4 - with: - images: dataquest/dspace-dependencies - tags: ${{ env.IMAGE_TAGS }} - flavor: ${{ env.TAGS_FLAVOR }} - - # https://github.com/docker/build-push-action - - name: Build and push 'dspace-dependencies' image - id: docker_build_deps - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile.dependencies - platforms: ${{ env.PLATFORMS }} - # For pull requests, we run the Docker build (to ensure no PR changes break the build), - # but we ONLY do an image push to DockerHub if it's NOT a PR - push: ${{ github.event_name != 'pull_request' }} - # Use tags / labels provided by 'docker/metadata-action' above - tags: ${{ steps.meta_build_deps.outputs.tags }} - labels: ${{ steps.meta_build_deps.outputs.labels }} - - ####################################### - # Build/Push the 'dataquest/dspace' image - ####################################### - # Get Metadata for docker_build step below - - name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace' image - id: meta_build - uses: docker/metadata-action@v4 - with: - images: dataquest/dspace - tags: ${{ env.IMAGE_TAGS }} - flavor: ${{ env.TAGS_FLAVOR }} - - - name: Build and push 'dspace' image - id: docker_build - uses: docker/build-push-action@v3 - with: - context: . - file: ./Dockerfile - platforms: ${{ env.PLATFORMS }} - # For pull requests, we run the Docker build (to ensure no PR changes break the build), - # but we ONLY do an image push to DockerHub if it's NOT a PR - push: ${{ github.event_name != 'pull_request' }} - # Use tags / labels provided by 'docker/metadata-action' above - tags: ${{ steps.meta_build.outputs.tags }} - labels: ${{ steps.meta_build.outputs.labels }} - - ##################################################### - # Build/Push the 'dataquest/dspace' image ('-test' tag) - ##################################################### - # Get Metadata for docker_build_test step below - - name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-test' image - id: meta_build_test - uses: docker/metadata-action@v4 - with: - images: dataquest/dspace - tags: ${{ env.IMAGE_TAGS }} - # As this is a test/development image, its tags are all suffixed with "-test". Otherwise, it uses the same - # tagging logic as the primary 'dataquest/dspace' image above. - flavor: ${{ env.TAGS_FLAVOR }} - suffix=-test - - - name: Build and push 'dspace-test' image - id: docker_build_test - uses: docker/build-push-action@v3 - with: - context: . - file: ./Dockerfile.test - platforms: ${{ env.PLATFORMS }} - # For pull requests, we run the Docker build (to ensure no PR changes break the build), - # but we ONLY do an image push to DockerHub if it's NOT a PR - push: ${{ github.event_name != 'pull_request' }} - # Use tags / labels provided by 'docker/metadata-action' above - tags: ${{ steps.meta_build_test.outputs.tags }} - labels: ${{ steps.meta_build_test.outputs.labels }} - - ########################################### - # Build/Push the 'dataquest/dspace-cli' image - ########################################### - # Get Metadata for docker_build_test step below - - name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-cli' image - id: meta_build_cli - uses: docker/metadata-action@v4 - with: - images: dataquest/dspace-cli - tags: ${{ env.IMAGE_TAGS }} - flavor: ${{ env.TAGS_FLAVOR }} - - - name: Build and push 'dspace-cli' image - id: docker_build_cli - uses: docker/build-push-action@v3 - with: - context: . - file: ./Dockerfile.cli - platforms: ${{ env.PLATFORMS }} - # For pull requests, we run the Docker build (to ensure no PR changes break the build), - # but we ONLY do an image push to DockerHub if it's NOT a PR - push: ${{ github.event_name != 'pull_request' }} - # Use tags / labels provided by 'docker/metadata-action' above - tags: ${{ steps.meta_build_cli.outputs.tags }} - labels: ${{ steps.meta_build_cli.outputs.labels }} - - - name: redeploy - if: '!cancelled()' - run: | - curl -H "Accept: application/vnd.github.everest-preview+json" \ - -H "Authorization: token ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }}" \ - --request POST \ - https://api.github.com/repos/dataquest-dev/\ - dspace-angular/actions/workflows/deploy.yml/dispatches \ - --data "{\"ref\":\"refs/heads/dtq-dev\"}" - - #################################################### - # Build/Push the 'dspace/dspace-dependencies' image. - # This image is used by all other jobs. + #################################################### + # Build/Push the 'dataquest/dspace-dependencies' image. + # This image is used by all other DSpace build jobs. #################################################### dspace-dependencies: - # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace' - if: github.repository == 'dspace/dspace' - runs-on: ubuntu-latest - - steps: - # https://github.com/actions/checkout - - name: Checkout codebase - uses: actions/checkout@v3 - - # https://github.com/docker/setup-buildx-action - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v2 - - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU emulation to build for multiple architectures - uses: docker/setup-qemu-action@v2 - - # https://github.com/docker/login-action - - name: Login to DockerHub - # Only login if not a PR, as PRs only trigger a Docker build and not a push - if: github.event_name != 'pull_request' - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - - # https://github.com/docker/metadata-action - # Get Metadata for docker_build_deps step below - - name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-dependencies' image - id: meta_build_deps - uses: docker/metadata-action@v4 - with: - images: dspace/dspace-dependencies - tags: ${{ env.IMAGE_TAGS }} - flavor: ${{ env.TAGS_FLAVOR }} + # Ensure this job never runs on forked repos. It's only executed for 'dataquest/dspace' + if: github.repository == 'dataquest-dev/dspace' + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} + uses: ./.github/workflows/reusable-docker-build.yml + with: + build_id: dspace-dependencies + image_name: dataquest/dspace-dependencies + dockerfile_path: ./Dockerfile.dependencies - # https://github.com/docker/build-push-action - - name: Build and push 'dspace-dependencies' image - id: docker_build_deps - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile.dependencies - platforms: ${{ env.PLATFORMS }} - # For pull requests, we run the Docker build (to ensure no PR changes break the build), - # but we ONLY do an image push to DockerHub if it's NOT a PR - push: ${{ github.event_name != 'pull_request' }} - # Use tags / labels provided by 'docker/metadata-action' above - tags: ${{ steps.meta_build_deps.outputs.tags }} - labels: ${{ steps.meta_build_deps.outputs.labels }} ####################################### # Build/Push the 'dataquest/dspace' image @@ -247,52 +41,20 @@ jobs: if: github.repository == 'dataquest-dev/dspace' # Must run after 'dspace-dependencies' job above needs: dspace-dependencies - runs-on: ubuntu-latest - - steps: - # https://github.com/actions/checkout - - name: Checkout codebase - uses: actions/checkout@v3 - - # https://github.com/docker/setup-buildx-action - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v2 - - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU emulation to build for multiple architectures - uses: docker/setup-qemu-action@v2 - - # https://github.com/docker/login-action - - name: Login to DockerHub - # Only login if not a PR, as PRs only trigger a Docker build and not a push - if: github.event_name != 'pull_request' - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - - # Get Metadata for docker_build step below - - name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace' image - id: meta_build - uses: docker/metadata-action@v4 - with: - images: dataquest/dspace - tags: ${{ env.IMAGE_TAGS }} - flavor: ${{ env.TAGS_FLAVOR }} - - - name: Build and push 'dspace' image - id: docker_build - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - platforms: ${{ env.PLATFORMS }} - # For pull requests, we run the Docker build (to ensure no PR changes break the build), - # but we ONLY do an image push to DockerHub if it's NOT a PR - push: ${{ github.event_name != 'pull_request' }} - # Use tags / labels provided by 'docker/metadata-action' above - tags: ${{ steps.meta_build.outputs.tags }} - labels: ${{ steps.meta_build.outputs.labels }} + uses: ./.github/workflows/reusable-docker-build.yml + with: + build_id: dspace + image_name: dataquest/dspace + dockerfile_path: ./Dockerfile + run_python_version_script: true + python_version_script_dest: dspace/config/VERSION_D.txt + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} + # Enable redeploy of sandbox & demo if the branch for this image matches the deployment branch of + # these sites as specified in reusable-docker-build.xml + REDEPLOY_SANDBOX_URL: ${{ secrets.REDEPLOY_SANDBOX_URL }} + REDEPLOY_DEMO_URL: ${{ secrets.REDEPLOY_DEMO_URL }} ############################################################# # Build/Push the 'dataquest/dspace' image ('-test' tag) @@ -300,117 +62,36 @@ jobs: dspace-test: # Ensure this job never runs on forked repos. It's only executed for 'dataquest/dspace' if: github.repository == 'dataquest-dev/dspace' - # Must run after 'dspace-dependencies' job above + # Must run after 'dspace-dependencies' job above needs: dspace-dependencies - runs-on: ubuntu-latest - - steps: - # https://github.com/actions/checkout - - name: Checkout codebase - uses: actions/checkout@v3 - - # https://github.com/docker/setup-buildx-action - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v2 - - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU emulation to build for multiple architectures - uses: docker/setup-qemu-action@v2 - - # https://github.com/docker/login-action - - name: Login to DockerHub - # Only login if not a PR, as PRs only trigger a Docker build and not a push - if: github.event_name != 'pull_request' - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - - # Get Metadata for docker_build_test step below - - name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-test' image - id: meta_build_test - uses: docker/metadata-action@v4 - with: - images: dataquest/dspace - tags: ${{ env.IMAGE_TAGS }} - # As this is a test/development image, its tags are all suffixed with "-test". Otherwise, it uses the same - # tagging logic as the primary 'dataquest/dspace' image above. - flavor: ${{ env.TAGS_FLAVOR }} - suffix=-test - - - name: Build and push 'dspace-test' image - id: docker_build_test - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile.test - platforms: ${{ env.PLATFORMS }} - # For pull requests, we run the Docker build (to ensure no PR changes break the build), - # but we ONLY do an image push to DockerHub if it's NOT a PR - push: ${{ github.event_name != 'pull_request' }} - # Use tags / labels provided by 'docker/metadata-action' above - tags: ${{ steps.meta_build_test.outputs.tags }} - labels: ${{ steps.meta_build_test.outputs.labels }} + uses: ./.github/workflows/reusable-docker-build.yml + with: + build_id: dspace-test + image_name: dataquest/dspace + dockerfile_path: ./Dockerfile.test + # As this is a test/development image, its tags are all suffixed with "-test". Otherwise, it uses the same + # tagging logic as the primary 'dataquest/dspace' image above. + tags_flavor: suffix=-test + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} ########################################### # Build/Push the 'dataquest/dspace-cli' image ########################################### dspace-cli: # Ensure this job never runs on forked repos. It's only executed for 'dataquest/dspace' - if: github.repository == 'dspace/dspace' - # Must run after 'dspace-dependencies' job above + if: github.repository == 'dataquest-dev/dspace' + # Must run after 'dspace-dependencies' job above needs: dspace-dependencies - runs-on: ubuntu-latest - - steps: - # https://github.com/actions/checkout - - name: Checkout codebase - uses: actions/checkout@v3 - - # https://github.com/docker/setup-buildx-action - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v2 - - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU emulation to build for multiple architectures - uses: docker/setup-qemu-action@v2 - - # https://github.com/docker/login-action - - name: Login to DockerHub - # Only login if not a PR, as PRs only trigger a Docker build and not a push - if: github.event_name != 'pull_request' - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - - # Get Metadata for docker_build_test step below - - name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-cli' image - id: meta_build_cli - uses: docker/metadata-action@v4 - with: - images: dataquest/dspace-cli - tags: ${{ env.IMAGE_TAGS }} - flavor: ${{ env.TAGS_FLAVOR }} - - - name: Build and push 'dspace-cli' image - id: docker_build_cli - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile.cli - platforms: ${{ env.PLATFORMS }} - # For pull requests, we run the Docker build (to ensure no PR changes break the build), - # but we ONLY do an image push to DockerHub if it's NOT a PR - push: ${{ github.event_name != 'pull_request' }} - # Use tags / labels provided by 'docker/metadata-action' above - tags: ${{ steps.meta_build_cli.outputs.tags }} - labels: ${{ steps.meta_build_cli.outputs.labels }} - - deploy: - needs: docker - uses: dataquest-dev/dspace-angular/.github/workflows/deploy.yml@customer/uk - secrets: inherit + uses: ./.github/workflows/reusable-docker-build.yml + with: + build_id: dspace-cli + image_name: dataquest/dspace-cli + dockerfile_path: ./Dockerfile.cli + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} ########################################### @@ -418,161 +99,72 @@ jobs: ########################################### dspace-solr: # Ensure this job never runs on forked repos. It's only executed for 'dataquest/dspace' - if: github.repository == 'dspace/dspace' - runs-on: ubuntu-latest - - steps: - # https://github.com/actions/checkout - - name: Checkout codebase - uses: actions/checkout@v3 - - # https://github.com/docker/setup-buildx-action - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v2 - - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU emulation to build for multiple architectures - uses: docker/setup-qemu-action@v2 - - # https://github.com/docker/login-action - - name: Login to DockerHub - # Only login if not a PR, as PRs only trigger a Docker build and not a push - if: github.event_name != 'pull_request' - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - - # Get Metadata for docker_build_solr step below - - name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-solr' image - id: meta_build_solr - uses: docker/metadata-action@v4 - with: - images: dspace/dspace-solr - tags: ${{ env.IMAGE_TAGS }} - flavor: ${{ env.TAGS_FLAVOR }} + if: github.repository == 'dataquest-dev/dspace' + uses: ./.github/workflows/reusable-docker-build.yml + with: + build_id: dspace-solr + image_name: dataquest/dspace-solr + dockerfile_path: ./dspace/src/main/docker/dspace-solr/Dockerfile + # Must pass solrconfigs to the Dockerfile so that it can find the required Solr config files + dockerfile_additional_contexts: 'solrconfigs=./dspace/solr/' + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} + # Enable redeploy of sandbox & demo SOLR instance whenever dspace-solr image changes for deployed branch. + # These URLs MUST use different secrets than 'dspace/dspace' image build above as they are deployed separately. + REDEPLOY_SANDBOX_URL: ${{ secrets.REDEPLOY_SANDBOX_SOLR_URL }} + REDEPLOY_DEMO_URL: ${{ secrets.REDEPLOY_DEMO_SOLR_URL }} - - name: Build and push 'dspace-solr' image - id: docker_build_solr - uses: docker/build-push-action@v4 - with: - context: . - file: ./dspace/src/main/docker/dspace-solr/Dockerfile - platforms: ${{ env.PLATFORMS }} - # For pull requests, we run the Docker build (to ensure no PR changes break the build), - # but we ONLY do an image push to DockerHub if it's NOT a PR - push: ${{ github.event_name != 'pull_request' }} - # Use tags / labels provided by 'docker/metadata-action' above - tags: ${{ steps.meta_build_solr.outputs.tags }} - labels: ${{ steps.meta_build_solr.outputs.labels }} ########################################################### # Build/Push the 'dataquest/dspace-postgres-pgcrypto' image ########################################################### dspace-postgres-pgcrypto: # Ensure this job never runs on forked repos. It's only executed for 'dataquest/dspace' - if: github.repository == 'dspace/dspace' - runs-on: ubuntu-latest - - steps: - # https://github.com/actions/checkout - - name: Checkout codebase - uses: actions/checkout@v3 - - # https://github.com/docker/setup-buildx-action - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v2 - - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU emulation to build for multiple architectures - uses: docker/setup-qemu-action@v2 - - # https://github.com/docker/login-action - - name: Login to DockerHub - # Only login if not a PR, as PRs only trigger a Docker build and not a push - if: github.event_name != 'pull_request' - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - - # Get Metadata for docker_build_postgres step below - - name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-postgres-pgcrypto' image - id: meta_build_postgres - uses: docker/metadata-action@v4 - with: - images: dspace/dspace-postgres-pgcrypto - tags: ${{ env.IMAGE_TAGS }} - flavor: ${{ env.TAGS_FLAVOR }} - - - name: Build and push 'dspace-postgres-pgcrypto' image - id: docker_build_postgres - uses: docker/build-push-action@v4 - with: - # Must build out of subdirectory to have access to install script for pgcrypto - context: ./dspace/src/main/docker/dspace-postgres-pgcrypto/ - dockerfile: Dockerfile - platforms: ${{ env.PLATFORMS }} - # For pull requests, we run the Docker build (to ensure no PR changes break the build), - # but we ONLY do an image push to DockerHub if it's NOT a PR - push: ${{ github.event_name != 'pull_request' }} - # Use tags / labels provided by 'docker/metadata-action' above - tags: ${{ steps.meta_build_postgres.outputs.tags }} - labels: ${{ steps.meta_build_postgres.outputs.labels }} + if: github.repository == 'dataquest-dev/dspace' + uses: ./.github/workflows/reusable-docker-build.yml + with: + build_id: dspace-postgres-pgcrypto + image_name: dataquest/dspace-postgres-pgcrypto + # Must build out of subdirectory to have access to install script for pgcrypto. + # NOTE: this context will build the image based on the Dockerfile in the specified directory + dockerfile_context: ./dspace/src/main/docker/dspace-postgres-pgcrypto/ + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} ######################################################################## # Build/Push the 'dataquest/dspace-postgres-pgcrypto' image (-loadsql tag) ######################################################################## dspace-postgres-pgcrypto-loadsql: # Ensure this job never runs on forked repos. It's only executed for 'dataquest/dspace' - if: github.repository == 'dspace/dspace' + if: github.repository == 'dataquest-dev/dspace' + uses: ./.github/workflows/reusable-docker-build.yml + with: + build_id: dspace-postgres-pgcrypto-loadsql + image_name: dataquest/dspace-postgres-pgcrypto + # Must build out of subdirectory to have access to install script for pgcrypto. + # NOTE: this context will build the image based on the Dockerfile in the specified directory + dockerfile_context: ./dspace/src/main/docker/dspace-postgres-pgcrypto-curl/ + # Suffix all tags with "-loadsql". Otherwise, it uses the same + # tagging logic as the primary 'dataquest/dspace-postgres-pgcrypto' image above. + tags_flavor: suffix=-loadsql + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} + + + dspace-redeploy: runs-on: ubuntu-latest - + needs: dspace + if: false steps: - # https://github.com/actions/checkout - - name: Checkout codebase - uses: actions/checkout@v3 - - # https://github.com/docker/setup-buildx-action - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v2 - - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU emulation to build for multiple architectures - uses: docker/setup-qemu-action@v2 - - # https://github.com/docker/login-action - - name: Login to DockerHub - # Only login if not a PR, as PRs only trigger a Docker build and not a push - if: github.event_name != 'pull_request' - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - - # Get Metadata for docker_build_postgres_loadsql step below - - name: Sync metadata (tags, labels) from GitHub to Docker for 'dspace-postgres-pgcrypto-loadsql' image - id: meta_build_postgres_loadsql - uses: docker/metadata-action@v4 - with: - images: dspace/dspace-postgres-pgcrypto - tags: ${{ env.IMAGE_TAGS }} - # Suffix all tags with "-loadsql". Otherwise, it uses the same - # tagging logic as the primary 'dspace/dspace-postgres-pgcrypto' image above. - flavor: ${{ env.TAGS_FLAVOR }} - suffix=-loadsql - - - name: Build and push 'dspace-postgres-pgcrypto-loadsql' image - id: docker_build_postgres_loadsql - uses: docker/build-push-action@v4 - with: - # Must build out of subdirectory to have access to install script for pgcrypto - context: ./dspace/src/main/docker/dspace-postgres-pgcrypto-curl/ - dockerfile: Dockerfile - platforms: ${{ env.PLATFORMS }} - # For pull requests, we run the Docker build (to ensure no PR changes break the build), - # but we ONLY do an image push to DockerHub if it's NOT a PR - push: ${{ github.event_name != 'pull_request' }} - # Use tags / labels provided by 'docker/metadata-action' above - tags: ${{ steps.meta_build_postgres_loadsql.outputs.tags }} - labels: ${{ steps.meta_build_postgres_loadsql.outputs.labels }} + - name: redeploy + if: '!cancelled()' + run: | + curl -H "Accept: application/vnd.github.everest-preview+json" \ + -H "Authorization: token ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }}" \ + --request POST \ + https://api.github.com/repos/dataquest-dev/\ + dspace-angular/actions/workflows/deploy.yml/dispatches \ + --data "{\"ref\":\"refs/heads/dtq-dev\"}" diff --git a/.github/workflows/reusable-docker-build.yml b/.github/workflows/reusable-docker-build.yml new file mode 100644 index 000000000000..870cad225ec5 --- /dev/null +++ b/.github/workflows/reusable-docker-build.yml @@ -0,0 +1,238 @@ +# +# DSpace's reusable Docker build/push workflow. +# +# This is used by docker.yml for all Docker image builds +name: Reusable DSpace Docker Build + +on: + workflow_call: + # Possible Inputs to this reusable job + inputs: + python_version_script_dest: + required: false + default: version.txt + type: string + run_python_version_script: + required: false + default: false + type: boolean + # Build name/id for this Docker build. Used for digest storage to avoid digest overlap between builds. + build_id: + required: true + type: string + # Requires the image name to build (e.g dspace/dspace-test) + image_name: + required: true + type: string + # Optionally the path to the Dockerfile to use for the build. (Default is [dockerfile_context]/Dockerfile) + dockerfile_path: + required: false + type: string + # Optionally the context directory to build the Dockerfile within. Defaults to "." (current directory) + dockerfile_context: + required: false + type: string + default: '.' + # Optionally a list of "additional_contexts" to pass to Dockerfile. Defaults to empty + dockerfile_additional_contexts: + required: false + type: string + default: '' + # If Docker image should have additional tag flavor details (e.g. a suffix), it may be passed in. + tags_flavor: + required: false + type: string + secrets: + # Requires that Docker login info be passed in as secrets. + DOCKER_USERNAME: + required: true + DOCKER_ACCESS_TOKEN: + required: true + # These URL secrets are optional. When specified & branch checks match, the redeployment code below will trigger. + # Therefore builds which need to trigger redeployment MUST specify these URLs. All others should leave them empty. + REDEPLOY_SANDBOX_URL: + required: false + REDEPLOY_DEMO_URL: + required: false + +# Define shared default settings as environment variables +env: + IMAGE_NAME: ${{ inputs.image_name }} + # Define tags to use for Docker images based on Git tags/branches (for docker/metadata-action) + # For a new commit on default branch (main), use the literal tag 'latest' on Docker image. + # For a new commit on other branches, use the branch name as the tag for Docker image. + # For a new tag, copy that tag name as the tag for Docker image. + IMAGE_TAGS: | + type=raw,value=dspace-7_x,enable=${{ github.ref_name == github.event.repository.default_branch }} + type=ref,event=branch,enable=${{ github.ref_name != github.event.repository.default_branch }} + type=ref,event=tag + # Define default tag "flavor" for docker/metadata-action per + # https://github.com/docker/metadata-action#flavor-input + # We manage the 'latest' tag ourselves to the 'main' branch (see settings above) + TAGS_FLAVOR: | + latest=false + ${{ inputs.tags_flavor }} + # When these URL variables are specified & required branch matches, then the sandbox or demo site will be redeployed. + # See "Redeploy" steps below for more details. + REDEPLOY_SANDBOX_URL: ${{ secrets.REDEPLOY_SANDBOX_URL }} + REDEPLOY_DEMO_URL: ${{ secrets.REDEPLOY_DEMO_URL }} + # Current DSpace maintenance branch (and architecture) which is deployed to demo.dspace.org / sandbox.dspace.org + # (NOTE: No deployment branch specified for sandbox.dspace.org as it uses the default_branch) + DEPLOY_DEMO_BRANCH: 'dspace-7_x' + DEPLOY_ARCH: 'linux/amd64' + +jobs: + docker-build: + + strategy: + matrix: + # Architectures / Platforms for which we will build Docker images + # arch: [ 'linux/amd64', 'linux/arm64' ] + arch: [ 'linux/amd64' ] + os: [ ubuntu-latest ] + isPr: + - ${{ github.event_name == 'pull_request' }} + # If this is a PR, we ONLY build for AMD64. For PRs we only do a sanity check test to ensure Docker builds work. + # The below exclude therefore ensures we do NOT build ARM64 for PRs. + # exclude: + # - isPr: true + # os: ubuntu-latest + # arch: linux/arm64 + + runs-on: ${{ matrix.os }} + + steps: + # https://github.com/actions/checkout + - name: Checkout codebase + uses: actions/checkout@v4 + + - name: Add version + if: ${{ inputs.run_python_version_script }} + run: python scripts/sourceversion.py > ${{ inputs.python_version_script_dest }} + + # https://github.com/docker/setup-buildx-action + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU emulation to build for multiple architectures + uses: docker/setup-qemu-action@v3 + + # https://github.com/docker/login-action + - name: Login to DockerHub + # Only login if not a PR, as PRs only trigger a Docker build and not a push + if: ${{ ! matrix.isPr }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + + # https://github.com/docker/metadata-action + # Get Metadata for docker_build_deps step below + - name: Sync metadata (tags, labels) from GitHub to Docker for image + id: meta_build + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_NAME }} + tags: ${{ env.IMAGE_TAGS }} + flavor: ${{ env.TAGS_FLAVOR }} + + # https://github.com/docker/build-push-action + - name: Build and push image + id: docker_build + uses: docker/build-push-action@v5 + with: + build-contexts: | + ${{ inputs.dockerfile_additional_contexts }} + context: ${{ inputs.dockerfile_context }} + file: ${{ inputs.dockerfile_path }} + platforms: ${{ matrix.arch }} + # For pull requests, we run the Docker build (to ensure no PR changes break the build), + # but we ONLY do an image push to DockerHub if it's NOT a PR + push: ${{ ! matrix.isPr }} + # Use tags / labels provided by 'docker/metadata-action' above + tags: ${{ steps.meta_build.outputs.tags }} + labels: ${{ steps.meta_build.outputs.labels }} + + # Export the digest of Docker build locally (for non PRs only) + # - name: Export Docker build digest + # if: ${{ ! matrix.isPr }} + # run: | + # mkdir -p /tmp/digests + # digest="${{ steps.docker_build.outputs.digest }}" + # touch "/tmp/digests/${digest#sha256:}" + + # Upload digest to an artifact, so that it can be used in manifest below + # - name: Upload Docker build digest to artifact + # if: ${{ ! matrix.isPr }} + # uses: actions/upload-artifact@v3 + # with: + # name: digests-${{ inputs.build_id }} + # path: /tmp/digests/* + # if-no-files-found: error + # retention-days: 1 + + # If this build is NOT a PR and passed in a REDEPLOY_SANDBOX_URL secret, + # Then redeploy https://sandbox.dspace.org if this build is for our deployment architecture and 'main' branch. + # - name: Redeploy sandbox.dspace.org (based on main branch) + # if: | + # !matrix.isPR && + # env.REDEPLOY_SANDBOX_URL != '' && + # matrix.arch == env.DEPLOY_ARCH && + # github.ref_name == github.event.repository.default_branch + # run: | + # curl -X POST $REDEPLOY_SANDBOX_URL + + # If this build is NOT a PR and passed in a REDEPLOY_DEMO_URL secret, + # Then redeploy https://demo.dspace.org if this build is for our deployment architecture and demo branch. + # - name: Redeploy demo.dspace.org (based on maintenace branch) + # if: | + # !matrix.isPR && + # env.REDEPLOY_DEMO_URL != '' && + # matrix.arch == env.DEPLOY_ARCH && + # github.ref_name == env.DEPLOY_DEMO_BRANCH + # run: | + # curl -X POST $REDEPLOY_DEMO_URL + + # Merge Docker digests (from various architectures) into a manifest. + # This runs after all Docker builds complete above, and it tells hub.docker.com + # that these builds should be all included in the manifest for this tag. + # (e.g. AMD64 and ARM64 should be listed as options under the same tagged Docker image) + # docker-build_manifest: + # if: ${{ github.event_name != 'pull_request' }} + # runs-on: ubuntu-latest + # needs: + # - docker-build + # steps: + # - name: Download Docker build digests + # uses: actions/download-artifact@v3 + # with: + # name: digests-${{ inputs.build_id }} + # path: /tmp/digests + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + + # - name: Add Docker metadata for image + # id: meta + # uses: docker/metadata-action@v5 + # with: + # images: ${{ env.IMAGE_NAME }} + # tags: ${{ env.IMAGE_TAGS }} + # flavor: ${{ env.TAGS_FLAVOR }} + + # - name: Login to Docker Hub + # uses: docker/login-action@v3 + # with: + # username: ${{ secrets.DOCKER_USERNAME }} + # password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + + # - name: Create manifest list from digests and push + # working-directory: /tmp/digests + # run: | + # docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + # $(printf '${{ env.IMAGE_NAME }}@sha256:%s ' *) + + # - name: Inspect image + # run: | + # docker buildx imagetools inspect ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BrowsesResourceControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BrowsesResourceControllerIT.java index 1618cbf001ab..57a8dbb3c2b7 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BrowsesResourceControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BrowsesResourceControllerIT.java @@ -64,15 +64,15 @@ public void findAll() throws Exception { //We expect the content type to be "application/hal+json;charset=UTF-8" .andExpect(content().contentType(contentType)) - //Our default Discovery config has 5 browse indexes, so we expect this to be reflected in the page + //Our default Discovery config has 6 browse indexes, so we expect this to be reflected in the page // object .andExpect(jsonPath("$.page.size", is(20))) - .andExpect(jsonPath("$.page.totalElements", is(5))) + .andExpect(jsonPath("$.page.totalElements", is(6))) .andExpect(jsonPath("$.page.totalPages", is(1))) .andExpect(jsonPath("$.page.number", is(0))) - //The array of browse index should have a size 5 - .andExpect(jsonPath("$._embedded.browses", hasSize(5))) + //The array of browse index should have a size 6 + .andExpect(jsonPath("$._embedded.browses", hasSize(6))) //Check that all (and only) the default browse indexes are present .andExpect(jsonPath("$._embedded.browses", containsInAnyOrder( @@ -80,7 +80,8 @@ public void findAll() throws Exception { BrowseIndexMatcher.contributorBrowseIndex("asc"), BrowseIndexMatcher.titleBrowseIndex("asc"), BrowseIndexMatcher.subjectBrowseIndex("asc"), - BrowseIndexMatcher.hierarchicalBrowseIndex("srsc") + BrowseIndexMatcher.hierarchicalBrowseIndex("srsc"), + BrowseIndexMatcher.languageBrowseIndex("asc") ))) ; } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/BrowseIndexMatcher.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/BrowseIndexMatcher.java index 59a53f475a0a..e7664ceaad78 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/BrowseIndexMatcher.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/BrowseIndexMatcher.java @@ -117,9 +117,9 @@ public static Matcher publisherBrowseIndex(final String order) { public static Matcher languageBrowseIndex(final String order) { return allOf( - hasJsonPath("$.metadata", contains("dc.language.iso")), - hasJsonPath("$.metadataBrowse", Matchers.is(true)), - hasJsonPath("$.dataType", equalToIgnoringCase("iso_lang")), + hasJsonPath("$.metadata", contains("local.language.name")), + hasJsonPath("$.browseType", equalToIgnoringCase(BROWSE_TYPE_VALUE_LIST)), + hasJsonPath("$.dataType", equalToIgnoringCase("text")), hasJsonPath("$.order", equalToIgnoringCase(order)), hasJsonPath("$.sortOptions[*].name", containsInAnyOrder("title", "dateissued", "dateaccessioned")), hasJsonPath("$._links.self.href", is(REST_SERVER_URL + "discover/browses/language")), diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index 0a53d98f2ab9..60e46607c21f 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -942,7 +942,7 @@ registry.metadata.load = metashare-schema.xml # can login as another user from the "edit eperson" page. This is useful for # debugging problems in a running dspace instance, especially in the workflow # process. The default value is false, i.e. no one may assume the login of another user. -#webui.user.assumelogin = true +webui.user.assumelogin = true # whether to display the contents of the licence bundle (often just the deposit # licence in standard DSpace installation) @@ -1160,8 +1160,8 @@ webui.browse.index.1 = dateissued:item:dateissued webui.browse.index.2 = author:metadata:dc.contributor.*\,dc.creator:text webui.browse.index.3 = title:item:title webui.browse.index.4 = subject:metadata:dc.subject.*:text -# webui.browse.index.5 = publisher:metadata:dc.publisher:text -# webui.browse.index.6 = language:metadata:dc.language.iso:iso_lang +webui.browse.index.5 = language:metadata:local.language.name:text +# webui.browse.index.6 = publisher:metadata:dc.publisher:text # webui.browse.index.7 = itemtype:metadata:dc.type:text # webui.browse.index.8 = rights:metadata:dc.rights.label:text