test.yml

name: CI/CD

on:
  schedule:
    - cron: "0 6 * * *"
  pull_request:
  workflow_dispatch:
  workflow_call:
    outputs:
      ckan:
        description: "Change in ckan container"
        value: ${{ jobs.detect-changes.outputs.ckan }}
      drupal:
        description: "Change in drupal container"
        value: ${{ jobs.detect-changes.outputs.drupal }}
      nginx:
        description: "Change in nginx container"
        value: ${{ jobs.detect-changes.outputs.nginx }}
      solr:
        description: "Change in solr container"
        value: ${{ jobs.detect-changes.outputs.solr }}
      datapusher:
        description: "Change in datapusher container"
        value: ${{ jobs.detect-changes.outputs.datapusher }}
      clamav:
        description: "Change in clamav container"
        value: ${{ jobs.detect-changes.outputs.clamav }}
      assets:
        description: "Change in assets"
        value: ${{ jobs.detect-changes.outputs.assets }}
      environment:
        description: "Changes in docker environment"
        value: ${{ jobs.detect-changes.outputs.environment }}


env:
  DOCKER_BUILDKIT: 1
  COMPOSE_DOCKER_CLI_BUILD: 1

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  detect-changes:
    uses: ./.github/workflows/changes.yml

  build-containers:
    needs:
      - detect-changes
    name: Build Containers
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    strategy:
      fail-fast: false
      matrix:
        include:
          - dockerfile: ./docker/solr/Dockerfile
            context: ./docker/solr
            submodules: ""
            build-frontend: false
            name: solr
            build-container: ${{ needs.detect-changes.outputs.solr == 'true' }}
          - dockerfile: ./docker/datapusher-plus/Dockerfile
            context: ./docker/datapusher-plus
            submodules: ""
            build-frontend: false
            name: datapusher
            build-container: ${{ needs.detect-changes.outputs.datapusher == 'true' }}
          - dockerfile: ./docker/nginx/Dockerfile
            context: ./docker/nginx
            submodules: ""
            build-frontend: false
            name: nginx
            build-container: ${{ needs.detect-changes.outputs.nginx == 'true' }}
          - dockerfile: ./drupal/Dockerfile
            context: ./drupal
            submodules: ""
            build-frontend: true
            name: drupal
            build-container: ${{ (needs.detect-changes.outputs.drupal == 'true') || (needs.detect-changes.outputs.assets == 'true') }}
          - dockerfile: ./ckan/Dockerfile
            context: ./ckan
            submodules: recursive
            build-frontend: true
            name: ckan
            build-container: ${{ (needs.detect-changes.outputs.ckan == 'true') || (needs.detect-changes.outputs.assets == 'true') }}
          - dockerfile: ./clamav/clamav-docker/Dockerfile
            context: ./clamav/clamav-docker
            submodules: ""
            build-frontend: false
            name: clamav
            build-container: ${{ needs.detect-changes.outputs.clamav == 'true' }}

    steps:
      - name: checkout
        if: ${{ matrix.build-container == true }}
        uses: actions/checkout@v4
        with:
          submodules: ${{ matrix.submodules }}

      - name: setup docker buildx
        if: ${{ matrix.build-container == true }}
        uses: docker/setup-buildx-action@v3

      - name: configure NPM credentials
        if: ${{ matrix.build-frontend == true && matrix.build-container == true }}
        run: |
          cat <<EOT >> ./opendata-assets/.npmrc
          @fortawesome:registry=https://npm.fontawesome.com/
          //npm.fontawesome.com/:_authToken=$NPM_TOKEN
          EOT
        env:
          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

      - name: install nodejs v20
        if: ${{ matrix.build-frontend == true && matrix.build-container == true }}
        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
          cache-dependency-path: opendata-assets/package-lock.json

      - name: install npm packages
        if: ${{ matrix.build-frontend == true && matrix.build-container == true }}
        run: npm ci
        working-directory: ./opendata-assets

      - name: build frontend with gulp
        if: ${{ matrix.build-frontend == true && matrix.build-container == true }}
        run: npx gulp
        working-directory: ./opendata-assets

      - name: build images
        if: ${{ matrix.build-container == true }}
        uses: docker/build-push-action@v6
        with:
          context: ${{ matrix.context }}
          file: ${{ matrix.dockerfile }}
          push: false
          cache-from: type=gha
          cache-to: type=gha,mode=max
          outputs: type=docker,dest=/tmp/${{ matrix.name }}.tar
          tags: ${{ env.REGISTRY }}/${{ env.REPOSITORY }}/${{ matrix.name }}:latest
        env:
          REGISTRY: ${{ secrets.REGISTRY }}
          REPOSITORY: ${{ secrets.REPOSITORY }}
          DOCKER_BUILD_RECORD_UPLOAD: false

      - name: upload images
        if: ${{ matrix.build-container == true }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.name }}
          path: /tmp/${{ matrix.name }}.tar

  test-e2e:
    name: test-e2e
    needs:
      - detect-changes
      - build-containers
    if: ${{ (needs.detect-changes.outputs.nginx == 'true') ||
            (needs.detect-changes.outputs.ckan == 'true') ||
            (needs.detect-changes.outputs.drupal == 'true') ||
            (needs.detect-changes.outputs.solr == 'true') ||
            (needs.detect-changes.outputs.datapusher == 'true') ||
            (needs.detect-changes.outputs.assets == 'true') ||
            (needs.detect-changes.outputs.environment == 'true')
      }}
    runs-on: ubuntu-latest
    timeout-minutes: 60
    permissions:
      id-token: write
      contents: read
    env:
      CI: 1
      TERM: xterm
    strategy:
      fail-fast: false
      matrix:
        containers: [1, 2, 3, 4, 5]
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: setup docker buildx
        uses: docker/setup-buildx-action@v3

      - name: install nodejs v20
        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
          cache-dependency-path: package-lock.json

      - run: npm ci

      - name: configure AWS credentials for running tests
        uses: aws-actions/configure-aws-credentials@v4
        id: aws-credentials
        with:
          role-to-assume: ${{ secrets.AWS_TEST_ROLE }}
          role-session-name: github-actions
          aws-region: eu-west-1
          output-credentials: true

      - name: get caller identity
        run: |
          aws sts get-caller-identity

      - name: configure Cypress
        shell: bash
        run: |
          # configure cypress
          cat <<EOT > cypress.config.js
          const { defineConfig } = require('cypress')
          const del = require('del')

          module.exports = defineConfig({
            projectId: 'ssb2ut',
            env: {
              resetDB: true,
              cloudStorageEnabled: true
            },
              videoCompression: 20,
              videoUploadOnPasses: false,
            e2e: {
              baseUrl: 'http://localhost',
              specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}',
              setupNodeEvents(on, config) {
                on('after:spec', (spec, results) => {
                  if (results && results.stats.failures === 0 && results.video) {
                    return del(results.video)
                  }
                })
              },
            },
          })
          EOT

      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: configure .env
        shell: bash
        run: |
          cp -f docker/.env.template docker/.env
          pip install "python-dotenv[cli]"
          dotenv -f docker/.env set REGISTRY ${REGISTRY}
          dotenv -f docker/.env set REPOSITORY ${REPOSITORY}
          dotenv -f docker/.env set MATOMO_ENABLED false
        env:
          REGISTRY: ${{ secrets.REGISTRY }}
          REPOSITORY: ${{ secrets.REPOSITORY }}

      - name: configure .env.ckan.local
        shell: bash
        run: |
          dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_ENABLED ${CKAN_CLOUDSTORAGE_ENABLED}
          dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_DRIVER_OPTIONS "{'key': '${AWS_ACCESS_KEY_ID}', 'secret': '${AWS_SECRET_ACCESS_KEY}', 'token': '${AWS_SESSION_TOKEN}' }"
          dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_CONTAINER_NAME ${CKAN_CLOUDSTORAGE_CONTAINER_NAME}
          dotenv -f docker/.env.ckan.local set AWS_ACCESS_KEY_ID ${AWS_ACCESS_KEY_ID}
          dotenv -f docker/.env.ckan.local set AWS_SECRET_ACCESS_KEY ${AWS_SECRET_ACCESS_KEY}
        env:
          CKAN_CLOUDSTORAGE_ENABLED: true
          CKAN_CLOUDSTORAGE_CONTAINER_NAME: ${{ secrets.CKAN_CLOUDSTORAGE_CONTAINER_NAME }}
          AWS_ACCESS_KEY_ID: ${{ steps.aws-credentials.outputs.aws-access-key-id }}
          AWS_SECRET_ACCESS_KEY: ${{ steps.aws-credentials.outputs.aws-secret-access-key }}
          AWS_SESSION_TOKEN: ${{ steps.aws-credentials.outputs.aws-session-token }}

      - name: configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_DEV_ROLE }}
          role-session-name: github-actions
          aws-region: eu-west-1

      - name: login to AWS ECR
        id: login
        uses: aws-actions/amazon-ecr-login@v2
        with:
          registries: ${{ secrets.AWS_PROD_ACCOUNT_ID }}



      - name: download built images
        uses: actions/download-artifact@v4
        with:
          path: /tmp

      - name: load built ckan image
        if: ${{ (needs.detect-changes.outputs.ckan == 'true') || (needs.detect-changes.outputs.assets == 'true')}}
        run: |
          docker load --input /tmp/ckan/ckan.tar

      - name: load built drupal image
        if: ${{ (needs.detect-changes.outputs.drupal == 'true') || (needs.detect-changes.outputs.assets == 'true') }}
        run: |
          docker load --input /tmp/drupal/drupal.tar

      - name: load built datapusher image
        if: ${{ needs.detect-changes.outputs.datapusher == 'true' }}
        run: |
          docker load --input /tmp/datapusher/datapusher.tar

      - name: load built solr image
        if: ${{ needs.detect-changes.outputs.solr == 'true' }}
        run: |
          docker load --input /tmp/solr/solr.tar

      - name: load built nginx image
        if: ${{ needs.detect-changes.outputs.nginx == 'true' }}
        run: |
          docker load --input /tmp/nginx/nginx.tar

      - name: bring services up
        working-directory: docker
        run: |
          docker compose -f docker-compose.yml -f docker-compose.build.yml -p opendata up -d

      - name: wait until services have started
        shell: bash
        run: |
          # wait for services to start properly
          while [[ $(curl -L --write-out '%{http_code}' --silent --output /dev/null http://localhost) != "200" ]]; do
            echo "waiting for services to start up and initialize ..."
            sleep 5s
          done
          sleep 5s
          # print the list of containers
          docker ps -a
          # print logs to debug errors
          docker logs opendata-ckan-1
          docker logs opendata-drupal-1
          docker logs opendata-solr-1
          docker logs opendata-datapusher-1
          docker logs opendata-nginx-1
          docker logs opendata-fuseki-1

      - name: Build cypress container
        uses: docker/build-push-action@v6
        with:
          context: ./cypress
          file: ./cypress/Dockerfile
          push: false
          cache-from: type=gha
          cache-to: type=gha,mode=max
          tags: cypress/test:latest
          load: true
        env:
          DOCKER_BUILD_RECORD_UPLOAD: false


      - name: run cypress e2e tests
        run: >
          docker run 
          --network host 
          -v $PWD:/e2e
          -w /e2e 
          -e COMMIT_INFO_BRANCH
          -e CYPRESS_PULL_REQUEST_ID
          -e CYPRESS_PULL_REQUEST_URL
          -e CYPRESS_CI_BUILD_URL
          --entrypoint cypress 
          cypress/test run 
          --browser chrome:stable 
          --record --key ${{ secrets.CYPRESS_RECORD_KEY }} 
          --parallel 
          --ci-build-id ${{ github.repository }}-${{ github.run_id }}-${{ github.run_attempt}}
        env:
          COMMIT_INFO_BRANCH: ${{ github.head_ref || github.ref_name }}
          CYPRESS_PULL_REQUEST_ID: ${{ github.event.number }}
          CYPRESS_PULL_REQUEST_URL: ${{ github.event.pull_request.url }}
          CYPRESS_CI_BUILD_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: export e2e test logs 
        if: failure()
        shell: bash
        run: |
          docker logs opendata-ckan-1 > /tmp/opendata-ckan-1.log 2>&1
          docker logs opendata-ckan_cron-1 > /tmp/opendata-ckan_cron-1.log 2>&1
          docker logs opendata-drupal-1 > /tmp/opendata-drupal-1.log 2>&1
          docker logs opendata-solr-1 > /tmp/opendata-solr-1.log 2>&1
          docker logs opendata-datapusher-1 > /tmp/opendata-datapusher-1.log 2>&1
          docker logs opendata-nginx-1 > /tmp/opendata-nginx-1.log 2>&1
          docker logs opendata-fuseki-1 > /tmp/opendata-fuseki-1.log 2>&1

      - name: upload log artifacts
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: docker-logs
          path: /tmp/opendata-*.log

      - name: upload cypress screenshot artifacts
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: screenshots
          path: cypress/screenshots

      - name: upload cypress video artifacts
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: videos
          path: cypress/videos

  test-e2e-results:
    if: ${{ always() }}
    runs-on: ubuntu-latest
    name: Final E2E Results
    needs: [test-e2e]
    steps:
      - run: exit 1
        if: >-
          ${{
              contains(needs.*.result, 'failure')
              || contains(needs.*.result, 'cancelled')
          }}

  test-cdk:
    name: test-cdk
    runs-on: ubuntu-latest
    timeout-minutes: 15
    concurrency:
      group: ${{ github.ref }}/test-cdk
      cancel-in-progress: true
    permissions:
      id-token: write
      contents: read
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: install nodejs v16
        uses: actions/setup-node@v4
        with:
          node-version: 16

      - name: cache node_modules
        uses: actions/cache@v4
        with:
          path: ~/.npm
          key: ${{ runner.os }}-node_cdk_v16-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-node_cdk_v16-

      - name: install cdk npm packages and verify installation
        working-directory: cdk
        run: |
          npm install
          $(npm bin)/cdk doctor

      - name: build cdk project
        working-directory: cdk
        run: |
          npm run build

      - name: test cdk project
        working-directory: cdk
        run: |
          npm run test

      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v4
        with:
          flags: cdk
          token: ${{ secrets.CODECOV_TOKEN }}

  test:
    name: Test opendata extensions
    runs-on: ubuntu-latest
    needs:
      - detect-changes
    if: ${{ needs.detect-changes.outputs.ckan == 'true' }}
    container:
      image: ckan/ckan-dev:2.9.9
    services:
      solr:
        image: ckan/ckan-solr:2.9
      postgres:
        image: postgres:12
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: postgres
        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
      redis:
        image: redis:3
    env:
      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
      CKAN_SOLR_URL: http://solr:8983/solr/ckan
      CKAN_REDIS_URL: redis://redis:6379/1
      PGPASSWORD: postgres

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - name: Create Database
      run: |
        psql --host=postgres --username=postgres --command="CREATE USER ckan_default WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
        createdb --encoding=utf-8 --host=postgres --username=postgres --owner=ckan_default ckan_test
        psql --host=postgres --username=postgres --command="CREATE USER datastore_write WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
        psql --host=postgres --username=postgres --command="CREATE USER datastore_read WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
        createdb --encoding=utf-8 --host=postgres --username=postgres --owner=datastore_write datastore_test
    - name: Install requirements
      run: |
        apk add proj proj-dev proj-util geos
        cd ckan/ckanext/ckanext-ytp_main
        pip install -r dev-requirements.txt
        pip install -r requirements.txt
        pip install -e .
        # Replace default path to CKAN core config file with the one on the container
        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
        cd ../ckanext-scheming
        pip install -e .
        cd ../ckanext-fluent
        pip install -r requirements.txt
        pip install -e .
        cd ../ckanext-dcat
        pip install -r requirements.txt
        pip install -e .
        cd ../ckanext-report
        pip install -r requirements.txt
        pip install -e .
        # Spatial is not currently used in tests
        # cd ../ckanext-spatial
        # pip install -r requirements.txt
        # pip install -e .
        cd ../ckanext-organizationapproval
        pip install -r requirements.txt
        pip install -e .
        cd ../ckanext-hierarchy
        pip install -r requirements.txt
        pip install -e .
        cd ../ckanext-harvest
        pip install -r requirements.txt
        pip install -e .
        cd ../ckanext-showcase
        pip install -r requirements.txt
        pip install -e .
        cd ../ckanext-sixodp_showcase
        pip install -r requirements.txt
        pip install -e .
        cd ../ckanext-sixodp_showcasesubmit
        pip install -r requirements.txt
        pip install -e .
        cd ../ckanext-sitesearch
        pip install -e .
    - name: Setup extension
      run: |
        ckan -c ckan/ckanext/ckanext-ytp_main/test.ini db init
        ckan -c ckan/ckanext/ckanext-ytp_main/test.ini db upgrade -p harvest
    - name: Run tests
      run: |
        pytest --ckan-ini=ckan/ckanext/ckanext-ytp_main/test.ini --cov=ckanext.ytp --disable-warnings ckan/ckanext/ckanext-ytp_main/ckanext/ytp/tests

    - name: install codecov requirements
      run: |
        apk add gpg gpg-agent

    - name: Upload coverage reports to Codecov
      uses: codecov/codecov-action@v4
      with:
        flags: ckan
        os: alpine
        token: ${{ secrets.CODECOV_TOKEN }}