CI/CD #2323
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD | |
| on: | |
| schedule: | |
| - cron: "0 6 * * *" | |
| pull_request: | |
| workflow_dispatch: | |
| workflow_call: | |
| outputs: | |
| ckan: | |
| description: "Change in ckan container" | |
| value: ${{ jobs.detect-changes.outputs.ckan }} | |
| drupal: | |
| description: "Change in drupal container" | |
| value: ${{ jobs.detect-changes.outputs.drupal }} | |
| nginx: | |
| description: "Change in nginx container" | |
| value: ${{ jobs.detect-changes.outputs.nginx }} | |
| solr: | |
| description: "Change in solr container" | |
| value: ${{ jobs.detect-changes.outputs.solr }} | |
| datapusher: | |
| description: "Change in datapusher container" | |
| value: ${{ jobs.detect-changes.outputs.datapusher }} | |
| clamav: | |
| description: "Change in clamav container" | |
| value: ${{ jobs.detect-changes.outputs.clamav }} | |
| assets: | |
| description: "Change in assets" | |
| value: ${{ jobs.detect-changes.outputs.assets }} | |
| environment: | |
| description: "Changes in docker environment" | |
| value: ${{ jobs.detect-changes.outputs.environment }} | |
| env: | |
| DOCKER_BUILDKIT: 1 | |
| COMPOSE_DOCKER_CLI_BUILD: 1 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| detect-changes: | |
| uses: ./.github/workflows/changes.yml | |
| build-containers: | |
| needs: | |
| - detect-changes | |
| name: Build Containers | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - dockerfile: ./docker/solr/Dockerfile | |
| context: ./docker/solr | |
| submodules: "" | |
| build-frontend: false | |
| name: solr | |
| build-container: ${{ needs.detect-changes.outputs.solr == 'true' }} | |
| - dockerfile: ./docker/datapusher-plus/Dockerfile | |
| context: ./docker/datapusher-plus | |
| submodules: "" | |
| build-frontend: false | |
| name: datapusher | |
| build-container: ${{ needs.detect-changes.outputs.datapusher == 'true' }} | |
| - dockerfile: ./docker/nginx/Dockerfile | |
| context: ./docker/nginx | |
| submodules: "" | |
| build-frontend: false | |
| name: nginx | |
| build-container: ${{ needs.detect-changes.outputs.nginx == 'true' }} | |
| - dockerfile: ./drupal/Dockerfile | |
| context: ./drupal | |
| submodules: "" | |
| build-frontend: true | |
| name: drupal | |
| build-container: ${{ (needs.detect-changes.outputs.drupal == 'true') || (needs.detect-changes.outputs.assets == 'true') }} | |
| - dockerfile: ./ckan/Dockerfile | |
| context: ./ckan | |
| submodules: recursive | |
| build-frontend: true | |
| name: ckan | |
| build-container: ${{ (needs.detect-changes.outputs.ckan == 'true') || (needs.detect-changes.outputs.assets == 'true') }} | |
| - dockerfile: ./clamav/clamav-docker/Dockerfile | |
| context: ./clamav/clamav-docker | |
| submodules: "" | |
| build-frontend: false | |
| name: clamav | |
| build-container: ${{ needs.detect-changes.outputs.clamav == 'true' }} | |
| steps: | |
| - name: checkout | |
| if: ${{ matrix.build-container == true }} | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: ${{ matrix.submodules }} | |
| - name: setup docker buildx | |
| if: ${{ matrix.build-container == true }} | |
| uses: docker/setup-buildx-action@v3 | |
| - name: configure NPM credentials | |
| if: ${{ matrix.build-frontend == true && matrix.build-container == true }} | |
| run: | | |
| cat <<EOT >> ./opendata-assets/.npmrc | |
| @fortawesome:registry=https://npm.fontawesome.com/ | |
| //npm.fontawesome.com/:_authToken=$NPM_TOKEN | |
| EOT | |
| env: | |
| NPM_TOKEN: ${{ secrets.NPM_TOKEN }} | |
| - name: install nodejs v20 | |
| if: ${{ matrix.build-frontend == true && matrix.build-container == true }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| cache: 'npm' | |
| cache-dependency-path: opendata-assets/package-lock.json | |
| - name: install npm packages | |
| if: ${{ matrix.build-frontend == true && matrix.build-container == true }} | |
| run: npm ci | |
| working-directory: ./opendata-assets | |
| - name: build frontend with gulp | |
| if: ${{ matrix.build-frontend == true && matrix.build-container == true }} | |
| run: npx gulp | |
| working-directory: ./opendata-assets | |
| - name: build images | |
| if: ${{ matrix.build-container == true }} | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ${{ matrix.context }} | |
| file: ${{ matrix.dockerfile }} | |
| push: false | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| outputs: type=docker,dest=/tmp/${{ matrix.name }}.tar | |
| tags: ${{ env.REGISTRY }}/${{ env.REPOSITORY }}/${{ matrix.name }}:latest | |
| env: | |
| REGISTRY: ${{ secrets.REGISTRY }} | |
| REPOSITORY: ${{ secrets.REPOSITORY }} | |
| DOCKER_BUILD_RECORD_UPLOAD: false | |
| - name: upload images | |
| if: ${{ matrix.build-container == true }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.name }} | |
| path: /tmp/${{ matrix.name }}.tar | |
| test-e2e: | |
| name: test-e2e | |
| needs: | |
| - detect-changes | |
| - build-containers | |
| if: ${{ (needs.detect-changes.outputs.nginx == 'true') || | |
| (needs.detect-changes.outputs.ckan == 'true') || | |
| (needs.detect-changes.outputs.drupal == 'true') || | |
| (needs.detect-changes.outputs.solr == 'true') || | |
| (needs.detect-changes.outputs.datapusher == 'true') || | |
| (needs.detect-changes.outputs.assets == 'true') || | |
| (needs.detect-changes.outputs.environment == 'true') | |
| }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| CI: 1 | |
| TERM: xterm | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| containers: [1, 2, 3, 4, 5] | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: setup docker buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: install nodejs v20 | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| cache: 'npm' | |
| cache-dependency-path: package-lock.json | |
| - run: npm ci | |
| - name: configure AWS credentials for running tests | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| id: aws-credentials | |
| with: | |
| role-to-assume: ${{ secrets.AWS_TEST_ROLE }} | |
| role-session-name: github-actions | |
| aws-region: eu-west-1 | |
| output-credentials: true | |
| - name: get caller identity | |
| run: | | |
| aws sts get-caller-identity | |
| - name: configure Cypress | |
| shell: bash | |
| run: | | |
| # configure cypress | |
| cat <<EOT > cypress.config.js | |
| const { defineConfig } = require('cypress') | |
| const del = require('del') | |
| module.exports = defineConfig({ | |
| projectId: 'ssb2ut', | |
| env: { | |
| resetDB: true, | |
| cloudStorageEnabled: true | |
| }, | |
| videoCompression: 20, | |
| videoUploadOnPasses: false, | |
| e2e: { | |
| baseUrl: 'http://localhost', | |
| specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}', | |
| setupNodeEvents(on, config) { | |
| on('after:spec', (spec, results) => { | |
| if (results && results.stats.failures === 0 && results.video) { | |
| return del(results.video) | |
| } | |
| }) | |
| }, | |
| }, | |
| }) | |
| EOT | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: configure .env | |
| shell: bash | |
| run: | | |
| cp -f docker/.env.template docker/.env | |
| pip install "python-dotenv[cli]" | |
| dotenv -f docker/.env set REGISTRY ${REGISTRY} | |
| dotenv -f docker/.env set REPOSITORY ${REPOSITORY} | |
| dotenv -f docker/.env set MATOMO_ENABLED false | |
| env: | |
| REGISTRY: ${{ secrets.REGISTRY }} | |
| REPOSITORY: ${{ secrets.REPOSITORY }} | |
| - name: configure .env.ckan.local | |
| shell: bash | |
| run: | | |
| dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_ENABLED ${CKAN_CLOUDSTORAGE_ENABLED} | |
| dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_DRIVER_OPTIONS "{'key': '${AWS_ACCESS_KEY_ID}', 'secret': '${AWS_SECRET_ACCESS_KEY}', 'token': '${AWS_SESSION_TOKEN}' }" | |
| dotenv -f docker/.env.ckan.local set CKAN_CLOUDSTORAGE_CONTAINER_NAME ${CKAN_CLOUDSTORAGE_CONTAINER_NAME} | |
| dotenv -f docker/.env.ckan.local set AWS_ACCESS_KEY_ID ${AWS_ACCESS_KEY_ID} | |
| dotenv -f docker/.env.ckan.local set AWS_SECRET_ACCESS_KEY ${AWS_SECRET_ACCESS_KEY} | |
| env: | |
| CKAN_CLOUDSTORAGE_ENABLED: true | |
| CKAN_CLOUDSTORAGE_CONTAINER_NAME: ${{ secrets.CKAN_CLOUDSTORAGE_CONTAINER_NAME }} | |
| AWS_ACCESS_KEY_ID: ${{ steps.aws-credentials.outputs.aws-access-key-id }} | |
| AWS_SECRET_ACCESS_KEY: ${{ steps.aws-credentials.outputs.aws-secret-access-key }} | |
| AWS_SESSION_TOKEN: ${{ steps.aws-credentials.outputs.aws-session-token }} | |
| - name: configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_DEV_ROLE }} | |
| role-session-name: github-actions | |
| aws-region: eu-west-1 | |
| - name: login to AWS ECR | |
| id: login | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| with: | |
| registries: ${{ secrets.AWS_PROD_ACCOUNT_ID }} | |
| - name: download built images | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: /tmp | |
| - name: load built ckan image | |
| if: ${{ (needs.detect-changes.outputs.ckan == 'true') || (needs.detect-changes.outputs.assets == 'true')}} | |
| run: | | |
| docker load --input /tmp/ckan/ckan.tar | |
| - name: load built drupal image | |
| if: ${{ (needs.detect-changes.outputs.drupal == 'true') || (needs.detect-changes.outputs.assets == 'true') }} | |
| run: | | |
| docker load --input /tmp/drupal/drupal.tar | |
| - name: load built datapusher image | |
| if: ${{ needs.detect-changes.outputs.datapusher == 'true' }} | |
| run: | | |
| docker load --input /tmp/datapusher/datapusher.tar | |
| - name: load built solr image | |
| if: ${{ needs.detect-changes.outputs.solr == 'true' }} | |
| run: | | |
| docker load --input /tmp/solr/solr.tar | |
| - name: load built nginx image | |
| if: ${{ needs.detect-changes.outputs.nginx == 'true' }} | |
| run: | | |
| docker load --input /tmp/nginx/nginx.tar | |
| - name: bring services up | |
| working-directory: docker | |
| run: | | |
| docker compose -f docker-compose.yml -f docker-compose.build.yml -p opendata up -d | |
| - name: wait until services have started | |
| shell: bash | |
| run: | | |
| # wait for services to start properly | |
| while [[ $(curl -L --write-out '%{http_code}' --silent --output /dev/null http://localhost) != "200" ]]; do | |
| echo "waiting for services to start up and initialize ..." | |
| sleep 5s | |
| done | |
| sleep 5s | |
| # print the list of containers | |
| docker ps -a | |
| # print logs to debug errors | |
| docker logs opendata-ckan-1 | |
| docker logs opendata-drupal-1 | |
| docker logs opendata-solr-1 | |
| docker logs opendata-datapusher-1 | |
| docker logs opendata-nginx-1 | |
| docker logs opendata-fuseki-1 | |
| - name: Build cypress container | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./cypress | |
| file: ./cypress/Dockerfile | |
| push: false | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| tags: cypress/test:latest | |
| load: true | |
| env: | |
| DOCKER_BUILD_RECORD_UPLOAD: false | |
| - name: run cypress e2e tests | |
| run: > | |
| docker run | |
| --network host | |
| -v $PWD:/e2e | |
| -w /e2e | |
| -e COMMIT_INFO_BRANCH | |
| -e CYPRESS_PULL_REQUEST_ID | |
| -e CYPRESS_PULL_REQUEST_URL | |
| -e CYPRESS_CI_BUILD_URL | |
| --entrypoint cypress | |
| cypress/test run | |
| --browser chrome:stable | |
| --record --key ${{ secrets.CYPRESS_RECORD_KEY }} | |
| --parallel | |
| --ci-build-id ${{ github.repository }}-${{ github.run_id }}-${{ github.run_attempt}} | |
| env: | |
| COMMIT_INFO_BRANCH: ${{ github.head_ref || github.ref_name }} | |
| CYPRESS_PULL_REQUEST_ID: ${{ github.event.number }} | |
| CYPRESS_PULL_REQUEST_URL: ${{ github.event.pull_request.url }} | |
| CYPRESS_CI_BUILD_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| - name: export e2e test logs | |
| if: failure() | |
| shell: bash | |
| run: | | |
| docker logs opendata-ckan-1 > /tmp/opendata-ckan-1.log 2>&1 | |
| docker logs opendata-ckan_cron-1 > /tmp/opendata-ckan_cron-1.log 2>&1 | |
| docker logs opendata-drupal-1 > /tmp/opendata-drupal-1.log 2>&1 | |
| docker logs opendata-solr-1 > /tmp/opendata-solr-1.log 2>&1 | |
| docker logs opendata-datapusher-1 > /tmp/opendata-datapusher-1.log 2>&1 | |
| docker logs opendata-nginx-1 > /tmp/opendata-nginx-1.log 2>&1 | |
| docker logs opendata-fuseki-1 > /tmp/opendata-fuseki-1.log 2>&1 | |
| - name: upload log artifacts | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: docker-logs | |
| path: /tmp/opendata-*.log | |
| - name: upload cypress screenshot artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: screenshots | |
| path: cypress/screenshots | |
| - name: upload cypress video artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: videos | |
| path: cypress/videos | |
| test-e2e-results: | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| name: Final E2E Results | |
| needs: [test-e2e] | |
| steps: | |
| - run: exit 1 | |
| if: >- | |
| ${{ | |
| contains(needs.*.result, 'failure') | |
| || contains(needs.*.result, 'cancelled') | |
| }} | |
| test-cdk: | |
| name: test-cdk | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| concurrency: | |
| group: ${{ github.ref }}/test-cdk | |
| cancel-in-progress: true | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: install nodejs v16 | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 16 | |
| - name: cache node_modules | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.npm | |
| key: ${{ runner.os }}-node_cdk_v16-${{ hashFiles('**/package-lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-node_cdk_v16- | |
| - name: install cdk npm packages and verify installation | |
| working-directory: cdk | |
| run: | | |
| npm install | |
| $(npm bin)/cdk doctor | |
| - name: build cdk project | |
| working-directory: cdk | |
| run: | | |
| npm run build | |
| - name: test cdk project | |
| working-directory: cdk | |
| run: | | |
| npm run test | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| flags: cdk | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| test: | |
| name: Test opendata extensions | |
| runs-on: ubuntu-latest | |
| needs: | |
| - detect-changes | |
| if: ${{ needs.detect-changes.outputs.ckan == 'true' }} | |
| container: | |
| image: ckan/ckan-dev:2.9.9 | |
| services: | |
| solr: | |
| image: ckan/ckan-solr:2.9 | |
| postgres: | |
| image: postgres:12 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| redis: | |
| image: redis:3 | |
| env: | |
| CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test | |
| CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test | |
| CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test | |
| CKAN_SOLR_URL: http://solr:8983/solr/ckan | |
| CKAN_REDIS_URL: redis://redis:6379/1 | |
| PGPASSWORD: postgres | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Create Database | |
| run: | | |
| psql --host=postgres --username=postgres --command="CREATE USER ckan_default WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" | |
| createdb --encoding=utf-8 --host=postgres --username=postgres --owner=ckan_default ckan_test | |
| psql --host=postgres --username=postgres --command="CREATE USER datastore_write WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" | |
| psql --host=postgres --username=postgres --command="CREATE USER datastore_read WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" | |
| createdb --encoding=utf-8 --host=postgres --username=postgres --owner=datastore_write datastore_test | |
| - name: Install requirements | |
| run: | | |
| apk add proj proj-dev proj-util geos | |
| cd ckan/ckanext/ckanext-ytp_main | |
| pip install -r dev-requirements.txt | |
| pip install -r requirements.txt | |
| pip install -e . | |
| # Replace default path to CKAN core config file with the one on the container | |
| sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini | |
| cd ../ckanext-scheming | |
| pip install -e . | |
| cd ../ckanext-fluent | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-dcat | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-report | |
| pip install -r requirements.txt | |
| pip install -e . | |
| # Spatial is not currently used in tests | |
| # cd ../ckanext-spatial | |
| # pip install -r requirements.txt | |
| # pip install -e . | |
| cd ../ckanext-organizationapproval | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-hierarchy | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-harvest | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-showcase | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-sixodp_showcase | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-sixodp_showcasesubmit | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-sitesearch | |
| pip install -e . | |
| - name: Setup extension | |
| run: | | |
| ckan -c ckan/ckanext/ckanext-ytp_main/test.ini db init | |
| ckan -c ckan/ckanext/ckanext-ytp_main/test.ini db upgrade -p harvest | |
| - name: Run tests | |
| run: | | |
| pytest --ckan-ini=ckan/ckanext/ckanext-ytp_main/test.ini --cov=ckanext.ytp --disable-warnings ckan/ckanext/ckanext-ytp_main/ckanext/ytp/tests | |
| - name: install codecov requirements | |
| run: | | |
| apk add gpg gpg-agent | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| flags: ckan | |
| os: alpine | |
| token: ${{ secrets.CODECOV_TOKEN }} |