CI/CD #1081
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD | |
| on: | |
| schedule: | |
| - cron: "0 6 * * *" | |
| pull_request: | |
| workflow_dispatch: | |
| workflow_call: | |
| outputs: | |
| ckan: | |
| description: "Change in ckan container" | |
| value: ${{ jobs.detect-changes.outputs.ckan }} | |
| drupal: | |
| description: "Change in drupal container" | |
| value: ${{ jobs.detect-changes.outputs.drupal }} | |
| nginx: | |
| description: "Change in nginx container" | |
| value: ${{ jobs.detect-changes.outputs.nginx }} | |
| solr: | |
| description: "Change in solr container" | |
| value: ${{ jobs.detect-changes.outputs.solr }} | |
| datapusher: | |
| description: "Change in datapusher container" | |
| value: ${{ jobs.detect-changes.outputs.datapusher }} | |
| assets: | |
| description: "Change in assets" | |
| value: ${{ jobs.detect-changes.outputs.assets }} | |
| environment: | |
| description: "Changes in docker environment" | |
| value: ${{ jobs.detect-changes.outputs.environment }} | |
| env: | |
| DOCKER_BUILDKIT: 1 | |
| COMPOSE_DOCKER_CLI_BUILD: 1 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| detect-changes: | |
| uses: ./.github/workflows/changes.yml | |
| build-containers: | |
| needs: | |
| - detect-changes | |
| name: Build Containers | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - dockerfile: ./docker/solr/Dockerfile | |
| context: ./docker/solr | |
| submodules: "" | |
| build-frontend: false | |
| name: solr | |
| build-container: ${{ needs.detect-changes.outputs.solr == 'true' }} | |
| - dockerfile: ./docker/datapusher-plus/Dockerfile | |
| context: ./docker/datapusher-plus | |
| submodules: "" | |
| build-frontend: false | |
| name: datapusher | |
| build-container: ${{ needs.detect-changes.outputs.datapusher == 'true' }} | |
| - dockerfile: ./docker/nginx/Dockerfile | |
| context: ./docker/nginx | |
| submodules: "" | |
| build-frontend: false | |
| name: nginx | |
| build-container: ${{ needs.detect-changes.outputs.nginx == 'true' }} | |
| - dockerfile: ./drupal/Dockerfile | |
| context: ./drupal | |
| submodules: "" | |
| build-frontend: true | |
| name: drupal | |
| build-container: ${{ (needs.detect-changes.outputs.drupal == 'true') || (needs.detect-changes.outputs.assets == 'true') }} | |
| - dockerfile: ./ckan/Dockerfile | |
| context: ./ckan | |
| submodules: recursive | |
| build-frontend: true | |
| name: ckan | |
| build-container: ${{ (needs.detect-changes.outputs.ckan == 'true') || (needs.detect-changes.outputs.assets == 'true') }} | |
| steps: | |
| - name: checkout | |
| if: ${{ matrix.build-container == true }} | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: ${{ matrix.submodules }} | |
| - name: setup docker buildx | |
| if: ${{ matrix.build-container == true }} | |
| uses: docker/setup-buildx-action@v2 | |
| - name: configure NPM credentials | |
| if: ${{ matrix.build-frontend == true && matrix.build-container == true }} | |
| run: | | |
| cat <<EOT >> ./opendata-assets/.npmrc | |
| @fortawesome:registry=https://npm.fontawesome.com/ | |
| //npm.fontawesome.com/:_authToken=$NPM_TOKEN | |
| EOT | |
| env: | |
| NPM_TOKEN: ${{ secrets.NPM_TOKEN }} | |
| - name: install nodejs v16 | |
| if: ${{ matrix.build-frontend == true && matrix.build-container == true }} | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| cache: 'npm' | |
| cache-dependency-path: opendata-assets/package-lock.json | |
| - name: install npm packages | |
| if: ${{ matrix.build-frontend == true && matrix.build-container == true }} | |
| run: npm ci | |
| working-directory: ./opendata-assets | |
| - name: build frontend with gulp | |
| if: ${{ matrix.build-frontend == true && matrix.build-container == true }} | |
| run: npx gulp | |
| working-directory: ./opendata-assets | |
| - name: build images | |
| if: ${{ matrix.build-container == true }} | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ${{ matrix.context }} | |
| file: ${{ matrix.dockerfile }} | |
| push: false | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| outputs: type=docker,dest=/tmp/${{ matrix.name }}.tar | |
| tags: ${{ env.REGISTRY }}/${{ env.REPOSITORY }}/${{ matrix.name }}:latest | |
| env: | |
| REGISTRY: ${{ secrets.REGISTRY }} | |
| REPOSITORY: ${{ secrets.REPOSITORY }} | |
| - name: upload images | |
| if: ${{ matrix.build-container == true }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ matrix.name }} | |
| path: /tmp/${{ matrix.name }}.tar | |
| test-e2e: | |
| name: test-e2e | |
| needs: | |
| - detect-changes | |
| - build-containers | |
| if: ${{ (needs.detect-changes.outputs.nginx == 'true') || | |
| (needs.detect-changes.outputs.ckan == 'true') || | |
| (needs.detect-changes.outputs.drupal == 'true') || | |
| (needs.detect-changes.outputs.solr == 'true') || | |
| (needs.detect-changes.outputs.datapusher == 'true') || | |
| (needs.detect-changes.outputs.assets == 'true') || | |
| (needs.detect-changes.outputs.environment == 'true') | |
| }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| CI: 1 | |
| TERM: xterm | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| containers: [1, 2, 3, 4, 5] | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v3 | |
| - name: setup docker buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: install nodejs v16 | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| cache: 'npm' | |
| cache-dependency-path: package-lock.json | |
| - run: npm ci | |
| - name: configure environment | |
| shell: bash | |
| run: | | |
| # configure cypress | |
| cat <<EOT > cypress.config.js | |
| const { defineConfig } = require('cypress') | |
| const del = require('del') | |
| module.exports = defineConfig({ | |
| projectId: 'ssb2ut', | |
| env: { | |
| resetDB: true, | |
| cloudStorageEnabled: true | |
| }, | |
| videoCompression: 20, | |
| videoUploadOnPasses: false, | |
| e2e: { | |
| baseUrl: 'http://localhost', | |
| specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}', | |
| setupNodeEvents(on, config) { | |
| on('after:spec', (spec, results) => { | |
| if (results && results.stats.failures === 0 && results.video) { | |
| return del(results.video) | |
| } | |
| }) | |
| }, | |
| }, | |
| }) | |
| EOT | |
| # configure docker | |
| cp -f docker/.env.template docker/.env | |
| sed -i.bak -E 's/^(REGISTRY[[:blank:]]*=[[:blank:]]*).*/\1\"'"${REGISTRY}"'\"/' docker/.env | |
| sed -i.bak -E 's/^(REPOSITORY[[:blank:]]*=[[:blank:]]*).*/\1\"'"${REPOSITORY}"'\"/' docker/.env | |
| sed -i.bak -E 's/^(MATOMO_ENABLED[[:blank:]]*=[[:blank:]]*).*/\1false/' docker/.env | |
| sed -i.bak -E 's/^(CKAN_CLOUDSTORAGE_ENABLED[[:blank:]]*=[[:blank:]]*).*/\1\"'"${CKAN_CLOUDSTORAGE_ENABLED}"'\"/' docker/.env.ckan.local | |
| sed -i.bak -E 's/^(CKAN_CLOUDSTORAGE_DRIVER_OPTIONS[[:blank:]]*=[[:blank:]]*).*/\1\"'"{'key': '${AWS_ACCESS_KEY_ID}', 'secret': '${AWS_SECRET_ACCESS_KEY}', 'token': ''}"'\"/' docker/.env.ckan.local | |
| sed -i.bak -E 's/^(CKAN_CLOUDSTORAGE_CONTAINER_NAME[[:blank:]]*=[[:blank:]]*).*/\1\"'"${CKAN_CLOUDSTORAGE_CONTAINER_NAME}"'\"/' docker/.env.ckan.local | |
| sed -i.bak -E 's/^(AWS_ACCESS_KEY_ID[[:blank:]]*=[[:blank:]]*).*/\1\"'"${AWS_ACCESS_KEY_ID}"'\"/' docker/.env.ckan.local | |
| sed -i.bak -E 's/^(AWS_SECRET_ACCESS_KEY[[:blank:]]*=[[:blank:]]*).*/\1\"'"${AWS_SECRET_ACCESS_KEY}"'\"/' docker/.env.ckan.local | |
| env: | |
| REGISTRY: ${{ secrets.REGISTRY }} | |
| REPOSITORY: ${{ secrets.REPOSITORY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| CKAN_CLOUDSTORAGE_ENABLED: ${{ secrets.CKAN_CLOUDSTORAGE_ENABLED }} | |
| CKAN_CLOUDSTORAGE_CONTAINER_NAME: ${{ secrets.CKAN_CLOUDSTORAGE_CONTAINER_NAME }} | |
| - name: configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1-node16 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_DEV_ROLE }} | |
| role-session-name: github-actions | |
| aws-region: eu-west-1 | |
| - name: login to AWS ECR | |
| id: login | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| with: | |
| registries: ${{ secrets.AWS_PROD_ACCOUNT_ID }} | |
| - name: download built images | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: /tmp | |
| - name: load built ckan image | |
| if: ${{ (needs.detect-changes.outputs.ckan == 'true') || (needs.detect-changes.outputs.assets == 'true')}} | |
| run: | | |
| docker load --input /tmp/ckan/ckan.tar | |
| - name: load built drupal image | |
| if: ${{ (needs.detect-changes.outputs.drupal == 'true') || (needs.detect-changes.outputs.assets == 'true') }} | |
| run: | | |
| docker load --input /tmp/drupal/drupal.tar | |
| - name: load built datapusher image | |
| if: ${{ needs.detect-changes.outputs.datapusher == 'true' }} | |
| run: | | |
| docker load --input /tmp/datapusher/datapusher.tar | |
| - name: load built solr image | |
| if: ${{ needs.detect-changes.outputs.solr == 'true' }} | |
| run: | | |
| docker load --input /tmp/solr/solr.tar | |
| - name: load built nginx image | |
| if: ${{ needs.detect-changes.outputs.nginx == 'true' }} | |
| run: | | |
| docker load --input /tmp/nginx/nginx.tar | |
| - name: bring services up | |
| working-directory: docker | |
| run: | | |
| docker-compose -f docker-compose.yml -f docker-compose.build.yml -p opendata up -d | |
| - name: wait until services have started | |
| shell: bash | |
| run: | | |
| # wait for services to start properly | |
| while [[ $(curl -L --write-out '%{http_code}' --silent --output /dev/null http://localhost) != "200" ]]; do | |
| echo "waiting for services to start up and initialize ..." | |
| sleep 5s | |
| done | |
| sleep 5s | |
| # print the list of containers | |
| docker ps -a | |
| # print logs to debug errors | |
| docker logs opendata_ckan_1 | |
| docker logs opendata_drupal_1 | |
| docker logs opendata_solr_1 | |
| docker logs opendata_datapusher_1 | |
| docker logs opendata_nginx_1 | |
| docker logs opendata_fuseki_1 | |
| # Might not be needed once https://github.com/cypress-io/cypress-docker-images/issues/873 is fixed | |
| - name: Get commit information | |
| id: commit_info | |
| run: | | |
| echo "commit_info_message=$(git show -s --pretty=oneline)" >> $GITHUB_OUTPUT | |
| echo "commit_info_sha=$(git show -s --pretty=%H)" >> $GITHUB_OUTPUT | |
| echo "commit_info_email=$(git show -s --pretty=%ae)" >> $GITHUB_OUTPUT | |
| echo "commit_info_author=$(git show -s --pretty=%an)" >> $GITHUB_OUTPUT | |
| echo "commit_info_remote=$(git config --get remote.origin.url)" >> $GITHUB_OUTPUT | |
| - name: run cypress e2e tests | |
| run: > | |
| docker run | |
| --network host | |
| -v $PWD:/e2e | |
| -w /e2e | |
| -e COMMIT_INFO_MESSAGE | |
| -e COMMIT_INFO_SHA | |
| -e COMMIT_INFO_BRANCH | |
| -e COMMIT_INFO_EMAIL | |
| -e COMMIT_INFO_AUTHOR | |
| -e COMMIT_INFO_REMOTE | |
| -e CYPRESS_CI_BUILD_URL | |
| --entrypoint cypress | |
| cypress/included:12.17.2 run | |
| --browser chrome:stable | |
| --record --key ${{ secrets.CYPRESS_RECORD_KEY }} | |
| --parallel | |
| --ci-build-id ${{ github.repository }}-${{ github.run_id }}-${{ github.run_attempt}} | |
| env: | |
| COMMIT_INFO_MESSAGE: ${{ steps.commit_info.outputs.commit_info_message }} | |
| COMMIT_INFO_SHA: ${{ steps.commit_info.outputs.commit_info_sha }} | |
| COMMIT_INFO_BRANCH: ${{ github.head_ref || github.ref_name }} | |
| COMMIT_INFO_EMAIL: ${{ steps.commit_info.outputs.commit_info_email }} | |
| COMMIT_INFO_AUTHOR: ${{ steps.commit_info.outputs.commit_info_author }} | |
| COMMIT_INFO_REMOTE: ${{ steps.commit_info.outputs.commit_info_remote }} | |
| CYPRESS_CI_BUILD_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| - name: export e2e test logs | |
| if: failure() | |
| shell: bash | |
| run: | | |
| docker logs opendata_ckan_1 > /tmp/opendata_ckan_1.log 2>&1 | |
| docker logs opendata_drupal_1 > /tmp/opendata_drupal_1.log 2>&1 | |
| docker logs opendata_solr_1 > /tmp/opendata_solr_1.log 2>&1 | |
| docker logs opendata_datapusher_1 > /tmp/opendata_datapusher_1.log 2>&1 | |
| docker logs opendata_nginx_1 > /tmp/opendata_nginx_1.log 2>&1 | |
| docker logs opendata_fuseki_1 > /tmp/opendata_fuseki_1.log 2>&1 | |
| - name: upload log artifacts | |
| if: failure() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: docker-logs | |
| path: /tmp/opendata_*.log | |
| - name: upload cypress screenshot artifacts | |
| uses: actions/upload-artifact@v3 | |
| if: failure() | |
| with: | |
| name: screenshots | |
| path: cypress/screenshots | |
| - name: upload cypress video artifacts | |
| uses: actions/upload-artifact@v3 | |
| if: failure() | |
| with: | |
| name: videos | |
| path: cypress/videos | |
| test-e2e-results: | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| name: Final E2E Results | |
| needs: [test-e2e] | |
| steps: | |
| - run: exit 1 | |
| if: >- | |
| ${{ | |
| contains(needs.*.result, 'failure') | |
| || contains(needs.*.result, 'cancelled') | |
| }} | |
| test-cdk: | |
| name: test-cdk | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| concurrency: | |
| group: ${{ github.ref }}/test-cdk | |
| cancel-in-progress: true | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v3 | |
| - name: install nodejs v16 | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| - name: cache node_modules | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.npm | |
| key: ${{ runner.os }}-node_cdk_v16-${{ hashFiles('**/package-lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-node_cdk_v16- | |
| - name: install cdk npm packages and verify installation | |
| working-directory: cdk | |
| run: | | |
| npm install | |
| $(npm bin)/cdk doctor | |
| - name: build cdk project | |
| working-directory: cdk | |
| run: | | |
| npm run build | |
| - name: test cdk project | |
| working-directory: cdk | |
| run: | | |
| npm run test | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| flags: cdk | |
| test: | |
| name: Test opendata extensions | |
| runs-on: ubuntu-latest | |
| needs: | |
| - detect-changes | |
| if: ${{ needs.detect-changes.outputs.ckan == 'true' }} | |
| container: | |
| image: ckan/ckan-dev:2.9.9 | |
| services: | |
| solr: | |
| image: ckan/ckan-solr-dev:2.9 | |
| postgres: | |
| image: postgres:12 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| redis: | |
| image: redis:3 | |
| env: | |
| CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test | |
| CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test | |
| CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test | |
| CKAN_SOLR_URL: http://solr:8983/solr/ckan | |
| CKAN_REDIS_URL: redis://redis:6379/1 | |
| PGPASSWORD: postgres | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Create Database | |
| run: | | |
| psql --host=postgres --username=postgres --command="CREATE USER ckan_default WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" | |
| createdb --encoding=utf-8 --host=postgres --username=postgres --owner=ckan_default ckan_test | |
| psql --host=postgres --username=postgres --command="CREATE USER datastore_write WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" | |
| psql --host=postgres --username=postgres --command="CREATE USER datastore_read WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" | |
| createdb --encoding=utf-8 --host=postgres --username=postgres --owner=datastore_write datastore_test | |
| - name: Install requirements | |
| run: | | |
| apk add proj proj-dev proj-util geos | |
| cd ckan/ckanext/ckanext-ytp_main | |
| pip install -r dev-requirements.txt | |
| pip install -r requirements.txt | |
| pip install -e . | |
| # Replace default path to CKAN core config file with the one on the container | |
| sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini | |
| cd ../ckanext-scheming | |
| pip install -e . | |
| cd ../ckanext-fluent | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-dcat | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-report | |
| pip install -r requirements.txt | |
| pip install -e . | |
| # Spatial is not currently used in tests | |
| # cd ../ckanext-spatial | |
| # pip install -r requirements.txt | |
| # pip install -e . | |
| cd ../ckanext-organizationapproval | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-hierarchy | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-harvest | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-showcase | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-sixodp_showcase | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-sixodp_showcasesubmit | |
| pip install -r requirements.txt | |
| pip install -e . | |
| cd ../ckanext-sitesearch | |
| pip install -e . | |
| - name: Setup extension | |
| run: | | |
| ckan -c ckan/ckanext/ckanext-ytp_main/test.ini db init | |
| - name: Run tests | |
| run: | | |
| pytest --ckan-ini=ckan/ckanext/ckanext-ytp_main/test.ini --cov=ckanext.ytp --disable-warnings ckan/ckanext/ckanext-ytp_main/ckanext/ytp/tests | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| flags: ckan |