test.yml

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Test CI

on:
  pull_request:
    paths-ignore:
      - 'docs/**'
  push:
    branches:
      - master
    paths-ignore:
      - 'docs/**'
  release:
    types:
      - created

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  DEBIAN_FRONTEND: noninteractive
  POSTGRES_DB: ci_test
  POSTGRES_PASSWORD: ci_test
  POSTGRES_USER: ci_test
  POSTGRES_HOST: 127.0.0.1
  SERVER_NAME: geotrek.local
  SECRET_KEY: test-test-test
  CONVERSION_HOST: localhost
  CAPTURE_HOST: localhost
  REDIS_HOST: 127.0.0.1
  VAR_DIR: /home/runner/work/Geotrek-admin/Geotrek-admin/var
  LANG: C.UTF-8

jobs:
  test:
    name: Python Unittests
    runs-on: ubuntu-latest
    container: makinacorpus/geodjango:${{ matrix.os }}

    strategy:
      matrix:
        os: ['bionic-3.8', 'focal-3.8']
        python-version: ['3.8']
        tests-env: ['tests', 'tests_nds']

    env:
      ENV: ${{ matrix.tests-env }}
      OS: ${{ matrix.os }}
      POSTGRES_HOST: postgres
      REDIS_HOST: redis

    services:
      postgres:
        image: postgis/postgis:10-2.5
        env:
          POSTGRES_DB: ci_test
          POSTGRES_PASSWORD: ci_test
          POSTGRES_USER: ci_test
        ports:
          - 5432:5432
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
      redis:
        image: redis:5.0-alpine
        ports:
          - 6379:6379

    steps:
    - uses: actions/checkout@v3
    - uses: actions/cache@v3
      with:
        path: |
          ./venv

        key: pip-${{ matrix.python-version }}
        restore-keys: |
          pip-${{ matrix.python-version }}

    - name: Prepare test env
      run: |
        apt-get update -q
        apt-get -qq -y install \
           unzip \
           gettext \
           binutils \
           libproj-dev \
           gdal-bin \
           sudo \
           less \
           curl \
           git \
           iproute2 \
           software-properties-common \
           shared-mime-info \
           fonts-liberation \
           libssl-dev \
           libfreetype6-dev \
           libxml2-dev \
           libxslt-dev \
           libcairo2 \
           libpango1.0-0 \
           libpangocairo-1.0-0 \
           libgdk-pixbuf2.0-dev \
           libffi-dev \
           libvips
        apt-get install -y --no-install-recommends postgis

    - name: Install python dependencies
      run: |
        python3.8 -m venv ./venv
        ./venv/bin/pip3 install --upgrade pip wheel setuptools
        ./venv/bin/pip3 install -r requirements.txt -U
        ./venv/bin/pip3 install -r dev-requirements.txt -U

    - name: Create test required directories
      run: |
        mkdir -p ./var/log ./var/cache/sessions

    - name: Check missing migrations
      run: |
        ENV=dev ./venv/bin/python3 ./manage.py makemigrations --check

    - name: Check running migrations
      run: |
        ENV=dev ./venv/bin/python3 ./manage.py migrate

    - name: Prepare environment
      run: |
        mkdir -p $VAR_DIR
        mkdir -p $VAR_DIR/conf/extra_static
        ./venv/bin/python3 ./manage.py collectstatic --clear --noinput --verbosity=0
        ./venv/bin/python3 ./manage.py compilemessages

    - name: Launch Celery
      run: |
        ./venv/bin/celery -A geotrek worker -c 1 &

    - name: Unit test
      run: |
        ./venv/bin/coverage run ./manage.py test -v 3
        ./venv/bin/coverage xml -o coverage.xml

    - uses: codecov/codecov-action@v3
      with:
        files: ./coverage.xml
        env_vars: OS,ENV
        token: ${{ secrets.CODECOV_TOKEN }} # not usually required for public repos
        fail_ci_if_error: true # optional (default = false)

  build_docker_image:
    name: Build docker image
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v3
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
      - name: Build Docker image
        run: |
          docker build -t geotrek:latest .
      - name: Upload image
        uses: ishworkh/docker-image-artifact-upload@v1
        with:
          image: "geotrek:latest"

  build_deb_18_04:
      name: Build 18.04 package
      runs-on: ubuntu-latest
      steps:
        - uses: actions/checkout@v3
        - name: Prepare debian 18.04 versioning
          run: |
            grep '^[0-9]\+\.[0-9]\+\.[0-9]\+$' VERSION || sed -i 's/+dev/.ubuntu18.04~dev'$GITHUB_RUN_ID'/' debian/changelog
            sed -i 's/geotrek-admin (\([0-9]\+\.[0-9]\+\.[0-9]\+\)\(.*\)) RELEASED;/geotrek-admin (\1.ubuntu18.04\2) bionic;/' debian/changelog

        - name: Building package debian 18.04
          run: |
            DISTRO=ubuntu:bionic make build_deb
            cp ./dpkg/*.deb /home/runner/work/Geotrek-admin/

        - name: Archive package artifact
          uses: actions/upload-artifact@v3
          with:
            name: debian-18-04
            path: |
              /home/runner/work/Geotrek-admin/*.deb

  build_deb_20_04:
    name: Build 20.04 package
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Prepare debian 20.04 versioning
        run: |
          sed -i 's/+dev/.ubuntu20.04~dev'$GITHUB_RUN_ID'/' debian/changelog
          sed -i 's/geotrek-admin (\([0-9]\+\.[0-9]\+\.[0-9]\+\)\(.*\)) RELEASED;/geotrek-admin (\1.ubuntu20.04\2) focal;/' debian/changelog

      - name: Building package debian 18.04
        run: |
          DISTRO=ubuntu:focal make build_deb
          cp ./dpkg/*.deb /home/runner/work/Geotrek-admin/

      - name: Archive package artifact
        uses: actions/upload-artifact@v3
        with:
          name: debian-20-04
          path: |
            /home/runner/work/Geotrek-admin/*.deb

  e2e_docker_image:
    name: Tests E2E docker
    runs-on: ubuntu-20.04
    needs: [build_docker_image]
    services:
      postgres:
        image: postgis/postgis:10-2.5
        env:
          POSTGRES_DB: ci_test
          POSTGRES_PASSWORD: ci_test
          POSTGRES_USER: ci_test
        ports:
          - 5432:5432
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
    steps:
      - uses: actions/checkout@v3
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "geotrek:latest"
      - name: Launch service
        run: |
          docker tag geotrek:latest geotrekce/admin:latest
          cp .env-prod.dist .env; cp docker-compose-prod.yml docker-compose.yml
          echo 'LANGUAGES=en fr' >> .env
          echo 'LANGUAGE_CODE=en' >> .env
          sudo sed -i 's/localhost/geotrek.local/' .env
          sudo sed -i 's/127.0.0.1 localhost/127.0.0.1 localhost geotrek.local/' /etc/hosts
          mkdir -p var/conf/extra_static
          cp cypress/custom.py var/conf/custom.py
          echo 'POSTGRES_USER=ci_test' >> .env
          echo 'POSTGRES_PASSWORD=ci_test' >> .env
          echo 'POSTGRES_DB=ci_test' >> .env
          docker-compose run web update.sh
          make load_data
          make load_test_integration
          docker-compose up -d

      - name: Prepare nginx
        run: |
          sudo apt-get install -q -y nginx
          sudo mkdir /var/www/geotrek -p
          ls -als
          pwd
          sudo ln -s /home/runner/work/Geotrek-admin/Geotrek-admin/var/media /var/www/geotrek/media
          sudo ln -s /home/runner/work/Geotrek-admin/Geotrek-admin/var/static /var/www/geotrek/static
          sudo ln -s /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/nginx.conf /etc/nginx/sites-enabled/geotrek.conf
          sudo systemctl restart nginx

      - name: install Cypress
        run: |
          cd /home/runner/work/Geotrek-admin/Geotrek-admin/cypress
          npm ci
          npm list
          echo "Waiting for container..."
          while ! nc -z "127.0.0.1" "8000"; do
            sleep 0.1
          done

      - name: E2E test
        run: |
          /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/node_modules/.bin/cypress run -P /home/runner/work/Geotrek-admin/Geotrek-admin/cypress --record --key 64a5a9b3-9869-4a2f-91e4-e3cd27c2f564
        continue-on-error: false

      - name: Archive results as artifacts
        uses: actions/upload-artifact@v3
        with:
          name: cypress-docker
          path: |
            /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/videos/*.mp4
            /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/screenshots/*.png

  e2e_deb_20_04:
    name: Tests E2E 20.04
    runs-on: ubuntu-20.04
    needs: [ build_deb_20_04 ]

    env:
      CYPRESS_BASE_URL: http://geotrek.local
      CYPRESS_CACHE_FOLDER: ~/cypress/cache

    services:
      postgres:
        image: postgis/postgis:10-2.5
        env:
          POSTGRES_DB: ci_test
          POSTGRES_PASSWORD: ci_test
          POSTGRES_USER: ci_test
        ports:
          - 5432:5432
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready

    steps:
      - uses: actions/checkout@v3
      - uses: actions/download-artifact@v3
        with:
          name: debian-20-04

      - name: Simulate install.sh procedure
        run: |
          sudo service apache2 stop
          sudo apt-get purge apache2 apache2-utils apache2.2-bin
          sudo apt-get autoremove
          echo "deb [arch=amd64] https://packages.geotrek.fr/ubuntu $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/geotrek.list
          wget -O- "https://packages.geotrek.fr/geotrek.gpg.key" | sudo apt-key add -
          sudo apt-get -q update
          echo "geotrek-admin geotrek-admin/LANGUAGES string en fr" >> .debian_settings
          echo "geotrek-admin geotrek-admin/POSTGRES_USER string $POSTGRES_USER" >> .debian_settings
          echo "geotrek-admin geotrek-admin/POSTGRES_DB string $POSTGRES_DB" >> .debian_settings
          echo "geotrek-admin geotrek-admin/POSTGRES_PASSWORD string $POSTGRES_PASSWORD" >> .debian_settings
          echo "geotrek-admin geotrek-admin/SERVER_NAME string geotrek.local" >> .debian_settings
          mkdir -p /opt/geotrek-admin/var/conf/
          cp /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/custom.py /opt/geotrek-admin/var/conf/custom.py
          sudo sed -i 's/127.0.0.1 localhost/127.0.0.1 localhost geotrek.local/' /etc/hosts
          sudo debconf-set-selections .debian_settings

      - name: Install package
        run: |
          sudo unlink /etc/nginx/sites-enabled/default
          sudo apt-get install /home/runner/work/Geotrek-admin/Geotrek-admin/*.deb || exit 0;
          sudo systemctl restart nginx

      - name: Load Data
        run: |
          sudo geotrek loaddata minimal
          sudo geotrek loaddata cirkwi
          sudo geotrek loaddata basic
          sudo geotrek loaddata test-integration
          for dir in `ls -d /opt/geotrek-admin/lib/python3*/site-packages/geotrek/*/fixtures/upload`; do
          	cd $dir > /dev/null
          	sudo cp -r * /opt/geotrek-admin/var/media/upload/
          done

      - name: Install cypress
        run: |
          cd /home/runner/work/Geotrek-admin/Geotrek-admin/cypress
          npm ci
          npm list

      - name: E2E test
        run: |
          /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/node_modules/.bin/cypress run -P /home/runner/work/Geotrek-admin/Geotrek-admin/cypress --record --key 64a5a9b3-9869-4a2f-91e4-e3cd27c2f564
        continue-on-error: false

      - name: Archive results as artifacts
        uses: actions/upload-artifact@v3
        with:
          name: cypress-20.04
          path: |
            /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/videos/*.mp4
            /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/screenshots/*.png
  deploy:
    name: Publish (on release only)
    runs-on: ubuntu-latest
    needs: [ test, e2e_docker_image, build_deb_18_04, e2e_deb_20_04 ]
    if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
    steps:
      - uses: actions/checkout@v3
      - name: Download 18.04 debian artifact
        uses: actions/download-artifact@v3
        with:
          name: debian-18-04
      - name: Download 20.04 debian artifact
        uses: actions/download-artifact@v3
        with:
          name: debian-20-04
      - name: Download docker image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: "geotrek:latest"
      - name: Attach debian packages as release binaries
        uses: skx/github-action-publish-binaries@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          args: '*.deb'
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_LOGIN }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - name: Publish Docker image (v2 and latest)
        run: |
          if [[ "${{ github.ref }}" == *"dev"* ]]; then
            export DOCKER_TAG=dev
          else
            docker tag geotrek:latest geotrekce/admin:$(cat VERSION)
            docker push geotrekce/admin:$(cat VERSION)
            export DOCKER_TAG=latest
          fi
          docker tag geotrek:latest geotrekce/admin:$DOCKER_TAG
          docker push geotrekce/admin:$DOCKER_TAG
      - name: Install SSH key
        uses: shimataro/ssh-key-action@v2
        with:
          key: ${{ secrets.SSH_PRIVATE_KEY }}
          known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }}
      - name: Publish debian packages
        run: |
          if [[ "${{ github.ref }}" == *"dev"* ]]; then
            export DEB_COMPONENT=dev
          else
            export DEB_COMPONENT=main
          fi
          echo "${{ github.ref }} : Publishing as $DEB_COMPONENT package"
          scp -P ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no /home/runner/work/Geotrek-admin/Geotrek-admin/geotrek-admin_*_amd64.deb ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }}:/srv/packages/incoming/$DEB_COMPONENT/
          if [[ "${{ github.ref }}" == *"dev"* ]]; then
            ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make bionic_dev -C /srv/packages
            ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make focal_dev -C /srv/packages
          else
            ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make bionic_main -C /srv/packages
            ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make focal_main -C /srv/packages
          fi