Skip to content

Use PGRouting to handle routing directly in Database through API #4606

Use PGRouting to handle routing directly in Database through API

Use PGRouting to handle routing directly in Database through API #4606

Workflow file for this run

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: Test CI
on:
pull_request:
paths-ignore:
- 'docs/**'
push:
branches:
- master
paths-ignore:
- 'docs/**'
release:
types:
- created
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
DEBIAN_FRONTEND: noninteractive
POSTGRES_DB: ci_test
POSTGRES_PASSWORD: ci_test
POSTGRES_USER: ci_test
POSTGRES_HOST: 127.0.0.1
SERVER_NAME: geotrek.local
SECRET_KEY: test-test-test
CONVERSION_HOST: localhost
CAPTURE_HOST: localhost
REDIS_HOST: 127.0.0.1
VAR_DIR: /home/runner/work/Geotrek-admin/Geotrek-admin/var
LANG: C.UTF-8
LANGUAGES: en fr
LANGUAGE_CODE: en
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
jobs:
test:
name: Python Unittests
runs-on: ubuntu-latest
container: makinacorpus/geodjango:${{ matrix.os }}
permissions:
packages: write # required to publish docker image
strategy:
matrix:
os: ['focal-3.8', 'jammy-3.10']
tests-env: ['tests', 'tests_nds']
env:
ENV: ${{ matrix.tests-env }}
OS: ${{ matrix.os }}
POSTGRES_HOST: postgres
REDIS_HOST: redis
services:
postgres:
image: pgrouting/pgrouting:12-3.0-3.0.0
env:
POSTGRES_DB: ci_test
POSTGRES_PASSWORD: ci_test
POSTGRES_USER: ci_test
ports:
- 5432:5432
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
redis:
image: redis:5.0-alpine
ports:
- 6379:6379
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
with:
path: |
./venv
key: pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt') }}-${{ hashFiles('./dev-requirements.txt') }}-${{ hashFiles('./docs/requirements.txt') }}
restore-keys: |
pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt') }}-${{ hashFiles('./dev-requirements.txt') }}-${{ hashFiles('./docs/requirements.txt') }}
pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt') }}-${{ hashFiles('./dev-requirements.txt') }}
pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt') }}
pip-${{ matrix.os }}
- name: Prepare test env
run: |
apt-get update -q
apt-get -qq -y install \
unzip \
gettext \
binutils \
libproj-dev \
gdal-bin \
sudo \
less \
curl \
git \
iproute2 \
software-properties-common \
shared-mime-info \
fonts-liberation \
libssl-dev \
libfreetype6-dev \
libxml2-dev \
libxslt-dev \
libcairo2 \
libpango1.0-0 \
libpangocairo-1.0-0 \
libgdk-pixbuf2.0-dev \
libffi-dev \
libvips
apt-get install -y --no-install-recommends postgis
- name: Install python venv
run: |
python3 -m venv ./venv
- name: Install python dependencies
run: |
./venv/bin/pip3 install --upgrade pip wheel setuptools pip-tools -c requirements.txt -c dev-requirements.txt
./venv/bin/pip-sync requirements.txt dev-requirements.txt
- name: Create test required directories
run: |
mkdir -p ./var/log ./var/cache/sessions
- name: Prepare environment
run: |
mkdir -p $VAR_DIR/conf/extra_static $VAR_DIR/tmp
./venv/bin/python3 ./manage.py collectstatic --clear --noinput --verbosity=0
./venv/bin/python3 ./manage.py compilemessages
- name: Check missing migrations
run: |
ENV=dev ./venv/bin/python3 ./manage.py makemigrations --check
- name: Check running migrations
run: |
ENV=dev ./venv/bin/python3 ./manage.py migrate
- name: Launch Celery
run: |
./venv/bin/celery -A geotrek worker -c 1 &
- name: Unit test
run: |
./venv/bin/coverage run --parallel-mode --concurrency=multiprocessing ./manage.py test --parallel -v 3
./venv/bin/coverage combine
./venv/bin/coverage xml -o coverage.xml
- name: Save coverage
uses: actions/upload-artifact@v4
with:
name: coverage_${{ matrix.os }}_${{ matrix.tests-env }}
path: coverage.xml
upload_coverage:
name: Upload coverage to codecov
runs-on: ubuntu-latest
needs: [ test ]
strategy:
matrix:
os: ['focal-3.8', 'jammy-3.10']
tests-env: ['tests', 'tests_nds']
env:
ENV: ${{ matrix.tests-env }}
OS: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
name: coverage_${{ matrix.os }}_${{ matrix.tests-env }}
path: .
- uses: codecov/codecov-action@v5
with:
files: ./coverage.xml
env_vars: OS,ENV
token: ${{ secrets.CODECOV_TOKEN }} # not usually required for public repos
fail_ci_if_error: true # optional (default = false)
build_docker_image:
name: Build docker image
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Build and push
id: docker_build
uses: docker/build-push-action@v6
with:
push: false
builder: ${{ steps.buildx.outputs.name }}
tags: "geotrek:latest"
load: true
file: ./docker/Dockerfile
- name: Upload image
uses: ishworkh/[email protected]
with:
image: "geotrek:latest"
build_deb:
name: Build debian package
runs-on: ubuntu-latest
strategy:
matrix:
os: ['bionic', 'focal', 'jammy']
include:
- os: 'bionic'
code: '18.04'
- os: 'focal'
code: '20.04'
- os: 'jammy'
code: '22.04'
env:
OS: ${{ matrix.os }}
CODE: ${{ matrix.code }}
DISTRO: ubuntu:${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- name: Prepare versioning
run: |
grep '^[0-9]\+\.[0-9]\+\.[0-9]\+$' VERSION || sed -i 's/+dev/.ubuntu'$CODE'~dev'$GITHUB_RUN_ID'/' debian/changelog
sed -i 's/geotrek-admin (\([0-9]\+\.[0-9]\+\.[0-9]\+\)\(.*\)) RELEASED;/geotrek-admin (\1.ubuntu'$CODE'\2) '$OS';/' debian/changelog
- name: Building package
run: |
make build_deb
cp ./dpkg/*.deb /home/runner/work/Geotrek-admin/
- name: Archive package artifact
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.os }}
path: |
/home/runner/work/Geotrek-admin/*.deb
e2e_docker_image:
name: Tests E2E docker
runs-on: ubuntu-20.04
needs: [build_docker_image]
env:
CYPRESS_BASE_URL: http://geotrek.local
CYPRESS_CACHE_FOLDER: ~/cypress/cache
services:
postgres:
image: pgrouting/pgrouting:12-3.0-3.0.0
env:
POSTGRES_DB: ci_test
POSTGRES_PASSWORD: ci_test
POSTGRES_USER: ci_test
ports:
- 5432:5432
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
steps:
- uses: actions/checkout@v4
- name: Download image
uses: ishworkh/[email protected]
with:
image: "geotrek:latest"
- name: Launch service
run: |
docker tag geotrek:latest geotrekce/admin:latest
cp docker/install/.env.dist .env
cp docker/install/docker-compose.yml docker-compose.yml
echo 'LANGUAGES=en fr' >> .env
echo 'LANGUAGE_CODE=en' >> .env
sudo sed -i 's/localhost/geotrek.local/' .env
sudo sed -i 's/127.0.0.1 localhost/127.0.0.1 localhost geotrek.local/' /etc/hosts
mkdir -p var/conf/extra_static
cp cypress/custom.py var/conf/custom.py
echo 'POSTGRES_USER=ci_test' >> .env
echo 'POSTGRES_PASSWORD=ci_test' >> .env
echo 'POSTGRES_DB=ci_test' >> .env
docker compose run web update.sh
make load_data
make load_test_integration
docker compose up -d
- name: Prepare nginx
run: |
sudo service apache2 stop
sudo apt-get purge apache2 apache2-utils apache2.2-bin
sudo apt-get autoremove
sudo apt-get install -q -y nginx
sudo unlink /etc/nginx/sites-enabled/default
sudo mkdir /var/www/geotrek -p
ls -als
pwd
sudo ln -s /home/runner/work/Geotrek-admin/Geotrek-admin/var/media /var/www/geotrek/media
sudo ln -s /home/runner/work/Geotrek-admin/Geotrek-admin/var/static /var/www/geotrek/static
sudo ln -s /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/nginx.conf /etc/nginx/sites-enabled/geotrek.conf
sudo systemctl restart nginx
- name: install Cypress
run: |
cd /home/runner/work/Geotrek-admin/Geotrek-admin/cypress
npm ci
npm list
echo "Waiting for container..."
while ! nc -z "127.0.0.1" "8001"; do
sleep 0.1
done
- name: E2E test
run: |
/home/runner/work/Geotrek-admin/Geotrek-admin/cypress/node_modules/.bin/cypress run -P /home/runner/work/Geotrek-admin/Geotrek-admin/cypress --record --key 64a5a9b3-9869-4a2f-91e4-e3cd27c2f564
continue-on-error: false
- name: Archive results as artifacts
uses: actions/upload-artifact@v4
with:
name: cypress-docker
path: |
/home/runner/work/Geotrek-admin/Geotrek-admin/cypress/videos/*.mp4
/home/runner/work/Geotrek-admin/Geotrek-admin/cypress/screenshots/*.png
e2e_deb_20_04:
name: Tests E2E 20.04
runs-on: ubuntu-20.04
needs: [ build_deb ]
env:
CYPRESS_BASE_URL: http://geotrek.local
CYPRESS_CACHE_FOLDER: ~/cypress/cache
services:
postgres:
image: pgrouting/pgrouting:12-3.0-3.0.0
env:
POSTGRES_DB: ci_test
POSTGRES_PASSWORD: ci_test
POSTGRES_USER: ci_test
ports:
- 5432:5432
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
name: 'focal'
- name: Simulate install.sh procedure
run: |
sudo service apache2 stop
sudo apt-get purge apache2 apache2-utils apache2.2-bin
sudo apt-get autoremove
echo "deb [arch=amd64] https://packages.geotrek.fr/ubuntu $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/geotrek.list
wget -O- "https://packages.geotrek.fr/geotrek.gpg.key" | sudo apt-key add -
sudo apt-get -q update
echo "geotrek-admin geotrek-admin/LANGUAGES string en fr" >> .debian_settings
echo "geotrek-admin geotrek-admin/POSTGRES_USER string $POSTGRES_USER" >> .debian_settings
echo "geotrek-admin geotrek-admin/POSTGRES_DB string $POSTGRES_DB" >> .debian_settings
echo "geotrek-admin geotrek-admin/POSTGRES_PASSWORD string $POSTGRES_PASSWORD" >> .debian_settings
echo "geotrek-admin geotrek-admin/SERVER_NAME string geotrek.local" >> .debian_settings
mkdir -p /opt/geotrek-admin/var/conf/
cp /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/custom.py /opt/geotrek-admin/var/conf/custom.py
sudo sed -i 's/127.0.0.1 localhost/127.0.0.1 localhost geotrek.local/' /etc/hosts
sudo debconf-set-selections .debian_settings
- name: Install package
run: |
sudo unlink /etc/nginx/sites-enabled/default
sudo apt-get install /home/runner/work/Geotrek-admin/Geotrek-admin/*.deb || exit 0;
sudo systemctl restart nginx
- name: Load Data
run: |
sudo geotrek loaddata minimal
sudo geotrek loaddata cirkwi
sudo geotrek loaddata basic
sudo geotrek loaddata test-integration
for dir in `ls -d /opt/geotrek-admin/lib/python3*/site-packages/geotrek/*/fixtures/upload`; do
cd $dir > /dev/null
sudo cp -r * /opt/geotrek-admin/var/media/upload/
done
- name: Install cypress
run: |
cd /home/runner/work/Geotrek-admin/Geotrek-admin/cypress
npm ci
npm list
- name: E2E test
run: |
/home/runner/work/Geotrek-admin/Geotrek-admin/cypress/node_modules/.bin/cypress run -P /home/runner/work/Geotrek-admin/Geotrek-admin/cypress --record --key 64a5a9b3-9869-4a2f-91e4-e3cd27c2f564
continue-on-error: false
- name: Archive results as artifacts
uses: actions/upload-artifact@v4
with:
name: cypress-20.04
path: |
/home/runner/work/Geotrek-admin/Geotrek-admin/cypress/videos/*.mp4
/home/runner/work/Geotrek-admin/Geotrek-admin/cypress/screenshots/*.png
deploy:
name: Publish (on release only)
runs-on: ubuntu-latest
needs: [ test, e2e_docker_image, build_deb, e2e_deb_20_04 ]
if: ${{ github.event_name != 'pull_request' }}
steps:
- uses: actions/checkout@v4
- name: Download 18.04 debian artifact
uses: actions/download-artifact@v4
if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
with:
name: bionic
- name: Download 20.04 debian artifact
uses: actions/download-artifact@v4
if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
with:
name: focal
- name: Download docker image
uses: ishworkh/[email protected]
with:
image: 'geotrek:latest'
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_LOGIN }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
with:
images: |
geotrekce/admin
ghcr.io/geotrekce/geotrek-admin
- name: Build and push image
uses: docker/build-push-action@v6
with:
push: true
provenance: mode=max
sbom: true
builder: ${{ steps.buildx.outputs.name }}
tags: ${{ steps.meta.outputs.tags }}
file: ./docker/Dockerfile
- name: Install SSH key
uses: shimataro/ssh-key-action@v2
if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
with:
key: ${{ secrets.SSH_PRIVATE_KEY }}
known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }}
- name: Publish debian packages
if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
run: |
if [[ "${{ github.ref }}" == *"dev"* ]]; then
export DEB_COMPONENT=dev
else
export DEB_COMPONENT=main
fi
echo "${{ github.ref }} : Publishing as $DEB_COMPONENT package"
scp -P ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no /home/runner/work/Geotrek-admin/Geotrek-admin/geotrek-admin_*_amd64.deb ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }}:/srv/packages/incoming/$DEB_COMPONENT/
if [[ "${{ github.ref }}" == *"dev"* ]]; then
ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make bionic_dev -C /srv/packages
ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make focal_dev -C /srv/packages
else
ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make bionic_main -C /srv/packages
ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make focal_main -C /srv/packages
fi
- name: Download 22.04 debian artifact
if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
uses: actions/download-artifact@v4
with:
name: jammy
- name: Attach debian packages as release binaries
if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
uses: skx/github-action-publish-binaries@master
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
args: '*.deb'