From f7a37c66da5c1b7bd17b708fb1468bbfa3313d14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Trimaille?= Date: Sun, 21 Aug 2022 15:32:13 +0200 Subject: [PATCH 01/32] Upgrade the stack from Django 2.2 to 3.2 (#233) * Upgrade the stack from Django 2.2 to 3.2 * Upgrade pip as well when installing Co-authored-by: Dimas Ciputra --- REQUIREMENTS.txt | 18 ++++++++++++------ dockerize/docker/Dockerfile | 2 +- dockerize/docker/REQUIREMENTS.txt | 13 ++++++++----- dockerize/production/Dockerfile | 1 + qgis-app/REQUIREMENTS_plugins.txt | 12 +++++++----- qgis-app/users/views.py | 21 ++++++++++----------- 6 files changed, 39 insertions(+), 28 deletions(-) diff --git a/REQUIREMENTS.txt b/REQUIREMENTS.txt index b8847c5a..f2240fb3 100644 --- a/REQUIREMENTS.txt +++ b/REQUIREMENTS.txt @@ -1,10 +1,14 @@ -Django==2.2.25 +Django==3.2.11 + # Currently broken with 'no module named defaults' error #Feedjack==0.9.18 # So use George's fork rather # git+https://github.com/Erve1879/feedjack.git -# George's is also broken: use my fork (django 1.8 ready) -git+https://github.com/elpaso/feedjack.git +# George's is also broken: use elpaso fork (django 1.8 ready) +# git+https://github.com/elpaso/feedjack.git +# His is also broken, use dimasciput (django 3.2 ready) +git+https://github.com/dimasciput/feedjack.git + Markdown==2.3.1 #PIL==1.1.7 Pillow @@ -16,7 +20,7 @@ argparse==1.2.1 django-annoying==0.7.7 django-auth-ldap==1.2.6 django-autoslug==1.7.1 -django-debug-toolbar==1.11.1 +django-debug-toolbar==3.2.4 django-endless-pagination==2.0 django-extensions==1.2.0 django-generic-aggregation==0.3.2 @@ -24,9 +28,11 @@ django-haystack==2.4.0 #django-olwidget==0.61.0 unmaintained, use this fork git+https://github.com/Christophe31/olwidget.git django-pagination==1.0.7 + # Unmaintained! #django-ratings==0.3.7 -git+https://github.com/enikesha/django-ratings.git +git+https://github.com/gelo-zhukov/django-ratings.git + django-simple-ratings==0.3.2 # SIMPLEMENU git+https://github.com/elpaso/django-simplemenu.git @@ -54,4 +60,4 @@ django-sortable-listview==0.43 django-user-map djangorestframework==3.12.2 django-rest-auth==0.9.5 -drf-yasg \ No newline at end of file +drf-yasg diff --git a/dockerize/docker/Dockerfile b/dockerize/docker/Dockerfile index f2be2c8c..a77ab97b 100644 --- a/dockerize/docker/Dockerfile +++ b/dockerize/docker/Dockerfile @@ -7,7 +7,7 @@ MAINTAINER Dimas Ciputra RUN apt-get clean all RUN apt-get update && apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev ADD REQUIREMENTS.txt /REQUIREMENTS.txt -RUN pip install -r /REQUIREMENTS.txt +RUN pip install --upgrade pip && pip install -r /REQUIREMENTS.txt RUN pip install uwsgi # https://docs.docker.com/examples/running_ssh_service/ diff --git a/dockerize/docker/REQUIREMENTS.txt b/dockerize/docker/REQUIREMENTS.txt index ebcd9493..e05c7661 100644 --- a/dockerize/docker/REQUIREMENTS.txt +++ b/dockerize/docker/REQUIREMENTS.txt @@ -1,4 +1,4 @@ -django==2.2.25 +django==3.2.11 django-auth-ldap python-ldap django-taggit @@ -6,8 +6,10 @@ django-tinymce==3.4.0 psycopg2 # Updates for Django 2 git+https://github.com/metamatik/django-templatetag-sugar.git -# Updates for Django 2 -git+https://github.com/elpaso/django-ratings.git@modernize + +# Updates for Django 3 +git+https://github.com/gelo-zhukov/django-ratings.git + django-taggit-autosuggest django-annoying # Updates for Django 2 @@ -20,7 +22,7 @@ django-bootstrap-pagination django-sortable-listview sorl-thumbnail django-extensions -django-debug-toolbar==1.11.1 +django-debug-toolbar==3.2.4 whoosh django-haystack @@ -29,8 +31,9 @@ django-haystack # git+https://github.com/Erve1879/feedjack.git # George's is also broken: use my fork (django 1.8 ready) # git+https://github.com/elpaso/feedjack.git -# His is also broken, use mine (django 2.2 ready) +# His is also broken, use dimasciput (django 3.2 ready) git+https://github.com/dimasciput/feedjack.git + feedparser==5.2.1 celery==4.3.1 requests==2.23.0 diff --git a/dockerize/production/Dockerfile b/dockerize/production/Dockerfile index 7a57d864..a51f6e85 100644 --- a/dockerize/production/Dockerfile +++ b/dockerize/production/Dockerfile @@ -14,6 +14,7 @@ RUN mkdir -p /usr/src; mkdir -p /home/web && \ ln -s /usr/src/plugins/qgis-app /home/web/django_project RUN cd /usr/src/plugins/dockerize/docker && \ + pip install --upgrade pip && \ pip install -r REQUIREMENTS.txt && \ pip install uwsgi && \ rm -rf /uwsgi.conf && \ diff --git a/qgis-app/REQUIREMENTS_plugins.txt b/qgis-app/REQUIREMENTS_plugins.txt index 0393e683..e3894f80 100644 --- a/qgis-app/REQUIREMENTS_plugins.txt +++ b/qgis-app/REQUIREMENTS_plugins.txt @@ -1,4 +1,4 @@ -django==2.2.25 +django==3.2.11 django-auth-ldap python-ldap django-taggit @@ -6,8 +6,10 @@ django-tinymce==3.4.0 psycopg2 # Updates for Django 2 git+https://github.com/metamatik/django-templatetag-sugar.git -# Updates for Django 2 -git+https://github.com/elpaso/django-ratings.git@modernize + +# Updates for Django 3 +git+https://github.com/gelo-zhukov/django-ratings.git + django-taggit-autosuggest django-annoying # Updates for Django 2 @@ -20,6 +22,6 @@ django-bootstrap-pagination django-sortable-listview sorl-thumbnail django-extensions -django-debug-toolbar==1.11.1 +django-debug-toolbar==3.2.4 whoosh -django-haystack \ No newline at end of file +django-haystack diff --git a/qgis-app/users/views.py b/qgis-app/users/views.py index 23344cf1..37604192 100644 --- a/qgis-app/users/views.py +++ b/qgis-app/users/views.py @@ -1,7 +1,7 @@ from django.http import HttpResponse from django.http import HttpResponseRedirect from django.contrib import messages -from django.shortcuts import render_to_response, get_object_or_404 +from django.shortcuts import render, get_object_or_404 from annoying.functions import get_object_or_None from django.template import RequestContext from django.utils.translation import ugettext_lazy as _ @@ -31,7 +31,9 @@ def usersMap(theRequest): myMap = InfoMap(users) - return render_to_response("view_users.html", + return render( + theRequest, + "view_users.html", { 'myMap' : myMap, 'myUserCount' : myUserCount, @@ -48,11 +50,11 @@ def createUser(theRequest): myForm.save() return HttpResponseRedirect("/community-map/view_users.html") else: - return render_to_response("create_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount }, + return render(theRequest, "create_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount }, context_instance=RequestContext(theRequest)) else: myForm = QgisUserForm() - return render_to_response("create_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount }, + return render(theRequest, "create_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount }, context_instance=RequestContext(theRequest)) @@ -64,12 +66,12 @@ def updateUser(theRequest, theId): if myForm.is_valid(): myForm.save() return HttpResponseRedirect("/community-map/view_users.html") - return render_to_response("update_user_form.html", { + return render(theRequest, "update_user_form.html", { 'myUser': myUser, 'myForm': myForm, 'myUserCount' : myUserCount }, context_instance=RequestContext(theRequest)) else: myForm = QgisUserForm(instance=myUser) - return render_to_response("update_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount}, + return render(theRequest, "update_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount}, context_instance=RequestContext(theRequest)) def emailEditAddress(theRequest): @@ -101,14 +103,11 @@ def emailEditAddress(theRequest): msg = _("User is NOT valid.") messages.warning(theRequest, msg, fail_silently=True) logging.info("User or form is NOT valid") - return render_to_response("update_user.html", {'myForm' : myForm, 'myUserCount' : myUserCount}, + return render(theRequest, "update_user.html", {'myForm' : myForm, 'myUserCount' : myUserCount}, context_instance=RequestContext(theRequest)) else: myUserCount = QgisUser.objects.all().count() myForm = EmailForm() - return render_to_response("update_user.html", {'myForm' : myForm}, + return render(theRequest, "update_user.html", {'myForm' : myForm}, context_instance=RequestContext(theRequest)) - - - From d4c855062896916bce274af03bfa2db5a2541801 Mon Sep 17 00:00:00 2001 From: Dimas Ciputra Date: Sun, 21 Aug 2022 15:55:44 +0200 Subject: [PATCH 02/32] Update docker compose (#256) * Fix min_qg_version query in plugins.xml (#224) * fix min_qg_version query * added 0 patch value for qgis version against max_qg_version * only add patch if it has major.minor version * Update docker-compose and dockerfile * Update nginx configuration * Update smtp * Add celery beat * Add feedjack update celery task * Add metabase configuration * Update test.yaml * Add test docker-compose Co-authored-by: sumandari --- .dockerignore | 1 + .github/workflows/test.yaml | 16 +- .gitignore | 2 + dockerize/.env.template | 6 + dockerize/Makefile | 46 +++- .../docker-compose.override.template.yml | 55 ++++ dockerize/docker-compose.override.test.yml | 19 ++ dockerize/docker-compose.yml | 252 ++++++++++-------- dockerize/docker/Dockerfile | 32 ++- dockerize/docker/REQUIREMENTS.txt | 3 +- dockerize/sites-enabled/default.conf | 17 ++ qgis-app/plugins/celery.py | 8 +- qgis-app/plugins/tasks/__init__.py | 3 +- .../plugins/tasks/generate_plugins_xml.py | 8 +- qgis-app/plugins/tasks/update_feedjack.py | 10 + qgis-app/plugins/tests/test_view.py | 27 ++ qgis-app/plugins/views.py | 45 +++- qgis-app/settings_docker.py | 23 +- 18 files changed, 423 insertions(+), 150 deletions(-) create mode 100644 .dockerignore create mode 100644 dockerize/.env.template create mode 100644 dockerize/docker-compose.override.template.yml create mode 100644 dockerize/docker-compose.override.test.yml create mode 100644 qgis-app/plugins/tasks/update_feedjack.py create mode 100644 qgis-app/plugins/tests/test_view.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..7ab91dc3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +dockerize/postgres_data diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index afc17f03..8166e470 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -48,23 +48,29 @@ jobs: - uses: actions/checkout@v2 - name: Run docker-compose build - run: docker-compose build + run: docker-compose build devweb - - name: Run the containers - run: docker-compose up -d db devweb + - name: Run docker-compose services + working-directory: dockerize + run: | + cp docker-compose.override.test.yml docker-compose.override.yml + make devweb-test + make wait-db + make create-test-db - name: Run Coverage test run: | - cat << EOF | docker-compose exec -T devweb bash + cat << EOF | docker-compose -p qgis-plugins exec -T devweb bash pip install coverage python manage.py makemigrations python manage.py migrate + python manage.py collectstatic --noinput --verbosity 0 coverage run manage.py test coverage xml EOF - name: Upload coverage to codecov - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 4267cc6f..073c345f 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,5 @@ qgis-app/api/tests/*/ # whoosh_index qgis-app/whoosh_index/ +docker-compose.override.yml +.env diff --git a/dockerize/.env.template b/dockerize/.env.template new file mode 100644 index 00000000..cd4fbcfe --- /dev/null +++ b/dockerize/.env.template @@ -0,0 +1,6 @@ +RABBITMQ_HOST=rabbitmq +DATABASE_NAME=gis +DATABASE_USERNAME=docker +DATABASE_PASSWORD=docker +DATABASE_HOST=db +DJANGO_SETTINGS_MODULE=settings_docker diff --git a/dockerize/Makefile b/dockerize/Makefile index aa3360a8..bdbb9ff0 100644 --- a/dockerize/Makefile +++ b/dockerize/Makefile @@ -15,6 +15,13 @@ build: @echo "------------------------------------------------------------------" @docker-compose -p $(PROJECT_ID) build +build-dev: + @echo + @echo "------------------------------------------------------------------" + @echo "Building in development mode only" + @echo "------------------------------------------------------------------" + @docker-compose -p $(PROJECT_ID) build devweb + db: @echo @echo "------------------------------------------------------------------" @@ -22,19 +29,33 @@ db: @echo "------------------------------------------------------------------" @docker-compose -p $(PROJECT_ID) up -d db +metabase: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running metabase in production mode" + @echo "------------------------------------------------------------------" + @docker-compose -p $(PROJECT_ID) up -d metabase + web: db @echo @echo "------------------------------------------------------------------" @echo "Running in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up -d web + @docker-compose -p $(PROJECT_ID) up -d uwsgi web smtp worker beat + +devweb-test: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running in TESTING mode" + @echo "------------------------------------------------------------------" + @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb devweb: db @echo @echo "------------------------------------------------------------------" @echo "Running in DEVELOPMENT mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb + @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb rabbitmq worker beat devweb-runserver: devweb @echo @@ -100,6 +121,20 @@ rm-only: kill @echo "------------------------------------------------------------------" @docker-compose -p $(PROJECT_ID) rm +maillogs: + @echo + @echo "------------------------------------------------------------------" + @echo "Showing smtp logs in production mode" + @echo "------------------------------------------------------------------" + @docker-compose exec smtp tail -f /var/log/mail.log + +mailerrorlogs: + @echo + @echo "------------------------------------------------------------------" + @echo "Showing smtp error logs in production mode" + @echo "------------------------------------------------------------------" + @docker-compose exec smtp tail -f /var/log/mail.err + dbrestore: @echo @echo "------------------------------------------------------------------" @@ -119,3 +154,10 @@ dbrestore: @docker exec -t $(PROJECT_ID)-db pg_restore /backups/latest.dmp | docker exec -i $(PROJECT_ID)-db su - postgres -c "psql gis" @docker-compose -p $(PROJECT_ID) start web @echo "starting web container" + +wait-db: + @docker-compose exec -p $(PROJECT_ID) db su - postgres -c "until pg_isready; do sleep 5; done" + +create-test-db: + @docker-compose exec -p $(PROJECT_ID) db su - postgres -c "psql -c 'create database test_db;'" + @docker-compose exec -p $(PROJECT_ID) db su - postgres -c "psql -d test_db -c 'create extension postgis;'" diff --git a/dockerize/docker-compose.override.template.yml b/dockerize/docker-compose.override.template.yml new file mode 100644 index 00000000..bb0fdd8a --- /dev/null +++ b/dockerize/docker-compose.override.template.yml @@ -0,0 +1,55 @@ +version: '3' +services: + devweb: + # Note you cannot scale if you use container_name + image: kartoza/qgis-plugins-uwsgi:dev-latest + container_name: qgis-plugins-devweb + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev + ports: + # for django test server + - "62202:8080" + # for ssh + - "62203:22" + + beat: + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + + worker: + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + + uwsgi: + container_name: qgis-plugins-uwsgi + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: prod + + db: + volumes: + - ./postgres_data:/var/lib/postgresql + - ./backups:/backups + + web: + volumes: + - ./sites-enabled:/etc/nginx/conf.d:ro + - ./static:/home/web/static:ro + - ./media:/home/web/media:ro + ports: + - "62201:8080" diff --git a/dockerize/docker-compose.override.test.yml b/dockerize/docker-compose.override.test.yml new file mode 100644 index 00000000..e10fe9a1 --- /dev/null +++ b/dockerize/docker-compose.override.test.yml @@ -0,0 +1,19 @@ +version: '3' +services: + devweb: + # Note you cannot scale if you use container_name + image: kartoza/qgis-plugins-uwsgi:dev-latest + container_name: qgis-plugins-devweb + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev + ports: + # for django test server + - "62202:8080" + # for ssh + - "62203:22" diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 2ee672fb..24854bc0 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -1,117 +1,145 @@ -db: - container_name: qgis-plugins-db - image: kartoza/postgis:9.6-2.4 - environment: - - ALLOW_IP_RANGE=0.0.0.0/0 - - POSTGRES_USER=docker - - POSTGRES_PASS=docker - volumes: - - ./backups:/backups - restart: unless-stopped +version: '3' +volumes: + backups-data: + static-data: + media-data: + logs-data: + nginx-conf: + db-backups: + rabbitmq: + celerybeat-schedule: + postgres-data: +services: -web: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-web - build: docker - hostname: uwsgi - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - DEBUG=False - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - ./static:/home/web/static:rw - - ./static:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq - - worker:worker - restart: unless-stopped - user: root - command: uwsgi --ini /uwsgi.conf + smtp: + image: catatnight/postfix:latest + hostname: postfix + environment: + - maildomain=qgis.org + - smtp_user=noreply:docker + restart: unless-stopped -devweb: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-devweb - build: docker - hostname: uwsgi - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - ./static:/home/web/static:rw - - ./static:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq - - worker:worker - restart: unless-stopped - user: root - ports: - # for django test server - - "62202:8080" - # for ssh - - "62203:22" + db: + container_name: qgis-plugins-db + image: kartoza/postgis:14-3.3 + environment: + - ALLOW_IP_RANGE=0.0.0.0/0 + - POSTGRES_USER=${DATABASE_USERNAME:-docker} + - POSTGRES_PASS=${DATABASE_PASSWORD:-docker} + - PASSWORD_AUTHENTICATION=${PASSWORD_AUTHENTICATION:-md5} + volumes: + - postgres-data:/var/lib/postgresql + - backups-data:/backups + restart: unless-stopped -rabbitmq: - image: library/rabbitmq:3.6 - hostname: rabbitmq - environment: - - RABBIT_PASSWORD=rabbit_test_password - - USER=rabbit_user - - RABBITMQ_NODENAME=rabbit - restart: unless-stopped + uwsgi: &uwsgi-common + container_name: qgis-plugins-uwsgi + image: kartoza/qgis-plugins-uwsgi + hostname: uwsgi + environment: + - DATABASE_NAME=${DATABASE_NAME:-gis} + - DATABASE_USERNAME=${DATABASE_USERNAME:-docker} + - DATABASE_PASSWORD=${DATABASE_PASSWORD:-docker} + - DATABASE_HOST=${DATABASE_HOST:-db} + - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-settings_docker} + - VIRTUAL_HOST=${VIRTUAL_HOST:-plugins.kartoza.com} + - VIRTUAL_PORT=${VIRTUAL_PORT:-8080} + - DEBUG=${DEBUG:-False} + - RABBITMQ_HOST=${RABBITMQ_HOST:-rabbitmq} + - BROKER_URL=amqp://rabbitmq:5672 + volumes: + - static-data:/home/web/static:rw + - media-data:/home/web/media:rw + - celerybeat-schedule:/home/web/celerybeat-schedule:rw + links: + - db:db + - rabbitmq:rabbitmq + restart: unless-stopped + user: root -worker: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-worker - build: docker - hostname: uwsgi - working_dir: /home/web/django_project - command: celery -A plugins worker -l info - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - ./static:/home/web/static:rw - - ./static:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq + # This is the entry point for a development server. + # Run with --no-deps to run attached to the services + # from prod environment if wanted + devweb: + <<: *uwsgi-common + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev -nginx: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-nginx - image: nginx - hostname: nginx - volumes: - - ./sites-enabled:/etc/nginx/conf.d:ro - # I dont use volumes_from as I want to use the ro modifier - - ./static:/home/web/static:ro - - ./static:/home/web/media:ro - - ./logs:/var/log/nginx - links: - - web:uwsgi - ports: - - "62201:8080" - restart: unless-stopped + rabbitmq: + image: rabbitmq:3.7-alpine + hostname: rabbitmq + volumes: + - rabbitmq:/var/lib/rabbitmq + restart: unless-stopped + + beat: + <<: *uwsgi-common + container_name: qgis-plugins-beat + working_dir: /home/web/django_project + entrypoint: [ ] + command: celery --app=plugins.celery:app beat -s /home/web/celerybeat-schedule/schedule -l INFO + + worker: + <<: *uwsgi-common + container_name: qgis-plugins-worker + links: + - db + - rabbitmq + - beat + working_dir: /home/web/django_project + entrypoint: [] + command: celery -A plugins worker -l INFO + + beat: + <<: *uwsgi-common + container_name: qgis-plugins-beat + working_dir: /home/web/django_project + entrypoint: [ ] + command: celery --app=plugins.celery:app beat -s /home/web/celerybeat-schedule/schedule -l INFO + + web: + # Note you cannot scale if you use container_name + container_name: qgis-plugins-web + image: nginx + hostname: web + volumes: + - nginx-conf:/etc/nginx/conf.d:ro + - static-data:/home/web/static:ro + - media-data:/home/web/media:ro + links: + - uwsgi:uwsgi + - metabase:metabase + logging: + driver: "json-file" + options: + max-size: "200k" + max-file: "10" + restart: unless-stopped + + dbbackups: + image: kartoza/pg-backup:14-3.3 + hostname: pg-backups + volumes: + - db-backups:/backups + links: + - db:db + environment: + # take care to let the project name below match that + # declared in the top of the makefile + - DUMPPREFIX=${DUMPPREFIX:-QGIS_PLUGINS} + - POSTGRES_USER=${DATABASE_USERNAME:-docker} + - POSTGRES_PASS=${DATABASE_PASSWORD:-docker} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_HOST=${DATABASE_HOST:-db} + - PGDATABASE=${DATABASE_NAME:-gis} + restart: unless-stopped + + metabase: + image: metabase/metabase:latest + environment: + - MB_DB_TYPE=postgres + - MB_DB_CONNECTION_URI=jdbc:postgresql://${DATABASE_HOST:-db}:5432/metabase?user=${DATABASE_USERNAME:-docker}&password=${DATABASE_PASSWORD:-docker} + links: + - db diff --git a/dockerize/docker/Dockerfile b/dockerize/docker/Dockerfile index a77ab97b..8cc74ab1 100644 --- a/dockerize/docker/Dockerfile +++ b/dockerize/docker/Dockerfile @@ -1,18 +1,34 @@ #--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ # Note this base image is based on debian -FROM kartoza/django-base:3.7 +FROM kartoza/django-base:3.7 as prod MAINTAINER Dimas Ciputra #RUN ln -s /bin/true /sbin/initctl RUN apt-get clean all RUN apt-get update && apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev -ADD REQUIREMENTS.txt /REQUIREMENTS.txt + +RUN rm -rf /uwsgi.conf +ADD dockerize/docker/uwsgi.conf /uwsgi.conf +ADD qgis-app /home/web/django_project +ADD dockerize/docker/REQUIREMENTS.txt /REQUIREMENTS.txt RUN pip install --upgrade pip && pip install -r /REQUIREMENTS.txt RUN pip install uwsgi +# Open port 8080 as we will be running our uwsgi socket on that +EXPOSE 8080 + +RUN mkdir -p /var/log/uwsgi + +WORKDIR /home/web/django_project +CMD ["uwsgi", "--ini", "/uwsgi.conf"] + + +FROM prod as dev + +# This section taken on 2 July 2015 from # https://docs.docker.com/examples/running_ssh_service/ # Sudo is needed by pycharm when it tries to pip install packages -RUN apt-get install -y openssh-server sudo +RUN apt-get update && apt-get install -y openssh-server sudo RUN mkdir /var/run/sshd RUN echo 'root:docker' | chpasswd RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config @@ -23,15 +39,13 @@ RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so ENV NOTVISIBLE "in users profile" RUN echo "export VISIBLE=now" >> /etc/profile -RUN rm -rf /uwsgi.conf -ADD uwsgi.conf /uwsgi.conf +# -------------------------------------------------------- +# Open ports as needed +# -------------------------------------------------------- -# Open port 8080 as we will be running our uwsgi socket on that +# Open port 8080 as we will be running our django dev server on EXPOSE 8080 - # Open port 22 as we will be using a remote interpreter from pycharm EXPOSE 22 -RUN mkdir -p /var/log/uwsgi -WORKDIR /home/web/django_project CMD ["/usr/sbin/sshd", "-D"] diff --git a/dockerize/docker/REQUIREMENTS.txt b/dockerize/docker/REQUIREMENTS.txt index e05c7661..70bb9dd9 100644 --- a/dockerize/docker/REQUIREMENTS.txt +++ b/dockerize/docker/REQUIREMENTS.txt @@ -35,7 +35,8 @@ django-haystack git+https://github.com/dimasciput/feedjack.git feedparser==5.2.1 -celery==4.3.1 +celery==5.2.7 +django-celery-beat==2.3.0 requests==2.23.0 markdown==3.2.1 diff --git a/dockerize/sites-enabled/default.conf b/dockerize/sites-enabled/default.conf index e870a2a6..8cf7cec7 100644 --- a/dockerize/sites-enabled/default.conf +++ b/dockerize/sites-enabled/default.conf @@ -28,6 +28,8 @@ server { } # max upload size, adjust to taste client_max_body_size 15M; + + # Django media location /media { # your Django project's media files - amend as required @@ -52,6 +54,15 @@ server { alias /home/web/archive; expires 21d; # cache for 6h } + + location /plugins/plugins.xml { + if ($request_uri !~ "&package_name(.*)") { + rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; + root /home; + expires 600s; + } + } + # Finally, send all non-media requests to the Django server. location / { uwsgi_pass uwsgi; @@ -74,4 +85,10 @@ server { uwsgi_param SERVER_PORT $server_port; uwsgi_param SERVER_NAME $server_name; } + + + location /metabase/ { + # set to webroot path + proxy_pass http://metabase:3000/; + } } diff --git a/qgis-app/plugins/celery.py b/qgis-app/plugins/celery.py index 030207ab..7a5c4468 100644 --- a/qgis-app/plugins/celery.py +++ b/qgis-app/plugins/celery.py @@ -1,6 +1,10 @@ from __future__ import absolute_import import os from celery import Celery +import logging + +logger = logging.getLogger('plugins') + # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings_docker') @@ -16,7 +20,3 @@ # Load task modules from all registered Django app configs. app.autodiscover_tasks() - -@app.task(bind=True) -def debug_task(self): - print('Request: {0!r}'.format(self.request)) diff --git a/qgis-app/plugins/tasks/__init__.py b/qgis-app/plugins/tasks/__init__.py index c06d2250..a0df9010 100644 --- a/qgis-app/plugins/tasks/__init__.py +++ b/qgis-app/plugins/tasks/__init__.py @@ -1 +1,2 @@ -from plugins.tasks.generate_plugins_xml import * \ No newline at end of file +from plugins.tasks.generate_plugins_xml import * # noqa +from plugins.tasks.update_feedjack import * # noqa diff --git a/qgis-app/plugins/tasks/generate_plugins_xml.py b/qgis-app/plugins/tasks/generate_plugins_xml.py index 42493392..ad0d1281 100644 --- a/qgis-app/plugins/tasks/generate_plugins_xml.py +++ b/qgis-app/plugins/tasks/generate_plugins_xml.py @@ -1,17 +1,23 @@ import os import requests from celery import shared_task +from celery.utils.log import get_task_logger from preferences import preferences from django.conf import settings +logger = get_task_logger(__name__) + + @shared_task -def generate_plugins_xml(site = ''): +def generate_plugins_xml(site=''): """ Fetch the xml list of plugins from the plugin site. :param site: site domain where the plugins will be fetched, default to http://plugins.qgis.org """ + logger.info('generate_plugins_xml : {}'.format(site)) + if not site: if settings.DEFAULT_PLUGINS_SITE: site = settings.DEFAULT_PLUGINS_SITE diff --git a/qgis-app/plugins/tasks/update_feedjack.py b/qgis-app/plugins/tasks/update_feedjack.py new file mode 100644 index 00000000..e12d14bd --- /dev/null +++ b/qgis-app/plugins/tasks/update_feedjack.py @@ -0,0 +1,10 @@ +from celery import shared_task +from celery.utils.log import get_task_logger + +logger = get_task_logger(__name__) + + +@shared_task +def update_feedjack(): + import subprocess + subprocess.call(['python', 'manage.py', 'feedjackupdate']) diff --git a/qgis-app/plugins/tests/test_view.py b/qgis-app/plugins/tests/test_view.py new file mode 100644 index 00000000..67b9fc32 --- /dev/null +++ b/qgis-app/plugins/tests/test_view.py @@ -0,0 +1,27 @@ +from django.test import TestCase + +from plugins.views import _add_patch_version + + +class TestTruncateVersion(TestCase): + """Test _add_patch_version function""" + + def test__add_patch_version_with_3_segment_version_number(self): + version = '1.2.3' + self.assertEqual(_add_patch_version(version, '99'), '1.2.3') + + def test__add_patch_version_with_2_segment_version_number(self): + version = '1.2' + self.assertEqual(_add_patch_version(version, '00'), '1.2.00') + + def test__add_patch_version_with_1_segment_version_number(self): + version = '1' + self.assertEqual(_add_patch_version(version, '99'), '1') + + def test__add_patch_version_with_None(self): + version = None + self.assertEqual(_add_patch_version(version, '99'), None) + + def test__add_patch_version_with_empty_string(self): + version = '' + self.assertEqual(_add_patch_version(version, '99'), '') diff --git a/qgis-app/plugins/views.py b/qgis-app/plugins/views.py index d625e2fd..45919175 100644 --- a/qgis-app/plugins/views.py +++ b/qgis-app/plugins/views.py @@ -909,6 +909,25 @@ def version_detail(request, package_name, version): from django.views.decorators.cache import cache_page +def _add_patch_version(version: str, additional_patch: str ) -> str: + """To add patch number in version. + + e.g qgis version = 3.16 we add patch number (99) in versioning -> 3.16.99 + We use this versioning to query against PluginVersion min_qg_version, + so that the query result will include all PluginVersion with + minimum QGIS version 3.16 regardless of the patch number. + """ + + if not version: + return version + separator = '.' + v = version.split(separator) + if len(v) == 2: + two_first_segment = separator.join(v[:2]) + version = f'{two_first_segment}.{additional_patch}' + return version + + @cache_page(60 * 15) def xml_plugins(request, qg_version=None, stable_only=None, package_name=None): """ @@ -930,10 +949,10 @@ def xml_plugins(request, qg_version=None, stable_only=None, package_name=None): object_list = [] if qg_version: - filters.update({'pluginversion__min_qg_version__lte' : qg_version}) - version_filters.update({'min_qg_version__lte' : qg_version}) - filters.update({'pluginversion__max_qg_version__gte' : qg_version}) - version_filters.update({'max_qg_version__gte' : qg_version}) + filters.update({'pluginversion__min_qg_version__lte' : _add_patch_version(qg_version, '99')}) + version_filters.update({'min_qg_version__lte' : _add_patch_version(qg_version, '99')}) + filters.update({'pluginversion__max_qg_version__gte' : _add_patch_version(qg_version, '0')}) + version_filters.update({'max_qg_version__gte' : _add_patch_version(qg_version, '0')}) # Get all versions for the given plugin) @@ -1007,10 +1026,10 @@ def xml_plugins_new(request, qg_version=None, stable_only=None, package_name=Non object_list = [] if qg_version: - filters.update({'pluginversion__min_qg_version__lte' : qg_version}) - version_filters.update({'min_qg_version__lte' : qg_version}) - filters.update({'pluginversion__max_qg_version__gte' : qg_version}) - version_filters.update({'max_qg_version__gte' : qg_version}) + filters.update({'pluginversion__min_qg_version__lte' : _add_patch_version(qg_version, '99')}) + version_filters.update({'min_qg_version__lte' : _add_patch_version(qg_version, '99')}) + filters.update({'pluginversion__max_qg_version__gte' : _add_patch_version(qg_version, '0')}) + version_filters.update({'max_qg_version__gte' : _add_patch_version(qg_version, '0')}) # Get all versions for the given plugin if package_name: @@ -1051,8 +1070,8 @@ def xml_plugins_new(request, qg_version=None, stable_only=None, package_name=Non FROM %(pv_table)s pv WHERE ( pv.approved = True - AND pv."max_qg_version" >= '%(qg_version)s' - AND pv."min_qg_version" <= '%(qg_version)s' + AND pv."max_qg_version" >= '%(qg_version_with_patch_0)s' + AND pv."min_qg_version" <= '%(qg_version_with_patch_99)s' AND pv.experimental = %(experimental)s ) ORDER BY pv.plugin_id, pv.version DESC @@ -1062,7 +1081,8 @@ def xml_plugins_new(request, qg_version=None, stable_only=None, package_name=Non object_list_new = PluginVersion.objects.raw(sql % { 'pv_table': PluginVersion._meta.db_table, 'p_table': Plugin._meta.db_table, - 'qg_version': qg_version, + 'qg_version_with_patch_0': _add_patch_version(qg_version, '0'), + 'qg_version_with_patch_99': _add_patch_version(qg_version, '99'), 'experimental': 'False', 'trusted_users_ids': str(trusted_users_ids), }) @@ -1074,7 +1094,8 @@ def xml_plugins_new(request, qg_version=None, stable_only=None, package_name=Non object_list_new += [o for o in PluginVersion.objects.raw(sql % { 'pv_table': PluginVersion._meta.db_table, 'p_table': Plugin._meta.db_table, - 'qg_version': qg_version, + 'qg_version_with_patch_0': _add_patch_version(qg_version, '0'), + 'qg_version_with_patch_99': _add_patch_version(qg_version, '99'), 'experimental': 'True', 'trusted_users_ids': str(trusted_users_ids), })] diff --git a/qgis-app/settings_docker.py b/qgis-app/settings_docker.py index 2200839a..896c8a93 100644 --- a/qgis-app/settings_docker.py +++ b/qgis-app/settings_docker.py @@ -1,3 +1,5 @@ +from celery.schedules import crontab + from settings import * import ast import os @@ -43,6 +45,8 @@ # full text search postgres 'django.contrib.postgres', + 'feedjack', + # ABP: 'plugins', 'django.contrib.humanize', @@ -61,8 +65,6 @@ 'tinymce', 'rpc4django', - 'feedjack', - 'preferences', 'rest_framework', @@ -80,7 +82,7 @@ 'layerdefinitions', # models (sharing .model3 file feature) 'models', - 'wavefronts' + 'wavefronts', ] DATABASES = { @@ -125,3 +127,18 @@ 'TEST_REQUEST_DEFAULT_FORMAT': 'json', } +CELERY_RESULT_BACKEND = 'rpc://' +CELERY_BROKER_URL = os.environ.get('BROKER_URL', 'amqp://rabbitmq:5672') +CELERY_BEAT_SCHEDULE = { + 'generate_plugins_xml': { + 'task': 'plugins.tasks.generate_plugins_xml.generate_plugins_xml', + 'schedule': crontab(minute='*/10'), # Execute every 10 minutes. + 'kwargs': { + 'site': 'https://plugins.qgis.org/' + } + }, + 'update_feedjack': { + 'task': 'plugins.tasks.update_feedjack.update_feedjack', + 'schedule': crontab(minute='*/30'), # Execute every 30 minutes. + } +} From 4a2431aa3be4c4078f91681a26ba960161b22b51 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 22 Nov 2023 15:30:55 +0300 Subject: [PATCH 03/32] Fix docker-compose and dockerfile --- dockerize/docker-compose.yml | 7 ------- dockerize/docker/Dockerfile | 4 ++++ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 24854bc0..900ec7fa 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -92,13 +92,6 @@ services: entrypoint: [] command: celery -A plugins worker -l INFO - beat: - <<: *uwsgi-common - container_name: qgis-plugins-beat - working_dir: /home/web/django_project - entrypoint: [ ] - command: celery --app=plugins.celery:app beat -s /home/web/celerybeat-schedule/schedule -l INFO - web: # Note you cannot scale if you use container_name container_name: qgis-plugins-web diff --git a/dockerize/docker/Dockerfile b/dockerize/docker/Dockerfile index 8cc74ab1..10eb7660 100644 --- a/dockerize/docker/Dockerfile +++ b/dockerize/docker/Dockerfile @@ -5,6 +5,10 @@ MAINTAINER Dimas Ciputra #RUN ln -s /bin/true /sbin/initctl RUN apt-get clean all + +# Debian stretch/updates release issue. please see https://serverfault.com/a/1130167 +RUN echo "deb http://archive.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list + RUN apt-get update && apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev RUN rm -rf /uwsgi.conf From a8349fd79f063dca99f3df0042d2be3e261a86f7 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 22 Nov 2023 20:06:38 +0300 Subject: [PATCH 04/32] Init starting docker dev environment --- dockerize/Makefile | 18 ++++++++++++------ dockerize/docker-compose.yml | 5 +++++ qgis-app/styles/models.py | 3 +++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/dockerize/Makefile b/dockerize/Makefile index 6f06d511..74ab2587 100644 --- a/dockerize/Makefile +++ b/dockerize/Makefile @@ -51,6 +51,12 @@ devweb-test: db @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb devweb: db + @echo + @echo "------------------------------------------------------------------" + @echo "Recreate qgis-plugins-uwsgi" + @echo "------------------------------------------------------------------" + @docker stop qgis-plugins-uwsgi + @docker rm qgis-plugins-uwsgi @echo @echo "------------------------------------------------------------------" @echo "Running in DEVELOPMENT mode" @@ -75,26 +81,26 @@ migrate: @# We add the '-' prefix to the next line as the migration may fail @# but we want to continue anyway. @#We need to migrate accounts first as it has a reference to user model - -@docker-compose -p $(PROJECT_ID) exec web python manage.py migrate auth - @docker-compose -p $(PROJECT_ID) exec web python manage.py migrate + -@docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py migrate auth + @docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py migrate update-migrations: @echo @echo "------------------------------------------------------------------" @echo "Running update migrations in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec web python manage.py makemigrations + @docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py makemigrations collectstatic: @echo @echo "------------------------------------------------------------------" @echo "Collecting static in production mode" @echo "------------------------------------------------------------------" - #@docker-compose -p $(PROJECT_ID) run uwsgi python manage.py collectstatic --noinput + @docker-compose -p $(PROJECT_ID) run uwsgi python manage.py collectstatic --noinput #We need to run collect static in the same context as the running # uwsgi container it seems so I use docker exec here # no -it flag so we can run over remote shell - @docker exec $(PROJECT_ID)-web python manage.py collectstatic --noinput + # @docker exec $(PROJECT_ID)-web python manage.py collectstatic --noinput reload: @echo @@ -174,4 +180,4 @@ rebuild_index: @echo "------------------------------------------------------------------" @echo "Rebuild search index in PRODUCTION mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec web bash -c 'python manage.py rebuild_index' + @docker-compose -p $(PROJECT_ID) exec uwsgi bash -c 'python manage.py rebuild_index' diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 900ec7fa..9938b03e 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -66,6 +66,11 @@ services: context: ${PWD}/../ dockerfile: dockerize/docker/Dockerfile target: dev + ports: + # for django test server + - "62202:8080" + # for ssh + - "62203:22" rabbitmq: image: rabbitmq:3.7-alpine diff --git a/qgis-app/styles/models.py b/qgis-app/styles/models.py index f1e4f753..bc649dc2 100644 --- a/qgis-app/styles/models.py +++ b/qgis-app/styles/models.py @@ -65,6 +65,9 @@ def __unicode__(self): def __str__(self): return self.__unicode__() + + def get_absolute_url(self): + return '' class Style(Resource): From ee066d27d83c954fa3eb156d97c3ce9b63f0303f Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Thu, 23 Nov 2023 11:25:55 +0300 Subject: [PATCH 05/32] Fixing docker container conflict, updating whoosh --- REQUIREMENTS.txt | 3 ++- dockerize/Makefile | 6 ------ dockerize/docker-compose.yml | 5 ++++- dockerize/docker/REQUIREMENTS.txt | 5 ++++- qgis-app/REQUIREMENTS_plugins.txt | 5 ++++- .../plugins/migrations/0004_merge_20231123_0018.py | 14 ++++++++++++++ qgis-app/plugins/tests/test_plugin_list.py | 2 +- qgis-app/styles/models.py | 3 --- 8 files changed, 29 insertions(+), 14 deletions(-) create mode 100644 qgis-app/plugins/migrations/0004_merge_20231123_0018.py diff --git a/REQUIREMENTS.txt b/REQUIREMENTS.txt index f2240fb3..783d71a7 100644 --- a/REQUIREMENTS.txt +++ b/REQUIREMENTS.txt @@ -13,7 +13,8 @@ Markdown==2.3.1 #PIL==1.1.7 Pillow Pygments==2.7.4 -Whoosh==2.5.6 +git+https://github.com/Xpirix/whoosh.git +pickle5==0.0.12 argparse==1.2.1 #cab==0.2.0 # Not used anymore..- #distribute==0.7.3 diff --git a/dockerize/Makefile b/dockerize/Makefile index 74ab2587..fa4334e2 100644 --- a/dockerize/Makefile +++ b/dockerize/Makefile @@ -51,12 +51,6 @@ devweb-test: db @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb devweb: db - @echo - @echo "------------------------------------------------------------------" - @echo "Recreate qgis-plugins-uwsgi" - @echo "------------------------------------------------------------------" - @docker stop qgis-plugins-uwsgi - @docker rm qgis-plugins-uwsgi @echo @echo "------------------------------------------------------------------" @echo "Running in DEVELOPMENT mode" diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 9938b03e..72c1dc14 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -33,7 +33,7 @@ services: restart: unless-stopped uwsgi: &uwsgi-common - container_name: qgis-plugins-uwsgi + container_name: qgis-plugins-uwsgi-common image: kartoza/qgis-plugins-uwsgi hostname: uwsgi environment: @@ -48,6 +48,7 @@ services: - RABBITMQ_HOST=${RABBITMQ_HOST:-rabbitmq} - BROKER_URL=amqp://rabbitmq:5672 volumes: + - ../qgis-app:/home/web/django_project - static-data:/home/web/static:rw - media-data:/home/web/media:rw - celerybeat-schedule:/home/web/celerybeat-schedule:rw @@ -66,6 +67,8 @@ services: context: ${PWD}/../ dockerfile: dockerize/docker/Dockerfile target: dev + volumes: + - ../qgis-app:/home/web/django_project ports: # for django test server - "62202:8080" diff --git a/dockerize/docker/REQUIREMENTS.txt b/dockerize/docker/REQUIREMENTS.txt index 5f966e50..85e59976 100644 --- a/dockerize/docker/REQUIREMENTS.txt +++ b/dockerize/docker/REQUIREMENTS.txt @@ -23,7 +23,10 @@ django-sortable-listview sorl-thumbnail django-extensions django-debug-toolbar==3.2.4 -whoosh + +# Updates for Django 3 & Python 3.7 +git+https://github.com/Xpirix/whoosh.git +pickle5==0.0.12 django-haystack # Feedjack==0.9.18 diff --git a/qgis-app/REQUIREMENTS_plugins.txt b/qgis-app/REQUIREMENTS_plugins.txt index e3894f80..ab719d94 100644 --- a/qgis-app/REQUIREMENTS_plugins.txt +++ b/qgis-app/REQUIREMENTS_plugins.txt @@ -23,5 +23,8 @@ django-sortable-listview sorl-thumbnail django-extensions django-debug-toolbar==3.2.4 -whoosh + +# Updates for Django 3 & Python 3.7 +git+https://github.com/Xpirix/whoosh.git +pickle5==0.0.12 django-haystack diff --git a/qgis-app/plugins/migrations/0004_merge_20231123_0018.py b/qgis-app/plugins/migrations/0004_merge_20231123_0018.py new file mode 100644 index 00000000..8e18926d --- /dev/null +++ b/qgis-app/plugins/migrations/0004_merge_20231123_0018.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.11 on 2023-11-23 00:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('plugins', '0002_plugins_feedback'), + ('plugins', '0003_plugin_allow_update_name'), + ] + + operations = [ + ] diff --git a/qgis-app/plugins/tests/test_plugin_list.py b/qgis-app/plugins/tests/test_plugin_list.py index b17c0e17..48972e11 100644 --- a/qgis-app/plugins/tests/test_plugin_list.py +++ b/qgis-app/plugins/tests/test_plugin_list.py @@ -1,6 +1,6 @@ from django.test import TestCase from django.urls import reverse -from ..models import Plugin +from plugins.models import Plugin class PluginsListViewTestCase(TestCase): fixtures = [ diff --git a/qgis-app/styles/models.py b/qgis-app/styles/models.py index bc649dc2..f1e4f753 100644 --- a/qgis-app/styles/models.py +++ b/qgis-app/styles/models.py @@ -65,9 +65,6 @@ def __unicode__(self): def __str__(self): return self.__unicode__() - - def get_absolute_url(self): - return '' class Style(Resource): From 303bad8e2e6e8dfbb4d678948997bfa3fdb77654 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Thu, 23 Nov 2023 12:10:13 +0300 Subject: [PATCH 06/32] Update testfiles and feedback test --- .../tests/test_plugin_version_feedback.py | 4 ++-- .../tests/testfiles/valid_plugin_0.0.2.zip_ | Bin 38483 -> 45559 bytes .../tests/testfiles/valid_plugin_0.0.3.zip_ | Bin 38483 -> 45559 bytes 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qgis-app/plugins/tests/test_plugin_version_feedback.py b/qgis-app/plugins/tests/test_plugin_version_feedback.py index c2034c98..36f6fc0f 100644 --- a/qgis-app/plugins/tests/test_plugin_version_feedback.py +++ b/qgis-app/plugins/tests/test_plugin_version_feedback.py @@ -244,11 +244,11 @@ def test_only_staff_can_see_new_feedback_form(self): self.client.force_login(user=self.creator) response = self.client.get(self.url) self.assertEqual(response.status_code, 200) - self.assertNotContains(response, '
') + self.assertNotContains(response, '
') self.client.force_login(user=self.staff) response = self.client.get(self.url) self.assertEqual(response.status_code, 200) - self.assertContains(response, '
') + self.assertContains(response, '
') def test_post_create_single_task_feedback(self): self.client.force_login(self.staff) diff --git a/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.2.zip_ b/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.2.zip_ index c694809e1cacf44578d482b84899caad2bcb6ab1..f5ba6212072650e8387b65d84f6a47d7c94168bf 100644 GIT binary patch delta 7884 zcmZvhWmHvPw8bx%?(PQZl)Q9zmvpyu3Wy%M^Ik$4>GV?4Qi6b#(j}dO(k<}(?MMInp1yR@;1wdx3aI zp%3b=snS>bKKJ9L_M__s6UB~afxHw6cep$IH@DUSw#Tbh0_478N;KtCmTvo#48jFL zI}?8HV}k|2!~CzclN_54j;Wb;zFlF`>uW8kt7|nMLlYO~frcP8RhQp!qL$ByGxoBq zCA$x^9xV)KE;DgyGu#7>)2eYF!-0VZX*ko(73bJkNgvo?V!3qWrQ7d@BG%VC?ViM3 zC#4>5Z_iIQwT`472DNC%&PtLqQQ^(Y#D!db4lW}aCIw z!yO&z9OW#_X4fxloT~&a#wOdw@&w~;EIeFsFpk5r;IQEI9S+!wrneUQ%)oHN#uztu z7Pi#&nf{_3|45Cvu(@e>B>zZ}043EhWP;_^!`i(4Yvd*SO%m=ZG;e{}Z{ZhbZ<<2= zd-!s;Wl!5XNNz{Fz)ny!F z9k%Sme^&fyufu&1cK;xe4=mtSe)!_k?qEt%&A0Hb#ZN|UZl;};8uBqTdu`ag z+@HZ*+fR~B=*6m7=|Jt-7-j{b5oF0vEPoq!c0FaMe_?XjK*-q40w6onop#9I3SQ3B zdHL>9wK1G3UUueiX>n%<`|!u&mRNm6;6d#EcSCu7=gel#efhPpuk}hMOTS#zzJ=&X z&X!mBpqh}y3sowMwdeVrvO|-mpYie5S0iGKwXy58*`(7&g@#+ra0m(SLrH`^DoriV z`y3$S%VqE;Rr7J@UjXu!9^cXn5|mim8{zle*CIigG`gj3`%Ej*6cpMbq@sc8%UhQk z%9n@T1a7AXZzkef>W+&F6{xwe9mI$yn+h{I#NhoY^HQQDn(mBA85#3}EN{FU5FmVN zq(5+oGR4X3r*~ZM+LJjI`e8vGN2dldeQd)?YLX@5$S=!IbpaRJcdaAPIKzUa231cd z@%n+xea^8-c7q9dHB8fvykcnL%>rY#K3a}VDz{2zI_<0LuMgR-&Ezywj!E2mKW`SF z<)2y2FG7j^#jfjBZ{`mbM<;EVR;r>LjY_NHzQKrGw%735vZIJc&4oDRf?l&Z4DuZ9 z3#}~F_g;j}DgfD09>WL7T>Y|dSd|LEi85YZ zMES0t*psz~v-dXRQWl=Puz;l#-M*CV#`_?+h%~+bxIp0pgwWz!Z56OYd%F;J9`Nl#_1{8ih) z1VK}b5g?|smXGhHRBjE5W@+{tI2m*qrQ7IO$gidQ35~ELLMr z^aR%%OC8QA9O#mL_o+z-+5(HH;t2^Kv0U}R&j*gT zaRBX~(*Utf%9Znd#D$A;HAVA5?Tjk+cu+I}==??6>P#TshX{etc>)`&nKy>$R(A1-Qhf z`m^{WviN5&B*DGaYjLt|YK4bs-Q%bIdIl z8pl**XO+eNW5W|%v*Vy$D~_y}ZSPtp9_;uPqQ%Uab<&JoXMTO_Mf?z#LaDHC`KIs4 ztA+i0xne94^=Y#n4SeOEjSl&RCwMEkVNgjEY!{<#=kq(S;T9!XLw3JFkI>D;G|Q!pDS z2ik>`ccvS}-2cgf87Vt1oh7nc8%J2$9a?a*mA(vpew`@aq7p-F(Z&{0|E6ra$T-S( zpUCX^y64R5?K*!59T*qYdyi~WcWomw4Szkvye5LUu*==`$xT*0{w8U4oTXr=Rp@gSNDIOsGW<`P-a=#x|=V z0+WlJX9!8N{JFBEJAxlQUXSZ$I3KUwtj)FAvs_{hti3RT=K^CHT|N8A2tVzxP={);aD8O4j>n+nFM8&skhI+zq)u05C%w=^7&LD@ z&%j3YYfW5qIRu!3r%fB!5!b84g`iW^4_sE5eMvvtsX&hG@&q?mOVAF?r1k%D(@g)} zj=|GpgTV4Q%)EhR;po^eQVpS2an_xS6OK`SjSzkBmY^kAG|V75Fx}E=Ia%be7Dkkp zV-o)?5lkc=thjT+VX$FF!9IS--S}3xpVQ@9p+?k{D}XM=~*ue_&U&v;TRRBZpz)PBm3? zuKB!3E02u>vMtPFvj+SjTRWn)eMHKQP;I)9-k(B7uCuwsWnZ{H73+tn95zgp7-IohgEW&j} zR<&0|zX{!YpHIJ>%*?+<@EFp0vw_;GgQ#<-%=t2XgKC60r$pTv=Od4V;DvIWGI;qz z&!7d;3XWDL5~6MH3O`&W<7?QO8A`XgSDw~HbEC>Ib}=)(HS2(d&72+?+=!!mJe#P? z=NG`k7c%LdGr`F!x;VM%vu#n+DUmeAfM9G>aH2!DnyI5kfkhxF|0t=Gn-NB{vq~`V zC#|Sduls*%g3^&%1vav zDs$3r_zc262@dTi%eZeAJWHh%&p|>iIR|Ju*-CBfm@h`7ld{n0rE_0&yDtP#R5qcD zELRZKyUKc&Mz@Mo%&hy{pnmW!xJ-g2zVocSKz5{C(rfK`XsGnSq}Gm_s8`bne6d&} zri|6NsG?>$I)}?VvqSOE+NmUZ#ilYv+Ao*6|04SbCeCZg9{e|zBx_`JI?jOP z2$alHAi7WH=sgJD9|R5yQ8OJWYN5Hm#7QEe#BHHsD{(4K?ucK8W6czYgl%4pv)yOv zmT+TBjzPTk{gur~6OCP=0I%k}bO-rZ{s?{S-jTL*4tj*w)*8=winQr52lTQ%uR7$c z)ms6lZ4B{ieb1t(W5{Ry=uaV0$iM;1%Q~|$_r{deo>P#4JS84Y{9&kxqDhEtEo!XQ zQtra+&iAcz;ZT3~>Q7kXrTS;*i9R|E z)L7OmQ~}bE?2Ej`Dvl~5W&742HnS%5HkZWUzb=wGP?DIoFO8uvAvT{a_5EtH z`8>N<@Z#V;5(Y*(eFUcCiAm9sN955z_T5?6|b3R zzN@&Eetzq6z#Wue-Kom=*1^9)HH9Sapa&;UeGjZ|w-%PWbOcb{9AnasI9uVr&~tu^ zcRA0Wm4ANQ-zt)(?VRq9^Zx97gpa%kX8PytvJ@*(uNZ|uSXrB|o@1RUhXlLxJG!nK zq6i^`_vGbU4KDoNPW1ZR=SoLxq7Xc$Vn28Zi@MG(0nTnElx~anm)>+%h-6lTpnbiZ zcm3-LIo#BGC2gS3LylX=%DdM`PQmH7vz}vU$n$a)g5S6i1u2nyr#(7bX&Z?U+V2Q2 zbTPhd2Z|#Wqq(VhaU_NPnICPC=`qM8nv@NVwZ9K13}_|&n2l3qcD=jk5b=qrpv<|) zEMc0DZhCFhCB<+2fHM?Rb0qMS*Wtv#uQ6zi*KZ*Usq7kHK-BE?oSN!Ue;R3la$v^! z+p?3e)QZt<>*p8GJxntX=acl$=!#6_Adk=7>1fiu##&JYi3?lAxJJB~z9mz>B-dug zbN)C*Lv#!>7~B3{eR+8x??W$YBVX*ncRNfgdRsLu$U)8SHhG>=$7HY#vh`GC8i`#Em6y zari~Q#fn#G;ET2ugh7c~YWLMioow@XT!*m~vgelyu!rUTHrA70IELQ3D@h!Y{(- zP5;7n+}9^N$9M9;M>>!bVXf_4BhBRZ4u;V7L5Z&z%x%E0l9A${2!nN>Yi(UPIE*+f z3qG4SqcQQG?Y=seL1Ew)pg@?m3jX4rSN2u4yWbeD0v4&cTTd-hrzLh;A`?$&y1^-B z0jgQ=spMREc=j9`j*~E#mCiTT{%D{oT-S+xiM&Vo!HqRfBq?@$r0xAWH@`Sj`%lb_ zRwj+>?fbyff6&sHaD;p*WnS22H2^}-Qw#7$1HS*l$O z-g_nrvF=rZo|UNb!>c%+S|Um{q21Ox7a#{*hB>ty^gPmrs#u&vFLr2!-k;( zG`6jgD+6wUh%DNJJ;PU$fShdu2DW&FKmNwt*BvNBtCac?1oetX4o*^hcRa2tuHJk% zMrl>7bXF@xbEPyzYLJ|qxfA3$1>|Fmp1-xq|B_rVc+cYiIYI1h*>M*qdp9WN1Nf}c z0yO7)Z6_i5PPK}g-QO_+4pRY|KW?IA8e>ydf9xT7Hf;x_Q$SGP&-;Vu(7-(au^biX zAQ#sdYq{&KSPCq`?!uP&Zns!29L0F%CZ3C9Czxnx>)hKL;woank zDgSnJ-;;OWUwU|M-R;iI@gAKOkm6sBnz|d@buKIzpQHWst{Y3KMS`E~R5iR?A`z9a z4kVB`c8nQAv$nLpx3hkeF_;Ul@89#nuvqsj^t~NAJej!NmcbF)^Y6m?VzuG4*i9pe zRnLa4xQboQ=-RhA^@jdo=jMH~k9cwc4GUrw0X}6tMyn)^=7Y^GB-i^2fKS}l5f~*X zbe$^+bAFKUnv5MA+2oHTr@CLyps`{;A?&XSZKGbsb}#!3PvN5(P=xLBAll#BeN_(% z;4SXOX8oOPIcmrIAlga)!dGmwR}Vm=|3-8jpgtm=)Jb^jt0EiGh1CV+6Hlhu8`|u( zEbyveWS6Nvmy$b*DNX$YbmV!+;YmXm=fY^Z!+9vU0zzdavU%=Fvm^l>33}bn!9k`H zh17{fmvGux{JBD>p@>kT4Gfi^&5xyzfxNx-;X=ndq5`Y@u=3uql!wrx_Yo;K;dLm? z7juqKX31{`>a;f7Q>m&|dqWH=2Z|QPb$4Cq`NBCz&I}wwpEo0bbDSmhyQtVNt8<7! z!Qy10j;^NnIXSz}4-@hWg{3F9$TNhg>h0R8MfoYOJcAEv!?!z8pWu$$Y#SFbwf=W)m2;FJ2zFs7RQd$Un^OW&V{!4lmEkRfE<8yJQrE9pni2|qZ~SZ;BX zy(y-q`)M78+2SpyT`P z`%iW5fTD1z2>|WDm=iNSC2(Cfi=Gmh_<3#hvp#-qOux-V=Bq>Yu|V)k?#LmD;O-H* zDQ}ql_|oPN(&*#cQzFuh-*5+4mux<$kQZ?e*(eg#ul;EZ0$SR+;qB}Bx5V6OPNvZ6 z0dicy+XX$@jixVeBT$k7rI36hjvENq38MQ5*XyoHb$~nzbGJO~O-=b2uab+!wo=$A zByuOz+Brvv>QfOCS^J$Be#3(WlJVi{qXL@udQ?R6{nW)khz}hbcVzJ zSBAm1EzxJXNRW>LXOYvhYsxM~&AWL8-JCEyxe?x&+|t!rlHovoLfnQ`FK?9+!CcJh z2J6WK9{`8$fn6kI?ndfid5Pb%!9y-;Oxn^X?|yD~Nc5cB?J?-|8gzbgOi;YXF^gw_ zqxY-u1lW!5^f=;Xlv{X3s1ac8;laNcGvBc*XovoZiv&Yk@=ZsaaBg?JO53@uz`>7< zQ`z4Yf^rA>ip-3i@Y_Qkh8Fl})9E;D?vN&P8R-1(uhvL^RYN_a{>qH%{R2%)QR@!>eP1yFl zUtC)GRnrYUrlH>9@kZO@0LVJUWOa4^v~P&(6>DPgv=q!;m0Rv+db><4Tq70UX68F< z26UeoF-G!??7u>Xv^5988!yo{d|dhEiSvJNWGdU=-=}8U7;^fzU38O1+N`>gUP4n# zqfna3`WSx}wsalZj;=D+0!p7C`K}RfMB6qF6UN8BnI(0g4Tv0GIx-9`vz#3@>t9`K zk-NN)Khb`-k5d-SyTnf$I`LIt)AFK^3{dFW|9&LmRoVUZQZR2xn_F^ST&`5{wZ2AV zL>gO$;GAMB-Nq4e^y)ZXdyxHM5y4NKOk}^IVe~Q8PJ*xE-j4|m%Qp_}3MjEo%O^x8 zYXO0$oGW})iygd7auUK8D~3h=dYw0IFM#zo2za}26zYe!cvc6+-E><{_2-WuZJ0M{ z>Qg+2LKXsk60sDuV8m-IOPC1t=coEe?etV%(zHAQ&uHELMSE!sYyZesO!H4Cy3D(X z|DlcI?3oBp&Mb%AKbDH;{lE0DhyM!oiD?&uV>}60#SD}u7+E76`N@gbef{q)=mz-z z=1{*__~+ah4I})=zy?fmpIAH7*Vs>D!|C6>8=)@t|Bw`4qDCCIk$*Fnz3ZRc#uQ&` z%%_HPfDIY(Z-NPfs1BwIw#0RsD)(4~V!{}m&U}Mob;8?-G(g7?2`CI5=l`#t7TL|np#s%yp z0yB@b1T%;}p(wOOu$@?GaH7Q1Ry@ua{7U+%r8Q0gtSQ>>^bku^DD z*+Hej?a{C@s6H4e2KEPPhM1TLg2BDXU_LNe#FZz_2wQ?lBSMn@`9f(Lh-v?;^V3ui zOSAv^NdN{6D_t7=kPXvJHv_j8z-rP>z-Wbk3mJ?qg9(Ql#0`qT2Z78B|9i&&0Tj>s Am;e9( delta 813 zcmXYwTS${(7=Zu1|E6i%Xk{L2;b7VDkSUQG#y@jrQ*q6S%_J3^EZofL6v{%u1KwoX z3q~2S+&s;l{zfHbkysZ|;YDO8sewc{hSH5UUG(q!`fuLnecl7#_rbrj0UvxIGgPqR zcqaPgcG|~f>5_L6qvbt1rV=o6Lpoklz%EIQ{Aef3K9?B7tawZu72%0pW(t3#)p6{q zU`i8@iEue@l$4G6{Xhmj9`_T#p@NmDz=}dEiH{4Pkj{=GJ6#v8Gf2-J{yuD>LuDgB=)NPPJT>b#n|G6DZc-y*h37+?>nL_i9m4*A zIQ+u(R)p=I7|iAOU{OF0jwFnAo8VazDrOV#M7{z`y(;v$l~9~2sC4}4*1_)s1gktc z2plB%$fF0f5=GOEu%;xVk!dkroi>10^*=R3rkeCN%;2fyOFSm5tkPl>FlQ$lRo8{Oz zFNLf!w9nVGuge+CX-UD2d5(Qq$)Lun6fVO-u@S$zl<4v5!EVBBZyh|1hW&VP!33+d z1S=i;aD9OT-+7X1oO@B{LzGywnr diff --git a/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.3.zip_ b/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.3.zip_ index f838d489169f88e460388e644d1a8f0e1c59d0bd..613f0f1f3b60473e627c61d5d494e2039b5a064d 100644 GIT binary patch delta 7884 zcmZvhWmHsew8n>_ySqU;C5P_r?r!N65IuC~3?YqlI)F$^2?A0|mvjnBx4`v}_uIW^ zt+V#?{`Rx?dDl80pQ&2#)g~BQTMZEj1N7emHG7$cO^+skZRPOa3#%oiPSb-Ra${RH zB>NyD@?cwiKtxJK`VV}_L^Js(PO-Gm5uN@Ld6MS{0MXwyXrNr@Hr=@rf^85b2sEVx z0ug|yKz>MC*yntBQb`sg6UFC(?qf`32eABYJ;K)OHxfsp>&k#BD1 zyEK&X#)W_I2HO~!5v4Z1~PqY!^vurrQ*mh%1?CxSK7C&BhYxG!lg!4 zFK6+FfvkPbu}OA=33)Y4vyS`{XwuCBV~##ru5B8(N>&E#%j>TXId0A5G*eE=+m?Mu_fFV(aV#Rln3>WtGjJ3VX!@fykjhsNu17my}^*8~@b>2~6B|HA5HN(czz z`TEa$l@yDpEA!Mi$YN&?m}#Jr`e>NNErvnazEwO{IwUmRLO^oWc=3y)CU`(7tOg}X zL}&h2C8l?SP?#+-%}~%;a9{h@I>_Bbn$M-cUc#-oq8Ja!k?z40GM_n6T!2YWWuyF6 z+rSh-Q;ZQHrn8Yx=%rL{4US=H_8&MIbRDObNc$Dssdd;KTl)fgFYGw^S15XH39&5J zhn|=TuGdyNoY6SYC5P@)(^AZG8<{sB((M@gf(DA_(S0@2ujpGRRVG8yy2=xBhWG{& ze7FvpY-%)=|QxFcb|RH?K(zK;O;7f;ex&(4lmV{iY9h)9L?{pojL4p zS{QSLNx#)|qVX5VCw7O8R79L)#qFnfjE!yX1wg1JlbLsZdneKsT1J;lNcf87sSkcW zaJr2LX#boBigi-1obMwpTvVtjS`2DuR==+%qfr zV9f$uE*ADg3FeJ@qn!xhQJfTW;Mr+mf2LQO4TJL!;L1{*4S> z`KzCSJ{`{9UCY3T<}D=af|+#CHyQX~! zgMdFPyT=&;YV};Z=O~s*L%Al_Apsr$*si7f#-pVGw)&CmUw0?6h>2#BO5}X=Q(C$;npsBJBBfl6;FwEU{%9TV%uQ^6g@i zXuo|T^W*EDGwV0&{GoJUTvVSuvQ6E!ji_`0GE&4~oUNT6esw|>N*H~XG^uXG%Ek$x z$7K)&e2QVlq!&!TDq$iA?91Dr@)@;0Vb11I+;wxp^D-g?RVX$caaBvTVF9UN9u6^s zpzepHCi8b9KhX(IWTZs$mGTMM71^iHoVi%z7bA5ER$85Z(wh(3(c&?o7C#hha|)W+ zu7(OsE^?kBB+v5a$&&5}e)N1buAAv{ymqrT*XF=-i8-)lXbjH-#xlEl_K^{O+GEE@ zQ(Z+U#-f#i(ps)p2q$JwRoA6sARE9zQ#wToXIsnSHu8QIeB}5uC7ZF=y_TKEH}jQ` z4aDOf2-0ZJ*TRro%DfFZ#73FEYTZzWX|Qm8WU)!WpyV%p=B<#t-5RV;S8Xr7&_ftJ zZ!*uoM)hk=Ty!}Un1ZKI8`u*!sKke&Q#1@*R+@jwINPa2j_UFPH`hqe4$P$Y|8m#N z_}z}d(`1Xl@;J=Afo19B)Gty4p;mFxor@QaRepsKbMKz0C0IPnAUQDI(rGnW?6?+A zl%H#w@GJ>TBp#x;bHZV;VSt#5EZtY;vGv?#%v+wF>pBz$n24CT7^$>Lr&|m5sO=Jf8_b5auHX}xS*4%)J8g!xI0wxOT5(fH>>Gd1g@{Ds;-N;`_Mg8 z4e|34cAc=RZPh{4xl`tRk+DHFLY!NwZiDlYM?&yIIbIpO z{Jv+<5@`iTD+>wHE^mb&u9EpRe9auC+rm3vYohss$}n~bGrbM#fTiu69vR%2qhdUV zsLS^kz{3|h>5)6Z$tt=yx#_!YS=%X*JjH-uVq18kL$;cwqeg*6ASnMRsgsu(PP4O0 zFz_e6xJ=|krc){Ximza1e#}QVw0@32%-bWJ&pcg?FBO(hc;={G|bvZl&S zWU?xA(r@$(!afNO>nF>+Zx%dDqZH3YLM}ZAXgb--Z0(sZMq`q*(deb~47)uR0x7DR z&_$Lji5lEwy~<)*MJi|318h;>`xIU#!;;>5Rb3!E(Jkq<_B=FJd16v)M^7}UX#^QA zmWnB3eOOdcg+Yu)^{iYXWS-fh1Z3}25xrznnIi3%%i1^0`GJY^O0oz4brs1P8J&&` zAUOghvl58ulR0_^g7*i5!$Z}~Mv7Z#?k{nYi70VfsMt!K%ThWLmf=`4C86P)SL1B= zS-Pd%*pg!q@BIK}bJ8ReHz>fXIWOHoK2|V7AGdd;?UIWg>AkhabDk<~cFY02Y|pO= zJ!|z*z-b#p{Myj7DC!jYSwH4eXf!f#!1AKre9YrRYFf`J$UvSFk0#+T%v8}d)UFOS z&Uz_ieCa?Qo$;sq-BvA$Ikpkip%s7R$@+)v$xp;MMBnZF6yco|U6mqYUike<=cU^c zRVuP?+b!!D;e{{ctP(17xd`4n^D>({z>$cb%ZCjBLOtO`sxQ=ZA7pNawu5DW&gbne z2smJ%^keDQ%Q-HC1d$tE$Qt!)B$vw1s)8JqgCwZd0_R`W8Xzh?`}YfVy06sE)5Hu_ zH1LZjwUzLhnSz>48UkVg$RG|%<3Pv>7v?2U$cQS`390hcZ!4UoKbF1;+x#Q zM1fu0lsP$76X%s)5u|GB4q`t7X0zGXIh-1)A=PM)bPVVFHhFMpfJekr#mQ~H}rLU1uyZ~L-?i;qiQ#dpKET~{>vz%SEq zl4Jc zDmLxnUv*P=BE}ysTQ4}sUGZx?ve-({k+jH1^Y8Tr>H=OBOC2alOgogtQkW83%$E6m zHQjuk(<^v!_ma7^=BMurz*L_cLG9}C-a<9ItAT^M5+4$ENcJej5+q1cm0vrgF`|QW z+VF-}bkPq-(om1mL_oJ0IaAFNT@@d9y6VO4KVxArM~P=BMXWZ^j|P?(&wd ztClEI2;m)h#a5#$zmGG$KKHrO5t}FkkEz5TUdp1bvrB-pTLq=t;{ByJogFHf9VzJ0 zAm`KYYC;Y-twBi}=<}50*0J{K^_5d_{_UdY6c+ltLWSTrZe(F<6yIr&&Q|(H5`^|U zf}t+Px9uQt#1b@jHE)jO@IUjTjWRt3SwxevVQ~)k5rl!Qq#tu|s?D!=7ab!%F%_1( z^q413^U+POjk>1#j~{S`VQP*9eeyn>82B{?t@ZvbWGR(X3k-;wpPo}wJ?c**El>{3 zxO`i77M5BuzHR+%_}tSh>u^3<|BSBKR1Wg^%!7_5!+We1Rgk!-MT~32o9SB$1E3 z@)pNm^joZWMFxIoTfrEVsAcwFoz=-UkH>Ww%OHFHX#jh8-ft6~+9vU7rT$69Y1Osf zT44=7r?sfp2Cv>Od^TqGo-;sAHY;8$;(Jr9=>z-8JvbzbSHfFZe$QXd~@KMB@77ew6@vA5frOaCR5EFhe zK5xbsw&T7&**U(G2R_n)+(;X3ms)8i|FxjHSe^AeeOV$%&y z84FOudQT`T-=$`5X=c_K-%<0EaKS9t{`S=xVM zXSA|t+-~134w;haP+<)oUl#=K%o^8CJ=lztUwO4UQw!G&Z@GyDc&QhSfFo~mL(J3c zWAWZGQHXV~67;M@R~%l&^VAVhstN73*1H0^;Bw5V<>2R0wp1Sq59oR2D<6yBMLTX7 zDL~`eK5%8iEfJALd$4EtN)wTDY{9@5kMPIenESc|KZ2p&3CJPIif@m{RmIht z@5U&tOO(#)q-d^`rbrD^QnGe}y{3QytkLr~)&*Zu3J33b93dx&-7PyF;$&|J#e4zZ zRa$`Ne6Q^!w7|JeakKk7M&Mx@K=a34luTo6%KDE3B;U5}fOHB7?)!Ov5EB-(2Ow6U z0v+Vy8e=VYy_HKrrPy8AGT-eND}W?gvN@&#k-PnLFL1vjS55tI<<;gS#$8h2wLypWb$3DYZ!OlbxzYbW0?m z64rwRlE#iPV`VzOpx%qN8XwP9`4%h(>}pW&%|Gy{sTT^>Y-Tl=r- z!GXLbz1Xb3Q>;eqc^^bO=?(qFHhc8|H2QBu*MaII;>n$aw|*+JkzH6_P(JY#n!TaT zUaLayN=9~>nsX_+qu8>vKR`#mryQO%ba5`6raOX%f-5jgRw9Szo-|t$(2=0m{Tvc( zCQ(G4RD21ijl-WSavq8dBig`F`Puwf_87$5+Yljiydx^G$`7mP9ZP)(J9-zHdJ|EP z!hA931Z9@|W}r@MyFHbrTD>>KpmLySX;OdJl~Ev^d*s5vG4y#e5;(_MQooCi`?5NR z7#t!_7Utw;cAuNO`}{Dmph#GHVv9Uen5x0Pom!Ni^2#gZpe`ZX{rBzXnB9}GW}5@0 z`m|a5I!s;kpF(YW4eW2%sf9Y)jL35`@w?x?8rlpF9lUL^irI>{XXHc_jrp3TWFea! zw1~Pf)Y+3Zf_ZneJOXSP75Z9LU22kRu6-u;S!7&e!OqgOoTclKemfq|W&26C*3n#` zow$-emg1#Rh)DDo)6_lXviAY>sVW3jB9cr@`h9B*H#v=Dy*hlm6Ag(TxXpI)kyBnZ zam;n52D5uY?t7kh4355O-;82w*uOU$N3`_)X&fxoeE=Cj)_s6sXo`}4%$)Fp3yswl zH`(hFYPz2`(U>hha!O7N2oTF(nv||XIC;h0j{P|-P-ia7{<^m4&w<}@*_EMQBx?iY z1kvUrj*ezt1p~ec;uS+qb!F6jFXhmgLiJ>39@^?B?U30Xp(%?k-yc<`@E-djn$!#2 z4!i%<>Q_mGVuQT^JV#vq`jog3c1o_|Bko$hP~ ztr;N46}(;0lig_g@+J}`1yBktFy^>{aGfA}jBvf`np6kKvoUuo!e7@`jPWYDT5cy+rPOGPXi{-%2@hH5$xdfF z?tf(%Y}*ojri%pmC~y`vJ-eptTHL&wU)aqF!;>4~jm;}tts@x@(kH}iT=n)*DHY7a ztZB5FJn#i@=pNWbLg#Lz9+sE*y&65`qQ|7IeDm+;c85gIx!oUwPp?7eC&vUOdmOWP z1~_`ZicWytgig;RZbrF*d z@pfToU-nOje5z*B09|4T9DZc0-@6lWz8+UdOB!>nAM;qgyV4ii!djlGI){CRE1(J6 zo%f4NE5B^Ip~p1RJ3QWKdmI4Sq?)d-&Y$)TalK?sDw&pod8l&B-Ar$ni$!Rp!Q0IJ zX3c@_6Jy3Go{{~R=#aMNVEBhibPZoOetF`8-y2!V4)^zIS++);0c{uEq*1o3Zlssc zw6bWFX0krUpG7TQhjyc@jCFw0CrE)?xiFj9af4vmUU()85To;!sQ+%bb z5fz!vmMJ)=*h;r?gdDRvj@KUSa9B+66DJGVe`pweOtq8XtGLf&qT}+7BfA1hob&Pt zk?C4s&?)B%U-e=KFO!^vu;q$Talc;YO`9RG{u%*q7mh;x@CMKNpro5_%emqF5u^?C zAx(SA=}^c*!A~NNqV7q+M5sUi2VjrX&QJ9PP0N$v8Lj)jXfK^%?Vs}{)BICMmwESJ zN1Qzi;fc(0$o(T!JnxX6NDu!N>XT>}gJV1iH^oepCxNUHf&3&AbYK0u3%UXRzdh83 z3;)QC@$eIY4VdOVdF{$Ab(ts1%S;C2NRS4LD?D|Q6BH1a6+y5% z4%)x74A5947-b>_m_hZ)td^(_Zq#@(wkK+Udo^L`Ne$1Jlef2e%f&YBNm1XhnYu8H_HI35OfR4T{7Efh>yt`;Gqt{r2M% delta 812 zcmXYwTS!x39KgT*&!*Gd=ydT~3x#FFOQl3<8E59qrlK7uHj`9vvT!pmQz#1sFZg7$ zKNw}iYV)$W+i6r%7K!x`J@^pWO-hjH#Zr3lrH9VG(|P&-{=ff)@B83vufuyksH)2u zc|3`JHQn}cRl4$bEN@|zyak;jDdSbAm<0tdjCC>03k4rx&}ME@r(nd_L!0X`c04?Q z13`J@rR$vx+dVOOMA(UiK{a$HVXWH(&y$dy*@MUPSuF8opvSF&qEu0(;}5q1e(j@J z=`lcXKgGu$BWN`!n`(p=4IN#ZlH;|hYS3r=PtA~}rM(T)1|SEiy{=Ej1E{0MtDO3!%By@zzVWV5Ltx)Em#p3WeF!zg?ieK->VK~e& mkNOFwO{Sq*;J`M7)j~ZiL_;#Z7PRUdl0&jtLfGNxukjB)w=uy0 From 8559721fd039db7c8764a9e02002416cd5aee24f Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Thu, 23 Nov 2023 12:33:16 +0300 Subject: [PATCH 07/32] Update dockerfile for dev and prod --- dockerize/Makefile | 1 + dockerize/docker/Dockerfile | 3 +++ dockerize/production/Dockerfile | 4 ++++ setup.cfg | 1 + 4 files changed, 9 insertions(+) diff --git a/dockerize/Makefile b/dockerize/Makefile index fa4334e2..1ad35681 100644 --- a/dockerize/Makefile +++ b/dockerize/Makefile @@ -141,6 +141,7 @@ dbrestore: @echo "Restore dump from backups/latest.dmp in production mode" @echo "------------------------------------------------------------------" @# - prefix causes command to continue even if it fails + @# - TODO: This command needs update since postgis has been updated. @echo "stopping web container" @docker-compose -p $(PROJECT_ID) stop web @echo "dropping gis" diff --git a/dockerize/docker/Dockerfile b/dockerize/docker/Dockerfile index 10eb7660..f7f77638 100644 --- a/dockerize/docker/Dockerfile +++ b/dockerize/docker/Dockerfile @@ -43,6 +43,9 @@ RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so ENV NOTVISIBLE "in users profile" RUN echo "export VISIBLE=now" >> /etc/profile +# Install freezegun for feedback test +RUN pip install freezegun + # -------------------------------------------------------- # Open ports as needed # -------------------------------------------------------- diff --git a/dockerize/production/Dockerfile b/dockerize/production/Dockerfile index a51f6e85..9453fffc 100644 --- a/dockerize/production/Dockerfile +++ b/dockerize/production/Dockerfile @@ -5,6 +5,10 @@ MAINTAINER Dimas Ciputra #RUN ln -s /bin/true /sbin/initctl RUN apt-get clean all + +# Debian stretch/updates release issue. please see https://serverfault.com/a/1130167 +RUN echo "deb http://archive.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list + RUN apt-get update && apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev ARG BRANCH_TAG=develop diff --git a/setup.cfg b/setup.cfg index fae25c68..2fa09d19 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,6 +3,7 @@ exclude = .git, .venv/, ./.venv/, + venv/, vagrant_assets, qgis-app/*/migrations/, qgis-app/plugins/tests/HelloWorld/, From 04c5ffe01de057d2e5c26ca57a3a3df8954e6757 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Tue, 13 Feb 2024 09:15:38 +0300 Subject: [PATCH 08/32] Use main branch for whoosh in requirements --- REQUIREMENTS.txt | 2 +- dockerize/docker/REQUIREMENTS.txt | 2 +- qgis-app/REQUIREMENTS_plugins.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/REQUIREMENTS.txt b/REQUIREMENTS.txt index f02fdf0e..a92fb708 100644 --- a/REQUIREMENTS.txt +++ b/REQUIREMENTS.txt @@ -15,7 +15,7 @@ Pillow Pygments==2.7.4 # Updates for Django 2 & Python 3.7 -git+https://github.com/Xpirix/whoosh.git@a306553 +git+https://github.com/Xpirix/whoosh.git@main pickle5==0.0.12 django-haystack==3.2.1 diff --git a/dockerize/docker/REQUIREMENTS.txt b/dockerize/docker/REQUIREMENTS.txt index 5fafb8b6..cfe52640 100644 --- a/dockerize/docker/REQUIREMENTS.txt +++ b/dockerize/docker/REQUIREMENTS.txt @@ -26,7 +26,7 @@ django-extensions django-debug-toolbar==3.2.4 # Updates for Django 2 & Python 3.7 -git+https://github.com/Xpirix/whoosh.git@a306553 +git+https://github.com/Xpirix/whoosh.git@main pickle5==0.0.12 django-haystack==3.2.1 diff --git a/qgis-app/REQUIREMENTS_plugins.txt b/qgis-app/REQUIREMENTS_plugins.txt index 40e69142..ebb39373 100644 --- a/qgis-app/REQUIREMENTS_plugins.txt +++ b/qgis-app/REQUIREMENTS_plugins.txt @@ -25,6 +25,6 @@ django-extensions django-debug-toolbar==3.2.4 # Updates for Django 2 & Python 3.7 -git+https://github.com/Xpirix/whoosh.git@a306553 +git+https://github.com/Xpirix/whoosh.git@main pickle5==0.0.12 django-haystack==3.2.1 From 16434e4f04480a42c9b6822adb7d1e2dced8bb53 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Thu, 21 Mar 2024 11:09:40 +0300 Subject: [PATCH 09/32] Django 4 update: Requirements, dockerfile and docker-compose --- dockerize/docker-compose.yml | 17 +++---- dockerize/docker/Dockerfile | 31 +++++++------ dockerize/docker/REQUIREMENTS.txt | 73 +++++++++++++++---------------- 3 files changed, 59 insertions(+), 62 deletions(-) diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 102bb6cb..4b7fd94b 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -11,17 +11,9 @@ volumes: postgres-data: services: - smtp: - image: catatnight/postfix:latest - hostname: postfix - environment: - - maildomain=qgis.org - - smtp_user=noreply:docker - restart: unless-stopped - db: container_name: qgis-plugins-db - image: kartoza/postgis:14-3.3 + image: kartoza/postgis:16-3.4 environment: - ALLOW_IP_RANGE=0.0.0.0/0 - POSTGRES_USER=${DATABASE_USERNAME:-docker} @@ -34,7 +26,10 @@ services: uwsgi: &uwsgi-common container_name: qgis-plugins-uwsgi-common - image: kartoza/qgis-plugins-uwsgi + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: prod hostname: uwsgi environment: - DATABASE_NAME=${DATABASE_NAME:-gis} @@ -120,7 +115,7 @@ services: restart: unless-stopped dbbackups: - image: kartoza/pg-backup:14-3.3 + image: kartoza/pg-backup:16-3.4 hostname: pg-backups volumes: - db-backups:/backups diff --git a/dockerize/docker/Dockerfile b/dockerize/docker/Dockerfile index 9e43d497..75dc5830 100644 --- a/dockerize/docker/Dockerfile +++ b/dockerize/docker/Dockerfile @@ -1,26 +1,31 @@ -#--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ -# Note this base image is based on debian -FROM kartoza/django-base:3.7 as prod -MAINTAINER Dimas Ciputra +# For more information, please refer to https://aka.ms/vscode-docker-python +FROM python:3.12-slim as prod -#RUN ln -s /bin/true /sbin/initctl -RUN apt-get clean all +EXPOSE 8000 -# Debian stretch/updates release issue. please see https://serverfault.com/a/1130167 -RUN echo "deb http://archive.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list +# Keeps Python from generating .pyc files in the container +ENV PYTHONDONTWRITEBYTECODE=1 -RUN apt-get update && apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev +# Turns off buffering for easier container logging +ENV PYTHONUNBUFFERED=1 + +RUN apt-get update && apt-get install -y \ + git python3-dev libxml2-dev \ + libsasl2-dev libldap2-dev libssl-dev \ + libxslt1-dev zlib1g-dev \ + build-essential \ + libffi-dev \ + libjpeg-dev libpq-dev \ + liblcms2-dev libblas-dev libatlas-base-dev RUN rm -rf /uwsgi.conf ADD dockerize/docker/uwsgi.conf /uwsgi.conf ADD qgis-app /home/web/django_project ADD dockerize/docker/REQUIREMENTS.txt /REQUIREMENTS.txt -RUN pip install --upgrade pip && pip install -r /REQUIREMENTS.txt -RUN pip install uwsgi freezegun==1.3.1 +RUN pip install --upgrade pip && pip install -r /REQUIREMENTS.txt -# Open port 8080 as we will be running our uwsgi socket on that -EXPOSE 8080 +RUN apt-get update && apt-get install -y gdal-bin RUN mkdir -p /var/log/uwsgi diff --git a/dockerize/docker/REQUIREMENTS.txt b/dockerize/docker/REQUIREMENTS.txt index cfe52640..e9ff83f3 100644 --- a/dockerize/docker/REQUIREMENTS.txt +++ b/dockerize/docker/REQUIREMENTS.txt @@ -1,64 +1,61 @@ -django==3.2.11 -django-auth-ldap -python-ldap -django-taggit==2.0.0 +django~=4.2 +django-auth-ldap~=4.6 +python-ldap~=3.4 +django-taggit~=5.0 django-tinymce==3.4.0 -psycopg2 +psycopg2-binary~=2.9 # Updates for Django 2 git+https://github.com/metamatik/django-templatetag-sugar.git +# Updates for Django 4 +git+https://github.com/Xpirix/django-ratings.git@modernize +django-taggit-autosuggest~=0.4 +django-annoying~=0.10 -# Updates for Django 3 -git+https://github.com/gelo-zhukov/django-ratings.git - -django-taggit-autosuggest -django-annoying # Updates for Django 2 -git+https://github.com/elpaso/rpc4django.git@modernize -Pillow +# git+https://github.com/elpaso/rpc4django.git@modernize +rpc4django~=0.6 +Pillow~=10.1 django-taggit-templatetags -# Updates for Django 2 -git+https://github.com/elpaso/django-simplemenu.git@modernize -django-bootstrap-pagination -django-sortable-listview -sorl-thumbnail -django-extensions - -django-debug-toolbar==3.2.4 - -# Updates for Django 2 & Python 3.7 -git+https://github.com/Xpirix/whoosh.git@main -pickle5==0.0.12 -django-haystack==3.2.1 +# Updates for Django 4 +git+https://github.com/Xpirix/django-simplemenu.git@modernize +django-bootstrap-pagination-forked~=1.7 +django-sortable-listview~=0.43 +sorl-thumbnail~=12.10 +django-extensions~=3.2 +django-debug-toolbar~=4.2 +whoosh~=2.7 +django-haystack~=3.2 # Feedjack==0.9.18 # So use George's fork rather # git+https://github.com/Erve1879/feedjack.git # George's is also broken: use my fork (django 1.8 ready) # git+https://github.com/elpaso/feedjack.git -# His is also broken, use dimasciput (django 3.2 ready) -git+https://github.com/dimasciput/feedjack.git - -feedparser==5.2.1 -celery==5.2.7 -django-celery-beat==2.3.0 +# His is also broken, use dimasciput (django 2.2 ready) +# git+https://github.com/dimasciput/feedjack.git +# For django 4, use Xpirix (django 4.2 ready) +git+https://github.com/Xpirix/feedjack.git +feedparser~=6.0 +celery~=5.3 # pin due to issues with a breaking change # https://github.com/celery/celery/issues/7783 importlib_metadata<5 -requests==2.23.0 - -markdown==3.2.1 +requests~=2.31 -djangorestframework==3.11.2 -pyjwt==1.7.1 -djangorestframework-simplejwt==4.4 +markdown~=3.5 +djangorestframework~=3.14 +pyjwt~=2.8 +djangorestframework-simplejwt~=5.3 sorl-thumbnail-serializer-field==0.2.1 django-rest-auth==0.9.5 -drf-yasg==1.17.1 +drf-yasg~=1.21 django-rest-multiple-models==2.1.3 django-preferences==1.0.0 PyWavefront==1.3.3 django-matomo==0.1.6 +uwsgi~=2.0 +freezegun~=1.4 \ No newline at end of file From c93fac69fe5b28350f0b33c65e27a6cd389213ee Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Thu, 21 Mar 2024 11:10:02 +0300 Subject: [PATCH 10/32] Django 4 update: fixes for ifequal, ugettext_lazy, django.conf.urls.url occurences --- qgis-app/base/forms/processing_forms.py | 2 +- qgis-app/base/models/processing_models.py | 2 +- qgis-app/base/validator.py | 2 +- qgis-app/base/views/processing_view.py | 2 +- qgis-app/custom_haystack_urls.py | 2 +- qgis-app/geopackages/models.py | 2 +- qgis-app/homepage.py | 2 +- qgis-app/layerdefinitions/file_handler.py | 2 +- qgis-app/layerdefinitions/models.py | 2 +- qgis-app/models/models.py | 2 +- qgis-app/models/validator.py | 2 +- qgis-app/plugins/api.py | 2 +- qgis-app/plugins/forms.py | 2 +- .../plugins/migrations/0009_merge_20240321_0207.py | 14 ++++++++++++++ qgis-app/plugins/models.py | 2 +- .../plugins/templates/plugins/form_snippet.html | 4 ++-- qgis-app/plugins/templates/plugins/pagination.html | 4 ++-- qgis-app/plugins/urls.py | 4 ++-- qgis-app/plugins/validator.py | 2 +- qgis-app/plugins/views.py | 2 +- qgis-app/qgis_context_processor.py | 4 +++- qgis-app/settings_docker.py | 7 ++++--- qgis-app/styles/file_handler.py | 2 +- qgis-app/styles/forms.py | 2 +- qgis-app/styles/models.py | 2 +- qgis-app/styles/views.py | 2 +- qgis-app/templates/cab/snippet_detail.html | 6 +++--- qgis-app/templates/snippet_base.html | 2 +- qgis-app/urls.py | 2 +- qgis-app/userexport/urls.py | 2 +- qgis-app/users/urls.py | 2 +- qgis-app/users/views.py | 2 +- qgis-app/wavefronts/models.py | 2 +- qgis-app/wavefronts/validator.py | 2 +- qgis-app/wavefronts/views.py | 2 +- 35 files changed, 58 insertions(+), 41 deletions(-) create mode 100644 qgis-app/plugins/migrations/0009_merge_20240321_0207.py diff --git a/qgis-app/base/forms/processing_forms.py b/qgis-app/base/forms/processing_forms.py index 771eb3eb..cdf85a7f 100644 --- a/qgis-app/base/forms/processing_forms.py +++ b/qgis-app/base/forms/processing_forms.py @@ -1,6 +1,6 @@ from base.validator import filesize_validator from django import forms -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class ResourceBaseReviewForm(forms.Form): diff --git a/qgis-app/base/models/processing_models.py b/qgis-app/base/models/processing_models.py index 0d6590c5..9b53c625 100644 --- a/qgis-app/base/models/processing_models.py +++ b/qgis-app/base/models/processing_models.py @@ -7,7 +7,7 @@ from django.contrib.auth.models import User from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class UnapprovedManager(models.Manager): diff --git a/qgis-app/base/validator.py b/qgis-app/base/validator.py index 534c4c08..4a491e9e 100644 --- a/qgis-app/base/validator.py +++ b/qgis-app/base/validator.py @@ -2,7 +2,7 @@ from django.conf import settings from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ RESOURCE_MAX_SIZE = getattr(settings, "RESOURCE_MAX_SIZE", 1000000) # 1MB ERROR_FILESIZE_TOO_BIG = ValidationError( diff --git a/qgis-app/base/views/processing_view.py b/qgis-app/base/views/processing_view.py index bddb4fb5..10087884 100644 --- a/qgis-app/base/views/processing_view.py +++ b/qgis-app/base/views/processing_view.py @@ -17,7 +17,7 @@ from django.urls import reverse, reverse_lazy from django.utils.decorators import method_decorator from django.utils.text import slugify -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.views.decorators.cache import never_cache from django.views.generic import ( CreateView, diff --git a/qgis-app/custom_haystack_urls.py b/qgis-app/custom_haystack_urls.py index a0ab8c23..c14fa7f5 100755 --- a/qgis-app/custom_haystack_urls.py +++ b/qgis-app/custom_haystack_urls.py @@ -1,6 +1,6 @@ # Custom haystack search to match partial strings -from django.conf.urls import include, url +from django.urls import re_path as url from haystack.query import SearchQuerySet from haystack.views import SearchView diff --git a/qgis-app/geopackages/models.py b/qgis-app/geopackages/models.py index 0cfc14e2..3475f08a 100644 --- a/qgis-app/geopackages/models.py +++ b/qgis-app/geopackages/models.py @@ -5,7 +5,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ GEOPACKAGES_STORAGE_PATH = getattr( settings, "GEOPACKAGE_STORAGE_PATH", "geopackages/%Y" diff --git a/qgis-app/homepage.py b/qgis-app/homepage.py index 7bf3d7c2..a7e9e51f 100644 --- a/qgis-app/homepage.py +++ b/qgis-app/homepage.py @@ -1,7 +1,7 @@ from django.contrib.flatpages.models import FlatPage from django.shortcuts import render from django.template import RequestContext -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from plugins.models import Plugin # from feedjack.models import Post diff --git a/qgis-app/layerdefinitions/file_handler.py b/qgis-app/layerdefinitions/file_handler.py index ddacc02d..ffc196c1 100644 --- a/qgis-app/layerdefinitions/file_handler.py +++ b/qgis-app/layerdefinitions/file_handler.py @@ -6,7 +6,7 @@ import xml.etree.ElementTree as ET from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ def parse_qlr(xmlfile): diff --git a/qgis-app/layerdefinitions/models.py b/qgis-app/layerdefinitions/models.py index 1c6be6e4..2383b5be 100644 --- a/qgis-app/layerdefinitions/models.py +++ b/qgis-app/layerdefinitions/models.py @@ -5,7 +5,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ LAYERDEFINITIONS_STORAGE_PATH = getattr( settings, "LAYERDEFINITION_STORAGE_PATH", "layerdefinitions" diff --git a/qgis-app/models/models.py b/qgis-app/models/models.py index 1786bd9e..4aa2809f 100644 --- a/qgis-app/models/models.py +++ b/qgis-app/models/models.py @@ -5,7 +5,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ MODELS_STORAGE_PATH = getattr(settings, "MODELS_STORAGE_PATH", "models/%Y") diff --git a/qgis-app/models/validator.py b/qgis-app/models/validator.py index e4d41829..6241633d 100644 --- a/qgis-app/models/validator.py +++ b/qgis-app/models/validator.py @@ -2,7 +2,7 @@ from django.conf import settings from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ MODEL_MAX_SIZE = getattr(settings, "MODEL_MAX_SIZE", 1000000) # 1MB diff --git a/qgis-app/plugins/api.py b/qgis-app/plugins/api.py index 62ff6b8e..72615d68 100644 --- a/qgis-app/plugins/api.py +++ b/qgis-app/plugins/api.py @@ -12,7 +12,7 @@ # Transaction from django.db import IntegrityError, connection -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from plugins.models import * from plugins.validator import validator from plugins.views import plugin_notify diff --git a/qgis-app/plugins/forms.py b/qgis-app/plugins/forms.py index 40c8ab5a..9f98f03f 100644 --- a/qgis-app/plugins/forms.py +++ b/qgis-app/plugins/forms.py @@ -5,7 +5,7 @@ from django.contrib.auth.models import User from django.forms import CharField, ModelForm, ValidationError from django.utils.safestring import mark_safe -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from plugins.models import Plugin, PluginOutstandingToken, PluginVersion, PluginVersionFeedback from plugins.validator import validator from taggit.forms import * diff --git a/qgis-app/plugins/migrations/0009_merge_20240321_0207.py b/qgis-app/plugins/migrations/0009_merge_20240321_0207.py new file mode 100644 index 00000000..45cd685a --- /dev/null +++ b/qgis-app/plugins/migrations/0009_merge_20240321_0207.py @@ -0,0 +1,14 @@ +# Generated by Django 4.2.11 on 2024-03-21 02:07 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('plugins', '0004_merge_20231123_0018'), + ('plugins', '0008_merge_20240206_0448'), + ] + + operations = [ + ] diff --git a/qgis-app/plugins/models.py b/qgis-app/plugins/models.py index 9f9cc209..72e8cc5a 100644 --- a/qgis-app/plugins/models.py +++ b/qgis-app/plugins/models.py @@ -8,7 +8,7 @@ from django.contrib.auth.models import User from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.utils import timezone from djangoratings.fields import AnonymousRatingField from taggit_autosuggest.managers import TaggableManager diff --git a/qgis-app/plugins/templates/plugins/form_snippet.html b/qgis-app/plugins/templates/plugins/form_snippet.html index f33ec0ce..ae794508 100755 --- a/qgis-app/plugins/templates/plugins/form_snippet.html +++ b/qgis-app/plugins/templates/plugins/form_snippet.html @@ -2,7 +2,7 @@
{% for field in form %}
- {% ifequal field.field.widget|klass 'CheckboxInput' %} + {% if field.field.widget|klass == 'CheckboxInput' %}
{% endif %} {{ field }} - {% endifequal %} + {% endif %}
{{ field.help_text }}
{% endfor %} diff --git a/qgis-app/plugins/templates/plugins/pagination.html b/qgis-app/plugins/templates/plugins/pagination.html index fc7e65c7..99bec111 100644 --- a/qgis-app/plugins/templates/plugins/pagination.html +++ b/qgis-app/plugins/templates/plugins/pagination.html @@ -8,11 +8,11 @@ {% endif %} {% for page in pages %} {% if page %} - {% ifequal page page_obj.number %} + {% if page == page_obj.number %} {{ page }} {% else %} {{ page }} - {% endifequal %} + {% endif %} {% else %} ... {% endif %} diff --git a/qgis-app/plugins/urls.py b/qgis-app/plugins/urls.py index dd021892..3d1c5da3 100644 --- a/qgis-app/plugins/urls.py +++ b/qgis-app/plugins/urls.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from django.conf.urls import include, url +from django.urls import re_path as url from django.contrib.auth.decorators import login_required, user_passes_test -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from plugins.models import Plugin, PluginVersion from plugins.views import * from rpc4django.views import serve_rpc_request diff --git a/qgis-app/plugins/validator.py b/qgis-app/plugins/validator.py index e1482bd4..78afa1fa 100644 --- a/qgis-app/plugins/validator.py +++ b/qgis-app/plugins/validator.py @@ -15,7 +15,7 @@ from django.conf import settings from django.core.files.uploadedfile import SimpleUploadedFile from django.forms import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ PLUGIN_MAX_UPLOAD_SIZE = getattr(settings, "PLUGIN_MAX_UPLOAD_SIZE", 25000000) # 25 mb PLUGIN_REQUIRED_METADATA = getattr( diff --git a/qgis-app/plugins/views.py b/qgis-app/plugins/views.py index d1897117..20dfca1c 100644 --- a/qgis-app/plugins/views.py +++ b/qgis-app/plugins/views.py @@ -21,7 +21,7 @@ from django.utils.timezone import now from django.utils.decorators import method_decorator from django.utils.encoding import DjangoUnicodeDecodeError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.views.decorators.cache import never_cache from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt, csrf_protect from django.views.decorators.http import require_POST diff --git a/qgis-app/qgis_context_processor.py b/qgis-app/qgis_context_processor.py index 11ff0481..8b8de12c 100644 --- a/qgis-app/qgis_context_processor.py +++ b/qgis-app/qgis_context_processor.py @@ -2,12 +2,14 @@ from django.contrib.sites.models import Site from django.core.exceptions import ImproperlyConfigured +def is_ajax(request): + return request.META.get('HTTP_X_REQUESTED_WITH') == 'XMLHttpRequest' def additions(request): """Insert some additional information into the template context from the settings and set the base template according to qs. """ - if request.is_ajax() or request.GET.get("ajax"): + if is_ajax(request=request) or request.GET.get("ajax"): base_template = "ajax_base.html" is_naked = True else: diff --git a/qgis-app/settings_docker.py b/qgis-app/settings_docker.py index a02e3bc4..55f5865a 100644 --- a/qgis-app/settings_docker.py +++ b/qgis-app/settings_docker.py @@ -10,13 +10,14 @@ from datetime import timedelta from django.contrib.staticfiles.storage import ManifestStaticFilesStorage -DEBUG = ast.literal_eval(os.environ.get("DEBUG", "True")) +# DEBUG = ast.literal_eval(os.environ.get("DEBUG", "True")) +DEBUG = True THUMBNAIL_DEBUG = DEBUG ALLOWED_HOSTS = ["*"] # Absolute filesystem path to the directory that will hold user-uploaded files. # Example: "/var/www/example.com/media/" -MEDIA_ROOT = os.environ.get("MEDIA_ROOT", "/home/web/media") +MEDIA_ROOT = os.environ.get("MEDIA_ROOT", "/home/web/media/") # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash. @@ -29,7 +30,7 @@ # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. # Example: "/var/www/example.com/static/" -STATIC_ROOT = os.environ.get("STATIC_ROOT", "/home/web/static") +STATIC_ROOT = os.environ.get("STATIC_ROOT", "/home/web/static/") # URL prefix for static files. # Example: "http://example.com/static/", "http://static.example.com/" diff --git a/qgis-app/styles/file_handler.py b/qgis-app/styles/file_handler.py index e35e855f..42b3c2de 100644 --- a/qgis-app/styles/file_handler.py +++ b/qgis-app/styles/file_handler.py @@ -4,7 +4,7 @@ import xml.etree.ElementTree as ET from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ def _check_name_type_attribute(element): diff --git a/qgis-app/styles/forms.py b/qgis-app/styles/forms.py index 3fe24047..a23655ff 100644 --- a/qgis-app/styles/forms.py +++ b/qgis-app/styles/forms.py @@ -1,6 +1,6 @@ from django import forms from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from styles.file_handler import validator from styles.models import Style diff --git a/qgis-app/styles/models.py b/qgis-app/styles/models.py index f1e4f753..52e62f51 100644 --- a/qgis-app/styles/models.py +++ b/qgis-app/styles/models.py @@ -3,7 +3,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ STYLES_STORAGE_PATH = getattr(settings, "PLUGINS_STORAGE_PATH", "styles/%Y") diff --git a/qgis-app/styles/views.py b/qgis-app/styles/views.py index 84cfb874..db156da4 100644 --- a/qgis-app/styles/views.py +++ b/qgis-app/styles/views.py @@ -19,7 +19,7 @@ from django.http import HttpResponse, HttpResponseRedirect from django.urls import reverse, reverse_lazy from django.utils.crypto import get_random_string -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.views.decorators.cache import never_cache from styles.file_handler import read_xml_style from styles.forms import UpdateForm, UploadForm diff --git a/qgis-app/templates/cab/snippet_detail.html b/qgis-app/templates/cab/snippet_detail.html index 38c97359..d47040ae 100644 --- a/qgis-app/templates/cab/snippet_detail.html +++ b/qgis-app/templates/cab/snippet_detail.html @@ -56,9 +56,9 @@

More like this

Tools

    {% if user.is_authenticated %} - {% ifequal user.id object.author.id %} + {% if user.id == object.author.id %}
  • Edit this snippet
    • - {% endifequal %} + {% endif %} {% if object|is_bookmarked:request.user %}
  • This snippet is in your bookmarks
    • {% else %} @@ -66,7 +66,7 @@

    Tools

    {% endif %} {% with object|rating_score:request.user as rating_score %} {% if rating_score %} -
  • You rated this snippet {% ifequal rating_score 1 %}useful{% else %}not useful{% endifequal %}.
    • +
  • You rated this snippet {% if rating_score == 1 %}useful{% else %}not useful{% endif %}.
    • {% else %}
  • Rate this snippet: useful or not useful
    • {% endif %} diff --git a/qgis-app/templates/snippet_base.html b/qgis-app/templates/snippet_base.html index d1a359fa..0e4c29b5 100644 --- a/qgis-app/templates/snippet_base.html +++ b/qgis-app/templates/snippet_base.html @@ -36,7 +36,7 @@

    QGIS Snippets

    {% if user.is_authenticated %} Logged in as {{ user.username }} (Add a snippet) (Log out) {% else %} - Log in or register + Log in or register {% endif %}
      diff --git a/qgis-app/urls.py b/qgis-app/urls.py index 7166f5c1..90acac68 100644 --- a/qgis-app/urls.py +++ b/qgis-app/urls.py @@ -1,6 +1,6 @@ import simplemenu from django.conf import settings -from django.conf.urls import url +from django.urls import re_path as url # Uncomment the next two lines to enable the admin: from django.contrib import admin diff --git a/qgis-app/userexport/urls.py b/qgis-app/userexport/urls.py index 84d6a35c..45ac6d08 100755 --- a/qgis-app/userexport/urls.py +++ b/qgis-app/userexport/urls.py @@ -1,5 +1,5 @@ # -* coding:utf-8 *- # -from django.conf.urls import * +from django.urls import re_path as url from userexport.views import * urlpatterns = [ diff --git a/qgis-app/users/urls.py b/qgis-app/users/urls.py index dbcb3317..dd51a6d9 100644 --- a/qgis-app/users/urls.py +++ b/qgis-app/users/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls import include, url +from django.urls import re_path as url from django.views.generic.base import TemplateView from .views import * diff --git a/qgis-app/users/views.py b/qgis-app/users/views.py index 05774ac4..25d08f3e 100644 --- a/qgis-app/users/views.py +++ b/qgis-app/users/views.py @@ -5,7 +5,7 @@ from django.shortcuts import render, get_object_or_404 from annoying.functions import get_object_or_None from django.template import RequestContext -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.core.mail import send_mail from django.template.loader import render_to_string from django.contrib.sites.models import Site diff --git a/qgis-app/wavefronts/models.py b/qgis-app/wavefronts/models.py index 7c5c52f4..65b10e85 100644 --- a/qgis-app/wavefronts/models.py +++ b/qgis-app/wavefronts/models.py @@ -6,7 +6,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from preferences.models import Preferences WAVEFRONTS_STORAGE_PATH = getattr(settings, "WAVEFRONTS_STORAGE_PATH", "wavefronts") diff --git a/qgis-app/wavefronts/validator.py b/qgis-app/wavefronts/validator.py index 09e8f338..deff6be8 100644 --- a/qgis-app/wavefronts/validator.py +++ b/qgis-app/wavefronts/validator.py @@ -9,7 +9,7 @@ import pywavefront from django.conf import settings from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from wavefronts.models import WAVEFRONTS_STORAGE_PATH diff --git a/qgis-app/wavefronts/views.py b/qgis-app/wavefronts/views.py index e7c11151..a93458ea 100644 --- a/qgis-app/wavefronts/views.py +++ b/qgis-app/wavefronts/views.py @@ -20,7 +20,7 @@ from django.template.response import TemplateResponse from django.urls import reverse_lazy from django.utils.text import slugify -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from wavefronts.forms import UpdateForm, UploadForm from wavefronts.models import Review, Wavefront from wavefronts.utilities import zipped_all_with_license From 6e50f7348136a2e4898a9ff36459324ef15f13ea Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Thu, 21 Mar 2024 11:23:22 +0300 Subject: [PATCH 11/32] Django 4 update: Fix deprecated readfp --- qgis-app/plugins/validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgis-app/plugins/validator.py b/qgis-app/plugins/validator.py index 78afa1fa..5f19852c 100644 --- a/qgis-app/plugins/validator.py +++ b/qgis-app/plugins/validator.py @@ -255,7 +255,7 @@ def validator(package): try: parser = configparser.ConfigParser() parser.optionxform = str - parser.readfp(StringIO(codecs.decode(zip.read(metadataname), "utf8"))) + parser.read_file(StringIO(codecs.decode(zip.read(metadataname), "utf8"))) if not parser.has_section("general"): raise ValidationError( _("Cannot find a section named 'general' in %s") % metadataname From 63cfd3e8599d81624edd2437ed144473e6a050fb Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:17:08 +0300 Subject: [PATCH 12/32] Django 4 update: update dbrestore in Makefile --- dockerize/Makefile | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/dockerize/Makefile b/dockerize/Makefile index 1ad35681..47e89f4d 100644 --- a/dockerize/Makefile +++ b/dockerize/Makefile @@ -41,7 +41,7 @@ web: db @echo "------------------------------------------------------------------" @echo "Running in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up -d uwsgi web smtp worker beat + @docker-compose -p $(PROJECT_ID) up -d uwsgi web worker beat devweb-test: db @echo @@ -141,20 +141,19 @@ dbrestore: @echo "Restore dump from backups/latest.dmp in production mode" @echo "------------------------------------------------------------------" @# - prefix causes command to continue even if it fails - @# - TODO: This command needs update since postgis has been updated. - @echo "stopping web container" - @docker-compose -p $(PROJECT_ID) stop web - @echo "dropping gis" - @docker exec -t $(PROJECT_ID)-db su - postgres -c "dropdb gis" - @echo "creating gis" - @docker exec -t $(PROJECT_ID)-db su - postgres -c "createdb -O docker -T template_postgis gis" - @echo "restoring gis" - @# Because we pipe from one docker command to another and we are going - @# to execute this Make command from a remote server at times, we need to using use interactive mode (-i) - @# in the first command and not use terminal (-t) in the second. Please do not change these! - @docker exec -t $(PROJECT_ID)-db pg_restore /backups/latest.dmp | docker exec -i $(PROJECT_ID)-db su - postgres -c "psql gis" - @docker-compose -p $(PROJECT_ID) start web - @echo "starting web container" + @echo "stopping uwsgi container" + @docker-compose -p $(PROJECT_ID) stop uwsgi + @echo "Dropping the gis and metabase databases" + -@docker-compose -p $(PROJECT_ID) exec db su - postgres -c "dropdb --force gis" + -@docker-compose -p $(PROJECT_ID) exec db su - postgres -c "dropdb --force metabase" + @echo "Creating the gis and metabase databases" + -@docker-compose -p $(PROJECT_ID) exec db su - postgres -c "createdb -O docker -T template1 gis" + -@docker-compose -p $(PROJECT_ID) exec db su - postgres -c "createdb -O docker -T template1 metabase" + @echo "Restore database from backups/latest-gis.dmp" + -@docker-compose -p $(PROJECT_ID) exec db su - postgres -c "pg_restore -c /backups/latest-gis.dmp -d gis" + -@docker-compose -p $(PROJECT_ID) exec db su - postgres -c "pg_restore -c /backups/latest-metabase.dmp -d metabase" + @echo "starting uwsgi container" + @docker-compose -p $(PROJECT_ID) up -d uwsgi wait-db: @docker-compose exec -p $(PROJECT_ID) db su - postgres -c "until pg_isready; do sleep 5; done" From 9d250fc18ce77f40adc28facc82fccd900ba0701 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Thu, 21 Mar 2024 15:33:32 +0300 Subject: [PATCH 13/32] Django 4 update: Get static, media and backup volumes from the environment variable --- dockerize/.env.template | 3 +++ dockerize/docker-compose.yml | 23 ++++++++++------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dockerize/.env.template b/dockerize/.env.template index cd4fbcfe..0a214a20 100644 --- a/dockerize/.env.template +++ b/dockerize/.env.template @@ -4,3 +4,6 @@ DATABASE_USERNAME=docker DATABASE_PASSWORD=docker DATABASE_HOST=db DJANGO_SETTINGS_MODULE=settings_docker +QGISPLUGINS_STATIC_VOLUME=static-data +QGISPLUGINS_MEDIA_VOLUME=media-data +QGISPLUGINS_BACKUP_VOLUME=backups-data \ No newline at end of file diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 4b7fd94b..62f09c23 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -1,14 +1,9 @@ version: '3.8' volumes: - backups-data: - static-data: - media-data: - logs-data: + postgres_data: nginx-conf: - db-backups: rabbitmq: celerybeat-schedule: - postgres-data: services: db: @@ -20,8 +15,8 @@ services: - POSTGRES_PASS=${DATABASE_PASSWORD:-docker} - PASSWORD_AUTHENTICATION=${PASSWORD_AUTHENTICATION:-md5} volumes: - - postgres-data:/var/lib/postgresql - - backups-data:/backups + - postgres_data:/var/lib/postgresql + - ${QGISPLUGINS_BACKUP_VOLUME}:/backups restart: unless-stopped uwsgi: &uwsgi-common @@ -44,8 +39,8 @@ services: - BROKER_URL=amqp://rabbitmq:5672 volumes: - ../qgis-app:/home/web/django_project - - static-data:/home/web/static:rw - - media-data:/home/web/media:rw + - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:rw + - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:rw - celerybeat-schedule:/home/web/celerybeat-schedule:rw links: - db:db @@ -64,6 +59,8 @@ services: target: dev volumes: - ../qgis-app:/home/web/django_project + - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:rw + - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:rw ports: # for django test server - "62202:8080" @@ -102,8 +99,8 @@ services: hostname: web volumes: - nginx-conf:/etc/nginx/conf.d:ro - - static-data:/home/web/static:ro - - media-data:/home/web/media:ro + - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:ro + - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:ro links: - uwsgi:uwsgi - metabase:metabase @@ -118,7 +115,7 @@ services: image: kartoza/pg-backup:16-3.4 hostname: pg-backups volumes: - - db-backups:/backups + - ${QGISPLUGINS_BACKUP_VOLUME}:/backups links: - db:db environment: From 9dfbd355fe96dd231d59b3dab72a6371a240885a Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Thu, 21 Mar 2024 15:34:13 +0300 Subject: [PATCH 14/32] Django 4 update: Use solr thumbnail default engine --- qgis-app/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgis-app/settings.py b/qgis-app/settings.py index c3e7a79b..23d50828 100644 --- a/qgis-app/settings.py +++ b/qgis-app/settings.py @@ -253,7 +253,7 @@ DEBUG_TOOLBAR_CONFIG = {"INTERCEPT_REDIRECTS": False} -THUMBNAIL_ENGINE = "sorl.thumbnail.engines.convert_engine.Engine" +# THUMBNAIL_ENGINE = "sorl.thumbnail.engines.convert_engine.Engine" USER_MAP = { "project_name": "QGIS", From bd6493090a47022c149cffb224384cbae5c07409 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Thu, 21 Mar 2024 15:49:07 +0300 Subject: [PATCH 15/32] Django 4 update: Fix Django warnings --- qgis-app/plugins/validator.py | 8 ++++---- qgis-app/settings_docker.py | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/qgis-app/plugins/validator.py b/qgis-app/plugins/validator.py index 5f19852c..f8ebd6ad 100644 --- a/qgis-app/plugins/validator.py +++ b/qgis-app/plugins/validator.py @@ -61,16 +61,16 @@ def _read_from_init(initcontent, initname): i = 0 lines = initcontent.split("\n") while i < len(lines): - if re.search("def\s+([^\(]+)", lines[i]): - k = re.search("def\s+([^\(]+)", lines[i]).groups()[0] + if re.search(r"def\s+([^\(]+)", lines[i]): + k = re.search(r"def\s+([^\(]+)", lines[i]).groups()[0] i += 1 while i < len(lines) and lines[i] != "": - if re.search("return\s+[\"']?([^\"']+)[\"']?", lines[i]): + if re.search(r"return\s+[\"']?([^\"']+)[\"']?", lines[i]): metadata.append( ( k, re.search( - "return\s+[\"']?([^\"']+)[\"']?", lines[i] + r"return\s+[\"']?([^\"']+)[\"']?", lines[i] ).groups()[0], ) ) diff --git a/qgis-app/settings_docker.py b/qgis-app/settings_docker.py index 55f5865a..9fb535ee 100644 --- a/qgis-app/settings_docker.py +++ b/qgis-app/settings_docker.py @@ -158,3 +158,6 @@ MATOMO_SITE_ID="1" MATOMO_URL="//matomo.qgis.org/" + +# Default primary key type +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' \ No newline at end of file From 56f0ca88363ad2643d81058907afe0be6edb4af8 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Fri, 22 Mar 2024 08:14:28 +0300 Subject: [PATCH 16/32] Django 4 update: fix depecated tests, new migrations --- qgis-app/api/tests/test_views.py | 2 +- .../migrations/0009_alter_review_reviewer.py | 21 +++++++++++++++++++ .../migrations/0002_alter_review_reviewer.py | 21 +++++++++++++++++++ qgis-app/layerdefinitions/tests/test_views.py | 2 +- .../migrations/0007_alter_review_reviewer.py | 21 +++++++++++++++++++ qgis-app/plugins/tests/tests.py | 2 +- .../migrations/0015_alter_review_reviewer.py | 21 +++++++++++++++++++ qgis-app/users/tests.py | 2 +- .../migrations/0002_alter_review_reviewer.py | 21 +++++++++++++++++++ qgis-app/wavefronts/tests/test_view.py | 2 +- 10 files changed, 110 insertions(+), 5 deletions(-) create mode 100644 qgis-app/geopackages/migrations/0009_alter_review_reviewer.py create mode 100644 qgis-app/layerdefinitions/migrations/0002_alter_review_reviewer.py create mode 100644 qgis-app/models/migrations/0007_alter_review_reviewer.py create mode 100644 qgis-app/styles/migrations/0015_alter_review_reviewer.py create mode 100644 qgis-app/wavefronts/migrations/0002_alter_review_reviewer.py diff --git a/qgis-app/api/tests/test_views.py b/qgis-app/api/tests/test_views.py index 9f0ff30b..acb2cb30 100644 --- a/qgis-app/api/tests/test_views.py +++ b/qgis-app/api/tests/test_views.py @@ -216,7 +216,7 @@ def test_download_resource_should_be_a_file_in_a_zip(self): url = reverse("resource-download", kwargs={"uuid": self.style.uuid}) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertEquals( + self.assertEqual( response.get("Content-Disposition"), "attachment; filename=style_zero.zip" ) with io.BytesIO(response.content) as file: diff --git a/qgis-app/geopackages/migrations/0009_alter_review_reviewer.py b/qgis-app/geopackages/migrations/0009_alter_review_reviewer.py new file mode 100644 index 00000000..0fc4f397 --- /dev/null +++ b/qgis-app/geopackages/migrations/0009_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('geopackages', '0008_remove_uuid_null'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/layerdefinitions/migrations/0002_alter_review_reviewer.py b/qgis-app/layerdefinitions/migrations/0002_alter_review_reviewer.py new file mode 100644 index 00000000..97576ae6 --- /dev/null +++ b/qgis-app/layerdefinitions/migrations/0002_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('layerdefinitions', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/layerdefinitions/tests/test_views.py b/qgis-app/layerdefinitions/tests/test_views.py index f46119b2..44f8dd28 100644 --- a/qgis-app/layerdefinitions/tests/test_views.py +++ b/qgis-app/layerdefinitions/tests/test_views.py @@ -193,7 +193,7 @@ def test_download_should_return_zipfile_with_custom_license(self): url = reverse("layerdefinition_download", kwargs={"pk": qlr.id}) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertEquals( + self.assertEqual( response.get("Content-Disposition"), "attachment; filename=test-qlr-file.zip", ) diff --git a/qgis-app/models/migrations/0007_alter_review_reviewer.py b/qgis-app/models/migrations/0007_alter_review_reviewer.py new file mode 100644 index 00000000..ad03f8c2 --- /dev/null +++ b/qgis-app/models/migrations/0007_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('models', '0006_remove_uuid_null'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/plugins/tests/tests.py b/qgis-app/plugins/tests/tests.py index 3748f41b..f51d798f 100644 --- a/qgis-app/plugins/tests/tests.py +++ b/qgis-app/plugins/tests/tests.py @@ -13,7 +13,7 @@ def test_basic_addition(self): """ Tests that 1 + 1 always equals 2. """ - self.failUnlessEqual(1 + 1, 2) + self.assertEqual(1 + 1, 2) __test__ = { diff --git a/qgis-app/styles/migrations/0015_alter_review_reviewer.py b/qgis-app/styles/migrations/0015_alter_review_reviewer.py new file mode 100644 index 00000000..8c6b0558 --- /dev/null +++ b/qgis-app/styles/migrations/0015_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('styles', '0014_remove_uuid_null'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/users/tests.py b/qgis-app/users/tests.py index 3748f41b..f51d798f 100644 --- a/qgis-app/users/tests.py +++ b/qgis-app/users/tests.py @@ -13,7 +13,7 @@ def test_basic_addition(self): """ Tests that 1 + 1 always equals 2. """ - self.failUnlessEqual(1 + 1, 2) + self.assertEqual(1 + 1, 2) __test__ = { diff --git a/qgis-app/wavefronts/migrations/0002_alter_review_reviewer.py b/qgis-app/wavefronts/migrations/0002_alter_review_reviewer.py new file mode 100644 index 00000000..79efde7f --- /dev/null +++ b/qgis-app/wavefronts/migrations/0002_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('wavefronts', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/wavefronts/tests/test_view.py b/qgis-app/wavefronts/tests/test_view.py index 364f4929..380848e6 100644 --- a/qgis-app/wavefronts/tests/test_view.py +++ b/qgis-app/wavefronts/tests/test_view.py @@ -170,7 +170,7 @@ def test_download_should_return_zipfile(self): url = reverse("wavefront_download", kwargs={"pk": wavefront.id}) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertEquals( + self.assertEqual( response.get("Content-Disposition"), "attachment; filename=odm-texturing.zip", ) From f0119d5d69721ede1e0981a7287c68377f9be06e Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Fri, 22 Mar 2024 09:41:38 +0300 Subject: [PATCH 17/32] Django 4 update: Update django unit tests --- qgis-app/plugins/tests/test_change_maintainer.py | 2 +- qgis-app/plugins/tests/test_plugin_update.py | 4 ++-- qgis-app/plugins/tests/test_plugin_upload.py | 2 +- qgis-app/plugins/tests/test_rename_plugin.py | 2 +- qgis-app/plugins/tests/test_validator.py | 5 +---- 5 files changed, 6 insertions(+), 9 deletions(-) diff --git a/qgis-app/plugins/tests/test_change_maintainer.py b/qgis-app/plugins/tests/test_change_maintainer.py index a9357f6f..515b5c46 100644 --- a/qgis-app/plugins/tests/test_change_maintainer.py +++ b/qgis-app/plugins/tests/test_change_maintainer.py @@ -50,7 +50,7 @@ def setUp(self): self.plugin = Plugin.objects.get(name='Test Plugin') self.plugin.save() - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_change_maintainer(self): """ diff --git a/qgis-app/plugins/tests/test_plugin_update.py b/qgis-app/plugins/tests/test_plugin_update.py index 2bc5979a..bdb700e9 100644 --- a/qgis-app/plugins/tests/test_plugin_update.py +++ b/qgis-app/plugins/tests/test_plugin_update.py @@ -51,7 +51,7 @@ def setUp(self): self.plugin = Plugin.objects.get(name='Test Plugin') - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_plugin_new_version(self): """ @@ -103,7 +103,7 @@ def test_plugin_new_version(self): settings.EMAIL_HOST_USER ) - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_plugin_version_update(self): """ diff --git a/qgis-app/plugins/tests/test_plugin_upload.py b/qgis-app/plugins/tests/test_plugin_upload.py index 3539701a..9b3d2814 100644 --- a/qgis-app/plugins/tests/test_plugin_upload.py +++ b/qgis-app/plugins/tests/test_plugin_upload.py @@ -34,7 +34,7 @@ def setUp(self): email='test@example.com' ) - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_plugin_upload_form(self): # Log in the test user diff --git a/qgis-app/plugins/tests/test_rename_plugin.py b/qgis-app/plugins/tests/test_rename_plugin.py index d1eb890b..78ff7d69 100644 --- a/qgis-app/plugins/tests/test_rename_plugin.py +++ b/qgis-app/plugins/tests/test_rename_plugin.py @@ -51,7 +51,7 @@ def setUp(self): self.plugin.name = "New name Test Plugin" self.plugin.save() - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_plugin_rename(self): """ diff --git a/qgis-app/plugins/tests/test_validator.py b/qgis-app/plugins/tests/test_validator.py index 7a0e7e4e..6b1e9051 100644 --- a/qgis-app/plugins/tests/test_validator.py +++ b/qgis-app/plugins/tests/test_validator.py @@ -120,10 +120,7 @@ def test_check_url_link_ssl_error(self, mock_request): @mock.patch("requests.get", side_effect=requests.exceptions.HTTPError()) def test_check_url_link_does_not_exist(self, mock_request): url = "http://example.com/" - self.assertRaises( - ValidationError, - _check_url_link(url, "forbidden_url", "metadata attribute"), - ) + self.assertIsNone(_check_url_link(url, "forbidden_url", "metadata attribute")) class TestValidatorForbiddenFileFolder(TestCase): From ecf9d40a1a71c96ac09680ac40aed4f8698a6147 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Fri, 22 Mar 2024 10:10:13 +0300 Subject: [PATCH 18/32] Django 4 update: Add email environment variables --- dockerize/.env.template | 18 +++++++++++++++++- dockerize/docker-compose.yml | 6 ++++++ qgis-app/settings_docker.py | 3 +-- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/dockerize/.env.template b/dockerize/.env.template index 0a214a20..def0b727 100644 --- a/dockerize/.env.template +++ b/dockerize/.env.template @@ -1,9 +1,25 @@ +# RabbitMQ host RABBITMQ_HOST=rabbitmq + +# Database variables DATABASE_NAME=gis DATABASE_USERNAME=docker DATABASE_PASSWORD=docker DATABASE_HOST=db + +# Django settings DJANGO_SETTINGS_MODULE=settings_docker +DEBUG=False + +# Docker volumes QGISPLUGINS_STATIC_VOLUME=static-data QGISPLUGINS_MEDIA_VOLUME=media-data -QGISPLUGINS_BACKUP_VOLUME=backups-data \ No newline at end of file +QGISPLUGINS_BACKUP_VOLUME=backups-data + +# Email variables +EMAIL_BACKEND='django.core.mail.backends.smtp.EmailBackend' +EMAIL_HOST='' +EMAIL_PORT=587 +EMAIL_USE_TLS=True +EMAIL_HOST_USER='' +EMAIL_HOST_PASSWORD='' \ No newline at end of file diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 62f09c23..aa80d393 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -37,6 +37,12 @@ services: - DEBUG=${DEBUG:-False} - RABBITMQ_HOST=${RABBITMQ_HOST:-rabbitmq} - BROKER_URL=amqp://rabbitmq:5672 + - EMAIL_BACKEND=${EMAIL_BACKEND} + - EMAIL_HOST=${EMAIL_HOST} + - EMAIL_PORT=${EMAIL_PORT} + - EMAIL_USE_TLS=${EMAIL_USE_TLS} + - EMAIL_HOST_USER=${EMAIL_HOST_USER:-automation} + - EMAIL_HOST_PASSWORD=${EMAIL_HOST_PASSWORD} volumes: - ../qgis-app:/home/web/django_project - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:rw diff --git a/qgis-app/settings_docker.py b/qgis-app/settings_docker.py index 9fb535ee..968cc1cb 100644 --- a/qgis-app/settings_docker.py +++ b/qgis-app/settings_docker.py @@ -10,8 +10,7 @@ from datetime import timedelta from django.contrib.staticfiles.storage import ManifestStaticFilesStorage -# DEBUG = ast.literal_eval(os.environ.get("DEBUG", "True")) -DEBUG = True +DEBUG = ast.literal_eval(os.environ.get("DEBUG", "True")) THUMBNAIL_DEBUG = DEBUG ALLOWED_HOSTS = ["*"] From e488644e16d4df687a39ce61a1b640eab63e5010 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:20:20 +0300 Subject: [PATCH 19/32] Django 4 update: Nginx and uwsgi updates --- dockerize/docker-compose.yml | 4 ++++ dockerize/docker/uwsgi.conf | 2 +- qgis-app/wsgi.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index aa80d393..fffa954a 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -26,6 +26,8 @@ services: dockerfile: dockerize/docker/Dockerfile target: prod hostname: uwsgi + expose: + - "8080" environment: - DATABASE_NAME=${DATABASE_NAME:-gis} - DATABASE_USERNAME=${DATABASE_USERNAME:-docker} @@ -103,6 +105,8 @@ services: container_name: qgis-plugins-web image: nginx hostname: web + ports: + - "80:8080" volumes: - nginx-conf:/etc/nginx/conf.d:ro - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:ro diff --git a/dockerize/docker/uwsgi.conf b/dockerize/docker/uwsgi.conf index 50a80d0d..7af0f302 100644 --- a/dockerize/docker/uwsgi.conf +++ b/dockerize/docker/uwsgi.conf @@ -14,7 +14,7 @@ env = DJANGO_SETTINGS_MODULE=settings_docker #daemonize = /tmp/django.log req-logger = file:/var/log/uwsgi-requests.log logger = file:/var/log/uwsgi-errors.log -reload-os-env +# reload-os-env #uid = 1000 #gid = 1000 memory-report = true diff --git a/qgis-app/wsgi.py b/qgis-app/wsgi.py index d56c6515..71f8df99 100644 --- a/qgis-app/wsgi.py +++ b/qgis-app/wsgi.py @@ -4,7 +4,7 @@ It exposes the WSGI callable as a module-level variable named ``application``. For more information on this file, see -https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ +https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ """ import os From 26de6e13291fd44169299224db3f116d3eb13c74 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Fri, 22 Mar 2024 16:23:20 +0300 Subject: [PATCH 20/32] Django 4 update: Refactoring dockerfile --- dockerize/docker/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dockerize/docker/Dockerfile b/dockerize/docker/Dockerfile index 75dc5830..67edbf12 100644 --- a/dockerize/docker/Dockerfile +++ b/dockerize/docker/Dockerfile @@ -14,9 +14,9 @@ RUN apt-get update && apt-get install -y \ libsasl2-dev libldap2-dev libssl-dev \ libxslt1-dev zlib1g-dev \ build-essential \ - libffi-dev \ + libffi-dev gdal-bin\ libjpeg-dev libpq-dev \ - liblcms2-dev libblas-dev libatlas-base-dev + liblcms2-dev libblas-dev libatlas-base-dev \ RUN rm -rf /uwsgi.conf ADD dockerize/docker/uwsgi.conf /uwsgi.conf @@ -25,8 +25,6 @@ ADD dockerize/docker/REQUIREMENTS.txt /REQUIREMENTS.txt RUN pip install --upgrade pip && pip install -r /REQUIREMENTS.txt -RUN apt-get update && apt-get install -y gdal-bin - RUN mkdir -p /var/log/uwsgi WORKDIR /home/web/django_project From 1921119fc64f60cbc3aed08d444c49157c1b9ea5 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 27 Mar 2024 12:14:34 +0300 Subject: [PATCH 21/32] Update makefile to use new docker compose --- dockerize/Makefile | 58 +++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/dockerize/Makefile b/dockerize/Makefile index 47e89f4d..09aa208d 100644 --- a/dockerize/Makefile +++ b/dockerize/Makefile @@ -13,56 +13,56 @@ build: @echo "------------------------------------------------------------------" @echo "Building in production and development mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) build + @docker compose -p $(PROJECT_ID) build build-dev: @echo @echo "------------------------------------------------------------------" @echo "Building in development mode only" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) build devweb + @docker compose -p $(PROJECT_ID) build devweb db: @echo @echo "------------------------------------------------------------------" @echo "Running db in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up -d db + @docker compose -p $(PROJECT_ID) up -d db metabase: db @echo @echo "------------------------------------------------------------------" @echo "Running metabase in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up -d metabase + @docker compose -p $(PROJECT_ID) up -d metabase web: db @echo @echo "------------------------------------------------------------------" @echo "Running in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up -d uwsgi web worker beat + @docker compose -p $(PROJECT_ID) up -d uwsgi web worker beat devweb-test: db @echo @echo "------------------------------------------------------------------" @echo "Running in TESTING mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb + @docker compose -p $(PROJECT_ID) up --no-deps -d devweb devweb: db @echo @echo "------------------------------------------------------------------" @echo "Running in DEVELOPMENT mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb rabbitmq worker beat + @docker compose -p $(PROJECT_ID) up --no-deps -d devweb rabbitmq worker beat devweb-runserver: devweb @echo @echo "------------------------------------------------------------------" @echo "Running in DEVELOPMENT mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec devweb python manage.py runserver 0.0.0.0:8080 + @docker compose -p $(PROJECT_ID) exec devweb python manage.py runserver 0.0.0.0:8080 migrate: @echo @@ -75,22 +75,22 @@ migrate: @# We add the '-' prefix to the next line as the migration may fail @# but we want to continue anyway. @#We need to migrate accounts first as it has a reference to user model - -@docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py migrate auth - @docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py migrate + -@docker compose -p $(PROJECT_ID) exec uwsgi python manage.py migrate auth + @docker compose -p $(PROJECT_ID) exec uwsgi python manage.py migrate update-migrations: @echo @echo "------------------------------------------------------------------" @echo "Running update migrations in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py makemigrations + @docker compose -p $(PROJECT_ID) exec uwsgi python manage.py makemigrations collectstatic: @echo @echo "------------------------------------------------------------------" @echo "Collecting static in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) run uwsgi python manage.py collectstatic --noinput + @docker compose -p $(PROJECT_ID) run uwsgi python manage.py collectstatic --noinput #We need to run collect static in the same context as the running # uwsgi container it seems so I use docker exec here # no -it flag so we can run over remote shell @@ -110,7 +110,7 @@ kill: @echo "------------------------------------------------------------------" @echo "Killing in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) kill + @docker compose -p $(PROJECT_ID) kill rm: rm-only @@ -119,21 +119,21 @@ rm-only: kill @echo "------------------------------------------------------------------" @echo "Removing production instance!!! " @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) rm + @docker compose -p $(PROJECT_ID) rm maillogs: @echo @echo "------------------------------------------------------------------" @echo "Showing smtp logs in production mode" @echo "------------------------------------------------------------------" - @docker-compose exec smtp tail -f /var/log/mail.log + @docker compose exec smtp tail -f /var/log/mail.log mailerrorlogs: @echo @echo "------------------------------------------------------------------" @echo "Showing smtp error logs in production mode" @echo "------------------------------------------------------------------" - @docker-compose exec smtp tail -f /var/log/mail.err + @docker compose exec smtp tail -f /var/log/mail.err dbrestore: @echo @@ -142,36 +142,36 @@ dbrestore: @echo "------------------------------------------------------------------" @# - prefix causes command to continue even if it fails @echo "stopping uwsgi container" - @docker-compose -p $(PROJECT_ID) stop uwsgi + @docker compose -p $(PROJECT_ID) stop uwsgi @echo "Dropping the gis and metabase databases" - -@docker-compose -p $(PROJECT_ID) exec db su - postgres -c "dropdb --force gis" - -@docker-compose -p $(PROJECT_ID) exec db su - postgres -c "dropdb --force metabase" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "dropdb --force gis" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "dropdb --force metabase" @echo "Creating the gis and metabase databases" - -@docker-compose -p $(PROJECT_ID) exec db su - postgres -c "createdb -O docker -T template1 gis" - -@docker-compose -p $(PROJECT_ID) exec db su - postgres -c "createdb -O docker -T template1 metabase" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "createdb -O docker -T template1 gis" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "createdb -O docker -T template1 metabase" @echo "Restore database from backups/latest-gis.dmp" - -@docker-compose -p $(PROJECT_ID) exec db su - postgres -c "pg_restore -c /backups/latest-gis.dmp -d gis" - -@docker-compose -p $(PROJECT_ID) exec db su - postgres -c "pg_restore -c /backups/latest-metabase.dmp -d metabase" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "pg_restore -c /backups/latest-gis.dmp -d gis" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "pg_restore -c /backups/latest-metabase.dmp -d metabase" @echo "starting uwsgi container" - @docker-compose -p $(PROJECT_ID) up -d uwsgi + @docker compose -p $(PROJECT_ID) up -d uwsgi wait-db: - @docker-compose exec -p $(PROJECT_ID) db su - postgres -c "until pg_isready; do sleep 5; done" + @docker compose exec -p $(PROJECT_ID) db su - postgres -c "until pg_isready; do sleep 5; done" create-test-db: - @docker-compose exec -p $(PROJECT_ID) db su - postgres -c "psql -c 'create database test_db;'" - @docker-compose exec -p $(PROJECT_ID) db su - postgres -c "psql -d test_db -c 'create extension postgis;'" + @docker compose exec -p $(PROJECT_ID) db su - postgres -c "psql -c 'create database test_db;'" + @docker compose exec -p $(PROJECT_ID) db su - postgres -c "psql -d test_db -c 'create extension postgis;'" dbseed: @echo @echo "------------------------------------------------------------------" @echo "Seed db with JSON data from /fixtures/*.json" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec devweb bash -c 'python manage.py loaddata fixtures/*.json' + @docker compose -p $(PROJECT_ID) exec devweb bash -c 'python manage.py loaddata fixtures/*.json' rebuild_index: @echo @echo "------------------------------------------------------------------" @echo "Rebuild search index in PRODUCTION mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec uwsgi bash -c 'python manage.py rebuild_index' + @docker compose -p $(PROJECT_ID) exec uwsgi bash -c 'python manage.py rebuild_index' From c1124318fa1460d3cb1971b75c9cf327888fbc22 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 27 Mar 2024 14:18:41 +0300 Subject: [PATCH 22/32] Django 4 update: Specify devweb container_name, fix typo --- dockerize/docker-compose.yml | 1 + dockerize/docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index fffa954a..48e6198e 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -61,6 +61,7 @@ services: # from prod environment if wanted devweb: <<: *uwsgi-common + container_name: qgis-plugins-devweb build: context: ${PWD}/../ dockerfile: dockerize/docker/Dockerfile diff --git a/dockerize/docker/Dockerfile b/dockerize/docker/Dockerfile index 67edbf12..dd51c53b 100644 --- a/dockerize/docker/Dockerfile +++ b/dockerize/docker/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y \ build-essential \ libffi-dev gdal-bin\ libjpeg-dev libpq-dev \ - liblcms2-dev libblas-dev libatlas-base-dev \ + liblcms2-dev libblas-dev libatlas-base-dev RUN rm -rf /uwsgi.conf ADD dockerize/docker/uwsgi.conf /uwsgi.conf From 4466e94e49e11027064829464443a304f35bbabd Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 27 Mar 2024 14:34:25 +0300 Subject: [PATCH 23/32] Use existing nginx configuration --- dockerize/docker-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 48e6198e..e3e4d329 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -1,7 +1,6 @@ version: '3.8' volumes: postgres_data: - nginx-conf: rabbitmq: celerybeat-schedule: services: @@ -109,7 +108,7 @@ services: ports: - "80:8080" volumes: - - nginx-conf:/etc/nginx/conf.d:ro + - ./sites-enabled:/etc/nginx/conf.d:ro - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:ro - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:ro links: From d5483ddae6c3698f84d1be9f43ca2e396cd6a236 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Thu, 28 Mar 2024 10:59:19 +0300 Subject: [PATCH 24/32] Add DEFAULT_PLUGINS_SITE to environment variables --- dockerize/.env.template | 5 ++++- dockerize/docker-compose.yml | 1 + qgis-app/plugins/management/commands/generate_plugins_xml.py | 4 ++-- qgis-app/settings_docker.py | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dockerize/.env.template b/dockerize/.env.template index def0b727..8751412a 100644 --- a/dockerize/.env.template +++ b/dockerize/.env.template @@ -22,4 +22,7 @@ EMAIL_HOST='' EMAIL_PORT=587 EMAIL_USE_TLS=True EMAIL_HOST_USER='' -EMAIL_HOST_PASSWORD='' \ No newline at end of file +EMAIL_HOST_PASSWORD='' + +# URL +DEFAULT_PLUGINS_SITE='https://plugins.qgis.org/' \ No newline at end of file diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index e3e4d329..a926a223 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -44,6 +44,7 @@ services: - EMAIL_USE_TLS=${EMAIL_USE_TLS} - EMAIL_HOST_USER=${EMAIL_HOST_USER:-automation} - EMAIL_HOST_PASSWORD=${EMAIL_HOST_PASSWORD} + - DEFAULT_PLUGINS_SITE=${DEFAULT_PLUGINS_SITE:-https://plugins.qgis.org/} volumes: - ../qgis-app:/home/web/django_project - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:rw diff --git a/qgis-app/plugins/management/commands/generate_plugins_xml.py b/qgis-app/plugins/management/commands/generate_plugins_xml.py index 4efd70e0..1795dcf9 100644 --- a/qgis-app/plugins/management/commands/generate_plugins_xml.py +++ b/qgis-app/plugins/management/commands/generate_plugins_xml.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from django.core.management.base import BaseCommand from plugins.tasks.generate_plugins_xml import generate_plugins_xml - +from django.conf import settings class Command(BaseCommand): @@ -12,7 +12,7 @@ def add_arguments(self, parser): "-s", "--site", dest="site", - default="http://plugins.qgis.org", + default=settings.DEFAULT_PLUGINS_SITE, help="Site url to get the source of plugins", ) diff --git a/qgis-app/settings_docker.py b/qgis-app/settings_docker.py index 968cc1cb..c0b7be1e 100644 --- a/qgis-app/settings_docker.py +++ b/qgis-app/settings_docker.py @@ -109,7 +109,7 @@ PAGINATION_DEFAULT_PAGINATION_HUB = 30 LOGIN_REDIRECT_URL = "/" SERVE_STATIC_MEDIA = DEBUG -DEFAULT_PLUGINS_SITE = os.environ.get("DEFAULT_PLUGINS_SITE", "") +DEFAULT_PLUGINS_SITE = os.environ.get("DEFAULT_PLUGINS_SITE", "https://plugins.qgis.org/") # See fig.yml file for postfix container definition # @@ -141,7 +141,7 @@ 'task': 'plugins.tasks.generate_plugins_xml.generate_plugins_xml', 'schedule': crontab(minute='*/10'), # Execute every 10 minutes. 'kwargs': { - 'site': 'https://plugins.qgis.org/' + 'site': DEFAULT_PLUGINS_SITE } }, 'update_feedjack': { From ce76edc00ba18ea86b7bd715a7a9155efa82543e Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Tue, 2 Apr 2024 08:40:29 +0300 Subject: [PATCH 25/32] Add nginx dev and prod configuration files --- dockerize/Makefile | 7 ++ dockerize/docker-compose.yml | 10 ++- dockerize/sites-enabled/dev.conf | 31 +++++++ dockerize/sites-enabled/docker-entrypoint.sh | 28 ++++++ dockerize/sites-enabled/prod.conf | 91 ++++++++++++++++++++ 5 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 dockerize/sites-enabled/dev.conf create mode 100755 dockerize/sites-enabled/docker-entrypoint.sh create mode 100644 dockerize/sites-enabled/prod.conf diff --git a/dockerize/Makefile b/dockerize/Makefile index 09aa208d..b5da7d75 100644 --- a/dockerize/Makefile +++ b/dockerize/Makefile @@ -43,6 +43,13 @@ web: db @echo "------------------------------------------------------------------" @docker compose -p $(PROJECT_ID) up -d uwsgi web worker beat +dbbackups: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running dbbackups in production mode" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) up -d dbbackups + devweb-test: db @echo @echo "------------------------------------------------------------------" diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index a926a223..d03821e9 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -106,10 +106,12 @@ services: container_name: qgis-plugins-web image: nginx hostname: web + entrypoint: + - /etc/nginx/sites-available/docker-entrypoint.sh ports: - - "80:8080" + - "80:80" volumes: - - ./sites-enabled:/etc/nginx/conf.d:ro + - ./sites-enabled:/etc/nginx/sites-available/:ro - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:ro - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:ro links: @@ -121,6 +123,8 @@ services: max-size: "200k" max-file: "10" restart: unless-stopped + command: + - prod dbbackups: image: kartoza/pg-backup:16-3.4 @@ -147,3 +151,5 @@ services: - MB_DB_CONNECTION_URI=jdbc:postgresql://${DATABASE_HOST:-db}:5432/metabase?user=${DATABASE_USERNAME:-docker}&password=${DATABASE_PASSWORD:-docker} links: - db + expose: + - "3000" diff --git a/dockerize/sites-enabled/dev.conf b/dockerize/sites-enabled/dev.conf new file mode 100644 index 00000000..ef93e13c --- /dev/null +++ b/dockerize/sites-enabled/dev.conf @@ -0,0 +1,31 @@ +# Define connection details for connecting to django running in +# a docker container. +upstream uwsgi { + server uwsgi:8080; +} +server { + # OTF gzip compression + gzip on; + gzip_min_length 860; + gzip_comp_level 5; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; + gzip_disable “MSIE [1-6].(?!.*SV1)”; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + # the port your site will be served on + listen 80; + # the domain name it will serve for + server_name ""; + charset utf-8; + + # Send all requests to the Django server. + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://uwsgi; + } +} diff --git a/dockerize/sites-enabled/docker-entrypoint.sh b/dockerize/sites-enabled/docker-entrypoint.sh new file mode 100755 index 00000000..61813a88 --- /dev/null +++ b/dockerize/sites-enabled/docker-entrypoint.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Clean up sites-enabled +echo "Clean sites-enabled" +rm -rf /etc/nginx/conf.d/*.conf +mkdir -p /etc/nginx/conf.d + +if [ $# -eq 1 ]; then + case $1 in + # Debug mode, enable dev.conf + [Dd][Ee][Bb][Uu][Gg]) + echo "Run in debug mode" + CONF_FILE=dev.conf + ln -s /etc/nginx/sites-available/$CONF_FILE /etc/nginx/conf.d/$CONF_FILE + exec nginx -g "daemon off;" + ;; + # Production mode, run using uwsgi + [Pp][Rr][Oo][Dd]) + echo "Run in prod mode" + CONF_FILE=prod.conf + ln -s /etc/nginx/sites-available/$CONF_FILE /etc/nginx/conf.d/$CONF_FILE + exec nginx -g "daemon off;" + ;; + esac +fi + +# Run as bash entrypoint +exec "$@" diff --git a/dockerize/sites-enabled/prod.conf b/dockerize/sites-enabled/prod.conf new file mode 100644 index 00000000..134b3d18 --- /dev/null +++ b/dockerize/sites-enabled/prod.conf @@ -0,0 +1,91 @@ +# Define connection details for connecting to django running in +# a docker container. +upstream uwsgi { + server uwsgi:8080; +} +server { + # OTF gzip compression + gzip on; + gzip_min_length 860; + gzip_comp_level 5; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; + gzip_disable “MSIE [1-6].(?!.*SV1)”; + + access_log /var/log/nginx/access.log; + + client_max_body_size 20M; + error_log /var/log/nginx/error.log; + + # the port your site will be served on + listen 80; + # the domain name it will serve for + server_name ""; + + charset utf-8; + + # Drop any non django related requests + # Its probably someone nefarious probing for vulnerabilities... + location ~ (\.php|\.asp|myadmin) { + return 404; + } + + # Django media + location /media { + # your Django project's media files - amend as required + alias /home/web/media; + expires 21d; # cache for 71 days + } + location /static { + # your Django project's static files - amend as required + alias /home/web/static; + expires 21d; # cache for 21 days + } + location /plugins/plugins.xml { + if ($request_uri !~ "&package_name(.*)") { + rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; + root /home; + expires 600s; + } + } + location /archive { + # Changed from http_host to host because of error messages when + # bots hit urls like this: + # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', + # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ + # for more details. + #proxy_set_header Host $http_host; + proxy_set_header Host $host; + autoindex on; + # your Django project's static files - amend as required + alias /home/web/archive; + expires 21d; # cache for 6h + } + # Finally, send all non-media requests to the Django server. + location / { + uwsgi_pass uwsgi; + # the uwsgi_params file you installed needs to be passed with each + # request. + # the uwsgi_params need to be passed with each uwsgi request + uwsgi_param QUERY_STRING $query_string; + uwsgi_param REQUEST_METHOD $request_method; + uwsgi_param CONTENT_TYPE $content_type; + uwsgi_param CONTENT_LENGTH $content_length; + + uwsgi_param REQUEST_URI $request_uri; + uwsgi_param PATH_INFO $document_uri; + uwsgi_param DOCUMENT_ROOT $document_root; + uwsgi_param SERVER_PROTOCOL $server_protocol; + uwsgi_param HTTPS $https if_not_empty; + + uwsgi_param REMOTE_ADDR $remote_addr; + uwsgi_param REMOTE_PORT $remote_port; + uwsgi_param SERVER_PORT $server_port; + uwsgi_param SERVER_NAME $server_name; + } + + location /metabase/ { + # set to webroot path + proxy_pass http://metabase:3000/; + } +} From 9abe60ffbaa2a2284652ebf50ed8254b92ef3447 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Tue, 2 Apr 2024 11:59:39 +0300 Subject: [PATCH 26/32] Django 4 update: Generate a .env file in GH actions --- .github/workflows/test.yaml | 3 +++ dockerize/docker-compose.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8166e470..eefe56db 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -47,6 +47,9 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Generate the .env file + run: cp .env.template .env + - name: Run docker-compose build run: docker-compose build devweb diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index d03821e9..d0d50fa9 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -3,6 +3,9 @@ volumes: postgres_data: rabbitmq: celerybeat-schedule: + static-data: + media-data: + backups-data: services: db: From d6a7b2beb5a43e6a71ce47577986f470248502af Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Tue, 2 Apr 2024 17:39:56 +0300 Subject: [PATCH 27/32] Django 4 update: Fix typo in Makefile --- dockerize/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dockerize/Makefile b/dockerize/Makefile index b5da7d75..19c060cf 100644 --- a/dockerize/Makefile +++ b/dockerize/Makefile @@ -55,7 +55,7 @@ devweb-test: db @echo "------------------------------------------------------------------" @echo "Running in TESTING mode" @echo "------------------------------------------------------------------" - @docker compose -p $(PROJECT_ID) up --no-deps -d devweb + @docker compose up --no-deps -d devweb devweb: db @echo @@ -163,11 +163,11 @@ dbrestore: @docker compose -p $(PROJECT_ID) up -d uwsgi wait-db: - @docker compose exec -p $(PROJECT_ID) db su - postgres -c "until pg_isready; do sleep 5; done" + @docker compose -p $(PROJECT_ID) exec db su - postgres -c "until pg_isready; do sleep 5; done" create-test-db: - @docker compose exec -p $(PROJECT_ID) db su - postgres -c "psql -c 'create database test_db;'" - @docker compose exec -p $(PROJECT_ID) db su - postgres -c "psql -d test_db -c 'create extension postgis;'" + @docker compose -p $(PROJECT_ID) exec db su - postgres -c "psql -c 'create database test_db;'" + @docker compose -p $(PROJECT_ID) exec db su - postgres -c "psql -d test_db -c 'create extension postgis;'" dbseed: @echo From fe4dd07f25d68bb4603010fbcc3f7b0844a5c0ec Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Tue, 2 Apr 2024 17:52:17 +0300 Subject: [PATCH 28/32] Django 4 update: Use updated docker compose in test.yaml --- .github/workflows/test.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index eefe56db..498e37be 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -50,10 +50,10 @@ jobs: - name: Generate the .env file run: cp .env.template .env - - name: Run docker-compose build - run: docker-compose build devweb + - name: Run docker compose build + run: docker compose build devweb - - name: Run docker-compose services + - name: Run docker compose services working-directory: dockerize run: | cp docker-compose.override.test.yml docker-compose.override.yml @@ -63,11 +63,10 @@ jobs: - name: Run Coverage test run: | - cat << EOF | docker-compose -p qgis-plugins exec -T devweb bash + cat << EOF | docker compose exec -T devweb bash pip install coverage python manage.py makemigrations python manage.py migrate - python manage.py collectstatic --noinput --verbosity 0 coverage run manage.py test coverage xml EOF From 0529d52bf5227a8a193cc3629ea2b2b4a2b3a5d1 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Fri, 5 Apr 2024 15:29:46 +0300 Subject: [PATCH 29/32] Django 4 update: Certbot service and SSL configuration --- .gitignore | 2 + INSTALL.md | 2 + dockerize/.env.template | 5 +- dockerize/Makefile | 7 + dockerize/docker-compose.yml | 15 +- dockerize/sites-enabled/default.conf | 94 --------- dockerize/sites-enabled/docker-entrypoint.sh | 2 +- dockerize/sites-enabled/prod-ssl.conf | 210 +++++++++++++++++++ dockerize/sites-enabled/prod.conf | 91 -------- 9 files changed, 240 insertions(+), 188 deletions(-) delete mode 100644 dockerize/sites-enabled/default.conf create mode 100644 dockerize/sites-enabled/prod-ssl.conf delete mode 100644 dockerize/sites-enabled/prod.conf diff --git a/.gitignore b/.gitignore index 073c345f..38e1880e 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,8 @@ static_media dockerize/static dockerize/media dockerize/logs +dockerize/certbot-etc +dockerize/webroot .editorconfig # test cache diff --git a/INSTALL.md b/INSTALL.md index 1a66d939..3a2f7421 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -196,6 +196,8 @@ $ make dbrestore ## Deploy Update with Rancher +**TO BE DEPRECATED** + - Go to [https://rancher.qgis.org](https://rancher.qgis.org) - Click the home icon in the top left corner, and choose the environment: `plugins.qgis.org` - You will be redirected to User Stacks page, choose the plugins stack by clicking on `plugins` link diff --git a/dockerize/.env.template b/dockerize/.env.template index 8751412a..23981fe9 100644 --- a/dockerize/.env.template +++ b/dockerize/.env.template @@ -25,4 +25,7 @@ EMAIL_HOST_USER='' EMAIL_HOST_PASSWORD='' # URL -DEFAULT_PLUGINS_SITE='https://plugins.qgis.org/' \ No newline at end of file +DEFAULT_PLUGINS_SITE='https://plugins.qgis.org/' + +# ENV: debug or prod +QGISPLUGINS_ENV=debug \ No newline at end of file diff --git a/dockerize/Makefile b/dockerize/Makefile index 19c060cf..417a360b 100644 --- a/dockerize/Makefile +++ b/dockerize/Makefile @@ -50,6 +50,13 @@ dbbackups: db @echo "------------------------------------------------------------------" @docker compose -p $(PROJECT_ID) up -d dbbackups +certbot: web + @echo + @echo "------------------------------------------------------------------" + @echo "Running cerbot in production mode" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) up -d certbot + devweb-test: db @echo @echo "------------------------------------------------------------------" diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index d0d50fa9..0f4f4dcf 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -113,10 +113,13 @@ services: - /etc/nginx/sites-available/docker-entrypoint.sh ports: - "80:80" + - "443:443" volumes: - ./sites-enabled:/etc/nginx/sites-available/:ro - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:ro - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:ro + - ./webroot:/var/www/webroot + - ./certbot-etc:/etc/letsencrypt links: - uwsgi:uwsgi - metabase:metabase @@ -127,7 +130,7 @@ services: max-file: "10" restart: unless-stopped command: - - prod + - ${QGISPLUGINS_ENV} dbbackups: image: kartoza/pg-backup:16-3.4 @@ -156,3 +159,13 @@ services: - db expose: - "3000" + + certbot: + image: certbot/certbot + container_name: certbot + volumes: + - ./webroot:/var/www/webroot + - ./certbot-etc:/etc/letsencrypt + depends_on: + - web + command: certonly --webroot --webroot-path=/var/www/webroot --email admin@qgis.org --agree-tos --no-eff-email --force-renewal -d plugins.qgis.org \ No newline at end of file diff --git a/dockerize/sites-enabled/default.conf b/dockerize/sites-enabled/default.conf deleted file mode 100644 index 8cf7cec7..00000000 --- a/dockerize/sites-enabled/default.conf +++ /dev/null @@ -1,94 +0,0 @@ -# Define connection details for connecting to django running in -# a docker container. -upstream uwsgi { - server uwsgi:8080; -} -server { - # OTF gzip compression - gzip on; - gzip_min_length 860; - gzip_comp_level 5; - gzip_proxied expired no-cache no-store private auth; - gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; - gzip_disable “MSIE [1-6].(?!.*SV1)”; - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - # the port your site will be served on - listen 8080; - # the domain name it will serve for - server_name ""; - charset utf-8; - - # Drop any non django related requests - # Its probably someone nefarious probing for vulnerabilities... - location ~ (\.php|.aspx|.asp|myadmin) { - return 404; - } - # max upload size, adjust to taste - client_max_body_size 15M; - - - # Django media - location /media { - # your Django project's media files - amend as required - alias /home/web/media; - expires 21d; # cache for 71 days - } - location /static { - # your Django project's static files - amend as required - alias /home/web/static; - expires 21d; # cache for 21 days - } - location /archive { - # Changed from http_host to host because of error messages when - # bots hit urls like this: - # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', - # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ - # for more details. - #proxy_set_header Host $http_host; - proxy_set_header Host $host; - autoindex on; - # your Django project's static files - amend as required - alias /home/web/archive; - expires 21d; # cache for 6h - } - - location /plugins/plugins.xml { - if ($request_uri !~ "&package_name(.*)") { - rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; - root /home; - expires 600s; - } - } - - # Finally, send all non-media requests to the Django server. - location / { - uwsgi_pass uwsgi; - # the uwsgi_params file you installed needs to be passed with each - # request. - # the uwsgi_params need to be passed with each uwsgi request - uwsgi_param QUERY_STRING $query_string; - uwsgi_param REQUEST_METHOD $request_method; - uwsgi_param CONTENT_TYPE $content_type; - uwsgi_param CONTENT_LENGTH $content_length; - - uwsgi_param REQUEST_URI $request_uri; - uwsgi_param PATH_INFO $document_uri; - uwsgi_param DOCUMENT_ROOT $document_root; - uwsgi_param SERVER_PROTOCOL $server_protocol; - uwsgi_param HTTPS $https if_not_empty; - - uwsgi_param REMOTE_ADDR $remote_addr; - uwsgi_param REMOTE_PORT $remote_port; - uwsgi_param SERVER_PORT $server_port; - uwsgi_param SERVER_NAME $server_name; - } - - - location /metabase/ { - # set to webroot path - proxy_pass http://metabase:3000/; - } -} diff --git a/dockerize/sites-enabled/docker-entrypoint.sh b/dockerize/sites-enabled/docker-entrypoint.sh index 61813a88..b4089f01 100755 --- a/dockerize/sites-enabled/docker-entrypoint.sh +++ b/dockerize/sites-enabled/docker-entrypoint.sh @@ -17,7 +17,7 @@ if [ $# -eq 1 ]; then # Production mode, run using uwsgi [Pp][Rr][Oo][Dd]) echo "Run in prod mode" - CONF_FILE=prod.conf + CONF_FILE=prod-ssl.conf ln -s /etc/nginx/sites-available/$CONF_FILE /etc/nginx/conf.d/$CONF_FILE exec nginx -g "daemon off;" ;; diff --git a/dockerize/sites-enabled/prod-ssl.conf b/dockerize/sites-enabled/prod-ssl.conf new file mode 100644 index 00000000..c8100ef6 --- /dev/null +++ b/dockerize/sites-enabled/prod-ssl.conf @@ -0,0 +1,210 @@ +# Define connection details for connecting to django running in +# a docker container. +upstream uwsgi { + server uwsgi:8080; +} + +server { + # OTF gzip compression + gzip on; + gzip_min_length 860; + gzip_comp_level 5; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; + gzip_disable “MSIE [1-6].(?!.*SV1)”; + + access_log /var/log/nginx/access.log; + + client_max_body_size 20M; + error_log /var/log/nginx/error.log; + + # the port your site will be served on + listen 80; + # the domain name it will serve for + server_name ""; + + charset utf-8; + + # Drop any non django related requests + # Its probably someone nefarious probing for vulnerabilities... + location ~ (\.php|\.asp|myadmin) { + return 404; + } + + # Django media + location /media { + # your Django project's media files - amend as required + alias /home/web/media; + expires 21d; # cache for 71 days + } + location /static { + # your Django project's static files - amend as required + alias /home/web/static; + expires 21d; # cache for 21 days + } + location /plugins/plugins.xml { + if ($request_uri !~ "&package_name(.*)") { + rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; + root /home; + expires 600s; + } + } + location /archive { + # Changed from http_host to host because of error messages when + # bots hit urls like this: + # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', + # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ + # for more details. + #proxy_set_header Host $http_host; + proxy_set_header Host $host; + autoindex on; + # your Django project's static files - amend as required + alias /home/web/archive; + expires 21d; # cache for 6h + } + # Finally, send all non-media requests to the Django server. + location / { + uwsgi_pass uwsgi; + # the uwsgi_params file you installed needs to be passed with each + # request. + # the uwsgi_params need to be passed with each uwsgi request + uwsgi_param QUERY_STRING $query_string; + uwsgi_param REQUEST_METHOD $request_method; + uwsgi_param CONTENT_TYPE $content_type; + uwsgi_param CONTENT_LENGTH $content_length; + + uwsgi_param REQUEST_URI $request_uri; + uwsgi_param PATH_INFO $document_uri; + uwsgi_param DOCUMENT_ROOT $document_root; + uwsgi_param SERVER_PROTOCOL $server_protocol; + uwsgi_param HTTPS $https if_not_empty; + + uwsgi_param REMOTE_ADDR $remote_addr; + uwsgi_param REMOTE_PORT $remote_port; + uwsgi_param SERVER_PORT $server_port; + uwsgi_param SERVER_NAME $server_name; + } + + location /metabase/ { + # set to webroot path + proxy_pass http://metabase:3000/; + } + + location ~ /.well-known/acme-challenge { + # set to webroot path + root /var/www/webroot; + default_type "text/plain"; + allow all; + } +} + + +server { + # SSL Cert + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name plugins.qgis.org; + + server_tokens off; + + ssl_certificate /etc/letsencrypt/live/plugins.qgis.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/plugins.qgis.org/privkey.pem; + + ssl_buffer_size 8k; + + # ssl_dhparam /etc/ssl/certs/dhparam-2048.pem; + + ssl_protocols TLSv1.2 TLSv1.1 TLSv1; + ssl_prefer_server_ciphers on; + + ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DH+3DES:!ADH:!AECDH:!MD5; + + ssl_ecdh_curve secp384r1; + ssl_session_tickets off; + + ssl_stapling on; + ssl_stapling_verify on; + resolver 8.8.8.8; + + # OTF gzip compression + gzip on; + gzip_min_length 860; + gzip_comp_level 5; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; + gzip_disable “MSIE [1-6].(?!.*SV1)”; + + access_log /var/log/nginx/access.log; + + client_max_body_size 20M; + error_log /var/log/nginx/error.log; + + charset utf-8; + + # Drop any non django related requests + # Its probably someone nefarious probing for vulnerabilities... + location ~ (\.php|\.asp|myadmin) { + return 404; + } + + # Django media + location /media { + # your Django project's media files - amend as required + alias /home/web/media; + expires 21d; # cache for 71 days + } + location /static { + # your Django project's static files - amend as required + alias /home/web/static; + expires 21d; # cache for 21 days + } + location /plugins/plugins.xml { + if ($request_uri !~ "&package_name(.*)") { + rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; + root /home; + expires 600s; + } + } + location /archive { + # Changed from http_host to host because of error messages when + # bots hit urls like this: + # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', + # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ + # for more details. + #proxy_set_header Host $http_host; + proxy_set_header Host $host; + autoindex on; + # your Django project's static files - amend as required + alias /home/web/archive; + expires 21d; # cache for 6h + } + # Finally, send all non-media requests to the Django server. + location / { + uwsgi_pass uwsgi; + # the uwsgi_params file you installed needs to be passed with each + # request. + # the uwsgi_params need to be passed with each uwsgi request + uwsgi_param QUERY_STRING $query_string; + uwsgi_param REQUEST_METHOD $request_method; + uwsgi_param CONTENT_TYPE $content_type; + uwsgi_param CONTENT_LENGTH $content_length; + + uwsgi_param REQUEST_URI $request_uri; + uwsgi_param PATH_INFO $document_uri; + uwsgi_param DOCUMENT_ROOT $document_root; + uwsgi_param SERVER_PROTOCOL $server_protocol; + uwsgi_param HTTPS $https if_not_empty; + + uwsgi_param REMOTE_ADDR $remote_addr; + uwsgi_param REMOTE_PORT $remote_port; + uwsgi_param SERVER_PORT $server_port; + uwsgi_param SERVER_NAME $server_name; + } + + location /metabase/ { + # set to webroot path + proxy_pass http://metabase:3000/; + } + + root /var/www/webroot; +} \ No newline at end of file diff --git a/dockerize/sites-enabled/prod.conf b/dockerize/sites-enabled/prod.conf deleted file mode 100644 index 134b3d18..00000000 --- a/dockerize/sites-enabled/prod.conf +++ /dev/null @@ -1,91 +0,0 @@ -# Define connection details for connecting to django running in -# a docker container. -upstream uwsgi { - server uwsgi:8080; -} -server { - # OTF gzip compression - gzip on; - gzip_min_length 860; - gzip_comp_level 5; - gzip_proxied expired no-cache no-store private auth; - gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; - gzip_disable “MSIE [1-6].(?!.*SV1)”; - - access_log /var/log/nginx/access.log; - - client_max_body_size 20M; - error_log /var/log/nginx/error.log; - - # the port your site will be served on - listen 80; - # the domain name it will serve for - server_name ""; - - charset utf-8; - - # Drop any non django related requests - # Its probably someone nefarious probing for vulnerabilities... - location ~ (\.php|\.asp|myadmin) { - return 404; - } - - # Django media - location /media { - # your Django project's media files - amend as required - alias /home/web/media; - expires 21d; # cache for 71 days - } - location /static { - # your Django project's static files - amend as required - alias /home/web/static; - expires 21d; # cache for 21 days - } - location /plugins/plugins.xml { - if ($request_uri !~ "&package_name(.*)") { - rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; - root /home; - expires 600s; - } - } - location /archive { - # Changed from http_host to host because of error messages when - # bots hit urls like this: - # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', - # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ - # for more details. - #proxy_set_header Host $http_host; - proxy_set_header Host $host; - autoindex on; - # your Django project's static files - amend as required - alias /home/web/archive; - expires 21d; # cache for 6h - } - # Finally, send all non-media requests to the Django server. - location / { - uwsgi_pass uwsgi; - # the uwsgi_params file you installed needs to be passed with each - # request. - # the uwsgi_params need to be passed with each uwsgi request - uwsgi_param QUERY_STRING $query_string; - uwsgi_param REQUEST_METHOD $request_method; - uwsgi_param CONTENT_TYPE $content_type; - uwsgi_param CONTENT_LENGTH $content_length; - - uwsgi_param REQUEST_URI $request_uri; - uwsgi_param PATH_INFO $document_uri; - uwsgi_param DOCUMENT_ROOT $document_root; - uwsgi_param SERVER_PROTOCOL $server_protocol; - uwsgi_param HTTPS $https if_not_empty; - - uwsgi_param REMOTE_ADDR $remote_addr; - uwsgi_param REMOTE_PORT $remote_port; - uwsgi_param SERVER_PORT $server_port; - uwsgi_param SERVER_NAME $server_name; - } - - location /metabase/ { - # set to webroot path - proxy_pass http://metabase:3000/; - } -} From b66044927354440dcbf181a514de467ba8ae1477 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Tue, 16 Apr 2024 16:16:53 +0300 Subject: [PATCH 30/32] Add a http configuration for Nginx --- dockerize/sites-enabled/docker-entrypoint.sh | 7 ++ dockerize/sites-enabled/prod.conf | 99 ++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 dockerize/sites-enabled/prod.conf diff --git a/dockerize/sites-enabled/docker-entrypoint.sh b/dockerize/sites-enabled/docker-entrypoint.sh index b4089f01..fbc27084 100755 --- a/dockerize/sites-enabled/docker-entrypoint.sh +++ b/dockerize/sites-enabled/docker-entrypoint.sh @@ -17,6 +17,13 @@ if [ $# -eq 1 ]; then # Production mode, run using uwsgi [Pp][Rr][Oo][Dd]) echo "Run in prod mode" + CONF_FILE=prod.conf + ln -s /etc/nginx/sites-available/$CONF_FILE /etc/nginx/conf.d/$CONF_FILE + exec nginx -g "daemon off;" + ;; + # Production SSL mode, run using uwsgi + [Pp][Rr][Oo][Dd][-][Ss][Ss][Ll]) + echo "Run in prod SSL mode" CONF_FILE=prod-ssl.conf ln -s /etc/nginx/sites-available/$CONF_FILE /etc/nginx/conf.d/$CONF_FILE exec nginx -g "daemon off;" diff --git a/dockerize/sites-enabled/prod.conf b/dockerize/sites-enabled/prod.conf new file mode 100644 index 00000000..84f9d8aa --- /dev/null +++ b/dockerize/sites-enabled/prod.conf @@ -0,0 +1,99 @@ +# Define connection details for connecting to django running in +# a docker container. +upstream uwsgi { + server uwsgi:8080; +} + +server { + # OTF gzip compression + gzip on; + gzip_min_length 860; + gzip_comp_level 5; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; + gzip_disable “MSIE [1-6].(?!.*SV1)”; + + access_log /var/log/nginx/access.log; + + client_max_body_size 20M; + error_log /var/log/nginx/error.log; + + # the port your site will be served on + listen 80; + # the domain name it will serve for + server_name ""; + + charset utf-8; + + # Drop any non django related requests + # Its probably someone nefarious probing for vulnerabilities... + location ~ (\.php|\.asp|myadmin) { + return 404; + } + + # Django media + location /media { + # your Django project's media files - amend as required + alias /home/web/media; + expires 21d; # cache for 71 days + } + location /static { + # your Django project's static files - amend as required + alias /home/web/static; + expires 21d; # cache for 21 days + } + location /plugins/plugins.xml { + if ($request_uri !~ "&package_name(.*)") { + rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; + root /home; + expires 600s; + } + } + location /archive { + # Changed from http_host to host because of error messages when + # bots hit urls like this: + # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', + # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ + # for more details. + #proxy_set_header Host $http_host; + proxy_set_header Host $host; + autoindex on; + # your Django project's static files - amend as required + alias /home/web/archive; + expires 21d; # cache for 6h + } + # Finally, send all non-media requests to the Django server. + location / { + uwsgi_pass uwsgi; + # the uwsgi_params file you installed needs to be passed with each + # request. + # the uwsgi_params need to be passed with each uwsgi request + uwsgi_param QUERY_STRING $query_string; + uwsgi_param REQUEST_METHOD $request_method; + uwsgi_param CONTENT_TYPE $content_type; + uwsgi_param CONTENT_LENGTH $content_length; + + uwsgi_param REQUEST_URI $request_uri; + uwsgi_param PATH_INFO $document_uri; + uwsgi_param DOCUMENT_ROOT $document_root; + uwsgi_param SERVER_PROTOCOL $server_protocol; + uwsgi_param HTTPS $https if_not_empty; + + uwsgi_param REMOTE_ADDR $remote_addr; + uwsgi_param REMOTE_PORT $remote_port; + uwsgi_param SERVER_PORT $server_port; + uwsgi_param SERVER_NAME $server_name; + } + + location /metabase/ { + # set to webroot path + proxy_pass http://metabase:3000/; + } + + location ~ /.well-known/acme-challenge { + # set to webroot path + root /var/www/webroot; + default_type "text/plain"; + allow all; + } +} From 68fc44ec2e66c37dad5e47703f4c2ac059b9b220 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Tue, 23 Apr 2024 10:10:21 +0300 Subject: [PATCH 31/32] Django 4 update: add SSL cert renewal script --- dockerize/scripts/renew-ssl.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 dockerize/scripts/renew-ssl.sh diff --git a/dockerize/scripts/renew-ssl.sh b/dockerize/scripts/renew-ssl.sh new file mode 100644 index 00000000..7004ead6 --- /dev/null +++ b/dockerize/scripts/renew-ssl.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + + +# Run daily on crontab e.g. +# Your cron job will be run at: (5 times displayed) +# +# 2021-11-08 11:10:00 UTC +# 2021-11-09 11:10:00 UTC +# 2021-11-10 11:10:00 UTC +# 2021-11-11 11:10:00 UTC +# 2021-11-12 11:10:00 UTC +# ...etc + +#25 11 * * * /bin/bash /home/web/QGIS-Django/dockerize/scripts/renew_ssl.sh > /tmp/ssl-renewal-logs.txt + + +docker compose -f /home/web/QGIS-Django/dockerize/docker-compose.yml -p qgis-plugins run certbot renew \ No newline at end of file From 8ff8de2712a612005754338a672ca06cdaebc472 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Fri, 3 May 2024 13:42:08 +0300 Subject: [PATCH 32/32] Redirect http to https --- dockerize/sites-enabled/prod-ssl.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dockerize/sites-enabled/prod-ssl.conf b/dockerize/sites-enabled/prod-ssl.conf index c8100ef6..313b9618 100644 --- a/dockerize/sites-enabled/prod-ssl.conf +++ b/dockerize/sites-enabled/prod-ssl.conf @@ -21,7 +21,10 @@ server { # the port your site will be served on listen 80; # the domain name it will serve for - server_name ""; + server_name plugins.qgis.org; + + # Redirect all HTTP traffic to HTTPS + return 301 https://$server_name$request_uri; charset utf-8;