diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..7ab91dc3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +dockerize/postgres_data diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 08e6211d..bfc5cf81 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -47,12 +47,13 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Run docker-compose build - run: docker-compose build + - name: Generate the .env file + run: cp .env.template .env + - name: Run the containers run: docker-compose up -d db devweb - + - name: Wait for the containers to start run: sleep 15 @@ -65,4 +66,3 @@ jobs: python manage.py migrate && python manage.py test ' - diff --git a/.gitignore b/.gitignore index 4267cc6f..38e1880e 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,8 @@ static_media dockerize/static dockerize/media dockerize/logs +dockerize/certbot-etc +dockerize/webroot .editorconfig # test cache @@ -51,3 +53,5 @@ qgis-app/api/tests/*/ # whoosh_index qgis-app/whoosh_index/ +docker-compose.override.yml +.env diff --git a/INSTALL.md b/INSTALL.md index 1a66d939..3a2f7421 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -196,6 +196,8 @@ $ make dbrestore ## Deploy Update with Rancher +**TO BE DEPRECATED** + - Go to [https://rancher.qgis.org](https://rancher.qgis.org) - Click the home icon in the top left corner, and choose the environment: `plugins.qgis.org` - You will be redirected to User Stacks page, choose the plugins stack by clicking on `plugins` link diff --git a/REQUIREMENTS.txt b/REQUIREMENTS.txt index 2e741ec8..a92fb708 100644 --- a/REQUIREMENTS.txt +++ b/REQUIREMENTS.txt @@ -1,17 +1,21 @@ -Django==2.2.25 +Django==3.2.11 + # Currently broken with 'no module named defaults' error #Feedjack==0.9.18 # So use George's fork rather # git+https://github.com/Erve1879/feedjack.git -# George's is also broken: use my fork (django 1.8 ready) -git+https://github.com/elpaso/feedjack.git +# George's is also broken: use elpaso fork (django 1.8 ready) +# git+https://github.com/elpaso/feedjack.git +# His is also broken, use dimasciput (django 3.2 ready) +git+https://github.com/dimasciput/feedjack.git + Markdown==2.3.1 #PIL==1.1.7 Pillow Pygments==2.7.4 # Updates for Django 2 & Python 3.7 -git+https://github.com/Xpirix/whoosh.git@a306553 +git+https://github.com/Xpirix/whoosh.git@main pickle5==0.0.12 django-haystack==3.2.1 @@ -21,16 +25,18 @@ argparse==1.2.1 django-annoying==0.7.7 django-auth-ldap==1.2.6 django-autoslug==1.7.1 -django-debug-toolbar==1.11.1 +django-debug-toolbar==3.2.4 django-endless-pagination==2.0 django-extensions==1.2.0 django-generic-aggregation==0.3.2 #django-olwidget==0.61.0 unmaintained, use this fork git+https://github.com/Christophe31/olwidget.git django-pagination==1.0.7 + # Unmaintained! #django-ratings==0.3.7 -git+https://github.com/enikesha/django-ratings.git +git+https://github.com/gelo-zhukov/django-ratings.git + django-simple-ratings==0.3.2 # SIMPLEMENU git+https://github.com/elpaso/django-simplemenu.git diff --git a/dockerize/.env.template b/dockerize/.env.template new file mode 100644 index 00000000..23981fe9 --- /dev/null +++ b/dockerize/.env.template @@ -0,0 +1,31 @@ +# RabbitMQ host +RABBITMQ_HOST=rabbitmq + +# Database variables +DATABASE_NAME=gis +DATABASE_USERNAME=docker +DATABASE_PASSWORD=docker +DATABASE_HOST=db + +# Django settings +DJANGO_SETTINGS_MODULE=settings_docker +DEBUG=False + +# Docker volumes +QGISPLUGINS_STATIC_VOLUME=static-data +QGISPLUGINS_MEDIA_VOLUME=media-data +QGISPLUGINS_BACKUP_VOLUME=backups-data + +# Email variables +EMAIL_BACKEND='django.core.mail.backends.smtp.EmailBackend' +EMAIL_HOST='' +EMAIL_PORT=587 +EMAIL_USE_TLS=True +EMAIL_HOST_USER='' +EMAIL_HOST_PASSWORD='' + +# URL +DEFAULT_PLUGINS_SITE='https://plugins.qgis.org/' + +# ENV: debug or prod +QGISPLUGINS_ENV=debug \ No newline at end of file diff --git a/dockerize/Makefile b/dockerize/Makefile index 21838313..417a360b 100644 --- a/dockerize/Makefile +++ b/dockerize/Makefile @@ -13,35 +13,70 @@ build: @echo "------------------------------------------------------------------" @echo "Building in production and development mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) build + @docker compose -p $(PROJECT_ID) build + +build-dev: + @echo + @echo "------------------------------------------------------------------" + @echo "Building in development mode only" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) build devweb db: @echo @echo "------------------------------------------------------------------" @echo "Running db in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up -d db + @docker compose -p $(PROJECT_ID) up -d db + +metabase: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running metabase in production mode" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) up -d metabase web: db @echo @echo "------------------------------------------------------------------" @echo "Running in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up -d web + @docker compose -p $(PROJECT_ID) up -d uwsgi web worker beat + +dbbackups: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running dbbackups in production mode" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) up -d dbbackups + +certbot: web + @echo + @echo "------------------------------------------------------------------" + @echo "Running cerbot in production mode" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) up -d certbot + +devweb-test: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running in TESTING mode" + @echo "------------------------------------------------------------------" + @docker compose up --no-deps -d devweb devweb: db @echo @echo "------------------------------------------------------------------" @echo "Running in DEVELOPMENT mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) up --no-deps -d devweb + @docker compose -p $(PROJECT_ID) up --no-deps -d devweb rabbitmq worker beat devweb-runserver: devweb @echo @echo "------------------------------------------------------------------" @echo "Running in DEVELOPMENT mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec devweb python manage.py runserver 0.0.0.0:8080 + @docker compose -p $(PROJECT_ID) exec devweb python manage.py runserver 0.0.0.0:8080 migrate: @echo @@ -54,26 +89,26 @@ migrate: @# We add the '-' prefix to the next line as the migration may fail @# but we want to continue anyway. @#We need to migrate accounts first as it has a reference to user model - -@docker-compose -p $(PROJECT_ID) exec web python manage.py migrate auth - @docker-compose -p $(PROJECT_ID) exec web python manage.py migrate + -@docker compose -p $(PROJECT_ID) exec uwsgi python manage.py migrate auth + @docker compose -p $(PROJECT_ID) exec uwsgi python manage.py migrate update-migrations: @echo @echo "------------------------------------------------------------------" @echo "Running update migrations in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec web python manage.py makemigrations + @docker compose -p $(PROJECT_ID) exec uwsgi python manage.py makemigrations collectstatic: @echo @echo "------------------------------------------------------------------" @echo "Collecting static in production mode" @echo "------------------------------------------------------------------" - #@docker-compose -p $(PROJECT_ID) run uwsgi python manage.py collectstatic --noinput + @docker compose -p $(PROJECT_ID) run uwsgi python manage.py collectstatic --noinput #We need to run collect static in the same context as the running # uwsgi container it seems so I use docker exec here # no -it flag so we can run over remote shell - @docker exec $(PROJECT_ID)-web python manage.py collectstatic --noinput + # @docker exec $(PROJECT_ID)-web python manage.py collectstatic --noinput reload: @echo @@ -89,7 +124,7 @@ kill: @echo "------------------------------------------------------------------" @echo "Killing in production mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) kill + @docker compose -p $(PROJECT_ID) kill rm: rm-only @@ -98,7 +133,21 @@ rm-only: kill @echo "------------------------------------------------------------------" @echo "Removing production instance!!! " @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) rm + @docker compose -p $(PROJECT_ID) rm + +maillogs: + @echo + @echo "------------------------------------------------------------------" + @echo "Showing smtp logs in production mode" + @echo "------------------------------------------------------------------" + @docker compose exec smtp tail -f /var/log/mail.log + +mailerrorlogs: + @echo + @echo "------------------------------------------------------------------" + @echo "Showing smtp error logs in production mode" + @echo "------------------------------------------------------------------" + @docker compose exec smtp tail -f /var/log/mail.err dbrestore: @echo @@ -106,30 +155,37 @@ dbrestore: @echo "Restore dump from backups/latest.dmp in production mode" @echo "------------------------------------------------------------------" @# - prefix causes command to continue even if it fails - @echo "stopping web container" - @docker-compose -p $(PROJECT_ID) stop web - @echo "dropping gis" - @docker exec -t $(PROJECT_ID)-db su - postgres -c "dropdb gis" - @echo "creating gis" - @docker exec -t $(PROJECT_ID)-db su - postgres -c "createdb -O docker -T template_postgis gis" - @echo "restoring gis" - @# Because we pipe from one docker command to another and we are going - @# to execute this Make command from a remote server at times, we need to using use interactive mode (-i) - @# in the first command and not use terminal (-t) in the second. Please do not change these! - @docker exec -t $(PROJECT_ID)-db pg_restore /backups/latest.dmp | docker exec -i $(PROJECT_ID)-db su - postgres -c "psql gis" - @docker-compose -p $(PROJECT_ID) start web - @echo "starting web container" + @echo "stopping uwsgi container" + @docker compose -p $(PROJECT_ID) stop uwsgi + @echo "Dropping the gis and metabase databases" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "dropdb --force gis" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "dropdb --force metabase" + @echo "Creating the gis and metabase databases" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "createdb -O docker -T template1 gis" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "createdb -O docker -T template1 metabase" + @echo "Restore database from backups/latest-gis.dmp" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "pg_restore -c /backups/latest-gis.dmp -d gis" + -@docker compose -p $(PROJECT_ID) exec db su - postgres -c "pg_restore -c /backups/latest-metabase.dmp -d metabase" + @echo "starting uwsgi container" + @docker compose -p $(PROJECT_ID) up -d uwsgi + +wait-db: + @docker compose -p $(PROJECT_ID) exec db su - postgres -c "until pg_isready; do sleep 5; done" + +create-test-db: + @docker compose -p $(PROJECT_ID) exec db su - postgres -c "psql -c 'create database test_db;'" + @docker compose -p $(PROJECT_ID) exec db su - postgres -c "psql -d test_db -c 'create extension postgis;'" dbseed: @echo @echo "------------------------------------------------------------------" @echo "Seed db with JSON data from /fixtures/*.json" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec devweb bash -c 'python manage.py loaddata fixtures/*.json' + @docker compose -p $(PROJECT_ID) exec devweb bash -c 'python manage.py loaddata fixtures/*.json' rebuild_index: @echo @echo "------------------------------------------------------------------" @echo "Rebuild search index in PRODUCTION mode" @echo "------------------------------------------------------------------" - @docker-compose -p $(PROJECT_ID) exec web bash -c 'python manage.py rebuild_index' + @docker compose -p $(PROJECT_ID) exec uwsgi bash -c 'python manage.py rebuild_index' diff --git a/dockerize/docker-compose.override.template.yml b/dockerize/docker-compose.override.template.yml new file mode 100644 index 00000000..bb0fdd8a --- /dev/null +++ b/dockerize/docker-compose.override.template.yml @@ -0,0 +1,55 @@ +version: '3' +services: + devweb: + # Note you cannot scale if you use container_name + image: kartoza/qgis-plugins-uwsgi:dev-latest + container_name: qgis-plugins-devweb + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev + ports: + # for django test server + - "62202:8080" + # for ssh + - "62203:22" + + beat: + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + + worker: + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + + uwsgi: + container_name: qgis-plugins-uwsgi + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: prod + + db: + volumes: + - ./postgres_data:/var/lib/postgresql + - ./backups:/backups + + web: + volumes: + - ./sites-enabled:/etc/nginx/conf.d:ro + - ./static:/home/web/static:ro + - ./media:/home/web/media:ro + ports: + - "62201:8080" diff --git a/dockerize/docker-compose.override.test.yml b/dockerize/docker-compose.override.test.yml new file mode 100644 index 00000000..e10fe9a1 --- /dev/null +++ b/dockerize/docker-compose.override.test.yml @@ -0,0 +1,19 @@ +version: '3' +services: + devweb: + # Note you cannot scale if you use container_name + image: kartoza/qgis-plugins-uwsgi:dev-latest + container_name: qgis-plugins-devweb + volumes: + - ../qgis-app:/home/web/django_project + - ./static:/home/web/static:rw + - ./media:/home/web/media:rw + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev + ports: + # for django test server + - "62202:8080" + # for ssh + - "62203:22" diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 80578f9f..0f4f4dcf 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -1,70 +1,78 @@ -version: "3.8" +version: '3.8' volumes: - django-statics-data: {} - django-media-data: {} + postgres_data: + rabbitmq: + celerybeat-schedule: + static-data: + media-data: + backups-data: services: + db: container_name: qgis-plugins-db - image: kartoza/postgis:9.6-2.4 + image: kartoza/postgis:16-3.4 environment: - ALLOW_IP_RANGE=0.0.0.0/0 - - POSTGRES_USER=docker - - POSTGRES_PASS=docker + - POSTGRES_USER=${DATABASE_USERNAME:-docker} + - POSTGRES_PASS=${DATABASE_PASSWORD:-docker} + - PASSWORD_AUTHENTICATION=${PASSWORD_AUTHENTICATION:-md5} volumes: - - ./backups:/backups + - postgres_data:/var/lib/postgresql + - ${QGISPLUGINS_BACKUP_VOLUME}:/backups restart: unless-stopped - web: - # Note you cannot scale if you use container_name - container_name: qgis-plugins-web - build: docker + uwsgi: &uwsgi-common + container_name: qgis-plugins-uwsgi-common + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: prod hostname: uwsgi + expose: + - "8080" environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - DEBUG=False - - RABBITMQ_HOST=rabbitmq + - DATABASE_NAME=${DATABASE_NAME:-gis} + - DATABASE_USERNAME=${DATABASE_USERNAME:-docker} + - DATABASE_PASSWORD=${DATABASE_PASSWORD:-docker} + - DATABASE_HOST=${DATABASE_HOST:-db} + - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-settings_docker} + - VIRTUAL_HOST=${VIRTUAL_HOST:-plugins.kartoza.com} + - VIRTUAL_PORT=${VIRTUAL_PORT:-8080} + - DEBUG=${DEBUG:-False} + - RABBITMQ_HOST=${RABBITMQ_HOST:-rabbitmq} + - BROKER_URL=amqp://rabbitmq:5672 + - EMAIL_BACKEND=${EMAIL_BACKEND} + - EMAIL_HOST=${EMAIL_HOST} + - EMAIL_PORT=${EMAIL_PORT} + - EMAIL_USE_TLS=${EMAIL_USE_TLS} + - EMAIL_HOST_USER=${EMAIL_HOST_USER:-automation} + - EMAIL_HOST_PASSWORD=${EMAIL_HOST_PASSWORD} + - DEFAULT_PLUGINS_SITE=${DEFAULT_PLUGINS_SITE:-https://plugins.qgis.org/} volumes: - ../qgis-app:/home/web/django_project - - django-statics-data:/home/web/static:rw - - django-media-data:/home/web/media:rw + - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:rw + - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:rw + - celerybeat-schedule:/home/web/celerybeat-schedule:rw links: - db:db - rabbitmq:rabbitmq - - worker:worker restart: unless-stopped user: root - command: uwsgi --ini /uwsgi.conf + # This is the entry point for a development server. + # Run with --no-deps to run attached to the services + # from prod environment if wanted devweb: - # Note you cannot scale if you use container_name + <<: *uwsgi-common container_name: qgis-plugins-devweb - build: docker - hostname: uwsgi - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - RABBITMQ_HOST=rabbitmq + build: + context: ${PWD}/../ + dockerfile: dockerize/docker/Dockerfile + target: dev volumes: - ../qgis-app:/home/web/django_project - - django-statics-data:/home/web/static:rw - - django-media-data:/home/web/media:rw - links: - - db:db - - rabbitmq:rabbitmq - - worker:worker - restart: unless-stopped - user: root + - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:rw + - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:rw ports: # for django test server - "62202:8080" @@ -72,50 +80,92 @@ services: - "62203:22" rabbitmq: - image: library/rabbitmq:3.6 + image: rabbitmq:3.7-alpine hostname: rabbitmq - environment: - - RABBIT_PASSWORD=rabbit_test_password - - USER=rabbit_user - - RABBITMQ_NODENAME=rabbit + volumes: + - rabbitmq:/var/lib/rabbitmq restart: unless-stopped + beat: + <<: *uwsgi-common + container_name: qgis-plugins-beat + working_dir: /home/web/django_project + entrypoint: [ ] + command: celery --app=plugins.celery:app beat -s /home/web/celerybeat-schedule/schedule -l INFO + worker: - # Note you cannot scale if you use container_name + <<: *uwsgi-common container_name: qgis-plugins-worker - build: docker - hostname: uwsgi - working_dir: /home/web/django_project - command: celery -A plugins worker -l info - environment: - - DATABASE_NAME=gis - - DATABASE_USERNAME=docker - - DATABASE_PASSWORD=docker - - DATABASE_HOST=db - - DJANGO_SETTINGS_MODULE=settings_docker - - VIRTUAL_HOST=plugins.kartoza.com - - VIRTUAL_PORT=8080 - - RABBITMQ_HOST=rabbitmq - volumes: - - ../qgis-app:/home/web/django_project - - django-statics-data:/home/web/static:rw - - django-media-data:/home/web/media:rw links: - - db:db - - rabbitmq:rabbitmq + - db + - rabbitmq + - beat + working_dir: /home/web/django_project + entrypoint: [] + command: celery -A plugins worker -l INFO - nginx: + web: # Note you cannot scale if you use container_name - container_name: qgis-plugins-nginx + container_name: qgis-plugins-web image: nginx - hostname: nginx + hostname: web + entrypoint: + - /etc/nginx/sites-available/docker-entrypoint.sh + ports: + - "80:80" + - "443:443" volumes: - - ./sites-enabled:/etc/nginx/conf.d:ro - - django-statics-data:/home/web/static:ro - - django-media-data:/home/web/media:ro - - ./logs:/var/log/nginx + - ./sites-enabled:/etc/nginx/sites-available/:ro + - ${QGISPLUGINS_STATIC_VOLUME}:/home/web/static:ro + - ${QGISPLUGINS_MEDIA_VOLUME}:/home/web/media:ro + - ./webroot:/var/www/webroot + - ./certbot-etc:/etc/letsencrypt links: - - web:uwsgi - ports: - - "62201:8080" + - uwsgi:uwsgi + - metabase:metabase + logging: + driver: "json-file" + options: + max-size: "200k" + max-file: "10" + restart: unless-stopped + command: + - ${QGISPLUGINS_ENV} + + dbbackups: + image: kartoza/pg-backup:16-3.4 + hostname: pg-backups + volumes: + - ${QGISPLUGINS_BACKUP_VOLUME}:/backups + links: + - db:db + environment: + # take care to let the project name below match that + # declared in the top of the makefile + - DUMPPREFIX=${DUMPPREFIX:-QGIS_PLUGINS} + - POSTGRES_USER=${DATABASE_USERNAME:-docker} + - POSTGRES_PASS=${DATABASE_PASSWORD:-docker} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - POSTGRES_HOST=${DATABASE_HOST:-db} + - PGDATABASE=${DATABASE_NAME:-gis} restart: unless-stopped + + metabase: + image: metabase/metabase:latest + environment: + - MB_DB_TYPE=postgres + - MB_DB_CONNECTION_URI=jdbc:postgresql://${DATABASE_HOST:-db}:5432/metabase?user=${DATABASE_USERNAME:-docker}&password=${DATABASE_PASSWORD:-docker} + links: + - db + expose: + - "3000" + + certbot: + image: certbot/certbot + container_name: certbot + volumes: + - ./webroot:/var/www/webroot + - ./certbot-etc:/etc/letsencrypt + depends_on: + - web + command: certonly --webroot --webroot-path=/var/www/webroot --email admin@qgis.org --agree-tos --no-eff-email --force-renewal -d plugins.qgis.org \ No newline at end of file diff --git a/dockerize/docker/Dockerfile b/dockerize/docker/Dockerfile index 3bd21f73..dd51c53b 100644 --- a/dockerize/docker/Dockerfile +++ b/dockerize/docker/Dockerfile @@ -1,22 +1,42 @@ -#--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ -# Note this base image is based on debian -FROM kartoza/django-base:3.7 -MAINTAINER Dimas Ciputra +# For more information, please refer to https://aka.ms/vscode-docker-python +FROM python:3.12-slim as prod -#RUN ln -s /bin/true /sbin/initctl -RUN apt-get clean all +EXPOSE 8000 -# Debian stretch/updates release issue. please see https://serverfault.com/a/1130167 -RUN echo "deb http://archive.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list +# Keeps Python from generating .pyc files in the container +ENV PYTHONDONTWRITEBYTECODE=1 -RUN apt-get update && apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev -ADD REQUIREMENTS.txt /REQUIREMENTS.txt -RUN pip install -r /REQUIREMENTS.txt -RUN pip install uwsgi freezegun==1.3.1 +# Turns off buffering for easier container logging +ENV PYTHONUNBUFFERED=1 +RUN apt-get update && apt-get install -y \ + git python3-dev libxml2-dev \ + libsasl2-dev libldap2-dev libssl-dev \ + libxslt1-dev zlib1g-dev \ + build-essential \ + libffi-dev gdal-bin\ + libjpeg-dev libpq-dev \ + liblcms2-dev libblas-dev libatlas-base-dev + +RUN rm -rf /uwsgi.conf +ADD dockerize/docker/uwsgi.conf /uwsgi.conf +ADD qgis-app /home/web/django_project +ADD dockerize/docker/REQUIREMENTS.txt /REQUIREMENTS.txt + +RUN pip install --upgrade pip && pip install -r /REQUIREMENTS.txt + +RUN mkdir -p /var/log/uwsgi + +WORKDIR /home/web/django_project +CMD ["uwsgi", "--ini", "/uwsgi.conf"] + + +FROM prod as dev + +# This section taken on 2 July 2015 from # https://docs.docker.com/examples/running_ssh_service/ # Sudo is needed by pycharm when it tries to pip install packages -RUN apt-get install -y openssh-server sudo +RUN apt-get update && apt-get install -y openssh-server sudo RUN mkdir /var/run/sshd RUN echo 'root:docker' | chpasswd RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config @@ -27,15 +47,16 @@ RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so ENV NOTVISIBLE "in users profile" RUN echo "export VISIBLE=now" >> /etc/profile -RUN rm -rf /uwsgi.conf -ADD uwsgi.conf /uwsgi.conf +# Install freezegun for feedback test +RUN pip install freezegun -# Open port 8080 as we will be running our uwsgi socket on that -EXPOSE 8080 +# -------------------------------------------------------- +# Open ports as needed +# -------------------------------------------------------- +# Open port 8080 as we will be running our django dev server on +EXPOSE 8080 # Open port 22 as we will be using a remote interpreter from pycharm EXPOSE 22 -RUN mkdir -p /var/log/uwsgi -WORKDIR /home/web/django_project CMD ["/usr/sbin/sshd", "-D"] diff --git a/dockerize/docker/REQUIREMENTS.txt b/dockerize/docker/REQUIREMENTS.txt index 4de38450..e9ff83f3 100644 --- a/dockerize/docker/REQUIREMENTS.txt +++ b/dockerize/docker/REQUIREMENTS.txt @@ -1,59 +1,61 @@ -django==2.2.25 -django-auth-ldap -python-ldap -django-taggit==2.0.0 +django~=4.2 +django-auth-ldap~=4.6 +python-ldap~=3.4 +django-taggit~=5.0 django-tinymce==3.4.0 -psycopg2 +psycopg2-binary~=2.9 # Updates for Django 2 git+https://github.com/metamatik/django-templatetag-sugar.git +# Updates for Django 4 +git+https://github.com/Xpirix/django-ratings.git@modernize +django-taggit-autosuggest~=0.4 +django-annoying~=0.10 + # Updates for Django 2 -git+https://github.com/elpaso/django-ratings.git@modernize -django-taggit-autosuggest -django-annoying -# Updates for Django 2 -git+https://github.com/elpaso/rpc4django.git@modernize -Pillow +# git+https://github.com/elpaso/rpc4django.git@modernize +rpc4django~=0.6 +Pillow~=10.1 django-taggit-templatetags -# Updates for Django 2 -git+https://github.com/elpaso/django-simplemenu.git@modernize -django-bootstrap-pagination -django-sortable-listview -sorl-thumbnail -django-extensions -django-debug-toolbar==1.11.1 - -# Updates for Django 2 & Python 3.7 -git+https://github.com/Xpirix/whoosh.git@a306553 -pickle5==0.0.12 -django-haystack==3.2.1 +# Updates for Django 4 +git+https://github.com/Xpirix/django-simplemenu.git@modernize +django-bootstrap-pagination-forked~=1.7 +django-sortable-listview~=0.43 +sorl-thumbnail~=12.10 +django-extensions~=3.2 +django-debug-toolbar~=4.2 +whoosh~=2.7 +django-haystack~=3.2 # Feedjack==0.9.18 # So use George's fork rather # git+https://github.com/Erve1879/feedjack.git # George's is also broken: use my fork (django 1.8 ready) # git+https://github.com/elpaso/feedjack.git -# His is also broken, use mine (django 2.2 ready) -git+https://github.com/dimasciput/feedjack.git -feedparser==5.2.1 -celery==4.3.1 +# His is also broken, use dimasciput (django 2.2 ready) +# git+https://github.com/dimasciput/feedjack.git +# For django 4, use Xpirix (django 4.2 ready) +git+https://github.com/Xpirix/feedjack.git +feedparser~=6.0 +celery~=5.3 # pin due to issues with a breaking change # https://github.com/celery/celery/issues/7783 importlib_metadata<5 -requests==2.23.0 - -markdown==3.2.1 +requests~=2.31 -djangorestframework==3.11.2 -pyjwt==1.7.1 -djangorestframework-simplejwt==4.4 +markdown~=3.5 +djangorestframework~=3.14 +pyjwt~=2.8 +djangorestframework-simplejwt~=5.3 sorl-thumbnail-serializer-field==0.2.1 django-rest-auth==0.9.5 -drf-yasg==1.17.1 +drf-yasg~=1.21 django-rest-multiple-models==2.1.3 django-preferences==1.0.0 PyWavefront==1.3.3 django-matomo==0.1.6 +uwsgi~=2.0 +freezegun~=1.4 \ No newline at end of file diff --git a/dockerize/docker/uwsgi.conf b/dockerize/docker/uwsgi.conf index 50a80d0d..7af0f302 100644 --- a/dockerize/docker/uwsgi.conf +++ b/dockerize/docker/uwsgi.conf @@ -14,7 +14,7 @@ env = DJANGO_SETTINGS_MODULE=settings_docker #daemonize = /tmp/django.log req-logger = file:/var/log/uwsgi-requests.log logger = file:/var/log/uwsgi-errors.log -reload-os-env +# reload-os-env #uid = 1000 #gid = 1000 memory-report = true diff --git a/dockerize/production/Dockerfile b/dockerize/production/Dockerfile index 7a57d864..9453fffc 100644 --- a/dockerize/production/Dockerfile +++ b/dockerize/production/Dockerfile @@ -5,6 +5,10 @@ MAINTAINER Dimas Ciputra #RUN ln -s /bin/true /sbin/initctl RUN apt-get clean all + +# Debian stretch/updates release issue. please see https://serverfault.com/a/1130167 +RUN echo "deb http://archive.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list + RUN apt-get update && apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev ARG BRANCH_TAG=develop @@ -14,6 +18,7 @@ RUN mkdir -p /usr/src; mkdir -p /home/web && \ ln -s /usr/src/plugins/qgis-app /home/web/django_project RUN cd /usr/src/plugins/dockerize/docker && \ + pip install --upgrade pip && \ pip install -r REQUIREMENTS.txt && \ pip install uwsgi && \ rm -rf /uwsgi.conf && \ diff --git a/dockerize/scripts/renew-ssl.sh b/dockerize/scripts/renew-ssl.sh new file mode 100644 index 00000000..7004ead6 --- /dev/null +++ b/dockerize/scripts/renew-ssl.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + + +# Run daily on crontab e.g. +# Your cron job will be run at: (5 times displayed) +# +# 2021-11-08 11:10:00 UTC +# 2021-11-09 11:10:00 UTC +# 2021-11-10 11:10:00 UTC +# 2021-11-11 11:10:00 UTC +# 2021-11-12 11:10:00 UTC +# ...etc + +#25 11 * * * /bin/bash /home/web/QGIS-Django/dockerize/scripts/renew_ssl.sh > /tmp/ssl-renewal-logs.txt + + +docker compose -f /home/web/QGIS-Django/dockerize/docker-compose.yml -p qgis-plugins run certbot renew \ No newline at end of file diff --git a/dockerize/sites-enabled/dev.conf b/dockerize/sites-enabled/dev.conf new file mode 100644 index 00000000..ef93e13c --- /dev/null +++ b/dockerize/sites-enabled/dev.conf @@ -0,0 +1,31 @@ +# Define connection details for connecting to django running in +# a docker container. +upstream uwsgi { + server uwsgi:8080; +} +server { + # OTF gzip compression + gzip on; + gzip_min_length 860; + gzip_comp_level 5; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; + gzip_disable “MSIE [1-6].(?!.*SV1)”; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + # the port your site will be served on + listen 80; + # the domain name it will serve for + server_name ""; + charset utf-8; + + # Send all requests to the Django server. + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://uwsgi; + } +} diff --git a/dockerize/sites-enabled/docker-entrypoint.sh b/dockerize/sites-enabled/docker-entrypoint.sh new file mode 100755 index 00000000..fbc27084 --- /dev/null +++ b/dockerize/sites-enabled/docker-entrypoint.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Clean up sites-enabled +echo "Clean sites-enabled" +rm -rf /etc/nginx/conf.d/*.conf +mkdir -p /etc/nginx/conf.d + +if [ $# -eq 1 ]; then + case $1 in + # Debug mode, enable dev.conf + [Dd][Ee][Bb][Uu][Gg]) + echo "Run in debug mode" + CONF_FILE=dev.conf + ln -s /etc/nginx/sites-available/$CONF_FILE /etc/nginx/conf.d/$CONF_FILE + exec nginx -g "daemon off;" + ;; + # Production mode, run using uwsgi + [Pp][Rr][Oo][Dd]) + echo "Run in prod mode" + CONF_FILE=prod.conf + ln -s /etc/nginx/sites-available/$CONF_FILE /etc/nginx/conf.d/$CONF_FILE + exec nginx -g "daemon off;" + ;; + # Production SSL mode, run using uwsgi + [Pp][Rr][Oo][Dd][-][Ss][Ss][Ll]) + echo "Run in prod SSL mode" + CONF_FILE=prod-ssl.conf + ln -s /etc/nginx/sites-available/$CONF_FILE /etc/nginx/conf.d/$CONF_FILE + exec nginx -g "daemon off;" + ;; + esac +fi + +# Run as bash entrypoint +exec "$@" diff --git a/dockerize/sites-enabled/prod-ssl.conf b/dockerize/sites-enabled/prod-ssl.conf new file mode 100644 index 00000000..313b9618 --- /dev/null +++ b/dockerize/sites-enabled/prod-ssl.conf @@ -0,0 +1,213 @@ +# Define connection details for connecting to django running in +# a docker container. +upstream uwsgi { + server uwsgi:8080; +} + +server { + # OTF gzip compression + gzip on; + gzip_min_length 860; + gzip_comp_level 5; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; + gzip_disable “MSIE [1-6].(?!.*SV1)”; + + access_log /var/log/nginx/access.log; + + client_max_body_size 20M; + error_log /var/log/nginx/error.log; + + # the port your site will be served on + listen 80; + # the domain name it will serve for + server_name plugins.qgis.org; + + # Redirect all HTTP traffic to HTTPS + return 301 https://$server_name$request_uri; + + charset utf-8; + + # Drop any non django related requests + # Its probably someone nefarious probing for vulnerabilities... + location ~ (\.php|\.asp|myadmin) { + return 404; + } + + # Django media + location /media { + # your Django project's media files - amend as required + alias /home/web/media; + expires 21d; # cache for 71 days + } + location /static { + # your Django project's static files - amend as required + alias /home/web/static; + expires 21d; # cache for 21 days + } + location /plugins/plugins.xml { + if ($request_uri !~ "&package_name(.*)") { + rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; + root /home; + expires 600s; + } + } + location /archive { + # Changed from http_host to host because of error messages when + # bots hit urls like this: + # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', + # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ + # for more details. + #proxy_set_header Host $http_host; + proxy_set_header Host $host; + autoindex on; + # your Django project's static files - amend as required + alias /home/web/archive; + expires 21d; # cache for 6h + } + # Finally, send all non-media requests to the Django server. + location / { + uwsgi_pass uwsgi; + # the uwsgi_params file you installed needs to be passed with each + # request. + # the uwsgi_params need to be passed with each uwsgi request + uwsgi_param QUERY_STRING $query_string; + uwsgi_param REQUEST_METHOD $request_method; + uwsgi_param CONTENT_TYPE $content_type; + uwsgi_param CONTENT_LENGTH $content_length; + + uwsgi_param REQUEST_URI $request_uri; + uwsgi_param PATH_INFO $document_uri; + uwsgi_param DOCUMENT_ROOT $document_root; + uwsgi_param SERVER_PROTOCOL $server_protocol; + uwsgi_param HTTPS $https if_not_empty; + + uwsgi_param REMOTE_ADDR $remote_addr; + uwsgi_param REMOTE_PORT $remote_port; + uwsgi_param SERVER_PORT $server_port; + uwsgi_param SERVER_NAME $server_name; + } + + location /metabase/ { + # set to webroot path + proxy_pass http://metabase:3000/; + } + + location ~ /.well-known/acme-challenge { + # set to webroot path + root /var/www/webroot; + default_type "text/plain"; + allow all; + } +} + + +server { + # SSL Cert + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name plugins.qgis.org; + + server_tokens off; + + ssl_certificate /etc/letsencrypt/live/plugins.qgis.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/plugins.qgis.org/privkey.pem; + + ssl_buffer_size 8k; + + # ssl_dhparam /etc/ssl/certs/dhparam-2048.pem; + + ssl_protocols TLSv1.2 TLSv1.1 TLSv1; + ssl_prefer_server_ciphers on; + + ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DH+3DES:!ADH:!AECDH:!MD5; + + ssl_ecdh_curve secp384r1; + ssl_session_tickets off; + + ssl_stapling on; + ssl_stapling_verify on; + resolver 8.8.8.8; + + # OTF gzip compression + gzip on; + gzip_min_length 860; + gzip_comp_level 5; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain application/xml application/x-javascript text/xml text/css application/json; + gzip_disable “MSIE [1-6].(?!.*SV1)”; + + access_log /var/log/nginx/access.log; + + client_max_body_size 20M; + error_log /var/log/nginx/error.log; + + charset utf-8; + + # Drop any non django related requests + # Its probably someone nefarious probing for vulnerabilities... + location ~ (\.php|\.asp|myadmin) { + return 404; + } + + # Django media + location /media { + # your Django project's media files - amend as required + alias /home/web/media; + expires 21d; # cache for 71 days + } + location /static { + # your Django project's static files - amend as required + alias /home/web/static; + expires 21d; # cache for 21 days + } + location /plugins/plugins.xml { + if ($request_uri !~ "&package_name(.*)") { + rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; + root /home; + expires 600s; + } + } + location /archive { + # Changed from http_host to host because of error messages when + # bots hit urls like this: + # 'REQUEST_URI': '/phpmyadmin/scripts/setup.php', + # See https://snakeycode.wordpress.com/2016/11/21/django-nginx-invalid-http_host-header/ + # for more details. + #proxy_set_header Host $http_host; + proxy_set_header Host $host; + autoindex on; + # your Django project's static files - amend as required + alias /home/web/archive; + expires 21d; # cache for 6h + } + # Finally, send all non-media requests to the Django server. + location / { + uwsgi_pass uwsgi; + # the uwsgi_params file you installed needs to be passed with each + # request. + # the uwsgi_params need to be passed with each uwsgi request + uwsgi_param QUERY_STRING $query_string; + uwsgi_param REQUEST_METHOD $request_method; + uwsgi_param CONTENT_TYPE $content_type; + uwsgi_param CONTENT_LENGTH $content_length; + + uwsgi_param REQUEST_URI $request_uri; + uwsgi_param PATH_INFO $document_uri; + uwsgi_param DOCUMENT_ROOT $document_root; + uwsgi_param SERVER_PROTOCOL $server_protocol; + uwsgi_param HTTPS $https if_not_empty; + + uwsgi_param REMOTE_ADDR $remote_addr; + uwsgi_param REMOTE_PORT $remote_port; + uwsgi_param SERVER_PORT $server_port; + uwsgi_param SERVER_NAME $server_name; + } + + location /metabase/ { + # set to webroot path + proxy_pass http://metabase:3000/; + } + + root /var/www/webroot; +} \ No newline at end of file diff --git a/dockerize/sites-enabled/default.conf b/dockerize/sites-enabled/prod.conf similarity index 81% rename from dockerize/sites-enabled/default.conf rename to dockerize/sites-enabled/prod.conf index e870a2a6..84f9d8aa 100644 --- a/dockerize/sites-enabled/default.conf +++ b/dockerize/sites-enabled/prod.conf @@ -3,6 +3,7 @@ upstream uwsgi { server uwsgi:8080; } + server { # OTF gzip compression gzip on; @@ -13,21 +14,23 @@ server { gzip_disable “MSIE [1-6].(?!.*SV1)”; access_log /var/log/nginx/access.log; + + client_max_body_size 20M; error_log /var/log/nginx/error.log; # the port your site will be served on - listen 8080; + listen 80; # the domain name it will serve for server_name ""; + charset utf-8; # Drop any non django related requests # Its probably someone nefarious probing for vulnerabilities... - location ~ (\.php|.aspx|.asp|myadmin) { + location ~ (\.php|\.asp|myadmin) { return 404; } - # max upload size, adjust to taste - client_max_body_size 15M; + # Django media location /media { # your Django project's media files - amend as required @@ -39,6 +42,13 @@ server { alias /home/web/static; expires 21d; # cache for 21 days } + location /plugins/plugins.xml { + if ($request_uri !~ "&package_name(.*)") { + rewrite ^/plugins/plugins.xml /web/media/cached_xmls/plugins_$arg_qgis.xml break; + root /home; + expires 600s; + } + } location /archive { # Changed from http_host to host because of error messages when # bots hit urls like this: @@ -74,4 +84,16 @@ server { uwsgi_param SERVER_PORT $server_port; uwsgi_param SERVER_NAME $server_name; } + + location /metabase/ { + # set to webroot path + proxy_pass http://metabase:3000/; + } + + location ~ /.well-known/acme-challenge { + # set to webroot path + root /var/www/webroot; + default_type "text/plain"; + allow all; + } } diff --git a/qgis-app/REQUIREMENTS_plugins.txt b/qgis-app/REQUIREMENTS_plugins.txt index fa9c03ac..ebb39373 100644 --- a/qgis-app/REQUIREMENTS_plugins.txt +++ b/qgis-app/REQUIREMENTS_plugins.txt @@ -1,4 +1,4 @@ -django==2.2.25 +django==3.2.11 django-auth-ldap python-ldap django-taggit==2.0.0 @@ -6,8 +6,10 @@ django-tinymce==3.4.0 psycopg2 # Updates for Django 2 git+https://github.com/metamatik/django-templatetag-sugar.git -# Updates for Django 2 -git+https://github.com/elpaso/django-ratings.git@modernize + +# Updates for Django 3 +git+https://github.com/gelo-zhukov/django-ratings.git + django-taggit-autosuggest django-annoying # Updates for Django 2 @@ -20,9 +22,9 @@ django-bootstrap-pagination django-sortable-listview sorl-thumbnail django-extensions -django-debug-toolbar==1.11.1 +django-debug-toolbar==3.2.4 # Updates for Django 2 & Python 3.7 -git+https://github.com/Xpirix/whoosh.git@a306553 +git+https://github.com/Xpirix/whoosh.git@main pickle5==0.0.12 django-haystack==3.2.1 diff --git a/qgis-app/api/tests/test_views.py b/qgis-app/api/tests/test_views.py index 9f0ff30b..acb2cb30 100644 --- a/qgis-app/api/tests/test_views.py +++ b/qgis-app/api/tests/test_views.py @@ -216,7 +216,7 @@ def test_download_resource_should_be_a_file_in_a_zip(self): url = reverse("resource-download", kwargs={"uuid": self.style.uuid}) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertEquals( + self.assertEqual( response.get("Content-Disposition"), "attachment; filename=style_zero.zip" ) with io.BytesIO(response.content) as file: diff --git a/qgis-app/base/forms/processing_forms.py b/qgis-app/base/forms/processing_forms.py index 771eb3eb..cdf85a7f 100644 --- a/qgis-app/base/forms/processing_forms.py +++ b/qgis-app/base/forms/processing_forms.py @@ -1,6 +1,6 @@ from base.validator import filesize_validator from django import forms -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class ResourceBaseReviewForm(forms.Form): diff --git a/qgis-app/base/models/processing_models.py b/qgis-app/base/models/processing_models.py index 0d6590c5..9b53c625 100644 --- a/qgis-app/base/models/processing_models.py +++ b/qgis-app/base/models/processing_models.py @@ -7,7 +7,7 @@ from django.contrib.auth.models import User from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class UnapprovedManager(models.Manager): diff --git a/qgis-app/base/validator.py b/qgis-app/base/validator.py index 534c4c08..4a491e9e 100644 --- a/qgis-app/base/validator.py +++ b/qgis-app/base/validator.py @@ -2,7 +2,7 @@ from django.conf import settings from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ RESOURCE_MAX_SIZE = getattr(settings, "RESOURCE_MAX_SIZE", 1000000) # 1MB ERROR_FILESIZE_TOO_BIG = ValidationError( diff --git a/qgis-app/base/views/processing_view.py b/qgis-app/base/views/processing_view.py index bddb4fb5..10087884 100644 --- a/qgis-app/base/views/processing_view.py +++ b/qgis-app/base/views/processing_view.py @@ -17,7 +17,7 @@ from django.urls import reverse, reverse_lazy from django.utils.decorators import method_decorator from django.utils.text import slugify -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.views.decorators.cache import never_cache from django.views.generic import ( CreateView, diff --git a/qgis-app/custom_haystack_urls.py b/qgis-app/custom_haystack_urls.py index a0ab8c23..c14fa7f5 100755 --- a/qgis-app/custom_haystack_urls.py +++ b/qgis-app/custom_haystack_urls.py @@ -1,6 +1,6 @@ # Custom haystack search to match partial strings -from django.conf.urls import include, url +from django.urls import re_path as url from haystack.query import SearchQuerySet from haystack.views import SearchView diff --git a/qgis-app/geopackages/migrations/0009_alter_review_reviewer.py b/qgis-app/geopackages/migrations/0009_alter_review_reviewer.py new file mode 100644 index 00000000..0fc4f397 --- /dev/null +++ b/qgis-app/geopackages/migrations/0009_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('geopackages', '0008_remove_uuid_null'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/geopackages/models.py b/qgis-app/geopackages/models.py index 0cfc14e2..3475f08a 100644 --- a/qgis-app/geopackages/models.py +++ b/qgis-app/geopackages/models.py @@ -5,7 +5,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ GEOPACKAGES_STORAGE_PATH = getattr( settings, "GEOPACKAGE_STORAGE_PATH", "geopackages/%Y" diff --git a/qgis-app/homepage.py b/qgis-app/homepage.py index 7bf3d7c2..a7e9e51f 100644 --- a/qgis-app/homepage.py +++ b/qgis-app/homepage.py @@ -1,7 +1,7 @@ from django.contrib.flatpages.models import FlatPage from django.shortcuts import render from django.template import RequestContext -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from plugins.models import Plugin # from feedjack.models import Post diff --git a/qgis-app/layerdefinitions/file_handler.py b/qgis-app/layerdefinitions/file_handler.py index ddacc02d..ffc196c1 100644 --- a/qgis-app/layerdefinitions/file_handler.py +++ b/qgis-app/layerdefinitions/file_handler.py @@ -6,7 +6,7 @@ import xml.etree.ElementTree as ET from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ def parse_qlr(xmlfile): diff --git a/qgis-app/layerdefinitions/migrations/0002_alter_review_reviewer.py b/qgis-app/layerdefinitions/migrations/0002_alter_review_reviewer.py new file mode 100644 index 00000000..97576ae6 --- /dev/null +++ b/qgis-app/layerdefinitions/migrations/0002_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('layerdefinitions', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/layerdefinitions/models.py b/qgis-app/layerdefinitions/models.py index 1c6be6e4..2383b5be 100644 --- a/qgis-app/layerdefinitions/models.py +++ b/qgis-app/layerdefinitions/models.py @@ -5,7 +5,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ LAYERDEFINITIONS_STORAGE_PATH = getattr( settings, "LAYERDEFINITION_STORAGE_PATH", "layerdefinitions" diff --git a/qgis-app/layerdefinitions/tests/test_views.py b/qgis-app/layerdefinitions/tests/test_views.py index f46119b2..44f8dd28 100644 --- a/qgis-app/layerdefinitions/tests/test_views.py +++ b/qgis-app/layerdefinitions/tests/test_views.py @@ -193,7 +193,7 @@ def test_download_should_return_zipfile_with_custom_license(self): url = reverse("layerdefinition_download", kwargs={"pk": qlr.id}) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertEquals( + self.assertEqual( response.get("Content-Disposition"), "attachment; filename=test-qlr-file.zip", ) diff --git a/qgis-app/models/migrations/0007_alter_review_reviewer.py b/qgis-app/models/migrations/0007_alter_review_reviewer.py new file mode 100644 index 00000000..ad03f8c2 --- /dev/null +++ b/qgis-app/models/migrations/0007_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('models', '0006_remove_uuid_null'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/models/models.py b/qgis-app/models/models.py index 1786bd9e..4aa2809f 100644 --- a/qgis-app/models/models.py +++ b/qgis-app/models/models.py @@ -5,7 +5,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ MODELS_STORAGE_PATH = getattr(settings, "MODELS_STORAGE_PATH", "models/%Y") diff --git a/qgis-app/models/validator.py b/qgis-app/models/validator.py index e4d41829..6241633d 100644 --- a/qgis-app/models/validator.py +++ b/qgis-app/models/validator.py @@ -2,7 +2,7 @@ from django.conf import settings from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ MODEL_MAX_SIZE = getattr(settings, "MODEL_MAX_SIZE", 1000000) # 1MB diff --git a/qgis-app/plugins/api.py b/qgis-app/plugins/api.py index 62ff6b8e..72615d68 100644 --- a/qgis-app/plugins/api.py +++ b/qgis-app/plugins/api.py @@ -12,7 +12,7 @@ # Transaction from django.db import IntegrityError, connection -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from plugins.models import * from plugins.validator import validator from plugins.views import plugin_notify diff --git a/qgis-app/plugins/celery.py b/qgis-app/plugins/celery.py index 3ec72655..cdc9b450 100644 --- a/qgis-app/plugins/celery.py +++ b/qgis-app/plugins/celery.py @@ -3,6 +3,10 @@ import os from celery import Celery +import logging + +logger = logging.getLogger('plugins') + # set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings_docker") diff --git a/qgis-app/plugins/forms.py b/qgis-app/plugins/forms.py index 40c8ab5a..9f98f03f 100644 --- a/qgis-app/plugins/forms.py +++ b/qgis-app/plugins/forms.py @@ -5,7 +5,7 @@ from django.contrib.auth.models import User from django.forms import CharField, ModelForm, ValidationError from django.utils.safestring import mark_safe -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from plugins.models import Plugin, PluginOutstandingToken, PluginVersion, PluginVersionFeedback from plugins.validator import validator from taggit.forms import * diff --git a/qgis-app/plugins/management/commands/generate_plugins_xml.py b/qgis-app/plugins/management/commands/generate_plugins_xml.py index 4efd70e0..1795dcf9 100644 --- a/qgis-app/plugins/management/commands/generate_plugins_xml.py +++ b/qgis-app/plugins/management/commands/generate_plugins_xml.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from django.core.management.base import BaseCommand from plugins.tasks.generate_plugins_xml import generate_plugins_xml - +from django.conf import settings class Command(BaseCommand): @@ -12,7 +12,7 @@ def add_arguments(self, parser): "-s", "--site", dest="site", - default="http://plugins.qgis.org", + default=settings.DEFAULT_PLUGINS_SITE, help="Site url to get the source of plugins", ) diff --git a/qgis-app/plugins/migrations/0004_merge_20231123_0018.py b/qgis-app/plugins/migrations/0004_merge_20231123_0018.py new file mode 100644 index 00000000..8e18926d --- /dev/null +++ b/qgis-app/plugins/migrations/0004_merge_20231123_0018.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.11 on 2023-11-23 00:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('plugins', '0002_plugins_feedback'), + ('plugins', '0003_plugin_allow_update_name'), + ] + + operations = [ + ] diff --git a/qgis-app/plugins/migrations/0009_merge_20240321_0207.py b/qgis-app/plugins/migrations/0009_merge_20240321_0207.py new file mode 100644 index 00000000..45cd685a --- /dev/null +++ b/qgis-app/plugins/migrations/0009_merge_20240321_0207.py @@ -0,0 +1,14 @@ +# Generated by Django 4.2.11 on 2024-03-21 02:07 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('plugins', '0004_merge_20231123_0018'), + ('plugins', '0008_merge_20240206_0448'), + ] + + operations = [ + ] diff --git a/qgis-app/plugins/models.py b/qgis-app/plugins/models.py index 9f9cc209..72e8cc5a 100644 --- a/qgis-app/plugins/models.py +++ b/qgis-app/plugins/models.py @@ -8,7 +8,7 @@ from django.contrib.auth.models import User from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.utils import timezone from djangoratings.fields import AnonymousRatingField from taggit_autosuggest.managers import TaggableManager diff --git a/qgis-app/plugins/tasks/__init__.py b/qgis-app/plugins/tasks/__init__.py index 7987374c..a0df9010 100644 --- a/qgis-app/plugins/tasks/__init__.py +++ b/qgis-app/plugins/tasks/__init__.py @@ -1 +1,2 @@ -from plugins.tasks.generate_plugins_xml import * +from plugins.tasks.generate_plugins_xml import * # noqa +from plugins.tasks.update_feedjack import * # noqa diff --git a/qgis-app/plugins/tasks/generate_plugins_xml.py b/qgis-app/plugins/tasks/generate_plugins_xml.py index b1e60cef..a7415b60 100644 --- a/qgis-app/plugins/tasks/generate_plugins_xml.py +++ b/qgis-app/plugins/tasks/generate_plugins_xml.py @@ -2,10 +2,15 @@ import requests from celery import shared_task +from celery.utils.log import get_task_logger +from preferences import preferences from django.conf import settings from preferences import preferences +logger = get_task_logger(__name__) + + @shared_task def generate_plugins_xml(site=""): """ @@ -13,6 +18,8 @@ def generate_plugins_xml(site=""): :param site: site domain where the plugins will be fetched, default to http://plugins.qgis.org """ + logger.info('generate_plugins_xml : {}'.format(site)) + if not site: if settings.DEFAULT_PLUGINS_SITE: site = settings.DEFAULT_PLUGINS_SITE diff --git a/qgis-app/plugins/tasks/update_feedjack.py b/qgis-app/plugins/tasks/update_feedjack.py new file mode 100644 index 00000000..e12d14bd --- /dev/null +++ b/qgis-app/plugins/tasks/update_feedjack.py @@ -0,0 +1,10 @@ +from celery import shared_task +from celery.utils.log import get_task_logger + +logger = get_task_logger(__name__) + + +@shared_task +def update_feedjack(): + import subprocess + subprocess.call(['python', 'manage.py', 'feedjackupdate']) diff --git a/qgis-app/plugins/templates/plugins/form_snippet.html b/qgis-app/plugins/templates/plugins/form_snippet.html index f33ec0ce..ae794508 100755 --- a/qgis-app/plugins/templates/plugins/form_snippet.html +++ b/qgis-app/plugins/templates/plugins/form_snippet.html @@ -2,7 +2,7 @@
{% for field in form %}
- {% ifequal field.field.widget|klass 'CheckboxInput' %} + {% if field.field.widget|klass == 'CheckboxInput' %}
{% endif %} {{ field }} - {% endifequal %} + {% endif %}
{{ field.help_text }}
{% endfor %} diff --git a/qgis-app/plugins/templates/plugins/pagination.html b/qgis-app/plugins/templates/plugins/pagination.html index fc7e65c7..99bec111 100644 --- a/qgis-app/plugins/templates/plugins/pagination.html +++ b/qgis-app/plugins/templates/plugins/pagination.html @@ -8,11 +8,11 @@ {% endif %} {% for page in pages %} {% if page %} - {% ifequal page page_obj.number %} + {% if page == page_obj.number %} {{ page }} {% else %} {{ page }} - {% endifequal %} + {% endif %} {% else %} ... {% endif %} diff --git a/qgis-app/plugins/tests/test_change_maintainer.py b/qgis-app/plugins/tests/test_change_maintainer.py index a9357f6f..515b5c46 100644 --- a/qgis-app/plugins/tests/test_change_maintainer.py +++ b/qgis-app/plugins/tests/test_change_maintainer.py @@ -50,7 +50,7 @@ def setUp(self): self.plugin = Plugin.objects.get(name='Test Plugin') self.plugin.save() - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_change_maintainer(self): """ diff --git a/qgis-app/plugins/tests/test_plugin_update.py b/qgis-app/plugins/tests/test_plugin_update.py index 97e842e2..d1e4c34b 100644 --- a/qgis-app/plugins/tests/test_plugin_update.py +++ b/qgis-app/plugins/tests/test_plugin_update.py @@ -51,7 +51,7 @@ def setUp(self): self.plugin = Plugin.objects.get(name='Test Plugin') - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_plugin_new_version(self): """ @@ -103,7 +103,7 @@ def test_plugin_new_version(self): settings.EMAIL_HOST_USER ) - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_plugin_version_update(self): """ diff --git a/qgis-app/plugins/tests/test_plugin_upload.py b/qgis-app/plugins/tests/test_plugin_upload.py index 3539701a..9b3d2814 100644 --- a/qgis-app/plugins/tests/test_plugin_upload.py +++ b/qgis-app/plugins/tests/test_plugin_upload.py @@ -34,7 +34,7 @@ def setUp(self): email='test@example.com' ) - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_plugin_upload_form(self): # Log in the test user diff --git a/qgis-app/plugins/tests/test_rename_plugin.py b/qgis-app/plugins/tests/test_rename_plugin.py index d1eb890b..78ff7d69 100644 --- a/qgis-app/plugins/tests/test_rename_plugin.py +++ b/qgis-app/plugins/tests/test_rename_plugin.py @@ -51,7 +51,7 @@ def setUp(self): self.plugin.name = "New name Test Plugin" self.plugin.save() - @patch("plugins.tasks.generate_plugins_xml.delay", new=do_nothing) + @patch("plugins.tasks.generate_plugins_xml", new=do_nothing) @patch("plugins.validator._check_url_link", new=do_nothing) def test_plugin_rename(self): """ diff --git a/qgis-app/plugins/tests/test_validator.py b/qgis-app/plugins/tests/test_validator.py index 7a0e7e4e..6b1e9051 100644 --- a/qgis-app/plugins/tests/test_validator.py +++ b/qgis-app/plugins/tests/test_validator.py @@ -120,10 +120,7 @@ def test_check_url_link_ssl_error(self, mock_request): @mock.patch("requests.get", side_effect=requests.exceptions.HTTPError()) def test_check_url_link_does_not_exist(self, mock_request): url = "http://example.com/" - self.assertRaises( - ValidationError, - _check_url_link(url, "forbidden_url", "metadata attribute"), - ) + self.assertIsNone(_check_url_link(url, "forbidden_url", "metadata attribute")) class TestValidatorForbiddenFileFolder(TestCase): diff --git a/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.2.zip_ b/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.2.zip_ index c694809e..f5ba6212 100644 Binary files a/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.2.zip_ and b/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.2.zip_ differ diff --git a/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.3.zip_ b/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.3.zip_ index f838d489..613f0f1f 100644 Binary files a/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.3.zip_ and b/qgis-app/plugins/tests/testfiles/valid_plugin_0.0.3.zip_ differ diff --git a/qgis-app/plugins/tests/tests.py b/qgis-app/plugins/tests/tests.py index 3748f41b..f51d798f 100644 --- a/qgis-app/plugins/tests/tests.py +++ b/qgis-app/plugins/tests/tests.py @@ -13,7 +13,7 @@ def test_basic_addition(self): """ Tests that 1 + 1 always equals 2. """ - self.failUnlessEqual(1 + 1, 2) + self.assertEqual(1 + 1, 2) __test__ = { diff --git a/qgis-app/plugins/urls.py b/qgis-app/plugins/urls.py index dd021892..3d1c5da3 100644 --- a/qgis-app/plugins/urls.py +++ b/qgis-app/plugins/urls.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from django.conf.urls import include, url +from django.urls import re_path as url from django.contrib.auth.decorators import login_required, user_passes_test -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from plugins.models import Plugin, PluginVersion from plugins.views import * from rpc4django.views import serve_rpc_request diff --git a/qgis-app/plugins/validator.py b/qgis-app/plugins/validator.py index e1482bd4..f8ebd6ad 100644 --- a/qgis-app/plugins/validator.py +++ b/qgis-app/plugins/validator.py @@ -15,7 +15,7 @@ from django.conf import settings from django.core.files.uploadedfile import SimpleUploadedFile from django.forms import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ PLUGIN_MAX_UPLOAD_SIZE = getattr(settings, "PLUGIN_MAX_UPLOAD_SIZE", 25000000) # 25 mb PLUGIN_REQUIRED_METADATA = getattr( @@ -61,16 +61,16 @@ def _read_from_init(initcontent, initname): i = 0 lines = initcontent.split("\n") while i < len(lines): - if re.search("def\s+([^\(]+)", lines[i]): - k = re.search("def\s+([^\(]+)", lines[i]).groups()[0] + if re.search(r"def\s+([^\(]+)", lines[i]): + k = re.search(r"def\s+([^\(]+)", lines[i]).groups()[0] i += 1 while i < len(lines) and lines[i] != "": - if re.search("return\s+[\"']?([^\"']+)[\"']?", lines[i]): + if re.search(r"return\s+[\"']?([^\"']+)[\"']?", lines[i]): metadata.append( ( k, re.search( - "return\s+[\"']?([^\"']+)[\"']?", lines[i] + r"return\s+[\"']?([^\"']+)[\"']?", lines[i] ).groups()[0], ) ) @@ -255,7 +255,7 @@ def validator(package): try: parser = configparser.ConfigParser() parser.optionxform = str - parser.readfp(StringIO(codecs.decode(zip.read(metadataname), "utf8"))) + parser.read_file(StringIO(codecs.decode(zip.read(metadataname), "utf8"))) if not parser.has_section("general"): raise ValidationError( _("Cannot find a section named 'general' in %s") % metadataname diff --git a/qgis-app/plugins/views.py b/qgis-app/plugins/views.py index d1897117..20dfca1c 100644 --- a/qgis-app/plugins/views.py +++ b/qgis-app/plugins/views.py @@ -21,7 +21,7 @@ from django.utils.timezone import now from django.utils.decorators import method_decorator from django.utils.encoding import DjangoUnicodeDecodeError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.views.decorators.cache import never_cache from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt, csrf_protect from django.views.decorators.http import require_POST diff --git a/qgis-app/qgis_context_processor.py b/qgis-app/qgis_context_processor.py index 11ff0481..8b8de12c 100644 --- a/qgis-app/qgis_context_processor.py +++ b/qgis-app/qgis_context_processor.py @@ -2,12 +2,14 @@ from django.contrib.sites.models import Site from django.core.exceptions import ImproperlyConfigured +def is_ajax(request): + return request.META.get('HTTP_X_REQUESTED_WITH') == 'XMLHttpRequest' def additions(request): """Insert some additional information into the template context from the settings and set the base template according to qs. """ - if request.is_ajax() or request.GET.get("ajax"): + if is_ajax(request=request) or request.GET.get("ajax"): base_template = "ajax_base.html" is_naked = True else: diff --git a/qgis-app/settings.py b/qgis-app/settings.py index c3e7a79b..23d50828 100644 --- a/qgis-app/settings.py +++ b/qgis-app/settings.py @@ -253,7 +253,7 @@ DEBUG_TOOLBAR_CONFIG = {"INTERCEPT_REDIRECTS": False} -THUMBNAIL_ENGINE = "sorl.thumbnail.engines.convert_engine.Engine" +# THUMBNAIL_ENGINE = "sorl.thumbnail.engines.convert_engine.Engine" USER_MAP = { "project_name": "QGIS", diff --git a/qgis-app/settings_docker.py b/qgis-app/settings_docker.py index f51dc50b..c0b7be1e 100644 --- a/qgis-app/settings_docker.py +++ b/qgis-app/settings_docker.py @@ -1,3 +1,6 @@ +from celery.schedules import crontab + +from settings import * import ast import os @@ -13,7 +16,7 @@ # Absolute filesystem path to the directory that will hold user-uploaded files. # Example: "/var/www/example.com/media/" -MEDIA_ROOT = os.environ.get("MEDIA_ROOT", "/home/web/media") +MEDIA_ROOT = os.environ.get("MEDIA_ROOT", "/home/web/media/") # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash. @@ -26,7 +29,7 @@ # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. # Example: "/var/www/example.com/static/" -STATIC_ROOT = os.environ.get("STATIC_ROOT", "/home/web/static") +STATIC_ROOT = os.environ.get("STATIC_ROOT", "/home/web/static/") # URL prefix for static files. # Example: "http://example.com/static/", "http://static.example.com/" @@ -106,7 +109,7 @@ PAGINATION_DEFAULT_PAGINATION_HUB = 30 LOGIN_REDIRECT_URL = "/" SERVE_STATIC_MEDIA = DEBUG -DEFAULT_PLUGINS_SITE = os.environ.get("DEFAULT_PLUGINS_SITE", "") +DEFAULT_PLUGINS_SITE = os.environ.get("DEFAULT_PLUGINS_SITE", "https://plugins.qgis.org/") # See fig.yml file for postfix container definition # @@ -131,6 +134,21 @@ "TEST_REQUEST_DEFAULT_FORMAT": "json", } +CELERY_RESULT_BACKEND = 'rpc://' +CELERY_BROKER_URL = os.environ.get('BROKER_URL', 'amqp://rabbitmq:5672') +CELERY_BEAT_SCHEDULE = { + 'generate_plugins_xml': { + 'task': 'plugins.tasks.generate_plugins_xml.generate_plugins_xml', + 'schedule': crontab(minute='*/10'), # Execute every 10 minutes. + 'kwargs': { + 'site': DEFAULT_PLUGINS_SITE + } + }, + 'update_feedjack': { + 'task': 'plugins.tasks.update_feedjack.update_feedjack', + 'schedule': crontab(minute='*/30'), # Execute every 30 minutes. + } +} # Set plugin token access and refresh validity to a very long duration SIMPLE_JWT = { 'ACCESS_TOKEN_LIFETIME': timedelta(days=365*1000), @@ -139,3 +157,6 @@ MATOMO_SITE_ID="1" MATOMO_URL="//matomo.qgis.org/" + +# Default primary key type +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' \ No newline at end of file diff --git a/qgis-app/styles/file_handler.py b/qgis-app/styles/file_handler.py index e35e855f..42b3c2de 100644 --- a/qgis-app/styles/file_handler.py +++ b/qgis-app/styles/file_handler.py @@ -4,7 +4,7 @@ import xml.etree.ElementTree as ET from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ def _check_name_type_attribute(element): diff --git a/qgis-app/styles/forms.py b/qgis-app/styles/forms.py index 3fe24047..a23655ff 100644 --- a/qgis-app/styles/forms.py +++ b/qgis-app/styles/forms.py @@ -1,6 +1,6 @@ from django import forms from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from styles.file_handler import validator from styles.models import Style diff --git a/qgis-app/styles/migrations/0015_alter_review_reviewer.py b/qgis-app/styles/migrations/0015_alter_review_reviewer.py new file mode 100644 index 00000000..8c6b0558 --- /dev/null +++ b/qgis-app/styles/migrations/0015_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('styles', '0014_remove_uuid_null'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/styles/models.py b/qgis-app/styles/models.py index f1e4f753..52e62f51 100644 --- a/qgis-app/styles/models.py +++ b/qgis-app/styles/models.py @@ -3,7 +3,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ STYLES_STORAGE_PATH = getattr(settings, "PLUGINS_STORAGE_PATH", "styles/%Y") diff --git a/qgis-app/styles/views.py b/qgis-app/styles/views.py index 84cfb874..db156da4 100644 --- a/qgis-app/styles/views.py +++ b/qgis-app/styles/views.py @@ -19,7 +19,7 @@ from django.http import HttpResponse, HttpResponseRedirect from django.urls import reverse, reverse_lazy from django.utils.crypto import get_random_string -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.views.decorators.cache import never_cache from styles.file_handler import read_xml_style from styles.forms import UpdateForm, UploadForm diff --git a/qgis-app/templates/cab/snippet_detail.html b/qgis-app/templates/cab/snippet_detail.html index 38c97359..d47040ae 100644 --- a/qgis-app/templates/cab/snippet_detail.html +++ b/qgis-app/templates/cab/snippet_detail.html @@ -56,9 +56,9 @@

More like this

Tools

    {% if user.is_authenticated %} - {% ifequal user.id object.author.id %} + {% if user.id == object.author.id %}
  • Edit this snippet
    • - {% endifequal %} + {% endif %} {% if object|is_bookmarked:request.user %}
  • This snippet is in your bookmarks
    • {% else %} @@ -66,7 +66,7 @@

    Tools

    {% endif %} {% with object|rating_score:request.user as rating_score %} {% if rating_score %} -
  • You rated this snippet {% ifequal rating_score 1 %}useful{% else %}not useful{% endifequal %}.
    • +
  • You rated this snippet {% if rating_score == 1 %}useful{% else %}not useful{% endif %}.
    • {% else %}
  • Rate this snippet: useful or not useful
    • {% endif %} diff --git a/qgis-app/templates/snippet_base.html b/qgis-app/templates/snippet_base.html index d1a359fa..0e4c29b5 100644 --- a/qgis-app/templates/snippet_base.html +++ b/qgis-app/templates/snippet_base.html @@ -36,7 +36,7 @@

    QGIS Snippets

    {% if user.is_authenticated %} Logged in as {{ user.username }} (Add a snippet) (Log out) {% else %} - Log in or register + Log in or register {% endif %}
      diff --git a/qgis-app/urls.py b/qgis-app/urls.py index 7166f5c1..90acac68 100644 --- a/qgis-app/urls.py +++ b/qgis-app/urls.py @@ -1,6 +1,6 @@ import simplemenu from django.conf import settings -from django.conf.urls import url +from django.urls import re_path as url # Uncomment the next two lines to enable the admin: from django.contrib import admin diff --git a/qgis-app/userexport/urls.py b/qgis-app/userexport/urls.py index 84d6a35c..45ac6d08 100755 --- a/qgis-app/userexport/urls.py +++ b/qgis-app/userexport/urls.py @@ -1,5 +1,5 @@ # -* coding:utf-8 *- # -from django.conf.urls import * +from django.urls import re_path as url from userexport.views import * urlpatterns = [ diff --git a/qgis-app/users/tests.py b/qgis-app/users/tests.py index 3748f41b..f51d798f 100644 --- a/qgis-app/users/tests.py +++ b/qgis-app/users/tests.py @@ -13,7 +13,7 @@ def test_basic_addition(self): """ Tests that 1 + 1 always equals 2. """ - self.failUnlessEqual(1 + 1, 2) + self.assertEqual(1 + 1, 2) __test__ = { diff --git a/qgis-app/users/urls.py b/qgis-app/users/urls.py index dbcb3317..dd51a6d9 100644 --- a/qgis-app/users/urls.py +++ b/qgis-app/users/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls import include, url +from django.urls import re_path as url from django.views.generic.base import TemplateView from .views import * diff --git a/qgis-app/users/views.py b/qgis-app/users/views.py index 46234176..25d08f3e 100644 --- a/qgis-app/users/views.py +++ b/qgis-app/users/views.py @@ -1,21 +1,23 @@ # python logging support to django logging middleware -import logging - -from annoying.functions import get_object_or_None -from django.conf import settings +from django.http import HttpResponse +from django.http import HttpResponseRedirect from django.contrib import messages -from django.contrib.sites.models import Site -from django.core.mail import send_mail -from django.http import HttpResponse, HttpResponseRedirect -from django.shortcuts import get_object_or_404, render_to_response +from django.shortcuts import render, get_object_or_404 +from annoying.functions import get_object_or_None from django.template import RequestContext +from django.utils.translation import gettext_lazy as _ +from django.core.mail import send_mail from django.template.loader import render_to_string -from django.utils.translation import ugettext_lazy as _ -from olwidget.fields import EditableLayerField, MapField -from olwidget.widgets import EditableLayer, InfoLayer, InfoMap, Map +from django.contrib.sites.models import Site +from django.conf import settings from users.forms import * from users.models import * +from olwidget.fields import MapField, EditableLayerField +from olwidget.widgets import Map, EditableLayer, InfoLayer, InfoMap +import os +# python logging support to django logging middleware +import logging def usersMap(theRequest): @@ -32,16 +34,15 @@ def usersMap(theRequest): myMap = InfoMap(users) - return render_to_response( + return render( + theRequest, "view_users.html", { - "myMap": myMap, - "myUserCount": myUserCount, - "myRandomUser": myRandomUser, + 'myMap' : myMap, + 'myUserCount' : myUserCount, + 'myRandomUser' : myRandomUser, }, - context_instance=RequestContext(theRequest), - ) - + context_instance=RequestContext(theRequest)) def createUser(theRequest): @@ -52,41 +53,29 @@ def createUser(theRequest): myForm.save() return HttpResponseRedirect("/community-map/view_users.html") else: - return render_to_response( - "create_user_form.html", - {"myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) + return render(theRequest, "create_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount }, + context_instance=RequestContext(theRequest)) else: myForm = QgisUserForm() - return render_to_response( - "create_user_form.html", - {"myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) + return render(theRequest, "create_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount }, + context_instance=RequestContext(theRequest)) def updateUser(theRequest, theId): - myUserCount = QgisUser.objects.all().count() - myUser = get_object_or_404(QgisUser, guid=theId) - if theRequest.method == "POST": - myForm = QgisUserForm(theRequest.POST, theRequest.FILES, instance=myUser) - if myForm.is_valid(): - myForm.save() - return HttpResponseRedirect("/community-map/view_users.html") - return render_to_response( - "update_user_form.html", - {"myUser": myUser, "myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) - else: - myForm = QgisUserForm(instance=myUser) - return render_to_response( - "update_user_form.html", - {"myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) - + myUserCount = QgisUser.objects.all().count() + myUser = get_object_or_404(QgisUser,guid=theId) + if theRequest.method == 'POST': + myForm = QgisUserForm(theRequest.POST, theRequest.FILES, instance=myUser) + if myForm.is_valid(): + myForm.save() + return HttpResponseRedirect("/community-map/view_users.html") + return render(theRequest, "update_user_form.html", { + 'myUser': myUser, 'myForm': myForm, 'myUserCount' : myUserCount + }, context_instance=RequestContext(theRequest)) + else: + myForm = QgisUserForm(instance=myUser) + return render(theRequest, "update_user_form.html", {'myForm' : myForm, 'myUserCount' : myUserCount}, + context_instance=RequestContext(theRequest)) def emailEditAddress(theRequest): @@ -115,22 +104,17 @@ def emailEditAddress(theRequest): send_mail(subject, message, sender, [recipient]) return HttpResponseRedirect("/community-map/edit/email_confirm.html") + else: myUserCount = QgisUser.objects.all().count() msg = _("User is NOT valid.") messages.warning(theRequest, msg, fail_silently=True) logging.info("User or form is NOT valid") - return render_to_response( - "update_user.html", - {"myForm": myForm, "myUserCount": myUserCount}, - context_instance=RequestContext(theRequest), - ) + return render(theRequest, "update_user.html", {'myForm' : myForm, 'myUserCount' : myUserCount}, + context_instance=RequestContext(theRequest)) else: myUserCount = QgisUser.objects.all().count() myForm = EmailForm() - return render_to_response( - "update_user.html", - {"myForm": myForm}, - context_instance=RequestContext(theRequest), - ) + return render(theRequest, "update_user.html", {'myForm' : myForm}, + context_instance=RequestContext(theRequest)) diff --git a/qgis-app/wavefronts/migrations/0002_alter_review_reviewer.py b/qgis-app/wavefronts/migrations/0002_alter_review_reviewer.py new file mode 100644 index 00000000..79efde7f --- /dev/null +++ b/qgis-app/wavefronts/migrations/0002_alter_review_reviewer.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.11 on 2024-03-22 00:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('wavefronts', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='review', + name='reviewer', + field=models.ForeignKey(help_text='The user who reviewed this %(app_label)s.', on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_related', to=settings.AUTH_USER_MODEL, verbose_name='Reviewed by'), + ), + ] diff --git a/qgis-app/wavefronts/models.py b/qgis-app/wavefronts/models.py index 7c5c52f4..65b10e85 100644 --- a/qgis-app/wavefronts/models.py +++ b/qgis-app/wavefronts/models.py @@ -6,7 +6,7 @@ from django.core.validators import FileExtensionValidator from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from preferences.models import Preferences WAVEFRONTS_STORAGE_PATH = getattr(settings, "WAVEFRONTS_STORAGE_PATH", "wavefronts") diff --git a/qgis-app/wavefronts/tests/test_view.py b/qgis-app/wavefronts/tests/test_view.py index 364f4929..380848e6 100644 --- a/qgis-app/wavefronts/tests/test_view.py +++ b/qgis-app/wavefronts/tests/test_view.py @@ -170,7 +170,7 @@ def test_download_should_return_zipfile(self): url = reverse("wavefront_download", kwargs={"pk": wavefront.id}) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertEquals( + self.assertEqual( response.get("Content-Disposition"), "attachment; filename=odm-texturing.zip", ) diff --git a/qgis-app/wavefronts/validator.py b/qgis-app/wavefronts/validator.py index 09e8f338..deff6be8 100644 --- a/qgis-app/wavefronts/validator.py +++ b/qgis-app/wavefronts/validator.py @@ -9,7 +9,7 @@ import pywavefront from django.conf import settings from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from wavefronts.models import WAVEFRONTS_STORAGE_PATH diff --git a/qgis-app/wavefronts/views.py b/qgis-app/wavefronts/views.py index e7c11151..a93458ea 100644 --- a/qgis-app/wavefronts/views.py +++ b/qgis-app/wavefronts/views.py @@ -20,7 +20,7 @@ from django.template.response import TemplateResponse from django.urls import reverse_lazy from django.utils.text import slugify -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from wavefronts.forms import UpdateForm, UploadForm from wavefronts.models import Review, Wavefront from wavefronts.utilities import zipped_all_with_license diff --git a/qgis-app/wsgi.py b/qgis-app/wsgi.py index d56c6515..71f8df99 100644 --- a/qgis-app/wsgi.py +++ b/qgis-app/wsgi.py @@ -4,7 +4,7 @@ It exposes the WSGI callable as a module-level variable named ``application``. For more information on this file, see -https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ +https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ """ import os diff --git a/setup.cfg b/setup.cfg index fae25c68..2fa09d19 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,6 +3,7 @@ exclude = .git, .venv/, ./.venv/, + venv/, vagrant_assets, qgis-app/*/migrations/, qgis-app/plugins/tests/HelloWorld/,