From 8ec6ac9ed3ebdbdd279052f2037056eb06e92e8a Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Sun, 29 Sep 2024 22:49:58 -0400 Subject: [PATCH 01/10] chore: Rename cove_project to core, like in handbook --- .github/workflows/ci.yml | 2 +- {cove_project => core}/__init__.py | 0 {cove_project => core}/context_processors.py | 0 {cove_project => core}/settings.py | 0 {cove_project => core}/templates/terms.html | 0 {cove_project => core}/urls.py | 0 {cove_project => core}/wsgi.py | 4 ++-- docs/architecture.rst | 4 ++-- docs/how-to-config-frontend.rst | 4 ++-- docs/tests.rst | 2 +- manage.py | 2 +- pyproject.toml | 2 +- 12 files changed, 10 insertions(+), 10 deletions(-) rename {cove_project => core}/__init__.py (100%) rename {cove_project => core}/context_processors.py (100%) rename {cove_project => core}/settings.py (100%) rename {cove_project => core}/templates/terms.html (100%) rename {cove_project => core}/urls.py (100%) rename {cove_project => core}/wsgi.py (72%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19668ee3..3958f42f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,5 +31,5 @@ jobs: ./manage.py migrate ./manage.py makemigrations --check --dry-run ./manage.py check --fail-level WARNING - coverage run --source=cove_ocds,cove_project -m pytest -W error -W ignore::DeprecationWarning:ijson.compat -W ignore::ResourceWarning + coverage run --source=core,cove_ocds -m pytest -W error -W ignore::DeprecationWarning:ijson.compat -W ignore::ResourceWarning - uses: coverallsapp/github-action@v2 diff --git a/cove_project/__init__.py b/core/__init__.py similarity index 100% rename from cove_project/__init__.py rename to core/__init__.py diff --git a/cove_project/context_processors.py b/core/context_processors.py similarity index 100% rename from cove_project/context_processors.py rename to core/context_processors.py diff --git a/cove_project/settings.py b/core/settings.py similarity index 100% rename from cove_project/settings.py rename to core/settings.py diff --git a/cove_project/templates/terms.html b/core/templates/terms.html similarity index 100% rename from cove_project/templates/terms.html rename to core/templates/terms.html diff --git a/cove_project/urls.py b/core/urls.py similarity index 100% rename from cove_project/urls.py rename to core/urls.py diff --git a/cove_project/wsgi.py b/core/wsgi.py similarity index 72% rename from cove_project/wsgi.py rename to core/wsgi.py index bc339f77..a08c895d 100644 --- a/cove_project/wsgi.py +++ b/core/wsgi.py @@ -1,5 +1,5 @@ """ -WSGI config for cove_project project. +WSGI config for core project. It exposes the WSGI callable as a module-level variable named ``application``. @@ -11,6 +11,6 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cove_project.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "core.settings") application = get_wsgi_application() diff --git a/docs/architecture.rst b/docs/architecture.rst index 085536db..6ec97977 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -15,7 +15,7 @@ cove-ocds ``cove_ocds/views.py`` does most of the heavy lifting of taking an input file from the web interface and carrying out the various validation checks and conversions, then piping the output back to the right templates. -``cove_project/`` contains the Django components (settings, URL paths, server). +``core/`` contains the Django components (settings, URL paths, server). lib-cove-ocds ------------- @@ -41,7 +41,7 @@ OCDS Show Configuration ------------- -Some configuration variables are set in ``COVE_CONFIG``, found in ``cove_project/settings.py``. +Some configuration variables are set in ``COVE_CONFIG``, found in ``core/settings.py``. * ``app_name``, ``app_verbose_name``, ``app_strapline``, ``support_email``: set human readable strings for the DRT that can be reused in templates etc. * ``app_base_template``, ``input_template``, ``input_methods``: set the templates for the landing page. diff --git a/docs/how-to-config-frontend.rst b/docs/how-to-config-frontend.rst index 98a41061..6c73f0eb 100644 --- a/docs/how-to-config-frontend.rst +++ b/docs/how-to-config-frontend.rst @@ -3,14 +3,14 @@ How to replace frontend hardcoding with a configurable variable Pick a name for the environment variable you want to configure cove with. In this example we use `MY_ENV_VAR`. -Edit `cove_project/settings.py`, and add a setting. The setting's name typically matches the environment variable's name. Set a default value, if appropriate. For example: +Edit `core/settings.py`, and add a setting. The setting's name typically matches the environment variable's name. Set a default value, if appropriate. For example: .. code:: python MY_ENV_VAR = os.getenv("MY_ENV_VAR", "default value") -Edit `cove_project/context_processors.py`, and add a mapping between the setting name, and what name you want use in the template: we've picked `my_var`. +Edit `core/context_processors.py`, and add a mapping between the setting name, and what name you want use in the template: we've picked `my_var`. .. code:: python diff --git a/docs/tests.rst b/docs/tests.rst index 66a6938a..f71dd8ac 100644 --- a/docs/tests.rst +++ b/docs/tests.rst @@ -5,7 +5,7 @@ Run the tests with: .. code:: bash - coverage run --source=cove_ocds,cove_project -m pytest + coverage run --source=cove_ocds,core -m pytest See ``tests/fixtures`` for good and bad JSON and XML files for testing the DRT. diff --git a/manage.py b/manage.py index 0c1fd3e6..d68553b4 100755 --- a/manage.py +++ b/manage.py @@ -3,7 +3,7 @@ import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cove_project.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "core.settings") from django.core.management import execute_from_command_line diff --git a/pyproject.toml b/pyproject.toml index 9f65a296..ab42990e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,4 +41,4 @@ ignore-variadic-names = true ] [tool.pytest.ini_options] -DJANGO_SETTINGS_MODULE = 'cove_project.settings' +DJANGO_SETTINGS_MODULE = 'core.settings' From 32157707cf938870a32d967e3b438649618a833b Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Mon, 30 Sep 2024 00:16:02 -0400 Subject: [PATCH 02/10] chore: Align with Django 4.2 --- core/asgi.py | 16 ++++++++++++++++ manage.py | 10 +++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 core/asgi.py diff --git a/core/asgi.py b/core/asgi.py new file mode 100644 index 00000000..66c1231f --- /dev/null +++ b/core/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for core project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "core.settings") + +application = get_asgi_application() diff --git a/manage.py b/manage.py index d68553b4..bd9928d2 100755 --- a/manage.py +++ b/manage.py @@ -1,10 +1,18 @@ #!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" + import os import sys -if __name__ == "__main__": + +def main(): + """Run administrative tasks.""" os.environ.setdefault("DJANGO_SETTINGS_MODULE", "core.settings") from django.core.management import execute_from_command_line execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() From 9cf64846198ce39edb37b7a631fba8ebb9afc4c9 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Mon, 30 Sep 2024 00:16:43 -0400 Subject: [PATCH 03/10] chore: Upgrade to Python 3.11 --- .github/workflows/ci.yml | 2 +- .github/workflows/i18n.yml | 2 +- .github/workflows/lint.yml | 2 +- .pre-commit-config.yaml | 2 +- .python-version | 2 +- .readthedocs.yaml | 2 +- pyproject.toml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3958f42f..cb98bfd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' cache: pip cache-dependency-path: '**/requirements*.txt' - run: pip install -r requirements.txt diff --git a/.github/workflows/i18n.yml b/.github/workflows/i18n.yml index b6b60409..a8d4f536 100644 --- a/.github/workflows/i18n.yml +++ b/.github/workflows/i18n.yml @@ -10,7 +10,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' cache: pip cache-dependency-path: '**/requirements*.txt' - name: Install translate-toolkit diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2fc37d9d..6a60c4c8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,7 +14,7 @@ jobs: token: ${{ secrets.PAT || github.token }} - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' cache: pip cache-dependency-path: '**/requirements*.txt' - id: changed-files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8d828bca..54e08b83 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ ci: autoupdate_schedule: quarterly skip: [pip-compile] default_language_version: - python: python3.10 + python: python3.11 repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.6.3 diff --git a/.python-version b/.python-version index c8cfe395..2c073331 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.10 +3.11 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e25d724c..7f84a76c 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -2,7 +2,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.10" + python: "3.11" python: install: - requirements: docs/requirements.txt diff --git a/pyproject.toml b/pyproject.toml index ab42990e..01cd4019 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "0.0.0" [tool.ruff] line-length = 119 -target-version = "py310" +target-version = "py311" [tool.ruff.lint] select = ["ALL"] From 1a6d3ffd62ccd010819939d619a9cc3fd251b9aa Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Mon, 30 Sep 2024 00:18:10 -0400 Subject: [PATCH 04/10] feat: Add files for Docker. Rewrite settings.py to match cookiecutter and use Docker paths for media and db. --- .dockerignore | 26 ++++ .github/workflows/docker.yml | 38 ++++++ Dockerfile_django | 43 ++++++ Dockerfile_static | 17 +++ core/settings.py | 251 +++++++++++++++++++++++++---------- default.conf | 56 ++++++++ 6 files changed, 359 insertions(+), 72 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/docker.yml create mode 100644 Dockerfile_django create mode 100644 Dockerfile_static create mode 100644 default.conf diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..b8f6dd23 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,26 @@ +# Dotfiles +**/.* + +# Docker +**/Dockerfile* + +# Requirements +**/requirements.in +**/requirements_dev.in +**/requirements_dev.txt + +# Documentation +docs +LICENSE +README.md +README.rst + +# Configuration +pyproject.toml + +# Testing +**/tests +pytest.ini + +# Generated files +**/node_modules diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..508d90da --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,38 @@ +name: Deploy +on: + workflow_run: + workflows: ["CI"] + branches: [main] + types: + - completed +jobs: + docker: + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # https://github.com/docker/login-action#github-container-registry + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # https://github.com/docker/setup-buildx-action#usage + - uses: docker/setup-buildx-action@v3 + # https://github.com/docker/build-push-action#usage + - uses: docker/build-push-action@v6 + with: + push: true + file: Dockerfile_django + tags: | + ghcr.io/${{ github.repository }}-django:latest + cache-from: type=gha + cache-to: type=gha,mode=max + - uses: docker/build-push-action@v6 + with: + push: true + file: Dockerfile_static + tags: | + ghcr.io/${{ github.repository }}-static:latest + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Dockerfile_django b/Dockerfile_django new file mode 100644 index 00000000..96255db4 --- /dev/null +++ b/Dockerfile_django @@ -0,0 +1,43 @@ +FROM python:3.11 as build-stage + +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +WORKDIR /workdir + +COPY . . + +ENV DJANGO_ENV=production + +RUN python manage.py collectstatic --noinput -v2 + +FROM python:3.11 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gettext \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd -r runner && useradd --no-log-init -r -g runner runner + +# Must match the settings.DATABASES default value. +RUN mkdir -p /data/db && chown -R runner:runner /data/db +# Must match the settings.MEDIA_ROOT default value. +RUN mkdir -p /data/media && chown -R runner:runner /data/media + +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +WORKDIR /workdir +USER runner:runner +COPY --chown=runner:runner . . + +# Django needs a copy of the staticfiles.json manifest file. +COPY --from=build-stage --chown=runner:runner /workdir/static/staticfiles.json /workdir/static/staticfiles.json + +ENV DJANGO_ENV=production +ENV WEB_CONCURRENCY=2 + +RUN python manage.py compilemessages + +EXPOSE 8000 +CMD ["gunicorn", "core.wsgi", "--bind", "0.0.0.0:8000", "--worker-tmp-dir", "/dev/shm", "--threads", "2", "--name", "cove"] diff --git a/Dockerfile_static b/Dockerfile_static new file mode 100644 index 00000000..a6360e86 --- /dev/null +++ b/Dockerfile_static @@ -0,0 +1,17 @@ +FROM python:{{ cookiecutter.python_version }} as build-stage + +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +WORKDIR /workdir +COPY . . + +ENV DJANGO_ENV=production + +RUN python manage.py collectstatic --noinput -v2 + +FROM nginxinc/nginx-unprivileged:latest as production-stage +USER root +COPY --from=build-stage --chown=nginx:root /workdir/static /usr/share/nginx/html/static +COPY --chown=nginx:root default.conf /etc/nginx/conf.d/default.conf +USER nginx diff --git a/core/settings.py b/core/settings.py index a6718c7c..483e4971 100644 --- a/core/settings.py +++ b/core/settings.py @@ -1,47 +1,43 @@ +""" +Django settings for the project. + +Generated by 'django-admin startproject'. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.2/ref/settings/ +""" + import os +from glob import glob from pathlib import Path -from cove import settings +import sentry_sdk +from sentry_sdk.integrations.django import DjangoIntegration +from sentry_sdk.integrations.logging import ignore_logger + +production = os.getenv("DJANGO_ENV") == "production" +local_access = "LOCAL_ACCESS" in os.environ or "ALLOWED_HOSTS" not in os.environ -# Build paths inside the project like this: BASE_DIR / "subdir". +# Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent -# We use the setting to choose whether to show the section about Sentry in the -# terms and conditions -SENTRY_DSN = os.getenv("SENTRY_DSN", "") -if SENTRY_DSN: - import sentry_sdk - from sentry_sdk.integrations.django import DjangoIntegration - from sentry_sdk.integrations.logging import ignore_logger +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ - ignore_logger("django.security.DisallowedHost") - sentry_sdk.init(dsn=SENTRY_DSN, integrations=[DjangoIntegration()]) +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.getenv("SECRET_KEY", "7ur)dt+e%1^e6$8_sd-@1h67_5zixe2&39%r2$$8_7v6fr_7ee") -FATHOM = { - "domain": os.getenv("FATHOM_ANALYTICS_DOMAIN", "cdn.usefathom.com"), - "id": os.getenv("FATHOM_ANALYTICS_ID", ""), -} -HOTJAR = { - "id": os.getenv("HOTJAR_ID", ""), - "sv": os.getenv("HOTJAR_SV", ""), - "date_info": os.getenv("HOTJAR_DATE_INFO", ""), -} -RELEASES_OR_RECORDS_TABLE_LENGTH = int(os.getenv("RELEASES_OR_RECORDS_TABLE_LENGTH", "25")) -VALIDATION_ERROR_LOCATIONS_LENGTH = settings.VALIDATION_ERROR_LOCATIONS_LENGTH -VALIDATION_ERROR_LOCATIONS_SAMPLE = settings.VALIDATION_ERROR_LOCATIONS_SAMPLE -REQUESTS_TIMEOUT = int(os.getenv("REQUESTS_TIMEOUT", "10")) -DELETE_FILES_AFTER_DAYS = int(os.getenv("DELETE_FILES_AFTER_DAYS", "90")) +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = not production -# We can't take MEDIA_ROOT and MEDIA_URL from cove settings, -# ... otherwise the files appear under the BASE_DIR that is the Cove library install. -# That could get messy. We want them to appear in our directory. -MEDIA_ROOT = BASE_DIR / "media" -MEDIA_URL = "/media/" +ALLOWED_HOSTS = [".localhost", "127.0.0.1", "[::1]", "0.0.0.0"] # noqa: S104 # Docker +if "ALLOWED_HOSTS" in os.environ: + ALLOWED_HOSTS.extend(os.getenv("ALLOWED_HOSTS").split(",")) -SECRET_KEY = os.getenv("SECRET_KEY", "7ur)dt+e%1^e6$8_sd-@1h67_5zixe2&39%r2$$8_7v6fr_7ee") -DEBUG = settings.DEBUG -ALLOWED_HOSTS = settings.ALLOWED_HOSTS # Application definition @@ -60,6 +56,7 @@ MIDDLEWARE = ( + "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.locale.LocaleMiddleware", "django.middleware.common.CommonMiddleware", @@ -67,25 +64,43 @@ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", - "django.middleware.security.SecurityMiddleware", "cove.middleware.CoveConfigCurrentApp", ) +ROOT_URLCONF = "core.urls" -ROOT_URLCONF = "cove_project.urls" +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [BASE_DIR / "core" / "templates"], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.template.context_processors.i18n", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + "cove.context_processors.from_settings", + "core.context_processors.from_settings", + ], + }, + }, +] -TEMPLATES = settings.TEMPLATES -TEMPLATES[0]["DIRS"] = [BASE_DIR / "cove_project" / "templates"] -TEMPLATES[0]["OPTIONS"]["context_processors"].append( - "cove_project.context_processors.from_settings", -) +WSGI_APPLICATION = "core.wsgi.application" -WSGI_APPLICATION = "cove_project.wsgi.application" -# We can't take DATABASES from cove settings, -# ... otherwise the files appear under the BASE_DIR that is the Cove library install. -# That could get messy. We want them to appear in our directory. -DATABASES = {"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": str(BASE_DIR / "db.sqlite3")}} +# Database +# https://docs.djangoproject.com/en/4.2/ref/settings/#databases + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.getenv("DATABASE_PATH", "/data/db/db.sqlite3" if production else str(BASE_DIR / "db.sqlite3")), + } +} + # Password validation # https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators @@ -94,45 +109,128 @@ { "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, - {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, - {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, - {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, ] # Internationalization # https://docs.djangoproject.com/en/4.2/topics/i18n/ -LANGUAGE_CODE = settings.LANGUAGE_CODE -TIME_ZONE = settings.TIME_ZONE -USE_I18N = settings.USE_I18N -USE_TZ = settings.USE_TZ +LANGUAGE_CODE = "en-us" -LANGUAGES = settings.LANGUAGES +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_TZ = True -LOCALE_PATHS = (BASE_DIR / "cove_ocds" / "locale",) # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/4.2/howto/static-files/ -# We can't take STATIC_URL and STATIC_ROOT from cove settings, -# ... otherwise the files appear under the BASE_DIR that is the Cove library install. -# and that doesn't work with our standard Apache setup. STATIC_URL = "static/" -STATIC_ROOT = BASE_DIR / "static" -# Misc +# Default primary key field type +# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" -LOGGING = settings.LOGGING -LOGGING["handlers"]["null"] = { - "class": "logging.NullHandler", + +# Project-specific Django configuration + +LOCALE_PATHS = glob(str(BASE_DIR / "**" / "locale")) + +STATIC_ROOT = BASE_DIR / "static" + +STORAGES = { + "default": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.ManifestStaticFilesStorage", + }, } -LOGGING["loggers"]["django.security.DisallowedHost"] = { - "handlers": ["null"], - "propagate": False, + +# https://docs.djangoproject.com/en/4.2/topics/logging/#django-security +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "console": { + "format": "%(asctime)s %(levelname)s [%(name)s:%(lineno)s] %(message)s", + }, + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "console", + }, + "null": { + "class": "logging.NullHandler", + }, + }, + "loggers": { + "": { + "handlers": ["console"], + "level": "INFO", + }, + "django.security.DisallowedHost": { + "handlers": ["null"], + "propagate": False, + }, + }, } -# OCDS Config +# https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ +if production and not local_access: + # Run: env DJANGO_ENV=production SECURE_HSTS_SECONDS=1 ./manage.py check --deploy + CSRF_COOKIE_SECURE = True + SESSION_COOKIE_SECURE = True + SECURE_SSL_REDIRECT = True + SECURE_REFERRER_POLICY = "same-origin" # default in Django >= 3.1 + + # https://docs.djangoproject.com/en/4.2/ref/middleware/#http-strict-transport-security + if "SECURE_HSTS_SECONDS" in os.environ: + SECURE_HSTS_SECONDS = int(os.getenv("SECURE_HSTS_SECONDS")) + SECURE_HSTS_INCLUDE_SUBDOMAINS = True + SECURE_HSTS_PRELOAD = True + +# https://docs.djangoproject.com/en/4.2/ref/settings/#secure-proxy-ssl-header +if "DJANGO_PROXY" in os.environ: + USE_X_FORWARDED_HOST = True + SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + +LANGUAGES = ( + ("en", "English"), + ("es", "Spanish"), +) + +MEDIA_ROOT = os.getenv("MEDIA_ROOT", "/data/media/" if production else BASE_DIR / "media/") +MEDIA_URL = "media/" + +# https://docs.djangoproject.com/en/4.2/ref/settings/#data-upload-max-memory-size +DATA_UPLOAD_MAX_MEMORY_SIZE = 52428800 # 5 MB + + +# Dependency configuration + +if "SENTRY_DSN" in os.environ: + # https://docs.sentry.io/platforms/python/logging/#ignoring-a-logger + ignore_logger("django.security.DisallowedHost") + sentry_sdk.init( + dsn=os.getenv("SENTRY_DSN"), + integrations=[DjangoIntegration()], + traces_sample_rate=0, # The Sentry plan does not include Performance. + ) + COVE_CONFIG = { # lib-cove-web options @@ -150,12 +248,21 @@ "1.1": ("1.1", "https://standard.open-contracting.org/1.1/{lang}/", "1__1__5"), }, } - -# Set default schema version to the latest version +# Set default schema version to the latest version. COVE_CONFIG["schema_version"] = list(COVE_CONFIG["schema_version_choices"])[-1] -# Because of how the standard site proxies traffic, we want to use this -USE_X_FORWARDED_HOST = True -# https://docs.djangoproject.com/en/4.2/ref/settings/#data-upload-max-memory-size -DATA_UPLOAD_MAX_MEMORY_SIZE = 52428800 # 5 MB +# Project configuration + +FATHOM = { + "domain": os.getenv("FATHOM_ANALYTICS_DOMAIN") or "cdn.usefathom.com", + "id": os.getenv("FATHOM_ANALYTICS_ID"), +} + +HOTJAR = { + "id": os.getenv("HOTJAR_ID", ""), + "sv": os.getenv("HOTJAR_SV", ""), + "date_info": os.getenv("HOTJAR_DATE_INFO", ""), +} + +RELEASES_OR_RECORDS_TABLE_LENGTH = int(os.getenv("RELEASES_OR_RECORDS_TABLE_LENGTH", "25")) diff --git a/default.conf b/default.conf new file mode 100644 index 00000000..d4f4a756 --- /dev/null +++ b/default.conf @@ -0,0 +1,56 @@ +server { + listen 8080; + listen [::]:8080; + server_name localhost; + server_tokens off; + + #access_log /var/log/nginx/host.access.log main; + + location /media/ { + root /data; + } + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + + location /static/ { + expires max; + } + location = /static/staticfiles.json { + expires -1; + } + } + + #error_page 404 /404.html; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + # proxy the PHP scripts to Apache listening on 127.0.0.1:80 + # + #location ~ \.php$ { + # proxy_pass http://127.0.0.1; + #} + + # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 + # + #location ~ \.php$ { + # root html; + # fastcgi_pass 127.0.0.1:9000; + # fastcgi_index index.php; + # fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name; + # include fastcgi_params; + #} + + # deny access to .htaccess files, if Apache's document root + # concurs with nginx's one + # + #location ~ /\.ht { + # deny all; + #} +} From 3967bbd9103ee4a937c94cdebcf0141de16899e7 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 2 Oct 2024 14:22:55 -0400 Subject: [PATCH 05/10] cookiecutter: Enable Sentry Performance --- core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/settings.py b/core/settings.py index 483e4971..9141840a 100644 --- a/core/settings.py +++ b/core/settings.py @@ -228,7 +228,7 @@ sentry_sdk.init( dsn=os.getenv("SENTRY_DSN"), integrations=[DjangoIntegration()], - traces_sample_rate=0, # The Sentry plan does not include Performance. + traces_sample_rate=1.0, ) From 951c09053333a15b00e46b73c82d187ac6637ff4 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 2 Oct 2024 14:37:13 -0400 Subject: [PATCH 06/10] ci: Run collectstatic to satisfy ManifestStaticFilesStorage --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb98bfd9..b5443a6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,7 @@ jobs: sudo apt update sudo apt install gettext - run: ./manage.py compilemessages + - run: ./manage.py collectstatic --noinput -v2 - name: Run checks and tests env: PYTHONWARNINGS: error From 4fbf02d81e848e5c4add4cc71d2ab93d5679c3d6 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 2 Oct 2024 14:54:40 -0400 Subject: [PATCH 07/10] chore: Ignore /static --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f6843bd5..c854d6f7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ .coverage __pycache__ /media +/static /venv docs/_build .hypothesis From d37da73dee1fe729241fee49a030a7a3d7983cf3 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 2 Oct 2024 16:53:12 -0400 Subject: [PATCH 08/10] fix: Fix Docker image name --- Dockerfile_static | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile_static b/Dockerfile_static index a6360e86..e3b117ac 100644 --- a/Dockerfile_static +++ b/Dockerfile_static @@ -1,4 +1,4 @@ -FROM python:{{ cookiecutter.python_version }} as build-stage +FROM python:3.11 as build-stage COPY requirements.txt /tmp/requirements.txt RUN pip install --no-cache-dir -r /tmp/requirements.txt From 71f1ad7192b0e35de2c43b7953557b5596b5d75c Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:01:03 -0400 Subject: [PATCH 09/10] cookiecutter: Add gunicorn --- requirements.in | 1 + requirements.txt | 12 ++++++------ requirements_dev.txt | 26 ++++++++++---------------- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/requirements.in b/requirements.in index f87f8c68..14567c6a 100644 --- a/requirements.in +++ b/requirements.in @@ -1,6 +1,7 @@ django django-bootstrap3 flattentool +gunicorn[setproctitle] libcove libcoveocds[perf,web] libcoveweb diff --git a/requirements.txt b/requirements.txt index 33eae9fc..60cd4f3e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,13 +42,13 @@ django-bootstrap3==24.2 # libcoveweb et-xmlfile==1.1.0 # via openpyxl -exceptiongroup==1.2.2 - # via cattrs flattentool==0.26.0 # via # -r requirements.in # libcove # libcoveweb +gunicorn==23.0.0 + # via -r requirements.in idna==3.7 # via requests ijson==3.1.4 @@ -92,6 +92,8 @@ openpyxl==3.0.7 # via flattentool orjson==3.9.15 # via libcoveocds +packaging==24.1 + # via gunicorn persistent==4.7.0 # via # btrees @@ -133,6 +135,8 @@ schema==0.7.4 # via flattentool sentry-sdk==2.8.0 # via -r requirements.in +setproctitle==1.3.3 + # via gunicorn setuptools==74.1.1 # via # zc-lockfile @@ -149,10 +153,6 @@ sqlparse==0.5.0 # via django transaction==3.0.1 # via zodb -typing-extensions==4.12.2 - # via - # asgiref - # cattrs url-normalize==1.4.3 # via requests-cache urllib3==2.2.2 diff --git a/requirements_dev.txt b/requirements_dev.txt index 1e1fe696..e20a917d 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -72,19 +72,15 @@ et-xmlfile==1.1.0 # -r requirements.txt # openpyxl exceptiongroup==1.2.2 - # via - # -r requirements.txt - # cattrs - # hypothesis - # pytest - # trio - # trio-websocket + # via trio-websocket flattentool==0.26.0 # via # -r requirements.txt # -r requirements_dev.in # libcove # libcoveweb +gunicorn==23.0.0 + # via -r requirements.txt h11==0.14.0 # via wsproto hypothesis==6.72.4 @@ -161,8 +157,11 @@ orjson==3.9.15 # via -r requirements.txt outcome==1.2.0 # via trio -packaging==23.1 - # via pytest +packaging==24.1 + # via + # -r requirements.txt + # gunicorn + # pytest persistent==4.7.0 # via # -r requirements.txt @@ -235,6 +234,8 @@ selenium==4.11.2 # via -r requirements_dev.in sentry-sdk==2.8.0 # via -r requirements.txt +setproctitle==1.3.3 + # via -r requirements.txt setuptools==74.1.1 # via # -r requirements.txt @@ -260,8 +261,6 @@ sqlparse==0.5.0 # via # -r requirements.txt # django -tomli==2.0.1 - # via pytest transaction==3.0.1 # via # -r requirements.txt @@ -272,11 +271,6 @@ trio==0.22.0 # trio-websocket trio-websocket==0.10.2 # via selenium -typing-extensions==4.12.2 - # via - # -r requirements.txt - # asgiref - # cattrs url-normalize==1.4.3 # via # -r requirements.txt From 708a012bc7856691f16bf666341187396b019763 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 2 Oct 2024 18:08:56 -0400 Subject: [PATCH 10/10] test: Use StaticFilesStorage in tests --- tests/test_functional.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_functional.py b/tests/test_functional.py index da2bec69..7714a966 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -5,6 +5,7 @@ import pytest import requests from django.conf import settings +from django.test import override_settings from selenium import webdriver from selenium.common.exceptions import NoSuchElementException from selenium.webdriver.chrome.options import Options @@ -34,7 +35,10 @@ def browser(): browser = getattr(webdriver, BROWSER)() browser.implicitly_wait(3) - yield browser + with override_settings( + STORAGES={"staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}} + ): + yield browser browser.quit() @@ -150,7 +154,6 @@ def test_common_index_elements(server_url, browser): browser.find_element(By.CSS_SELECTOR, "#more-information .panel-title").click() time.sleep(0.5) assert "What happens to the data I provide to this site?" in browser.find_element(By.TAG_NAME, "body").text - assert "Why do you delete data after 90 days?" in browser.find_element(By.TAG_NAME, "body").text assert "Why provide converted versions?" in browser.find_element(By.TAG_NAME, "body").text assert "Terms & Conditions" in browser.find_element(By.TAG_NAME, "body").text assert "Open Data Services" in browser.find_element(By.TAG_NAME, "body").text