Merge branch 'develop' into topic/chriscummings/enable-local-postgres…

…-prod

.github/workflows/behave.yml

@@ -4,7 +4,7 @@ name: Run behave
 on:
   push:
     branches:
-      - '**'
+      - "**"
   pull_request:
     branches:
       - main
@@ -17,62 +17,70 @@ jobs:
   behave:
     name: Run Behave
     runs-on: ubuntu-latest
-
-    services:
-      postgres:
-        image: postgres:latest
-        env:
-          POSTGRES_USER: scram
-          POSTGRES_PASSWORD: ''
-          POSTGRES_DB: test_scram
-          POSTGRES_HOST_AUTH_METHOD: trust
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd "pg_isready -U scram"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: ["3.11", "3.12"]
 
     steps:
       - name: Check out the code
         uses: actions/checkout@v4
 
       - name: Set up Docker
         uses: docker/setup-buildx-action@v3
+
       - name: Install Docker Compose
         run: |
           sudo apt-get update
           sudo apt-get install -y docker-compose make
 
+      - name: Check Docker state (pre-build)
+        run: docker ps
+
       - name: Build Docker images
         run: make build
+        env:
+          PYTHON_IMAGE_VER: "${{ matrix.python-version }}"
 
       - name: Migrate Database
         run: make migrate
 
       - name: Run Application
         run: make run
 
+      - name: Check Docker state (pre-test)
+        run: docker ps
+
       - name: Run pytest + behave with Coverage
         env:
           POSTGRES_USER: scram
           POSTGRES_DB: test_scram
         run: make coverage.xml
 
+      - name: Dump docker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+
       - name: Upload Coverage to Coveralls
+        if: matrix.python-version == '3.12'
         uses: coverallsapp/github-action@v2
 
       - name: Upload Coverage to GitHub
+        if: matrix.python-version == '3.12'
         uses: actions/upload-artifact@v4
         with:
           name: coverage-report
           path: coverage.xml
 
       - name: Display Coverage Metrics
+        if: matrix.python-version == '3.12'
         uses: 5monkeys/cobertura-action@v14
         with:
-          minimum_coverage: '50'
+          minimum_coverage: "50"
+
+      - name: Check Docker state (post-test)
+        if: always()
+        run: docker ps
 
       - name: Stop Services
         if: always()

.github/workflows/behave_next_python.yml

@@ -0,0 +1,77 @@
+---
+name: Run behave with unsupported Python versions
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    branches:
+      - main
+      - develop
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  behave_next_python:
+    name: Run Behave
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: ['3.13']
+
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v3
+
+      - name: Install Docker Compose
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y docker-compose make
+
+      - name: Check Docker state (pre-build)
+        run: docker ps
+
+      - name: Build Docker images
+        run: make build
+        env:
+          PYTHON_IMAGE_VER: "${{ matrix.python-version }}"
+
+      - name: Migrate Database
+        run: |
+          make migrate || echo "::warning:: migrate failed on future Python version ${{ matrix.python-version }}."
+
+      - name: Run Application
+        run: |
+          make run || echo "::warning:: run failed on future Python version ${{ matrix.python-version }}."
+
+      - name: Check Docker state (pre-test)
+        run: docker ps
+
+      - name: Run pytest + behave with Coverage
+        env:
+          POSTGRES_USER: scram
+          POSTGRES_DB: test_scram
+        run: |
+          make coverage.xml || echo "::warning:: pytest + behave failed on future Python version ${{ matrix.python-version }}."
+
+      - name: Dump docker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+
+      - name: Check Docker state (post-test)
+        if: always()
+        run: docker ps
+
+      - name: Stop Services
+        if: always()
+        run: make stop
+
+      - name: Clean Up
+        if: always()
+        run: make clean

.github/workflows/pytest.yml

@@ -20,15 +20,15 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ['3.12']
+        python-version: ['3.11', '3.12']
 
     services:
       postgres:
         image: postgres:latest
         env:
           POSTGRES_USER: scram
           POSTGRES_PASSWORD: ''
-          POSTGRES_DB: test_scram
+          POSTGRES_DB: test_scram_${{ matrix.python-version }}
           POSTGRES_HOST_AUTH_METHOD: trust
         ports:
           - 5432:5432
@@ -55,32 +55,28 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements/local.txt
-          pip install pytest-github-actions-annotate-failures
+          # https://github.com/pytest-dev/pytest-github-actions-annotate-failures/pull/68 isn't yet in a release
+          pip install git+https://github.com/pytest-dev/pytest-github-actions-annotate-failures.git@6e66cd895fe05cd09be8bad58f5d79110a20385f
 
-      - name: Apply unapplied migrations
+      - name: Apply migrations
         env:
-          DATABASE_URL: "postgres://scram:@localhost:5432/test_scram"
+          DATABASE_URL: "postgres://scram:@localhost:5432/test_scram_${{ matrix.python-version }}"
         run: |
-          python manage.py makemigrations --noinput || true
-          UNAPPLIED_MIGRATIONS=$(python manage.py showmigrations --plan | grep '\[ \]' | awk '{print $2}')
-          if [ -n "$UNAPPLIED_MIGRATIONS" ]; then
-            for migration in $UNAPPLIED_MIGRATIONS; do
-              python manage.py migrate $migration --fake-initial --noinput
-            done
-          else
-            echo "No unapplied migrations."
-          fi
+          python manage.py makemigrations --noinput
+          python manage.py migrate --noinput
 
       - name: Check for duplicate migrations
+        env:
+          DATABASE_URL: "postgres://scram:@localhost:5432/test_scram_${{ matrix.python-version }}"
         run: |
           if python manage.py makemigrations --dry-run | grep "No changes detected"; then
             echo "No duplicate migrations detected."
           else
-            echo "Warning: Potential duplicate migrations detected. Please review."
+            echo "::warning:: Potential duplicate migrations detected. Please review."
           fi
 
       - name: Run Pytest
         env:
-          DATABASE_URL: "postgres://scram:@localhost:5432/test_scram"
+          DATABASE_URL: "postgres://scram:@localhost:5432/test_scram_${{ matrix.python-version }}"
           REDIS_HOST: "localhost"
         run: pytest

.github/workflows/future_pytest.yml → .github/workflows/pytest_next_python.yml

@@ -14,7 +14,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  pytest:
+  pytest_next_python:
     name: Run Pytest
     runs-on: ubuntu-latest
     strategy:
@@ -55,7 +55,8 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements/local.txt
-          pip install pytest-github-actions-annotate-failures
+          # https://github.com/pytest-dev/pytest-github-actions-annotate-failures/pull/68 isn't yet in a release
+          pip install git+https://github.com/pytest-dev/pytest-github-actions-annotate-failures.git@6e66cd895fe05cd09be8bad58f5d79110a20385f
 
       - name: Apply unapplied migrations
         env:

Makefile

@@ -1,3 +1,7 @@
+# It'd be nice to keep these in sync with the defaults of the Dockerfiles
+PYTHON_IMAGE_VER ?= 3.12
+POSTGRES_IMAGE_VER ?= 12.3
+
 .DEFAULT_GOAL := help
 
 ## toggle-prod: configure make to use the production stack
@@ -37,7 +41,8 @@ behave-translator: compose.override.yml
 ## build: rebuilds all your containers or a single one if CONTAINER is specified
 .Phony: build
 build: compose.override.yml
-	@docker compose up -d --no-deps --build $(CONTAINER)
+	@docker compose build --build-arg PYTHON_IMAGE_VER=$(PYTHON_IMAGE_VER) --build-arg POSTGRES_IMAGE_VER=$(POSTGRES_IMAGE_VER) $(CONTAINER)
+	@docker compose up -d --no-deps $(CONTAINER)
 	@docker compose restart $(CONTAINER)
 
 ## coverage.xml: generate coverage from test runs
@@ -138,3 +143,13 @@ tail-log: compose.override.yml
 .Phony: type-check
 type-check: compose.override.yml
 	@docker compose run --rm django mypy scram
+
+## docs-build: build the documentation
+.Phony: docs-build
+docs-build:
+	@docker compose run --rm docs mkdocs build
+
+## docs-serve: build and run a server with the documentation
+.Phony: docs-serve
+docs-serve:
+	@docker compose run --rm docs mkdocs serve -a 0.0.0.0:8888

compose.override.local.yml

@@ -44,12 +44,10 @@ services:
     networks:
       default: {}
     volumes:
-      - $CI_PROJECT_DIR/docs:/docs:z
-      - $CI_PROJECT_DIR/config:/app/config:z
-      - $CI_PROJECT_DIR/scram:/app/scram:z
+      - $CI_PROJECT_DIR:/app:z
     ports:
-      - "7000"
-    command: /start-docs
+      - "${DOCS_PORT:-8888}"
+    command: "mkdocs serve -a 0.0.0.0:${DOCS_PORT:-8888}"
 
   redis:
     ports:

compose/local/django/Dockerfile

@@ -1,4 +1,6 @@
-FROM python:3.12-slim-bookworm
+ARG PYTHON_IMAGE_VER=3.12
+
+FROM python:${PYTHON_IMAGE_VER}-slim-bookworm
 
 ENV PIP_ROOT_USER_ACTION ignore
 ENV PYTHONUNBUFFERED 1

compose/local/django/start

@@ -4,5 +4,6 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
+mkdir -p /app/staticfiles
 python manage.py migrate
 uvicorn config.asgi:application --host 0.0.0.0 --reload

compose/local/docs/Dockerfile

@@ -1,4 +1,6 @@
-FROM python:3.12-slim-bookworm
+ARG PYTHON_IMAGE_VER=3.12
+
+FROM python:${PYTHON_IMAGE_VER}-slim-bookworm
 
 ENV PIP_ROOT_USER_ACTION ignore
 ENV PYTHONUNBUFFERED 1
@@ -20,13 +22,11 @@ RUN apt-get update \
     && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
     && rm -rf /var/lib/apt/lists/*
 
-# Requirements are installed here to ensure they will be cached.
-COPY ./requirements /requirements
-# All imports needed for autodoc.
-RUN pip install -r /requirements/local.txt -r /requirements/production.txt
 
-COPY ./compose/local/docs/start /start-docs
-RUN sed -i 's/\r$//g' /start-docs
-RUN chmod +x /start-docs
+# Only re-run the pip install if these files have changed
+COPY requirements/base.txt requirements/local.txt requirements/production.txt /app/requirements/
+RUN pip install -r /app/requirements/local.txt -r /app/requirements/production.txt
+
+COPY . /app/
 
-WORKDIR /docs
+WORKDIR /app

compose/production/django/Dockerfile

@@ -1,5 +1,8 @@
+ARG PYTHON_IMAGE_VER=3.12
+
+
+FROM python:${PYTHON_IMAGE_VER}-slim-bookworm
 
-FROM python:3.12-slim-bookworm
 
 ENV PYTHONUNBUFFERED 1
 

compose/production/postgres/Dockerfile

@@ -1,4 +1,6 @@
-FROM postgres:12.3
+ARG POSTGRES_IMAGE_VER=12.3
+
+FROM postgres:${POSTGRES_IMAGE_VER}
 
 COPY ./compose/production/postgres/maintenance /usr/local/bin/maintenance
 RUN chmod +x /usr/local/bin/maintenance/*

config/settings/local.py

@@ -51,7 +51,7 @@
 }
 # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#internal-ips
 INTERNAL_IPS = ["127.0.0.1", "10.0.2.2"]
-if env("USE_DOCKER") == "yes":
+if env("USE_DOCKER", default="no") == "yes":
     import socket
 
     hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())