Merge pull request #277 from lsst-sqre/tickets/DM-45281

DM-45281: Switch to testcontainers from tox-docker

.github/workflows/ci.yaml

@@ -58,7 +58,7 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
           tox-envs: "py,typing"
-          tox-plugins: "tox-docker tox-uv"
+          tox-plugins: "tox-uv"
 
   docs:
 

.github/workflows/periodic-ci.yaml

@@ -35,7 +35,7 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
           tox-envs: "lint,typing,py"
-          tox-plugins: "tox-docker tox-uv"
+          tox-plugins: "tox-uv"
           use-cache: false
 
       - name: Report status

Makefile

@@ -16,7 +16,7 @@ clean:
 .PHONY: init
 init:
 	pip install --upgrade uv
-	uv pip install --upgrade pre-commit tox tox-docker tox-uv
+	uv pip install --upgrade pre-commit tox tox-uv
 	uv pip install --upgrade -e ".[arq,db,dev,gcs,kubernetes]"
 	pre-commit install
 	rm -rf .tox

pyproject.toml

@@ -57,6 +57,7 @@ dev = [
     "respx",
     "scriv",
     "sqlalchemy[mypy]",
+    "testcontainers[postgres,redis]",
     "uvicorn",
     # documentation
     "documenteer[guide]>=1",

tests/conftest.py

@@ -2,26 +2,33 @@
 
 from __future__ import annotations
 
-import os
 from collections.abc import AsyncIterator, Iterator
 from datetime import timedelta
 
 import pytest
 import pytest_asyncio
-import redis.asyncio as redis
 import respx
+from redis.asyncio import Redis
+from testcontainers.postgres import PostgresContainer
+from testcontainers.redis import RedisContainer
 
 from safir.testing.gcs import MockStorageClient, patch_google_storage
 from safir.testing.kubernetes import MockKubernetesApi, patch_kubernetes
 from safir.testing.slack import MockSlackWebhook, mock_slack_webhook
 
 
-@pytest.fixture
-def database_url() -> str:
-    """Dynamically construct the test database URL from tox-docker envvars."""
-    host = os.environ["POSTGRES_HOST"]
-    port = os.environ["POSTGRES_5432_TCP_PORT"]
-    return f"postgresql://safir@{host}:{port}/safir"
+@pytest.fixture(scope="session")
+def database_password() -> str:
+    return "INSECURE@%PASSWORD/"
+
+
+@pytest.fixture(scope="session")
+def database_url(database_password: str) -> Iterator[str]:
+    """Start a PostgreSQL database and return a URL for it."""
+    with PostgresContainer(
+        driver="asyncpg", username="safir", password=database_password
+    ) as postgres:
+        yield postgres.get_connection_url()
 
 
 @pytest.fixture
@@ -41,15 +48,23 @@ def mock_slack(respx_mock: respx.Router) -> MockSlackWebhook:
     return mock_slack_webhook("https://example.com/slack", respx_mock)
 
 
+@pytest.fixture(scope="session")
+def redis() -> Iterator[RedisContainer]:
+    """Start a Redis container."""
+    with RedisContainer() as redis:
+        yield redis
+
+
 @pytest_asyncio.fixture
-async def redis_client() -> AsyncIterator[redis.Redis]:
-    """Redis client for testing.
+async def redis_client(redis: RedisContainer) -> AsyncIterator[Redis]:
+    """Create a Redis client for testing.
 
-    This fixture connects to the Redis server that runs via tox-docker.
+    This must be done separately for each test since it's tied to the per-test
+    event loop, and therefore must be separated from the session-shared Redis
+    server container.
     """
-    host = os.environ["REDIS_HOST"]
-    port = os.environ["REDIS_6379_TCP_PORT"]
-    client = redis.Redis(host=host, port=int(port), db=0)
+    host = redis.get_container_host_ip()
+    port = redis.get_exposed_port(6379)
+    client = Redis(host=host, port=port, db=0)
     yield client
-
     await client.aclose()

tests/database_test.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import os
 from datetime import UTC, datetime, timedelta, timezone
 from urllib.parse import unquote, urlparse
 
@@ -25,8 +24,6 @@
 )
 from safir.database._connection import _build_database_url
 
-TEST_DATABASE_PASSWORD = os.environ["TEST_DATABASE_PASSWORD"]
-
 Base = declarative_base()
 
 
@@ -39,9 +36,11 @@ class User(Base):
 
 
 @pytest.mark.asyncio
-async def test_database_init(database_url: str) -> None:
+async def test_database_init(
+    database_url: str, database_password: str
+) -> None:
     logger = structlog.get_logger(__name__)
-    engine = create_database_engine(database_url, TEST_DATABASE_PASSWORD)
+    engine = create_database_engine(database_url, database_password)
     await initialize_database(engine, logger, schema=Base.metadata, reset=True)
     session = await create_async_session(engine, logger)
     async with session.begin():
@@ -58,7 +57,7 @@ async def test_database_init(database_url: str) -> None:
 
     # Reinitializing the database with reset should delete the data. Try
     # passing in the password as a SecretStr.
-    password = SecretStr(TEST_DATABASE_PASSWORD)
+    password = SecretStr(database_password)
     engine = create_database_engine(database_url, password)
     await initialize_database(engine, logger, schema=Base.metadata, reset=True)
     session = await create_async_session(engine, logger)
@@ -111,9 +110,11 @@ def test_build_database_url(database_url: str) -> None:
 
 
 @pytest.mark.asyncio
-async def test_create_async_session(database_url: str) -> None:
+async def test_create_async_session(
+    database_url: str, database_password: str
+) -> None:
     logger = structlog.get_logger(__name__)
-    engine = create_database_engine(database_url, TEST_DATABASE_PASSWORD)
+    engine = create_database_engine(database_url, database_password)
     await initialize_database(engine, logger, schema=Base.metadata, reset=True)
 
     session = await create_async_session(
@@ -165,9 +166,11 @@ class Test(BaseModel):
 
 
 @pytest.mark.asyncio
-async def test_retry_async_transaction(database_url: str) -> None:
+async def test_retry_async_transaction(
+    database_url: str, database_password: str
+) -> None:
     logger = structlog.get_logger(__name__)
-    engine = create_database_engine(database_url, TEST_DATABASE_PASSWORD)
+    engine = create_database_engine(database_url, database_password)
     await initialize_database(engine, logger, schema=Base.metadata, reset=True)
     session = await create_async_session(engine, logger)
     async with session.begin():

tests/dependencies/db_session_test.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import os
 from typing import Annotated
 
 import pytest
@@ -21,8 +20,6 @@
 )
 from safir.dependencies.db_session import db_session_dependency
 
-TEST_DATABASE_PASSWORD = os.getenv("TEST_DATABASE_PASSWORD")
-
 Base = declarative_base()
 
 
@@ -35,14 +32,12 @@ class User(Base):
 
 
 @pytest.mark.asyncio
-async def test_session(database_url: str) -> None:
+async def test_session(database_url: str, database_password: str) -> None:
     logger = structlog.get_logger(__name__)
-    engine = create_database_engine(database_url, TEST_DATABASE_PASSWORD)
+    engine = create_database_engine(database_url, database_password)
     await initialize_database(engine, logger, schema=Base.metadata, reset=True)
     session = await create_async_session(engine, logger)
-    await db_session_dependency.initialize(
-        database_url, TEST_DATABASE_PASSWORD
-    )
+    await db_session_dependency.initialize(database_url, database_password)
 
     app = FastAPI()
 

tox.ini

@@ -2,32 +2,6 @@
 envlist = py,coverage-report,typing,lint,docs
 isolated_build = True
 
-[docker:postgres]
-image = postgres:latest
-environment =
-    POSTGRES_PASSWORD=INSECURE@%PASSWORD/
-    POSTGRES_USER=safir
-    POSTGRES_DB=safir
-# The healthcheck ensures that tox-docker won't run tests until the
-# container is up and the command finishes with exit code 0 (success)
-healthcheck_cmd = PGPASSWORD=$POSTGRES_PASSWORD psql \
-    --user=$POSTGRES_USER --dbname=$POSTGRES_DB \
-    --host=127.0.0.1 --quiet --no-align --tuples-only \
-    -1 --command="SELECT 1"
-healthcheck_timeout = 1
-healthcheck_retries = 30
-healthcheck_interval = 1
-healthcheck_start_period = 1
-
-[docker:redis]
-image = redis:latest
-healthcheck_cmd =
-    redis-cli ping
-healthcheck_timeout = 1
-healthcheck_retries = 30
-healthcheck_interval = 1
-healthcheck_start_period = 1
-
 [testenv]
 description = Run pytest against {envname}.
 extras =
@@ -40,13 +14,8 @@ extras =
 
 [testenv:py]
 description = Run pytest with PostgreSQL via Docker.
-docker =
-    postgres
-    redis
 commands =
     coverage run -m pytest {posargs}
-setenv =
-    TEST_DATABASE_PASSWORD = INSECURE@%PASSWORD/
 
 [testenv:coverage-report]
 description = Compile coverage from each test run.