From b93e39ee6db144c94c6f7cf40a2a361f9003ef01 Mon Sep 17 00:00:00 2001 From: Russ Allbery Date: Fri, 19 Jul 2024 12:56:58 -0700 Subject: [PATCH] Switch to testcontainers from tox-docker In preparation for converting Safir to a monorepo with multiple PyPI modules and thus the nox build system, switch from tox-docker to testcontainers for testing. --- .github/workflows/ci.yaml | 2 +- .github/workflows/periodic-ci.yaml | 2 +- Makefile | 2 +- pyproject.toml | 1 + tests/conftest.py | 45 ++++++++++++++++++--------- tests/database_test.py | 23 ++++++++------ tests/dependencies/db_session_test.py | 11 ++----- tox.ini | 31 ------------------ 8 files changed, 50 insertions(+), 67 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8f4765af..852f382c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -58,7 +58,7 @@ jobs: with: python-version: ${{ matrix.python }} tox-envs: "py,typing" - tox-plugins: "tox-docker tox-uv" + tox-plugins: "tox-uv" docs: diff --git a/.github/workflows/periodic-ci.yaml b/.github/workflows/periodic-ci.yaml index 3d8e0e4b..b787b6ab 100644 --- a/.github/workflows/periodic-ci.yaml +++ b/.github/workflows/periodic-ci.yaml @@ -35,7 +35,7 @@ jobs: with: python-version: ${{ matrix.python }} tox-envs: "lint,typing,py" - tox-plugins: "tox-docker tox-uv" + tox-plugins: "tox-uv" use-cache: false - name: Report status diff --git a/Makefile b/Makefile index 0c34f007..83832e4a 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ clean: .PHONY: init init: pip install --upgrade uv - uv pip install --upgrade pre-commit tox tox-docker tox-uv + uv pip install --upgrade pre-commit tox tox-uv uv pip install --upgrade -e ".[arq,db,dev,gcs,kubernetes]" pre-commit install rm -rf .tox diff --git a/pyproject.toml b/pyproject.toml index 6e49e0e2..55e53350 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ dev = [ "respx", "scriv", "sqlalchemy[mypy]", + "testcontainers[postgres,redis]", "uvicorn", # documentation "documenteer[guide]>=1", diff --git a/tests/conftest.py b/tests/conftest.py index 8691288a..6a8c50a8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,26 +2,33 @@ from __future__ import annotations -import os from collections.abc import AsyncIterator, Iterator from datetime import timedelta import pytest import pytest_asyncio -import redis.asyncio as redis import respx +from redis.asyncio import Redis +from testcontainers.postgres import PostgresContainer +from testcontainers.redis import RedisContainer from safir.testing.gcs import MockStorageClient, patch_google_storage from safir.testing.kubernetes import MockKubernetesApi, patch_kubernetes from safir.testing.slack import MockSlackWebhook, mock_slack_webhook -@pytest.fixture -def database_url() -> str: - """Dynamically construct the test database URL from tox-docker envvars.""" - host = os.environ["POSTGRES_HOST"] - port = os.environ["POSTGRES_5432_TCP_PORT"] - return f"postgresql://safir@{host}:{port}/safir" +@pytest.fixture(scope="session") +def database_password() -> str: + return "INSECURE@%PASSWORD/" + + +@pytest.fixture(scope="session") +def database_url(database_password: str) -> Iterator[str]: + """Start a PostgreSQL database and return a URL for it.""" + with PostgresContainer( + driver="asyncpg", username="safir", password=database_password + ) as postgres: + yield postgres.get_connection_url() @pytest.fixture @@ -41,15 +48,23 @@ def mock_slack(respx_mock: respx.Router) -> MockSlackWebhook: return mock_slack_webhook("https://example.com/slack", respx_mock) +@pytest.fixture(scope="session") +def redis() -> Iterator[RedisContainer]: + """Start a Redis container.""" + with RedisContainer() as redis: + yield redis + + @pytest_asyncio.fixture -async def redis_client() -> AsyncIterator[redis.Redis]: - """Redis client for testing. +async def redis_client(redis: RedisContainer) -> AsyncIterator[Redis]: + """Create a Redis client for testing. - This fixture connects to the Redis server that runs via tox-docker. + This must be done separately for each test since it's tied to the per-test + event loop, and therefore must be separated from the session-shared Redis + server container. """ - host = os.environ["REDIS_HOST"] - port = os.environ["REDIS_6379_TCP_PORT"] - client = redis.Redis(host=host, port=int(port), db=0) + host = redis.get_container_host_ip() + port = redis.get_exposed_port(6379) + client = Redis(host=host, port=port, db=0) yield client - await client.aclose() diff --git a/tests/database_test.py b/tests/database_test.py index f50a6e25..14e19b4d 100644 --- a/tests/database_test.py +++ b/tests/database_test.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os from datetime import UTC, datetime, timedelta, timezone from urllib.parse import unquote, urlparse @@ -25,8 +24,6 @@ ) from safir.database._connection import _build_database_url -TEST_DATABASE_PASSWORD = os.environ["TEST_DATABASE_PASSWORD"] - Base = declarative_base() @@ -39,9 +36,11 @@ class User(Base): @pytest.mark.asyncio -async def test_database_init(database_url: str) -> None: +async def test_database_init( + database_url: str, database_password: str +) -> None: logger = structlog.get_logger(__name__) - engine = create_database_engine(database_url, TEST_DATABASE_PASSWORD) + engine = create_database_engine(database_url, database_password) await initialize_database(engine, logger, schema=Base.metadata, reset=True) session = await create_async_session(engine, logger) async with session.begin(): @@ -58,7 +57,7 @@ async def test_database_init(database_url: str) -> None: # Reinitializing the database with reset should delete the data. Try # passing in the password as a SecretStr. - password = SecretStr(TEST_DATABASE_PASSWORD) + password = SecretStr(database_password) engine = create_database_engine(database_url, password) await initialize_database(engine, logger, schema=Base.metadata, reset=True) session = await create_async_session(engine, logger) @@ -111,9 +110,11 @@ def test_build_database_url(database_url: str) -> None: @pytest.mark.asyncio -async def test_create_async_session(database_url: str) -> None: +async def test_create_async_session( + database_url: str, database_password: str +) -> None: logger = structlog.get_logger(__name__) - engine = create_database_engine(database_url, TEST_DATABASE_PASSWORD) + engine = create_database_engine(database_url, database_password) await initialize_database(engine, logger, schema=Base.metadata, reset=True) session = await create_async_session( @@ -165,9 +166,11 @@ class Test(BaseModel): @pytest.mark.asyncio -async def test_retry_async_transaction(database_url: str) -> None: +async def test_retry_async_transaction( + database_url: str, database_password: str +) -> None: logger = structlog.get_logger(__name__) - engine = create_database_engine(database_url, TEST_DATABASE_PASSWORD) + engine = create_database_engine(database_url, database_password) await initialize_database(engine, logger, schema=Base.metadata, reset=True) session = await create_async_session(engine, logger) async with session.begin(): diff --git a/tests/dependencies/db_session_test.py b/tests/dependencies/db_session_test.py index 61faeba9..7f8bd3a4 100644 --- a/tests/dependencies/db_session_test.py +++ b/tests/dependencies/db_session_test.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os from typing import Annotated import pytest @@ -21,8 +20,6 @@ ) from safir.dependencies.db_session import db_session_dependency -TEST_DATABASE_PASSWORD = os.getenv("TEST_DATABASE_PASSWORD") - Base = declarative_base() @@ -35,14 +32,12 @@ class User(Base): @pytest.mark.asyncio -async def test_session(database_url: str) -> None: +async def test_session(database_url: str, database_password: str) -> None: logger = structlog.get_logger(__name__) - engine = create_database_engine(database_url, TEST_DATABASE_PASSWORD) + engine = create_database_engine(database_url, database_password) await initialize_database(engine, logger, schema=Base.metadata, reset=True) session = await create_async_session(engine, logger) - await db_session_dependency.initialize( - database_url, TEST_DATABASE_PASSWORD - ) + await db_session_dependency.initialize(database_url, database_password) app = FastAPI() diff --git a/tox.ini b/tox.ini index 733f0d2d..7ba16290 100644 --- a/tox.ini +++ b/tox.ini @@ -2,32 +2,6 @@ envlist = py,coverage-report,typing,lint,docs isolated_build = True -[docker:postgres] -image = postgres:latest -environment = - POSTGRES_PASSWORD=INSECURE@%PASSWORD/ - POSTGRES_USER=safir - POSTGRES_DB=safir -# The healthcheck ensures that tox-docker won't run tests until the -# container is up and the command finishes with exit code 0 (success) -healthcheck_cmd = PGPASSWORD=$POSTGRES_PASSWORD psql \ - --user=$POSTGRES_USER --dbname=$POSTGRES_DB \ - --host=127.0.0.1 --quiet --no-align --tuples-only \ - -1 --command="SELECT 1" -healthcheck_timeout = 1 -healthcheck_retries = 30 -healthcheck_interval = 1 -healthcheck_start_period = 1 - -[docker:redis] -image = redis:latest -healthcheck_cmd = - redis-cli ping -healthcheck_timeout = 1 -healthcheck_retries = 30 -healthcheck_interval = 1 -healthcheck_start_period = 1 - [testenv] description = Run pytest against {envname}. extras = @@ -40,13 +14,8 @@ extras = [testenv:py] description = Run pytest with PostgreSQL via Docker. -docker = - postgres - redis commands = coverage run -m pytest {posargs} -setenv = - TEST_DATABASE_PASSWORD = INSECURE@%PASSWORD/ [testenv:coverage-report] description = Compile coverage from each test run.