From c22831603732fa00c6d0dfe42b03143e0e0dbee3 Mon Sep 17 00:00:00 2001 From: Carl Simon Adorf Date: Tue, 1 Mar 2022 14:22:34 +0100 Subject: [PATCH] Tests: Reduce test flakiness (#109) * Tests: Do not allow docker pull by default, but pre-load default image per session. * Improve profile detection algorithm. * Tests: Mark the test_instance_pull test as slow. * Use getsentry/responses for request mocking. Instead of requests-mock due to flakiness. * Tests: Expand start_stop_reset test. --- aiidalab_launch/profile.py | 14 ++++++-- setup.cfg | 2 +- tests/conftest.py | 70 ++++++++++++++++++++++++++++++-------- tests/test_cli.py | 17 +++++++-- tests/test_instance.py | 3 +- 5 files changed, 85 insertions(+), 21 deletions(-) diff --git a/aiidalab_launch/profile.py b/aiidalab_launch/profile.py index cd99920..0cb54ac 100644 --- a/aiidalab_launch/profile.py +++ b/aiidalab_launch/profile.py @@ -29,6 +29,9 @@ def _default_port() -> int: # explicit function required to enable test patchin return DEFAULT_PORT +_DEFAULT_IMAGE = "aiidalab/aiidalab-docker-stack:latest" + + def _get_configured_host_port(container: Container) -> int | None: try: host_config = container.attrs["HostConfig"] @@ -51,7 +54,7 @@ class Profile: port: int | None = field(default_factory=_default_port) default_apps: list[str] = field(default_factory=lambda: ["aiidalab-widgets-base"]) system_user: str = "aiida" - image: str = "aiidalab/aiidalab-docker-stack:latest" + image: str = _DEFAULT_IMAGE home_mount: str | None = None def __post_init__(self): @@ -96,6 +99,13 @@ def from_container(cls, container: Container) -> Profile: ) system_user = get_docker_env(container, "SYSTEM_USER") + + image_tag = ( + _DEFAULT_IMAGE + if _DEFAULT_IMAGE in container.image.tags + else container.image.tags[0] + ) + return Profile( name=profile_name, port=_get_configured_host_port(container), @@ -103,6 +113,6 @@ def from_container(cls, container: Container) -> Profile: home_mount=str( docker_mount_for(container, PurePosixPath("/", "home", system_user)) ), - image=container.image.tags[0], + image=image_tag, system_user=system_user, ) diff --git a/setup.cfg b/setup.cfg index 07a11df..8576fe6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,7 +49,7 @@ tests = pytest==6.2.5 pytest-asyncio==0.17.2 pytest-cov==3.0.0 - requests-mock==1.9.3 + responses==0.18.0 [flake8] ignore = diff --git a/tests/conftest.py b/tests/conftest.py index 35c14df..3616695 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,7 +20,7 @@ import docker import pytest import requests -from requests_mock import ANY +import responses import aiidalab_launch from aiidalab_launch.application_state import ApplicationState @@ -57,6 +57,14 @@ def docker_client(): pytest.skip("docker not available") +@pytest.fixture(scope="session", autouse=True) +def _pull_docker_image(docker_client): + try: + docker_client.images.pull(aiidalab_launch.profile._DEFAULT_IMAGE) + except docker.errors.APIError: + pytest.skip("unable to pull docker image") + + # Avoid interfering with used ports on the host system. @pytest.fixture(scope="session", autouse=True) def _default_port(monkeypatch_session): @@ -147,29 +155,49 @@ async def started_instance(instance): @pytest.fixture(autouse=True) -def _enable_docker_requests(requests_mock): +def _mocked_responses(): + "Setup mocker for all HTTP requests." + with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps: + yield rsps + + +@pytest.fixture(autouse=True) +def _enable_docker_requests(_mocked_responses): + "Pass-through all docker requests." docker_uris = re.compile(r"http\+docker:\/\/") - requests_mock.register_uri(ANY, docker_uris, real_http=True) + _mocked_responses.add_passthru(docker_uris) + + +@pytest.fixture +def _pypi_response(): + "A minimal, but valid PyPI response for this package." + return dict( + url="https://pypi.python.org/pypi/aiidalab-launch/json", + json={"releases": {"2022.1010": [{"yanked": False}]}}, + ) # Do not request package information from PyPI @pytest.fixture(autouse=True) -def mock_pypi_request(monkeypatch, requests_mock): +def mock_pypi_request(monkeypatch, _mocked_responses, _pypi_response): + "Mock the PyPI request." + # Need to monkeypatch to prevent caching to interfere with the test. monkeypatch.setattr(aiidalab_launch.util, "SESSION", requests.Session()) - requests_mock.register_uri( - "GET", - "https://pypi.python.org/pypi/aiidalab-launch/json", - json={"releases": {"2022.1010": [{"yanked": False}]}}, - ) + # Setup the mocked response for PyPI. + _mocked_responses.upsert(responses.GET, **_pypi_response) @pytest.fixture -def mock_pypi_request_timeout(requests_mock): - requests_mock.register_uri( - "GET", - "https://pypi.python.org/pypi/aiidalab-launch/json", - exc=requests.exceptions.Timeout, +def mock_pypi_request_timeout(_mocked_responses, _pypi_response): + "Simulate a timeout while trying to reach the PyPI server." + # Setup the timeout response. + timeout_response = dict( + url=_pypi_response["url"], body=requests.exceptions.Timeout() ) + _mocked_responses.upsert(responses.GET, **timeout_response) + yield + # Restore the valid mocked response for PyPI. + _mocked_responses.upsert(responses.GET, **_pypi_response) @pytest.fixture(scope="session") @@ -185,6 +213,20 @@ def invalid_image_id(docker_client): pytest.xfail("Unable to generate invalid Docker image id.") +@pytest.fixture(autouse=True) +def _disable_docker_pull(monkeypatch): + def no_pull(self, *args, **kwargs): + pytest.skip("Test tried to pull docker image.") + + monkeypatch.setattr(docker.api.image.ImageApiMixin, "pull", no_pull) + return monkeypatch + + +@pytest.fixture() +def enable_docker_pull(_disable_docker_pull): + _disable_docker_pull.undo() + + def pytest_addoption(parser): parser.addoption( "--slow", diff --git a/tests/test_cli.py b/tests/test_cli.py index 49ffcc1..526f6a1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -188,9 +188,18 @@ def assert_status_down(): assert "down" in result.output assert "http" not in result.output - # Start instance. + # Start instance (non-blocking). runner: CliRunner = CliRunner() - result: Result = runner.invoke(cli.cli, ["start", "--no-browser", "--wait=300"]) + result: Result = runner.invoke( + cli.cli, ["start", "--no-browser", "--no-pull", "--wait=0"] + ) + assert result.exit_code == 0 + + # Start instance again (blocking, should be no-op). + runner: CliRunner = CliRunner() + result: Result = runner.invoke( + cli.cli, ["start", "--no-browser", "--no-pull", "--wait=300"] + ) assert result.exit_code == 0 assert_status_up() @@ -198,7 +207,9 @@ def assert_status_down(): assert get_volume(instance.profile.conda_volume_name()) # Start instance again – should be noop. - result: Result = runner.invoke(cli.cli, ["start", "--no-browser", "--wait=300"]) + result: Result = runner.invoke( + cli.cli, ["start", "--no-browser", "--no-pull", "--wait=300"] + ) assert "Container was already running" in result.output.strip() assert result.exit_code == 0 assert_status_up() diff --git a/tests/test_instance.py b/tests/test_instance.py index 9b96eaf..c337137 100644 --- a/tests/test_instance.py +++ b/tests/test_instance.py @@ -19,7 +19,8 @@ async def test_instance_init(instance): assert await instance.status() is instance.AiidaLabInstanceStatus.DOWN -def test_instance_pull(instance): +@pytest.mark.slow +def test_instance_pull(instance, enable_docker_pull): assert ( "hello-world:latest" in replace(instance, profile=replace(instance.profile, image="hello-world"))