From 467883ff4db0c9db83349ff173135c659dca275d Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 12 Mar 2024 13:06:13 +0100 Subject: [PATCH] Refactored tests to enable waiting for specific states of the docker container startup --- .../roles/entrypoint/files/entrypoint.py | 13 +++- .../test_create_dss_docker_image.py | 67 ++++++++++++------- ...test_cloud.py => disabled_nbtest_cloud.py} | 0 3 files changed, 55 insertions(+), 25 deletions(-) rename test/notebooks/{nbtest_cloud.py => disabled_nbtest_cloud.py} (100%) diff --git a/exasol/ds/sandbox/runtime/ansible/roles/entrypoint/files/entrypoint.py b/exasol/ds/sandbox/runtime/ansible/roles/entrypoint/files/entrypoint.py index 77976907..e30a2924 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/entrypoint/files/entrypoint.py +++ b/exasol/ds/sandbox/runtime/ansible/roles/entrypoint/files/entrypoint.py @@ -18,7 +18,10 @@ _logger = logging.getLogger(__name__) _logger.setLevel(logging.DEBUG) -logging.basicConfig(format="%(levelname)s %(filename)s: %(message)s") +logging.basicConfig( + format="%(asctime)s %(levelname)-7s %(filename)s: %(message)s", + datefmt="%Y-%m-%d %X", +) def arg_parser(): @@ -172,6 +175,7 @@ def ensure_file(src: Path, dst: Path): def disable_core_dumps(): resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + _logger.info("Disabled coredumps") def sleep_infinity(): @@ -244,7 +248,7 @@ def _find_group(self, id: int) -> str: return None def _run(self, command: str) -> int: - _logger.debug(f"executing {command}") + _logger.debug(f"Executing {command}") return subprocess.run(command.split()).returncode def enable(self) -> Group: @@ -288,6 +292,7 @@ def enable_group_access(self, path: str): ).enable() os.setgroups([group.id]) self.docker_group = group + _logger.info(f"Enabled access to {path}") return self def switch_to(self): @@ -300,6 +305,7 @@ def switch_to(self): f" gid = {os.getresgid()}" f" extra groups = {os.getgroups()}" ) + _logger.info(f"Switched uid/gid to {self.name}/{self.group.name}") return self @@ -314,6 +320,9 @@ def main(): args.notebooks, args.warning_as_error, ) + _logger.info( + "Copied notebooks from" + f" {args.notebook_defaults} to {args.notebooks}") disable_core_dumps() if (args.jupyter_server and args.notebooks diff --git a/test/integration/test_create_dss_docker_image.py b/test/integration/test_create_dss_docker_image.py index 6678591d..9c72d683 100644 --- a/test/integration/test_create_dss_docker_image.py +++ b/test/integration/test_create_dss_docker_image.py @@ -9,11 +9,16 @@ import time import typing -from tenacity.retry import retry_if_exception_type +from tenacity.retry import ( + retry_if_exception_type, + retry_if_not_result, +) +from tenacity import Retrying +from re import Pattern from contextlib import contextmanager from tenacity.wait import wait_fixed from tenacity.stop import stop_after_delay -from typing import Set, Tuple +from typing import Set, Tuple, Union from datetime import datetime, timedelta from exasol.ds.sandbox.lib.dss_docker import DssDockerImage from exasol.ds.sandbox.lib.logging import set_log_level @@ -39,9 +44,6 @@ def dss_docker_container(dss_docker_image, jupyter_port): 'mode': 'rw', }, }, ) container.start() - # wait for entrypoint to be finished and - # for potential modification of socket by entrypoint - time.sleep(1) try: yield container finally: @@ -49,6 +51,36 @@ def dss_docker_container(dss_docker_image, jupyter_port): container.remove() +def wait_for( + container, + log_message: Union[str, Pattern], + timeout: timedelta = timedelta(seconds=5), +): + """ + Wait until container log contains the specified string or regular + expression. + """ + for attempt in Retrying( + wait=wait_fixed(timeout/10), + stop=stop_after_delay(timeout), + ): + with attempt: + logs = container.logs().decode("utf-8").strip() + if isinstance(log_message, Pattern): + matches = log_message.search(logs) + else: + matches = log_message in logs + if not matches: + raise Exception() + + +def wait_for_socket_access(container): + wait_for( + container, + f"entrypoint.py: Enabled access to {DOCKER_SOCKET_CONTAINER}", + ) + + def retry(exception: typing.Type[BaseException], timeout: timedelta): return tenacity.retry( retry=retry_if_exception_type(exception), @@ -79,7 +111,8 @@ def request_with_retry(url: str) -> requests.Response: def test_install_notebook_connector(dss_docker_container): container = dss_docker_container - command = '/home/jupyter/jupyterenv/bin/python -c "import exasol.nb_connector.secret_store"' + command = ('/home/jupyter/jupyterenv/bin/python' + ' -c "import exasol.nb_connector.secret_store"') exit_code, output = container.exec_run(command) output = output.decode('utf-8').strip() assert exit_code == 0, f'Got output "{output}".' @@ -89,6 +122,7 @@ def test_install_notebooks(dss_docker_container): def filename_set(string: str) -> Set[str]: return set(re.split(r'\s+', string.strip())) + wait_for(dss_docker_container, "entrypoint.py: Copied notebooks") exit_code, output = dss_docker_container.exec_run( "ls --indicator-style=slash /home/jupyter/notebooks" ) @@ -105,24 +139,10 @@ def filename_set(string: str) -> Set[str]: assert actual.issuperset(expected) -class RetryException(Exception): - pass - - def test_docker_socket_access(dss_docker_container): - """ - Verify that when mounting the docker socket from the host's file - system into the container, the code inside the container can use the - docker CLI. - """ - @retry(RetryException, timedelta(seconds=5)) - def docker_exec_with_retry(*args, **kwargs) -> Tuple[int, str]: - exit_code, output = dss_docker_container.exec_run(*args, **kwargs) - if exit_code != 0: - raise RetryException() - return exit_code, output.decode("utf-8").strip() - - exit_code, output = docker_exec_with_retry("docker ps", user="jupyter") + wait_for_socket_access(dss_docker_container) + exit_code, output = dss_docker_container.exec_run("docker ps", user="jupyter") + output = output.decode("utf-8").strip() assert exit_code == 0 and re.match(r"^CONTAINER ID +IMAGE .*", output) @@ -155,6 +175,7 @@ def my_container(docker_socket_host): stat_before = socket_on_host.stat() with my_container(socket_on_host) as c: + wait_for_socket_access(c) exit_code, output = c.exec_run(f"docker ps") output = output.decode("utf-8").strip() assert exit_code == 1 and \ diff --git a/test/notebooks/nbtest_cloud.py b/test/notebooks/disabled_nbtest_cloud.py similarity index 100% rename from test/notebooks/nbtest_cloud.py rename to test/notebooks/disabled_nbtest_cloud.py