From 4c6fc9b85927987503a047b973106131d2ae1e44 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Tue, 30 Jan 2024 07:09:04 +0100 Subject: [PATCH] #139: Create Docker Test Setup for notebook tests (#142) * Update Notebook Connector to the commit that allows the AI Lab container to access the ITDE Co-authored-by: Christoph Pirkl Co-authored-by: Mikhail Beck --- doc/developer_guide/developer_guide.md | 3 + doc/developer_guide/notebooks.md | 30 ++++++++ .../ds/sandbox/lib/dss_docker/create_image.py | 36 ++++----- .../files/requirements_dependencies.txt | 2 +- .../test_notebooks_in_dss_docker_image.py | 73 +++++++++++++++++++ test/notebooks/__init__.py | 0 test/notebooks/nbtest_environment_test.py | 9 +++ test/notebooks/nbtest_itde.py | 22 ++++++ test/notebooks/pytest.ini | 2 + test/notebooks/test_dependencies.txt | 5 ++ 10 files changed, 164 insertions(+), 18 deletions(-) create mode 100644 doc/developer_guide/notebooks.md create mode 100644 test/integration/test_notebooks_in_dss_docker_image.py create mode 100644 test/notebooks/__init__.py create mode 100644 test/notebooks/nbtest_environment_test.py create mode 100644 test/notebooks/nbtest_itde.py create mode 100644 test/notebooks/pytest.ini create mode 100644 test/notebooks/test_dependencies.txt diff --git a/doc/developer_guide/developer_guide.md b/doc/developer_guide/developer_guide.md index b04d657b..db3f3359 100644 --- a/doc/developer_guide/developer_guide.md +++ b/doc/developer_guide/developer_guide.md @@ -50,3 +50,6 @@ Script `start-test-release-build` requires environment variable `GH_TOKEN` to co 3. [Testing](testing.md) 4. [Running tests in the CI](ci.md) 5. [Updating Packages](updating_packages.md) +6. [Notebooks](notebooks.md) + + diff --git a/doc/developer_guide/notebooks.md b/doc/developer_guide/notebooks.md new file mode 100644 index 00000000..d11b3523 --- /dev/null +++ b/doc/developer_guide/notebooks.md @@ -0,0 +1,30 @@ +# Notebook Files + +DSS repository includes some Jupyter notebooks and scripts to add these notebooks to DSS images, e.g. AMI or Docker Images. + +Please add or update the notebook files in folder [exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook](../../exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook). + +## Notebook Testing + +We are running tests for the notebooks in the Docker Edition of the AI Lab. For this we are creating a Docker test setup in +[test_notebooks_in_dss_docker_image.py](test/integration/test_notebooks_in_dss_docker_image.py) which installs test libraries into the AI Lab Docker Image. +It further creates a new test and Docker Container for each notebook test in [test/notebooks](test/notebooks). +Notebook test names need to fit the pattern `nbtest_*.py`, to prevent pytest running them outside of Docker setup. + +Environment variables with the prefix `NBTEST_` with which you call +[test_notebooks_in_dss_docker_image.py](test/integration/test_notebooks_in_dss_docker_image.py) are forwarded +into the Docker container and to the notebook test. You can use this to forward secrets to the notebook tests. + +By default all created containers and images are removed after running the tests regardless of success or failure. +However, with the following pytest commandline parameters you can keep them or reuse them to speed up local testing: + +``` + --dss-docker-image=DSS_DOCKER_IMAGE + Name and version of existing Docker image to use for tests + --keep-dss-docker-image + Keep the created dss docker image for inspection or reuse. + --docker-image-notebook-test=DOCKER_IMAGE_NOTEBOOK_TEST + Name and version of existing Docker image for Notebook testing to use for tests + --keep-docker-image-notebook-test + Keep the created notebook-test docker image for inspection or reuse. +``` \ No newline at end of file diff --git a/exasol/ds/sandbox/lib/dss_docker/create_image.py b/exasol/ds/sandbox/lib/dss_docker/create_image.py index d80b0291..0bcd8a8b 100644 --- a/exasol/ds/sandbox/lib/dss_docker/create_image.py +++ b/exasol/ds/sandbox/lib/dss_docker/create_image.py @@ -1,25 +1,22 @@ -import docker -import humanfriendly -import importlib_resources - -from functools import reduce from datetime import datetime -from docker.types import Mount -from exasol.ds.sandbox.lib import pretty_print -from importlib_metadata import version -from pathlib import Path +from functools import reduce from typing import Dict, List, Optional +import docker +import humanfriendly +import importlib_resources from docker.models.containers import Container as DockerContainer from docker.models.images import Image as DockerImage +from importlib_metadata import version -from exasol.ds.sandbox.lib.config import ConfigObject, SLC_VERSION -from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib import pretty_print from exasol.ds.sandbox.lib.ansible import ansible_repository -from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess, AnsibleFacts -from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext +from exasol.ds.sandbox.lib.config import ConfigObject, SLC_VERSION +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies DEFAULT_ORG_AND_REPOSITORY = "exasol/data-science-sandbox" DSS_VERSION = version("exasol-data-science-sandbox") @@ -151,13 +148,18 @@ def _commit_container( _logger.info("Committing changes to docker container") virtualenv = get_fact(facts, "jupyter", "virtualenv") port = get_fact(facts, "jupyter", "port") - notebook_folder = get_fact(facts, "notebook_folder", "final") + notebook_folder_final = get_fact(facts, "notebook_folder", "final") + notebook_folder_initial = get_fact(facts, "notebook_folder", "initial") conf = { "Entrypoint": entrypoint(facts), "Cmd": [], - "Volumes": { notebook_folder: {}, }, - "ExposedPorts": { f"{port}/tcp": {} }, - "Env": [ f"VIRTUAL_ENV={virtualenv}" ], + "Volumes": {notebook_folder_final: {}, }, + "ExposedPorts": {f"{port}/tcp": {}}, + "Env": [ + f"VIRTUAL_ENV={virtualenv}", + f"NOTEBOOK_FOLDER_FINAL={notebook_folder_final}", + f"NOTEBOOK_FOLDER_INITIAL={notebook_folder_initial}" + ], } return container.commit( repository=self.image_name, diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt index a2a0b339..5a7c8c3e 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt @@ -8,4 +8,4 @@ jupysql==0.10.7 sqlalchemy_exasol==4.6.3 stopwatch.py==2.0.1 --extra-index-url https://download.pytorch.org/whl/cpu -git+https://github.com/exasol/notebook-connector.git@0.2.6 +git+https://github.com/exasol/notebook-connector.git@aa1496f \ No newline at end of file diff --git a/test/integration/test_notebooks_in_dss_docker_image.py b/test/integration/test_notebooks_in_dss_docker_image.py new file mode 100644 index 00000000..7fb77308 --- /dev/null +++ b/test/integration/test_notebooks_in_dss_docker_image.py @@ -0,0 +1,73 @@ +import io +import os +from inspect import cleandoc +from pathlib import Path + +import pytest + +from test.integration.docker.exec_run import exec_command +from test.integration.docker.image import image +from test.integration.docker.in_memory_build_context import InMemoryBuildContext +from test.integration.docker.container import container + +TEST_RESOURCE_PATH = Path(__file__).parent.parent / "notebooks" + + +@pytest.fixture(scope="session") +def notebook_test_dockerfile_content(dss_docker_image) -> str: + yield cleandoc( + f""" + FROM {dss_docker_image.image_name} + COPY notebooks/* /tmp/notebooks/ + RUN mv /tmp/notebooks/* "$NOTEBOOK_FOLDER_INITIAL" && rmdir /tmp/notebooks/ + WORKDIR $NOTEBOOK_FOLDER_INITIAL + RUN "$VIRTUAL_ENV/bin/python3" -m pip install -r test_dependencies.txt + """ + ) + + +@pytest.fixture(scope="session") +def notebook_test_build_context(notebook_test_dockerfile_content) -> io.BytesIO: + with InMemoryBuildContext() as context: + context.add_string_to_file(name="Dockerfile", string=notebook_test_dockerfile_content) + context.add_host_path(host_path=str(TEST_RESOURCE_PATH), path_in_tar="../notebooks", recursive=True) + yield context.fileobj + + +@pytest.fixture(scope="session") +def notebook_test_image(request, notebook_test_build_context): + yield from image(request, + name="notebook_test", + fileobj=notebook_test_build_context, + custom_context=True, + print_log=True) + + +@pytest.fixture() +def notebook_test_container(request, notebook_test_image): + yield from container(request, + base_name="notebook_test_container", + image=notebook_test_image, + volumes={ + '/var/run/docker.sock': {'bind': '/var/run/docker.sock', 'mode': 'rw'}, + }, + ) + + +@pytest.mark.parametrize( + "notebook_test_file", + [ + python_file.name + for python_file in TEST_RESOURCE_PATH.glob("nbtest_*.py") + if python_file.is_file() + ] +) +def test_notebook(notebook_test_container, notebook_test_file): + container = notebook_test_container + command_echo_virtual_env = 'bash -c "echo $VIRTUAL_ENV"' + virtual_env = exec_command(command_echo_virtual_env, container) + command_run_test = f'{virtual_env}/bin/python -m pytest --setup-show -s {notebook_test_file}' + environ = os.environ.copy() + environ["NBTEST_ACTIVE"] = "TRUE" + nbtest_environ = {key: value for key, value in environ.items() if key.startswith("NBTEST_")} + exec_command(command_run_test, container, print_output=True, environment=nbtest_environ) diff --git a/test/notebooks/__init__.py b/test/notebooks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/notebooks/nbtest_environment_test.py b/test/notebooks/nbtest_environment_test.py new file mode 100644 index 00000000..32b28331 --- /dev/null +++ b/test/notebooks/nbtest_environment_test.py @@ -0,0 +1,9 @@ +import os + + +def test_assert_working_directory(): + assert os.getcwd() == os.environ["NOTEBOOK_FOLDER_INITIAL"] + + +def test_assert_environ_nbtest_active(): + assert os.environ["NBTEST_ACTIVE"] == "TRUE" diff --git a/test/notebooks/nbtest_itde.py b/test/notebooks/nbtest_itde.py new file mode 100644 index 00000000..6a9b8b02 --- /dev/null +++ b/test/notebooks/nbtest_itde.py @@ -0,0 +1,22 @@ +from exasol.secret_store import Secrets +from exasol.itde_manager import ( + bring_itde_up, + take_itde_down +) +from exasol.connections import open_pyexasol_connection + +def test_itde(tmp_path): + store_path = tmp_path / 'tmp_config.sqlite' + store_password = "password" + secrets = Secrets(store_path, master_password=store_password) + bring_itde_up(secrets) + try: + con = open_pyexasol_connection(secrets) + try: + result = con.execute("select 1").fetchmany() + assert result[0][0] == 1 + finally: + con.close() + finally: + take_itde_down(secrets) + diff --git a/test/notebooks/pytest.ini b/test/notebooks/pytest.ini new file mode 100644 index 00000000..be8d0936 --- /dev/null +++ b/test/notebooks/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +python_files = nbtest_*.py \ No newline at end of file diff --git a/test/notebooks/test_dependencies.txt b/test/notebooks/test_dependencies.txt new file mode 100644 index 00000000..75eb6f9f --- /dev/null +++ b/test/notebooks/test_dependencies.txt @@ -0,0 +1,5 @@ +nbclient +nbformat +pytest +testbook +pytest-check-links \ No newline at end of file