#139: Create Docker Test Setup for notebook tests (#142)

* Update Notebook Connector to the commit that allows the AI Lab container to access the ITDE

Co-authored-by: Christoph Pirkl <[email protected]>
Co-authored-by: Mikhail Beck <[email protected]>

doc/developer_guide/developer_guide.md

@@ -50,3 +50,6 @@ Script `start-test-release-build` requires environment variable `GH_TOKEN` to co
 3. [Testing](testing.md)
 4. [Running tests in the CI](ci.md)
 5. [Updating Packages](updating_packages.md)
+6. [Notebooks](notebooks.md)
+
+

doc/developer_guide/notebooks.md

@@ -0,0 +1,30 @@
+# Notebook Files
+
+DSS repository includes some Jupyter notebooks and scripts to add these notebooks to DSS images, e.g. AMI or Docker Images.
+
+Please add or update the notebook files in folder [exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook](../../exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook).
+
+## Notebook Testing
+
+We are running tests for the notebooks in the Docker Edition of the AI Lab. For this we are creating a Docker test setup in 
+[test_notebooks_in_dss_docker_image.py](test/integration/test_notebooks_in_dss_docker_image.py) which installs test libraries into the AI Lab Docker Image.
+It further creates a new test and Docker Container for each notebook test in [test/notebooks](test/notebooks). 
+Notebook test names need to fit the pattern `nbtest_*.py`, to prevent pytest running them outside of Docker setup.
+
+Environment variables with the prefix `NBTEST_` with which you call
+[test_notebooks_in_dss_docker_image.py](test/integration/test_notebooks_in_dss_docker_image.py) are forwarded 
+into the Docker container and to the notebook test. You can use this to forward secrets to the notebook tests.
+
+By default all created containers and images are removed after running the tests regardless of success or failure.
+However, with the following pytest commandline parameters you can keep them or reuse them to speed up local testing:
+
+```
+  --dss-docker-image=DSS_DOCKER_IMAGE
+                        Name and version of existing Docker image to use for tests
+  --keep-dss-docker-image
+                        Keep the created dss docker image for inspection or reuse.
+  --docker-image-notebook-test=DOCKER_IMAGE_NOTEBOOK_TEST
+                        Name and version of existing Docker image for Notebook testing to use for tests
+  --keep-docker-image-notebook-test
+                        Keep the created notebook-test docker image for inspection or reuse.
+```

exasol/ds/sandbox/lib/dss_docker/create_image.py

@@ -1,25 +1,22 @@
-import docker
-import humanfriendly
-import importlib_resources
-
-from functools import reduce
 from datetime import datetime
-from docker.types import Mount
-from exasol.ds.sandbox.lib import pretty_print
-from importlib_metadata import version
-from pathlib import Path
+from functools import reduce
 from typing import Dict, List, Optional
 
+import docker
+import humanfriendly
+import importlib_resources
 from docker.models.containers import Container as DockerContainer
 from docker.models.images import Image as DockerImage
+from importlib_metadata import version
 
-from exasol.ds.sandbox.lib.config import ConfigObject, SLC_VERSION
-from exasol.ds.sandbox.lib.logging import get_status_logger, LogType
+from exasol.ds.sandbox.lib import pretty_print
 from exasol.ds.sandbox.lib.ansible import ansible_repository
-from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext
 from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess, AnsibleFacts
-from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies
+from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext
+from exasol.ds.sandbox.lib.config import ConfigObject, SLC_VERSION
+from exasol.ds.sandbox.lib.logging import get_status_logger, LogType
 from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo
+from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies
 
 DEFAULT_ORG_AND_REPOSITORY = "exasol/data-science-sandbox"
 DSS_VERSION = version("exasol-data-science-sandbox")
@@ -151,13 +148,18 @@ def _commit_container(
         _logger.info("Committing changes to docker container")
         virtualenv = get_fact(facts, "jupyter", "virtualenv")
         port = get_fact(facts, "jupyter", "port")
-        notebook_folder = get_fact(facts, "notebook_folder", "final")
+        notebook_folder_final = get_fact(facts, "notebook_folder", "final")
+        notebook_folder_initial = get_fact(facts, "notebook_folder", "initial")
         conf = {
             "Entrypoint": entrypoint(facts),
             "Cmd": [],
-            "Volumes": { notebook_folder: {}, },
-            "ExposedPorts": { f"{port}/tcp": {} },
-            "Env": [ f"VIRTUAL_ENV={virtualenv}" ],
+            "Volumes": {notebook_folder_final: {}, },
+            "ExposedPorts": {f"{port}/tcp": {}},
+            "Env": [
+                f"VIRTUAL_ENV={virtualenv}",
+                f"NOTEBOOK_FOLDER_FINAL={notebook_folder_final}",
+                f"NOTEBOOK_FOLDER_INITIAL={notebook_folder_initial}"
+            ],
         }
         return container.commit(
             repository=self.image_name,

exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt

@@ -8,4 +8,4 @@ jupysql==0.10.7
 sqlalchemy_exasol==4.6.3
 stopwatch.py==2.0.1
 --extra-index-url https://download.pytorch.org/whl/cpu
-git+https://github.com/exasol/notebook-connector.git@0.2.6
+git+https://github.com/exasol/notebook-connector.git@aa1496f

test/integration/test_notebooks_in_dss_docker_image.py

@@ -0,0 +1,73 @@
+import io
+import os
+from inspect import cleandoc
+from pathlib import Path
+
+import pytest
+
+from test.integration.docker.exec_run import exec_command
+from test.integration.docker.image import image
+from test.integration.docker.in_memory_build_context import InMemoryBuildContext
+from test.integration.docker.container import container
+
+TEST_RESOURCE_PATH = Path(__file__).parent.parent / "notebooks"
+
+
+@pytest.fixture(scope="session")
+def notebook_test_dockerfile_content(dss_docker_image) -> str:
+    yield cleandoc(
+        f"""
+        FROM {dss_docker_image.image_name} 
+        COPY notebooks/* /tmp/notebooks/
+        RUN mv /tmp/notebooks/* "$NOTEBOOK_FOLDER_INITIAL" && rmdir /tmp/notebooks/
+        WORKDIR $NOTEBOOK_FOLDER_INITIAL
+        RUN "$VIRTUAL_ENV/bin/python3" -m pip install -r test_dependencies.txt
+        """
+    )
+
+
+@pytest.fixture(scope="session")
+def notebook_test_build_context(notebook_test_dockerfile_content) -> io.BytesIO:
+    with InMemoryBuildContext() as context:
+        context.add_string_to_file(name="Dockerfile", string=notebook_test_dockerfile_content)
+        context.add_host_path(host_path=str(TEST_RESOURCE_PATH), path_in_tar="../notebooks", recursive=True)
+    yield context.fileobj
+
+
+@pytest.fixture(scope="session")
+def notebook_test_image(request, notebook_test_build_context):
+    yield from image(request,
+                     name="notebook_test",
+                     fileobj=notebook_test_build_context,
+                     custom_context=True,
+                     print_log=True)
+
+
+@pytest.fixture()
+def notebook_test_container(request, notebook_test_image):
+    yield from container(request,
+                         base_name="notebook_test_container",
+                         image=notebook_test_image,
+                         volumes={
+                             '/var/run/docker.sock': {'bind': '/var/run/docker.sock', 'mode': 'rw'},
+                         },
+                         )
+
+
+@pytest.mark.parametrize(
+    "notebook_test_file",
+    [
+        python_file.name
+        for python_file in TEST_RESOURCE_PATH.glob("nbtest_*.py")
+        if python_file.is_file()
+    ]
+)
+def test_notebook(notebook_test_container, notebook_test_file):
+    container = notebook_test_container
+    command_echo_virtual_env = 'bash -c "echo $VIRTUAL_ENV"'
+    virtual_env = exec_command(command_echo_virtual_env, container)
+    command_run_test = f'{virtual_env}/bin/python -m pytest --setup-show -s {notebook_test_file}'
+    environ = os.environ.copy()
+    environ["NBTEST_ACTIVE"] = "TRUE"
+    nbtest_environ = {key: value for key, value in environ.items() if key.startswith("NBTEST_")}
+    exec_command(command_run_test, container, print_output=True, environment=nbtest_environ)

test/notebooks/nbtest_environment_test.py

@@ -0,0 +1,9 @@
+import os
+
+
+def test_assert_working_directory():
+    assert os.getcwd() == os.environ["NOTEBOOK_FOLDER_INITIAL"]
+
+
+def test_assert_environ_nbtest_active():
+    assert os.environ["NBTEST_ACTIVE"] == "TRUE"

test/notebooks/nbtest_itde.py

@@ -0,0 +1,22 @@
+from exasol.secret_store import Secrets
+from exasol.itde_manager import (
+    bring_itde_up,
+    take_itde_down
+)
+from exasol.connections import open_pyexasol_connection
+
+def test_itde(tmp_path):
+    store_path = tmp_path / 'tmp_config.sqlite'
+    store_password = "password"
+    secrets = Secrets(store_path, master_password=store_password)
+    bring_itde_up(secrets)
+    try:
+        con = open_pyexasol_connection(secrets)
+        try:
+            result = con.execute("select 1").fetchmany()
+            assert result[0][0] == 1
+        finally:
+            con.close()
+    finally:
+        take_itde_down(secrets)
+

test/notebooks/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+python_files = nbtest_*.py

test/notebooks/test_dependencies.txt

@@ -0,0 +1,5 @@
+nbclient
+nbformat
+pytest
+testbook
+pytest-check-links