#324 Using the pytest-exasol-backend (#325)

* #317 Simplified the notebook_test_utils.py * #324 Simplified the notebook_test_utils.py * #324 Simplified the notebook_test_utils.py * #324 Simplified the notebook_test_utils.py * #324 Set up the backends to run the tests on * #324 Set up the backends to run the tests on * #324 Passing backend as a parameter * #324 Some debugging * #324 Some debugging * #324 Re-enabled all tests * #324 Addressed review issues * [CodeBuild]
exasol · Sep 4, 2024 · 776bb08 · 776bb08
1 parent c18369d
commit 776bb08
Show file tree

Hide file tree

Showing 17 changed files with 375 additions and 419 deletions.
diff --git a/.github/workflows/notebook_tests.yaml b/.github/workflows/notebook_tests.yaml
@@ -11,13 +11,15 @@ jobs:
       fail-fast: false
       matrix:
         nb_test:
-          - { name: "CSE notebook", test_file: "nbtest_cloud.py" }
-          - { name: "sklearn notebook", test_file: "nbtest_sklearn.py" }
-          - { name: "ibis notebook", test_file: "nbtest_ibis.py" }
-          - { name: "SLC notebook", test_file: "nbtest_script_languages_container.py" }
-          - { name: "SME notebooks", test_file: "nbtest_sagemaker.py"}
-          - { name: "TE notebooks", test_file: "nbtest_transformers.py"}
-          - { name: "short notebook tests", test_file: "\"nbtest_environment_test.py nbtest_itde.py\""}
+          - { name: "CSE notebook", test_file: "nbtest_cloud.py", test_backend: "all" }
+          - { name: "sklearn notebook", test_file: "nbtest_sklearn.py", test_backend: "all" }
+          - { name: "ibis notebook", test_file: "nbtest_ibis.py", test_backend: "all" }
+          - { name: "SLC notebook", test_file: "nbtest_script_languages_container.py", test_backend: "onprem" }
+          - { name: "SME notebooks onprem", test_file: "nbtest_sagemaker.py", test_backend: "onprem" }
+          - { name: "SME notebooks saas", test_file: "nbtest_sagemaker.py", test_backend: "saas" }
+          - { name: "TE notebooks onprem", test_file: "nbtest_transformers.py", test_backend: "onprem" }
+          - { name: "TE notebooks saas", test_file: "nbtest_transformers.py", test_backend: "saas"}
+          - { name: "short notebook tests", test_file: "\"nbtest_environment_test.py nbtest_itde.py\"", test_backend: ""}
     name: Running ${{ matrix.nb_test.name }}
     steps:
       - uses: actions/checkout@v4
@@ -32,6 +34,7 @@ jobs:
             --capture=no \
             --override-ini=log_cli=true \
             --override-ini=log_cli_level=INFO \
+            --nb-test-backend=${{ matrix.nb_test.test_backend }} \
             --nb-test-file=${{ matrix.nb_test.test_file }} \
             test/notebook_test_runner/test_notebooks_in_dss_docker_image.py
         env:

diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md
@@ -1,5 +1,6 @@
 # Changes
 
+* [3.1.0](changes_3.1.0.md)
 * [3.0.0](changes_3.0.0.md)
 * [2.0.0](changes_2.0.0.md)
 * [1.0.0](changes_1.0.0.md)

diff --git a/doc/changes/changes_3.1.0.md b/doc/changes/changes_3.1.0.md
@@ -0,0 +1,6 @@
+# AI-Lab 3.1.0 released T.B.D.
+
+## Refactoring
+
+* #324 Used pytest-plugins in notebook tests.
+
diff --git a/doc/developer_guide/notebooks.md b/doc/developer_guide/notebooks.md
@@ -1,9 +1,19 @@
 # Notebook Files
 
-DSS repository includes some Jupyter notebooks and scripts to add these notebooks to DSS images, e.g. AMI or Docker Images.
+AI-Lab repository includes some Jupyter notebooks and scripts to add these notebooks to AI-Lab images (the old name - DSS images), e.g. AMI or Docker Images.
 
 Please add or update the notebook files in folder [exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook](../../exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook).
 
+## Notebook Connector
+
+Most notebooks use the `exasol-notebook-connector` - the component providing various helper functions for notebooks.
+We typically put there the code we don't want to show in a notebook. Logically, this component is a part of the AI-Lab.
+It is listed in the [notebook_requirements.txt](../../exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt) and gets installed in the container with pip.
+In order to simplify making simultaneous changes to both notebooks and the notebook-connector we allow referencing the latter temporarily through a git dependency,
+e.g. `exasol-notebook-connector @ git+https://github.com/exasol/notebook-connector@e3a8525`.
+It is important to reference a commit in the `main` branch, not the `main` branch itself. The git dependency should be
+replaced with the proper pypi dependency before releasing of the AI-Lab.
+
 ## Notebook Testing
 
 We are running tests for the notebooks in the Docker Edition of the AI Lab. For this we are creating a Docker test setup in 

diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt
@@ -2,7 +2,7 @@ scikit-learn==1.5.1 # required for notebook sklearn
 matplotlib==3.7.4 # required for notebook sklearn
 jupysql==0.10.10 # required for multiple notebooks
 stopwatch.py>=2.0.1 # also required by ITDE
-exasol-notebook-connector==0.2.9
+exasol-notebook-connector @ git+https://github.com/exasol/notebook-connector@e3a8525
 pickleshare==0.7.5 # See https://github.com/exasol/ai-lab/issues/291 for details.
 ipyfilechooser==0.6.0 # required for SLC notebooks
 ipywidgets==8.1.1 # enable interactive Javascript widgets in the notebooks
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "exasol-ai-lab"
-version = "3.0.0"
+version = "3.1.0"
 description = "Provide AI-Lab editions."
 packages = [ {include = "exasol"}, ]
 license = "MIT"

diff --git a/test/notebook_test_runner/conftest.py b/test/notebook_test_runner/conftest.py
@@ -3,8 +3,14 @@
 
 def pytest_addoption(parser):
     parser.addoption("--nb-test-file", action="store", help="Notebook test file nbtest_*.py")
+    parser.addoption("--nb-test-backend", action="store", help="Backend to run the notebook test on")
 
 
 @pytest.fixture
 def notebook_test_file(request):
     return request.config.getoption("--nb-test-file")
+
+
+@pytest.fixture
+def notebook_test_backend(request):
+    return request.config.getoption("--nb-test-backend")
diff --git a/test/notebook_test_runner/test_notebooks_in_dss_docker_image.py b/test/notebook_test_runner/test_notebooks_in_dss_docker_image.py
@@ -74,14 +74,14 @@ def notebook_test_container_with_log(notebook_test_container):
     yield container
 
 
-def test_notebook(notebook_test_container_with_log, notebook_test_file):
-    _logger.info(f"Running notebook tests for {notebook_test_file}")
+def test_notebook(notebook_test_container_with_log, notebook_test_file, notebook_test_backend):
+    _logger.info(f"Running notebook tests for {notebook_test_file} at {notebook_test_backend}")
     container = notebook_test_container_with_log
     command_echo_virtual_env = 'bash -c "echo $VIRTUAL_ENV"'
     virtual_env = exec_command(command_echo_virtual_env, container)
     command_run_test = (
         f"{virtual_env}/bin/python"
-        f" -m pytest --setup-show -s {notebook_test_file}"
+        f" -m pytest --setup-show -s --backend={notebook_test_backend} {notebook_test_file}"
     )
     environ = os.environ.copy()
     environ["NBTEST_ACTIVE"] = "TRUE"

diff --git a/test/notebooks/nbtest_cloud.py b/test/notebooks/nbtest_cloud.py
@@ -1,18 +1,16 @@
 import os
 
-import pytest
-
+# We need to manually import all fixtures that we use, directly or indirectly,
+# since the pytest won't do this for us.
 from notebook_test_utils import (
-    access_to_temp_secret_store,
-    access_to_temp_saas_secret_store,
+    backend_setup,
     notebook_runner,
     set_log_level_for_libraries,
 )
-from exasol.nb_connector.ai_lab_config import StorageBackend
 
 set_log_level_for_libraries()
 
-@pytest.mark.parametrize('notebook_runner', [StorageBackend.onprem, StorageBackend.saas], indirect=True)
+
 def test_cloud_notebook(notebook_runner) -> None:
 
     current_dir = os.getcwd()

diff --git a/test/notebooks/nbtest_ibis.py b/test/notebooks/nbtest_ibis.py
@@ -1,11 +1,10 @@
 import os
 import textwrap
-import pytest
 
-from exasol.nb_connector.ai_lab_config import StorageBackend
+# We need to manually import all fixtures that we use, directly or indirectly,
+# since the pytest won't do this for us.
 from notebook_test_utils import (
-    access_to_temp_secret_store,
-    access_to_temp_saas_secret_store,
+    backend_setup,
     notebook_runner,
     set_log_level_for_libraries,
 )
@@ -14,7 +13,6 @@
 set_log_level_for_libraries()
 
 
-@pytest.mark.parametrize('notebook_runner', [StorageBackend.onprem, StorageBackend.saas], indirect=True)
 def test_quickstart(notebook_runner) -> None:
 
     data_import_hack = (

diff --git a/test/notebooks/nbtest_sagemaker.py b/test/notebooks/nbtest_sagemaker.py
@@ -5,13 +5,13 @@
 from inspect import cleandoc
 import textwrap
 
-import pytest
 from exasol.nb_connector.secret_store import Secrets
-from exasol.nb_connector.ai_lab_config import AILabConfig as CKey, StorageBackend
+from exasol.nb_connector.ai_lab_config import AILabConfig as CKey
 
+# We need to manually import all fixtures that we use, directly or indirectly,
+# since the pytest won't do this for us.
 from notebook_test_utils import (
-    access_to_temp_secret_store,
-    access_to_temp_saas_secret_store,
+    backend_setup,
     run_notebook,
     uploading_hack,
     set_log_level_for_libraries,
@@ -182,10 +182,9 @@ def continuous_job_polling():
     )
 
 
-@pytest.mark.parametrize('access_to_temp_secret_store', [StorageBackend.onprem, StorageBackend.saas], indirect=True)
-def test_sagemaker(access_to_temp_secret_store, uploading_hack):
+def test_sagemaker(backend_setup, uploading_hack):
 
-    store_path, store_password = access_to_temp_secret_store
+    store_path, store_password = backend_setup
     store_file = str(store_path)
 
     _copy_aws_credentials()

diff --git a/test/notebooks/nbtest_script_languages_container.py b/test/notebooks/nbtest_script_languages_container.py
@@ -1,14 +1,16 @@
 import os
 from pathlib import Path
+import shutil
 
 import pytest
 
-from notebook_test_utils import (access_to_temp_secret_store,
-                                 access_to_temp_saas_secret_store,
-                                 run_notebook,
-                                 uploading_hack)
-from exasol.nb_connector.ai_lab_config import AILabConfig as CKey, StorageBackend
+# We need to manually import all fixtures that we use, directly or indirectly,
+# since the pytest won't do this for us.
+from notebook_test_utils import (backend_setup,
+                                 run_notebook)
+from exasol.nb_connector.ai_lab_config import AILabConfig as CKey
 from exasol.nb_connector.secret_store import Secrets
+from exasol.pytest_backend import BACKEND_ONPREM
 
 
 def _slc_repo_dir() -> Path:
@@ -23,19 +25,22 @@ def _store_slc_config(store_path: Path, store_password: str, clone_repo: bool):
     conf.save(CKey.slc_source, slc_source)
     conf.save(CKey.slc_target_dir, str(_slc_repo_dir()))
 
+
 @pytest.fixture()
-def cleanup_slc_repo_dir():
-    import shutil
+def cleanup_slc_repo_dir(backend):
     yield
-    p = Path.cwd() / "script_languages_container" / "script_languages_release"
-    shutil.rmtree(p)
+    if backend == BACKEND_ONPREM:
+        p = Path.cwd() / "script_languages_container" / "script_languages_release"
+        shutil.rmtree(p)
 
 
-@pytest.mark.parametrize('access_to_temp_secret_store', [StorageBackend.onprem], indirect=True)
-def test_script_languages_container_cloning_slc_repo(access_to_temp_secret_store,
+def test_script_languages_container_cloning_slc_repo(backend,
+                                                     backend_setup,
                                                      cleanup_slc_repo_dir) -> None:
+    if backend != BACKEND_ONPREM:
+        pytest.skip()
     current_dir = Path.cwd()
-    store_path, store_password = access_to_temp_secret_store
+    store_path, store_password = backend_setup
     store_file = str(store_path)
     try:
         run_notebook('main_config.ipynb', store_file, store_password)
@@ -57,11 +62,13 @@ def _clone_slc_repo():
     repo.submodule_update(recursive=True)
 
 
-@pytest.mark.parametrize('access_to_temp_secret_store', [StorageBackend.onprem], indirect=True)
-def test_script_languages_container_with_existing_slc_repo(access_to_temp_secret_store,
+def test_script_languages_container_with_existing_slc_repo(backend,
+                                                           backend_setup,
                                                            cleanup_slc_repo_dir) -> None:
+    if backend != BACKEND_ONPREM:
+        pytest.skip()
     current_dir = Path.cwd()
-    store_path, store_password = access_to_temp_secret_store
+    store_path, store_password = backend_setup
     store_file = str(store_path)
     try:
         run_notebook('main_config.ipynb', store_file, store_password)

diff --git a/test/notebooks/nbtest_sklearn.py b/test/notebooks/nbtest_sklearn.py
@@ -1,10 +1,9 @@
 import os
-import pytest
 
-from exasol.nb_connector.ai_lab_config import StorageBackend
+# We need to manually import all fixtures that we use, directly or indirectly,
+# since the pytest won't do this for us.
 from notebook_test_utils import (
-    access_to_temp_secret_store,
-    access_to_temp_saas_secret_store,
+    backend_setup,
     notebook_runner,
     set_log_level_for_libraries,
 )
@@ -13,7 +12,6 @@
 set_log_level_for_libraries()
 
 
-@pytest.mark.parametrize('notebook_runner', [StorageBackend.onprem, StorageBackend.saas], indirect=True)
 def test_regression(notebook_runner) -> None:
 
     current_dir = os.getcwd()
@@ -30,7 +28,6 @@ def test_regression(notebook_runner) -> None:
         os.chdir(current_dir)
 
 
-@pytest.mark.parametrize('notebook_runner', [StorageBackend.onprem, StorageBackend.saas], indirect=True)
 def test_classification(notebook_runner) -> None:
 
     current_dir = os.getcwd()

diff --git a/test/notebooks/nbtest_transformers.py b/test/notebooks/nbtest_transformers.py
@@ -2,10 +2,10 @@
 import textwrap
 import pytest
 
-from exasol.nb_connector.ai_lab_config import StorageBackend
+# We need to manually import all fixtures that we use, directly or indirectly,
+# since the pytest won't do this for us.
 from notebook_test_utils import (
-    access_to_temp_secret_store,
-    access_to_temp_saas_secret_store,
+    backend_setup,
     notebook_runner,
     uploading_hack,
     set_log_level_for_libraries,
@@ -27,7 +27,6 @@
         'zero_shot_classification.ipynb'
     ]
 )
-@pytest.mark.parametrize('notebook_runner', [StorageBackend.onprem, StorageBackend.saas], indirect=True)
 def test_transformers(notebook_runner, uploading_hack, notebook_file) -> None:
 
     running_hack = (