From 941c0a1a93267712d9d38314b49f680211d9d77e Mon Sep 17 00:00:00 2001 From: Mikhail Beck Date: Wed, 12 Jun 2024 08:28:25 +0100 Subject: [PATCH] #279 Made the nodebook testing running on SaaS [run-notebook-tests] (#281) * #279 Made the nodebook testing running on SaaS [run-notebook-tests] * #279 Enabled new fixture [run-notebook-tests] * #279 Added SaaS secrets to ci [run-notebook-tests] * #279 Added SaaS secrets to ci [run-notebook-tests] * #279 Fixed a bug with a cfg name [run-notebook-tests] * #279 Updated the dependency again [run-notebook-tests] * #279 Updated the dependency again [run-notebook-tests] * #279 Updated the dependency again [run-notebook-tests] * #279 Updated the dependency again [run-notebook-tests] * #279 Updated the dependency again [run-notebook-tests] * #279 Updated the dependency again [run-notebook-tests] * #279 Updated the dependency again [run-notebook-tests] * #279 Updated the dependency again [run-notebook-tests] * #279 Updated the dependency again [run-notebook-tests] * #279 Looking at the permission error [run-notebook-tests] * #279 Looking at the permission error [run-notebook-tests] * #279 Fixing the fixtures [run-notebook-tests] * #279 Fixing the fixtures [run-notebook-tests] * #279 Enabled SageMaker test [run-notebook-tests] * #279 Increased the database time-out [run-notebook-tests] --- .github/workflows/check_ci.yaml | 3 + doc/changes/changes_2.1.0.md | 1 + .../test_notebooks_in_dss_docker_image.py | 3 +- test/notebooks/nbtest_sagemaker.py | 8 +- test/notebooks/nbtest_sklearn.py | 8 +- test/notebooks/nbtest_transformers.py | 6 +- test/notebooks/notebook_test_utils.py | 107 ++++++++++++++++-- test/notebooks/test_dependencies.txt | 4 +- 8 files changed, 125 insertions(+), 15 deletions(-) diff --git a/.github/workflows/check_ci.yaml b/.github/workflows/check_ci.yaml index 82f7698b..004a3916 100644 --- a/.github/workflows/check_ci.yaml +++ b/.github/workflows/check_ci.yaml @@ -113,3 +113,6 @@ jobs: NBTEST_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} NBTEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }} NBTEST_AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} + SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} + SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} diff --git a/doc/changes/changes_2.1.0.md b/doc/changes/changes_2.1.0.md index eaf58400..1ff541a4 100644 --- a/doc/changes/changes_2.1.0.md +++ b/doc/changes/changes_2.1.0.md @@ -13,6 +13,7 @@ Version: 2.1.0 ## Features * 277 Added the SaaS database parameters to the configuration page. +* 279 Made the notebooks tests running in SaaS as well as in the Docker-DB. ## Security diff --git a/test/notebook_test_runner/test_notebooks_in_dss_docker_image.py b/test/notebook_test_runner/test_notebooks_in_dss_docker_image.py index dec0aced..bd4b97e0 100644 --- a/test/notebook_test_runner/test_notebooks_in_dss_docker_image.py +++ b/test/notebook_test_runner/test_notebooks_in_dss_docker_image.py @@ -107,5 +107,6 @@ def test_notebook(notebook_test_container_with_log, notebook_test_file): ) environ = os.environ.copy() environ["NBTEST_ACTIVE"] = "TRUE" - nbtest_environ = {key: value for key, value in environ.items() if key.startswith("NBTEST_")} + nbtest_environ = {key: value for key, value in environ.items() if ( + key.startswith("NBTEST_") or key.startswith("SAAS_"))} exec_command(command_run_test, container, print_output=True, environment=nbtest_environ, user="jupyter") diff --git a/test/notebooks/nbtest_sagemaker.py b/test/notebooks/nbtest_sagemaker.py index bf919750..f1129dfc 100644 --- a/test/notebooks/nbtest_sagemaker.py +++ b/test/notebooks/nbtest_sagemaker.py @@ -7,9 +7,12 @@ import pytest from exasol.nb_connector.secret_store import Secrets -from exasol.nb_connector.ai_lab_config import AILabConfig as CKey +from exasol.nb_connector.ai_lab_config import AILabConfig as CKey, StorageBackend -from notebook_test_utils import (access_to_temp_secret_store, run_notebook, uploading_hack) +from notebook_test_utils import (access_to_temp_secret_store, + access_to_temp_saas_secret_store, + run_notebook, + uploading_hack) def _create_aws_s3_bucket() -> str: @@ -173,6 +176,7 @@ def continuous_job_polling(): ) +@pytest.mark.parametrize('access_to_temp_secret_store', [StorageBackend.onprem, StorageBackend.saas], indirect=True) def test_sagemaker(access_to_temp_secret_store, uploading_hack): store_path, store_password = access_to_temp_secret_store diff --git a/test/notebooks/nbtest_sklearn.py b/test/notebooks/nbtest_sklearn.py index 60c93faa..fb75396d 100644 --- a/test/notebooks/nbtest_sklearn.py +++ b/test/notebooks/nbtest_sklearn.py @@ -1,8 +1,13 @@ import os +import pytest -from notebook_test_utils import (access_to_temp_secret_store, notebook_runner) +from exasol.nb_connector.ai_lab_config import StorageBackend +from notebook_test_utils import (access_to_temp_secret_store, + access_to_temp_saas_secret_store, + notebook_runner) +@pytest.mark.parametrize('notebook_runner', [StorageBackend.onprem, StorageBackend.saas], indirect=True) def test_regression(notebook_runner) -> None: current_dir = os.getcwd() @@ -18,6 +23,7 @@ def test_regression(notebook_runner) -> None: os.chdir(current_dir) +@pytest.mark.parametrize('notebook_runner', [StorageBackend.onprem, StorageBackend.saas], indirect=True) def test_classification(notebook_runner) -> None: current_dir = os.getcwd() diff --git a/test/notebooks/nbtest_transformers.py b/test/notebooks/nbtest_transformers.py index a05fa43e..141bc194 100644 --- a/test/notebooks/nbtest_transformers.py +++ b/test/notebooks/nbtest_transformers.py @@ -2,7 +2,10 @@ import textwrap import pytest -from notebook_test_utils import (access_to_temp_secret_store, notebook_runner, uploading_hack) +from notebook_test_utils import (access_to_temp_secret_store, + access_to_temp_saas_secret_store, + notebook_runner, + uploading_hack) @pytest.mark.parametrize( @@ -17,6 +20,7 @@ 'zero_shot_classification.ipynb' ] ) +@pytest.mark.skip(reason="The expected functionality is not yet implemented in the Transformers Extension") def test_transformers(notebook_runner, uploading_hack, notebook_file) -> None: running_hack = ( diff --git a/test/notebooks/notebook_test_utils.py b/test/notebooks/notebook_test_utils.py index 9c69522c..573a8d30 100644 --- a/test/notebooks/notebook_test_utils.py +++ b/test/notebooks/notebook_test_utils.py @@ -4,6 +4,9 @@ import random import string import textwrap +from contextlib import contextmanager, ExitStack +from datetime import timedelta +import os import pytest import nbformat @@ -11,11 +14,23 @@ import requests from exasol.nb_connector.secret_store import Secrets -from exasol.nb_connector.ai_lab_config import AILabConfig as CKey +from exasol.nb_connector.ai_lab_config import AILabConfig as CKey, StorageBackend from exasol.nb_connector.itde_manager import ( bring_itde_up, take_itde_down ) +from exasol.saas.client.api_access import ( + OpenApiAccess, + create_saas_client, + timestamp_name, +) + + +def _env(var: str) -> str: + result = os.environ.get(var) + if result: + return result + raise RuntimeError(f"Environment variable {var} is empty.") def generate_password(pwd_length): @@ -31,13 +46,22 @@ def url_exists(url): return False -def _init_secret_store(secrets: Secrets) -> None: +def _init_onprem_secret_store(secrets: Secrets) -> None: secrets.save(CKey.use_itde, 'yes') secrets.save(CKey.mem_size, '4') secrets.save(CKey.disk_size, '4') secrets.save(CKey.db_schema, 'NOTEBOOK_TESTS') +def _init_saas_secret_store(secrets: Secrets) -> None: + secrets.save(CKey.storage_backend, StorageBackend.saas.name) + secrets.save(CKey.saas_url, _env("SAAS_HOST")) + secrets.save(CKey.saas_token, _env("SAAS_PAT")) + secrets.save(CKey.saas_account_id, _env("SAAS_ACCOUNT_ID")) + secrets.save(CKey.saas_database_name, timestamp_name('NBTEST')) + secrets.save(CKey.db_schema, 'NOTEBOOK_TESTS_SAAS') + + def _insert_hacks(nb: nbformat.NotebookNode, hacks: List[Tuple[str, str]]): def cell_match(nb_cell, ins_tag: str) -> bool: @@ -89,8 +113,8 @@ def init_notebook_test(): nb_client.execute() -@pytest.fixture -def access_to_temp_secret_store(tmp_path: Path) -> Tuple[Path, str]: +@contextmanager +def access_to_temp_onprem_secret_store(tmp_path: Path) -> Tuple[Path, str]: """ Creates a temporary configuration store. Brings up and subsequently destroys the Exasol Docker-DB. @@ -109,7 +133,7 @@ def access_to_temp_secret_store(tmp_path: Path) -> Tuple[Path, str]: # Set the configuration required by the ITDE manager and those the # manager will not set after starting the Exasol Docker-DB. - _init_secret_store(secrets) + _init_onprem_secret_store(secrets) # Start the Exasol Docker-DB and then destroy it after the test finishes. bring_itde_up(secrets) @@ -119,14 +143,79 @@ def access_to_temp_secret_store(tmp_path: Path) -> Tuple[Path, str]: take_itde_down(secrets) +@pytest.fixture(scope='session') +def access_to_temp_saas_secret_store(tmp_path_factory) -> Tuple[Path, str]: + """ + Creates a temporary configuration store. + Initiates the creation of a temporary SaaS database and waits till this database + becomes operational. + Saves the SaaS connection parameters in the configuration store. + """ + + store_path = tmp_path_factory.mktemp('tmp_config_dir') / 'tmp_config_saas.sqlite' + # See access_to_temp_onprem_secret_store for considerations about the store password. + store_password = generate_password(12) + secrets = Secrets(store_path, master_password=store_password) + + _init_saas_secret_store(secrets) + + with ExitStack() as stack: + client = stack.enter_context(create_saas_client( + host=secrets.get(CKey.saas_url), + pat=secrets.get(CKey.saas_token))) + api_access = OpenApiAccess( + client=client, + account_id=secrets.get(CKey.saas_account_id)) + stack.enter_context(api_access.allowed_ip()) + db = stack.enter_context(api_access.database( + name=secrets.get(CKey.saas_database_name), + idle_time=timedelta(hours=12))) + api_access.wait_until_running(db.id) + yield store_path, store_password + + @pytest.fixture -def notebook_runner(access_to_temp_secret_store) -> Callable: +def access_to_temp_secret_store(request, + tmp_path: Path, + access_to_temp_saas_secret_store + ) -> Tuple[Path, str]: """ - A fixture for running a notebook. + Creates a temporary configuration store. + Ensures that the database (either on-prem or SaaS, depending on the request parameter) + is running for the duration of the fixture. """ + if request.param == StorageBackend.onprem: + with access_to_temp_onprem_secret_store(tmp_path) as onprem_store: + yield onprem_store + elif request.param == StorageBackend.saas: + yield access_to_temp_saas_secret_store + else: + raise ValueError(('Unrecognised testing backend in the access_to_temp_secret_store. ' + 'Should be either "onprem" or "saas"')) - store_path, store_password = access_to_temp_secret_store - return partial(run_notebook, store_file=str(store_path), store_password=store_password) + +@pytest.fixture +def notebook_runner(request, + tmp_path: Path, + access_to_temp_saas_secret_store + ) -> Callable: + """ + A fixture for running a notebook. + """ + if request.param == StorageBackend.onprem: + with access_to_temp_onprem_secret_store(tmp_path) as onprem_store: + store_path, store_password = onprem_store + yield partial(run_notebook, + store_file=str(store_path), + store_password=store_password) + elif request.param == StorageBackend.saas: + store_path, store_password = access_to_temp_saas_secret_store + yield partial(run_notebook, + store_file=str(store_path), + store_password=store_password) + else: + raise ValueError(('Unrecognised testing backend in the notebook_runner. ' + 'Should be either "onprem" or "saas"')) @pytest.fixture diff --git a/test/notebooks/test_dependencies.txt b/test/notebooks/test_dependencies.txt index 75eb6f9f..44cd8bc0 100644 --- a/test/notebooks/test_dependencies.txt +++ b/test/notebooks/test_dependencies.txt @@ -2,4 +2,6 @@ nbclient nbformat pytest testbook -pytest-check-links \ No newline at end of file +pytest-check-links +exasol-saas-api @ git+https://github.com/exasol/saas-api-python.git@main +