From cf9fe76e85a3ecaf928409ce0de92fb930e79d73 Mon Sep 17 00:00:00 2001 From: mibe Date: Thu, 30 May 2024 16:52:39 +0100 Subject: [PATCH 1/3] Addressed review comments [run-notebook-tests] --- doc/changes/changes_2.1.0.md | 4 +- .../files/notebook/utils/main_config_ui.ipynb | 121 ++++++++++++------ .../jupyter/files/notebook_requirements.txt | 2 +- 3 files changed, 84 insertions(+), 43 deletions(-) diff --git a/doc/changes/changes_2.1.0.md b/doc/changes/changes_2.1.0.md index 0c56d205..1b6e186b 100644 --- a/doc/changes/changes_2.1.0.md +++ b/doc/changes/changes_2.1.0.md @@ -12,7 +12,8 @@ Version: 2.1.0 ## Features -n/a +* 277 Added configuration for a SaaS Database. + Added SaaS as the third option in the database selection UI. ## Security @@ -29,3 +30,4 @@ n/a ## Refactoring * #267: Switched CodeBuildWaiter to use tenacity +* #276: Started using the new ITDE Manager interface in the notebook-connector 0.2.9 diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/utils/main_config_ui.ipynb b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/utils/main_config_ui.ipynb index 42695ae7..057219bc 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/utils/main_config_ui.ipynb +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/utils/main_config_ui.ipynb @@ -36,20 +36,22 @@ "source": [ "from typing import List\n", "import os\n", - "from exasol.nb_connector.secret_store import Secrets\n", - "from exasol.nb_connector.ai_lab_config import AILabConfig as CKey\n", + "\n", "import ipywidgets as widgets\n", "\n", - "from exasol.nb_connector.ai_lab_config import AILabConfig as CKey\n", - "from exasol.nb_connector.itde_manager import (bring_itde_up, is_itde_running, start_itde, take_itde_down)\n", + "from exasol.nb_connector.secret_store import Secrets\n", + "from exasol.nb_connector.ai_lab_config import AILabConfig as CKey, StorageBackend\n", + "from exasol.nb_connector.itde_manager import (bring_itde_up, get_itde_status, restart_itde, take_itde_down, ItdeContainerStatus)\n", + "from exasol.nb_connector.connections import get_backend\n", "\n", "\n", "class ITDEStatus(Enum):\n", " \"\"\"\n", " Display status of the Exasol Docker-DB\n", " \"\"\"\n", - " running = 'Exasol Docker-DB is RUNNING'\n", + " ready = 'Exasol Docker-DB is READY'\n", " stopped = 'Exasol Docker-DB is STOPPED'\n", + " disconnected = 'Exasol Docker-DB is NOT CONNECTED'\n", " missing = 'Exasol Docker-DB is NOT CREATED'\n", " inaccessible = 'Exasol Docker-DB is INACCESSIBLE'\n", "\n", @@ -61,8 +63,14 @@ "\n", " ui_look = get_config_styles()\n", "\n", - " db_options = ['Exasol Docker-DB', 'External Exasol Database']\n", - " db_choice = 0 if conf.get(CKey.use_itde, 'True') == 'True' else 1\n", + " db_options = ['Exasol Docker-DB', 'Exasol On-Prem Database', 'Exasol SaaS Database']\n", + " storage_backend = get_backend(conf)\n", + " if storage_backend == StorageBackend.saas:\n", + " db_choice = 2\n", + " elif conf.get(CKey.use_itde, 'True') == 'True':\n", + " db_choice = 0\n", + " else:\n", + " db_choice = 1\n", " db_selector = widgets.RadioButtons(options=db_options, value=db_options[db_choice], \n", " layout=ui_look.input_layout, style=ui_look.input_style)\n", " select_btn = widgets.Button(description='Select', style=ui_look.button_style, layout=ui_look.button_layout)\n", @@ -70,7 +78,11 @@ "\n", "\n", " def select_database(btn):\n", - " conf.save(CKey.use_itde, str(db_selector.value == db_options[0]))\n", + " if db_selector.value == db_options[2]:\n", + " conf.save(CKey.storage_backend, StorageBackend.saas.name)\n", + " else:\n", + " conf.save(CKey.storage_backend, StorageBackend.onprem.name)\n", + " conf.save(CKey.use_itde, str(db_selector.value == db_options[0]))\n", " btn.icon = 'check'\n", "\n", " def on_value_change(change):\n", @@ -85,7 +97,7 @@ " return ui\n", "\n", "\n", - "def get_external_db_config_ui(conf: Secrets) -> widgets.Widget:\n", + "def get_onprem_db_config_ui(conf: Secrets) -> widgets.Widget:\n", " \"\"\"\n", " Creates a UI form for editing an external Exasol Database configuration.\n", " \"\"\"\n", @@ -124,6 +136,32 @@ " return get_generic_config_ui(conf, inputs, group_names)\n", "\n", "\n", + "def get_saas_db_config_ui(conf: Secrets) -> widgets.Widget:\n", + " \"\"\"\n", + " Creates a UI form for editing the Exasol Docker-DB configuration.\n", + " \"\"\"\n", + "\n", + " inputs = [\n", + " [\n", + " ('Service URL', widgets.Text(value=conf.get(CKey.saas_url, 'https://cloud.exasol.com')), CKey.saas_url),\n", + " ('Account ID', widgets.Password(value=conf.get(CKey.saas_account_id)), CKey.saas_account_id),\n", + " ('Database ID', widgets.Text(value=conf.get(CKey.saas_database_id)), CKey.saas_database_id),\n", + " ('Database Name', widgets.Text(value=conf.get(CKey.saas_database_name)), CKey.saas_database_name),\n", + " ('Personal Access Token', widgets.Password(value=conf.get(CKey.saas_token)), CKey.saas_token),\n", + " ('Default Schema', widgets.Text(value=conf.get(CKey.db_schema, 'AI_LAB')), CKey.db_schema)\n", + " ],\n", + " [\n", + " ('Validate Certificate', widgets.Checkbox(value=conf.get(CKey.cert_vld, 'True') == 'True', indent=False),\n", + " CKey.cert_vld),\n", + " ('Trusted CA File/Dir', widgets.Text(value=conf.get(CKey.trusted_ca)), CKey.trusted_ca)\n", + " ]\n", + " ]\n", + "\n", + " group_names = ['SaaS DB Configuration', 'TLS/SSL Configuration']\n", + "\n", + " return get_generic_config_ui(conf, inputs, group_names)\n", + "\n", + "\n", "def get_docker_db_config_ui(conf: Secrets) -> widgets.Widget:\n", " \"\"\"\n", " Creates a UI form for editing the Exasol Docker-DB configuration.\n", @@ -147,18 +185,20 @@ " \"\"\"\n", " Creates a db configuration UI, depending on the choice of the database.\n", " \"\"\"\n", - "\n", - " if conf.get(CKey.use_itde, 'True') == 'True':\n", + " storage_backend = get_backend(conf)\n", + " if storage_backend == StorageBackend.saas:\n", + " return get_saas_db_config_ui(conf)\n", + " elif conf.get(CKey.use_itde, 'True') == 'True':\n", " return get_docker_db_config_ui(conf)\n", " else:\n", - " return get_external_db_config_ui(conf)\n", + " return get_onprem_db_config_ui(conf)\n", "\n", "\n", - "def _get_docker_db_action_buttons(conf: Secrets, itde_exists: bool, itde_running: bool, \n", + "def _get_docker_db_action_buttons(conf: Secrets, itde_exists: bool, itde_ready: bool, \n", " display_status: widgets.Widget) -> List[widgets.Button]:\n", " \"\"\"\n", " Creates one or two action buttons with the correspondent on_click functions for managing the\n", - " Exasol Docker-DB. Depending on the current status (idte_exists, itde_running) of the docker\n", + " Exasol Docker-DB. Depending on the current status (idte_exists, itde_ready) of the docker\n", " container, the \"Start\", \"Restart\" or both buttons are created.\n", " When the action is completed successfully, the running status is displayed in the provided\n", " widget (display_status).\n", @@ -168,14 +208,14 @@ " try:\n", " # Need to check if the Exasol Docker-DB still exists and not running because\n", " # the situation might have changed while the the widgets were hanging around.\n", - " itde_exists_now, itde_running_now = is_itde_running(conf)\n", - " if not itde_running_now:\n", - " if itde_exists_now:\n", - " start_itde(conf)\n", - " else:\n", + " itde_status_now = get_itde_status(conf)\n", + " if itde_status_now != ItdeContainerStatus.READY:\n", + " if itde_status_now == ItdeContainerStatus.ABSENT:\n", " bring_itde_up(conf)\n", + " else:\n", + " restart_itde(conf)\n", " # Indicate the successful completion.\n", - " display_status.value = ITDEStatus.running.value\n", + " display_status.value = ITDEStatus.ready.value\n", " btn.icon = 'check'\n", " except Exception as e:\n", " popup_message('Failed to start the Exasol Docker-DB:' + str(e))\n", @@ -184,17 +224,17 @@ " try:\n", " # Need to check again if the Exasol Docker-DB exists or not because\n", " # the situation might have changed while the widgets were hanging around.\n", - " itde_exists_now, _ = is_itde_running(conf)\n", - " if itde_exists_now:\n", + " itde_status_now = get_itde_status(conf)\n", + " if itde_status_now != ItdeContainerStatus.ABSENT:\n", " take_itde_down(conf)\n", " bring_itde_up(conf)\n", " # Indicate the successful completion.\n", - " display_status.value = ITDEStatus.running.value\n", + " display_status.value = ITDEStatus.ready.value\n", " btn.icon = 'check'\n", " except Exception as e:\n", " popup_message('Failed to restart the Exasol Docker-DB:' + str(e))\n", "\n", - " if itde_running:\n", + " if itde_ready:\n", " btn_restart = widgets.Button(description='Recreate and Start')\n", " btn_restart.on_click(restart_docker_db)\n", " return [btn_restart]\n", @@ -230,40 +270,39 @@ " # Check if the docker-socket has been mounted\n", " socket_mounted = os.path.exists('/var/run/docker.sock')\n", "\n", - " # Get the current status of the Exasol Docker-DB.\n", - " if socket_mounted:\n", - " itde_exists, itde_running = is_itde_running(conf)\n", - " else:\n", - " itde_exists, itde_running = False, False\n", - "\n", - " # Display the status.\n", " header_lbl = widgets.Label(style=ui_look.header_style, layout=ui_look.header_layout)\n", - " if itde_running:\n", - " header_lbl.value = ITDEStatus.running.value\n", - " elif itde_exists:\n", - " header_lbl.value = ITDEStatus.stopped.value\n", - " elif not socket_mounted:\n", - " header_lbl.value = ITDEStatus.inaccessible.value\n", - " else:\n", - " header_lbl.value = ITDEStatus.missing.value\n", " group_items = [header_lbl]\n", " \n", " if socket_mounted:\n", + " # Get and display the current status of the Exasol Docker-DB.\n", + " itde_status = get_itde_status(conf)\n", + " if itde_status == ItdeContainerStatus.READY:\n", + " header_lbl.value = ITDEStatus.ready.value\n", + " elif itde_status == ItdeContainerStatus.RUNNING:\n", + " header_lbl.value = ITDEStatus.disconnected.value\n", + " elif itde_status == ItdeContainerStatus.STOPPED:\n", + " header_lbl.value = ITDEStatus.stopped.value\n", + " else:\n", + " header_lbl.value = ITDEStatus.missing.value\n", + "\n", " # Add a warning message about recreating an existing Exasol Docker-DB.\n", + " itde_exists = itde_status != ItdeContainerStatus.ABSENT\n", + " itde_ready = itde_status == ItdeContainerStatus.READY\n", " if itde_exists:\n", " warning_text = 'Please note that recreating the Exasol Docker-DB will result in the loss of all data stored in the ' \\\n", - " f'{\"running\" if itde_running else \"existing\"} instance of the database!'\n", + " f'{\"running\" if itde_ready else \"existing\"} instance of the database!'\n", " warning_html = _create_warning(warning_text)\n", " group_items.append(widgets.Box([warning_html], layout=ui_look.row_layout))\n", " \n", " # Create action buttons.\n", - " action_buttons = _get_docker_db_action_buttons(conf, itde_exists, itde_running, header_lbl)\n", + " action_buttons = _get_docker_db_action_buttons(conf, itde_exists, itde_ready, header_lbl)\n", " for btn in action_buttons:\n", " btn.style = ui_look.button_style\n", " btn.layout = ui_look.button_layout\n", " else:\n", " from utils.useful_urls import UsefulURLs\n", "\n", + " header_lbl.value = ITDEStatus.inaccessible.value\n", " warning_text = f'The docker socket is not mounted. Please consult the ' \\\n", " f'documentation ' \\\n", " 'on how to start the AI-Lab that will use the Integrated Exasol Docker-DB.'\n", diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt index 211f857d..a0531383 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt @@ -7,4 +7,4 @@ jupysql==0.10.10 sqlalchemy_exasol==4.6.3 stopwatch.py==2.0.1 boto3==1.26.163 -exasol-notebook-connector==0.2.8 +exasol-notebook-connector @ git+https://github.com/exasol/notebook-connector.git@main From f9dfe0b4ea774b91a56100eee0a800b8c311559f Mon Sep 17 00:00:00 2001 From: mibe Date: Fri, 31 May 2024 09:59:21 +0100 Subject: [PATCH 2/3] #277 updated git dependency [run-notebook-tests] --- .../ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml index bca77c7d..f57b8782 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml @@ -14,4 +14,4 @@ jupyterlab_notebook_folder: "{{ user_home }}/notebooks" apt_dependencies: - virtualenv=20.0.17-1ubuntu0.4 - - git=1:2.25.1-1ubuntu3.11 + - git=1:2.25.1-1ubuntu3.12 From 0a28149cbba92eca81183d27e1e05c7b3df67ad8 Mon Sep 17 00:00:00 2001 From: mibe Date: Mon, 3 Jun 2024 08:29:55 +0100 Subject: [PATCH 3/3] #277 Addressed the review issues [run-notebook-tests] --- doc/changes/changes_2.1.0.md | 3 +-- .../roles/jupyter/files/notebook/utils/main_config_ui.ipynb | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/changes/changes_2.1.0.md b/doc/changes/changes_2.1.0.md index 1b6e186b..eaf58400 100644 --- a/doc/changes/changes_2.1.0.md +++ b/doc/changes/changes_2.1.0.md @@ -12,8 +12,7 @@ Version: 2.1.0 ## Features -* 277 Added configuration for a SaaS Database. - Added SaaS as the third option in the database selection UI. +* 277 Added the SaaS database parameters to the configuration page. ## Security diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/utils/main_config_ui.ipynb b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/utils/main_config_ui.ipynb index 057219bc..ab4ef1b4 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/utils/main_config_ui.ipynb +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/utils/main_config_ui.ipynb @@ -138,7 +138,7 @@ "\n", "def get_saas_db_config_ui(conf: Secrets) -> widgets.Widget:\n", " \"\"\"\n", - " Creates a UI form for editing the Exasol Docker-DB configuration.\n", + " Creates a UI form for editing the Exasol SaaS database configuration.\n", " \"\"\"\n", "\n", " inputs = [\n",