Nox issue investigation

.github/workflows/ci.yml

@@ -39,4 +39,4 @@ jobs:
           SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }}
           SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }}
           SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }}
-          PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.pytest-markers.outputs.slow-tests }}'
+          PYTEST_ADDOPTS: '${{ steps.pytest-markers.outputs.slow-tests }}'

justfile

@@ -1,4 +1,5 @@
-PROJECTS := "pytest-slc pytest-extension pytest-backend pytest-saas pytest-itde"
+# PROJECTS := "pytest-slc pytest-extension pytest-backend pytest-saas pytest-itde"
+PROJECTS := "pytest-extension"
 
 # Default target
 default:
@@ -16,7 +17,8 @@ test +projects=PROJECTS:
 
     for p in "{{projects}}".split():
         run(f"poetry -C {p}/ install")
-        run(f"poetry -C {p}/ run nox -f {p}/noxfile.py -s coverage")
+        run(f"poetry -C {p}/ run nox -f {p}/noxfile.py -s my-integration-tests")
+        # run(f"poetry -C {p}/ run pytest -s {p}/test")
     sys.exit(rc)
 
 relock +projects=PROJECTS:

pytest-backend/pyproject.toml

@@ -16,8 +16,8 @@ exasol-saas-api = ">=0.6.0,<1.0.0"
 backend = "exasol.pytest_backend"
 
 [tool.poetry.group.dev.dependencies]
-exasol-toolbox = "0.9.0"
-exasol-bucketfs = ">=0.12.0"
+exasol-toolbox = ">=0.15.0,<1"
+exasol-bucketfs = ">=0.13.0,<1"
 pyexasol = ">=0.26.0"
 
 [build-system]

pytest-extension/README.md

@@ -36,4 +36,55 @@ def test_something_else(bucketfs_connection_factory):
     ...
 ```
 
+The following fixtures are used to test various deployment scenarios where the connection parameters
+for the Database and the BucketFS are supplied in a command line. The first two fixtures provide dictionaries
+of standard cli parameters (`StdParams`) defined in the `exasol-python-extension-common`.
+
+    `database_std_params` - the Database connection parameters.
+    `bucketfs_std_params` - the BucketFs connection parameters.
+
+The next two fixtures - `database_cli_args` and `bucketfs_cli_args` - give the same parameters as the previous two
+but in the form of command line arguments. They are helpful for testing the CLI directly, for example using the
+click.CliRunner as in the samples below. There is also a fixture - `cli_args` - that combines these two argument
+strings.
+
+```python
+import click
+from click.testing import CliRunner
+from exasol.python_extension_common.cli.std_options import (StdParams, StdTags, select_std_options)
+from exasol.pytest_backend import (BACKEND_ONPREM, BACKEND_SAAS)
+
+def test_db_connection_cli(backend, database_cli_args):
+    if backend == BACKEND_ONPREM:
+        tags = StdTags.DB | StdTags.ONPREM
+    elif backend == BACKEND_SAAS:
+        tags = StdTags.DB | StdTags.SAAS
+    else:
+        ValueError(f'Unknown backend {backend}')
+
+    def test_something_with_db(**kwargs):
+        pass
+
+    opts = select_std_options(tags)
+    cmd = click.Command('whatever', params=opts, callback=test_something_with_db)
+    runner = CliRunner()
+    runner.invoke(cmd, args=database_cli_args, catch_exceptions=False, standalone_mode=False)
+
+def test_bucketfs_connection_cli(backend, bucketfs_cli_args):
+    if backend == BACKEND_ONPREM:
+        tags = StdTags.BFS | StdTags.ONPREM
+    elif backend == BACKEND_SAAS:
+        tags = StdTags.BFS | StdTags.SAAS
+    else:
+        ValueError(f'Unknown backend {backend}')
+
+    def test_something_with_bucketfs(**kwargs):
+        pass
+
+    opts = select_std_options(tags)
+    cmd = click.Command('whatever', params=opts, callback=test_something_with_bucketfs)
+    runner = CliRunner()
+    runner.invoke(cmd, args=bucketfs_cli_args, catch_exceptions=False, standalone_mode=False)
+```
+
 Note, that by default the tests will run twice - once for each backend.

pytest-extension/doc/changes/unreleased.md

@@ -3,3 +3,7 @@
 ## Internal
 
 * Relock dependencies
+
+## Features
+
+* #61 Added CLI parameters.

pytest-extension/exasol/pytest_extension/__init__.py

@@ -1,54 +1,15 @@
 from __future__ import annotations
 from typing import Any, Callable
-import json
 import random
 import string
+from urllib.parse import urlparse
 import pyexasol
 import pytest
 
 from exasol.pytest_backend import BACKEND_ONPREM, BACKEND_SAAS
-
-
-def _to_json_str(bucketfs_params: dict[str, Any], selected: list[str]) -> str:
-    filtered_kwargs = {k: v for k, v in bucketfs_params.items()
-                       if (k in selected) and (v is not None)}
-    return json.dumps(filtered_kwargs)
-
-
-def _create_bucketfs_connection(pyexasol_connection: pyexasol.ExaConnection,
-                                conn_name: str,
-                                conn_to: str,
-                                conn_user: str,
-                                conn_password: str) -> None:
-
-    query = (f"CREATE OR REPLACE  CONNECTION {conn_name} "
-             f"TO '{conn_to}' "
-             f"USER '{conn_user}' "
-             f"IDENTIFIED BY '{conn_password}'")
-    pyexasol_connection.execute(query)
-
-
-def _create_bucketfs_connection_onprem(pyexasol_connection: pyexasol.ExaConnection,
-                                       conn_name: str,
-                                       bucketfs_params: dict[str, Any]) -> None:
-    conn_to = _to_json_str(bucketfs_params, [
-        'backend', 'url', 'service_name', 'bucket_name', 'path', 'verify'])
-    conn_user = _to_json_str(bucketfs_params, ['username'])
-    conn_password = _to_json_str(bucketfs_params, ['password'])
-
-    _create_bucketfs_connection(pyexasol_connection, conn_name,
-                                conn_to, conn_user, conn_password)
-
-
-def _create_bucketfs_connection_saas(pyexasol_connection: pyexasol.ExaConnection,
-                                     conn_name: str,
-                                     bucketfs_params: dict[str, Any]) -> None:
-    conn_to = _to_json_str(bucketfs_params, ['backend', 'url', 'path'])
-    conn_user = _to_json_str(bucketfs_params, ['account_id', 'database_id'])
-    conn_password = _to_json_str(bucketfs_params, ['pat'])
-
-    _create_bucketfs_connection(pyexasol_connection, conn_name,
-                                conn_to, conn_user, conn_password)
+from exasol.python_extension_common.cli.std_options import StdParams
+from exasol.python_extension_common.connections.bucketfs_location import (
+    create_bucketfs_conn_object_onprem, create_bucketfs_conn_object_saas)
 
 
 @pytest.fixture(scope="session")
@@ -105,10 +66,129 @@ def func(conn_name: str, path_in_bucket: str | None = None) -> None:
         else:
             bucketfs_params = backend_aware_bucketfs_params
         if backend == BACKEND_ONPREM:
-            _create_bucketfs_connection_onprem(pyexasol_connection, conn_name, bucketfs_params)
+            create_bucketfs_conn_object_onprem(pyexasol_connection, conn_name, bucketfs_params)
         elif backend == BACKEND_SAAS:
-            _create_bucketfs_connection_saas(pyexasol_connection, conn_name, bucketfs_params)
+            create_bucketfs_conn_object_saas(pyexasol_connection, conn_name, bucketfs_params)
         else:
             raise ValueError(f'Unsupported backend {backend}')
 
     return func
+
+
+@pytest.fixture(scope="session")
+def onprem_database_std_params(use_onprem,
+                               backend_aware_onprem_database,
+                               exasol_config) -> dict[str, Any]:
+    if use_onprem:
+        return {
+            StdParams.dsn.name: f'{exasol_config.host}:{exasol_config.port}',
+            StdParams.db_user.name: exasol_config.username,
+            StdParams.db_password.name: exasol_config.password,
+            StdParams.use_ssl_cert_validation.name: False
+        }
+    return {}
+
+
+@pytest.fixture(scope="session")
+def onprem_bucketfs_std_params(use_onprem,
+                               backend_aware_onprem_database,
+                               bucketfs_config) -> dict[str, Any]:
+    if use_onprem:
+        parsed_url = urlparse(bucketfs_config.url)
+        host, port = parsed_url.netloc.split(":")
+        return {
+            StdParams.bucketfs_host.name: host,
+            StdParams.bucketfs_port.name: port,
+            StdParams.bucketfs_use_https.name: parsed_url.scheme.lower() == 'https',
+            StdParams.bucketfs_user.name: bucketfs_config.username,
+            StdParams.bucketfs_password.name: bucketfs_config.password,
+            StdParams.bucketfs_name.name: 'bfsdefault',
+            StdParams.bucket.name: 'default',
+            StdParams.use_ssl_cert_validation.name: False
+        }
+    return {}
+
+
+@pytest.fixture(scope="session")
+def saas_std_params(use_saas,
+                    saas_host,
+                    saas_pat,
+                    saas_account_id,
+                    backend_aware_saas_database_id) -> dict[str, Any]:
+    if use_saas:
+        return {
+            StdParams.saas_url.name: saas_host,
+            StdParams.saas_account_id.name: saas_account_id,
+            StdParams.saas_database_id.name: backend_aware_saas_database_id,
+            StdParams.saas_token.name: saas_pat
+        }
+    return {}
+
+
+@pytest.fixture(scope="session")
+def database_std_params(backend,
+                        onprem_database_std_params,
+                        saas_std_params) -> dict[str, Any]:
+    """
+    This is a collection of StdParams parameters required to open a
+    database connection for either DockerDB or SaaS test database.
+    """
+    if backend == BACKEND_ONPREM:
+        return onprem_database_std_params
+    elif backend == BACKEND_SAAS:
+        return saas_std_params
+    else:
+        ValueError(f'Unknown backend {backend}')
+
+
+@pytest.fixture(scope="session")
+def bucketfs_std_params(backend,
+                        onprem_bucketfs_std_params,
+                        saas_std_params) -> dict[str, Any]:
+    """
+    This is a collection of StdParams parameters required to connect
+    to the BucketFS on either DockerDB or SaaS test database.
+    """
+    if backend == BACKEND_ONPREM:
+        return onprem_bucketfs_std_params
+    elif backend == BACKEND_SAAS:
+        return saas_std_params
+    else:
+        ValueError(f'Unknown backend {backend}')
+
+
+def _cli_params_to_args(cli_params) -> str:
+    def arg_string(k: str, v: Any):
+        k = k.replace("_", "-")
+        if isinstance(v, bool):
+            return f'--{k}' if v else f'--no-{k}'
+        return f'--{k} "{v}"'
+
+    return ' '.join(arg_string(k, v) for k, v in cli_params.items())
+
+
+@pytest.fixture(scope='session')
+def database_cli_args(database_std_params) -> str:
+    """
+    CLI argument string for testing a command that involves connecting to the database.
+    """
+    return _cli_params_to_args(database_std_params)
+
+
+@pytest.fixture(scope='session')
+def bucketfs_cli_args(bucketfs_std_params) -> str:
+    """
+    CLI argument string for testing a command that involves connecting to the BucketFS .
+    """
+    return _cli_params_to_args(bucketfs_std_params)
+
+
+@pytest.fixture(scope='session')
+def cli_args(database_std_params, bucketfs_std_params):
+    """
+    CLI argument string for testing a command that involves connecting to both
+    the database and the BucketFS.
+    """
+    std_params = dict(database_std_params)
+    std_params.update(bucketfs_std_params)
+    return _cli_params_to_args(database_std_params)

pytest-extension/noxfile.py

@@ -2,10 +2,21 @@
 import sys
 
 import nox
+from nox import Session
+from noxconfig import PROJECT_CONFIG
 
 print(sys.path)
 # imports all nox task provided by the toolbox
 from exasol.toolbox.nox.tasks import *  # pylint: disable=wildcard-import disable=unused-wildcard-import
 
 # default actions to be run if nothing is explicitly specified with the -s option
 nox.options.sessions = ["fix"]
+
+
+@nox.session(name="my-integration-tests", python=False)
+def my_integration_tests(session: Session) -> None:
+    path = PROJECT_CONFIG.root / "test" / "integration" / "tmp_test.py"
+    base_command = ["poetry", "run"]
+    pytest_command = ["pytest", "-v", "--backend=onprem", f"{path}"]
+    command = base_command + pytest_command
+    session.run(*command)