#54 pytest-extension

.github/workflows/cd.yml

@@ -9,6 +9,7 @@ on:
         default: 'pytest-saas'
         type: choice
         options:
+        - "pytest-extension"
         - "pytest-slc"
         - "pytest-backend"
         - "pytest-saas"

README.md

@@ -14,12 +14,13 @@ Whether you're looking to use database interactions, enhance test reporting, or
 
 ## Plugins
 
-| Plugin                  | Description                                                                                                                | PYPI                                                                     |
-|-------------------------|----------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------|
-| `pytest-exasol-itde`    | Fixture to enable simple usage with Exasol's project [ITDE](https://github.com/exasol/integration-test-docker-environment) | [pytest-exasol-itde](https://pypi.org/project/pytest-exasol-itde/)       |
-| `pytest-exasol-saas`    | Fixture to enable simple usage with Exasol's project [saas-api-python](https://github.com/exasol/saas-api-python/)         | [pytest-exasol-saas](https://pypi.org/project/pytest-exasol-saas/)       |
-| `pytest-exasol-backend` | Fixture aggregating functionality of both of the above plugins                                                             | [pytest-exasol-backend](https://pypi.org/project/pytest-exasol-backend/) |
-| `pytest-exasol-slc`     | Fixture for uploading a script language container                                                                          | [pytest-exasol-slc](https://pypi.org/project/pytest-exasol-slc/)         |
+| Plugin                    | Description                                                                                                                | PYPI                                                                         |
+|---------------------------|----------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------|
+| `pytest-exasol-itde`      | Fixture to enable simple usage with Exasol's project [ITDE](https://github.com/exasol/integration-test-docker-environment) | [pytest-exasol-itde](https://pypi.org/project/pytest-exasol-itde/)           |
+| `pytest-exasol-saas`      | Fixture to enable simple usage with Exasol's project [saas-api-python](https://github.com/exasol/saas-api-python/)         | [pytest-exasol-saas](https://pypi.org/project/pytest-exasol-saas/)           |
+| `pytest-exasol-backend`   | Fixture aggregating functionality of both of the above plugins                                                             | [pytest-exasol-backend](https://pypi.org/project/pytest-exasol-backend/)     |
+| `pytest-exasol-slc`       | Fixture for uploading a script language container                                                                          | [pytest-exasol-slc](https://pypi.org/project/pytest-exasol-slc/)             |
+| `pytest-exasol-extension` | Fixture for setting up a database for an extension test                                                                    | [pytest-exasol-extension](https://pypi.org/project/pytest-exasol-extension/) |
 
 
 ## Installation

justfile

@@ -1,4 +1,4 @@
-PROJECTS := "pytest-slc pytest-backend pytest-saas pytest-itde"
+PROJECTS := "pytest-extension pytest-slc pytest-backend pytest-saas pytest-itde"
 
 # Default target
 default:

pytest-extension/.gitignore

@@ -0,0 +1 @@
+.html-documentation

pytest-extension/README.md

@@ -0,0 +1,39 @@
+# pytest-exasol-extension Plugin
+
+The `pytest-exasol-extension` plugin provides pytest fixtures for preparing a database for the extension tests.
+The fixtures are backend agnostic. They run for the selected backends
+(see the documentation for the `pytest-exasol-backend` plugin).
+
+## Installation
+
+The pytest-exasol-extension plugin can be installed using pip:
+
+```shell
+pip install pytest-exasol-extension
+```
+
+## Usage in Tests
+
+Below is an example of a test that requires a database connection with an open test schema.
+
+```python
+import pytest
+
+@pytest.fixture(scope="session")
+def db_schema_name() -> str:
+    """Let's override a randomly generated db schema for the test, giving it a meaningful name."""
+    return 'MY_TEST_SCHEMA'
+
+def test_something(pyexasol_connection):
+    ...
+```
+
+Next, is an example of a test that needs to store a bucket-fs connection object in the database.
+
+```python
+def test_something_else(bucketfs_connection_factory):
+    bucketfs_connection_factory('my_connection_name,' 'some_path/in_the_bucket')
+    ...
+```
+
+Note, that by default the tests will run twice - once for each backend.

pytest-extension/doc/changes/changelog.md

@@ -0,0 +1,14 @@
+# Changes
+
+* [unreleased](unreleased.md)
+* [0.1.0](changes_0.1.0.md)
+
+<!--- This MyST Parser Sphinx directive is necessary to keep Sphinx happy. We need list here all release letters again, because release droid and other scripts assume Markdown --->
+```{toctree}
+---
+hidden:
+---
+unreleased
+changes_0.1.0
+
+```

pytest-extension/doc/changes/changes_0.1.0.md

@@ -0,0 +1,9 @@
+# 0.1.0 - 2024-09-06
+
+## Summary
+
+🚀 Initial Release of the pytest-exasol-extension Plugin
+
+## Feature
+
+* #54: Added the pytest-extension project

pytest-extension/doc/changes/unreleased.md

@@ -0,0 +1 @@
+# Unreleased

pytest-extension/error_code_config.yml

@@ -0,0 +1,3 @@
+error-tags:
+  PYTEXT:
+    highest-index: 0

pytest-extension/exasol/pytest_extension/__init__.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+from typing import Any, Callable
+import json
+import random
+import string
+import pyexasol
+import pytest
+
+from exasol.pytest_backend import BACKEND_ONPREM, BACKEND_SAAS
+
+
+def _to_json_str(bucketfs_params: dict[str, Any], selected: list[str]) -> str:
+    filtered_kwargs = {k: v for k, v in bucketfs_params.items()
+                       if (k in selected) and (v is not None)}
+    return json.dumps(filtered_kwargs)
+
+
+def _create_bucketfs_connection(pyexasol_connection: pyexasol.ExaConnection,
+                                conn_name: str,
+                                conn_to: str,
+                                conn_user: str,
+                                conn_password: str) -> None:
+
+    query = (f"CREATE OR REPLACE  CONNECTION {conn_name} "
+             f"TO '{conn_to}' "
+             f"USER '{conn_user}' "
+             f"IDENTIFIED BY '{conn_password}'")
+    pyexasol_connection.execute(query)
+
+
+def _create_bucketfs_connection_onprem(pyexasol_connection: pyexasol.ExaConnection,
+                                       conn_name: str,
+                                       bucketfs_params: dict[str, Any]) -> None:
+    conn_to = _to_json_str(bucketfs_params, [
+        'backend', 'url', 'service_name', 'bucket_name', 'path', 'verify'])
+    conn_user = _to_json_str(bucketfs_params, ['username'])
+    conn_password = _to_json_str(bucketfs_params, ['password'])
+
+    _create_bucketfs_connection(pyexasol_connection, conn_name,
+                                conn_to, conn_user, conn_password)
+
+
+def _create_bucketfs_connection_saas(pyexasol_connection: pyexasol.ExaConnection,
+                                     conn_name: str,
+                                     bucketfs_params: dict[str, Any]) -> None:
+    conn_to = _to_json_str(bucketfs_params, ['backend', 'url', 'path'])
+    conn_user = _to_json_str(bucketfs_params, ['account_id', 'database_id'])
+    conn_password = _to_json_str(bucketfs_params, ['pat'])
+
+    _create_bucketfs_connection(pyexasol_connection, conn_name,
+                                conn_to, conn_user, conn_password)
+
+
+@pytest.fixture(scope="session")
+def db_schema_name() -> str:
+    """
+    The fixture gives a test schema name.
+    The user can override this fixture and provide a meaningful name, which can be
+    useful when looking at the test results. Otherwise, the schema name will be a
+    randomly generated string.
+    """
+    return ''.join(random.choice(string.ascii_uppercase) for _ in range(12))
+
+
+@pytest.fixture(scope="session")
+def pyexasol_connection(backend_aware_database_params,
+                        db_schema_name
+                        ) -> pyexasol.ExaConnection:
+    """
+    The fixture provides a database connection. It opens the test schema,
+    creating it if it doesn't exist. In the latter case the schema gets
+    deleted and the end of the fixture's life span.
+    """
+    with pyexasol.connect(**backend_aware_database_params, compression=True) as conn:
+        sql = f"SELECT * FROM SYS.EXA_SCHEMAS WHERE SCHEMA_NAME = '{db_schema_name}'"
+        use_temp_schema = len(conn.execute(sql).fetchall()) == 0
+        if use_temp_schema:
+            conn.execute(f'CREATE SCHEMA "{db_schema_name}"')
+        conn.execute(f'OPEN SCHEMA "{db_schema_name}"')
+        try:
+            yield conn
+        finally:
+            if use_temp_schema:
+                conn.execute(f'DROP SCHEMA "{db_schema_name}" CASCADE')
+
+
+@pytest.fixture(scope='session')
+def bucketfs_connection_factory(backend,
+                                pyexasol_connection,
+                                backend_aware_bucketfs_params
+                                ) -> Callable[[str, str | None], None]:
+    """
+    This is a factory fixture that creates a bucket-fs connection object in a database.
+    It takes the following parameters:
+
+    - conn_name:        The name of the connection object.
+    - path_in_bucket:   Optional path in the bucket. Bucket root if not specified.
+
+    It will override any existing object with the same name.
+    """
+    def func(conn_name: str, path_in_bucket: str | None = None) -> None:
+        if path_in_bucket:
+            bucketfs_params = dict(backend_aware_bucketfs_params)
+            bucketfs_params['path'] = path_in_bucket
+        else:
+            bucketfs_params = backend_aware_bucketfs_params
+        if backend == BACKEND_ONPREM:
+            _create_bucketfs_connection_onprem(pyexasol_connection, conn_name, bucketfs_params)
+        elif backend == BACKEND_SAAS:
+            _create_bucketfs_connection_saas(pyexasol_connection, conn_name, bucketfs_params)
+        else:
+            raise ValueError(f'Unsupported backend {backend}')
+
+    return func

pytest-extension/exasol/pytest_extension/version.py

@@ -0,0 +1,10 @@
+# ATTENTION:
+# This file is generated by exasol/toolbox/pre_commit_hooks/package_version.py when using:
+#   * either "poetry run nox -s fix"
+#   * or "poetry run version-check <path/version.py> --fix"
+# Do not edit this file manually!
+# If you need to change the version, do so in the project.toml, e.g. by using `poetry version X.Y.Z`.
+MAJOR = 0
+MINOR = 1
+PATCH = 0
+VERSION = f"{MAJOR}.{MINOR}.{PATCH}"

pytest-extension/noxconfig.py

@@ -0,0 +1,39 @@
+"""Configuration for nox based task runner"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import (
+    Any,
+    Iterable,
+    MutableMapping,
+)
+
+from nox import Session
+
+
+@dataclass(frozen=True)
+class Config:
+    """Project specific configuration used by nox infrastructure"""
+
+    root: Path = Path(__file__).parent
+    doc: Path = Path(__file__).parent / "doc"
+    version_file: Path = Path(__file__).parent / "exasol" / "pytest_extension" / "version.py"
+    path_filters: Iterable[str] = ("dist", ".eggs", "venv", "metrics-schema")
+
+    @staticmethod
+    def pre_integration_tests_hook(
+        _session: Session, _config: Config, _context: MutableMapping[str, Any]
+    ) -> bool:
+        """Implement if project specific behaviour is required"""
+        return True
+
+    @staticmethod
+    def post_integration_tests_hook(
+        _session: Session, _config: Config, _context: MutableMapping[str, Any]
+    ) -> bool:
+        """Implement if project specific behaviour is required"""
+        return True
+
+
+PROJECT_CONFIG = Config()

pytest-extension/noxfile.py

@@ -0,0 +1,11 @@
+"""defines nox tasks/targets for this project"""
+import sys
+
+import nox
+
+print(sys.path)
+# imports all nox task provided by the toolbox
+from exasol.toolbox.nox.tasks import *  # pylint: disable=wildcard-import disable=unused-wildcard-import
+
+# default actions to be run if nothing is explicitly specified with the -s option
+nox.options.sessions = ["fix"]