diff --git a/pyproject.toml b/pyproject.toml index 66b44ff52..c1334ddef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,6 @@ name = "exasol-integration-test-docker-environment" packages = [ { include = "exasol_integration_test_docker_environment" }, - { include = "pytest_itde" } ] version = "2.1.0" description = "Integration Test Docker Environment for Exasol" @@ -42,15 +41,10 @@ importlib_resources = ">=5.4.0" #Needed to copy resource files, can be removed a "stopwatch.py" = ">=1.0.0" exasol-bucketfs = ">=0.6.0,<2.0.0" pytest = "^7.2.2" -pyexasol = "^0.25.2" fabric = "^3.0.1" portalocker = "^2.7.0" exasol-error-reporting = "^0.4.0" -[tool.poetry.plugins.pytest11] -itde = "pytest_itde" - - [tool.poetry.group.dev.dependencies] toml = ">=0.10.2" nox = "^2022.1.7" diff --git a/pytest_itde/__init__.py b/pytest_itde/__init__.py deleted file mode 100644 index 50a9c7ddb..000000000 --- a/pytest_itde/__init__.py +++ /dev/null @@ -1,250 +0,0 @@ -import os -from typing import Tuple - -import pyexasol -import pytest - -from pytest_itde import config -from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports - -EXASOL = config.OptionGroup( - prefix="exasol", - options=( - { - "name": "host", - "type": str, - "default": "localhost", - "help_text": "Host to connect to", - }, - { - "name": "port", - "type": int, - "default": Ports.forward.database, - "help_text": "Port on which the exasol db is listening", - }, - { - "name": "username", - "type": str, - "default": "SYS", - "help_text": "Username used to authenticate against the exasol db", - }, - { - "name": "password", - "type": str, - "default": "exasol", - "help_text": "Password used to authenticate against the exasol db", - }, - ), -) - -BUCKETFS = config.OptionGroup( - prefix="bucketfs", - options=( - { - "name": "url", - "type": str, - "default": f"http://127.0.0.1:{Ports.forward.bucketfs}", - "help_text": "Base url used to connect to the bucketfs service", - }, - { - "name": "username", - "type": str, - "default": "w", - "help_text": "Username used to authenticate against the bucketfs service", - }, - { - "name": "password", - "type": str, - "default": "write", - "help_text": "Password used to authenticate against the bucketfs service", - }, - ), -) - -SSH = config.OptionGroup( - prefix="ssh", - options=( - { - "name": "port", - "type": int, - "default": Ports.forward.ssh, - "help_text": "Port on which external processes can access the database via SSH protocol", - }, - ), -) - -def TestSchemas(value) -> Tuple[str]: - """ - Parses a test schema spec string. - - Args: - value: spec string for the test schemas. - e.g. "Schema1, Schema2, " or "Schema" - """ - - seperator = "," - if seperator in value: - schemas = value.split(seperator) - else: - schemas = [value] - schemas = (s.strip() for s in schemas) - schemas = (s for s in schemas if s != "") - return tuple(schemas) - - -ITDE = config.OptionGroup( - prefix="itde", - options=( - { - "name": "db_version", - "type": str, - "default": "8.18.1", - "help_text": "DB version to start, if value is 'external' an existing instance will be used", - }, - { - "name": "schemas", - "type": TestSchemas, - "default": ("TEST", "TEST_SCHEMA"), - "help_text": "Schemas which should be created for the session", - }, - ), -) - - -@pytest.fixture(scope="session") -def exasol_config(request) -> config.Exasol: - """Returns the configuration settings of the exasol db for this session.""" - cli_arguments = request.config.option - kwargs = EXASOL.kwargs(os.environ, cli_arguments) - return config.Exasol(**kwargs) - - -@pytest.fixture(scope="session") -def bucketfs_config(request) -> config.BucketFs: - """Returns the configuration settings of the bucketfs for this session.""" - cli_arguments = request.config.option - kwargs = BUCKETFS.kwargs(os.environ, cli_arguments) - return config.BucketFs(**kwargs) - - -@pytest.fixture(scope="session") -def ssh_config(request) -> config.Ssh: - """Returns the configuration settings for SSH access in this session.""" - cli_arguments = request.config.option - kwargs = SSH.kwargs(os.environ, cli_arguments) - return config.Ssh(**kwargs) - - -@pytest.fixture(scope="session") -def itde_config(request) -> config.Itde: - """Returns the configuration settings of the ITDE for this session.""" - cli_arguments = request.config.option - kwargs = ITDE.kwargs(os.environ, cli_arguments) - return config.Itde(**kwargs) - - -@pytest.fixture(scope="session") -def connection_factory(): - """ - Returns a database connection factory. - - Attention: - All created connections will be cleaned up (closed) at the end of the session. - """ - connections = [] - - def factory(config: config.Exasol): - con = pyexasol.connect( - dsn=f"{config.host}:{config.port}", - user=config.username, - password=config.password, - ) - connections.append(con) - return con - - yield factory - - for connection in connections: - connection.close() - - -@pytest.fixture(scope="session") -def _bootstrap_db(itde_config, exasol_config, bucketfs_config, ssh_config): - """Bootstraps the database should not be used from outside the itde plugin.""" - - def nop(): - pass - - def start_db(name, itde, exasol, bucketfs, ssh_access): - from urllib.parse import urlparse - - from exasol_integration_test_docker_environment.lib import api - - bucketfs_url = urlparse(bucketfs.url) - _, cleanup_function = api.spawn_test_environment( - environment_name=name, - database_port_forward=exasol.port, - bucketfs_port_forward=bucketfs_url.port, - ssh_port_forward=ssh_access.port, - db_mem_size="4GB", - docker_db_image_version=itde.db_version, - ) - return cleanup_function - - db_name = "pytest_exasol_db" - bootstrap_db = itde_config.db_version != "external" - - start = ( - lambda: start_db(db_name, itde_config, exasol_config, bucketfs_config, ssh_config) - if bootstrap_db - else lambda: nop - ) - stop = start() - yield - stop() - - -@pytest.fixture(scope="session") -def itde( - _bootstrap_db, - itde_config, - exasol_config, - bucketfs_config, - connection_factory, -) -> config.TestConfig: - """Starts a docker based test environment and returns the associated test config.""" - connection = connection_factory(exasol_config) - - for schema in itde_config.schemas: - connection.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE;") - connection.execute(f"CREATE SCHEMA {schema};") - connection.commit() - - yield config.TestConfig( - db=exasol_config, - bucketfs=bucketfs_config, - itde=itde_config, - ctrl_connection=connection, - ) - - for schema in itde_config.schemas: - connection.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE;") - connection.commit() - - -OPTION_GROUPS = (EXASOL, BUCKETFS, ITDE, SSH) - - -def _add_option_group(parser, group): - parser_group = parser.getgroup(group.prefix) - for option in group.options: - parser_group.addoption( - option.cli, - type=option.type, - help=option.help, - ) - - -def pytest_addoption(parser): - for group in OPTION_GROUPS: - _add_option_group(parser, group) diff --git a/pytest_itde/config.py b/pytest_itde/config.py deleted file mode 100644 index faa00e402..000000000 --- a/pytest_itde/config.py +++ /dev/null @@ -1,146 +0,0 @@ -from collections import ChainMap -from dataclasses import dataclass -from typing import Generic, List, Optional, TypeVar - -from pyexasol.connection import ExaConnection - -T = TypeVar("T") - - -@dataclass(frozen=True) -class Option(Generic[T]): - name: str - prefix: str - type: T - default: Optional[T] = None - help_text: str = "" - - @property - def env(self): - """Environment variable name""" - - def normalize(name): - name = name.replace("-", "_") - name = name.upper() - return name - - return f"{normalize(self.prefix)}_{normalize(self.name)}" - - @property - def cli(self): - """Cli argument name""" - - def normalize(name): - name = name.replace("_", "-") - name = name.lower() - return name - - return f"--{normalize(self.prefix)}-{normalize(self.name)}" - - @property - def pytest(self): - """Pytest option name""" - - def normalize(name): - name = name.replace("-", "_") - name = name.lower() - return name - - return f"{normalize(self.prefix)}_{normalize(self.name)}" - - @property - def help(self): - """Help text including information about default value.""" - if not self.default: - return f"{self.help_text}." - return f"{self.help_text} (default: {self.default})." - - -class OptionGroup: - """ - Wraps a set of pytest options. - """ - - def __init__(self, prefix, options): - self._prefix = prefix - self._options = tuple(Option(prefix=prefix, **kwargs) for kwargs in options) - self._default = {o.name: o.default for o in self._options} - self._env = {} - self._cli = {} - self._kwargs = ChainMap(self._cli, self._env, self._default) - - @property - def prefix(self): - """The option group prefix.""" - return self._prefix - - @property - def options(self): - """A tuple of all options which are part of this group.""" - return self._options - - def kwargs(self, environment, cli_arguments): - """ - Given the default values, the passed environment and cli arguments it will - take care of the prioritization for the option values in regard of their - source(s) and return a kwargs dictionary with all options and their - appropriate value. - """ - env = { - o.name: o.type(environment[o.env]) - for o in self._options - if o.env in environment - } - cli = { - o.name: getattr(cli_arguments, o.pytest) - for o in self.options - if hasattr(cli_arguments, o.pytest) - and getattr(cli_arguments, o.pytest) is not None - } - self._env.update(env) - self._cli.update(cli) - return self._kwargs - - -@dataclass -class Exasol: - """Exasol database configuration""" - - host: str - port: int - username: str - password: str - - -@dataclass -class BucketFs: - """Bucketfs configuration""" - - url: str - username: str - password: str - - -@dataclass -class Ssh: - """SSH configuration""" - - port: int - - -@dataclass -class Itde: - """Itde configuration settings""" - - db_version: str - schemas: List[str] - - -@dataclass -class TestConfig: - """Full test configuration for itde based tests""" - - db: Exasol - bucketfs: BucketFs - itde: Itde - ctrl_connection: ExaConnection diff --git a/starter_scripts/Dockerfile b/starter_scripts/Dockerfile index 4573266b2..7ddafff22 100644 --- a/starter_scripts/Dockerfile +++ b/starter_scripts/Dockerfile @@ -22,7 +22,6 @@ ENV LANG=C.UTF-8 RUN mkdir /exasol_integration_test_docker_environment COPY exasol_integration_test_docker_environment /integration_test_docker_environment/exasol_integration_test_docker_environment -COPY pytest_itde /integration_test_docker_environment/pytest_itde COPY poetry.lock /integration_test_docker_environment/poetry.lock COPY pyproject.toml /integration_test_docker_environment/pyproject.toml COPY LICENSE /integration_test_docker_environment/LICENSE diff --git a/starter_scripts/checksums/Dockerfile.sha512sum b/starter_scripts/checksums/Dockerfile.sha512sum index 8904cbff9..5924f9657 100644 --- a/starter_scripts/checksums/Dockerfile.sha512sum +++ b/starter_scripts/checksums/Dockerfile.sha512sum @@ -1 +1 @@ -64586b7191bf816f7d230414538f054b89895af0681037ea84b26852ac6ab6a5a6ec0ff5b30630cac0685c9824f8e0aaff287e984bd9e7a8b0dcf0ba5133d844 Dockerfile +11ff01926b3c94b4cd20bb754d2014772aa26e53b0bd2fac1af28227542b5997891346a6237fa59c1c30df5715c3e28ad991b58d4b3293ab1c75f954f5b457dd Dockerfile diff --git a/test/integration/pytest_itde_test.py b/test/integration/pytest_itde_test.py deleted file mode 100644 index 682ace020..000000000 --- a/test/integration/pytest_itde_test.py +++ /dev/null @@ -1,156 +0,0 @@ -import os -from inspect import cleandoc -from itertools import chain - -import pytest - -pytest_plugins = "pytester" - - -@pytest.fixture -def make_test_files(): - def make(pytester, files): - pytester.makepyfile(**files) - - return make - - -def _ids(params): - keys = (k for k in params.keys()) - return next(keys) - - -default_version = "8.18.1" -default_version_only=pytest.mark.skipif( - "EXASOL_VERSION" in os.environ and os.environ["EXASOL_VERSION"] != default_version, - reason="""This test always uses default version of Exasol database. If - the current run of a matrix build uses a different version then executing - all tests requires to download two docker images in total. For Exasol - versions 8 and higher the size of the Docker Containers did drastically - increase which in turn causes error "no space left on device" in the GitHub Action Runners.""", -) - - -@default_version_only -@pytest.mark.slow -@pytest.mark.parametrize( - "files", - [ - { - "test_smoke_test": cleandoc( - """ - def test_itde_smoke_test(itde): - # This smoke test just makes sure db spin up etc does not fail - assert True - """ - ) - }], ids=_ids -) -def test_itde_smoke_test(make_test_files, pytester, files): - make_test_files(pytester, files) - result = pytester.runpytest() - assert result.ret == pytest.ExitCode.OK - - -@default_version_only -@pytest.mark.parametrize( - "files", - [ - { - "test_exasol_settings": cleandoc( - """ - def test_default_settings_of_exasol(exasol_config): - assert exasol_config.host == 'localhost' - assert exasol_config.port == 8563 - assert exasol_config.username == 'SYS' - assert exasol_config.password == 'exasol' - """ - ) - }, - { - "test_bucketfs_settings": cleandoc( - """ - def test_default_settings_of_bucketfs(bucketfs_config): - assert bucketfs_config.url == 'http://127.0.0.1:2580' - assert bucketfs_config.username == 'w' - assert bucketfs_config.password == 'write' - """ - ) - }, - { - "test_itde_settings": cleandoc( - f""" -def test_default_settings_of_itde(itde_config): - assert itde_config.db_version == '{default_version}' - assert set(itde_config.schemas) == set(('TEST', 'TEST_SCHEMA')) -""" - ) - }, - ], - ids=_ids, -) -def test_default_settings_for_options(pytester, make_test_files, files): - make_test_files(pytester, files) - result = pytester.runpytest() - assert result.ret == pytest.ExitCode.OK - - -@pytest.mark.parametrize( - "files,cli_args", - [ - ( - { - "test_exasol_settings": cleandoc( - """ - def test_default_settings_of_exasol(exasol_config): - assert exasol_config.host == '127.0.0.1' - assert exasol_config.port == 7777 - assert exasol_config.username == 'foo' - assert exasol_config.password == 'bar' - """ - ) - }, - { - "--exasol-port": 7777, - "--exasol-host": "127.0.0.1", - "--exasol-username": "foo", - "--exasol-password": "bar", - }, - ), - ( - { - "test_bucketfs_settings": cleandoc( - """ - def test_default_settings_of_bucketfs(bucketfs_config): - assert bucketfs_config.url == 'https://127.0.0.1:7777' - assert bucketfs_config.username == 'user' - assert bucketfs_config.password == 'pw' - """ - ) - }, - { - "--bucketfs-url": "https://127.0.0.1:7777", - "--bucketfs-username": "user", - "--bucketfs-password": "pw", - }, - ), - ( - { - "test_itde_settings": cleandoc( - """ - def test_default_settings_of_itde(itde_config): - assert itde_config.db_version == '7.1.0' - assert set(itde_config.schemas) == set(('TEST_FOO', 'TEST_BAR')) - """ - ) - }, - {"--itde-db-version": "7.1.0", "--itde-schemas": "TEST_FOO, TEST_BAR"}, - ), - ], - ids=_ids, -) -def test_pass_options_via_cli(pytester, make_test_files, files, cli_args): - make_test_files(pytester, files) - args = chain.from_iterable(cli_args.items()) - result = pytester.runpytest(*args) - assert result.ret == pytest.ExitCode.OK diff --git a/test/unit/pytest_plugin_itde_test.py b/test/unit/pytest_plugin_itde_test.py deleted file mode 100644 index aa1e2236e..000000000 --- a/test/unit/pytest_plugin_itde_test.py +++ /dev/null @@ -1,246 +0,0 @@ -import pytest - -from pytest_itde import TestSchemas -from pytest_itde.config import Option, OptionGroup -from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports - -OPTIONS = ( - Option( - name="port", - prefix="exasol", - type=int, - default=9999, - help_text="Port to connect to", - ), - Option( - name="URL", - prefix="exasol", - type=str, - default="http://foo.bar", - help_text="A url", - ), -) - - -@pytest.mark.parametrize("option,expected", zip(OPTIONS, ("EXASOL_PORT", "EXASOL_URL"))) -def test_get_environment_variable_name_for_option(option, expected): - assert option.env == expected - - -@pytest.mark.parametrize( - "option,expected", zip(OPTIONS, ("--exasol-port", "--exasol-url")) -) -def test_get_cli_argument_name_for_option(option, expected): - assert option.cli == expected - - -@pytest.mark.parametrize("option,expected", zip(OPTIONS, ("exasol_port", "exasol_url"))) -def test_get_pytest_option_name_for_option(option, expected): - assert option.pytest == expected - - -def test_help_of_option_without_default_value(): - option = Option( - name="port", - prefix="exasol", - type=int, - default=9999, - help_text="Port to connect to", - ) - expected = "Port to connect to (default: 9999)." - assert option.help == expected - - -def test_help_of_option_with_default_value(): - option = Option( - name="port", - prefix="exasol", - type=int, - help_text="Port to connect to", - ) - expected = "Port to connect to." - assert option.help == expected - - -@pytest.mark.parametrize( - "definition,expected", - ( - ("", []), - ("TEST", ["TEST"]), - ("TEST1,TEST2", ["TEST1", "TEST2"]), - ("TEST1, TEST2", ["TEST1", "TEST2"]), - ("TEST1, TEST2,", ["TEST1", "TEST2"]), - ("TEST1, TEST2, TEST3", ["TEST1", "TEST2", "TEST3"]), - ), -) -def test_test_schema_parser(definition, expected): - actual = TestSchemas(definition) - assert set(actual) == set(expected) - - -EXASOL_OPTIONS = ( - { - "name": "host", - "type": str, - "default": "localhost", - "help_text": "Host to connect to", - }, - { - "name": "port", - "type": int, - "default": Ports.default_ports.database, - "help_text": "Port on which the exasol db is listening", - }, - { - "name": "username", - "type": str, - "default": "SYS", - "help_text": "Username used to authenticate against the exasol db", - }, - { - "name": "password", - "type": str, - "default": "exasol", - "help_text": "Password used to authenticate against the exasol db", - }, -) - -BUCKETFS_OPTIONS = ( - { - "name": "url", - "type": str, - "default": f"http://127.0.0.1:{Ports.default_ports.bucketfs}", - "help_text": "Base url used to connect to the bucketfs service", - }, - { - "name": "username", - "type": str, - "default": "w", - "help_text": "Username used to authenticate against the bucketfs service", - }, - { - "name": "password", - "type": str, - "default": "write", - "help_text": "Password used to authenticate against the bucketfs service", - }, -) - - -@pytest.mark.parametrize( - "group,expected", - ( - (OptionGroup(prefix="exasol", options=EXASOL_OPTIONS), "exasol"), - (OptionGroup(prefix="bucket", options=BUCKETFS_OPTIONS), "bucket"), - ), -) -def test_option_group_prefix_property(group, expected): - actual = group.prefix - assert actual == expected - - -@pytest.mark.parametrize( - "group,expected", - ( - ( - OptionGroup(prefix="exasol", options=EXASOL_OPTIONS), - ( - Option( - prefix="exasol", - name="host", - type=str, - default="localhost", - help_text="Host to connect to", - ), - Option( - prefix="exasol", - name="port", - type=int, - default=8563, - help_text="Port on which the exasol db is listening", - ), - Option( - prefix="exasol", - name="username", - type=str, - default="SYS", - help_text="Username used to authenticate against the exasol db", - ), - Option( - prefix="exasol", - name="password", - type=str, - default="exasol", - help_text="Password used to authenticate against the exasol db", - ), - ), - ), - ), -) -def test_option_group_options_property(group, expected): - actual = group.options - assert set(actual) == set(expected) - - -class PyTestArgs: - def __init__(self, **kwargs): - for name, value in kwargs.items(): - setattr(self, name, value) - - -@pytest.mark.parametrize( - "group,env,cli,expected", - ( - ( - OptionGroup( - prefix="db", - options=( - { - "name": "port", - "type": int, - "default": 9999, - "help_text": "some help", - }, - ), - ), - {}, - PyTestArgs(), - {"port": 9999}, - ), - ( - OptionGroup( - prefix="db", - options=( - { - "name": "port", - "type": int, - "default": 9999, - "help_text": "some help", - }, - ), - ), - {"DB_PORT": "7777"}, - PyTestArgs(), - {"port": 7777}, - ), - ( - OptionGroup( - prefix="db", - options=( - { - "name": "port", - "type": int, - "default": 9999, - "help_text": "some help", - }, - ), - ), - {"DB_PORT": "7777"}, - PyTestArgs(db_port=8888), - {"port": 8888}, - ), - ), -) -def test_option_group_prefix_kwargs_prioritization(group, env, cli, expected): - actual = dict(group.kwargs(env, cli)) - assert actual == expected