From ad11849e215946cc992893602718dcdbaa5bb06f Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 26 Nov 2024 08:26:01 -0300 Subject: [PATCH] #423: updated formatting (#424) closes #423 --- doc/changes/unreleased.md | 1 + .../cli/commands/health.py | 2 +- .../cli/commands/spawn_test_environment.py | 62 ++-- .../cli/options/build_options.py | 61 ++-- .../cli/options/docker_repository_options.py | 122 ++++--- .../cli/options/push_options.py | 17 +- .../cli/options/system_options.py | 60 ++-- .../cli/options/test_environment_options.py | 179 ++++++---- .../cli/termination_handler.py | 6 +- .../doctor.py | 24 +- .../lib/__init__.py | 2 +- .../lib/api/__init__.py | 17 +- .../lib/api/api_errors.py | 6 +- .../lib/api/build_test_container.py | 132 +++++--- .../lib/api/common.py | 268 +++++++++------ .../lib/api/push_test_container.py | 140 +++++--- .../lib/api/spawn_test_environment.py | 218 +++++++----- ...wn_test_environment_with_test_container.py | 222 +++++++----- .../lib/base/abstract_task_future.py | 3 +- .../lib/base/base_task.py | 136 +++++--- .../lib/base/db_os_executor.py | 43 ++- .../lib/base/dependency_logger_base_task.py | 78 +++-- .../lib/base/docker_base_task.py | 8 +- .../lib/base/docker_parameter.py | 4 +- .../lib/base/flavor_task.py | 28 +- .../lib/base/frozendict_to_dict.py | 2 +- .../lib/base/info.py | 14 +- .../lib/base/json_pickle_parameter.py | 47 ++- .../lib/base/json_pickle_target.py | 15 +- .../lib/base/luigi_log_config.py | 56 +++- .../lib/base/pickle_target.py | 1 + .../lib/base/ssh_access.py | 34 +- .../lib/base/still_running_logger.py | 13 +- .../lib/base/stoppable_base_task.py | 36 +- .../lib/base/task_dependency.py | 31 +- .../lib/base/task_logger_wrapper.py | 2 +- .../lib/base/task_state.py | 21 +- .../lib/base/wrong_task_state_exception.py | 4 +- .../lib/config/build_config.py | 28 +- .../lib/config/docker_config.py | 28 +- .../lib/config/log_config.py | 5 +- .../lib/data/container_info.py | 22 +- .../lib/data/database_credentials.py | 14 +- .../lib/data/database_info.py | 20 +- .../lib/data/docker_network_info.py | 4 +- .../lib/data/docker_volume_info.py | 10 +- .../lib/data/environment_info.py | 25 +- .../lib/data/environment_type.py | 9 +- .../lib/data/ssh_info.py | 1 + .../test_container_content_description.py | 20 +- .../lib/docker/__init__.py | 19 +- .../lib/docker/container/utils.py | 2 + .../lib/docker/images/clean/clean_images.py | 59 +++- .../docker/images/create/docker_build_base.py | 149 +++++--- .../create/docker_image_analyze_task.py | 195 +++++++---- .../images/create/docker_image_build_task.py | 71 ++-- .../images/create/docker_image_create_task.py | 91 +++-- .../create/docker_image_creator_base_task.py | 22 +- .../images/create/docker_image_load_task.py | 23 +- .../images/create/docker_image_pull_task.py | 51 ++- .../create/utils/build_context_creator.py | 47 ++- .../create/utils/build_context_hasher.py | 76 +++-- .../images/create/utils/build_log_handler.py | 48 ++- .../create/utils/character_length_checker.py | 14 +- .../utils/docker_registry_image_checker.py | 11 +- .../utils/file_directory_list_hasher.py | 208 ++++++++---- .../images/create/utils/pull_log_handler.py | 52 ++- .../create/utils/symlink_loop_checker.py | 4 +- .../lib/docker/images/image_info.py | 76 +++-- .../push/docker_image_push_base_task.py | 50 ++- .../images/push/docker_image_push_task.py | 21 +- .../images/push/docker_push_parameter.py | 2 +- .../docker/images/push/push_log_handler.py | 44 ++- .../push/push_task_creator_for_build_tasks.py | 19 +- .../save/docker_image_save_base_task.py | 49 ++- .../images/save/docker_image_save_task.py | 21 +- .../save/save_task_creator_for_build_tasks.py | 21 +- .../images/task_creator_from_build_tasks.py | 50 ++- .../lib/docker/images/utils.py | 15 +- .../lib/logging/abstract_log_handler.py | 4 +- .../lib/logging/command_log_handler.py | 42 ++- .../lib/logging/container_log_handler.py | 13 +- .../abstract_spawn_test_environment.py | 204 +++++++---- .../analyze_test_container.py | 54 ++- .../analyze_certificate_container.py | 42 ++- .../create_ssl_certificates_task.py | 112 ++++--- .../database_setup/bucketfs_sync_checker.py | 4 +- ...docker_db_log_based_bucket_sync_checker.py | 41 ++- .../find_exaplus_in_db_container.py | 12 +- .../database_setup/populate_data.py | 54 ++- .../time_based_bucketfs_sync_waiter.py | 5 +- .../db_container_log_thread.py | 45 ++- .../is_database_ready_thread.py | 57 ++-- .../wait_for_external_database.py | 27 +- .../wait_for_test_docker_database.py | 120 ++++--- .../lib/test_environment/db_version.py | 15 +- .../test_environment/docker_container_copy.py | 26 +- .../docker_db_test_environment_parameter.py | 39 ++- .../external_test_environment_parameter.py | 35 +- ...eneral_spawn_test_environment_parameter.py | 27 +- .../spawn_test_environment_parameter.py | 27 +- .../parameter/test_container_parameter.py | 22 +- .../lib/test_environment/ports.py | 30 +- .../prepare_network_for_test_environment.py | 85 +++-- .../setup_external_database_host.py | 110 +++--- .../lib/test_environment/shell_variables.py | 61 ++-- .../test_environment/spawn_test_container.py | 242 ++++++++----- .../test_environment/spawn_test_database.py | 317 +++++++++++------- .../spawn_test_environment.py | 49 +-- .../spawn_test_environment_with_docker_db.py | 113 ++++--- ...spawn_test_environment_with_external_db.py | 106 +++--- .../lib/utils/resource_directory.py | 14 +- .../main.py | 3 +- .../test/get_test_container_content.py | 23 +- .../test/test_api_build_test_container.py | 49 ++- .../test_api_external_test_environment.py | 129 ++++--- .../test/test_api_logging.py | 150 ++++++--- .../test/test_api_push_test_container.py | 33 +- .../test/test_api_test_environment.py | 136 +++++--- .../test_api_test_environment_certificate.py | 55 ++- ...est_api_test_environment_docker_runtime.py | 56 +++- .../test/test_base_task.py | 63 ++-- .../test/test_base_task_cleanup.py | 79 +++-- .../test/test_certificate_docker_build.py | 60 +++- ..._cli_test_environment_additional_params.py | 61 ++-- .../test_cli_test_environment_db_disk_size.py | 52 +-- .../test_cli_test_environment_db_mem_size.py | 54 +-- .../test_cli_test_environment_nameservers.py | 60 ++-- .../test/test_click_api_consistency.py | 45 ++- .../test/test_common_run_task.py | 19 +- ...db_version_supports_custom_certificates.py | 7 +- .../test/test_docker_access_method.py | 22 +- .../test/test_docker_build_base.py | 75 +++-- .../test_docker_registry_image_checker.py | 7 +- .../test/test_doctor.py | 1 - .../test/test_find_free_port.py | 8 +- .../test/test_generate_graph_plot.py | 25 +- .../test/test_hash_temp_dir.py | 177 +++++----- .../test/test_hash_temp_dir_with_files.py | 168 ++++++---- .../test/test_populate_data.py | 80 +++-- .../test/test_symlink_loops.py | 83 +++-- .../test/test_termination_handler.py | 19 +- .../test/test_test_container_reuse.py | 153 +++++---- .../test/test_test_env_reuse.py | 123 ++++--- .../testing/api_consistency_utils.py | 43 ++- .../testing/api_test_environment.py | 54 +-- .../testing/docker_registry.py | 29 +- .../exaslct_docker_test_environment.py | 37 +- .../testing/exaslct_test_environment.py | 121 ++++--- .../testing/luigi_utils.py | 8 +- .../testing/spawned_test_environments.py | 12 +- .../testing/utils.py | 8 +- noxfile.py | 17 +- pyproject.toml | 15 + test/integration/conftest.py | 63 ++-- test/integration/helpers.py | 24 +- test/integration/test_cli_environment.py | 70 ++-- .../test_db_container_log_thread.py | 39 ++- test/integration/test_ssh_access.py | 64 ++-- test/integration/test_udf_execution.py | 8 +- test/unit/test_base_ssh_keys.py | 8 +- test/unit/test_db_os_executor.py | 44 +-- test/unit/test_env_variable.py | 64 +++- test/unit/test_shell_variables.py | 45 +-- test/unit/test_task_runtime_error.py | 57 +++- test/unit/test_termination_handler.py | 41 ++- 166 files changed, 5734 insertions(+), 3177 deletions(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 252ba4848..710de667e 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -11,3 +11,4 @@ Code name: * #418: Use exasol/python-toolbox * #411: Removed usage of exasol-bucketfs * #425: Fixed type checks found by MyPy +* #423: Updated formatting diff --git a/exasol_integration_test_docker_environment/cli/commands/health.py b/exasol_integration_test_docker_environment/cli/commands/health.py index dbca38c44..885c0a29e 100644 --- a/exasol_integration_test_docker_environment/cli/commands/health.py +++ b/exasol_integration_test_docker_environment/cli/commands/health.py @@ -1,8 +1,8 @@ import sys from exasol_integration_test_docker_environment.cli.cli import cli -from exasol_integration_test_docker_environment.lib.api.api_errors import HealthProblem from exasol_integration_test_docker_environment.lib import api +from exasol_integration_test_docker_environment.lib.api.api_errors import HealthProblem @cli.command() diff --git a/exasol_integration_test_docker_environment/cli/commands/spawn_test_environment.py b/exasol_integration_test_docker_environment/cli/commands/spawn_test_environment.py index f714f1f05..a22a0a1b9 100644 --- a/exasol_integration_test_docker_environment/cli/commands/spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/cli/commands/spawn_test_environment.py @@ -1,4 +1,7 @@ -from typing import Optional, Tuple +from typing import ( + Optional, + Tuple, +) import click @@ -7,9 +10,10 @@ docker_repository_options, ) from exasol_integration_test_docker_environment.cli.options.system_options import ( + luigi_logging_options, output_directory_option, system_options, - tempory_base_directory_option, luigi_logging_options, + tempory_base_directory_option, ) from exasol_integration_test_docker_environment.cli.options.test_environment_options import ( docker_db_options, @@ -87,33 +91,33 @@ @add_options(system_options) @add_options(luigi_logging_options) def spawn_test_environment( - environment_name: str, - database_port_forward: Optional[int], - bucketfs_port_forward: Optional[int], - ssh_port_forward: Optional[int], - db_mem_size: str, - db_disk_size: str, - nameserver: Tuple[str, ...], - docker_runtime: Optional[str], - docker_db_image_version: str, - docker_db_image_name: str, - db_os_access: Optional[str], - create_certificates: bool, - additional_db_parameter: Tuple[str, ...], - source_docker_repository_name: str, - source_docker_tag_prefix: str, - source_docker_username: Optional[str], - source_docker_password: Optional[str], - target_docker_repository_name: str, - target_docker_tag_prefix: str, - target_docker_username: Optional[str], - target_docker_password: Optional[str], - output_directory: str, - temporary_base_directory: str, - workers: int, - task_dependencies_dot_file: Optional[str], - log_level: Optional[str], - use_job_specific_log_file: bool + environment_name: str, + database_port_forward: Optional[int], + bucketfs_port_forward: Optional[int], + ssh_port_forward: Optional[int], + db_mem_size: str, + db_disk_size: str, + nameserver: Tuple[str, ...], + docker_runtime: Optional[str], + docker_db_image_version: str, + docker_db_image_name: str, + db_os_access: Optional[str], + create_certificates: bool, + additional_db_parameter: Tuple[str, ...], + source_docker_repository_name: str, + source_docker_tag_prefix: str, + source_docker_username: Optional[str], + source_docker_password: Optional[str], + target_docker_repository_name: str, + target_docker_tag_prefix: str, + target_docker_username: Optional[str], + target_docker_password: Optional[str], + output_directory: str, + temporary_base_directory: str, + workers: int, + task_dependencies_dot_file: Optional[str], + log_level: Optional[str], + use_job_specific_log_file: bool, ): """ This command spawns a test environment with a docker-db container and a connected test-container. diff --git a/exasol_integration_test_docker_environment/cli/options/build_options.py b/exasol_integration_test_docker_environment/cli/options/build_options.py index b49df006c..9b099da1e 100644 --- a/exasol_integration_test_docker_environment/cli/options/build_options.py +++ b/exasol_integration_test_docker_environment/cli/options/build_options.py @@ -1,27 +1,48 @@ import click -from exasol_integration_test_docker_environment.cli.options.system_options import tempory_base_directory_option, \ - output_directory_option +from exasol_integration_test_docker_environment.cli.options.system_options import ( + output_directory_option, + tempory_base_directory_option, +) build_options = [ - click.option('--force-rebuild/--no-force-rebuild', default=False, - help="Forces the system to complete rebuild all stages down to the stages " - "specified with the options --force-rebuild-from."), - click.option('--force-rebuild-from', multiple=True, type=str, - help="If the option --force-rebuild is given, " - "this options specifies for which stages and dependent stages system will force a rebuild. " - "The option can be repeated with different stages. " - "The system will than force the rebuild of these stages and their. dependet stages." - ), - click.option('--force-pull/--no-force-pull', default=False, - help="Forces the system to pull all stages if available, otherwise it rebuilds a stage."), + click.option( + "--force-rebuild/--no-force-rebuild", + default=False, + help="Forces the system to complete rebuild all stages down to the stages " + "specified with the options --force-rebuild-from.", + ), + click.option( + "--force-rebuild-from", + multiple=True, + type=str, + help="If the option --force-rebuild is given, " + "this options specifies for which stages and dependent stages system will force a rebuild. " + "The option can be repeated with different stages. " + "The system will than force the rebuild of these stages and their. dependet stages.", + ), + click.option( + "--force-pull/--no-force-pull", + default=False, + help="Forces the system to pull all stages if available, otherwise it rebuilds a stage.", + ), output_directory_option, tempory_base_directory_option, - click.option('--log-build-context-content/--no-log-build-context-content', - default=False, - help="For Debugging: Logs the files and directories in the build context of a stage"), - click.option('--cache-directory', default=None, type=click.Path(file_okay=False, dir_okay=True, exists=False), - help="Directory from where saved docker images can be loaded"), - click.option('--build-name', default=None, type=str, - help="Name of the build. For example: Repository + CI Build Number"), + click.option( + "--log-build-context-content/--no-log-build-context-content", + default=False, + help="For Debugging: Logs the files and directories in the build context of a stage", + ), + click.option( + "--cache-directory", + default=None, + type=click.Path(file_okay=False, dir_okay=True, exists=False), + help="Directory from where saved docker images can be loaded", + ), + click.option( + "--build-name", + default=None, + type=str, + help="Name of the build. For example: Repository + CI Build Number", + ), ] diff --git a/exasol_integration_test_docker_environment/cli/options/docker_repository_options.py b/exasol_integration_test_docker_environment/cli/options/docker_repository_options.py index 8a73e9d0b..490ab29a5 100644 --- a/exasol_integration_test_docker_environment/cli/options/docker_repository_options.py +++ b/exasol_integration_test_docker_environment/cli/options/docker_repository_options.py @@ -1,54 +1,84 @@ import click -DEFAULT_DOCKER_REPOSITORY_NAME = 'exasol/script-language-container' +DEFAULT_DOCKER_REPOSITORY_NAME = "exasol/script-language-container" docker_repository_options = [ - click.option('--source-docker-repository-name', type=str, - default=DEFAULT_DOCKER_REPOSITORY_NAME, - show_default=True, - help="Name of the docker repository for pulling cached stages. " - "The repository name may contain the URL of the docker registry, " - "the username and the actual repository name. " - "A common structure is //"), - click.option('--source-docker-tag-prefix', type=str, - default="", - show_default=True, - help="Prefix for the tags which are used for pulling of cached stages"), - click.option('--source-docker-username', type=str, - help="Username for the docker registry from where the system pulls cached stages.", - required=False), - click.option('--source-docker-password', type=str, - help="Password for the docker registry from where the system pulls cached stages. " - "Without password option the system prompts for the password."), - click.option('--target-docker-repository-name', type=str, - default=DEFAULT_DOCKER_REPOSITORY_NAME, - show_default=True, - help="Name of the docker repository for naming and pushing images of stages. " - "The repository name may contain the URL of the docker registry, " - "the username and the actual repository name. " - "A common structure is //"), - click.option('--target-docker-tag-prefix', type=str, - default="", - show_default=True, - help="Prefix for the tags which are used for naming and pushing of stages"), - click.option('--target-docker-username', type=str, - help="Username for the docker registry where the system pushes images of stages.", - required=False), - click.option('--target-docker-password', type=str, - help="Password for the docker registry where the system pushes images of stages. " - "Without password option the system prompts for the password."), + click.option( + "--source-docker-repository-name", + type=str, + default=DEFAULT_DOCKER_REPOSITORY_NAME, + show_default=True, + help="Name of the docker repository for pulling cached stages. " + "The repository name may contain the URL of the docker registry, " + "the username and the actual repository name. " + "A common structure is //", + ), + click.option( + "--source-docker-tag-prefix", + type=str, + default="", + show_default=True, + help="Prefix for the tags which are used for pulling of cached stages", + ), + click.option( + "--source-docker-username", + type=str, + help="Username for the docker registry from where the system pulls cached stages.", + required=False, + ), + click.option( + "--source-docker-password", + type=str, + help="Password for the docker registry from where the system pulls cached stages. " + "Without password option the system prompts for the password.", + ), + click.option( + "--target-docker-repository-name", + type=str, + default=DEFAULT_DOCKER_REPOSITORY_NAME, + show_default=True, + help="Name of the docker repository for naming and pushing images of stages. " + "The repository name may contain the URL of the docker registry, " + "the username and the actual repository name. " + "A common structure is //", + ), + click.option( + "--target-docker-tag-prefix", + type=str, + default="", + show_default=True, + help="Prefix for the tags which are used for naming and pushing of stages", + ), + click.option( + "--target-docker-username", + type=str, + help="Username for the docker registry where the system pushes images of stages.", + required=False, + ), + click.option( + "--target-docker-password", + type=str, + help="Password for the docker registry where the system pushes images of stages. " + "Without password option the system prompts for the password.", + ), ] simple_docker_repository_options = [ - click.option('--docker-repository-name', type=str, - default=DEFAULT_DOCKER_REPOSITORY_NAME, - show_default=True, - help="Name of the docker repository for naming images. " - "The repository name may contain the URL of the docker registry, " - "the username and the actual repository name. " - "A common structure is //"), - click.option('--docker-tag-prefix', type=str, - default="", - show_default=True, - help="Prefix for the tags of the images"), + click.option( + "--docker-repository-name", + type=str, + default=DEFAULT_DOCKER_REPOSITORY_NAME, + show_default=True, + help="Name of the docker repository for naming images. " + "The repository name may contain the URL of the docker registry, " + "the username and the actual repository name. " + "A common structure is //", + ), + click.option( + "--docker-tag-prefix", + type=str, + default="", + show_default=True, + help="Prefix for the tags of the images", + ), ] diff --git a/exasol_integration_test_docker_environment/cli/options/push_options.py b/exasol_integration_test_docker_environment/cli/options/push_options.py index b2f29448c..95d6b036b 100644 --- a/exasol_integration_test_docker_environment/cli/options/push_options.py +++ b/exasol_integration_test_docker_environment/cli/options/push_options.py @@ -1,9 +1,14 @@ import click push_options = [ - click.option('--force-push/--no-force-push', default=False, - help="Forces the system to overwrite existing images in registry for build steps that run"), - click.option('--push-all/--no-push-all', default=False, - help="Forces the system to push all images of build-steps that are specified by the goals") - -] \ No newline at end of file + click.option( + "--force-push/--no-force-push", + default=False, + help="Forces the system to overwrite existing images in registry for build steps that run", + ), + click.option( + "--push-all/--no-push-all", + default=False, + help="Forces the system to push all images of build-steps that are specified by the goals", + ), +] diff --git a/exasol_integration_test_docker_environment/cli/options/system_options.py b/exasol_integration_test_docker_environment/cli/options/system_options.py index 36f51ec69..d611a59a7 100644 --- a/exasol_integration_test_docker_environment/cli/options/system_options.py +++ b/exasol_integration_test_docker_environment/cli/options/system_options.py @@ -2,30 +2,50 @@ DEFAULT_OUTPUT_DIRECTORY = ".build_output" -output_directory_option = click.option('--output-directory', type=click.Path(file_okay=False, dir_okay=True), - default=DEFAULT_OUTPUT_DIRECTORY, - show_default=True, - help="Output directory where the system stores all output and log files.") +output_directory_option = click.option( + "--output-directory", + type=click.Path(file_okay=False, dir_okay=True), + default=DEFAULT_OUTPUT_DIRECTORY, + show_default=True, + help="Output directory where the system stores all output and log files.", +) -tempory_base_directory_option = click.option('--temporary-base-directory', - type=click.Path(file_okay=False, - dir_okay=True), - default="/tmp", - show_default=True, - help="Directory where the system creates temporary directories.") +tempory_base_directory_option = click.option( + "--temporary-base-directory", + type=click.Path(file_okay=False, dir_okay=True), + default="/tmp", + show_default=True, + help="Directory where the system creates temporary directories.", +) system_options = [ - click.option('--workers', type=int, - default=5, show_default=True, - help="Number of parallel workers"), - click.option('--task-dependencies-dot-file', type=click.Path(file_okay=True), - default=None, help="Path where to store the Task Dependency Graph as dot file"), + click.option( + "--workers", + type=int, + default=5, + show_default=True, + help="Number of parallel workers", + ), + click.option( + "--task-dependencies-dot-file", + type=click.Path(file_okay=True), + default=None, + help="Path where to store the Task Dependency Graph as dot file", + ), ] luigi_logging_options = [ - click.option('--log-level', type=click.Choice(['DEBUG', 'INFO', 'WARNING', 'ERROR', 'FATAL']), - default=None, help="Log level used for console logging"), - click.option('--use-job-specific-log-file', type=bool, - default=True, help="Use a job specific log file which write the debug log " - "to the job directory in the build directory") + click.option( + "--log-level", + type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "FATAL"]), + default=None, + help="Log level used for console logging", + ), + click.option( + "--use-job-specific-log-file", + type=bool, + default=True, + help="Use a job specific log file which write the debug log " + "to the job directory in the build directory", + ), ] diff --git a/exasol_integration_test_docker_environment/cli/options/test_environment_options.py b/exasol_integration_test_docker_environment/cli/options/test_environment_options.py index 2e23dc9f9..941c6b431 100644 --- a/exasol_integration_test_docker_environment/cli/options/test_environment_options.py +++ b/exasol_integration_test_docker_environment/cli/options/test_environment_options.py @@ -1,71 +1,136 @@ import click -from exasol_integration_test_docker_environment \ - .lib.test_environment.parameter \ - .docker_db_test_environment_parameter \ - import DbOsAccess -from exasol_integration_test_docker_environment \ - .lib.test_environment.ports import Ports -test_environment_options = [ - click.option('--environment-type', type=click.Choice(['docker_db', 'external_db']), default="""docker_db""", - show_default=True, - help="""Environment type for tests."""), - click.option('--max_start_attempts', type=int, default=2, - show_default=True, - help="""Maximum start attempts for environment""") +from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter import ( + DbOsAccess, +) +from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports +test_environment_options = [ + click.option( + "--environment-type", + type=click.Choice(["docker_db", "external_db"]), + default="""docker_db""", + show_default=True, + help="""Environment type for tests.""", + ), + click.option( + "--max_start_attempts", + type=int, + default=2, + show_default=True, + help="""Maximum start attempts for environment""", + ), ] LATEST_DB_VERSION = """8.31.0""" docker_db_options = [ - click.option('--docker-db-image-version', type=str, default=LATEST_DB_VERSION, - show_default=True, - help="""Docker DB Image Version against which the tests should run."""), - click.option('--docker-db-image-name', type=str, default="""exasol/docker-db""", - show_default=True, - help="""Docker DB Image Name against which the tests should run."""), - click.option('--db-os-access', type=click.Choice([e.name for e in DbOsAccess]), - metavar="METHOD", default="""DOCKER_EXEC""", show_default=True, - help="""How to access file system and command line of the + click.option( + "--docker-db-image-version", + type=str, + default=LATEST_DB_VERSION, + show_default=True, + help="""Docker DB Image Version against which the tests should run.""", + ), + click.option( + "--docker-db-image-name", + type=str, + default="""exasol/docker-db""", + show_default=True, + help="""Docker DB Image Name against which the tests should run.""", + ), + click.option( + "--db-os-access", + type=click.Choice([e.name for e in DbOsAccess]), + metavar="METHOD", + default="""DOCKER_EXEC""", + show_default=True, + help="""How to access file system and command line of the database operating system. Experimental option, will show no effect until implementation of feature SSH access is - completed."""), - click.option('--create-certificates/--no-create-certificates', default=False, - help="""Creates and injects SSL certificates to the Docker DB container."""), - click.option('--additional-db-parameter', '-p', type=str, multiple=True, - help="""Additional database parameter which will be injected to EXAConf. Value should have format '-param=value'.""") + completed.""", + ), + click.option( + "--create-certificates/--no-create-certificates", + default=False, + help="""Creates and injects SSL certificates to the Docker DB container.""", + ), + click.option( + "--additional-db-parameter", + "-p", + type=str, + multiple=True, + help="""Additional database parameter which will be injected to EXAConf. Value should have format '-param=value'.""", + ), ] external_db_options = [ - click.option('--external-exasol-db-host', type=str, - help="""Host name or IP of external Exasol DB, needs to be set if --environment-type=external_db"""), - click.option('--external-exasol-db-port', type=int, - default=Ports.external.database, - help="""Database port of external Exasol DB, needs to be set if --environment-type=external_db"""), - click.option('--external-exasol-bucketfs-port', type=int, - default=Ports.external.bucketfs, - help="""Bucketfs port of external Exasol DB, needs to be set if --environment-type=external_db"""), - click.option('--external-exasol-ssh-port', type=int, - help="""SSH port of external Exasol DB, needs to be set if --environment-type=external_db"""), - click.option('--external-exasol-db-user', type=str, - help="""User for external Exasol DB, needs to be set if --environment-type=external_db"""), - click.option('--external-exasol-db-password', type=str, - help="""Database Password for external Exasol DB"""), - click.option('--external-exasol-bucketfs-write-password', type=str, - help="""BucketFS write Password for external Exasol DB"""), - click.option('--external-exasol-xmlrpc-host', type=str, - help="""Hostname for the xmlrpc server"""), - click.option('--external-exasol-xmlrpc-port', type=int, - default=443, show_default=True, - help="""Port for the xmlrpc server"""), - click.option('--external-exasol-xmlrpc-user', type=str, - default="""admin""", show_default=True, - help="""User for the xmlrpc server"""), - click.option('--external-exasol-xmlrpc-password', type=str, - help="""Password for the xmlrpc server"""), - click.option('--external-exasol-xmlrpc-cluster-name', type=str, - default="""cluster1""", show_default=True, - help="""Password for the xmlrpc server""") - + click.option( + "--external-exasol-db-host", + type=str, + help="""Host name or IP of external Exasol DB, needs to be set if --environment-type=external_db""", + ), + click.option( + "--external-exasol-db-port", + type=int, + default=Ports.external.database, + help="""Database port of external Exasol DB, needs to be set if --environment-type=external_db""", + ), + click.option( + "--external-exasol-bucketfs-port", + type=int, + default=Ports.external.bucketfs, + help="""Bucketfs port of external Exasol DB, needs to be set if --environment-type=external_db""", + ), + click.option( + "--external-exasol-ssh-port", + type=int, + help="""SSH port of external Exasol DB, needs to be set if --environment-type=external_db""", + ), + click.option( + "--external-exasol-db-user", + type=str, + help="""User for external Exasol DB, needs to be set if --environment-type=external_db""", + ), + click.option( + "--external-exasol-db-password", + type=str, + help="""Database Password for external Exasol DB""", + ), + click.option( + "--external-exasol-bucketfs-write-password", + type=str, + help="""BucketFS write Password for external Exasol DB""", + ), + click.option( + "--external-exasol-xmlrpc-host", + type=str, + help="""Hostname for the xmlrpc server""", + ), + click.option( + "--external-exasol-xmlrpc-port", + type=int, + default=443, + show_default=True, + help="""Port for the xmlrpc server""", + ), + click.option( + "--external-exasol-xmlrpc-user", + type=str, + default="""admin""", + show_default=True, + help="""User for the xmlrpc server""", + ), + click.option( + "--external-exasol-xmlrpc-password", + type=str, + help="""Password for the xmlrpc server""", + ), + click.option( + "--external-exasol-xmlrpc-cluster-name", + type=str, + default="""cluster1""", + show_default=True, + help="""Password for the xmlrpc server""", + ), ] diff --git a/exasol_integration_test_docker_environment/cli/termination_handler.py b/exasol_integration_test_docker_environment/cli/termination_handler.py index 89cef1e3d..abb14e658 100644 --- a/exasol_integration_test_docker_environment/cli/termination_handler.py +++ b/exasol_integration_test_docker_environment/cli/termination_handler.py @@ -2,7 +2,9 @@ from datetime import datetime from traceback import print_tb -from exasol_integration_test_docker_environment.lib.api.api_errors import TaskRuntimeError +from exasol_integration_test_docker_environment.lib.api.api_errors import ( + TaskRuntimeError, +) def print_err(*args, **kwargs): @@ -48,7 +50,7 @@ def _handle_failure(self, task_error: TaskRuntimeError): def _print_task_failures(task_error: TaskRuntimeError): print_err() print_err("Task failure message: %s" % task_error.msg) - print_err(task_error.__cause__.args[0]) # type: ignore + print_err(task_error.__cause__.args[0]) # type: ignore print_err() def _handle_success(self): diff --git a/exasol_integration_test_docker_environment/doctor.py b/exasol_integration_test_docker_environment/doctor.py index 4fe8bc21b..04c1dd290 100644 --- a/exasol_integration_test_docker_environment/doctor.py +++ b/exasol_integration_test_docker_environment/doctor.py @@ -2,14 +2,19 @@ The doctor module provides functionality to check the health of the `exasol_integration_test_docker_environment` package and also provide help to find potential fixes. """ + import sys from collections.abc import Callable -from typing import Iterable, List, Tuple -from exasol import error from enum import Enum +from typing import ( + Iterable, + List, + Tuple, +) import docker from docker.errors import DockerException +from exasol import error SUPPORTED_PLATFORMS = ["linux", "darwin"] @@ -19,21 +24,21 @@ class Error(Enum): "E-ITDE-0", "Unknown issue.", ["An unknown error occurred, please contact the maintainer."], - {} + {}, ) UnixSocketNotAvailable = error.ExaError( "E-ITDE-1", "Could not find unix socket to connect to.", ["Make sure environment variable DOCKER_HOST is configured correctly."], - {} + {}, ) TargetPlatformNotSupported = error.ExaError( "E-ITDE-2", "The platform ITDE is running on is not supported.", ["Make sure you are using one of the following platforms: [linux, darwin]."], - {} + {}, ) @@ -79,13 +84,12 @@ def health_checkup() -> Iterable[error.ExaError]: return an iterator of error codes specifying which problems have been identified. """ - check_function = Callable[[],bool] - diagnosis_function = Callable[[],Iterable[error.ExaError]] - examinations : List[Tuple[check_function, diagnosis_function]] = [ + check_function = Callable[[], bool] + diagnosis_function = Callable[[], Iterable[error.ExaError]] + examinations: List[Tuple[check_function, diagnosis_function]] = [ (is_docker_daemon_available, diagnose_docker_daemon_not_available), (is_supported_platform, lambda: [Error.TargetPlatformNotSupported]), ] for is_fine, diagnosis in examinations: if not is_fine(): - for error_code in diagnosis(): - yield error_code + yield from diagnosis() diff --git a/exasol_integration_test_docker_environment/lib/__init__.py b/exasol_integration_test_docker_environment/lib/__init__.py index 1d11f433b..35520dd62 100644 --- a/exasol_integration_test_docker_environment/lib/__init__.py +++ b/exasol_integration_test_docker_environment/lib/__init__.py @@ -1,5 +1,5 @@ - def extract_modulename_for_build_steps(path: str): return path.replace("/", "_").replace(".", "_") + PACKAGE_NAME = "exasol_integration_test_docker_environment" diff --git a/exasol_integration_test_docker_environment/lib/api/__init__.py b/exasol_integration_test_docker_environment/lib/api/__init__.py index 8ce06bb5f..e8c38888f 100644 --- a/exasol_integration_test_docker_environment/lib/api/__init__.py +++ b/exasol_integration_test_docker_environment/lib/api/__init__.py @@ -1,6 +1,13 @@ - +from exasol_integration_test_docker_environment.lib.api.build_test_container import ( + build_test_container, +) from exasol_integration_test_docker_environment.lib.api.health import health -from exasol_integration_test_docker_environment.lib.api.build_test_container import build_test_container -from exasol_integration_test_docker_environment.lib.api.push_test_container import push_test_container -from exasol_integration_test_docker_environment.lib.api.spawn_test_environment import spawn_test_environment -from exasol_integration_test_docker_environment.lib.api.spawn_test_environment_with_test_container import spawn_test_environment_with_test_container +from exasol_integration_test_docker_environment.lib.api.push_test_container import ( + push_test_container, +) +from exasol_integration_test_docker_environment.lib.api.spawn_test_environment import ( + spawn_test_environment, +) +from exasol_integration_test_docker_environment.lib.api.spawn_test_environment_with_test_container import ( + spawn_test_environment_with_test_container, +) diff --git a/exasol_integration_test_docker_environment/lib/api/api_errors.py b/exasol_integration_test_docker_environment/lib/api/api_errors.py index d1d30475d..1f5f0a8c1 100644 --- a/exasol_integration_test_docker_environment/lib/api/api_errors.py +++ b/exasol_integration_test_docker_environment/lib/api/api_errors.py @@ -1,4 +1,8 @@ -from typing import Optional, List, Iterable +from typing import ( + Iterable, + List, + Optional, +) class ArgumentConstraintError(ValueError): diff --git a/exasol_integration_test_docker_environment/lib/api/build_test_container.py b/exasol_integration_test_docker_environment/lib/api/build_test_container.py index 854b7ac66..ee6f4cc53 100644 --- a/exasol_integration_test_docker_environment/lib/api/build_test_container.py +++ b/exasol_integration_test_docker_environment/lib/api/build_test_container.py @@ -1,40 +1,56 @@ -from typing import Tuple, Optional +from typing import ( + Optional, + Tuple, +) -from exasol_integration_test_docker_environment.lib.api.common import set_build_config, \ - set_docker_repository_config, run_task, generate_root_task, no_cli_function -from exasol_integration_test_docker_environment.cli.options.docker_repository_options import \ - DEFAULT_DOCKER_REPOSITORY_NAME -from exasol_integration_test_docker_environment.cli.options.system_options import DEFAULT_OUTPUT_DIRECTORY -from exasol_integration_test_docker_environment.lib.data.test_container_content_description import \ - TestContainerContentDescription -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo -from exasol_integration_test_docker_environment.lib.test_environment.analyze_test_container import AnalyzeTestContainer, \ - DockerTestContainerBuild +from exasol_integration_test_docker_environment.cli.options.docker_repository_options import ( + DEFAULT_DOCKER_REPOSITORY_NAME, +) +from exasol_integration_test_docker_environment.cli.options.system_options import ( + DEFAULT_OUTPUT_DIRECTORY, +) +from exasol_integration_test_docker_environment.lib.api.common import ( + generate_root_task, + no_cli_function, + run_task, + set_build_config, + set_docker_repository_config, +) +from exasol_integration_test_docker_environment.lib.data.test_container_content_description import ( + TestContainerContentDescription, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, +) +from exasol_integration_test_docker_environment.lib.test_environment.analyze_test_container import ( + AnalyzeTestContainer, + DockerTestContainerBuild, +) @no_cli_function def build_test_container( - test_container_content: TestContainerContentDescription, - force_rebuild: bool = False, - force_rebuild_from: Tuple[str, ...] = tuple(), - force_pull: bool = False, - output_directory: str = DEFAULT_OUTPUT_DIRECTORY, - temporary_base_directory: str = "/tmp", - log_build_context_content: bool = False, - cache_directory: Optional[str] = None, - build_name: Optional[str] = None, - source_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, - source_docker_tag_prefix: str = '', - source_docker_username: Optional[str] = None, - source_docker_password: Optional[str] = None, - target_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, - target_docker_tag_prefix: str = '', - target_docker_username: Optional[str] = None, - target_docker_password: Optional[str] = None, - workers: int = 5, - task_dependencies_dot_file: Optional[str] = None, - log_level: Optional[str] = None, - use_job_specific_log_file: bool = False + test_container_content: TestContainerContentDescription, + force_rebuild: bool = False, + force_rebuild_from: Tuple[str, ...] = tuple(), + force_pull: bool = False, + output_directory: str = DEFAULT_OUTPUT_DIRECTORY, + temporary_base_directory: str = "/tmp", + log_build_context_content: bool = False, + cache_directory: Optional[str] = None, + build_name: Optional[str] = None, + source_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, + source_docker_tag_prefix: str = "", + source_docker_username: Optional[str] = None, + source_docker_password: Optional[str] = None, + target_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, + target_docker_tag_prefix: str = "", + target_docker_username: Optional[str] = None, + target_docker_password: Optional[str] = None, + workers: int = 5, + task_dependencies_dot_file: Optional[str] = None, + log_level: Optional[str] = None, + use_job_specific_log_file: bool = False, ) -> ImageInfo: """ This function builds all stages of the test container for the test environment. @@ -42,24 +58,42 @@ def build_test_container( instead of building them. It returns the image info of the test-container. """ - set_build_config(force_rebuild, - force_rebuild_from, - force_pull, - log_build_context_content, - output_directory, - temporary_base_directory, - cache_directory, - build_name) + set_build_config( + force_rebuild, + force_rebuild_from, + force_pull, + log_build_context_content, + output_directory, + temporary_base_directory, + cache_directory, + build_name, + ) # Use AnalyzeTestContainer to ensure that all luigi processes got it loaded analyze_task = AnalyzeTestContainer.__class__.__name__ - set_docker_repository_config(source_docker_password, source_docker_repository_name, source_docker_username, - source_docker_tag_prefix, "source") - set_docker_repository_config(target_docker_password, target_docker_repository_name, target_docker_username, - target_docker_tag_prefix, "target") - task_creator = lambda: generate_root_task(task_class=DockerTestContainerBuild, - test_container_content=test_container_content) - image_infos = run_task(task_creator, workers, task_dependencies_dot_file, - log_level=log_level, - use_job_specific_log_file=use_job_specific_log_file) + set_docker_repository_config( + source_docker_password, + source_docker_repository_name, + source_docker_username, + source_docker_tag_prefix, + "source", + ) + set_docker_repository_config( + target_docker_password, + target_docker_repository_name, + target_docker_username, + target_docker_tag_prefix, + "target", + ) + task_creator = lambda: generate_root_task( + task_class=DockerTestContainerBuild, + test_container_content=test_container_content, + ) + image_infos = run_task( + task_creator, + workers, + task_dependencies_dot_file, + log_level=log_level, + use_job_specific_log_file=use_job_specific_log_file, + ) return image_infos["test-container"] diff --git a/exasol_integration_test_docker_environment/lib/api/common.py b/exasol_integration_test_docker_environment/lib/api/common.py index 37921ca32..6f8e710a0 100644 --- a/exasol_integration_test_docker_environment/lib/api/common.py +++ b/exasol_integration_test_docker_environment/lib/api/common.py @@ -7,10 +7,15 @@ from datetime import datetime from pathlib import Path from typing import ( + Any, Callable, - Tuple, + Dict, + Iterable, + Iterator, + List, + Optional, Set, - Optional, Any, Dict, Iterator, Iterable, List + Tuple, ) import luigi @@ -19,14 +24,27 @@ from luigi.setup_logging import InterfaceLogging from networkx import DiGraph -from exasol_integration_test_docker_environment.lib import extract_modulename_for_build_steps -from exasol_integration_test_docker_environment.lib.api.api_errors import TaskRuntimeError, TaskFailures -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask -from exasol_integration_test_docker_environment.lib.base.luigi_log_config import get_luigi_log_config, get_log_path -from exasol_integration_test_docker_environment.lib.base.task_dependency import TaskDependency, DependencyState +from exasol_integration_test_docker_environment.lib import ( + extract_modulename_for_build_steps, +) +from exasol_integration_test_docker_environment.lib.api.api_errors import ( + TaskFailures, + TaskRuntimeError, +) +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.luigi_log_config import ( + get_log_path, + get_luigi_log_config, +) +from exasol_integration_test_docker_environment.lib.base.task_dependency import ( + DependencyState, + TaskDependency, +) -class JobCounterSingleton(object): +class JobCounterSingleton: """ We use here a Singleton to avoid an unprotected global variable. However, this counter needs to be global counter to guarantee unique job ids. @@ -37,60 +55,88 @@ class JobCounterSingleton(object): def __new__(cls): if cls._instance is None: - cls._instance = super(JobCounterSingleton, cls).__new__(cls) + cls._instance = super().__new__(cls) cls._instance._counter = 0 return cls._instance def get_next_value(self) -> int: # self._counter is a class variable and because of this we need to suppress type checks - self._counter += 1 # type: ignore - return self._counter # type: ignore - - -def set_build_config(force_rebuild: bool, - force_rebuild_from: Tuple[str, ...], - force_pull: bool, - log_build_context_content: bool, - output_directory: str, - temporary_base_directory: Optional[str], - cache_directory: Optional[str], - build_name: Optional[str], ): - luigi.configuration.get_config().set('build_config', 'force_rebuild', str(force_rebuild)) - luigi.configuration.get_config().set('build_config', 'force_rebuild_from', json.dumps(force_rebuild_from)) - luigi.configuration.get_config().set('build_config', 'force_pull', str(force_pull)) + self._counter += 1 # type: ignore + return self._counter # type: ignore + + +def set_build_config( + force_rebuild: bool, + force_rebuild_from: Tuple[str, ...], + force_pull: bool, + log_build_context_content: bool, + output_directory: str, + temporary_base_directory: Optional[str], + cache_directory: Optional[str], + build_name: Optional[str], +): + luigi.configuration.get_config().set( + "build_config", "force_rebuild", str(force_rebuild) + ) + luigi.configuration.get_config().set( + "build_config", "force_rebuild_from", json.dumps(force_rebuild_from) + ) + luigi.configuration.get_config().set("build_config", "force_pull", str(force_pull)) set_output_directory(output_directory) if temporary_base_directory is not None: - luigi.configuration.get_config().set('build_config', 'temporary_base_directory', temporary_base_directory) + luigi.configuration.get_config().set( + "build_config", "temporary_base_directory", temporary_base_directory + ) if cache_directory is not None: - luigi.configuration.get_config().set('build_config', 'cache_directory', cache_directory) + luigi.configuration.get_config().set( + "build_config", "cache_directory", cache_directory + ) if build_name is not None: - luigi.configuration.get_config().set('build_config', 'build_name', build_name) - luigi.configuration.get_config().set('build_config', 'log_build_context_content', str(log_build_context_content)) + luigi.configuration.get_config().set("build_config", "build_name", build_name) + luigi.configuration.get_config().set( + "build_config", "log_build_context_content", str(log_build_context_content) + ) def set_output_directory(output_directory): if output_directory is not None: - luigi.configuration.get_config().set('build_config', 'output_directory', output_directory) - - -def set_docker_repository_config(docker_password: Optional[str], docker_repository_name: Optional[str], - docker_username: Optional[str], tag_prefix: str, kind: str): - config_class = f'{kind}_docker_repository_config' - luigi.configuration.get_config().set(config_class, 'tag_prefix', tag_prefix) + luigi.configuration.get_config().set( + "build_config", "output_directory", output_directory + ) + + +def set_docker_repository_config( + docker_password: Optional[str], + docker_repository_name: Optional[str], + docker_username: Optional[str], + tag_prefix: str, + kind: str, +): + config_class = f"{kind}_docker_repository_config" + luigi.configuration.get_config().set(config_class, "tag_prefix", tag_prefix) if docker_repository_name is not None: - luigi.configuration.get_config().set(config_class, 'repository_name', docker_repository_name) + luigi.configuration.get_config().set( + config_class, "repository_name", docker_repository_name + ) password_environment_variable_name = f"{kind.upper()}_DOCKER_PASSWORD" if docker_username is not None: - luigi.configuration.get_config().set(config_class, 'username', docker_username) + luigi.configuration.get_config().set(config_class, "username", docker_username) if docker_password is not None: - luigi.configuration.get_config().set(config_class, 'password', docker_password) + luigi.configuration.get_config().set( + config_class, "password", docker_password + ) elif password_environment_variable_name in os.environ: - print(f"Using password from environment variable {password_environment_variable_name}") + print( + f"Using password from environment variable {password_environment_variable_name}" + ) password = os.environ[password_environment_variable_name] - luigi.configuration.get_config().set(config_class, 'password', password) + luigi.configuration.get_config().set(config_class, "password", password) else: - password = getpass.getpass(f"{kind.capitalize()} Docker Registry Password for User %s:" % docker_username) - luigi.configuration.get_config().set(config_class, 'password', password) + password = getpass.getpass( + f"{kind.capitalize()} Docker Registry Password for User %s:" + % docker_username + ) + luigi.configuration.get_config().set(config_class, "password", password) def import_build_steps(flavor_path: Tuple[str, ...]): @@ -101,10 +147,13 @@ def import_build_steps(flavor_path: Tuple[str, ...]): # to create the scheduler and worker processes, such that the imported classes available # in the scheduler and worker processes import importlib.util + for path in flavor_path: path_to_build_steps = Path(path).joinpath("flavor_base/build_steps.py") module_name_for_build_steps = extract_modulename_for_build_steps(path) - spec = importlib.util.spec_from_file_location(module_name_for_build_steps, path_to_build_steps) + spec = importlib.util.spec_from_file_location( + module_name_for_build_steps, path_to_build_steps + ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -112,52 +161,64 @@ def import_build_steps(flavor_path: Tuple[str, ...]): def generate_root_task(task_class, *args, **kwargs) -> DependencyLoggerBaseTask: job_counter = JobCounterSingleton().get_next_value() - strftime = datetime.now().strftime('%Y_%m_%d_%H_%M_%S') + strftime = datetime.now().strftime("%Y_%m_%d_%H_%M_%S") params = {"job_id": f"{strftime}_{job_counter}_{task_class.__name__}"} params.update(kwargs) return task_class(**params) -def run_task(task_creator: Callable[[], DependencyLoggerBaseTask], workers: int = 2, - task_dependencies_dot_file: Optional[str] = None, - log_level: Optional[str] = None, use_job_specific_log_file: bool = False) \ - -> Any: +def run_task( + task_creator: Callable[[], DependencyLoggerBaseTask], + workers: int = 2, + task_dependencies_dot_file: Optional[str] = None, + log_level: Optional[str] = None, + use_job_specific_log_file: bool = False, +) -> Any: setup_worker() task = task_creator() success = False log_file_path = get_log_path(task.job_id) try: - no_scheduling_errors = \ - _run_task_with_logging_config( - task, log_file_path, log_level, use_job_specific_log_file, workers) + no_scheduling_errors = _run_task_with_logging_config( + task, log_file_path, log_level, use_job_specific_log_file, workers + ) success = not task.failed_target.exists() and no_scheduling_errors - return _handle_task_result(no_scheduling_errors, success, task, - task_dependencies_dot_file) + return _handle_task_result( + no_scheduling_errors, success, task, task_dependencies_dot_file + ) except BaseException as e: logging.error("Going to abort the task %s" % task) raise e finally: if use_job_specific_log_file: logging.info( - f"The detailed log of the integration-test-docker-environment can be found at: {log_file_path}") + f"The detailed log of the integration-test-docker-environment can be found at: {log_file_path}" + ) task.cleanup(success) + def _run_task_with_logging_config( - task: DependencyLoggerBaseTask, - log_file_path: Path, - log_level: Optional[str], - use_job_specific_log_file: bool, - workers: int) -> bool: - with _configure_logging(log_file_path, log_level, use_job_specific_log_file) as run_kwargs: - no_scheduling_errors = luigi.build([task], workers=workers, - local_scheduler=True, **run_kwargs) + task: DependencyLoggerBaseTask, + log_file_path: Path, + log_level: Optional[str], + use_job_specific_log_file: bool, + workers: int, +) -> bool: + with _configure_logging( + log_file_path, log_level, use_job_specific_log_file + ) as run_kwargs: + no_scheduling_errors = luigi.build( + [task], workers=workers, local_scheduler=True, **run_kwargs + ) return no_scheduling_errors + def _handle_task_result( - no_scheduling_errors: bool, - success: bool, - task: DependencyLoggerBaseTask, - task_dependencies_dot_file: Optional[str]) -> Any: + no_scheduling_errors: bool, + success: bool, + task: DependencyLoggerBaseTask, + task_dependencies_dot_file: Optional[str], +) -> Any: generate_graph_from_task_dependencies(task, task_dependencies_dot_file) if success: return task.get_result() @@ -165,81 +226,100 @@ def _handle_task_result( logging.error(f"Task {task} failed. failed target exists.") task_failures = list(task.collect_failures().keys()) raise TaskRuntimeError( - msg=f"Task {task} (or any of it's subtasks) failed.", - inner=task_failures) \ - from TaskFailures(inner=task_failures) + msg=f"Task {task} (or any of it's subtasks) failed.", inner=task_failures + ) from TaskFailures(inner=task_failures) elif not no_scheduling_errors: logging.error(f"Task {task} failed. : luigi reported a scheduling error.") - raise TaskRuntimeError(msg=f"Task {task} failed. reason: luigi reported a scheduling error.") + raise TaskRuntimeError( + msg=f"Task {task} failed. reason: luigi reported a scheduling error." + ) @contextlib.contextmanager def _configure_logging( - log_file_path: Path, - log_level: Optional[str], - use_job_specific_log_file: bool) -> Iterator[Dict[str, str]]: - with get_luigi_log_config(log_file_target=log_file_path, - log_level=log_level, - use_job_specific_log_file=use_job_specific_log_file) as luigi_config: + log_file_path: Path, log_level: Optional[str], use_job_specific_log_file: bool +) -> Iterator[Dict[str, str]]: + with get_luigi_log_config( + log_file_target=log_file_path, + log_level=log_level, + use_job_specific_log_file=use_job_specific_log_file, + ) as luigi_config: no_configure_logging, run_kwargs = _configure_logging_parameter( log_level=log_level, luigi_config=luigi_config, - use_job_specific_log_file=use_job_specific_log_file) + use_job_specific_log_file=use_job_specific_log_file, + ) # We need to set InterfaceLogging._configured to false, # because otherwise luigi doesn't accept the new config. InterfaceLogging._configured = False - luigi.configuration.get_config().set('core', 'no_configure_logging', str(no_configure_logging)) + luigi.configuration.get_config().set( + "core", "no_configure_logging", str(no_configure_logging) + ) with warnings.catch_warnings(): # This filter is necessary, because luigi uses the config no_configure_logging, # but doesn't define it, which seems to be a bug in luigi - warnings.filterwarnings(action="ignore", - category=UnconsumedParameterWarning, - message=".*no_configure_logging.*") + warnings.filterwarnings( + action="ignore", + category=UnconsumedParameterWarning, + message=".*no_configure_logging.*", + ) yield run_kwargs -def _configure_logging_parameter(log_level: Optional[str], luigi_config: Path, use_job_specific_log_file: bool) \ - -> Tuple[bool, Dict[str, str]]: +def _configure_logging_parameter( + log_level: Optional[str], luigi_config: Path, use_job_specific_log_file: bool +) -> Tuple[bool, Dict[str, str]]: if use_job_specific_log_file: no_configure_logging = False - run_kwargs = {"logging_conf_file": f'{luigi_config}'} + run_kwargs = {"logging_conf_file": f"{luigi_config}"} else: no_configure_logging = True run_kwargs = {} return no_configure_logging, run_kwargs -def generate_graph_from_task_dependencies(task: DependencyLoggerBaseTask, task_dependencies_dot_file: Optional[str]): +def generate_graph_from_task_dependencies( + task: DependencyLoggerBaseTask, task_dependencies_dot_file: Optional[str] +): if task_dependencies_dot_file is not None: print(f"Generate Task Dependency Graph to {task_dependencies_dot_file}") print() dependencies = collect_dependencies(task) g = DiGraph() for dependency in dependencies: - g.add_node(dependency.source.formatted_id, label=dependency.source.formatted_representation) - g.add_node(dependency.target.formatted_id, label=dependency.target.formatted_representation) - g.add_edge(dependency.source.formatted_id, dependency.target.formatted_id, - dependency=dependency.formatted, - label=f"\"type={dependency.type}, index={dependency.index}\"") + g.add_node( + dependency.source.formatted_id, + label=dependency.source.formatted_representation, + ) + g.add_node( + dependency.target.formatted_id, + label=dependency.target.formatted_representation, + ) + g.add_edge( + dependency.source.formatted_id, + dependency.target.formatted_id, + dependency=dependency.formatted, + label=f'"type={dependency.type}, index={dependency.index}"', + ) networkx.nx_pydot.write_dot(g, task_dependencies_dot_file) def collect_dependencies(task: DependencyLoggerBaseTask) -> Set[TaskDependency]: - dependencies : Set[TaskDependency] = set() + dependencies: Set[TaskDependency] = set() for root, directories, files in os.walk(task._get_dependencies_path_for_job()): for file in files: file_path = Path(root).joinpath(file) with open(file_path) as f: for line in f.readlines(): - task_dependency : TaskDependency = TaskDependency.from_json(line) # type: ignore + task_dependency: TaskDependency = TaskDependency.from_json(line) # type: ignore if task_dependency.state == DependencyState.requested.name: dependencies.add(task_dependency) return dependencies def setup_worker(): - luigi.configuration.get_config().set('worker', 'wait_interval', str(0.1)) - luigi.configuration.get_config().set('worker', 'wait_jitter', str(0.5)) + luigi.configuration.get_config().set("worker", "wait_interval", str(0.1)) + luigi.configuration.get_config().set("worker", "wait_jitter", str(0.5)) def add_options(options): diff --git a/exasol_integration_test_docker_environment/lib/api/push_test_container.py b/exasol_integration_test_docker_environment/lib/api/push_test_container.py index 3eb5e5231..ba1423b0e 100644 --- a/exasol_integration_test_docker_environment/lib/api/push_test_container.py +++ b/exasol_integration_test_docker_environment/lib/api/push_test_container.py @@ -1,67 +1,101 @@ -from typing import Tuple, Optional +from typing import ( + Optional, + Tuple, +) -from exasol_integration_test_docker_environment.lib.api.common import set_docker_repository_config, \ - run_task, set_build_config, generate_root_task, no_cli_function -from exasol_integration_test_docker_environment.cli.options.docker_repository_options import \ - DEFAULT_DOCKER_REPOSITORY_NAME -from exasol_integration_test_docker_environment.cli.options.system_options import DEFAULT_OUTPUT_DIRECTORY -from exasol_integration_test_docker_environment.lib.data.test_container_content_description import \ - TestContainerContentDescription -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo -from exasol_integration_test_docker_environment.lib.test_environment.analyze_test_container import \ - AnalyzeTestContainer, DockerTestContainerPush +from exasol_integration_test_docker_environment.cli.options.docker_repository_options import ( + DEFAULT_DOCKER_REPOSITORY_NAME, +) +from exasol_integration_test_docker_environment.cli.options.system_options import ( + DEFAULT_OUTPUT_DIRECTORY, +) +from exasol_integration_test_docker_environment.lib.api.common import ( + generate_root_task, + no_cli_function, + run_task, + set_build_config, + set_docker_repository_config, +) +from exasol_integration_test_docker_environment.lib.data.test_container_content_description import ( + TestContainerContentDescription, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, +) +from exasol_integration_test_docker_environment.lib.test_environment.analyze_test_container import ( + AnalyzeTestContainer, + DockerTestContainerPush, +) @no_cli_function def push_test_container( - test_container_content: TestContainerContentDescription, - force_push: bool = False, - push_all: bool = False, - force_rebuild: bool = False, - force_rebuild_from: Tuple[str, ...] = tuple(), - force_pull: bool = False, - output_directory: str = DEFAULT_OUTPUT_DIRECTORY, - temporary_base_directory: str = "/tmp", - log_build_context_content: bool = False, - cache_directory: Optional[str] = None, - build_name: Optional[str] = None, - source_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, - source_docker_tag_prefix: str = '', - source_docker_username: Optional[str] = None, - source_docker_password: Optional[str] = None, - target_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, - target_docker_tag_prefix: str = '', - target_docker_username: Optional[str] = None, - target_docker_password: Optional[str] = None, - workers: int = 5, - task_dependencies_dot_file: Optional[str] = None, - log_level: Optional[str] = None, - use_job_specific_log_file: bool = False + test_container_content: TestContainerContentDescription, + force_push: bool = False, + push_all: bool = False, + force_rebuild: bool = False, + force_rebuild_from: Tuple[str, ...] = tuple(), + force_pull: bool = False, + output_directory: str = DEFAULT_OUTPUT_DIRECTORY, + temporary_base_directory: str = "/tmp", + log_build_context_content: bool = False, + cache_directory: Optional[str] = None, + build_name: Optional[str] = None, + source_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, + source_docker_tag_prefix: str = "", + source_docker_username: Optional[str] = None, + source_docker_password: Optional[str] = None, + target_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, + target_docker_tag_prefix: str = "", + target_docker_username: Optional[str] = None, + target_docker_password: Optional[str] = None, + workers: int = 5, + task_dependencies_dot_file: Optional[str] = None, + log_level: Optional[str] = None, + use_job_specific_log_file: bool = False, ) -> ImageInfo: """ This function pushes all stages of the test container for the test environment. If the stages do not exist locally, the system will build or pull them before the push. """ - set_build_config(force_rebuild, - force_rebuild_from, - force_pull, - log_build_context_content, - output_directory, - temporary_base_directory, - cache_directory, - build_name) + set_build_config( + force_rebuild, + force_rebuild_from, + force_pull, + log_build_context_content, + output_directory, + temporary_base_directory, + cache_directory, + build_name, + ) # Use AnalyzeTestContainer to ensure that all luigi processes got it loaded analyze_task = AnalyzeTestContainer.__class__.__name__ - set_docker_repository_config(source_docker_password, source_docker_repository_name, source_docker_username, - source_docker_tag_prefix, "source") - set_docker_repository_config(target_docker_password, target_docker_repository_name, target_docker_username, - target_docker_tag_prefix, "target") - task_creator = lambda: generate_root_task(task_class=DockerTestContainerPush, - test_container_content=test_container_content, - force_push=force_push, - push_all=push_all) - image_infos = run_task(task_creator, workers, task_dependencies_dot_file, - log_level=log_level, - use_job_specific_log_file=use_job_specific_log_file) + set_docker_repository_config( + source_docker_password, + source_docker_repository_name, + source_docker_username, + source_docker_tag_prefix, + "source", + ) + set_docker_repository_config( + target_docker_password, + target_docker_repository_name, + target_docker_username, + target_docker_tag_prefix, + "target", + ) + task_creator = lambda: generate_root_task( + task_class=DockerTestContainerPush, + test_container_content=test_container_content, + force_push=force_push, + push_all=push_all, + ) + image_infos = run_task( + task_creator, + workers, + task_dependencies_dot_file, + log_level=log_level, + use_job_specific_log_file=use_job_specific_log_file, + ) return image_infos[0] diff --git a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py index 28e57a41f..d8a13e4ec 100644 --- a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment.py @@ -1,67 +1,93 @@ import functools -from typing import Tuple, Optional, Callable, Any +from typing import ( + Any, + Callable, + Optional, + Tuple, +) + import humanfriendly +from exasol_integration_test_docker_environment.cli.options.docker_repository_options import ( + DEFAULT_DOCKER_REPOSITORY_NAME, +) +from exasol_integration_test_docker_environment.cli.options.system_options import ( + DEFAULT_OUTPUT_DIRECTORY, +) +from exasol_integration_test_docker_environment.cli.options.test_environment_options import ( + LATEST_DB_VERSION, +) +from exasol_integration_test_docker_environment.lib.api.api_errors import ( + ArgumentConstraintError, +) from exasol_integration_test_docker_environment.lib.api.common import ( + cli_function, + generate_root_task, + run_task, set_build_config, set_docker_repository_config, - run_task, - generate_root_task, - cli_function, ) -from exasol_integration_test_docker_environment.cli.options.docker_repository_options import \ - DEFAULT_DOCKER_REPOSITORY_NAME -from exasol_integration_test_docker_environment.cli.options.system_options import DEFAULT_OUTPUT_DIRECTORY -from exasol_integration_test_docker_environment.cli.options.test_environment_options import LATEST_DB_VERSION -from exasol_integration_test_docker_environment.lib.api.api_errors import ArgumentConstraintError -from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo -from exasol_integration_test_docker_environment.lib.docker.container.utils import remove_docker_container -from exasol_integration_test_docker_environment.lib.docker.volumes.utils import remove_docker_volumes -from exasol_integration_test_docker_environment.lib.docker.networks.utils import remove_docker_networks -from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_environment_with_docker_db import \ - SpawnTestEnvironmentWithDockerDB -from exasol_integration_test_docker_environment \ - .lib.test_environment.parameter \ - .docker_db_test_environment_parameter import DbOsAccess +from exasol_integration_test_docker_environment.lib.data.environment_info import ( + EnvironmentInfo, +) +from exasol_integration_test_docker_environment.lib.docker.container.utils import ( + remove_docker_container, +) +from exasol_integration_test_docker_environment.lib.docker.networks.utils import ( + remove_docker_networks, +) +from exasol_integration_test_docker_environment.lib.docker.volumes.utils import ( + remove_docker_volumes, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter import ( + DbOsAccess, +) +from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_environment_with_docker_db import ( + SpawnTestEnvironmentWithDockerDB, +) def _cleanup(environment_info: EnvironmentInfo) -> None: if environment_info.database_info.container_info is not None: - remove_docker_container([environment_info.database_info.container_info.container_name]) + remove_docker_container( + [environment_info.database_info.container_info.container_name] + ) if environment_info.database_info.container_info.volume_name is not None: - remove_docker_volumes([environment_info.database_info.container_info.volume_name]) + remove_docker_volumes( + [environment_info.database_info.container_info.volume_name] + ) remove_docker_networks([environment_info.network_info.network_name]) @cli_function def spawn_test_environment( - environment_name: str, - database_port_forward: Optional[int] = None, - bucketfs_port_forward: Optional[int] = None, - ssh_port_forward: Optional[int] = None, - db_mem_size: str = "2 GiB", - db_disk_size: str = "2 GiB", - nameserver: Tuple[str, ...] = tuple(), - docker_runtime: Optional[str] = None, - docker_db_image_version: str = LATEST_DB_VERSION, - docker_db_image_name: str = "exasol/docker-db", - db_os_access: Optional[str] = "DOCKER_EXEC", - create_certificates: bool = False, - additional_db_parameter: Tuple[str, ...] = tuple(), - source_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, - source_docker_tag_prefix: str = '', - source_docker_username: Optional[str] = None, - source_docker_password: Optional[str] = None, - target_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, - target_docker_tag_prefix: str = '', - target_docker_username: Optional[str] = None, - target_docker_password: Optional[str] = None, - output_directory: str = DEFAULT_OUTPUT_DIRECTORY, - temporary_base_directory: str = "/tmp", - workers: int = 5, - task_dependencies_dot_file: Optional[str] = None, - log_level: Optional[str] = None, - use_job_specific_log_file: bool = False + environment_name: str, + database_port_forward: Optional[int] = None, + bucketfs_port_forward: Optional[int] = None, + ssh_port_forward: Optional[int] = None, + db_mem_size: str = "2 GiB", + db_disk_size: str = "2 GiB", + nameserver: Tuple[str, ...] = tuple(), + docker_runtime: Optional[str] = None, + docker_db_image_version: str = LATEST_DB_VERSION, + docker_db_image_name: str = "exasol/docker-db", + db_os_access: Optional[str] = "DOCKER_EXEC", + create_certificates: bool = False, + additional_db_parameter: Tuple[str, ...] = tuple(), + source_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, + source_docker_tag_prefix: str = "", + source_docker_username: Optional[str] = None, + source_docker_password: Optional[str] = None, + target_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, + target_docker_tag_prefix: str = "", + target_docker_username: Optional[str] = None, + target_docker_password: Optional[str] = None, + output_directory: str = DEFAULT_OUTPUT_DIRECTORY, + temporary_base_directory: str = "/tmp", + workers: int = 5, + task_dependencies_dot_file: Optional[str] = None, + log_level: Optional[str] = None, + use_job_specific_log_file: bool = False, ) -> Tuple[EnvironmentInfo, Callable[[], None]]: """ This function spawns a test environment with a docker-db container and a connected test-container. @@ -71,6 +97,7 @@ def spawn_test_environment( raises: TaskRuntimeError if spawning the test environment fails """ + def str_or_none(x: Any) -> Optional[str]: return str(x) if x is not None else None @@ -80,44 +107,63 @@ def str_or_none(x: Any) -> Optional[str]: parsed_db_disk_size = humanfriendly.parse_size(db_disk_size) if parsed_db_disk_size < humanfriendly.parse_size("100 MiB"): raise ArgumentConstraintError("db_disk_size", "needs to be at least 100 MiB") - db_os_access_value = DbOsAccess[db_os_access] if db_os_access else DbOsAccess.DOCKER_EXEC + db_os_access_value = ( + DbOsAccess[db_os_access] if db_os_access else DbOsAccess.DOCKER_EXEC + ) - set_build_config(False, - tuple(), - False, - False, - output_directory, - temporary_base_directory, - None, - None) - set_docker_repository_config(source_docker_password, source_docker_repository_name, source_docker_username, - source_docker_tag_prefix, "source") - set_docker_repository_config(target_docker_password, target_docker_repository_name, target_docker_username, - target_docker_tag_prefix, "target") - task_creator = lambda: generate_root_task(task_class=SpawnTestEnvironmentWithDockerDB, - environment_name=environment_name, - database_port_forward=str_or_none(database_port_forward), - bucketfs_port_forward=str_or_none(bucketfs_port_forward), - ssh_port_forward=str_or_none(ssh_port_forward), - mem_size=db_mem_size, - disk_size=db_disk_size, - nameservers=nameserver, - docker_runtime=docker_runtime, - docker_db_image_version=docker_db_image_version, - docker_db_image_name=docker_db_image_name, - db_os_access=db_os_access_value, - db_user="sys", - db_password="exasol", - bucketfs_write_password="write", - no_test_container_cleanup_after_success=True, - no_test_container_cleanup_after_failure=False, - no_database_cleanup_after_success=True, - no_database_cleanup_after_failure=False, - create_certificates=create_certificates, - test_container_content=None, - additional_db_parameter=additional_db_parameter - ) - environment_info = run_task(task_creator, workers, task_dependencies_dot_file, - log_level=log_level, - use_job_specific_log_file=use_job_specific_log_file) + set_build_config( + False, + tuple(), + False, + False, + output_directory, + temporary_base_directory, + None, + None, + ) + set_docker_repository_config( + source_docker_password, + source_docker_repository_name, + source_docker_username, + source_docker_tag_prefix, + "source", + ) + set_docker_repository_config( + target_docker_password, + target_docker_repository_name, + target_docker_username, + target_docker_tag_prefix, + "target", + ) + task_creator = lambda: generate_root_task( + task_class=SpawnTestEnvironmentWithDockerDB, + environment_name=environment_name, + database_port_forward=str_or_none(database_port_forward), + bucketfs_port_forward=str_or_none(bucketfs_port_forward), + ssh_port_forward=str_or_none(ssh_port_forward), + mem_size=db_mem_size, + disk_size=db_disk_size, + nameservers=nameserver, + docker_runtime=docker_runtime, + docker_db_image_version=docker_db_image_version, + docker_db_image_name=docker_db_image_name, + db_os_access=db_os_access_value, + db_user="sys", + db_password="exasol", + bucketfs_write_password="write", + no_test_container_cleanup_after_success=True, + no_test_container_cleanup_after_failure=False, + no_database_cleanup_after_success=True, + no_database_cleanup_after_failure=False, + create_certificates=create_certificates, + test_container_content=None, + additional_db_parameter=additional_db_parameter, + ) + environment_info = run_task( + task_creator, + workers, + task_dependencies_dot_file, + log_level=log_level, + use_job_specific_log_file=use_job_specific_log_file, + ) return environment_info, functools.partial(_cleanup, environment_info) diff --git a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py index c75a87ada..02f4e822d 100644 --- a/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py +++ b/exasol_integration_test_docker_environment/lib/api/spawn_test_environment_with_test_container.py @@ -1,25 +1,54 @@ import functools -from typing import Tuple, Optional, Callable, Any +from typing import ( + Any, + Callable, + Optional, + Tuple, +) + import humanfriendly -from exasol_integration_test_docker_environment.lib.api.common import set_build_config, set_docker_repository_config, \ - run_task, generate_root_task, no_cli_function -from exasol_integration_test_docker_environment.cli.options.docker_repository_options import \ - DEFAULT_DOCKER_REPOSITORY_NAME -from exasol_integration_test_docker_environment.cli.options.system_options import DEFAULT_OUTPUT_DIRECTORY -from exasol_integration_test_docker_environment.cli.options.test_environment_options import LATEST_DB_VERSION -from exasol_integration_test_docker_environment.lib.api.api_errors import ArgumentConstraintError -from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo -from exasol_integration_test_docker_environment.lib.data.test_container_content_description import \ - TestContainerContentDescription -from exasol_integration_test_docker_environment.lib.docker.container.utils import remove_docker_container -from exasol_integration_test_docker_environment.lib.docker.volumes.utils import remove_docker_volumes -from exasol_integration_test_docker_environment.lib.docker.networks.utils import remove_docker_networks -from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_environment_with_docker_db import \ - SpawnTestEnvironmentWithDockerDB -from exasol_integration_test_docker_environment \ - .lib.test_environment.parameter \ - .docker_db_test_environment_parameter import DbOsAccess +from exasol_integration_test_docker_environment.cli.options.docker_repository_options import ( + DEFAULT_DOCKER_REPOSITORY_NAME, +) +from exasol_integration_test_docker_environment.cli.options.system_options import ( + DEFAULT_OUTPUT_DIRECTORY, +) +from exasol_integration_test_docker_environment.cli.options.test_environment_options import ( + LATEST_DB_VERSION, +) +from exasol_integration_test_docker_environment.lib.api.api_errors import ( + ArgumentConstraintError, +) +from exasol_integration_test_docker_environment.lib.api.common import ( + generate_root_task, + no_cli_function, + run_task, + set_build_config, + set_docker_repository_config, +) +from exasol_integration_test_docker_environment.lib.data.environment_info import ( + EnvironmentInfo, +) +from exasol_integration_test_docker_environment.lib.data.test_container_content_description import ( + TestContainerContentDescription, +) +from exasol_integration_test_docker_environment.lib.docker.container.utils import ( + remove_docker_container, +) +from exasol_integration_test_docker_environment.lib.docker.networks.utils import ( + remove_docker_networks, +) +from exasol_integration_test_docker_environment.lib.docker.volumes.utils import ( + remove_docker_volumes, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter import ( + DbOsAccess, +) +from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_environment_with_docker_db import ( + SpawnTestEnvironmentWithDockerDB, +) + def _cleanup(environment_info: EnvironmentInfo) -> None: if test_container_info := environment_info.test_container_info: @@ -32,38 +61,38 @@ def _cleanup(environment_info: EnvironmentInfo) -> None: remove_docker_networks([environment_info.network_info.network_name]) + @no_cli_function def spawn_test_environment_with_test_container( - environment_name: str, - test_container_content: TestContainerContentDescription, - database_port_forward: Optional[int] = None, - bucketfs_port_forward: Optional[int] = None, - ssh_port_forward: Optional[int] = None, - db_mem_size: str = "2 GiB", - db_disk_size: str = "2 GiB", - nameserver: Tuple[str, ...] = tuple(), - docker_runtime: Optional[str] = None, - docker_db_image_version: str = LATEST_DB_VERSION, - docker_db_image_name: str = "exasol/docker-db", - db_os_access: str = "DOCKER_EXEC", - create_certificates: bool = False, - additional_db_parameter: Tuple[str, ...] = tuple(), - source_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, - source_docker_tag_prefix: str = '', - source_docker_username: Optional[str] = None, - source_docker_password: Optional[str] = None, - target_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, - target_docker_tag_prefix: str = '', - target_docker_username: Optional[str] = None, - target_docker_password: Optional[str] = None, - output_directory: str = DEFAULT_OUTPUT_DIRECTORY, - temporary_base_directory: str = "/tmp", - workers: int = 5, - task_dependencies_dot_file: Optional[str] = None, - log_level: Optional[str] = None, - use_job_specific_log_file: bool = False -) \ - -> Tuple[EnvironmentInfo, Callable[[], None]]: + environment_name: str, + test_container_content: TestContainerContentDescription, + database_port_forward: Optional[int] = None, + bucketfs_port_forward: Optional[int] = None, + ssh_port_forward: Optional[int] = None, + db_mem_size: str = "2 GiB", + db_disk_size: str = "2 GiB", + nameserver: Tuple[str, ...] = tuple(), + docker_runtime: Optional[str] = None, + docker_db_image_version: str = LATEST_DB_VERSION, + docker_db_image_name: str = "exasol/docker-db", + db_os_access: str = "DOCKER_EXEC", + create_certificates: bool = False, + additional_db_parameter: Tuple[str, ...] = tuple(), + source_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, + source_docker_tag_prefix: str = "", + source_docker_username: Optional[str] = None, + source_docker_password: Optional[str] = None, + target_docker_repository_name: str = DEFAULT_DOCKER_REPOSITORY_NAME, + target_docker_tag_prefix: str = "", + target_docker_username: Optional[str] = None, + target_docker_password: Optional[str] = None, + output_directory: str = DEFAULT_OUTPUT_DIRECTORY, + temporary_base_directory: str = "/tmp", + workers: int = 5, + task_dependencies_dot_file: Optional[str] = None, + log_level: Optional[str] = None, + use_job_specific_log_file: bool = False, +) -> Tuple[EnvironmentInfo, Callable[[], None]]: """ This function spawns a test environment with a docker-db container and a connected test-container. The test-container is reachable by the database for output redirects of UDFs. @@ -72,50 +101,69 @@ def spawn_test_environment_with_test_container( raises: TaskRuntimeError if spawning the test environment fails """ + def str_or_none(x: Any) -> Optional[str]: return str(x) if x is not None else None + parsed_db_mem_size = humanfriendly.parse_size(db_mem_size) if parsed_db_mem_size < humanfriendly.parse_size("1 GiB"): raise ArgumentConstraintError("db_mem_size", "needs to be at least 1 GiB") parsed_db_disk_size = humanfriendly.parse_size(db_disk_size) if parsed_db_disk_size < humanfriendly.parse_size("100 MiB"): raise ArgumentConstraintError("db_disk_size", "needs to be at least 100 MiB") - set_build_config(False, - tuple(), - False, - False, - output_directory, - temporary_base_directory, - None, - None) - set_docker_repository_config(source_docker_password, source_docker_repository_name, source_docker_username, - source_docker_tag_prefix, "source") - set_docker_repository_config(target_docker_password, target_docker_repository_name, target_docker_username, - target_docker_tag_prefix, "target") - task_creator = lambda: generate_root_task(task_class=SpawnTestEnvironmentWithDockerDB, - environment_name=environment_name, - database_port_forward=str_or_none(database_port_forward), - bucketfs_port_forward=str_or_none(bucketfs_port_forward), - ssh_port_forward=str_or_none(ssh_port_forward), - mem_size=db_mem_size, - disk_size=db_disk_size, - nameservers=nameserver, - docker_runtime=docker_runtime, - docker_db_image_version=docker_db_image_version, - docker_db_image_name=docker_db_image_name, - db_os_access=DbOsAccess[db_os_access], - db_user="sys", - db_password="exasol", - bucketfs_write_password="write", - no_test_container_cleanup_after_success=True, - no_test_container_cleanup_after_failure=False, - no_database_cleanup_after_success=True, - no_database_cleanup_after_failure=False, - create_certificates=create_certificates, - test_container_content=test_container_content, - additional_db_parameter=additional_db_parameter - ) - environment_info = run_task(task_creator, workers, task_dependencies_dot_file, - log_level=log_level, - use_job_specific_log_file=use_job_specific_log_file) + set_build_config( + False, + tuple(), + False, + False, + output_directory, + temporary_base_directory, + None, + None, + ) + set_docker_repository_config( + source_docker_password, + source_docker_repository_name, + source_docker_username, + source_docker_tag_prefix, + "source", + ) + set_docker_repository_config( + target_docker_password, + target_docker_repository_name, + target_docker_username, + target_docker_tag_prefix, + "target", + ) + task_creator = lambda: generate_root_task( + task_class=SpawnTestEnvironmentWithDockerDB, + environment_name=environment_name, + database_port_forward=str_or_none(database_port_forward), + bucketfs_port_forward=str_or_none(bucketfs_port_forward), + ssh_port_forward=str_or_none(ssh_port_forward), + mem_size=db_mem_size, + disk_size=db_disk_size, + nameservers=nameserver, + docker_runtime=docker_runtime, + docker_db_image_version=docker_db_image_version, + docker_db_image_name=docker_db_image_name, + db_os_access=DbOsAccess[db_os_access], + db_user="sys", + db_password="exasol", + bucketfs_write_password="write", + no_test_container_cleanup_after_success=True, + no_test_container_cleanup_after_failure=False, + no_database_cleanup_after_success=True, + no_database_cleanup_after_failure=False, + create_certificates=create_certificates, + test_container_content=test_container_content, + additional_db_parameter=additional_db_parameter, + ) + environment_info = run_task( + task_creator, + workers, + task_dependencies_dot_file, + log_level=log_level, + use_job_specific_log_file=use_job_specific_log_file, + ) return environment_info, functools.partial(_cleanup, environment_info) diff --git a/exasol_integration_test_docker_environment/lib/base/abstract_task_future.py b/exasol_integration_test_docker_environment/lib/base/abstract_task_future.py index ce0769bc8..6a496d322 100644 --- a/exasol_integration_test_docker_environment/lib/base/abstract_task_future.py +++ b/exasol_integration_test_docker_environment/lib/base/abstract_task_future.py @@ -3,5 +3,4 @@ class AbstractTaskFuture(abc.ABC): @abc.abstractmethod - def get_output(self): - ... + def get_output(self): ... diff --git a/exasol_integration_test_docker_environment/lib/base/base_task.py b/exasol_integration_test_docker_environment/lib/base/base_task.py index ed840cd01..1a6978d7e 100644 --- a/exasol_integration_test_docker_environment/lib/base/base_task.py +++ b/exasol_integration_test_docker_environment/lib/base/base_task.py @@ -3,21 +3,43 @@ import logging import shutil from pathlib import Path -from typing import Dict, List, Generator, Any, Union, Set +from typing import ( + Any, + Dict, + Generator, + List, + Set, + Union, +) import luigi import six -from luigi import Task, util +from luigi import ( + Task, + util, +) from luigi.parameter import ParameterVisibility from luigi.task import TASK_ID_TRUNCATE_HASH -from exasol_integration_test_docker_environment.abstract_method_exception import AbstractMethodException -from exasol_integration_test_docker_environment.lib.base.abstract_task_future import AbstractTaskFuture -from exasol_integration_test_docker_environment.lib.base.pickle_target import PickleTarget -from exasol_integration_test_docker_environment.lib.base.task_logger_wrapper import TaskLoggerWrapper +from exasol_integration_test_docker_environment.abstract_method_exception import ( + AbstractMethodException, +) +from exasol_integration_test_docker_environment.lib.base.abstract_task_future import ( + AbstractTaskFuture, +) +from exasol_integration_test_docker_environment.lib.base.pickle_target import ( + PickleTarget, +) +from exasol_integration_test_docker_environment.lib.base.task_logger_wrapper import ( + TaskLoggerWrapper, +) from exasol_integration_test_docker_environment.lib.base.task_state import TaskState -from exasol_integration_test_docker_environment.lib.base.wrong_task_state_exception import WrongTaskStateException -from exasol_integration_test_docker_environment.lib.config.build_config import build_config +from exasol_integration_test_docker_environment.lib.base.wrong_task_state_exception import ( + WrongTaskStateException, +) +from exasol_integration_test_docker_environment.lib.config.build_config import ( + build_config, +) RETURN_TARGETS = "return_targets" @@ -57,7 +79,9 @@ def get_output(self) -> Any: self._outputs_cache = completion_target.read() return self._outputs_cache else: - raise WrongTaskStateException(self._current_task._task_state, "RequiresTaskFuture.read_outputs_dict") + raise WrongTaskStateException( + self._current_task._task_state, "RequiresTaskFuture.read_outputs_dict" + ) class RunTaskFuture(AbstractTaskFuture): @@ -85,26 +109,31 @@ def get_output(self) -> Any: class BaseTask(Task): - caller_output_path : List[str] = luigi.ListParameter([], significant=False, visibility=ParameterVisibility.HIDDEN) # type: ignore - job_id : str = luigi.Parameter() # type: ignore + caller_output_path: List[str] = luigi.ListParameter([], significant=False, visibility=ParameterVisibility.HIDDEN) # type: ignore + job_id: str = luigi.Parameter() # type: ignore def __init__(self, *args, **kwargs): self._registered_tasks = [] self._run_dependencies_tasks = [] self._task_state = TaskState.INIT super().__init__(*args, **kwargs) - self.task_id = self.task_id_str(self.get_task_family(), - self.get_parameter_as_string_dict()) + self.task_id = self.task_id_str( + self.get_task_family(), self.get_parameter_as_string_dict() + ) self.__hash = hash(self.task_id) self._init_non_pickle_attributes() self.register_required() self._task_state = TaskState.AFTER_INIT def _init_non_pickle_attributes(self): - logger = logging.getLogger(f'luigi-interface.{self.__class__.__name__}') + logger = logging.getLogger(f"luigi-interface.{self.__class__.__name__}") self.logger = TaskLoggerWrapper(logger, self.__repr__()) - self._run_dependencies_target = PickleTarget(path=self._get_tmp_path_for_run_dependencies()) - self._complete_target = PickleTarget(path=self._get_tmp_path_for_completion_target()) + self._run_dependencies_target = PickleTarget( + path=self._get_tmp_path_for_run_dependencies() + ) + self._complete_target = PickleTarget( + path=self._get_tmp_path_for_completion_target() + ) self._registered_return_target = None def __getstate__(self): @@ -128,10 +157,10 @@ def task_id_str(self, task_family, params): """ # task_id is a concatenation of task family, the first values of the first 3 parameters # sorted by parameter name and a md5hash of the family/parameters as a cananocalised json. - param_str = json.dumps(params, separators=(',', ':'), sort_keys=True) + param_str = json.dumps(params, separators=(",", ":"), sort_keys=True) hash_input = param_str - param_hash = hashlib.sha3_256(hash_input.encode('utf-8')).hexdigest() - return '{}_{}'.format(task_family, param_hash[:TASK_ID_TRUNCATE_HASH]) + param_hash = hashlib.sha3_256(hash_input.encode("utf-8")).hexdigest() + return f"{task_family}_{param_hash[:TASK_ID_TRUNCATE_HASH]}" def get_parameter_as_string_dict(self): """ @@ -139,21 +168,22 @@ def get_parameter_as_string_dict(self): """ params_str = {} params = dict(self.get_params()) - for param_name, param_value in six.iteritems(self.param_kwargs): - if (params[param_name].significant): + for param_name, param_value in self.param_kwargs.items(): + if params[param_name].significant: params_str[param_name] = params[param_name].serialize(param_value) return params_str def get_output_path(self) -> Path: - path = Path(self._get_output_path_for_job(), - "outputs", - Path(*self._extend_output_path())) + path = Path( + self._get_output_path_for_job(), + "outputs", + Path(*self._extend_output_path()), + ) path.mkdir(parents=True, exist_ok=True) return path def _get_output_path_for_job(self) -> Path: - return Path(build_config().output_directory, - "jobs", self.job_id) + return Path(build_config().output_directory, "jobs", self.job_id) def _extend_output_path(self): extension = self.extend_output_path() @@ -169,8 +199,7 @@ def _get_tmp_path_for_job(self) -> Path: return Path(self._get_output_path_for_job(), "temp") def _get_tmp_path_for_task(self) -> Path: - return Path(self._get_tmp_path_for_job(), - self.task_id) + return Path(self._get_tmp_path_for_job(), self.task_id) def _get_tmp_path_for_completion_target(self) -> Path: return Path(self._get_tmp_path_for_task(), COMPLETION_TARGET) @@ -206,7 +235,9 @@ def register_dependency(self, task: "BaseTask") -> RequiresTaskFuture: def register_dependencies(self, tasks): if isinstance(tasks, dict): - return {key: self.register_dependencies(task) for key, task in tasks.items()} + return { + key: self.register_dependencies(task) for key, task in tasks.items() + } elif isinstance(tasks, list): return [self.register_dependencies(task) for task in tasks] elif isinstance(tasks, BaseTask): @@ -216,7 +247,9 @@ def register_dependencies(self, tasks): def get_values_from_futures(self, futures): if isinstance(futures, dict): - return {key: self.get_values_from_futures(task) for key, task in futures.items()} + return { + key: self.get_values_from_futures(task) for key, task in futures.items() + } elif isinstance(futures, list): return [self.get_values_from_futures(task) for task in futures] elif isinstance(futures, AbstractTaskFuture): @@ -224,7 +257,9 @@ def get_values_from_futures(self, futures): else: return futures - def get_values_from_future(self, future: AbstractTaskFuture) -> Union[Any, Set[str]]: + def get_values_from_future( + self, future: AbstractTaskFuture + ) -> Union[Any, Set[str]]: return future.get_output() def requires(self): @@ -276,12 +311,18 @@ def _register_run_dependencies(self, tasks): elif isinstance(tasks, BaseTask): self._run_dependencies_tasks.append(tasks) - def _generate_run_task_futures(self, completion_targets: Union[Any]) -> Union[ - List[Any], Dict[Any, Any], RunTaskFuture, Any]: + def _generate_run_task_futures( + self, completion_targets: Union[Any] + ) -> Union[List[Any], Dict[Any, Any], RunTaskFuture, Any]: if isinstance(completion_targets, dict): - return {key: self._generate_run_task_futures(task) for key, task in completion_targets.items()} + return { + key: self._generate_run_task_futures(task) + for key, task in completion_targets.items() + } elif isinstance(completion_targets, list): - return [self._generate_run_task_futures(task) for task in completion_targets] + return [ + self._generate_run_task_futures(task) for task in completion_targets + ] elif isinstance(completion_targets, PickleTarget): return RunTaskFuture(completion_targets) else: @@ -320,11 +361,17 @@ def __repr__(self): repr_parts = [] param_objs = dict(params) for param_name, param_value in param_values: - if param_objs[param_name].significant and \ - param_objs[param_name].visibility == ParameterVisibility.PUBLIC: - repr_parts.append('%s=%s' % (param_name, param_objs[param_name].serialize(param_value))) - - task_str = '{}({})'.format(self.task_id, ', '.join(repr_parts)) + if ( + param_objs[param_name].significant + and param_objs[param_name].visibility == ParameterVisibility.PUBLIC + ): + repr_parts.append( + "{}={}".format( + param_name, param_objs[param_name].serialize(param_value) + ) + ) + + task_str = "{}({})".format(self.task_id, ", ".join(repr_parts)) return task_str @@ -349,7 +396,10 @@ def cleanup_internal(self, success: bool, cleanup_checklist: Set[str]): self.logger.debug("Cleaning up") if str(self) not in cleanup_checklist: cleanup_checklist.add(str(self)) - if self._task_state != TaskState.CLEANUP and self._task_state != TaskState.CLEANED: + if ( + self._task_state != TaskState.CLEANUP + and self._task_state != TaskState.CLEANED + ): self._task_state = TaskState.CLEANUP try: self.cleanup_child_task(success, cleanup_checklist) @@ -371,7 +421,9 @@ def cleanup_child_task(self, success: bool, cleanup_checklist: Set[str]): _run_dependencies_tasks_from_target = self._run_dependencies_target.read() else: _run_dependencies_tasks_from_target = [] - _run_dependencies_tasks = self._run_dependencies_tasks + _run_dependencies_tasks_from_target + _run_dependencies_tasks = ( + self._run_dependencies_tasks + _run_dependencies_tasks_from_target + ) reversed_run_dependencies_task_list = list(_run_dependencies_tasks) reversed_run_dependencies_task_list.reverse() for task in reversed_run_dependencies_task_list: diff --git a/exasol_integration_test_docker_environment/lib/base/db_os_executor.py b/exasol_integration_test_docker_environment/lib/base/db_os_executor.py index 50d8463a9..9671a3dd2 100644 --- a/exasol_integration_test_docker_environment/lib/base/db_os_executor.py +++ b/exasol_integration_test_docker_environment/lib/base/db_os_executor.py @@ -1,23 +1,32 @@ +import time from abc import abstractmethod -import fabric +from typing import ( + Optional, + Protocol, + runtime_checkable, +) + import docker -import time +import fabric from docker import DockerClient -from typing import Protocol, runtime_checkable, Optional -from docker.models.containers import Container, ExecResult -from exasol_integration_test_docker_environment \ - .lib.base.ssh_access import SshKey -from exasol_integration_test_docker_environment \ - .lib.data.database_info import DatabaseInfo -from exasol_integration_test_docker_environment.lib.docker \ - import ContextDockerClient +from docker.models.containers import ( + Container, + ExecResult, +) from paramiko.ssh_exception import NoValidConnectionsError +from exasol_integration_test_docker_environment.lib.base.ssh_access import SshKey +from exasol_integration_test_docker_environment.lib.data.database_info import ( + DatabaseInfo, +) +from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient + class DockerClientFactory: """ Create a Docker client. """ + def __init__(self, timeout: int = 100000): self._timeout = timeout @@ -37,18 +46,17 @@ class DbOsExecutor(Protocol): concrete implementations in sub-classes ``DockerExecutor`` and ``SshExecutor``. """ - def exec(self, cmd: str) -> ExecResult: - ... - def prepare(self): - ... + def exec(self, cmd: str) -> ExecResult: ... + + def prepare(self): ... class DockerExecutor(DbOsExecutor): def __init__(self, docker_client: DockerClient, container_name: str): self._client = docker_client self._container_name = container_name - self._container : Optional[Container] = None + self._container: Optional[Container] = None def __enter__(self): self._container = self._client.containers.get(self._container_name) @@ -78,13 +86,13 @@ class SshExecutor(DbOsExecutor): def __init__(self, connect_string: str, key_file: str): self._connect_string = connect_string self._key_file = key_file - self._connection : Optional[fabric.Connection] = None + self._connection: Optional[fabric.Connection] = None def __enter__(self): key = SshKey.read_from(self._key_file) self._connection = fabric.Connection( self._connect_string, - connect_kwargs={ "pkey": key.private }, + connect_kwargs={"pkey": key.private}, ) return self @@ -147,7 +155,6 @@ def executor(self) -> DbOsExecutor: return DockerExecutor(client, self._container_name) - class SshExecFactory(DbOsExecFactory): @classmethod def from_database_info(cls, info: DatabaseInfo): diff --git a/exasol_integration_test_docker_environment/lib/base/dependency_logger_base_task.py b/exasol_integration_test_docker_environment/lib/base/dependency_logger_base_task.py index b232f390f..ae45937ec 100644 --- a/exasol_integration_test_docker_environment/lib/base/dependency_logger_base_task.py +++ b/exasol_integration_test_docker_environment/lib/base/dependency_logger_base_task.py @@ -1,12 +1,21 @@ from pathlib import Path -from typing import List, Generator +from typing import ( + Generator, + List, +) import luigi from luigi import Task -from exasol_integration_test_docker_environment.lib.base.stoppable_base_task import StoppableBaseTask -from exasol_integration_test_docker_environment.lib.base.task_dependency import DependencyType, DependencyState, \ - TaskDependency, TaskDescription +from exasol_integration_test_docker_environment.lib.base.stoppable_base_task import ( + StoppableBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.task_dependency import ( + DependencyState, + DependencyType, + TaskDependency, + TaskDescription, +) class DependencyLoggerBaseTask(StoppableBaseTask): @@ -33,7 +42,9 @@ def handle_requires_value(self, tasks): dependencies_file=dependencies_file, dependency_type=DependencyType.requires, dependency_state=DependencyState.requested, - index=0, value=tasks) + index=0, + value=tasks, + ) return tasks else: return tasks @@ -42,18 +53,20 @@ def handle_requires_generator(self, tasks): dependency_path = self._get_dependencies_requires_path() if not dependency_path.exists(): with dependency_path.open("w") as dependencies_file: - result = list(self.write_dependencies_for_generator( - dependencies_file=dependencies_file, - task_generator=tasks, - dependency_type=DependencyType.requires)) + result = list( + self.write_dependencies_for_generator( + dependencies_file=dependencies_file, + task_generator=tasks, + dependency_type=DependencyType.requires, + ) + ) return result else: return tasks - def write_dependencies_for_generator(self, - dependencies_file, - task_generator, - dependency_type: DependencyType): + def write_dependencies_for_generator( + self, dependencies_file, task_generator, dependency_type: DependencyType + ): index = 0 try: element = next(task_generator) @@ -63,25 +76,30 @@ def write_dependencies_for_generator(self, dependency_type=dependency_type, dependency_state=DependencyState.requested, index=index, - value=element) + value=element, + ) result = yield element element = task_generator.send(result) index += 1 except StopIteration: pass - def write_dependency(self, - dependencies_file, - dependency_type: DependencyType, - dependency_state: DependencyState, - index: int, - value): + def write_dependency( + self, + dependencies_file, + dependency_type: DependencyType, + dependency_state: DependencyState, + index: int, + value, + ): for task in self.flatten_tasks(value): - dependency = TaskDependency(source=self.get_task_description(), - target=task.get_task_description(), - type=dependency_type, - index=index, - state=dependency_state) + dependency = TaskDependency( + source=self.get_task_description(), + target=task.get_task_description(), + type=dependency_type, + index=index, + state=dependency_state, + ) dependencies_file.write(f"{dependency.to_json()}") dependencies_file.write("\n") @@ -103,11 +121,15 @@ def run(self): yield from self.write_dependencies_for_generator( dependencies_file=dependencies_file, task_generator=task_generator, - dependency_type=DependencyType.dynamic) + dependency_type=DependencyType.dynamic, + ) def get_task_description(self) -> TaskDescription: return TaskDescription(id=self.task_id, representation=str(self)) def flatten_tasks(self, generator: Generator) -> List["DependencyLoggerBaseTask"]: - return [task for task in luigi.task.flatten(generator) - if isinstance(task, DependencyLoggerBaseTask)] + return [ + task + for task in luigi.task.flatten(generator) + if isinstance(task, DependencyLoggerBaseTask) + ] diff --git a/exasol_integration_test_docker_environment/lib/base/docker_base_task.py b/exasol_integration_test_docker_environment/lib/base/docker_base_task.py index f18fc42f1..577574575 100644 --- a/exasol_integration_test_docker_environment/lib/base/docker_base_task.py +++ b/exasol_integration_test_docker_environment/lib/base/docker_base_task.py @@ -1,5 +1,9 @@ -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask -from exasol_integration_test_docker_environment.lib.base.docker_parameter import DockerParameter +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.docker_parameter import ( + DockerParameter, +) from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient diff --git a/exasol_integration_test_docker_environment/lib/base/docker_parameter.py b/exasol_integration_test_docker_environment/lib/base/docker_parameter.py index 25a9b9308..f16888b4c 100644 --- a/exasol_integration_test_docker_environment/lib/base/docker_parameter.py +++ b/exasol_integration_test_docker_environment/lib/base/docker_parameter.py @@ -8,5 +8,7 @@ class DockerParameter(Config): Docker parameters used for Tasks accessing Docker client. """ - timeout = luigi.IntParameter(100000, significant=False, visibility=ParameterVisibility.PRIVATE) + timeout = luigi.IntParameter( + 100000, significant=False, visibility=ParameterVisibility.PRIVATE + ) no_cache = luigi.BoolParameter(False) diff --git a/exasol_integration_test_docker_environment/lib/base/flavor_task.py b/exasol_integration_test_docker_environment/lib/base/flavor_task.py index 940f00b44..f82312529 100644 --- a/exasol_integration_test_docker_environment/lib/base/flavor_task.py +++ b/exasol_integration_test_docker_environment/lib/base/flavor_task.py @@ -1,14 +1,22 @@ from pathlib import Path -from typing import Dict, Any, List +from typing import ( + Any, + Dict, + List, +) import luigi -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) class FlavorsBaseTask(DependencyLoggerBaseTask): - flavor_paths : List[str] = luigi.ListParameter() # type: ignore + flavor_paths: List[str] = luigi.ListParameter() # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -16,9 +24,15 @@ def __init__(self, *args, **kwargs): if not Path(flavor_path).is_dir(): raise OSError("Flavor path %s not a directory." % flavor_path) - def create_tasks_for_flavors_with_common_params(self, cls, **kwargs) -> Dict[str, Any]: - return {flavor_path: self._create_task_for_with_common_params(cls, flavor_path, kwargs) - for flavor_path in self.flavor_paths} + def create_tasks_for_flavors_with_common_params( + self, cls, **kwargs + ) -> Dict[str, Any]: + return { + flavor_path: self._create_task_for_with_common_params( + cls, flavor_path, kwargs + ) + for flavor_path in self.flavor_paths + } def _create_task_for_with_common_params(self, cls, flavor_path, kwargs): params = {**kwargs, "flavor_path": flavor_path} diff --git a/exasol_integration_test_docker_environment/lib/base/frozendict_to_dict.py b/exasol_integration_test_docker_environment/lib/base/frozendict_to_dict.py index 9e33b7de4..4972063d3 100644 --- a/exasol_integration_test_docker_environment/lib/base/frozendict_to_dict.py +++ b/exasol_integration_test_docker_environment/lib/base/frozendict_to_dict.py @@ -1,7 +1,7 @@ from collections.abc import Mapping -class FrozenDictToDict(): +class FrozenDictToDict: def convert(self, obj): if isinstance(obj, Mapping): return {k: self.convert(v) for k, v in obj.items()} diff --git a/exasol_integration_test_docker_environment/lib/base/info.py b/exasol_integration_test_docker_environment/lib/base/info.py index 9657fe208..ac60e94b6 100644 --- a/exasol_integration_test_docker_environment/lib/base/info.py +++ b/exasol_integration_test_docker_environment/lib/base/info.py @@ -2,13 +2,15 @@ import jsonpickle -from exasol_integration_test_docker_environment.lib.base.frozendict_to_dict import FrozenDictToDict +from exasol_integration_test_docker_environment.lib.base.frozendict_to_dict import ( + FrozenDictToDict, +) class Info: def to_json(self, indent=4): - jsonpickle.set_preferred_backend('simplejson') - jsonpickle.set_encoder_options('simplejson', sort_keys=True, indent=indent) + jsonpickle.set_preferred_backend("simplejson") + jsonpickle.set_encoder_options("simplejson", sort_keys=True, indent=indent) return jsonpickle.encode(self) def to_dict(self): @@ -21,7 +23,11 @@ def __repr__(self): def from_json(cls, json_string): loaded_object = jsonpickle.decode(json_string) if not isinstance(loaded_object, cls): - raise TypeError("Type %s of loaded object does not match %s" % (type(loaded_object), cls)) + raise TypeError( + "Type {} of loaded object does not match {}".format( + type(loaded_object), cls + ) + ) return loaded_object @classmethod diff --git a/exasol_integration_test_docker_environment/lib/base/json_pickle_parameter.py b/exasol_integration_test_docker_environment/lib/base/json_pickle_parameter.py index 2b20152f6..1a10a35cb 100644 --- a/exasol_integration_test_docker_environment/lib/base/json_pickle_parameter.py +++ b/exasol_integration_test_docker_environment/lib/base/json_pickle_parameter.py @@ -1,27 +1,54 @@ import jsonpickle -from luigi.parameter import Parameter, ParameterVisibility, _no_value +from luigi.parameter import ( + Parameter, + ParameterVisibility, + _no_value, +) class JsonPickleParameter(Parameter): - def __init__(self, cls, default=_no_value, is_global=False, significant=True, description=None, - config_path=None, positional=True, always_in_help=False, batch_method=None, is_optional=False, - visibility=ParameterVisibility.PUBLIC): - super().__init__(default, is_global, significant, description, - config_path, positional, always_in_help, - batch_method, visibility) + def __init__( + self, + cls, + default=_no_value, + is_global=False, + significant=True, + description=None, + config_path=None, + positional=True, + always_in_help=False, + batch_method=None, + is_optional=False, + visibility=ParameterVisibility.PUBLIC, + ): + super().__init__( + default, + is_global, + significant, + description, + config_path, + positional, + always_in_help, + batch_method, + visibility, + ) self.cls = cls self._is_optional = is_optional def parse(self, s): - jsonpickle.set_preferred_backend('simplejson') + jsonpickle.set_preferred_backend("simplejson") loaded_object = jsonpickle.decode(s) if self._is_optional and loaded_object is None: return None elif not isinstance(loaded_object, self.cls): - raise TypeError("Type %s of loaded object does not match %s" % (type(loaded_object), self.cls)) + raise TypeError( + "Type {} of loaded object does not match {}".format( + type(loaded_object), self.cls + ) + ) return loaded_object def serialize(self, x): - jsonpickle.set_preferred_backend('simplejson') + jsonpickle.set_preferred_backend("simplejson") return jsonpickle.encode(x) diff --git a/exasol_integration_test_docker_environment/lib/base/json_pickle_target.py b/exasol_integration_test_docker_environment/lib/base/json_pickle_target.py index 067d7e662..4c34d4526 100644 --- a/exasol_integration_test_docker_environment/lib/base/json_pickle_target.py +++ b/exasol_integration_test_docker_environment/lib/base/json_pickle_target.py @@ -1,5 +1,8 @@ from pathlib import Path -from typing import Any, Optional +from typing import ( + Any, + Optional, +) import jsonpickle from luigi import LocalTarget @@ -10,15 +13,15 @@ class JsonPickleTarget(LocalTarget): def __init__(self, path: Path, is_tmp: bool = False): super().__init__(path=str(path), is_tmp=is_tmp) - def write(self, obj:Any, indent: Optional[int] = None): - jsonpickle.set_preferred_backend('simplejson') - jsonpickle.set_encoder_options('simplejson', indent=indent) - json_str=jsonpickle.encode(obj) + def write(self, obj: Any, indent: Optional[int] = None): + jsonpickle.set_preferred_backend("simplejson") + jsonpickle.set_encoder_options("simplejson", indent=indent) + json_str = jsonpickle.encode(obj) with self.open("w") as f: f.write(json_str) def read(self): - jsonpickle.set_preferred_backend('simplejson') + jsonpickle.set_preferred_backend("simplejson") with self.open("r") as f: json_str = f.read() obj = jsonpickle.decode(json_str) diff --git a/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py b/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py index 359d1fbdd..13e318cad 100644 --- a/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py +++ b/exasol_integration_test_docker_environment/lib/base/luigi_log_config.py @@ -1,15 +1,23 @@ import contextlib +import logging import os import tempfile -from pathlib import Path -from typing import Optional, Callable, Generator, List, Any from dataclasses import dataclass +from pathlib import Path +from typing import ( + Any, + Callable, + Generator, + List, + Optional, +) import jinja2 -import logging from exasol_integration_test_docker_environment.lib import PACKAGE_NAME -from exasol_integration_test_docker_environment.lib.config.build_config import build_config +from exasol_integration_test_docker_environment.lib.config.build_config import ( + build_config, +) LOG_ENV_VARIABLE_NAME = "EXA_BUILD_LOG" @@ -42,18 +50,24 @@ def get_log_path(job_id: str) -> Path: log_path_dir.mkdir(parents=True, exist_ok=True) return log_path + @dataclass class LogInfoStorage: - level : int + level: int handlers: List[logging.Handler] filters: List[Any] propagate: bool + @contextlib.contextmanager def restore_logger(logger_creator: Callable[[], logging.Logger]): before_logger = logger_creator() - logger_info_before = LogInfoStorage(level=before_logger.level, handlers=list(before_logger.handlers), - filters=list(before_logger.filters), propagate=before_logger.propagate) + logger_info_before = LogInfoStorage( + level=before_logger.level, + handlers=list(before_logger.handlers), + filters=list(before_logger.filters), + propagate=before_logger.propagate, + ) yield after_logger = logger_creator() after_logger.level = logger_info_before.level @@ -63,9 +77,11 @@ def restore_logger(logger_creator: Callable[[], logging.Logger]): @contextlib.contextmanager -def get_luigi_log_config(log_file_target: Path, - use_job_specific_log_file: bool, - log_level: Optional[str] = None) -> Generator[Path, None, None]: +def get_luigi_log_config( + log_file_target: Path, + use_job_specific_log_file: bool, + log_level: Optional[str] = None, +) -> Generator[Path, None, None]: """ Yields a context manager containing the path of the log-config file. log_file_target contains the location of the log-file. @@ -74,20 +90,24 @@ def get_luigi_log_config(log_file_target: Path, """ if log_level is None and use_job_specific_log_file: log_level = logging.getLevelName(logging.WARNING) - env = jinja2.Environment(loader=jinja2.PackageLoader(PACKAGE_NAME), - autoescape=jinja2.select_autoescape()) + env = jinja2.Environment( + loader=jinja2.PackageLoader(PACKAGE_NAME), autoescape=jinja2.select_autoescape() + ) template = env.get_template("luigi_log.conf") - rendered_template = template.render(console_log_level=log_level, - log_file_target=str(log_file_target)) + rendered_template = template.render( + console_log_level=log_level, log_file_target=str(log_file_target) + ) with tempfile.TemporaryDirectory() as temp_dir: temp_luigi_conf_path = Path(temp_dir) / "luigi_log.conf" with open(temp_luigi_conf_path, "w") as f: f.write(rendered_template) - with restore_logger(logger_creator=lambda: logging.root), \ - restore_logger(logger_creator=lambda: logging.getLogger(LUIGI_INTERFACE_LOGGER)), \ - restore_logger(logger_creator=lambda: logging.getLogger(LUIGI_LOGGER)): + with restore_logger(logger_creator=lambda: logging.root), restore_logger( + logger_creator=lambda: logging.getLogger(LUIGI_INTERFACE_LOGGER) + ), restore_logger(logger_creator=lambda: logging.getLogger(LUIGI_LOGGER)): if log_level is not None and not use_job_specific_log_file: - logging.getLogger(LUIGI_INTERFACE_LOGGER).level = logging.getLevelName(log_level) + logging.getLogger(LUIGI_INTERFACE_LOGGER).level = logging.getLevelName( + log_level + ) logging.getLogger(LUIGI_LOGGER).level = logging.getLevelName(log_level) yield temp_luigi_conf_path diff --git a/exasol_integration_test_docker_environment/lib/base/pickle_target.py b/exasol_integration_test_docker_environment/lib/base/pickle_target.py index 4277cb018..da85e5e52 100644 --- a/exasol_integration_test_docker_environment/lib/base/pickle_target.py +++ b/exasol_integration_test_docker_environment/lib/base/pickle_target.py @@ -6,6 +6,7 @@ FORMAT = NopFormat() + class PickleTarget(LocalTarget): def __init__(self, path: Path, is_tmp: bool = False): diff --git a/exasol_integration_test_docker_environment/lib/base/ssh_access.py b/exasol_integration_test_docker_environment/lib/base/ssh_access.py index 0a792cab9..9a22fc715 100644 --- a/exasol_integration_test_docker_environment/lib/base/ssh_access.py +++ b/exasol_integration_test_docker_environment/lib/base/ssh_access.py @@ -1,12 +1,15 @@ import base64 import os -import paramiko import tempfile - -import portalocker from pathlib import Path from string import Template -from typing import Optional, Union +from typing import ( + Optional, + Union, +) + +import paramiko +import portalocker _LOCK_FILE = "$TMP/$MODULE-ssh-access.lock" _DEFAULT_CACHE_DIR = "$HOME/.cache/exasol/$MODULE" @@ -14,10 +17,9 @@ def _path(template: str) -> Path: return Path( - Template(template) - .substitute( + Template(template).substitute( TMP=tempfile.gettempdir(), - HOME=os.path.expanduser('~'), + HOME=os.path.expanduser("~"), MODULE="itde", ) ) @@ -62,20 +64,22 @@ class SshKey: ITDE uses python library portalocker to guarantee that the key files are accessed only by a single process at a time. """ + def __init__(self, private_key: paramiko.RSAKey): self.private = private_key - def write_private_key(self, path: str) -> 'SshKey': - self.private.write_private_key_file(path) # uses 0o600 = user r/w + def write_private_key(self, path: str) -> "SshKey": + self.private.write_private_key_file(path) # uses 0o600 = user r/w return self def public_key_as_string(self, comment="") -> str: b64 = base64.b64encode(self.private.asbytes()).decode("ascii") return f"ssh-rsa {b64} {comment}" - def write_public_key(self, path: str, comment="") -> 'SshKey': + def write_public_key(self, path: str, comment="") -> "SshKey": def opener(path, flags): return os.open(path, flags, 0o600) + content = self.public_key_as_string(comment) # Windows does not support kwarg mode=0o600 here with open(path, "w", opener=opener) as file: @@ -83,21 +87,21 @@ def opener(path, flags): return self @classmethod - def read_from(cls, private_key_file: Union[Path, str]) -> 'SshKey': - with open(private_key_file, "r") as file: + def read_from(cls, private_key_file: Union[Path, str]) -> "SshKey": + with open(private_key_file) as file: rsa_key = paramiko.RSAKey.from_private_key(file) return SshKey(rsa_key) @classmethod - def generate(cls) -> 'SshKey': + def generate(cls) -> "SshKey": rsa_key = paramiko.RSAKey.generate(bits=4096) return SshKey(rsa_key) @classmethod - def from_cache(cls, cache_directory: Optional[Path] = None) -> 'SshKey': + def from_cache(cls, cache_directory: Optional[Path] = None) -> "SshKey": cache = SshKeyCache(cache_directory) priv = cache.private_key - with portalocker.Lock(_path(_LOCK_FILE), 'wb', timeout=10) as fh: + with portalocker.Lock(_path(_LOCK_FILE), "wb", timeout=10) as fh: if priv.exists(): return cls.read_from(priv) # mode 0o700 = rwx permissions only for the current user diff --git a/exasol_integration_test_docker_environment/lib/base/still_running_logger.py b/exasol_integration_test_docker_environment/lib/base/still_running_logger.py index 37e4c7911..833547c4d 100644 --- a/exasol_integration_test_docker_environment/lib/base/still_running_logger.py +++ b/exasol_integration_test_docker_environment/lib/base/still_running_logger.py @@ -1,6 +1,9 @@ import threading import time -from datetime import datetime, timedelta +from datetime import ( + datetime, + timedelta, +) from exasol_integration_test_docker_environment.lib.config.log_config import log_config @@ -30,10 +33,14 @@ def __init__(self, logger, description): def log(self, message=None): - timedelta_between = timedelta(seconds=self._log_config.seconds_between_still_running_logs) + timedelta_between = timedelta( + seconds=self._log_config.seconds_between_still_running_logs + ) if self._previous_time + timedelta_between <= datetime.now(): if message is None: self._logger.info("Still running %s.", self._description) else: - self._logger.info("Still running %s. Message: %s", self._description, message) + self._logger.info( + "Still running %s. Message: %s", self._description, message + ) self._previous_time = datetime.now() diff --git a/exasol_integration_test_docker_environment/lib/base/stoppable_base_task.py b/exasol_integration_test_docker_environment/lib/base/stoppable_base_task.py index e7d61407a..773aa30bf 100644 --- a/exasol_integration_test_docker_environment/lib/base/stoppable_base_task.py +++ b/exasol_integration_test_docker_environment/lib/base/stoppable_base_task.py @@ -1,8 +1,14 @@ import traceback -from pathlib import Path -from typing import List, Dict from collections import OrderedDict -from exasol_integration_test_docker_environment.lib.base.timeable_base_task import TimeableBaseTask +from pathlib import Path +from typing import ( + Dict, + List, +) + +from exasol_integration_test_docker_environment.lib.base.timeable_base_task import ( + TimeableBaseTask, +) class StoppingFurtherExecution(Exception): @@ -34,8 +40,12 @@ def fail_if_any_task_failed(self): if self.failed_target.exists(): with self.failed_target.open("r") as f: failed_task = f.read() - self.logger.error("Task %s failed. Stopping further execution." % failed_task) - raise StoppingFurtherExecution("Task %s failed. Stopping further execution." % failed_task) + self.logger.error( + "Task %s failed. Stopping further execution." % failed_task + ) + raise StoppingFurtherExecution( + "Task %s failed. Stopping further execution." % failed_task + ) def handle_failure(self, exception, exception_tb): if not isinstance(exception, StoppingFurtherExecution): @@ -45,26 +55,28 @@ def handle_failure(self, exception, exception_tb): with self.failed_target.open("w") as f: f.write("%s" % self.task_id) - def collect_failures(self) -> Dict[str,None]: - failures : Dict[str, None] = OrderedDict() + def collect_failures(self) -> Dict[str, None]: + failures: Dict[str, None] = OrderedDict() failures.update(self.collect_failures_of_child_tasks()) if self.get_failure_log_path().exists(): with self.get_failure_log_path().open("r") as f: exception = f.read().strip() - prefix = ' ' + prefix = " " formatted_exception = prefix + prefix.join(exception.splitlines(True)) - failure_message = "- %s:\n%s" % (self.task_id, formatted_exception) + failure_message = f"- {self.task_id}:\n{formatted_exception}" failures[failure_message] = None return failures - def collect_failures_of_child_tasks(self) -> Dict[str,None]: - failures_of_child_tasks : Dict[str, None] = OrderedDict() + def collect_failures_of_child_tasks(self) -> Dict[str, None]: + failures_of_child_tasks: Dict[str, None] = OrderedDict() if self._run_dependencies_target.exists(): _run_dependencies_tasks_from_target = self._run_dependencies_target.read() else: _run_dependencies_tasks_from_target = [] - _run_dependencies_tasks = self._run_dependencies_tasks + _run_dependencies_tasks_from_target + _run_dependencies_tasks = ( + self._run_dependencies_tasks + _run_dependencies_tasks_from_target + ) reversed_run_dependencies_task_list = list(_run_dependencies_tasks) reversed_run_dependencies_task_list.reverse() for task in reversed_run_dependencies_task_list: diff --git a/exasol_integration_test_docker_environment/lib/base/task_dependency.py b/exasol_integration_test_docker_environment/lib/base/task_dependency.py index 03d8cd91f..11c472024 100644 --- a/exasol_integration_test_docker_environment/lib/base/task_dependency.py +++ b/exasol_integration_test_docker_environment/lib/base/task_dependency.py @@ -22,7 +22,7 @@ def formatted_id(self): class DependencyType(Enum): - requires = 1, + requires = (1,) dynamic = 2 @@ -33,8 +33,14 @@ class DependencyState(Enum): class TaskDependency: - def __init__(self, source: TaskDescription, target: TaskDescription, - type: DependencyType, index: int, state: DependencyState): + def __init__( + self, + source: TaskDescription, + target: TaskDescription, + type: DependencyType, + index: int, + state: DependencyState, + ): self.state = state.name self.index = index self.type = type.name @@ -42,15 +48,19 @@ def __init__(self, source: TaskDescription, target: TaskDescription, self.source = source def to_json(self): - jsonpickle.set_preferred_backend('simplejson') - jsonpickle.set_encoder_options('simplejson', sort_keys=True) + jsonpickle.set_preferred_backend("simplejson") + jsonpickle.set_encoder_options("simplejson", sort_keys=True) return jsonpickle.encode(self) @classmethod def from_json(cls, json_string: object) -> object: loaded_object = jsonpickle.decode(json_string) if not isinstance(loaded_object, cls): - raise TypeError("Type %s of loaded object does not match %s" % (type(loaded_object), cls)) + raise TypeError( + "Type {} of loaded object does not match {}".format( + type(loaded_object), cls + ) + ) return loaded_object @property @@ -60,8 +70,7 @@ def formatted(self): def __str__(self): return ( - f"TaskDependency(source={self.source}, " - f"target={self.target}, type={self.type}, " - f"index={self.index}, state={self.state})" - ) - + f"TaskDependency(source={self.source}, " + f"target={self.target}, type={self.type}, " + f"index={self.index}, state={self.state})" + ) diff --git a/exasol_integration_test_docker_environment/lib/base/task_logger_wrapper.py b/exasol_integration_test_docker_environment/lib/base/task_logger_wrapper.py index c354fe738..11d589af4 100644 --- a/exasol_integration_test_docker_environment/lib/base/task_logger_wrapper.py +++ b/exasol_integration_test_docker_environment/lib/base/task_logger_wrapper.py @@ -1,7 +1,7 @@ import logging -class TaskLoggerWrapper(): +class TaskLoggerWrapper: def __init__(self, logger: logging.Logger, task_id): self.task_id = task_id self.logger = logger diff --git a/exasol_integration_test_docker_environment/lib/base/task_state.py b/exasol_integration_test_docker_environment/lib/base/task_state.py index b6b9797b2..6cc7b3299 100644 --- a/exasol_integration_test_docker_environment/lib/base/task_state.py +++ b/exasol_integration_test_docker_environment/lib/base/task_state.py @@ -1,12 +1,15 @@ -from enum import Enum +from enum import ( + Enum, + auto, +) class TaskState(Enum): - NONE = 0, - INIT = 1, - AFTER_INIT =2, - RUN = 3, - FINISHED = 4, - CLEANUP = 5, - CLEANED = 6, - ERROR = 7 + NONE = auto() + INIT = auto() + AFTER_INIT = auto() + RUN = auto() + FINISHED = auto() + CLEANUP = auto() + CLEANED = auto() + ERROR = auto() diff --git a/exasol_integration_test_docker_environment/lib/base/wrong_task_state_exception.py b/exasol_integration_test_docker_environment/lib/base/wrong_task_state_exception.py index 130546e40..c56855cd2 100644 --- a/exasol_integration_test_docker_environment/lib/base/wrong_task_state_exception.py +++ b/exasol_integration_test_docker_environment/lib/base/wrong_task_state_exception.py @@ -4,4 +4,6 @@ class WrongTaskStateException(Exception): def __init__(self, task_state: TaskState, method: str): - super().__init__(f"Calling method {method} in task state {task_state} not allowed") + super().__init__( + f"Calling method {method} in task state {task_state} not allowed" + ) diff --git a/exasol_integration_test_docker_environment/lib/config/build_config.py b/exasol_integration_test_docker_environment/lib/config/build_config.py index d1076f324..4d05a6874 100644 --- a/exasol_integration_test_docker_environment/lib/config/build_config.py +++ b/exasol_integration_test_docker_environment/lib/config/build_config.py @@ -1,17 +1,23 @@ +from typing import ( + List, + Optional, +) + import luigi -from typing import List, Optional -from exasol_integration_test_docker_environment.cli.options.system_options import DEFAULT_OUTPUT_DIRECTORY +from exasol_integration_test_docker_environment.cli.options.system_options import ( + DEFAULT_OUTPUT_DIRECTORY, +) class build_config(luigi.Config): - force_pull : bool = luigi.BoolParameter(False) # type: ignore - force_load : bool = luigi.BoolParameter(False) # type: ignore - force_rebuild : bool = luigi.BoolParameter(False) # type: ignore - force_rebuild_from : List[str] = luigi.ListParameter([]) # type: ignore - log_build_context_content : bool = luigi.BoolParameter(False) # type: ignore + force_pull: bool = luigi.BoolParameter(False) # type: ignore + force_load: bool = luigi.BoolParameter(False) # type: ignore + force_rebuild: bool = luigi.BoolParameter(False) # type: ignore + force_rebuild_from: List[str] = luigi.ListParameter([]) # type: ignore + log_build_context_content: bool = luigi.BoolParameter(False) # type: ignore # keep_build_context = luigi.BoolParameter(False) - temporary_base_directory : Optional[str] = luigi.OptionalParameter(None) # type: ignore - output_directory : str = luigi.Parameter(DEFAULT_OUTPUT_DIRECTORY) # type: ignore - cache_directory : Optional[str] = luigi.OptionalParameter("") # type: ignore - build_name : Optional[str] = luigi.OptionalParameter("") # type: ignore + temporary_base_directory: Optional[str] = luigi.OptionalParameter(None) # type: ignore + output_directory: str = luigi.Parameter(DEFAULT_OUTPUT_DIRECTORY) # type: ignore + cache_directory: Optional[str] = luigi.OptionalParameter("") # type: ignore + build_name: Optional[str] = luigi.OptionalParameter("") # type: ignore diff --git a/exasol_integration_test_docker_environment/lib/config/docker_config.py b/exasol_integration_test_docker_environment/lib/config/docker_config.py index 99f5221a3..2efda228d 100644 --- a/exasol_integration_test_docker_environment/lib/config/docker_config.py +++ b/exasol_integration_test_docker_environment/lib/config/docker_config.py @@ -1,26 +1,38 @@ import luigi from luigi.parameter import ParameterVisibility -from exasol_integration_test_docker_environment.cli.options.docker_repository_options import DEFAULT_DOCKER_REPOSITORY_NAME +from exasol_integration_test_docker_environment.cli.options.docker_repository_options import ( + DEFAULT_DOCKER_REPOSITORY_NAME, +) class source_docker_repository_config(luigi.Config): repository_name = luigi.Parameter(DEFAULT_DOCKER_REPOSITORY_NAME) tag_prefix = luigi.Parameter("") - username = luigi.OptionalParameter(None, significant=False, visibility=ParameterVisibility.PRIVATE) - password = luigi.OptionalParameter(None, significant=False, visibility=ParameterVisibility.PRIVATE) + username = luigi.OptionalParameter( + None, significant=False, visibility=ParameterVisibility.PRIVATE + ) + password = luigi.OptionalParameter( + None, significant=False, visibility=ParameterVisibility.PRIVATE + ) class target_docker_repository_config(luigi.Config): repository_name = luigi.Parameter(DEFAULT_DOCKER_REPOSITORY_NAME) tag_prefix = luigi.Parameter("") - username = luigi.OptionalParameter(None, significant=False, visibility=ParameterVisibility.PRIVATE) - password = luigi.OptionalParameter(None, significant=False, visibility=ParameterVisibility.PRIVATE) + username = luigi.OptionalParameter( + None, significant=False, visibility=ParameterVisibility.PRIVATE + ) + password = luigi.OptionalParameter( + None, significant=False, visibility=ParameterVisibility.PRIVATE + ) class docker_build_arguments(luigi.Config): transparent = luigi.DictParameter(dict()) image_changing = luigi.DictParameter(dict()) - secret = luigi.DictParameter(dict(), - description="Will not be saved somewhere, but are also assumed to be transparent", - visibility=ParameterVisibility.PRIVATE) + secret = luigi.DictParameter( + dict(), + description="Will not be saved somewhere, but are also assumed to be transparent", + visibility=ParameterVisibility.PRIVATE, + ) diff --git a/exasol_integration_test_docker_environment/lib/config/log_config.py b/exasol_integration_test_docker_environment/lib/config/log_config.py index 1860a12a0..c201c64e2 100644 --- a/exasol_integration_test_docker_environment/lib/config/log_config.py +++ b/exasol_integration_test_docker_environment/lib/config/log_config.py @@ -9,7 +9,8 @@ class WriteLogFilesToConsole(Enum): class log_config(luigi.Config): - write_log_files_to_console = luigi.EnumParameter(enum=WriteLogFilesToConsole, - default=WriteLogFilesToConsole.only_error) + write_log_files_to_console = luigi.EnumParameter( + enum=WriteLogFilesToConsole, default=WriteLogFilesToConsole.only_error + ) log_task_is_still_running = luigi.BoolParameter(False) seconds_between_still_running_logs = luigi.IntParameter(60) diff --git a/exasol_integration_test_docker_environment/lib/data/container_info.py b/exasol_integration_test_docker_environment/lib/data/container_info.py index 8e19a82e9..8882580db 100644 --- a/exasol_integration_test_docker_environment/lib/data/container_info.py +++ b/exasol_integration_test_docker_environment/lib/data/container_info.py @@ -1,16 +1,24 @@ -from typing import List, Optional +from typing import ( + List, + Optional, +) from exasol_integration_test_docker_environment.lib.base.info import Info -from exasol_integration_test_docker_environment.lib.data.docker_network_info import DockerNetworkInfo +from exasol_integration_test_docker_environment.lib.data.docker_network_info import ( + DockerNetworkInfo, +) class ContainerInfo(Info): - def __init__(self, container_name: str, - ip_address: str, - network_aliases: List[str], - network_info: DockerNetworkInfo, - volume_name: Optional[str] = None): + def __init__( + self, + container_name: str, + ip_address: str, + network_aliases: List[str], + network_info: DockerNetworkInfo, + volume_name: Optional[str] = None, + ): self.network_aliases = network_aliases self.ip_address = ip_address self.network_info = network_info diff --git a/exasol_integration_test_docker_environment/lib/data/database_credentials.py b/exasol_integration_test_docker_environment/lib/data/database_credentials.py index 34a1d1a67..eb5e3f4f3 100644 --- a/exasol_integration_test_docker_environment/lib/data/database_credentials.py +++ b/exasol_integration_test_docker_environment/lib/data/database_credentials.py @@ -11,10 +11,14 @@ def __init__(self, db_user: str, db_password: str, bucketfs_write_password: str) class DatabaseCredentialsParameter(Config): db_user = luigi.Parameter() - db_password = luigi.Parameter(significant=False, - visibility=luigi.parameter.ParameterVisibility.HIDDEN) - bucketfs_write_password = luigi.Parameter(significant=False, - visibility=luigi.parameter.ParameterVisibility.HIDDEN) + db_password = luigi.Parameter( + significant=False, visibility=luigi.parameter.ParameterVisibility.HIDDEN + ) + bucketfs_write_password = luigi.Parameter( + significant=False, visibility=luigi.parameter.ParameterVisibility.HIDDEN + ) def get_database_credentials(self): - return DatabaseCredentials(self.db_user, self.db_password, self.bucketfs_write_password) + return DatabaseCredentials( + self.db_user, self.db_password, self.bucketfs_write_password + ) diff --git a/exasol_integration_test_docker_environment/lib/data/database_info.py b/exasol_integration_test_docker_environment/lib/data/database_info.py index 941adcb19..8f8b55e52 100644 --- a/exasol_integration_test_docker_environment/lib/data/database_info.py +++ b/exasol_integration_test_docker_environment/lib/data/database_info.py @@ -1,20 +1,22 @@ from typing import Optional from exasol_integration_test_docker_environment.lib.base.info import Info -from exasol_integration_test_docker_environment.lib.data.container_info import ContainerInfo -from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports +from exasol_integration_test_docker_environment.lib.data.container_info import ( + ContainerInfo, +) from exasol_integration_test_docker_environment.lib.data.ssh_info import SshInfo +from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports class DatabaseInfo(Info): def __init__( - self, - host: str, - ports: Ports, - reused: bool, - container_info: Optional[ContainerInfo] = None, - ssh_info: Optional[SshInfo] = None, - forwarded_ports: Optional[Ports] = None, + self, + host: str, + ports: Ports, + reused: bool, + container_info: Optional[ContainerInfo] = None, + ssh_info: Optional[SshInfo] = None, + forwarded_ports: Optional[Ports] = None, ): self.container_info = container_info self.ports = ports diff --git a/exasol_integration_test_docker_environment/lib/data/docker_network_info.py b/exasol_integration_test_docker_environment/lib/data/docker_network_info.py index f166d3c39..b7b057959 100644 --- a/exasol_integration_test_docker_environment/lib/data/docker_network_info.py +++ b/exasol_integration_test_docker_environment/lib/data/docker_network_info.py @@ -3,7 +3,9 @@ class DockerNetworkInfo(Info): - def __init__(self, network_name: str, subnet: str, gateway: str, reused: bool = False): + def __init__( + self, network_name: str, subnet: str, gateway: str, reused: bool = False + ): self.gateway = gateway self.subnet = subnet self.network_name = network_name diff --git a/exasol_integration_test_docker_environment/lib/data/docker_volume_info.py b/exasol_integration_test_docker_environment/lib/data/docker_volume_info.py index 4a77f69ff..6d3597560 100644 --- a/exasol_integration_test_docker_environment/lib/data/docker_volume_info.py +++ b/exasol_integration_test_docker_environment/lib/data/docker_volume_info.py @@ -4,10 +4,12 @@ class DockerVolumeInfo(Info): def __str__(self): - return f'DockerVolumeInfo: \n' \ - f'volume_name="{self.volume_name}"\n' \ - f'reused={self.reused}\n' \ - f'mount_point={self.mount_point}' + return ( + f"DockerVolumeInfo: \n" + f'volume_name="{self.volume_name}"\n' + f"reused={self.reused}\n" + f"mount_point={self.mount_point}" + ) def __init__(self, volume_name: str, mount_point: str, reused: bool = False): self.volume_name = volume_name diff --git a/exasol_integration_test_docker_environment/lib/data/environment_info.py b/exasol_integration_test_docker_environment/lib/data/environment_info.py index 0163f7938..1d7ea13a7 100644 --- a/exasol_integration_test_docker_environment/lib/data/environment_info.py +++ b/exasol_integration_test_docker_environment/lib/data/environment_info.py @@ -1,18 +1,27 @@ from typing import Optional from exasol_integration_test_docker_environment.lib.base.info import Info -from exasol_integration_test_docker_environment.lib.data.container_info import ContainerInfo -from exasol_integration_test_docker_environment.lib.data.database_info import DatabaseInfo -from exasol_integration_test_docker_environment.lib.data.docker_network_info import DockerNetworkInfo +from exasol_integration_test_docker_environment.lib.data.container_info import ( + ContainerInfo, +) +from exasol_integration_test_docker_environment.lib.data.database_info import ( + DatabaseInfo, +) +from exasol_integration_test_docker_environment.lib.data.docker_network_info import ( + DockerNetworkInfo, +) class EnvironmentInfo(Info): - def __init__(self, - name: str, env_type: str, - database_info: DatabaseInfo, - test_container_info: Optional[ContainerInfo], - network_info: DockerNetworkInfo): + def __init__( + self, + name: str, + env_type: str, + database_info: DatabaseInfo, + test_container_info: Optional[ContainerInfo], + network_info: DockerNetworkInfo, + ): self.name = name self.type = env_type self.test_container_info = test_container_info diff --git a/exasol_integration_test_docker_environment/lib/data/environment_type.py b/exasol_integration_test_docker_environment/lib/data/environment_type.py index f0899a9b0..6c7ce6aa3 100644 --- a/exasol_integration_test_docker_environment/lib/data/environment_type.py +++ b/exasol_integration_test_docker_environment/lib/data/environment_type.py @@ -1,6 +1,9 @@ -from enum import Enum +from enum import ( + Enum, + auto, +) class EnvironmentType(Enum): - docker_db = 1, - external_db = 2 \ No newline at end of file + docker_db = auto() + external_db = auto() diff --git a/exasol_integration_test_docker_environment/lib/data/ssh_info.py b/exasol_integration_test_docker_environment/lib/data/ssh_info.py index 7db0da6a2..f7b8a151b 100644 --- a/exasol_integration_test_docker_environment/lib/data/ssh_info.py +++ b/exasol_integration_test_docker_environment/lib/data/ssh_info.py @@ -2,6 +2,7 @@ class SshInfo: """ key_file contains path to the file containing the private key for SSH access. """ + def __init__(self, user: str, key_file: str): self.user = user self.key_file = key_file diff --git a/exasol_integration_test_docker_environment/lib/data/test_container_content_description.py b/exasol_integration_test_docker_environment/lib/data/test_container_content_description.py index ffcac4346..58bff6a90 100644 --- a/exasol_integration_test_docker_environment/lib/data/test_container_content_description.py +++ b/exasol_integration_test_docker_environment/lib/data/test_container_content_description.py @@ -1,5 +1,8 @@ from pathlib import Path -from typing import Optional, List +from typing import ( + List, + Optional, +) from exasol_integration_test_docker_environment.lib.base.info import Info @@ -10,6 +13,7 @@ class TestContainerBuildMapping(Info): The artifact will be copied to location "target", parallel to the Dockerfile and is hence accessible from within the Dockerfile during the build time of the test-container. """ + def __init__(self, source: Path, target: str): self.source = source self.target = target @@ -22,7 +26,10 @@ class TestContainerRuntimeMapping(Info): Optionally, the content will be copied within the test-container to the location indicated by parameter "deployement_target": This is useful if the source path must not be polluted with runtime artifacts (logs, etc.). """ - def __init__(self, source: Path, target: str, deployment_target: Optional[str] = None): + + def __init__( + self, source: Path, target: str, deployment_target: Optional[str] = None + ): self.source = source self.target = target self.deployment_target = deployment_target @@ -33,8 +40,13 @@ class TestContainerContentDescription(Info): This class contains all information necessary to build, start and set up the test-container. Its purpose is to give the client control about the build- and runtime-artifacts. """ - def __init__(self, docker_file: Optional[str], build_files_and_directories: List[TestContainerBuildMapping], - runtime_mappings: List[TestContainerRuntimeMapping]): + + def __init__( + self, + docker_file: Optional[str], + build_files_and_directories: List[TestContainerBuildMapping], + runtime_mappings: List[TestContainerRuntimeMapping], + ): self.docker_file = docker_file self.build_files_and_directories = build_files_and_directories self.runtime_mappings = runtime_mappings diff --git a/exasol_integration_test_docker_environment/lib/docker/__init__.py b/exasol_integration_test_docker_environment/lib/docker/__init__.py index 68bdec4c7..275974389 100644 --- a/exasol_integration_test_docker_environment/lib/docker/__init__.py +++ b/exasol_integration_test_docker_environment/lib/docker/__init__.py @@ -1,10 +1,12 @@ import docker -from exasol_integration_test_docker_environment.lib.config.docker_config import source_docker_repository_config, \ - target_docker_repository_config +from exasol_integration_test_docker_environment.lib.config.docker_config import ( + source_docker_repository_config, + target_docker_repository_config, +) -class ContextDockerClient(object): +class ContextDockerClient: def __init__(self, **kwargs): self.kwargs = kwargs self._client = None @@ -17,9 +19,14 @@ def __enter__(self): return self._client def login(self, client: docker.client.DockerClient, docker_repository_config): - if docker_repository_config.username is not None and \ - docker_repository_config.password is not None: - client.login(username=docker_repository_config.username, password=docker_repository_config.password) + if ( + docker_repository_config.username is not None + and docker_repository_config.password is not None + ): + client.login( + username=docker_repository_config.username, + password=docker_repository_config.password, + ) def __exit__(self, type_, value, traceback): if self._client is not None: diff --git a/exasol_integration_test_docker_environment/lib/docker/container/utils.py b/exasol_integration_test_docker_environment/lib/docker/container/utils.py index d376ef6ac..422c31308 100644 --- a/exasol_integration_test_docker_environment/lib/docker/container/utils.py +++ b/exasol_integration_test_docker_environment/lib/docker/container/utils.py @@ -1,9 +1,11 @@ import logging from typing import List + from docker.models.containers import Container from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient + def remove_docker_container(containers: List[str]): """ Removes the given container using docker API. diff --git a/exasol_integration_test_docker_environment/lib/docker/images/clean/clean_images.py b/exasol_integration_test_docker_environment/lib/docker/images/clean/clean_images.py index be12e136b..9b5fb4fb0 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/clean/clean_images.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/clean/clean_images.py @@ -1,6 +1,11 @@ import luigi -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.docker.images.utils import find_images_by_tag + +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) +from exasol_integration_test_docker_environment.lib.docker.images.utils import ( + find_images_by_tag, +) class CleanImageTask(DockerBaseTask): @@ -11,7 +16,9 @@ def __init__(self, *args, **kwargs): def run_task(self): self.logger.info("Try to remove dependent images of %s" % self.image_id) - yield from self.run_dependencies(self.get_clean_image_tasks_for_dependent_images()) + yield from self.run_dependencies( + self.get_clean_image_tasks_for_dependent_images() + ) for i in range(3): try: self.logger.info("Try to remove image %s" % self.image_id) @@ -20,20 +27,30 @@ def run_task(self): self.logger.info("Removed image %s" % self.image_id) break except Exception as e: - self.logger.info("Could not removed image %s got exception %s" % (self.image_id, e)) + self.logger.info( + "Could not removed image {} got exception {}".format( + self.image_id, e + ) + ) def get_clean_image_tasks_for_dependent_images(self): with self._get_docker_client() as docker_client: - image_ids = [str(possible_child).replace("sha256:", "") for possible_child - in docker_client.api.images(all=True, quiet=True) - if self.is_child_image(possible_child)] - return [self.create_child_task(CleanImageTask, image_id=image_id) - for image_id in image_ids] + image_ids = [ + str(possible_child).replace("sha256:", "") + for possible_child in docker_client.api.images(all=True, quiet=True) + if self.is_child_image(possible_child) + ] + return [ + self.create_child_task(CleanImageTask, image_id=image_id) + for image_id in image_ids + ] def is_child_image(self, possible_child): try: with self._get_docker_client() as docker_client: - inspect = docker_client.api.inspect_image(image=str(possible_child).replace("sha256:", "")) + inspect = docker_client.api.inspect_image( + image=str(possible_child).replace("sha256:", "") + ) return str(inspect["Parent"]).replace("sha256:", "") == self.image_id except: return False @@ -44,15 +61,25 @@ class CleanImagesStartingWith(DockerBaseTask): starts_with_pattern = luigi.Parameter() def register_required(self): - image_ids = [str(image.id).replace("sha256:", "") - for image in self.find_images_to_clean()] - self.register_dependencies([self.create_child_task(CleanImageTask, image_id=image_id) - for image_id in image_ids]) + image_ids = [ + str(image.id).replace("sha256:", "") + for image in self.find_images_to_clean() + ] + self.register_dependencies( + [ + self.create_child_task(CleanImageTask, image_id=image_id) + for image_id in image_ids + ] + ) def find_images_to_clean(self): - self.logger.info("Going to remove all images starting with %s" % self.starts_with_pattern) + self.logger.info( + "Going to remove all images starting with %s" % self.starts_with_pattern + ) with self._get_docker_client() as docker_client: - filter_images = find_images_by_tag(docker_client, lambda tag: tag.startswith(self.starts_with_pattern)) + filter_images = find_images_by_tag( + docker_client, lambda tag: tag.startswith(self.starts_with_pattern) + ) for i in filter_images: self.logger.info("Going to remove following image: %s" % i.tags) return filter_images diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py index 7daa9a2fa..36c5afb40 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py @@ -1,16 +1,33 @@ import copy -from typing import Dict, Set - -from exasol_integration_test_docker_environment.abstract_method_exception import AbstractMethodException -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask -from exasol_integration_test_docker_environment.lib.config.build_config import build_config -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_analyze_task import \ - DockerAnalyzeImageTask -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_create_task import \ - DockerCreateImageTask, DockerCreateImageTaskWithDeps -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo, ImageState -from exasol_integration_test_docker_environment.lib.docker.images.required_task_info import RequiredTaskInfoDict, \ - RequiredTaskInfo +from typing import ( + Dict, + Set, +) + +from exasol_integration_test_docker_environment.abstract_method_exception import ( + AbstractMethodException, +) +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) +from exasol_integration_test_docker_environment.lib.config.build_config import ( + build_config, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_analyze_task import ( + DockerAnalyzeImageTask, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_create_task import ( + DockerCreateImageTask, + DockerCreateImageTaskWithDeps, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, + ImageState, +) +from exasol_integration_test_docker_environment.lib.docker.images.required_task_info import ( + RequiredTaskInfo, + RequiredTaskInfoDict, +) class DockerBuildBase(DependencyLoggerBaseTask): @@ -30,7 +47,9 @@ def get_goal_class_map(self) -> Dict[str, DockerAnalyzeImageTask]: raise AbstractMethodException() def register_required(self): - self.analyze_tasks_futures = self.register_dependencies(self.create_analyze_tasks()) + self.analyze_tasks_futures = self.register_dependencies( + self.create_analyze_tasks() + ) def create_analyze_tasks(self) -> Dict[str, DockerAnalyzeImageTask]: goals = set(self.get_goals()) @@ -45,75 +64,99 @@ def create_analyze_tasks(self) -> Dict[str, DockerAnalyzeImageTask]: return tasks else: difference = goals.difference(self.available_goals) - raise Exception(f"Unknown goal(s) {difference}, " - f"following goals are avaialable {self.available_goals}") + raise Exception( + f"Unknown goal(s) {difference}, " + f"following goals are avaialable {self.available_goals}" + ) def _check_if_build_steps_to_rebuild_are_valid_goals(self): build_steps_to_rebuild = self._get_build_steps_to_rebuild() if not build_steps_to_rebuild.issubset(self.available_goals): difference = build_steps_to_rebuild.difference(self.available_goals) - raise Exception(f"Unknown build stages {difference} forced to rebuild, " - f"following stages are avaialable {self.available_goals}") - - def create_build_tasks(self, shortcut_build: bool = True) \ - -> Dict[str, DockerCreateImageTask]: - image_infos = {goal: analyze_task_future.get_output() - for goal, analyze_task_future - in self.analyze_tasks_futures.items()} + raise Exception( + f"Unknown build stages {difference} forced to rebuild, " + f"following stages are avaialable {self.available_goals}" + ) + + def create_build_tasks( + self, shortcut_build: bool = True + ) -> Dict[str, DockerCreateImageTask]: + image_infos = { + goal: analyze_task_future.get_output() + for goal, analyze_task_future in self.analyze_tasks_futures.items() + } tasks = self._create_build_tasks_for_image_infos(image_infos, shortcut_build) return tasks - def _create_build_tasks_for_image_infos(self, image_infos: Dict[str, ImageInfo], shortcut_build: bool): - result = {goal: self._create_build_task_for_image_info(image_info, shortcut_build) - for goal, image_info in image_infos.items()} + def _create_build_tasks_for_image_infos( + self, image_infos: Dict[str, ImageInfo], shortcut_build: bool + ): + result = { + goal: self._create_build_task_for_image_info(image_info, shortcut_build) + for goal, image_info in image_infos.items() + } return result def _create_build_task_for_image_info( - self, image_info: ImageInfo, - shortcut_build: bool = True) -> DockerCreateImageTask: + self, image_info: ImageInfo, shortcut_build: bool = True + ) -> DockerCreateImageTask: if self._build_with_depenencies_is_requested(image_info, shortcut_build): - task_for_image_info = self._create_build_task_with_dependencies(image_info, shortcut_build) + task_for_image_info = self._create_build_task_with_dependencies( + image_info, shortcut_build + ) return task_for_image_info else: image_name = f"{image_info.target_repository_name}:{image_info.target_tag}" - task_for_image_info = \ - self.create_child_task(DockerCreateImageTask, - image_name=image_name, - image_info=image_info) + task_for_image_info = self.create_child_task( + DockerCreateImageTask, image_name=image_name, image_info=image_info + ) return task_for_image_info - def _build_with_depenencies_is_requested(self, image_info: ImageInfo, shortcut_build: bool): + def _build_with_depenencies_is_requested( + self, image_info: ImageInfo, shortcut_build: bool + ): needs_to_be_build = image_info.image_state == ImageState.NEEDS_TO_BE_BUILD.name - result = (not shortcut_build or needs_to_be_build) and \ - image_info.depends_on_images and len(image_info.depends_on_images) > 0 + result = ( + (not shortcut_build or needs_to_be_build) + and image_info.depends_on_images + and len(image_info.depends_on_images) > 0 + ) return result def _create_build_task_with_dependencies( - self, image_info: ImageInfo, - shortcut_build: bool = True) -> DockerCreateImageTask: + self, image_info: ImageInfo, shortcut_build: bool = True + ) -> DockerCreateImageTask: assert image_info.depends_on_images - required_tasks = \ - self._create_build_tasks_for_image_infos( - image_info.depends_on_images, shortcut_build) + required_tasks = self._create_build_tasks_for_image_infos( + image_info.depends_on_images, shortcut_build + ) required_task_infos = self._create_required_task_infos(required_tasks) image_info_copy = copy.copy(image_info) # TODO looks not nice image_info_copy.depends_on_images = {} image_name = f"{image_info.target_repository_name}:{image_info.target_tag}" - task_for_image_info = \ - self.create_child_task(DockerCreateImageTaskWithDeps, - image_name=image_name, - image_info=image_info_copy, - required_task_infos=required_task_infos) + task_for_image_info = self.create_child_task( + DockerCreateImageTaskWithDeps, + image_name=image_name, + image_info=image_info_copy, + required_task_infos=required_task_infos, + ) return task_for_image_info def _create_required_task_infos( - self, required_tasks: Dict[str, DockerCreateImageTask]) -> RequiredTaskInfoDict: - result = {key: self._create_required_task_info(required_task) - for key, required_task in required_tasks.items()} + self, required_tasks: Dict[str, DockerCreateImageTask] + ) -> RequiredTaskInfoDict: + result = { + key: self._create_required_task_info(required_task) + for key, required_task in required_tasks.items() + } return RequiredTaskInfoDict(result) - def _create_required_task_info(self, required_task: DockerCreateImageTask) -> RequiredTaskInfo: - required_task_info = RequiredTaskInfo(module_name=required_task.__module__, - class_name=required_task.__class__.__name__, - params=required_task.param_kwargs) + def _create_required_task_info( + self, required_task: DockerCreateImageTask + ) -> RequiredTaskInfo: + required_task_info = RequiredTaskInfo( + module_name=required_task.__module__, + class_name=required_task.__class__.__name__, + params=required_task.param_kwargs, + ) return required_task_info diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py index 376222075..6a89f2198 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py @@ -1,22 +1,40 @@ import abc from pathlib import Path -from typing import Dict, Type, Optional +from typing import ( + Dict, + Optional, + Type, +) import docker import git -from exasol_integration_test_docker_environment.abstract_method_exception import AbstractMethodException -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.config.build_config import build_config -from exasol_integration_test_docker_environment.lib.config.docker_config import docker_build_arguments -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.build_context_hasher import \ - BuildContextHasher -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.docker_image_target import \ - DockerImageTarget -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.docker_registry_image_checker import \ - DockerRegistryImageChecker -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageDescription, ImageInfo, \ - ImageState +from exasol_integration_test_docker_environment.abstract_method_exception import ( + AbstractMethodException, +) +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) +from exasol_integration_test_docker_environment.lib.config.build_config import ( + build_config, +) +from exasol_integration_test_docker_environment.lib.config.docker_config import ( + docker_build_arguments, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.build_context_hasher import ( + BuildContextHasher, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.docker_image_target import ( + DockerImageTarget, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.docker_registry_image_checker import ( + DockerRegistryImageChecker, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageDescription, + ImageInfo, + ImageState, +) class DockerAnalyzeImageTask(DockerBaseTask): @@ -27,15 +45,19 @@ def __init__(self, *args, **kwargs): self._target_repository_name = self.get_target_repository_name() self._source_image_tag = self.get_source_image_tag() self._target_image_tag = self.get_target_image_tag() - merged_transparent_build_arguments = {**self.get_transparent_build_arguments(), - **docker_build_arguments().transparent} - merged_image_changing_build_arguments = {**self.get_image_changing_build_arguments(), - **docker_build_arguments().image_changing} + merged_transparent_build_arguments = { + **self.get_transparent_build_arguments(), + **docker_build_arguments().transparent, + } + merged_image_changing_build_arguments = { + **self.get_image_changing_build_arguments(), + **docker_build_arguments().image_changing, + } self.image_description = ImageDescription( dockerfile=self.get_dockerfile(), mapping_of_build_files_and_directories=self.get_mapping_of_build_files_and_directories(), image_changing_build_arguments=merged_image_changing_build_arguments, - transparent_build_arguments=merged_transparent_build_arguments + transparent_build_arguments=merged_transparent_build_arguments, ) self._dockerfile = self.get_dockerfile() @@ -119,35 +141,43 @@ def is_rebuild_requested(self) -> bool: def register_required(self): task_classes = self.requires_tasks() if task_classes is not None: - if not isinstance(task_classes, dict) or \ - not self.keys_are_string(task_classes) or \ - not self.values_are_subclass_of_baseclass(task_classes): - raise TypeError(f"Expected Dict[str,DockerAnalyzeImageTask] got {task_classes}") - tasks = {key: self.create_child_task_with_common_params(value) - for key, value in task_classes.items()} + if ( + not isinstance(task_classes, dict) + or not self.keys_are_string(task_classes) + or not self.values_are_subclass_of_baseclass(task_classes) + ): + raise TypeError( + f"Expected Dict[str,DockerAnalyzeImageTask] got {task_classes}" + ) + tasks = { + key: self.create_child_task_with_common_params(value) + for key, value in task_classes.items() + } else: tasks = None self.dependencies_futures = self.register_dependencies(tasks) def keys_are_string(self, task_classes) -> bool: - return all(isinstance(key, str) - for key in task_classes.keys()) + return all(isinstance(key, str) for key in task_classes.keys()) def values_are_subclass_of_baseclass(self, task_classes) -> bool: - return all(issubclass(value, DockerAnalyzeImageTask) - for value in task_classes.values()) + return all( + issubclass(value, DockerAnalyzeImageTask) for value in task_classes.values() + ) def requires_tasks(self) -> Dict[str, Type["DockerAnalyzeImageTask"]]: return dict() def run_task(self): - image_info_of_dependencies = self.get_values_from_futures(self.dependencies_futures) + image_info_of_dependencies = self.get_values_from_futures( + self.dependencies_futures + ) if image_info_of_dependencies is None: image_info_of_dependencies = dict() - _build_context_hasher = BuildContextHasher(self.logger, - self.image_description) - image_hash = _build_context_hasher \ - .generate_image_hash(image_info_of_dependencies) + _build_context_hasher = BuildContextHasher(self.logger, self.image_description) + image_hash = _build_context_hasher.generate_image_hash( + image_info_of_dependencies + ) image_info = ImageInfo( source_repository_name=self._source_repository_name, target_repository_name=self._target_repository_name, @@ -158,13 +188,17 @@ def run_task(self): build_name=build_config().build_name, depends_on_images=image_info_of_dependencies, image_state=None, - image_description=self.image_description + image_description=self.image_description, + ) + target_image_target = DockerImageTarget( + self._target_repository_name, image_info.get_target_complete_tag() + ) + source_image_target = DockerImageTarget( + self._source_repository_name, image_info.get_source_complete_tag() + ) + image_state = self.get_image_state( + source_image_target, target_image_target, image_info_of_dependencies ) - target_image_target = DockerImageTarget(self._target_repository_name, image_info.get_target_complete_tag()) - source_image_target = DockerImageTarget(self._source_repository_name, image_info.get_source_complete_tag()) - image_state = self.get_image_state(source_image_target, - target_image_target, - image_info_of_dependencies) image_info.image_state = image_state.name # TODO setter for image_state self.return_object(image_info) @@ -174,24 +208,32 @@ def get_commit_id(self): sha = repo.head.object.hexsha return sha except Exception as e: - self.logger.info("Not a Git Repository, can't determine the commit id for the image_info") + self.logger.info( + "Not a Git Repository, can't determine the commit id for the image_info" + ) return "" - def get_image_state(self, - source_image_target: DockerImageTarget, - target_image_target: DockerImageTarget, - image_info_of_dependencies: Dict[str, ImageInfo]) -> ImageState: + def get_image_state( + self, + source_image_target: DockerImageTarget, + target_image_target: DockerImageTarget, + image_info_of_dependencies: Dict[str, ImageInfo], + ) -> ImageState: if self.is_rebuild_necessary(image_info_of_dependencies): return ImageState.NEEDS_TO_BE_BUILD else: - if self.is_image_locally_available(target_image_target) \ - and not build_config().force_pull \ - and not build_config().force_load: + if ( + self.is_image_locally_available(target_image_target) + and not build_config().force_pull + and not build_config().force_load + ): return ImageState.TARGET_LOCALLY_AVAILABLE - if self.is_image_locally_available(source_image_target) \ - and not build_config().force_pull \ - and not build_config().force_load: + if ( + self.is_image_locally_available(source_image_target) + and not build_config().force_pull + and not build_config().force_load + ): return ImageState.SOURCE_LOCALLY_AVAILABLE elif self.can_image_be_loaded(source_image_target): return ImageState.CAN_BE_LOADED @@ -201,47 +243,62 @@ def get_image_state(self, return ImageState.NEEDS_TO_BE_BUILD def needs_any_dependency_to_be_build( - self, image_info_of_dependencies: Dict[str, ImageInfo]) -> bool: - return any(image_info.image_state == ImageState.NEEDS_TO_BE_BUILD - for image_info in image_info_of_dependencies.values()) + self, image_info_of_dependencies: Dict[str, ImageInfo] + ) -> bool: + return any( + image_info.image_state == ImageState.NEEDS_TO_BE_BUILD + for image_info in image_info_of_dependencies.values() + ) def is_rebuild_necessary(self, image_info_of_dependencies: Dict[str, ImageInfo]): - return self.needs_any_dependency_to_be_build(image_info_of_dependencies) or \ - self.is_rebuild_requested() + return ( + self.needs_any_dependency_to_be_build(image_info_of_dependencies) + or self.is_rebuild_requested() + ) def is_image_locally_available(self, image_target: DockerImageTarget): exists = image_target.exists() self.logger.info( f"Checking if image {image_target.get_complete_name()} " - f"is locally available, result {exists}") + f"is locally available, result {exists}" + ) return exists def can_image_be_loaded(self, image_target: DockerImageTarget): if build_config().cache_directory is not None: image_path = self.get_path_to_cached_image(image_target) - self.logger.info(f"Checking if image archive {image_path} " - f"is available in cache directory, " - f"result {image_path.exists()}") + self.logger.info( + f"Checking if image archive {image_path} " + f"is available in cache directory, " + f"result {image_path.exists()}" + ) return image_path.exists() else: return False def get_path_to_cached_image(self, image_target): - image_path = \ - Path(build_config().cache_directory) \ - .joinpath(Path(image_target.get_complete_name() + ".tar")) + image_path = Path(build_config().cache_directory).joinpath( + Path(image_target.get_complete_name() + ".tar") + ) return image_path def is_image_in_registry(self, image_target: DockerImageTarget): try: - self.logger.info("Try to find image %s in registry", - image_target.get_complete_name()) - exists = DockerRegistryImageChecker().check(image_target.get_complete_name()) + self.logger.info( + "Try to find image %s in registry", image_target.get_complete_name() + ) + exists = DockerRegistryImageChecker().check( + image_target.get_complete_name() + ) if exists: - self.logger.info("Found image %s in registry", - image_target.get_complete_name()) + self.logger.info( + "Found image %s in registry", image_target.get_complete_name() + ) return exists except docker.errors.DockerException as e: - self.logger.warning("Image %s not in registry, got exception %s", - image_target.get_complete_name(), e) + self.logger.warning( + "Image %s not in registry, got exception %s", + image_target.get_complete_name(), + e, + ) return False diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_build_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_build_task.py index 7acd6638e..ea1682e75 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_build_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_build_task.py @@ -2,39 +2,59 @@ import tempfile from pathlib import Path -from exasol_integration_test_docker_environment.lib.base.still_running_logger import StillRunningLogger -from exasol_integration_test_docker_environment.lib.config.build_config import build_config -from exasol_integration_test_docker_environment.lib.config.docker_config import docker_build_arguments -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_creator_base_task import \ - DockerImageCreatorBaseTask -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.build_context_creator import \ - BuildContextCreator -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.build_log_handler import BuildLogHandler -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo +from exasol_integration_test_docker_environment.lib.base.still_running_logger import ( + StillRunningLogger, +) +from exasol_integration_test_docker_environment.lib.config.build_config import ( + build_config, +) +from exasol_integration_test_docker_environment.lib.config.docker_config import ( + docker_build_arguments, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_creator_base_task import ( + DockerImageCreatorBaseTask, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.build_context_creator import ( + BuildContextCreator, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.build_log_handler import ( + BuildLogHandler, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, +) class DockerBuildImageTask(DockerImageCreatorBaseTask): def run_task(self): - self.logger.info("Build docker image %s, log file can be found here %s", - self.image_info.get_target_complete_name(), self.get_log_path()) - temp_directory = tempfile.mkdtemp(prefix="script_language_container_tmp_dir", - dir=build_config().temporary_base_directory) + self.logger.info( + "Build docker image %s, log file can be found here %s", + self.image_info.get_target_complete_name(), + self.get_log_path(), + ) + temp_directory = tempfile.mkdtemp( + prefix="script_language_container_tmp_dir", + dir=build_config().temporary_base_directory, + ) try: image_description = self.image_info.image_description - build_context_creator = BuildContextCreator(temp_directory, - self.image_info, - self.get_log_path()) + build_context_creator = BuildContextCreator( + temp_directory, self.image_info, self.get_log_path() + ) build_context_creator.prepare_build_context_to_temp_dir() with self._get_docker_client() as docker_client: - output_generator = \ - docker_client.api.build(path=temp_directory, - nocache=self.no_cache, - tag=self.image_info.get_target_complete_name(), - rm=True, - buildargs=dict(**image_description.transparent_build_arguments, - **image_description.image_changing_build_arguments, - **docker_build_arguments().secret)) + output_generator = docker_client.api.build( + path=temp_directory, + nocache=self.no_cache, + tag=self.image_info.get_target_complete_name(), + rm=True, + buildargs=dict( + **image_description.transparent_build_arguments, + **image_description.image_changing_build_arguments, + **docker_build_arguments().secret + ), + ) self._handle_output(output_generator, self.image_info) finally: shutil.rmtree(temp_directory) @@ -43,7 +63,8 @@ def _handle_output(self, output_generator, image_info: ImageInfo): log_file_path = Path(self.get_log_path(), "docker-build.log") with BuildLogHandler(log_file_path, self.logger, image_info) as log_handler: still_running_logger = StillRunningLogger( - self.logger, "build image %s" % image_info.get_target_complete_name()) + self.logger, "build image %s" % image_info.get_target_complete_name() + ) for log_line in output_generator: still_running_logger.log() log_handler.handle_log_lines(log_line) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_create_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_create_task.py index 71c313238..2945556c5 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_create_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_create_task.py @@ -3,25 +3,39 @@ import luigi -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import JsonPickleParameter -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_build_task import \ - DockerBuildImageTask -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_load_task import \ - DockerLoadImageTask -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_pull_task import \ - DockerPullImageTask -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo, ImageState -from exasol_integration_test_docker_environment.lib.docker.images.required_task_info import RequiredTaskInfoDict, \ - RequiredTaskInfo +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import ( + JsonPickleParameter, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_build_task import ( + DockerBuildImageTask, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_load_task import ( + DockerLoadImageTask, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_pull_task import ( + DockerPullImageTask, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, + ImageState, +) +from exasol_integration_test_docker_environment.lib.docker.images.required_task_info import ( + RequiredTaskInfo, + RequiredTaskInfoDict, +) class DockerCreateImageTask(DockerBaseTask): - image_name : str = luigi.Parameter() # type: ignore + image_name: str = luigi.Parameter() # type: ignore # ParameterVisibility needs to be hidden instead of private, because otherwise a MissingParameter gets thrown - image_info : ImageInfo = JsonPickleParameter(ImageInfo, - visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type: ignore + image_info: ImageInfo = JsonPickleParameter( + ImageInfo, + visibility=luigi.parameter.ParameterVisibility.HIDDEN, + significant=True, + ) # type: ignore def run_task(self): new_image_info = yield from self.build(self.image_info) @@ -29,23 +43,23 @@ def run_task(self): def build(self, image_info: ImageInfo): if image_info.image_state == ImageState.NEEDS_TO_BE_BUILD.name: - task = self.create_child_task(DockerBuildImageTask, - image_name=self.image_name, - image_info=image_info) + task = self.create_child_task( + DockerBuildImageTask, image_name=self.image_name, image_info=image_info + ) yield from self.run_dependencies(task) image_info.image_state = ImageState.WAS_BUILD.name # TODO clone and change return image_info elif image_info.image_state == ImageState.CAN_BE_LOADED.name: - task = self.create_child_task(DockerLoadImageTask, - image_name=self.image_name, - image_info=image_info) + task = self.create_child_task( + DockerLoadImageTask, image_name=self.image_name, image_info=image_info + ) yield from self.run_dependencies(task) image_info.image_state = ImageState.WAS_LOADED.name return image_info elif image_info.image_state == ImageState.REMOTE_AVAILABLE.name: - task = self.create_child_task(DockerPullImageTask, - image_name=self.image_name, - image_info=image_info) + task = self.create_child_task( + DockerPullImageTask, image_name=self.image_name, image_info=image_info + ) yield from self.run_dependencies(task) image_info.image_state = ImageState.WAS_PULLED.name return image_info @@ -57,30 +71,39 @@ def build(self, image_info: ImageInfo): self.rename_source_image_to_target_image(image_info) return image_info else: - raise Exception("Task %s: Image state %s not supported for image %s", - self.task_id, image_info.image_state, image_info.get_target_complete_name()) + raise Exception( + "Task %s: Image state %s not supported for image %s", + self.task_id, + image_info.image_state, + image_info.get_target_complete_name(), + ) def rename_source_image_to_target_image(self, image_info): with self._get_docker_client() as docker_client: docker_client.images.get(image_info.get_source_complete_name()).tag( repository=image_info.target_repository_name, - tag=image_info.get_target_complete_tag() + tag=image_info.get_target_complete_tag(), ) class DockerCreateImageTaskWithDeps(DockerCreateImageTask): # ParameterVisibility needs to be hidden instead of private, because otherwise a MissingParameter gets thrown - required_task_infos : RequiredTaskInfoDict = JsonPickleParameter(RequiredTaskInfoDict, - visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type: ignore + required_task_infos: RequiredTaskInfoDict = JsonPickleParameter( + RequiredTaskInfoDict, + visibility=luigi.parameter.ParameterVisibility.HIDDEN, + significant=True, + ) # type: ignore def register_required(self): - self.required_tasks = {key: self.create_required_task(required_task_info) - for key, required_task_info - in self.required_task_infos.infos.items()} + self.required_tasks = { + key: self.create_required_task(required_task_info) + for key, required_task_info in self.required_task_infos.infos.items() + } self.futures = self.register_dependencies(self.required_tasks) - def create_required_task(self, required_task_info: RequiredTaskInfo) -> DockerCreateImageTask: + def create_required_task( + self, required_task_info: RequiredTaskInfo + ) -> DockerCreateImageTask: module = importlib.import_module(required_task_info.module_name) class_ = getattr(module, required_task_info.class_name) instance = self.create_child_task(class_, **required_task_info.params) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_creator_base_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_creator_base_task.py index 071f8c0c1..b6e00d899 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_creator_base_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_creator_base_task.py @@ -1,13 +1,21 @@ import luigi -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import JsonPickleParameter -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import ( + JsonPickleParameter, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, +) class DockerImageCreatorBaseTask(DockerBaseTask): - image_name : str = luigi.Parameter() # type: ignore + image_name: str = luigi.Parameter() # type: ignore # ParameterVisibility needs to be hidden instead of private, because otherwise a MissingParameter gets thrown - image_info : ImageInfo= JsonPickleParameter(ImageInfo, - visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type: ignore + image_info: ImageInfo = JsonPickleParameter( + ImageInfo, + visibility=luigi.parameter.ParameterVisibility.HIDDEN, + significant=True, + ) # type: ignore diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_load_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_load_task.py index 2d7d602b7..b6298c598 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_load_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_load_task.py @@ -1,21 +1,28 @@ from pathlib import Path -from exasol_integration_test_docker_environment.lib.config.build_config import build_config -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_creator_base_task import \ - DockerImageCreatorBaseTask +from exasol_integration_test_docker_environment.lib.config.build_config import ( + build_config, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_creator_base_task import ( + DockerImageCreatorBaseTask, +) class DockerLoadImageTask(DockerImageCreatorBaseTask): def run_task(self): - image_archive_path = Path(build_config().cache_directory) \ - .joinpath(self.image_info.get_source_complete_name() + ".tar") - self.logger.info("Try to load docker image %s from %s", - self.image_info.get_source_complete_name(), image_archive_path) + image_archive_path = Path(build_config().cache_directory).joinpath( + self.image_info.get_source_complete_name() + ".tar" + ) + self.logger.info( + "Try to load docker image %s from %s", + self.image_info.get_source_complete_name(), + image_archive_path, + ) with self._get_docker_client() as docker_client: with image_archive_path.open("rb") as f: docker_client.images.load(f) docker_client.images.get(self.image_info.get_source_complete_name()).tag( repository=self.image_info.target_repository_name, - tag=self.image_info.get_target_complete_tag() + tag=self.image_info.get_target_complete_tag(), ) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_pull_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_pull_task.py index 586e66524..f51284b70 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_pull_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_pull_task.py @@ -1,25 +1,37 @@ import time -from exasol_integration_test_docker_environment.lib.base.still_running_logger import StillRunningLogger -from exasol_integration_test_docker_environment.lib.config.docker_config import source_docker_repository_config -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_creator_base_task import \ - DockerImageCreatorBaseTask -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.docker_image_target import \ - DockerImageTarget -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.pull_log_handler import PullLogHandler +from exasol_integration_test_docker_environment.lib.base.still_running_logger import ( + StillRunningLogger, +) +from exasol_integration_test_docker_environment.lib.config.docker_config import ( + source_docker_repository_config, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_creator_base_task import ( + DockerImageCreatorBaseTask, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.docker_image_target import ( + DockerImageTarget, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.pull_log_handler import ( + PullLogHandler, +) class DockerPullImageTask(DockerImageCreatorBaseTask): RETRIES = 3 def run_task(self): - image_target = DockerImageTarget(image_name=self.image_info.source_repository_name, - image_tag=self.image_info.get_source_complete_tag()) - if source_docker_repository_config().username is not None and \ - source_docker_repository_config().password is not None: + image_target = DockerImageTarget( + image_name=self.image_info.source_repository_name, + image_tag=self.image_info.get_source_complete_tag(), + ) + if ( + source_docker_repository_config().username is not None + and source_docker_repository_config().password is not None + ): auth_config = { "username": source_docker_repository_config().username, - "password": source_docker_repository_config().password + "password": source_docker_repository_config().password, } else: auth_config = None @@ -38,23 +50,28 @@ def run_task(self): with self._get_docker_client() as docker_client: docker_client.images.get(self.image_info.get_source_complete_name()).tag( repository=self.image_info.target_repository_name, - tag=self.image_info.get_target_complete_tag() + tag=self.image_info.get_target_complete_tag(), ) def pull(self, image_target, auth_config): - self.logger.info("Try to pull docker image %s", image_target.get_complete_name()) + self.logger.info( + "Try to pull docker image %s", image_target.get_complete_name() + ) with self._get_docker_client() as docker_client: output_generator = docker_client.api.pull( - repository=image_target.image_name, tag=image_target.image_tag, + repository=image_target.image_name, + tag=image_target.image_tag, auth_config=auth_config, - stream=True) + stream=True, + ) self._handle_output(output_generator, self.image_info) def _handle_output(self, output_generator, image_info): log_file_path = self.get_log_path().joinpath("pull_docker_db_image.log") with PullLogHandler(log_file_path, self.logger, image_info) as log_handler: still_running_logger = StillRunningLogger( - self.logger, "pull image %s" % image_info.get_source_complete_name()) + self.logger, "pull image %s" % image_info.get_source_complete_name() + ) for log_line in output_generator: still_running_logger.log() log_handler.handle_log_lines(log_line) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/build_context_creator.py b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/build_context_creator.py index 55fec68c5..71c318825 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/build_context_creator.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/build_context_creator.py @@ -6,15 +6,17 @@ from jinja2 import Template -from exasol_integration_test_docker_environment.lib.config.build_config import build_config -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo, ImageState +from exasol_integration_test_docker_environment.lib.config.build_config import ( + build_config, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, + ImageState, +) class BuildContextCreator: - def __init__(self, - temp_directory, - image_info: ImageInfo, - log_file_path: Path): + def __init__(self, temp_directory, image_info: ImageInfo, log_file_path: Path): self._image_info = image_info self._log_file_path = log_file_path self._image_info_of_dependencies = self._image_info.depends_on_images @@ -29,29 +31,34 @@ def prepare_build_context_to_temp_dir(self): def _prepare_image_info(self): self._image_info.image_state = ImageState.WAS_BUILD.name - with open(self._temp_directory + "/image_info", "wt") as file: + with open(self._temp_directory + "/image_info", "w") as file: file.write(self._image_info.to_json()) def _prepare_dockerfile(self): - with open(self._image_description.dockerfile, "rt") as file: + with open(self._image_description.dockerfile) as file: dockerfile_content = file.read() template = Template(dockerfile_content) - image_names_of_dependencies = \ - {key: image_info.get_target_complete_name() - for key, image_info - in self._image_info_of_dependencies.items()} + image_names_of_dependencies = { + key: image_info.get_target_complete_name() + for key, image_info in self._image_info_of_dependencies.items() + } final_dockerfile = template.render(**image_names_of_dependencies) - final_dockerfile += textwrap.dedent(f""" + final_dockerfile += textwrap.dedent( + f""" RUN mkdir -p /build_info/image_info COPY image_info /build_info/image_info/{self._image_info.target_tag} RUN mkdir -p /build_info/dockerfiles COPY Dockerfile /build_info/dockerfiles/{self._image_info.target_tag} - """) - with open(self._temp_directory + "/Dockerfile", "wt") as file: + """ + ) + with open(self._temp_directory + "/Dockerfile", "w") as file: file.write(final_dockerfile) def _copy_build_files_and_directories(self): - for dest, src in self._image_description.mapping_of_build_files_and_directories.items(): + for ( + dest, + src, + ) in self._image_description.mapping_of_build_files_and_directories.items(): src_path = pathlib.Path(src) dest_path = self._temp_directory + "/" + dest if src_path.is_dir(): @@ -63,7 +70,9 @@ def _copy_build_files_and_directories(self): def _log_build_context(self): if build_config().log_build_context_content: - build_context_log_file = self._log_file_path.joinpath("docker-build-context.log") + build_context_log_file = self._log_file_path.joinpath( + "docker-build-context.log" + ) with build_context_log_file.open("wt") as log_file: for file in self._get_files_in_build_context(self._temp_directory): log_file.write(file) @@ -72,4 +81,6 @@ def _log_build_context(self): shutil.copy(self._temp_directory + "/Dockerfile", str(dockerfile_log_file)) def _get_files_in_build_context(self, temp_directory): - return [os.path.join(r, file) for r, d, f in os.walk(temp_directory) for file in f] + return [ + os.path.join(r, file) for r, d, f in os.walk(temp_directory) for file in f + ] diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/build_context_hasher.py b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/build_context_hasher.py index 70727e68f..a0ad8053b 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/build_context_hasher.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/build_context_hasher.py @@ -1,12 +1,20 @@ import base64 import hashlib -from pathlib import Path, PurePath +from pathlib import ( + Path, + PurePath, +) from typing import Dict # TODO add hash config to the hash -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.file_directory_list_hasher import \ - FileDirectoryListHasher, PathMapping -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageDescription, ImageInfo +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.file_directory_list_hasher import ( + FileDirectoryListHasher, + PathMapping, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageDescription, + ImageInfo, +) class BuildContextHasher: @@ -17,30 +25,43 @@ def __init__(self, logger, image_description: ImageDescription): def generate_image_hash(self, image_info_of_dependencies: Dict[str, ImageInfo]): hash_of_build_context = self._generate_build_context_hash() - final_hash = self._generate_final_hash(hash_of_build_context, image_info_of_dependencies) + final_hash = self._generate_final_hash( + hash_of_build_context, image_info_of_dependencies + ) return self._encode_hash(final_hash) def _generate_build_context_hash(self): - files_directories_list_hasher = \ - FileDirectoryListHasher(followlinks=True, - hashfunc="sha256", - hash_file_names=True, - hash_directory_names=True, - hash_permissions=True) - files_directories_to_hash = [PathMapping(PurePath(destination), Path(source)) for destination, source in - self.image_description.mapping_of_build_files_and_directories.items()] + files_directories_list_hasher = FileDirectoryListHasher( + followlinks=True, + hashfunc="sha256", + hash_file_names=True, + hash_directory_names=True, + hash_permissions=True, + ) + files_directories_to_hash = [ + PathMapping(PurePath(destination), Path(source)) + for destination, source in self.image_description.mapping_of_build_files_and_directories.items() + ] # Use only the dockerfile itself for hashing. In order to accomplish that, # set the dockerfile only as destination in the mapping dockerfile = Path(self.image_description.dockerfile).name - files_directories_to_hash.append(PathMapping(PurePath(dockerfile), - Path(self.image_description.dockerfile))) + files_directories_to_hash.append( + PathMapping(PurePath(dockerfile), Path(self.image_description.dockerfile)) + ) self.logger.debug("files_directories_list_hasher %s", files_directories_to_hash) - hash_of_build_context = files_directories_list_hasher.hash(files_directories_to_hash) - self.logger.debug("hash_of_build_context %s", self._encode_hash(hash_of_build_context)) + hash_of_build_context = files_directories_list_hasher.hash( + files_directories_to_hash + ) + self.logger.debug( + "hash_of_build_context %s", self._encode_hash(hash_of_build_context) + ) return hash_of_build_context - def _generate_final_hash(self, hash_of_build_context: bytes, - image_info_of_dependencies: Dict[str, ImageInfo]): + def _generate_final_hash( + self, + hash_of_build_context: bytes, + image_info_of_dependencies: Dict[str, ImageInfo], + ): hasher = hashlib.sha256() self.add_image_changing_build_arguments(hasher) self.add_dependencies(hasher, image_info_of_dependencies) @@ -48,9 +69,13 @@ def _generate_final_hash(self, hash_of_build_context: bytes, final_hash = hasher.digest() return final_hash - def add_dependencies(self, hasher, image_info_of_dependencies: Dict[str, ImageInfo]): - hashes_of_dependencies = \ - [(key, image_info.hash) for key, image_info in image_info_of_dependencies.items()] + def add_dependencies( + self, hasher, image_info_of_dependencies: Dict[str, ImageInfo] + ): + hashes_of_dependencies = [ + (key, image_info.hash) + for key, image_info in image_info_of_dependencies.items() + ] hashes_to_hash = sorted(hashes_of_dependencies, key=lambda t: t[0]) self.logger.debug("hashes_to_hash %s", hashes_to_hash) for key, image_name in hashes_to_hash: @@ -58,9 +83,10 @@ def add_dependencies(self, hasher, image_info_of_dependencies: Dict[str, ImageIn hasher.update(image_name.encode("utf-8")) def add_image_changing_build_arguments(self, hasher): - arguments = \ - [(key, value) for key, value - in self.image_description.image_changing_build_arguments.items()] + arguments = [ + (key, value) + for key, value in self.image_description.image_changing_build_arguments.items() + ] sorted_arguemnts = sorted(arguments, key=lambda t: t[0]) for key, value in sorted_arguemnts: hasher.update(key.encode("utf-8")) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/build_log_handler.py b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/build_log_handler.py index a2f117a59..c2bbef214 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/build_log_handler.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/build_log_handler.py @@ -3,9 +3,15 @@ import docker -from exasol_integration_test_docker_environment.lib.config.log_config import WriteLogFilesToConsole -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo -from exasol_integration_test_docker_environment.lib.logging.abstract_log_handler import AbstractLogHandler +from exasol_integration_test_docker_environment.lib.config.log_config import ( + WriteLogFilesToConsole, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, +) +from exasol_integration_test_docker_environment.lib.logging.abstract_log_handler import ( + AbstractLogHandler, +) class BuildLogHandler(AbstractLogHandler): @@ -21,7 +27,7 @@ def handle_log_line(self, log_line, error: bool = False): self._log_file.write(json_output["stream"]) self._log_file.write("\n") self._log_file.flush() - if 'errorDetail' in json_output: + if "errorDetail" in json_output: self._error_message = json_output["errorDetail"]["message"] def finish(self): @@ -30,23 +36,33 @@ def finish(self): def write_log_to_console_if_requested(self): if self._log_config.write_log_files_to_console == WriteLogFilesToConsole.all: - self._logger.info("Build Log of image %s\n%s", - self._image_info.get_target_complete_name(), - "\n".join(self._complete_log)) + self._logger.info( + "Build Log of image %s\n%s", + self._image_info.get_target_complete_name(), + "\n".join(self._complete_log), + ) def handle_error(self): if self._error_message is not None: self.write_error_log_to_console_if_requested() raise docker.errors.BuildError( - "Error occurred during the build of the image %s. Received error \"%s\" ." + 'Error occurred during the build of the image %s. Received error "%s" .' "The whole log can be found in %s" - % (self._image_info.get_target_complete_name(), - self._error_message, - self._log_file_path.absolute()), - self._log_file_path.absolute()) + % ( + self._image_info.get_target_complete_name(), + self._error_message, + self._log_file_path.absolute(), + ), + self._log_file_path.absolute(), + ) def write_error_log_to_console_if_requested(self): - if self._log_config.write_log_files_to_console == WriteLogFilesToConsole.only_error: - self._logger.error("Build of image %s failed\nBuild Log:\n%s", - self._image_info.get_target_complete_name(), - "\n".join(self._complete_log)) + if ( + self._log_config.write_log_files_to_console + == WriteLogFilesToConsole.only_error + ): + self._logger.error( + "Build of image %s failed\nBuild Log:\n%s", + self._image_info.get_target_complete_name(), + "\n".join(self._complete_log), + ) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/character_length_checker.py b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/character_length_checker.py index c453a2f21..af4f7ed80 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/character_length_checker.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/character_length_checker.py @@ -9,8 +9,14 @@ class CharacterLengthChecker: The goal is to avoid high memory consumption for calculation of huge directories which contain many sub-directories/files (broad directory trees) and long names. """ - def __init__(self, root_directory: PurePath, max_characters_paths: int, - count_directory_names: bool, count_file_names: bool): + + def __init__( + self, + root_directory: PurePath, + max_characters_paths: int, + count_directory_names: bool, + count_file_names: bool, + ): self._num_characters = 0 self._max_characters_paths = max_characters_paths self._root_directory = root_directory @@ -31,4 +37,6 @@ def add_and_check(self, directories: List[str], files: List[str]) -> None: self._num_characters += sum([len(f) for f in files]) if self._num_characters > self._max_characters_paths: - raise OSError(f"Walking through too many directories. Aborting. Please verify: {self._root_directory}") + raise OSError( + f"Walking through too many directories. Aborting. Please verify: {self._root_directory}" + ) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/docker_registry_image_checker.py b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/docker_registry_image_checker.py index 3353a1317..2d20b4750 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/docker_registry_image_checker.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/docker_registry_image_checker.py @@ -1,9 +1,10 @@ import json import multiprocessing as mp - from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.lib.logging.abstract_log_handler import AbstractLogHandler +from exasol_integration_test_docker_environment.lib.logging.abstract_log_handler import ( + AbstractLogHandler, +) class DockerRegistryImageCheckerPullLogHandler(AbstractLogHandler): @@ -35,7 +36,7 @@ def map(self, image: str, queue: mp.Queue): def check(self, image: str): log_handler = DockerRegistryImageCheckerPullLogHandler() - queue : mp.Queue = mp.Queue() + queue: mp.Queue = mp.Queue() process = mp.Process(target=self.map, args=(image, queue)) process.start() try: @@ -48,6 +49,8 @@ def check(self, image: str): elif value is None: return False else: - raise RuntimeError(f"Should not happen. Programming Error. Unknown value {value}") + raise RuntimeError( + f"Should not happen. Programming Error. Unknown value {value}" + ) finally: process.terminate() diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py index 57a8569b0..34c69c104 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py @@ -3,21 +3,29 @@ import stat from dataclasses import dataclass from multiprocessing import Pool -from pathlib import Path, PurePath -from typing import List, Callable +from pathlib import ( + Path, + PurePath, +) +from typing import ( + Callable, + List, +) import humanfriendly -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.character_length_checker import \ - CharacterLengthChecker -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.symlink_loop_checker import \ - SymlinkLoopChecker +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.character_length_checker import ( + CharacterLengthChecker, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.symlink_loop_checker import ( + SymlinkLoopChecker, +) HASH_FUNCTIONS = { - 'md5': hashlib.md5, - 'sha1': hashlib.sha1, - 'sha256': hashlib.sha256, - 'sha512': hashlib.sha512 + "md5": hashlib.md5, + "sha1": hashlib.sha1, + "sha256": hashlib.sha256, + "sha512": hashlib.sha512, } @@ -26,6 +34,7 @@ class PathMapping: """ Describes a mapping of a file or directory from source to destination. """ + destination: PurePath source: Path @@ -35,6 +44,7 @@ class DestinationMapping: """ Represents one file/directory found by traversing the source directory tree. """ + destination_path: PurePath """The path of the file/directory in the destination (including destination_root_path)""" source_path: Path @@ -56,25 +66,27 @@ class DirectoryMappingResult: """ Contains all entries found by traversing all mapping directory paths. """ + sources: List[Path] paths_for_hashing: List[DestinationMapping] class FileDirectoryListHasher: - def __init__(self, - hashfunc: str = 'md5', - followlinks: bool = False, - hash_file_names: bool = False, - hash_permissions: bool = False, - hash_directory_names: bool = False, - excluded_directories=None, - excluded_files=None, - excluded_extensions=None, - blocksize: str = "64kb", - workers: int = 4, - max_characters_paths: int = 500000000 - ): + def __init__( + self, + hashfunc: str = "md5", + followlinks: bool = False, + hash_file_names: bool = False, + hash_permissions: bool = False, + hash_directory_names: bool = False, + excluded_directories=None, + excluded_files=None, + excluded_extensions=None, + blocksize: str = "64kb", + workers: int = 4, + max_characters_paths: int = 500000000, + ): self.MAX_CHARACTERS_PATHS = max_characters_paths self.workers = workers self.excluded_files = excluded_files @@ -87,7 +99,7 @@ def __init__(self, self.hash_func = HASH_FUNCTIONS.get(hashfunc) if not self.hash_func: - raise NotImplementedError('{} not implemented.'.format(hashfunc)) + raise NotImplementedError(f"{hashfunc} not implemented.") if not self.excluded_files: self.excluded_files = [] @@ -98,16 +110,19 @@ def __init__(self, if not self.excluded_extensions: self.excluded_extensions = [] - self.path_hasher = PathHasher(hashfunc, - hash_permissions=hash_permissions) + self.path_hasher = PathHasher(hashfunc, hash_permissions=hash_permissions) self.file_content_hasher = FileContentHasher(hashfunc, blocksize) def hash(self, files_and_directories: List[PathMapping]) -> bytes: if not isinstance(files_and_directories, List): - raise Exception("List with paths expected and not '%s' with type %s" - % (files_and_directories, type(files_and_directories))) - - directory_mapping_result = self.collect_dest_path_and_src_files(files_and_directories) + raise Exception( + "List with paths expected and not '%s' with type %s" + % (files_and_directories, type(files_and_directories)) + ) + + directory_mapping_result = self.collect_dest_path_and_src_files( + files_and_directories + ) hashes = self.compute_hashes(directory_mapping_result) return self._reduce_hash(hashes) @@ -119,11 +134,15 @@ def check_no_duplicate_destinations(mappings: List[DestinationMapping]) -> None: structure; or if subpaths of destinations and sources somehow match; or if the destination of two mappings is the same file. """ - destination_paths_set = set(p.destination_path for p in mappings) + destination_paths_set = {p.destination_path for p in mappings} if len(destination_paths_set) != len(mappings): - raise AssertionError(f"Directory content for hashing contains duplicates: {mappings}") + raise AssertionError( + f"Directory content for hashing contains duplicates: {mappings}" + ) - def collect_dest_path_and_src_files(self, files_and_directories: List[PathMapping]) -> DirectoryMappingResult: + def collect_dest_path_and_src_files( + self, files_and_directories: List[PathMapping] + ) -> DirectoryMappingResult: """ Traverse the source paths of all mappings and assemble two lists: 1) Destination paths-names (for Hashing): @@ -136,9 +155,13 @@ def collect_dest_path_and_src_files(self, files_and_directories: List[PathMappin """ collected_dest_paths: List[DestinationMapping] = list() - def replace_src_by_dest_path(src: PurePath, dest: PurePath, target: str) -> PurePath: + def replace_src_by_dest_path( + src: PurePath, dest: PurePath, target: str + ) -> PurePath: if not target.startswith(str(src)): - raise RuntimeError(f"path target {target} does not start with source: {src}") + raise RuntimeError( + f"path target {target} does not start with source: {src}" + ) p = Path(target).relative_to(src) return Path(dest) / p @@ -147,41 +170,69 @@ def replace_src_by_dest_path(src: PurePath, dest: PurePath, target: str) -> Pure destination = file_or_directory.destination def handle_directory(directories: List[str]) -> None: - new_dest_paths_mappings = [DestinationMapping( - destination_path=replace_src_by_dest_path(source, destination, p), - source_path=Path(p)) for p in directories] + new_dest_paths_mappings = [ + DestinationMapping( + destination_path=replace_src_by_dest_path( + source, destination, p + ), + source_path=Path(p), + ) + for p in directories + ] collected_dest_paths.extend(new_dest_paths_mappings) def handle_files(files: List[str]) -> None: - collected_dest_paths.extend([DestinationMapping( - destination_path=replace_src_by_dest_path(source, destination, f), - source_path=Path(f)) for f in files]) + collected_dest_paths.extend( + [ + DestinationMapping( + destination_path=replace_src_by_dest_path( + source, destination, f + ), + source_path=Path(f), + ) + for f in files + ] + ) if os.path.isdir(source): self.traverse_directory(source, handle_directory, handle_files) elif os.path.isfile(source): - collected_dest_paths.append(DestinationMapping(destination_path=Path(destination), - source_path=Path(source))) + collected_dest_paths.append( + DestinationMapping( + destination_path=Path(destination), source_path=Path(source) + ) + ) else: raise FileNotFoundError("Could not find file or directory %s" % source) collected_dest_paths.sort(key=lambda x: x.destination_path) self.check_no_duplicate_destinations(collected_dest_paths) # Now, after we sorted the collected paths, filter out paths which are not needed for current configuration - filtered_dest_paths = [d for d in collected_dest_paths - if d.use_for_hashing(self.hash_file_names, self.hash_directory_names)] - - collected_src_files = [p.source_path for p in collected_dest_paths if p.source_path.is_file()] - return DirectoryMappingResult(sources=collected_src_files, paths_for_hashing=filtered_dest_paths) - - def compute_hashes(self, directory_mapping_result: DirectoryMappingResult) -> List[str]: + filtered_dest_paths = [ + d + for d in collected_dest_paths + if d.use_for_hashing(self.hash_file_names, self.hash_directory_names) + ] + + collected_src_files = [ + p.source_path for p in collected_dest_paths if p.source_path.is_file() + ] + return DirectoryMappingResult( + sources=collected_src_files, paths_for_hashing=filtered_dest_paths + ) + + def compute_hashes( + self, directory_mapping_result: DirectoryMappingResult + ) -> List[str]: paths_for_hashing = directory_mapping_result.paths_for_hashing collected_src_files = directory_mapping_result.sources pool = Pool(processes=self.workers) - dest_path_hashes_future = \ - pool.map_async(self.path_hasher.hash, paths_for_hashing, chunksize=2) - file_content_hashes_future = \ - pool.map_async(self.file_content_hasher.hash, collected_src_files, chunksize=2) + dest_path_hashes_future = pool.map_async( + self.path_hasher.hash, paths_for_hashing, chunksize=2 + ) + file_content_hashes_future = pool.map_async( + self.file_content_hasher.hash, collected_src_files, chunksize=2 + ) hashes = [] hashes.extend(file_content_hashes_future.get()) file_path_hashes_of_directories = dest_path_hashes_future.get() @@ -189,7 +240,7 @@ def compute_hashes(self, directory_mapping_result: DirectoryMappingResult) -> Li return hashes def has_excluded_extension(self, f: str): - return f.split('.')[-1:][0] in self.excluded_extensions + return f.split(".")[-1:][0] in self.excluded_extensions def is_excluded_file(self, f: str): return f in self.excluded_files @@ -197,21 +248,35 @@ def is_excluded_file(self, f: str): def is_excluded_directory(self, f: str): return f in self.excluded_directories - def traverse_directory(self, directory: PurePath, - directory_handler: Callable[[List[str]], None], - file_handler: Callable[[List[str]], None]) -> None: + def traverse_directory( + self, + directory: PurePath, + directory_handler: Callable[[List[str]], None], + file_handler: Callable[[List[str]], None], + ) -> None: symlink_loop_checker = SymlinkLoopChecker() - character_length_checker = CharacterLengthChecker(directory, self.MAX_CHARACTERS_PATHS, - self.hash_directory_names, self.hash_file_names) - - for root, dirs, files in os.walk(directory, topdown=True, followlinks=self.followlinks): + character_length_checker = CharacterLengthChecker( + directory, + self.MAX_CHARACTERS_PATHS, + self.hash_directory_names, + self.hash_file_names, + ) + + for root, dirs, files in os.walk( + directory, topdown=True, followlinks=self.followlinks + ): symlink_loop_checker.check_and_add(root) - new_directories = [os.path.join(root, d) for d in dirs if not self.is_excluded_directory(d)] + new_directories = [ + os.path.join(root, d) for d in dirs if not self.is_excluded_directory(d) + ] directory_handler(new_directories) - new_files = [os.path.join(root, f) for f in files - if not self.is_excluded_file(f) and not self.has_excluded_extension(f)] + new_files = [ + os.path.join(root, f) + for f in files + if not self.is_excluded_file(f) and not self.has_excluded_extension(f) + ] character_length_checker.add_and_check(new_directories, new_files) file_handler(new_files) @@ -223,19 +288,18 @@ def _reduce_hash(self, hashes): class PathHasher: - def __init__(self, hashfunc: str = 'md5', - hash_permissions: bool = False): + def __init__(self, hashfunc: str = "md5", hash_permissions: bool = False): self.hash_permissions = hash_permissions self.hash_func = HASH_FUNCTIONS.get(hashfunc) if not self.hash_func: - raise NotImplementedError('{} not implemented.'.format(hashfunc)) + raise NotImplementedError(f"{hashfunc} not implemented.") def hash(self, path_mapping: DestinationMapping): src_path = path_mapping.source_path dest_path = path_mapping.destination_path assert self.hash_func hasher = self.hash_func() - hasher.update(str(dest_path).encode('utf-8')) + hasher.update(str(dest_path).encode("utf-8")) if self.hash_permissions: stat_result = os.stat(src_path) # we only check the executable right of the user, because git only remembers this @@ -246,16 +310,16 @@ def hash(self, path_mapping: DestinationMapping): class FileContentHasher: - def __init__(self, hashfunc: str = 'md5', blocksize: str = "64kb"): + def __init__(self, hashfunc: str = "md5", blocksize: str = "64kb"): self.blocksize = humanfriendly.parse_size(blocksize) self.hash_func = HASH_FUNCTIONS.get(hashfunc) if not self.hash_func: - raise NotImplementedError('{} not implemented.'.format(hashfunc)) + raise NotImplementedError(f"{hashfunc} not implemented.") def hash(self, filepath: Path): assert self.hash_func hasher = self.hash_func() - with open(filepath, 'rb') as fp: + with open(filepath, "rb") as fp: while True: data = fp.read(self.blocksize) if not data: diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/pull_log_handler.py b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/pull_log_handler.py index 06077a00d..cd9b85488 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/pull_log_handler.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/pull_log_handler.py @@ -1,8 +1,14 @@ import json -from exasol_integration_test_docker_environment.lib.config.log_config import WriteLogFilesToConsole -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo -from exasol_integration_test_docker_environment.lib.logging.abstract_log_handler import AbstractLogHandler +from exasol_integration_test_docker_environment.lib.config.log_config import ( + WriteLogFilesToConsole, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, +) +from exasol_integration_test_docker_environment.lib.logging.abstract_log_handler import ( + AbstractLogHandler, +) class PullLogHandler(AbstractLogHandler): @@ -13,14 +19,16 @@ def __init__(self, log_file_path, logger, image_info: ImageInfo): def handle_log_line(self, log_line, error: bool = False): json_output = json.loads(log_line) - if "status" in json_output \ - and json_output["status"] != "Downloading" \ - and json_output["status"] != "Extracting": + if ( + "status" in json_output + and json_output["status"] != "Downloading" + and json_output["status"] != "Extracting" + ): self._complete_log.append(json_output["status"]) self._log_file.write(json_output["status"]) self._log_file.write("\n") self._log_file.flush() - if 'errorDetail' in json_output: + if "errorDetail" in json_output: self._error_message = json_output["errorDetail"]["message"] def finish(self): @@ -28,18 +36,30 @@ def finish(self): if self._error_message is not None: self.write_error_log_to_console_if_requested() raise Exception( - "Error occurred during the pull of the image %s. Received error \"%s\" ." + 'Error occurred during the pull of the image %s. Received error "%s" .' "The whole log can be found in %s" - % (self._image_info.get_source_complete_name(), self._error_message, self._log_file_path)) + % ( + self._image_info.get_source_complete_name(), + self._error_message, + self._log_file_path, + ) + ) def write_error_log_to_console_if_requested(self): - if self._log_config.write_log_files_to_console == WriteLogFilesToConsole.only_error: - self._logger.error("Pull of image %s failed\nPush Log:\n%s", - self._image_info.get_source_complete_name(), - "\n".join(self._complete_log)) + if ( + self._log_config.write_log_files_to_console + == WriteLogFilesToConsole.only_error + ): + self._logger.error( + "Pull of image %s failed\nPush Log:\n%s", + self._image_info.get_source_complete_name(), + "\n".join(self._complete_log), + ) def write_log_to_console_if_requested(self): if self._log_config.write_log_files_to_console == WriteLogFilesToConsole.all: - self._logger.info("Pull Log of image %s\n%s", - self._image_info.get_source_complete_name(), - "\n".join(self._complete_log)) + self._logger.info( + "Pull Log of image %s\n%s", + self._image_info.get_source_complete_name(), + "\n".join(self._complete_log), + ) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/symlink_loop_checker.py b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/symlink_loop_checker.py index afe8e10f9..0c7c8c263 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/symlink_loop_checker.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/symlink_loop_checker.py @@ -6,6 +6,7 @@ class SymlinkLoopChecker: Collects all visited directory inodes in a set and checks for duplicates. Can be used to check for symlink loops. """ + def __init__(self): self._inodes = set() @@ -19,5 +20,6 @@ def check_and_add(self, directory: str) -> None: stat = os.stat(directory) if stat.st_ino > 0 and stat.st_ino in self._inodes: raise OSError( - f"Directory: {directory} contains symlink loops (Symlinks pointing to a parent directory). Please fix!") + f"Directory: {directory} contains symlink loops (Symlinks pointing to a parent directory). Please fix!" + ) self._inodes.add(stat.st_ino) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/image_info.py b/exasol_integration_test_docker_environment/lib/docker/images/image_info.py index 28203129b..279ad7651 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/image_info.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/image_info.py @@ -1,35 +1,46 @@ from datetime import datetime -from enum import Enum -from typing import Dict, Any, Optional +from enum import ( + Enum, + auto, +) +from typing import ( + Any, + Dict, + Optional, +) from exasol_integration_test_docker_environment.lib.base.info import Info class ImageState(Enum): - NOT_EXISTING = 0, + NOT_EXISTING = auto() # After analyze phase or if build phase did touch the image - NEEDS_TO_BE_BUILD = 1, - TARGET_LOCALLY_AVAILABLE = 2, - SOURCE_LOCALLY_AVAILABLE = 3, - REMOTE_AVAILABLE = 4, - CAN_BE_LOADED = 5, + NEEDS_TO_BE_BUILD = auto() + TARGET_LOCALLY_AVAILABLE = auto() + SOURCE_LOCALLY_AVAILABLE = auto() + REMOTE_AVAILABLE = auto() + CAN_BE_LOADED = auto() # After build phase - WAS_BUILD = 6, - USED_LOCAL = 7, - WAS_PULLED = 8, - WAS_LOADED = 9 - WAS_TAGED = 10 + WAS_BUILD = auto() + USED_LOCAL = auto() + WAS_PULLED = auto() + WAS_LOADED = auto() + WAS_TAGED = auto() class ImageDescription: - def __init__(self, - dockerfile: str, - image_changing_build_arguments: Dict[str, Any], - transparent_build_arguments: Dict[str, Any], - mapping_of_build_files_and_directories: Dict[str, str]): + def __init__( + self, + dockerfile: str, + image_changing_build_arguments: Dict[str, Any], + transparent_build_arguments: Dict[str, Any], + mapping_of_build_files_and_directories: Dict[str, str], + ): self.transparent_build_arguments = transparent_build_arguments self.image_changing_build_arguments = image_changing_build_arguments - self.mapping_of_build_files_and_directories = mapping_of_build_files_and_directories + self.mapping_of_build_files_and_directories = ( + mapping_of_build_files_and_directories + ) self.dockerfile = dockerfile def __repr__(self): @@ -40,15 +51,20 @@ class ImageInfo(Info): DOCKER_TAG_LENGTH_LIMIT = 128 MAX_TAG_SURPLUS = 30 - def __init__(self, - source_repository_name: str, target_repository_name: str, - source_tag: str, target_tag: str, - hash_value: str, commit: str, - image_description: ImageDescription, - build_name: str = "", - build_date_time: datetime = datetime.utcnow(), - image_state: ImageState = ImageState.NOT_EXISTING, - depends_on_images: Optional[Dict[str, "ImageInfo"]] = None): + def __init__( + self, + source_repository_name: str, + target_repository_name: str, + source_tag: str, + target_tag: str, + hash_value: str, + commit: str, + image_description: ImageDescription, + build_name: str = "", + build_date_time: datetime = datetime.utcnow(), + image_state: ImageState = ImageState.NOT_EXISTING, + depends_on_images: Optional[Dict[str, "ImageInfo"]] = None, + ): self.build_name = build_name self.date_time = str(build_date_time) self.commit = commit @@ -74,7 +90,9 @@ def check_complete_tag_length(self, tag): complete_tag_length_limit = self.DOCKER_TAG_LENGTH_LIMIT + self.MAX_TAG_SURPLUS complete_tag = self._create_complete_tag(tag) if len(complete_tag) > complete_tag_length_limit: - raise Exception(f"Complete Tag to long by {len(complete_tag) - complete_tag_length_limit}: {complete_tag}") + raise Exception( + f"Complete Tag to long by {len(complete_tag) - complete_tag_length_limit}: {complete_tag}" + ) def get_target_complete_name(self): return f"{self.target_repository_name}:{self.get_target_complete_tag()}" diff --git a/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_base_task.py b/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_base_task.py index 6320f09c0..474c8126d 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_base_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_base_task.py @@ -2,17 +2,32 @@ import luigi -from exasol_integration_test_docker_environment.abstract_method_exception import AbstractMethodException -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.base.still_running_logger import StillRunningLogger -from exasol_integration_test_docker_environment.lib.config.docker_config import target_docker_repository_config -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageState, ImageInfo -from exasol_integration_test_docker_environment.lib.docker.images.push.push_log_handler import PushLogHandler +from exasol_integration_test_docker_environment.abstract_method_exception import ( + AbstractMethodException, +) +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.still_running_logger import ( + StillRunningLogger, +) +from exasol_integration_test_docker_environment.lib.config.docker_config import ( + target_docker_repository_config, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, + ImageState, +) +from exasol_integration_test_docker_environment.lib.docker.images.push.push_log_handler import ( + PushLogHandler, +) class DockerPushImageBaseTask(DockerBaseTask): image_name = luigi.Parameter() - force_push = luigi.BoolParameter(False, visibility=luigi.parameter.ParameterVisibility.HIDDEN) + force_push = luigi.BoolParameter( + False, visibility=luigi.parameter.ParameterVisibility.HIDDEN + ) def register_required(self): task = self.get_docker_image_task() @@ -28,15 +43,19 @@ def run_task(self): self.logger.info("Push images") auth_config = { "username": target_docker_repository_config().username, - "password": target_docker_repository_config().password + "password": target_docker_repository_config().password, } with self._get_docker_client() as docker_client: - self.logger.info(f"Push images to repo={image_info.get_target_complete_name()}, " - f"tag={image_info.get_target_complete_tag()}") - generator = docker_client.images.push(repository=image_info.get_target_complete_name(), - tag=image_info.get_target_complete_tag(), - auth_config=auth_config, - stream=True) + self.logger.info( + f"Push images to repo={image_info.get_target_complete_name()}, " + f"tag={image_info.get_target_complete_tag()}" + ) + generator = docker_client.images.push( + repository=image_info.get_target_complete_name(), + tag=image_info.get_target_complete_tag(), + auth_config=auth_config, + stream=True, + ) self._handle_output(generator, image_info) self.return_object(image_info) @@ -44,7 +63,8 @@ def _handle_output(self, output_generator, image_info: ImageInfo): log_file_path = Path(self.get_log_path(), "push.log") with PushLogHandler(log_file_path, self.logger, image_info) as log_hanlder: still_running_logger = StillRunningLogger( - self.logger, "push image %s" % image_info.get_target_complete_name()) + self.logger, "push image %s" % image_info.get_target_complete_name() + ) for log_line in output_generator: still_running_logger.log() log_hanlder.handle_log_lines(log_line) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_task.py b/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_task.py index fc6d38ea7..5e5cdc01f 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/push/docker_image_push_task.py @@ -2,10 +2,15 @@ import luigi -from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import JsonPickleParameter -from exasol_integration_test_docker_environment.lib.docker.images.push.docker_image_push_base_task import \ - DockerPushImageBaseTask -from exasol_integration_test_docker_environment.lib.docker.images.required_task_info import RequiredTaskInfo +from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import ( + JsonPickleParameter, +) +from exasol_integration_test_docker_environment.lib.docker.images.push.docker_image_push_base_task import ( + DockerPushImageBaseTask, +) +from exasol_integration_test_docker_environment.lib.docker.images.required_task_info import ( + RequiredTaskInfo, +) class DockerPushImageTask(DockerPushImageBaseTask): @@ -14,9 +19,11 @@ class DockerPushImageTask(DockerPushImageBaseTask): # don't want to wait for the push finishing before starting to build depended images, # but we also need to create a DockerPushImageTask for each DockerCreateImageTask of a goal - required_task_info : RequiredTaskInfo = JsonPickleParameter(RequiredTaskInfo, - visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type:ignore + required_task_info: RequiredTaskInfo = JsonPickleParameter( + RequiredTaskInfo, + visibility=luigi.parameter.ParameterVisibility.HIDDEN, + significant=True, + ) # type:ignore def get_docker_image_task(self): module = importlib.import_module(self.required_task_info.module_name) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/push/docker_push_parameter.py b/exasol_integration_test_docker_environment/lib/docker/images/push/docker_push_parameter.py index 36283cb41..6f81be181 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/push/docker_push_parameter.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/push/docker_push_parameter.py @@ -4,4 +4,4 @@ class DockerPushParameter(Config): force_push = luigi.BoolParameter(False) - push_all = luigi.BoolParameter(False) \ No newline at end of file + push_all = luigi.BoolParameter(False) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/push/push_log_handler.py b/exasol_integration_test_docker_environment/lib/docker/images/push/push_log_handler.py index 58bccbac5..a53175732 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/push/push_log_handler.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/push/push_log_handler.py @@ -1,8 +1,14 @@ import json -from exasol_integration_test_docker_environment.lib.config.log_config import WriteLogFilesToConsole -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo -from exasol_integration_test_docker_environment.lib.logging.abstract_log_handler import AbstractLogHandler +from exasol_integration_test_docker_environment.lib.config.log_config import ( + WriteLogFilesToConsole, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, +) +from exasol_integration_test_docker_environment.lib.logging.abstract_log_handler import ( + AbstractLogHandler, +) class PushLogHandler(AbstractLogHandler): @@ -17,7 +23,7 @@ def handle_log_line(self, log_line, error: bool = False): self._complete_log.append(json_output["status"]) self._log_file.write(json_output["status"]) self._log_file.write("\n") - if 'errorDetail' in json_output: + if "errorDetail" in json_output: self._error_message = json_output["errorDetail"]["message"] def finish(self): @@ -25,18 +31,30 @@ def finish(self): if self._error_message is not None: self.write_error_log_to_console_if_requested() raise Exception( - "Error occurred during the push of the image %s. Received error \"%s\" ." + 'Error occurred during the push of the image %s. Received error "%s" .' "The whole log can be found in %s" - % (self._image_info.get_target_complete_name(), self._error_message, self._log_file_path)) + % ( + self._image_info.get_target_complete_name(), + self._error_message, + self._log_file_path, + ) + ) def write_error_log_to_console_if_requested(self): - if self._log_config.write_log_files_to_console == WriteLogFilesToConsole.only_error: - self._logger.error("Push of image %s failed\nPush Log:\n%s", - self._image_info.get_target_complete_name(), - "\n".join(self._complete_log)) + if ( + self._log_config.write_log_files_to_console + == WriteLogFilesToConsole.only_error + ): + self._logger.error( + "Push of image %s failed\nPush Log:\n%s", + self._image_info.get_target_complete_name(), + "\n".join(self._complete_log), + ) def write_log_to_console_if_requested(self): if self._log_config.write_log_files_to_console == WriteLogFilesToConsole.all: - self._logger.info("Push Log of image %s\n%s", - self._image_info.get_target_complete_name(), - "\n".join(self._complete_log)) + self._logger.info( + "Push Log of image %s\n%s", + self._image_info.get_target_complete_name(), + "\n".join(self._complete_log), + ) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/push/push_task_creator_for_build_tasks.py b/exasol_integration_test_docker_environment/lib/docker/images/push/push_task_creator_for_build_tasks.py index 09f035cde..9d5cf6bc3 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/push/push_task_creator_for_build_tasks.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/push/push_task_creator_for_build_tasks.py @@ -1,7 +1,10 @@ from exasol_integration_test_docker_environment.lib.base.base_task import BaseTask -from exasol_integration_test_docker_environment.lib.docker.images.push.docker_image_push_task import DockerPushImageTask -from exasol_integration_test_docker_environment.lib.docker.images.task_creator_from_build_tasks import \ - TaskCreatorFromBuildTasks +from exasol_integration_test_docker_environment.lib.docker.images.push.docker_image_push_task import ( + DockerPushImageTask, +) +from exasol_integration_test_docker_environment.lib.docker.images.task_creator_from_build_tasks import ( + TaskCreatorFromBuildTasks, +) class PushTaskCreatorFromBuildTasks(TaskCreatorFromBuildTasks): @@ -10,9 +13,9 @@ def __init__(self, task: BaseTask): self.task = task def create_task_with_required_tasks(self, build_task, required_task_info): - push_task = \ - self.task.create_child_task_with_common_params( - DockerPushImageTask, - image_name=build_task.image_name, - required_task_info=required_task_info) + push_task = self.task.create_child_task_with_common_params( + DockerPushImageTask, + image_name=build_task.image_name, + required_task_info=required_task_info, + ) return push_task diff --git a/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_base_task.py b/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_base_task.py index d12a15baa..ba3812ec7 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_base_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_base_task.py @@ -5,10 +5,19 @@ import luigi -from exasol_integration_test_docker_environment.abstract_method_exception import AbstractMethodException -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.base.still_running_logger import StillRunningLogger -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageState, ImageInfo +from exasol_integration_test_docker_environment.abstract_method_exception import ( + AbstractMethodException, +) +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.still_running_logger import ( + StillRunningLogger, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, + ImageState, +) DOCKER_HUB_REGISTRY_URL_REGEX = r"^.*docker.io/" @@ -16,7 +25,9 @@ # TODO align and extract save_path of DockerSaveImageTask and load_path of DockerLoadImageTask class DockerSaveImageBaseTask(DockerBaseTask): image_name = luigi.Parameter() - force_save = luigi.BoolParameter(False, visibility=luigi.parameter.ParameterVisibility.HIDDEN) + force_save = luigi.BoolParameter( + False, visibility=luigi.parameter.ParameterVisibility.HIDDEN + ) save_path = luigi.Parameter(visibility=luigi.parameter.ParameterVisibility.HIDDEN) def register_required(self): @@ -29,17 +40,23 @@ def get_docker_image_task(self): def run_task(self): image_info = self.get_values_from_future(self._image_info_future) tag_for_save = self.get_tag_for_save(image_info) - save_file_path = pathlib.Path(self.save_path, f"{image_info.get_target_complete_name()}.tar") + save_file_path = pathlib.Path( + self.save_path, f"{image_info.get_target_complete_name()}.tar" + ) was_build = image_info.image_state == ImageState.WAS_BUILD.name if was_build or self.force_save or not save_file_path.exists(): self.save_image(image_info, tag_for_save, save_file_path) self.return_object(image_info) def get_tag_for_save(self, image_info: ImageInfo): - tag_for_save = re.sub(DOCKER_HUB_REGISTRY_URL_REGEX, "", image_info.get_target_complete_name()) + tag_for_save = re.sub( + DOCKER_HUB_REGISTRY_URL_REGEX, "", image_info.get_target_complete_name() + ) return tag_for_save - def save_image(self, image_info: ImageInfo, tag_for_save: str, save_file_path: pathlib.Path): + def save_image( + self, image_info: ImageInfo, tag_for_save: str, save_file_path: pathlib.Path + ): self.remove_save_file_if_necassary(save_file_path) with self._get_docker_client() as docker_client: image = docker_client.images.get(image_info.get_target_complete_name()) @@ -51,15 +68,19 @@ def remove_save_file_if_necassary(self, save_file_path: pathlib.Path): if save_file_path.exists(): os.remove(str(save_file_path)) - def write_image_to_file(self, - save_file_path: pathlib.Path, - image_info: ImageInfo, - output_generator: Generator): + def write_image_to_file( + self, + save_file_path: pathlib.Path, + image_info: ImageInfo, + output_generator: Generator, + ): self.logger.info( - f"Saving image {image_info.get_target_complete_name()} to file {save_file_path}") + f"Saving image {image_info.get_target_complete_name()} to file {save_file_path}" + ) with save_file_path.open("wb") as file: still_running_logger = StillRunningLogger( - self.logger, "save image %s" % image_info.get_target_complete_name()) + self.logger, "save image %s" % image_info.get_target_complete_name() + ) for chunk in output_generator: still_running_logger.log() file.write(chunk) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_task.py b/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_task.py index 9accd37d4..41842e051 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/save/docker_image_save_task.py @@ -2,10 +2,15 @@ import luigi -from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import JsonPickleParameter -from exasol_integration_test_docker_environment.lib.docker.images.required_task_info import RequiredTaskInfo -from exasol_integration_test_docker_environment.lib.docker.images.save.docker_image_save_base_task import \ - DockerSaveImageBaseTask +from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import ( + JsonPickleParameter, +) +from exasol_integration_test_docker_environment.lib.docker.images.required_task_info import ( + RequiredTaskInfo, +) +from exasol_integration_test_docker_environment.lib.docker.images.save.docker_image_save_base_task import ( + DockerSaveImageBaseTask, +) class DockerSaveImageTask(DockerSaveImageBaseTask): @@ -14,9 +19,11 @@ class DockerSaveImageTask(DockerSaveImageBaseTask): # don't want to wait for the save finishing before starting to build depended images, # but we also need to create a DockerSaveImageTask for each DockerCreateImageTask of a goal - required_task_info : RequiredTaskInfo = JsonPickleParameter(RequiredTaskInfo, - visibility=luigi.parameter.ParameterVisibility.HIDDEN, - significant=True) # type: ignore + required_task_info: RequiredTaskInfo = JsonPickleParameter( + RequiredTaskInfo, + visibility=luigi.parameter.ParameterVisibility.HIDDEN, + significant=True, + ) # type: ignore def get_docker_image_task(self): module = importlib.import_module(self.required_task_info.module_name) diff --git a/exasol_integration_test_docker_environment/lib/docker/images/save/save_task_creator_for_build_tasks.py b/exasol_integration_test_docker_environment/lib/docker/images/save/save_task_creator_for_build_tasks.py index 9b7efc9f6..08bb2bf33 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/save/save_task_creator_for_build_tasks.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/save/save_task_creator_for_build_tasks.py @@ -1,7 +1,10 @@ from exasol_integration_test_docker_environment.lib.base.base_task import BaseTask -from exasol_integration_test_docker_environment.lib.docker.images.save.docker_image_save_task import DockerSaveImageTask -from exasol_integration_test_docker_environment.lib.docker.images.task_creator_from_build_tasks import \ - TaskCreatorFromBuildTasks +from exasol_integration_test_docker_environment.lib.docker.images.save.docker_image_save_task import ( + DockerSaveImageTask, +) +from exasol_integration_test_docker_environment.lib.docker.images.task_creator_from_build_tasks import ( + TaskCreatorFromBuildTasks, +) class SaveTaskCreatorFromBuildTasks(TaskCreatorFromBuildTasks): @@ -10,11 +13,9 @@ def __init__(self, task: BaseTask): self.task = task def create_task_with_required_tasks(self, build_task, required_task_info): - push_task = \ - self.task.create_child_task_with_common_params( - DockerSaveImageTask, - image_name=build_task.image_name, - required_task_info=required_task_info, - - ) + push_task = self.task.create_child_task_with_common_params( + DockerSaveImageTask, + image_name=build_task.image_name, + required_task_info=required_task_info, + ) return push_task diff --git a/exasol_integration_test_docker_environment/lib/docker/images/task_creator_from_build_tasks.py b/exasol_integration_test_docker_environment/lib/docker/images/task_creator_from_build_tasks.py index d3d5283d0..645edc463 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/task_creator_from_build_tasks.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/task_creator_from_build_tasks.py @@ -1,23 +1,36 @@ import itertools -from typing import Dict, List - -from exasol_integration_test_docker_environment.abstract_method_exception import AbstractMethodException +from typing import ( + Dict, + List, +) + +from exasol_integration_test_docker_environment.abstract_method_exception import ( + AbstractMethodException, +) from exasol_integration_test_docker_environment.lib.base.base_task import BaseTask -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_create_task import \ - DockerCreateImageTask, DockerCreateImageTaskWithDeps -from exasol_integration_test_docker_environment.lib.docker.images.required_task_info import RequiredTaskInfo +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_create_task import ( + DockerCreateImageTask, + DockerCreateImageTaskWithDeps, +) +from exasol_integration_test_docker_environment.lib.docker.images.required_task_info import ( + RequiredTaskInfo, +) class TaskCreatorFromBuildTasks: - def create_tasks_for_build_tasks(self, build_tasks: Dict[str, DockerCreateImageTask]) \ - -> List[BaseTask]: - tasks_per_goal = [self._create_tasks_for_build_task(build_task) - for goal, build_task in build_tasks.items()] + def create_tasks_for_build_tasks( + self, build_tasks: Dict[str, DockerCreateImageTask] + ) -> List[BaseTask]: + tasks_per_goal = [ + self._create_tasks_for_build_task(build_task) + for goal, build_task in build_tasks.items() + ] return list(itertools.chain.from_iterable(tasks_per_goal)) - def _create_tasks_for_build_task(self, build_task: DockerCreateImageTask) \ - -> List[BaseTask]: + def _create_tasks_for_build_task( + self, build_task: DockerCreateImageTask + ) -> List[BaseTask]: if isinstance(build_task, DockerCreateImageTaskWithDeps): tasks = self.create_tasks_for_build_tasks(build_task.required_tasks) task = self._create_task(build_task) @@ -32,11 +45,14 @@ def _create_task(self, build_task): return task def _create_required_task_info(self, build_task: DockerCreateImageTask): - required_task_info = \ - RequiredTaskInfo(module_name=build_task.__module__, - class_name=build_task.__class__.__name__, - params=build_task.param_kwargs) + required_task_info = RequiredTaskInfo( + module_name=build_task.__module__, + class_name=build_task.__class__.__name__, + params=build_task.param_kwargs, + ) return required_task_info - def create_task_with_required_tasks(self, build_task, required_task_info) -> BaseTask: + def create_task_with_required_tasks( + self, build_task, required_task_info + ) -> BaseTask: raise AbstractMethodException() diff --git a/exasol_integration_test_docker_environment/lib/docker/images/utils.py b/exasol_integration_test_docker_environment/lib/docker/images/utils.py index c9f14763b..288d28713 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/utils.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/utils.py @@ -1,9 +1,16 @@ -from typing import Callable, List +from typing import ( + Callable, + List, +) def find_images_by_tag(docker_client, condition: Callable[[str], bool]) -> List: images = docker_client.images.list() - filter_images = [image for image in images - if image.tags is not None and len(image.tags) > 0 and - any([condition(tag) for tag in image.tags])] + filter_images = [ + image + for image in images + if image.tags is not None + and len(image.tags) > 0 + and any([condition(tag) for tag in image.tags]) + ] return filter_images diff --git a/exasol_integration_test_docker_environment/lib/logging/abstract_log_handler.py b/exasol_integration_test_docker_environment/lib/logging/abstract_log_handler.py index ece1aaf04..359740574 100644 --- a/exasol_integration_test_docker_environment/lib/logging/abstract_log_handler.py +++ b/exasol_integration_test_docker_environment/lib/logging/abstract_log_handler.py @@ -22,10 +22,10 @@ def __exit__(self, type, value, traceback): def handle_log_lines(self, log_lines, error: bool = False): log_lines = log_lines.decode("utf-8") - log_lines = log_lines.strip('\r\n') + log_lines = log_lines.strip("\r\n") result = [] for log_line in log_lines.split("\n"): - log_line = log_line.strip('\r\n') + log_line = log_line.strip("\r\n") result.append(self.handle_log_line(log_line, error)) return result diff --git a/exasol_integration_test_docker_environment/lib/logging/command_log_handler.py b/exasol_integration_test_docker_environment/lib/logging/command_log_handler.py index bf1c0e58f..db277dac8 100644 --- a/exasol_integration_test_docker_environment/lib/logging/command_log_handler.py +++ b/exasol_integration_test_docker_environment/lib/logging/command_log_handler.py @@ -1,7 +1,11 @@ import pathlib -from exasol_integration_test_docker_environment.lib.config.log_config import WriteLogFilesToConsole -from exasol_integration_test_docker_environment.lib.logging.abstract_log_handler import AbstractLogHandler +from exasol_integration_test_docker_environment.lib.config.log_config import ( + WriteLogFilesToConsole, +) +from exasol_integration_test_docker_environment.lib.logging.abstract_log_handler import ( + AbstractLogHandler, +) class CommandLogHandler(AbstractLogHandler): @@ -18,18 +22,28 @@ def handle_log_line(self, log_line, error: bool = False): def finish(self): if self._log_config.write_log_files_to_console == WriteLogFilesToConsole.all: - self._logger.info("Command log for %s \n%s", - self._description, - "".join(self._complete_log)) + self._logger.info( + "Command log for %s \n%s", + self._description, + "".join(self._complete_log), + ) if self._error_message is not None: - if self._log_config.write_log_files_to_console == WriteLogFilesToConsole.only_error: - self._logger.error("Command failed %s failed\nCommand Log:\n%s", - self._description, - "\n".join(self._complete_log)) + if ( + self._log_config.write_log_files_to_console + == WriteLogFilesToConsole.only_error + ): + self._logger.error( + "Command failed %s failed\nCommand Log:\n%s", + self._description, + "\n".join(self._complete_log), + ) raise Exception( - "Error occurred during %s. Received error \"%s\" ." + 'Error occurred during %s. Received error "%s" .' "The whole log can be found in %s" - % (self._description, - self._error_message, - self._log_file_path.absolute()), - self._log_file_path.absolute()) + % ( + self._description, + self._error_message, + self._log_file_path.absolute(), + ), + self._log_file_path.absolute(), + ) diff --git a/exasol_integration_test_docker_environment/lib/logging/container_log_handler.py b/exasol_integration_test_docker_environment/lib/logging/container_log_handler.py index 49874fa6c..5b7393bf9 100644 --- a/exasol_integration_test_docker_environment/lib/logging/container_log_handler.py +++ b/exasol_integration_test_docker_environment/lib/logging/container_log_handler.py @@ -1,7 +1,11 @@ from typing import List -from exasol_integration_test_docker_environment.lib.config.log_config import WriteLogFilesToConsole -from exasol_integration_test_docker_environment.lib.logging.abstract_log_handler import AbstractLogHandler +from exasol_integration_test_docker_environment.lib.config.log_config import ( + WriteLogFilesToConsole, +) +from exasol_integration_test_docker_environment.lib.logging.abstract_log_handler import ( + AbstractLogHandler, +) class ContainerLogHandler(AbstractLogHandler): @@ -19,5 +23,6 @@ def get_complete_log(self) -> List[str]: def finish(self): if self._log_config.write_log_files_to_console == WriteLogFilesToConsole.all: - self._logger.info("Log %s\n%s", self.db_container_name, - "\n".join(self._complete_log)) + self._logger.info( + "Log %s\n%s", self.db_container_name, "\n".join(self._complete_log) + ) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py index 4a88daa70..b26f151ea 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/abstract_spawn_test_environment.py @@ -1,34 +1,63 @@ from pathlib import Path -from typing import Generator, Tuple, Optional, Any -from exasol_integration_test_docker_environment.lib.docker.container.utils import default_bridge_ip_address +from typing import ( + Any, + Generator, + Optional, + Tuple, +) import luigi -from exasol_integration_test_docker_environment.abstract_method_exception import AbstractMethodException +from exasol_integration_test_docker_environment.abstract_method_exception import ( + AbstractMethodException, +) from exasol_integration_test_docker_environment.lib.base.base_task import BaseTask -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.data.container_info import ContainerInfo -from exasol_integration_test_docker_environment.lib.data.database_credentials import DatabaseCredentialsParameter -from exasol_integration_test_docker_environment.lib.data.database_info import DatabaseInfo -from exasol_integration_test_docker_environment.lib.data.docker_network_info import DockerNetworkInfo -from exasol_integration_test_docker_environment.lib.data.docker_volume_info import DockerVolumeInfo -from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo -from exasol_integration_test_docker_environment.lib.test_environment.docker_container_copy import DockerContainerCopy -from exasol_integration_test_docker_environment.lib.test_environment.parameter.general_spawn_test_environment_parameter import \ - GeneralSpawnTestEnvironmentParameter -from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_container import SpawnTestContainer -from exasol_integration_test_docker_environment.lib.test_environment.shell_variables import ShellVariables - +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) +from exasol_integration_test_docker_environment.lib.data.container_info import ( + ContainerInfo, +) +from exasol_integration_test_docker_environment.lib.data.database_credentials import ( + DatabaseCredentialsParameter, +) +from exasol_integration_test_docker_environment.lib.data.database_info import ( + DatabaseInfo, +) +from exasol_integration_test_docker_environment.lib.data.docker_network_info import ( + DockerNetworkInfo, +) +from exasol_integration_test_docker_environment.lib.data.docker_volume_info import ( + DockerVolumeInfo, +) +from exasol_integration_test_docker_environment.lib.data.environment_info import ( + EnvironmentInfo, +) +from exasol_integration_test_docker_environment.lib.docker.container.utils import ( + default_bridge_ip_address, +) +from exasol_integration_test_docker_environment.lib.test_environment.docker_container_copy import ( + DockerContainerCopy, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.general_spawn_test_environment_parameter import ( + GeneralSpawnTestEnvironmentParameter, +) +from exasol_integration_test_docker_environment.lib.test_environment.shell_variables import ( + ShellVariables, +) +from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_container import ( + SpawnTestContainer, +) DATABASE = "database" TEST_CONTAINER = "test_container" -class AbstractSpawnTestEnvironment(DockerBaseTask, - GeneralSpawnTestEnvironmentParameter, - DatabaseCredentialsParameter): - environment_name : str = luigi.Parameter() # type: ignore +class AbstractSpawnTestEnvironment( + DockerBaseTask, GeneralSpawnTestEnvironmentParameter, DatabaseCredentialsParameter +): + environment_name: str = luigi.Parameter() # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -48,39 +77,50 @@ def _attempt_database_start(self): database_info = None test_container_info = None while not is_database_ready and attempt < self.max_start_attempts: - network_info, database_info, is_database_ready, test_container_info = \ + network_info, database_info, is_database_ready, test_container_info = ( yield from self._start_database(attempt) + ) attempt += 1 if not is_database_ready and not attempt < self.max_start_attempts: - raise Exception(f"Maximum attempts {attempt} to start the database reached.") - test_environment_info = \ - EnvironmentInfo(name=self.environment_name, - env_type=self.get_environment_type(), - database_info=database_info, - test_container_info=test_container_info, - network_info=network_info) - self.create_test_environment_info_in_test_container_and_on_host(test_environment_info) + raise Exception( + f"Maximum attempts {attempt} to start the database reached." + ) + test_environment_info = EnvironmentInfo( + name=self.environment_name, + env_type=self.get_environment_type(), + database_info=database_info, + test_container_info=test_container_info, + network_info=network_info, + ) + self.create_test_environment_info_in_test_container_and_on_host( + test_environment_info + ) return test_environment_info def create_test_environment_info_in_test_container( - self, - test_environment_info: EnvironmentInfo, - shell_variables: ShellVariables, - json: str, + self, + test_environment_info: EnvironmentInfo, + shell_variables: ShellVariables, + json: str, ): assert test_environment_info.test_container_info test_container_name = test_environment_info.test_container_info.container_name with self._get_docker_client() as docker_client: test_container = docker_client.containers.get(test_container_name) - self.logger.info(f"Create test environment info in test container '{test_container_name}' at '/'") + self.logger.info( + f"Create test environment info in test container '{test_container_name}' at '/'" + ) copy = DockerContainerCopy(test_container) copy.add_string_to_file("environment_info.json", json) copy.add_string_to_file("environment_info.conf", shell_variables.render()) - copy.add_string_to_file("environment_info.sh", shell_variables.render("export ")) + copy.add_string_to_file( + "environment_info.sh", shell_variables.render("export ") + ) copy.copy("/") def create_test_environment_info_in_test_container_and_on_host( - self, test_environment_info: EnvironmentInfo): + self, test_environment_info: EnvironmentInfo + ): path = Path(self.get_cache_path(), f"environments/{self.environment_name}") path.mkdir(exist_ok=True, parents=True) self.logger.info(f"Create test environment info on the host at '{path}'") @@ -105,7 +145,9 @@ def create_test_environment_info_in_test_container_and_on_host( def _default_bridge_ip_address(self, test_environment_info) -> Optional[str]: if test_environment_info.database_info.container_info is not None: - container_name = test_environment_info.database_info.container_info.container_name + container_name = ( + test_environment_info.database_info.container_info.container_name + ) with self._get_docker_client() as docker_client: db_container = docker_client.containers.get(container_name) return default_bridge_ip_address(db_container) @@ -117,26 +159,39 @@ def collect_shell_variables(self, test_environment_info) -> ShellVariables: test_environment_info, ) - def _start_database(self, attempt) \ - -> Generator[Any, None, Tuple[DockerNetworkInfo, DatabaseInfo, bool, Optional[ContainerInfo]]]: + def _start_database( + self, attempt + ) -> Generator[ + Any, None, Tuple[DockerNetworkInfo, DatabaseInfo, bool, Optional[ContainerInfo]] + ]: network_info = yield from self._create_network(attempt) ssl_volume_info = None if self.create_certificates: ssl_volume_info = yield from self._create_ssl_certificates() - database_info, test_container_info = yield from self._spawn_database_and_test_container(network_info, ssl_volume_info, attempt) + database_info, test_container_info = ( + yield from self._spawn_database_and_test_container( + network_info, ssl_volume_info, attempt + ) + ) is_database_ready = yield from self._wait_for_database(database_info, attempt) return network_info, database_info, is_database_ready, test_container_info - def _create_ssl_certificates(self) -> Generator[BaseTask, None, Optional[DockerVolumeInfo]]: - ssl_volume_info_future = yield from self.run_dependencies(self.create_ssl_certificates()) - ssl_volume_info : Optional[DockerVolumeInfo] = self.get_values_from_future(ssl_volume_info_future) # type: ignore + def _create_ssl_certificates( + self, + ) -> Generator[BaseTask, None, Optional[DockerVolumeInfo]]: + ssl_volume_info_future = yield from self.run_dependencies( + self.create_ssl_certificates() + ) + ssl_volume_info: Optional[DockerVolumeInfo] = self.get_values_from_future(ssl_volume_info_future) # type: ignore return ssl_volume_info def create_ssl_certificates(self): raise AbstractMethodException() def _create_network(self, attempt): - network_info_future = yield from self.run_dependencies(self.create_network_task(attempt)) + network_info_future = yield from self.run_dependencies( + self.create_network_task(attempt) + ) network_info = self.get_values_from_future(network_info_future) return network_info @@ -144,10 +199,10 @@ def create_network_task(self, attempt: int): raise AbstractMethodException() def _spawn_database_and_test_container( - self, - network_info: DockerNetworkInfo, - certificate_volume_info: Optional[DockerVolumeInfo], - attempt: int, + self, + network_info: DockerNetworkInfo, + certificate_volume_info: Optional[DockerVolumeInfo], + attempt: int, ) -> Generator[BaseTask, None, Tuple[DatabaseInfo, Optional[ContainerInfo]]]: def volume_name(info): return None if info is None else info.volume_name @@ -168,37 +223,40 @@ def volume_name(info): ) futures = yield from self.run_dependencies(child_tasks) results = self.get_values_from_futures(futures) - database_info : DatabaseInfo = results[DATABASE] # type: ignore - test_container_info : Optional[ContainerInfo] = results[TEST_CONTAINER] if self.test_container_content is not None else None # type: ignore + database_info: DatabaseInfo = results[DATABASE] # type: ignore + test_container_info: Optional[ContainerInfo] = results[TEST_CONTAINER] if self.test_container_content is not None else None # type: ignore return database_info, test_container_info - def create_spawn_database_task(self, - network_info: DockerNetworkInfo, - certificate_volume_info: Optional[DockerVolumeInfo], - attempt: int): + def create_spawn_database_task( + self, + network_info: DockerNetworkInfo, + certificate_volume_info: Optional[DockerVolumeInfo], + attempt: int, + ): raise AbstractMethodException() - def create_spawn_test_container_task(self, network_info: DockerNetworkInfo, - certificate_volume_name: str, attempt: int): + def create_spawn_test_container_task( + self, + network_info: DockerNetworkInfo, + certificate_volume_name: str, + attempt: int, + ): return self.create_child_task_with_common_params( - SpawnTestContainer, - test_container_name=self.test_container_name, - network_info=network_info, - ip_address_index_in_subnet=1, - certificate_volume_name=certificate_volume_name, - attempt=attempt, - test_container_content=self.test_container_content - ) - - def _wait_for_database(self, - database_info: DatabaseInfo, - attempt: int): - database_ready_target_future = \ - yield from self.run_dependencies(self.create_wait_for_database_task(attempt, database_info)) + SpawnTestContainer, + test_container_name=self.test_container_name, + network_info=network_info, + ip_address_index_in_subnet=1, + certificate_volume_name=certificate_volume_name, + attempt=attempt, + test_container_content=self.test_container_content, + ) + + def _wait_for_database(self, database_info: DatabaseInfo, attempt: int): + database_ready_target_future = yield from self.run_dependencies( + self.create_wait_for_database_task(attempt, database_info) + ) is_database_ready = self.get_values_from_futures(database_ready_target_future) return is_database_ready - def create_wait_for_database_task(self, - attempt: int, - database_info: DatabaseInfo): + def create_wait_for_database_task(self, attempt: int, database_info: DatabaseInfo): raise AbstractMethodException() diff --git a/exasol_integration_test_docker_environment/lib/test_environment/analyze_test_container.py b/exasol_integration_test_docker_environment/lib/test_environment/analyze_test_container.py index df4291662..02500a2ca 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/analyze_test_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/analyze_test_container.py @@ -1,16 +1,30 @@ -from typing import Set, Dict - -from exasol_integration_test_docker_environment.lib.config.build_config import build_config -from exasol_integration_test_docker_environment.lib.config.docker_config import target_docker_repository_config, \ - source_docker_repository_config -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_build_base import DockerBuildBase -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_analyze_task import \ - DockerAnalyzeImageTask -from exasol_integration_test_docker_environment.lib.docker.images.push.docker_push_parameter import DockerPushParameter -from exasol_integration_test_docker_environment.lib.docker.images.push.push_task_creator_for_build_tasks import \ - PushTaskCreatorFromBuildTasks -from exasol_integration_test_docker_environment.lib.test_environment.parameter.test_container_parameter import \ - TestContainerParameter +from typing import ( + Dict, + Set, +) + +from exasol_integration_test_docker_environment.lib.config.build_config import ( + build_config, +) +from exasol_integration_test_docker_environment.lib.config.docker_config import ( + source_docker_repository_config, + target_docker_repository_config, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_build_base import ( + DockerBuildBase, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_analyze_task import ( + DockerAnalyzeImageTask, +) +from exasol_integration_test_docker_environment.lib.docker.images.push.docker_push_parameter import ( + DockerPushParameter, +) +from exasol_integration_test_docker_environment.lib.docker.images.push.push_task_creator_for_build_tasks import ( + PushTaskCreatorFromBuildTasks, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.test_container_parameter import ( + TestContainerParameter, +) class AnalyzeTestContainer(DockerAnalyzeImageTask, TestContainerParameter): @@ -34,8 +48,10 @@ def get_target_image_tag(self): return f"db-test-container" def get_mapping_of_build_files_and_directories(self): - return {mapping.target: str(mapping.source) for mapping - in self.test_container_content.build_files_and_directories} + return { + mapping.target: str(mapping.source) + for mapping in self.test_container_content.build_files_and_directories + } def get_dockerfile(self): return str(self.test_container_content.docker_file) @@ -48,8 +64,12 @@ def is_rebuild_requested(self) -> bool: class DockerTestContainerBuildBase(DockerBuildBase, TestContainerParameter): def get_goal_class_map(self) -> Dict[str, DockerAnalyzeImageTask]: - goal_class_map = {"test-container": self.create_child_task(task_class=AnalyzeTestContainer, - test_container_content=self.test_container_content)} + goal_class_map = { + "test-container": self.create_child_task( + task_class=AnalyzeTestContainer, + test_container_content=self.test_container_content, + ) + } return goal_class_map def get_default_goals(self) -> Set[str]: diff --git a/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/analyze_certificate_container.py b/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/analyze_certificate_container.py index 8637dc0c4..2bc09130d 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/analyze_certificate_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/analyze_certificate_container.py @@ -1,15 +1,26 @@ -from typing import Set, Dict +from typing import ( + Dict, + Set, +) import luigi -from exasol_integration_test_docker_environment.lib.config.docker_config import target_docker_repository_config, \ - source_docker_repository_config -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_build_base import DockerBuildBase -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_analyze_task import \ - DockerAnalyzeImageTask -from exasol_integration_test_docker_environment.lib.docker.images.push.docker_push_parameter import DockerPushParameter -from exasol_integration_test_docker_environment.lib.docker.images.push.push_task_creator_for_build_tasks import \ - PushTaskCreatorFromBuildTasks +from exasol_integration_test_docker_environment.lib.config.docker_config import ( + source_docker_repository_config, + target_docker_repository_config, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_build_base import ( + DockerBuildBase, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_analyze_task import ( + DockerAnalyzeImageTask, +) +from exasol_integration_test_docker_environment.lib.docker.images.push.docker_push_parameter import ( + DockerPushParameter, +) +from exasol_integration_test_docker_environment.lib.docker.images.push.push_task_creator_for_build_tasks import ( + PushTaskCreatorFromBuildTasks, +) TAG_SUFFIX = "certificate_resources" @@ -36,7 +47,9 @@ def get_target_image_tag(self): return TAG_SUFFIX def get_mapping_of_build_files_and_directories(self): - return {"create_certificates.sh": f"{self.certificate_container_root_directory}/create_certificates.sh"} + return { + "create_certificates.sh": f"{self.certificate_container_root_directory}/create_certificates.sh" + } def get_dockerfile(self): return f"{self.certificate_container_root_directory}/Dockerfile" @@ -50,9 +63,12 @@ class DockerCertificateBuildBase(DockerBuildBase): certificate_container_root_directory = luigi.Parameter() def get_goal_class_map(self) -> Dict[str, DockerAnalyzeImageTask]: - goal_class_map = {self.GOAL: self.create_child_task(task_class=AnalyzeCertificateContainer, - certificate_container_root_directory= - self.certificate_container_root_directory)} + goal_class_map = { + self.GOAL: self.create_child_task( + task_class=AnalyzeCertificateContainer, + certificate_container_root_directory=self.certificate_container_root_directory, + ) + } return goal_class_map def get_default_goals(self) -> Set[str]: diff --git a/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py b/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py index ae5a6f6ec..9dea8cef0 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/create_certificates/create_ssl_certificates_task.py @@ -1,43 +1,61 @@ -from typing import Dict, Optional, Generator, Set +from typing import ( + Dict, + Generator, + Optional, + Set, +) import docker import luigi -from exasol_integration_test_docker_environment.lib.base.base_task import BaseTask -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.base.pickle_target import PickleTarget -from exasol_integration_test_docker_environment.lib.data.docker_volume_info import DockerVolumeInfo -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo -from exasol_integration_test_docker_environment.lib.test_environment.create_certificates.analyze_certificate_container import \ - DockerCertificateContainerBuild, DockerCertificateBuildBase -from exasol_integration_test_docker_environment.lib.utils.resource_directory import ResourceDirectory import exasol_integration_test_docker_environment.certificate_resources.container +from exasol_integration_test_docker_environment.lib.base.base_task import BaseTask +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.pickle_target import ( + PickleTarget, +) +from exasol_integration_test_docker_environment.lib.data.docker_volume_info import ( + DockerVolumeInfo, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, +) +from exasol_integration_test_docker_environment.lib.test_environment.create_certificates.analyze_certificate_container import ( + DockerCertificateBuildBase, + DockerCertificateContainerBuild, +) +from exasol_integration_test_docker_environment.lib.utils.resource_directory import ( + ResourceDirectory, +) CERTIFICATES_MOUNT_PATH = "/certificates" class CreateSSLCertificatesTask(DockerBaseTask): - environment_name : str = luigi.Parameter() # type: ignore - docker_runtime : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore - db_container_name : str = luigi.Parameter(significant=False) # type: ignore - network_name : str = luigi.Parameter() # type: ignore - reuse : bool = luigi.BoolParameter(False, significant=False) # type: ignore - no_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) # type: ignore - no_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) # type: ignore - volume_name : str = luigi.Parameter() # type: ignore + environment_name: str = luigi.Parameter() # type: ignore + docker_runtime: Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + db_container_name: str = luigi.Parameter(significant=False) # type: ignore + network_name: str = luigi.Parameter() # type: ignore + reuse: bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_cleanup_after_success: bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_cleanup_after_failure: bool = luigi.BoolParameter(False, significant=False) # type: ignore + volume_name: str = luigi.Parameter() # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._temp_resource_directory = \ - ResourceDirectory(exasol_integration_test_docker_environment.certificate_resources.container) + self._temp_resource_directory = ResourceDirectory( + exasol_integration_test_docker_environment.certificate_resources.container + ) self._temp_resource_directory.create() def on_failure(self, exception): - super(CreateSSLCertificatesTask, self).on_failure(exception) + super().on_failure(exception) self._temp_resource_directory.cleanup() def on_success(self): - super(CreateSSLCertificatesTask, self).on_success() + super().on_success() self._temp_resource_directory.cleanup() def run_task(self): @@ -48,8 +66,10 @@ def run_task(self): try: self.volume_info = self.get_volume_info(reused=True) except Exception as e: - self.logger.warning(f"Tried to reuse volume {self.volume_name}, but got Exeception {e}. " - "Fallback to create new volume.") + self.logger.warning( + f"Tried to reuse volume {self.volume_name}, but got Exeception {e}. " + "Fallback to create new volume." + ) if self.volume_info is None: self.volume_info = self.create_docker_volume() self.create_certificate(image_infos) @@ -57,19 +77,26 @@ def run_task(self): self.return_object(self.volume_info) def build_image(self) -> Generator[BaseTask, None, Set[ImageInfo]]: - task = self.create_child_task(task_class=DockerCertificateContainerBuild, - certificate_container_root_directory=self._temp_resource_directory.tmp_directory) + task = self.create_child_task( + task_class=DockerCertificateContainerBuild, + certificate_container_root_directory=self._temp_resource_directory.tmp_directory, + ) image_infos_future = yield from self.run_dependencies(task) - image_infos: Set[ImageInfo] = self.get_values_from_future(image_infos_future) # type: ignore + image_infos: Set[ImageInfo] = self.get_values_from_future(image_infos_future) # type: ignore return image_infos def get_volume_info(self, reused: bool) -> DockerVolumeInfo: with self._get_docker_client() as docker_client: volume_properties = docker_client.api.inspect_volume(self.volume_name) - if volume_properties['Name'] == self.volume_name: - return DockerVolumeInfo(volume_name=str(self.volume_name), - mount_point=volume_properties['Mountpoint'], reused=reused) - raise RuntimeError(f"Volume Info not found for created volume {self.volume_name}") + if volume_properties["Name"] == self.volume_name: + return DockerVolumeInfo( + volume_name=str(self.volume_name), + mount_point=volume_properties["Mountpoint"], + reused=reused, + ) + raise RuntimeError( + f"Volume Info not found for created volume {self.volume_name}" + ) def create_docker_volume(self) -> DockerVolumeInfo: self.remove_volume(self.volume_name) @@ -89,13 +116,16 @@ def remove_volume(self, volume_name): pass def cleanup_task(self, success): - if (success and not self.no_cleanup_after_success) or \ - (not success and not self.no_cleanup_after_failure): + if (success and not self.no_cleanup_after_success) or ( + not success and not self.no_cleanup_after_failure + ): try: self.logger.info(f"Cleaning up volume %s:", self.volume_name) self.remove_volume(self.volume_name) except Exception as e: - self.logger.error(f"Error during removing volume %s: %s:", self.volume_name, e) + self.logger.error( + f"Error during removing volume %s: %s:", self.volume_name, e + ) @property def _construct_complete_host_name(self): @@ -110,7 +140,7 @@ def create_certificate(self, image_infos: Dict[str, ImageInfo]) -> None: volumes = { self.volume_info.volume_name: { "bind": CERTIFICATES_MOUNT_PATH, - "mode": "rw" + "mode": "rw", } } @@ -127,16 +157,20 @@ def create_certificate(self, image_infos: Dict[str, ImageInfo]) -> None: "test_environment_name": self.environment_name, "container_type": "certificate_resources", }, - runtime=self.docker_runtime + runtime=self.docker_runtime, ) container.start() self.logger.info("Creating certificates...") - cmd = f"bash /scripts/create_certificates.sh " \ - f"{self._construct_complete_host_name} {CERTIFICATES_MOUNT_PATH}" + cmd = ( + f"bash /scripts/create_certificates.sh " + f"{self._construct_complete_host_name} {CERTIFICATES_MOUNT_PATH}" + ) exit_code, output = container.exec_run(cmd) - self.logger.info(output.decode('utf-8')) + self.logger.info(output.decode("utf-8")) if exit_code != 0: - raise RuntimeError(f"Error creating certificates:'{output.decode('utf-8')}'") + raise RuntimeError( + f"Error creating certificates:'{output.decode('utf-8')}'" + ) finally: container.stop() container.remove() diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/bucketfs_sync_checker.py b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/bucketfs_sync_checker.py index 4f66eb766..308500275 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/bucketfs_sync_checker.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/bucketfs_sync_checker.py @@ -1,4 +1,6 @@ -from exasol_integration_test_docker_environment.abstract_method_exception import AbstractMethodException +from exasol_integration_test_docker_environment.abstract_method_exception import ( + AbstractMethodException, +) class BucketFSSyncChecker: diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/docker_db_log_based_bucket_sync_checker.py b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/docker_db_log_based_bucket_sync_checker.py index a1947d7e5..0b4ef1f87 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/docker_db_log_based_bucket_sync_checker.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/docker_db_log_based_bucket_sync_checker.py @@ -3,21 +3,25 @@ from docker.models.containers import Container -from exasol_integration_test_docker_environment \ - .lib.test_environment.database_setup.bucketfs_sync_checker \ - import BucketFSSyncChecker -from exasol_integration_test_docker_environment \ - .lib.base.db_os_executor import DbOsExecutor +from exasol_integration_test_docker_environment.lib.base.db_os_executor import ( + DbOsExecutor, +) +from exasol_integration_test_docker_environment.lib.test_environment.database_setup.bucketfs_sync_checker import ( + BucketFSSyncChecker, +) class DockerDBLogBasedBucketFSSyncChecker(BucketFSSyncChecker): - def __init__(self, logger, - database_container: Container, - pattern_to_wait_for: str, - log_file_to_check: str, - bucketfs_write_password: str, - executor: DbOsExecutor): + def __init__( + self, + logger, + database_container: Container, + pattern_to_wait_for: str, + log_file_to_check: str, + bucketfs_write_password: str, + executor: DbOsExecutor, + ): self.logger = logger self.pattern_to_wait_for = pattern_to_wait_for self.log_file_to_check = log_file_to_check @@ -34,18 +38,21 @@ def wait_for_bucketfs_sync(self): ready = False while not ready: exit_code, output = self.find_pattern_in_logfile() - if self.exit_code_changed(exit_code, self.start_exit_code) or \ - self.found_new_log_line(exit_code, self.start_exit_code, - self.start_output, output): + if self.exit_code_changed( + exit_code, self.start_exit_code + ) or self.found_new_log_line( + exit_code, self.start_exit_code, self.start_output, output + ): ready = True time.sleep(1) def exit_code_changed(self, exit_code, start_exit_code): return exit_code == 0 and start_exit_code != 0 - def found_new_log_line(self, exit_code, start_exit_code, - start_output, output): - return exit_code == 0 and start_exit_code == 0 and len(start_output) < len(output) + def found_new_log_line(self, exit_code, start_exit_code, start_output, output): + return ( + exit_code == 0 and start_exit_code == 0 and len(start_output) < len(output) + ) def find_pattern_in_logfile(self): cmd = f"""grep '{self.pattern_to_wait_for}' {self.log_file_to_check}""" diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/find_exaplus_in_db_container.py b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/find_exaplus_in_db_container.py index 84af03192..dedc6d85d 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/find_exaplus_in_db_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/find_exaplus_in_db_container.py @@ -2,13 +2,15 @@ from typing import List import docker -from exasol_integration_test_docker_environment.lib.base.db_os_executor \ - import DbOsExecutor + +from exasol_integration_test_docker_environment.lib.base.db_os_executor import ( + DbOsExecutor, +) def find_exaplus( - db_container: docker.models.containers.Container, - os_executor: DbOsExecutor, + db_container: docker.models.containers.Container, + os_executor: DbOsExecutor, ) -> PurePath: """ Tries to find path of exaplus in given container in directories where exaplus is normally installed. @@ -22,7 +24,7 @@ def find_exaplus( exit, output = os_executor.exec("find /opt -name 'exaplus' -type f") if exit != 0: raise RuntimeError(f"Exaplus not found on docker db! Output is {output}") - found_paths : List[str] = list(filter(None, output.decode("UTF-8").split("\n"))) + found_paths: List[str] = list(filter(None, output.decode("UTF-8").split("\n"))) if len(found_paths) != 1: raise RuntimeError(f"Error determining exaplus path! Output is {output}") exaplus_path = PurePath(found_paths[0]) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py index 1a472d941..2ed2b5f03 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/populate_data.py @@ -1,21 +1,35 @@ import logging -from pathlib import Path, PurePath +from pathlib import ( + Path, + PurePath, +) import luigi -from exasol_integration_test_docker_environment.abstract_method_exception import AbstractMethodException -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import JsonPickleParameter -from exasol_integration_test_docker_environment.lib.data.database_credentials import DatabaseCredentialsParameter -from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo +from exasol_integration_test_docker_environment.abstract_method_exception import ( + AbstractMethodException, +) +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import ( + JsonPickleParameter, +) +from exasol_integration_test_docker_environment.lib.data.database_credentials import ( + DatabaseCredentialsParameter, +) +from exasol_integration_test_docker_environment.lib.data.environment_info import ( + EnvironmentInfo, +) class PopulateTestDataToDatabase(DockerBaseTask, DatabaseCredentialsParameter): - logger = logging.getLogger('luigi-interface') + logger = logging.getLogger("luigi-interface") - environment_name : str = luigi.Parameter() # type: ignore - test_environment_info : EnvironmentInfo = JsonPickleParameter( - EnvironmentInfo, significant=False) # type: ignore + environment_name: str = luigi.Parameter() # type: ignore + test_environment_info: EnvironmentInfo = JsonPickleParameter( + EnvironmentInfo, significant=False + ) # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -29,17 +43,25 @@ def run_task(self): data_path_within_test_container = self.get_data_path_within_test_container() data_file_within_data_path = self.get_data_file_within_data_path() with self._get_docker_client() as docker_client: - test_container = docker_client.containers.get(self._test_container_info.container_name) - cmd = f"cd {data_path_within_test_container}; " \ - f"$EXAPLUS -c '{self._database_info.host}:{self._database_info.ports.database}' " \ - f"-x -u '{username}' -p '{password}' -f {data_file_within_data_path} " \ - f"-jdbcparam 'validateservercertificate=0'" + test_container = docker_client.containers.get( + self._test_container_info.container_name + ) + cmd = ( + f"cd {data_path_within_test_container}; " + f"$EXAPLUS -c '{self._database_info.host}:{self._database_info.ports.database}' " + f"-x -u '{username}' -p '{password}' -f {data_file_within_data_path} " + f"-jdbcparam 'validateservercertificate=0'" + ) bash_cmd = f"""bash -c "{cmd}" """ exit_code, output = test_container.exec_run(cmd=bash_cmd) self.write_logs(output.decode("utf-8")) if exit_code != 0: - raise Exception("Failed to populate the database with data.\nLog: %s" % cmd + "\n" + output.decode("utf-8")) + raise Exception( + "Failed to populate the database with data.\nLog: %s" % cmd + + "\n" + + output.decode("utf-8") + ) def write_logs(self, output: str): log_file = Path(self.get_log_path(), "log") diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/time_based_bucketfs_sync_waiter.py b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/time_based_bucketfs_sync_waiter.py index 0b3bf9499..8857aa592 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_setup/time_based_bucketfs_sync_waiter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_setup/time_based_bucketfs_sync_waiter.py @@ -1,7 +1,8 @@ import time -from exasol_integration_test_docker_environment.lib.test_environment.database_setup.bucketfs_sync_checker import \ - BucketFSSyncChecker +from exasol_integration_test_docker_environment.lib.test_environment.database_setup.bucketfs_sync_checker import ( + BucketFSSyncChecker, +) class TimeBasedBucketFSSyncWaiter(BucketFSSyncChecker): diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py index 61c08bca5..bb39cb880 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py @@ -2,29 +2,37 @@ import time from pathlib import Path from threading import Thread -from typing import Callable, Optional, List +from typing import ( + Callable, + List, + Optional, +) from docker.models.containers import Container -from exasol_integration_test_docker_environment.lib.base.still_running_logger import StillRunningLogger -from exasol_integration_test_docker_environment.lib.logging.container_log_handler import ContainerLogHandler +from exasol_integration_test_docker_environment.lib.base.still_running_logger import ( + StillRunningLogger, +) +from exasol_integration_test_docker_environment.lib.logging.container_log_handler import ( + ContainerLogHandler, +) class DBContainerLogThread(Thread): def __init__(self, container: Container, logger, log_file: Path, description: str): super().__init__() - self.complete_log : List[str] = list() + self.complete_log: List[str] = list() self.description = description self.logger = logger self.log_file = log_file self.container = container self.finish = False - self.previous_timestamp : Optional[float] = None - self.current_timestamp : Optional[float] = None - self.error_message : Optional[str] = None + self.previous_timestamp: Optional[float] = None + self.current_timestamp: Optional[float] = None + self.error_message: Optional[str] = None self.ignore_error_return_codes = ( "(membership) returned with state 1", # exclude webui not found in 7.0.0 - "rsyslogd) returned with state 1" # exclude rsyslogd which might crash when running itde under lima + "rsyslogd) returned with state 1", # exclude rsyslogd which might crash when running itde under lima ) def _contains_error(self, log_line: str) -> bool: @@ -40,9 +48,9 @@ def contains(substr: str, ignore: Optional[Callable[[str], bool]] = None): return ignore is None or not ignore(log_line) return ( - contains("error", ignore_sshd) - or contains("exception") - or contains("returned with state 1", ignore_return_code) + contains("error", ignore_sshd) + or contains("exception") + or contains("returned with state 1", ignore_return_code) ) def stop(self): @@ -51,18 +59,23 @@ def stop(self): def run(self): try: - with ContainerLogHandler(self.log_file, self.logger, self.description) as log_handler: - still_running_logger = StillRunningLogger( - self.logger, self.description) + with ContainerLogHandler( + self.log_file, self.logger, self.description + ) as log_handler: + still_running_logger = StillRunningLogger(self.logger, self.description) while not self.finish: self.current_timestamp = math.floor(time.time()) - log = self.container.logs(since=self.previous_timestamp, until=self.current_timestamp) + log = self.container.logs( + since=self.previous_timestamp, until=self.current_timestamp + ) if len(log) != 0: still_running_logger.log() log_handler.handle_log_lines(log) log_line = log.decode("utf-8").lower() if self._contains_error(log_line): - self.logger.info("ContainerLogHandler error message, %s", log_line) + self.logger.info( + "ContainerLogHandler error message, %s", log_line + ) self.error_message = log_line self.finish = True self.previous_timestamp = self.current_timestamp diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/is_database_ready_thread.py b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/is_database_ready_thread.py index 4c3676b28..84f319694 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/is_database_ready_thread.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/is_database_ready_thread.py @@ -5,22 +5,30 @@ from docker.models.containers import Container -from exasol_integration_test_docker_environment.lib.data.database_credentials import DatabaseCredentials -from exasol_integration_test_docker_environment.lib.data.database_info import DatabaseInfo -from exasol_integration_test_docker_environment.lib.test_environment.database_setup.find_exaplus_in_db_container import \ - find_exaplus -from exasol_integration_test_docker_environment.lib.base.db_os_executor import \ - DbOsExecFactory +from exasol_integration_test_docker_environment.lib.base.db_os_executor import ( + DbOsExecFactory, +) +from exasol_integration_test_docker_environment.lib.data.database_credentials import ( + DatabaseCredentials, +) +from exasol_integration_test_docker_environment.lib.data.database_info import ( + DatabaseInfo, +) +from exasol_integration_test_docker_environment.lib.test_environment.database_setup.find_exaplus_in_db_container import ( + find_exaplus, +) class IsDatabaseReadyThread(Thread): - def __init__(self, - logger: Logger, - database_info: DatabaseInfo, - database_container: Container, - database_credentials: DatabaseCredentials, - docker_db_image_version: str, - executor_factory: DbOsExecFactory): + def __init__( + self, + logger: Logger, + database_info: DatabaseInfo, + database_container: Container, + database_credentials: DatabaseCredentials, + docker_db_image_version: str, + executor_factory: DbOsExecFactory, + ): super().__init__() self.logger = logger self.database_credentials = database_credentials @@ -45,17 +53,28 @@ def run(self): try: executor.prepare() exaplus_path = find_exaplus(self._db_container, executor) - db_connection_command = self.create_db_connection_command(exaplus_path) - bucket_fs_connection_command = self.create_bucketfs_connection_command() + db_connection_command = self.create_db_connection_command( + exaplus_path + ) + bucket_fs_connection_command = ( + self.create_bucketfs_connection_command() + ) except RuntimeError as e: - self.logger.exception("Caught exception while searching for exaplus.") + self.logger.exception( + "Caught exception while searching for exaplus." + ) self.finish = True while not self.finish: - (exit_code_db_connection, self.output_db_connection) = \ + (exit_code_db_connection, self.output_db_connection) = ( executor.exec(db_connection_command) - (exit_code_bucketfs_connection, self.output_bucketfs_connection) = \ + ) + (exit_code_bucketfs_connection, self.output_bucketfs_connection) = ( executor.exec(bucket_fs_connection_command) - if exit_code_db_connection == 0 and exit_code_bucketfs_connection == 0: + ) + if ( + exit_code_db_connection == 0 + and exit_code_bucketfs_connection == 0 + ): self.finish = True self.is_ready = True time.sleep(1) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py index b39ce02bc..77b1d89c2 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py @@ -1,17 +1,24 @@ import luigi -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import JsonPickleParameter -from exasol_integration_test_docker_environment.lib.data.database_credentials import DatabaseCredentialsParameter -from exasol_integration_test_docker_environment.lib.data.database_info import DatabaseInfo +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import ( + JsonPickleParameter, +) +from exasol_integration_test_docker_environment.lib.data.database_credentials import ( + DatabaseCredentialsParameter, +) +from exasol_integration_test_docker_environment.lib.data.database_info import ( + DatabaseInfo, +) -class WaitForTestExternalDatabase(DockerBaseTask, - DatabaseCredentialsParameter): - environment_name : str = luigi.Parameter() # type: ignore - database_info : DatabaseInfo = JsonPickleParameter(DatabaseInfo, significant=False) # type: ignore - db_startup_timeout_in_seconds : int = luigi.IntParameter(1 * 60, significant=False) # type: ignore - attempt : int = luigi.IntParameter(1) # type: ignore +class WaitForTestExternalDatabase(DockerBaseTask, DatabaseCredentialsParameter): + environment_name: str = luigi.Parameter() # type: ignore + database_info: DatabaseInfo = JsonPickleParameter(DatabaseInfo, significant=False) # type: ignore + db_startup_timeout_in_seconds: int = luigi.IntParameter(1 * 60, significant=False) # type: ignore + attempt: int = luigi.IntParameter(1) # type: ignore def run_task(self): # Since we can't assume that the test container exists, we cannot connect easily here diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_test_docker_database.py b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_test_docker_database.py index 9f6bc900d..65e7a5428 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_test_docker_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_test_docker_database.py @@ -1,56 +1,74 @@ import gzip import pathlib import time -from datetime import datetime, timedelta +from datetime import ( + datetime, + timedelta, +) import luigi from docker.models.containers import Container -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import JsonPickleParameter -from exasol_integration_test_docker_environment.lib.data.database_credentials import DatabaseCredentialsParameter -from exasol_integration_test_docker_environment.lib.data.database_info import DatabaseInfo -from exasol_integration_test_docker_environment.lib.test_environment.database_waiters.db_container_log_thread import \ - DBContainerLogThread -from exasol_integration_test_docker_environment.lib.test_environment.database_waiters.is_database_ready_thread import \ - IsDatabaseReadyThread -from exasol_integration_test_docker_environment.lib.base.db_os_executor import \ - DbOsExecFactory +from exasol_integration_test_docker_environment.lib.base.db_os_executor import ( + DbOsExecFactory, +) +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import ( + JsonPickleParameter, +) +from exasol_integration_test_docker_environment.lib.data.database_credentials import ( + DatabaseCredentialsParameter, +) +from exasol_integration_test_docker_environment.lib.data.database_info import ( + DatabaseInfo, +) +from exasol_integration_test_docker_environment.lib.test_environment.database_waiters.db_container_log_thread import ( + DBContainerLogThread, +) +from exasol_integration_test_docker_environment.lib.test_environment.database_waiters.is_database_ready_thread import ( + IsDatabaseReadyThread, +) class WaitForTestDockerDatabase(DockerBaseTask, DatabaseCredentialsParameter): - environment_name : str = luigi.Parameter() # type: ignore - database_info : DatabaseInfo = JsonPickleParameter(DatabaseInfo, significant=False) # type: ignore - db_startup_timeout_in_seconds : int = luigi.IntParameter(10 * 60, significant=False) # type: ignore - attempt : int = luigi.IntParameter(1) # type: ignore - docker_db_image_version : str = luigi.Parameter() # type: ignore - executor_factory : DbOsExecFactory = JsonPickleParameter(DbOsExecFactory, significant=False) # type: ignore + environment_name: str = luigi.Parameter() # type: ignore + database_info: DatabaseInfo = JsonPickleParameter(DatabaseInfo, significant=False) # type: ignore + db_startup_timeout_in_seconds: int = luigi.IntParameter(10 * 60, significant=False) # type: ignore + attempt: int = luigi.IntParameter(1) # type: ignore + docker_db_image_version: str = luigi.Parameter() # type: ignore + executor_factory: DbOsExecFactory = JsonPickleParameter(DbOsExecFactory, significant=False) # type: ignore def run_task(self): with self._get_docker_client() as docker_client: db_container_name = self.database_info.container_info.container_name db_container = docker_client.containers.get(db_container_name) - is_database_ready = \ - self.wait_for_database_startup(db_container) - after_startup_db_log_file = self.get_log_path().joinpath("after_startup_db_log.tar.gz") + is_database_ready = self.wait_for_database_startup(db_container) + after_startup_db_log_file = self.get_log_path().joinpath( + "after_startup_db_log.tar.gz" + ) self.save_db_log_files_as_gzip_tar(after_startup_db_log_file, db_container) self.return_object(is_database_ready) - def wait_for_database_startup(self, - db_container: Container): - container_log_thread, is_database_ready_thread = \ - self.start_wait_threads(db_container) - is_database_ready = \ - self.wait_for_threads(container_log_thread, is_database_ready_thread) + def wait_for_database_startup(self, db_container: Container): + container_log_thread, is_database_ready_thread = self.start_wait_threads( + db_container + ) + is_database_ready = self.wait_for_threads( + container_log_thread, is_database_ready_thread + ) self.join_threads(container_log_thread, is_database_ready_thread) return is_database_ready def start_wait_threads(self, db_container: Container): startup_log_file = self.get_log_path().joinpath("startup.log") - container_log_thread = DBContainerLogThread(db_container, - self.logger, - startup_log_file, - "Database Startup %s" % db_container.name) + container_log_thread = DBContainerLogThread( + db_container, + self.logger, + startup_log_file, + "Database Startup %s" % db_container.name, + ) container_log_thread.start() is_database_ready_thread = IsDatabaseReadyThread( self.logger, @@ -63,19 +81,25 @@ def start_wait_threads(self, db_container: Container): is_database_ready_thread.start() return container_log_thread, is_database_ready_thread - def join_threads(self, container_log_thread: DBContainerLogThread, - is_database_ready_thread: IsDatabaseReadyThread): + def join_threads( + self, + container_log_thread: DBContainerLogThread, + is_database_ready_thread: IsDatabaseReadyThread, + ): container_log_thread.stop() is_database_ready_thread.stop() container_log_thread.join() is_database_ready_thread.join() - def wait_for_threads(self, container_log_thread: DBContainerLogThread, - is_database_ready_thread: IsDatabaseReadyThread): + def wait_for_threads( + self, + container_log_thread: DBContainerLogThread, + is_database_ready_thread: IsDatabaseReadyThread, + ): is_database_ready = False reason = None start_time = datetime.now() - while (True): + while True: if container_log_thread.error_message != None: is_database_ready = False reason = "error message in container log" @@ -87,19 +111,27 @@ def wait_for_threads(self, container_log_thread: DBContainerLogThread, reason = "error in is_database_ready_thread" break if self.timeout_occured(start_time): - reason = f"timeout after after {self.db_startup_timeout_in_seconds} seconds" + reason = ( + f"timeout after after {self.db_startup_timeout_in_seconds} seconds" + ) is_database_ready = False break time.sleep(1) if not is_database_ready: - self.log_database_not_ready(container_log_thread, is_database_ready_thread, reason) + self.log_database_not_ready( + container_log_thread, is_database_ready_thread, reason + ) container_log_thread.stop() is_database_ready_thread.stop() return is_database_ready - def log_database_not_ready(self, container_log_thread: DBContainerLogThread, - is_database_ready_thread: IsDatabaseReadyThread, reason): - container_log = '\n'.join(container_log_thread.complete_log) + def log_database_not_ready( + self, + container_log_thread: DBContainerLogThread, + is_database_ready_thread: IsDatabaseReadyThread, + reason, + ): + container_log = "\n".join(container_log_thread.complete_log) log_information = f""" ========== IsDatabaseReadyThread output db connection: ============ {is_database_ready_thread.output_db_connection} @@ -110,13 +142,17 @@ def log_database_not_ready(self, container_log_thread: DBContainerLogThread, """ self.logger.warning( 'Database startup failed for following reason "%s", here some debug information \n%s', - reason, log_information) + reason, + log_information, + ) def timeout_occured(self, start_time): timeout = timedelta(seconds=self.db_startup_timeout_in_seconds) return datetime.now() - start_time > timeout - def save_db_log_files_as_gzip_tar(self, path: pathlib.Path, database_container: Container): + def save_db_log_files_as_gzip_tar( + self, path: pathlib.Path, database_container: Container + ): stream, stat = database_container.get_archive("/exa/logs") with gzip.open(path, "wb") as file: for chunk in stream: diff --git a/exasol_integration_test_docker_environment/lib/test_environment/db_version.py b/exasol_integration_test_docker_environment/lib/test_environment/db_version.py index fe4efcaf4..a57d12340 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/db_version.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/db_version.py @@ -1,6 +1,11 @@ -from typing import Optional, Tuple +from typing import ( + Optional, + Tuple, +) -from exasol_integration_test_docker_environment.cli.options.test_environment_options import LATEST_DB_VERSION +from exasol_integration_test_docker_environment.cli.options.test_environment_options import ( + LATEST_DB_VERSION, +) DEFAULT_VERSION = "default" @@ -13,14 +18,16 @@ def __init__(self, major, minor, stable): @classmethod def from_db_version_str(cls, db_version_str: Optional[str]): - db_version : str = LATEST_DB_VERSION if db_version_str is None else db_version_str + db_version: str = ( + LATEST_DB_VERSION if db_version_str is None else db_version_str + ) if db_version_str == DEFAULT_VERSION: db_version = LATEST_DB_VERSION if db_version.endswith("-d1"): db_version = "-".join(db_version.split("-")[0:-1]) if db_version.startswith("prerelease-"): db_version = "-".join(db_version.split("-")[1:]) - version : Tuple[int,...] = tuple([int(v) for v in db_version.split(".")]) + version: Tuple[int, ...] = tuple([int(v) for v in db_version.split(".")]) if len(version) != 3: raise ValueError(f"Invalid db version given: {db_version}") return DbVersion(version[0], version[1], version[2]) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/docker_container_copy.py b/exasol_integration_test_docker_environment/lib/test_environment/docker_container_copy.py index ab28d9a37..377f1bf00 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/docker_container_copy.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/docker_container_copy.py @@ -1,17 +1,20 @@ -from typing import Optional - -from docker.models.containers import Container import io import tarfile import time +from typing import Optional + +from docker.models.containers import Container + class DockerContainerCopy: - def __init__(self, container:Container): + def __init__(self, container: Container): super().__init__() self.open = True - self.file_like_object : Optional[io.BytesIO] = io.BytesIO() - self.tar : Optional[tarfile.TarFile] = tarfile.open(fileobj=self.file_like_object, mode="x") + self.file_like_object: Optional[io.BytesIO] = io.BytesIO() + self.tar: Optional[tarfile.TarFile] = tarfile.open( + fileobj=self.file_like_object, mode="x" + ) self.container = container def __del__(self): @@ -21,14 +24,13 @@ def __del__(self): self.tar = None self.file_like_object = None - def is_open_or_raise(self): if not self.open: raise Exception("DockerContainerCopy not open") def add_string_to_file(self, name: str, string: str): self.is_open_or_raise() - encoded = string.encode('utf-8') + encoded = string.encode("utf-8") bytes_io = io.BytesIO(encoded) tar_info = tarfile.TarInfo(name=name) tar_info.mtime = time.time() @@ -36,19 +38,21 @@ def add_string_to_file(self, name: str, string: str): assert self.tar self.tar.addfile(tarinfo=tar_info, fileobj=bytes_io) - def add_file(self, host_path:str, path_in_tar:str): + def add_file(self, host_path: str, path_in_tar: str): self.is_open_or_raise() assert self.tar self.tar.add(host_path, path_in_tar) - def copy(self, path_in_container:str): + def copy(self, path_in_container: str): self.is_open_or_raise() assert self.tar self.tar.close() self.open = False self.tar = None assert self.file_like_object - self.container.put_archive(path_in_container, self.file_like_object.getbuffer().tobytes()) + self.container.put_archive( + path_in_container, self.file_like_object.getbuffer().tobytes() + ) self.file_like_object = None diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py index 738c7a281..ac11f3572 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/docker_db_test_environment_parameter.py @@ -1,7 +1,15 @@ +from enum import ( + Enum, + auto, +) +from typing import ( + List, + Optional, +) + import luigi -from enum import Enum, auto from luigi import Config -from typing import List, Optional + class DbOsAccess(Enum): """ @@ -15,20 +23,23 @@ class DbOsAccess(Enum): case will verify that when using SSH access a file with the required private key is generated. """ + DOCKER_EXEC = auto() SSH = auto() class DockerDBTestEnvironmentParameter(Config): - docker_db_image_name : Optional[str] = luigi.OptionalParameter(None) # type: ignore - docker_db_image_version : Optional[str] = luigi.OptionalParameter(None) # type: ignore - reuse_database : bool = luigi.BoolParameter(False, significant=False) # type: ignore - db_os_access = luigi.EnumParameter(DbOsAccess.DOCKER_EXEC, enum=DbOsAccess, significant=False) - no_database_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) # type: ignore - no_database_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) # type: ignore - database_port_forward : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore - bucketfs_port_forward : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore - ssh_port_forward : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore - mem_size : Optional[str] = luigi.OptionalParameter("2 GiB", significant=False) # type: ignore - disk_size : Optional[str] = luigi.OptionalParameter("2 GiB", significant=False) # type: ignore - nameservers : List[str] = luigi.ListParameter([], significant=False) # type: ignore + docker_db_image_name: Optional[str] = luigi.OptionalParameter(None) # type: ignore + docker_db_image_version: Optional[str] = luigi.OptionalParameter(None) # type: ignore + reuse_database: bool = luigi.BoolParameter(False, significant=False) # type: ignore + db_os_access = luigi.EnumParameter( + DbOsAccess.DOCKER_EXEC, enum=DbOsAccess, significant=False + ) + no_database_cleanup_after_success: bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_database_cleanup_after_failure: bool = luigi.BoolParameter(False, significant=False) # type: ignore + database_port_forward: Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + bucketfs_port_forward: Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + ssh_port_forward: Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + mem_size: Optional[str] = luigi.OptionalParameter("2 GiB", significant=False) # type: ignore + disk_size: Optional[str] = luigi.OptionalParameter("2 GiB", significant=False) # type: ignore + nameservers: List[str] = luigi.ListParameter([], significant=False) # type: ignore diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py index 2b89f121a..6014d25ee 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/external_test_environment_parameter.py @@ -2,39 +2,38 @@ import luigi from luigi import Config - from luigi.parameter import ParameterVisibility class ExternalDatabaseXMLRPCParameter(Config): - external_exasol_xmlrpc_host : Optional[str] = luigi.OptionalParameter() #type: ignore - external_exasol_xmlrpc_port : int = luigi.IntParameter(443) #type: ignore - external_exasol_xmlrpc_user : Optional[str] = luigi.OptionalParameter() #type: ignore - external_exasol_xmlrpc_cluster_name : Optional[str] = luigi.OptionalParameter() #type: ignore - external_exasol_xmlrpc_password : Optional[str] = luigi.OptionalParameter( + external_exasol_xmlrpc_host: Optional[str] = luigi.OptionalParameter() # type: ignore + external_exasol_xmlrpc_port: int = luigi.IntParameter(443) # type: ignore + external_exasol_xmlrpc_user: Optional[str] = luigi.OptionalParameter() # type: ignore + external_exasol_xmlrpc_cluster_name: Optional[str] = luigi.OptionalParameter() # type: ignore + external_exasol_xmlrpc_password: Optional[str] = luigi.OptionalParameter( significant=False, visibility=ParameterVisibility.HIDDEN, - ) #type: ignore + ) # type: ignore # See ticket https://github.com/exasol/integration-test-docker-environment/issues/341 class ExternalDatabaseHostParameter(Config): - external_exasol_db_host : Optional[str] = luigi.OptionalParameter() #type: ignore - external_exasol_db_port : int = luigi.IntParameter() #type: ignore - external_exasol_bucketfs_port : int = luigi.IntParameter() #type: ignore - external_exasol_ssh_port : int = luigi.IntParameter() #type: ignore + external_exasol_db_host: Optional[str] = luigi.OptionalParameter() # type: ignore + external_exasol_db_port: int = luigi.IntParameter() # type: ignore + external_exasol_bucketfs_port: int = luigi.IntParameter() # type: ignore + external_exasol_ssh_port: int = luigi.IntParameter() # type: ignore class ExternalDatabaseCredentialsParameter( - ExternalDatabaseHostParameter, - ExternalDatabaseXMLRPCParameter, + ExternalDatabaseHostParameter, + ExternalDatabaseXMLRPCParameter, ): - external_exasol_db_user : Optional[str] = luigi.OptionalParameter() #type: ignore - external_exasol_db_password : Optional[str] = luigi.OptionalParameter( + external_exasol_db_user: Optional[str] = luigi.OptionalParameter() # type: ignore + external_exasol_db_password: Optional[str] = luigi.OptionalParameter( significant=False, visibility=ParameterVisibility.HIDDEN, - ) #type: ignore - external_exasol_bucketfs_write_password : Optional[str] = luigi.OptionalParameter( + ) # type: ignore + external_exasol_bucketfs_write_password: Optional[str] = luigi.OptionalParameter( significant=False, visibility=ParameterVisibility.HIDDEN, - ) #type: ignore + ) # type: ignore diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/general_spawn_test_environment_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/general_spawn_test_environment_parameter.py index f4917a3ff..9e72010f8 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/general_spawn_test_environment_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/general_spawn_test_environment_parameter.py @@ -1,17 +1,22 @@ +from typing import ( + List, + Optional, +) + import luigi from luigi import Config -from typing import List, Optional -from exasol_integration_test_docker_environment.lib.test_environment.parameter.test_container_parameter import \ - OptionalTestContainerParameter +from exasol_integration_test_docker_environment.lib.test_environment.parameter.test_container_parameter import ( + OptionalTestContainerParameter, +) class GeneralSpawnTestEnvironmentParameter(OptionalTestContainerParameter): - reuse_database_setup : bool = luigi.BoolParameter(False, significant=False) #type: ignore - reuse_test_container : bool = luigi.BoolParameter(False, significant=False) #type: ignore - no_test_container_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) #type: ignore - no_test_container_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) #type: ignore - max_start_attempts : int = luigi.IntParameter(2, significant=False) #type: ignore - docker_runtime : Optional[str] = luigi.OptionalParameter(None, significant=False) #type: ignore - create_certificates : bool = luigi.BoolParameter() #type: ignore - additional_db_parameter : List[str] = luigi.ListParameter() #type: ignore + reuse_database_setup: bool = luigi.BoolParameter(False, significant=False) # type: ignore + reuse_test_container: bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_test_container_cleanup_after_success: bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_test_container_cleanup_after_failure: bool = luigi.BoolParameter(False, significant=False) # type: ignore + max_start_attempts: int = luigi.IntParameter(2, significant=False) # type: ignore + docker_runtime: Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + create_certificates: bool = luigi.BoolParameter() # type: ignore + additional_db_parameter: List[str] = luigi.ListParameter() # type: ignore diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/spawn_test_environment_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/spawn_test_environment_parameter.py index 29a6b6ec3..f5b48543c 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/spawn_test_environment_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/spawn_test_environment_parameter.py @@ -1,15 +1,22 @@ import luigi -from exasol_integration_test_docker_environment.lib.data.environment_type import EnvironmentType -from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter import \ - DockerDBTestEnvironmentParameter -from exasol_integration_test_docker_environment.lib.test_environment.parameter.external_test_environment_parameter import \ - ExternalDatabaseCredentialsParameter -from exasol_integration_test_docker_environment.lib.test_environment.parameter.general_spawn_test_environment_parameter import \ - GeneralSpawnTestEnvironmentParameter +from exasol_integration_test_docker_environment.lib.data.environment_type import ( + EnvironmentType, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter import ( + DockerDBTestEnvironmentParameter, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.external_test_environment_parameter import ( + ExternalDatabaseCredentialsParameter, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.general_spawn_test_environment_parameter import ( + GeneralSpawnTestEnvironmentParameter, +) -class SpawnTestEnvironmentParameter(GeneralSpawnTestEnvironmentParameter, - ExternalDatabaseCredentialsParameter, - DockerDBTestEnvironmentParameter): +class SpawnTestEnvironmentParameter( + GeneralSpawnTestEnvironmentParameter, + ExternalDatabaseCredentialsParameter, + DockerDBTestEnvironmentParameter, +): environment_type = luigi.EnumParameter(enum=EnvironmentType) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/parameter/test_container_parameter.py b/exasol_integration_test_docker_environment/lib/test_environment/parameter/test_container_parameter.py index 445f908b3..3c3f29db5 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/parameter/test_container_parameter.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/parameter/test_container_parameter.py @@ -1,16 +1,22 @@ from luigi.parameter import ParameterVisibility -from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import JsonPickleParameter -from exasol_integration_test_docker_environment.lib.data.test_container_content_description import \ - TestContainerContentDescription +from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import ( + JsonPickleParameter, +) +from exasol_integration_test_docker_environment.lib.data.test_container_content_description import ( + TestContainerContentDescription, +) class TestContainerParameter: - test_container_content : TestContainerContentDescription = JsonPickleParameter(TestContainerContentDescription, - visibility=ParameterVisibility.HIDDEN) # type: ignore + test_container_content: TestContainerContentDescription = JsonPickleParameter( + TestContainerContentDescription, visibility=ParameterVisibility.HIDDEN + ) # type: ignore class OptionalTestContainerParameter: - test_container_content : TestContainerContentDescription = JsonPickleParameter(TestContainerContentDescription, - visibility=ParameterVisibility.HIDDEN, - is_optional=True) # type: ignore + test_container_content: TestContainerContentDescription = JsonPickleParameter( + TestContainerContentDescription, + visibility=ParameterVisibility.HIDDEN, + is_optional=True, + ) # type: ignore diff --git a/exasol_integration_test_docker_environment/lib/test_environment/ports.py b/exasol_integration_test_docker_environment/lib/test_environment/ports.py index 92afa5665..01b0a3675 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/ports.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/ports.py @@ -1,25 +1,35 @@ import socket from contextlib import ExitStack -from typing import List, Optional, Generator +from typing import ( + Generator, + List, + Optional, +) def find_free_ports(num_ports: int) -> List[int]: def new_socket(): return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + def bind(sock: socket.socket, port: int): - sock.bind(('', port)) + sock.bind(("", port)) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + def acquire_port_numbers(num_ports: int) -> Generator[int, None, None]: with ExitStack() as stack: sockets = [stack.enter_context(new_socket()) for dummy in range(num_ports)] for sock in sockets: bind(sock, 0) yield sock.getsockname()[1] + def check_port_numbers(ports): with ExitStack() as stack: - sockets_and_ports = [(stack.enter_context(new_socket()), port) for port in ports] + sockets_and_ports = [ + (stack.enter_context(new_socket()), port) for port in ports + ] for sock, port in sockets_and_ports: bind(sock, port) + ports = list(acquire_port_numbers(num_ports)) check_port_numbers(ports) return ports @@ -34,16 +44,16 @@ class PortsType(type): """ @property - def default_ports(self) -> 'Ports': + def default_ports(self) -> "Ports": return Ports(database=8563, bucketfs=2580, ssh=22) @property - def external(self) -> 'Ports': + def external(self) -> "Ports": # For external databases SSH port might depend on version database. return Ports(database=8563, bucketfs=2580, ssh=None) @property - def forward(self) -> 'Ports': + def forward(self) -> "Ports": return Ports(database=8563, bucketfs=2580, ssh=20002) @@ -54,7 +64,11 @@ def __init__(self, database: int, bucketfs: int, ssh: Optional[int] = None): self.ssh = ssh @classmethod - def random_free(cls, ssh: bool = True) -> 'Ports': + def random_free(cls, ssh: bool = True) -> "Ports": count = 3 if ssh else 2 ports = find_free_ports(count) - return Ports(ports[0], ports[1], None) if not ssh else Ports(ports[0], ports[1], ports[2]) + return ( + Ports(ports[0], ports[1], None) + if not ssh + else Ports(ports[0], ports[1], ports[2]) + ) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py b/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py index a906573d3..ba91e0a80 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/prepare_network_for_test_environment.py @@ -1,20 +1,25 @@ from typing import Optional + import docker import luigi -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.data.docker_network_info import DockerNetworkInfo +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) +from exasol_integration_test_docker_environment.lib.data.docker_network_info import ( + DockerNetworkInfo, +) class PrepareDockerNetworkForTestEnvironment(DockerBaseTask): - environment_name : str = luigi.Parameter() # type: ignore - network_name : str = luigi.Parameter() # type: ignore - test_container_name : str = luigi.Parameter(significant=False) # type: ignore - db_container_name : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore - reuse : bool = luigi.BoolParameter(False, significant=False) # type: ignore - no_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) # type: ignore - no_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) # type: ignore - attempt : int = luigi.IntParameter(-1) # type: ignore + environment_name: str = luigi.Parameter() # type: ignore + network_name: str = luigi.Parameter() # type: ignore + test_container_name: str = luigi.Parameter(significant=False) # type: ignore + db_container_name: Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + reuse: bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_cleanup_after_success: bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_cleanup_after_failure: bool = luigi.BoolParameter(False, significant=False) # type: ignore + attempt: int = luigi.IntParameter(-1) # type: ignore def run_task(self): self.network_info = None @@ -23,8 +28,12 @@ def run_task(self): try: self.network_info = self.get_network_info(reused=True) except Exception as e: - self.logger.warning("Tried to reuse network %s, but got Exeception %s. " - "Fallback to create new network.", self.network_name, e) + self.logger.warning( + "Tried to reuse network %s, but got Exeception %s. " + "Fallback to create new network.", + self.network_name, + e, + ) if self.network_info is None: self.network_info = self.create_docker_network() self.return_object(self.network_info) @@ -33,8 +42,12 @@ def get_network_info(self, reused: bool): with self._get_docker_client() as docker_client: network_properties = docker_client.api.inspect_network(self.network_name) network_config = network_properties["IPAM"]["Config"][0] - return DockerNetworkInfo(network_name=self.network_name, subnet=network_config["Subnet"], - gateway=network_config["Gateway"], reused=reused) + return DockerNetworkInfo( + network_name=self.network_name, + subnet=network_config["Subnet"], + gateway=network_config["Gateway"], + reused=reused, + ) def create_docker_network(self) -> DockerNetworkInfo: self.remove_container(self.test_container_name) @@ -49,18 +62,13 @@ def create_docker_network(self) -> DockerNetworkInfo: network_info = self.get_network_info(reused=False) subnet = network_info.subnet gateway = network_info.gateway - ipam_pool = docker.types.IPAMPool( - subnet=subnet, - gateway=gateway - ) - ipam_config = docker.types.IPAMConfig( - pool_configs=[ipam_pool] - ) - self.remove_network(self.network_name) # TODO race condition possible, add retry + ipam_pool = docker.types.IPAMPool(subnet=subnet, gateway=gateway) + ipam_config = docker.types.IPAMConfig(pool_configs=[ipam_pool]) + self.remove_network( + self.network_name + ) # TODO race condition possible, add retry network = docker_client.networks.create( - name=self.network_name, - driver="bridge", - ipam=ipam_config + name=self.network_name, driver="bridge", ipam=ipam_config ) return network_info @@ -82,23 +90,36 @@ def remove_container(self, container_name: str): pass def cleanup_task(self, success): - if (success and not self.no_cleanup_after_success) or \ - (not success and not self.no_cleanup_after_failure): + if (success and not self.no_cleanup_after_success) or ( + not success and not self.no_cleanup_after_failure + ): try: - self.logger.info(f"Cleaning up container %s:",self.test_container_name) + self.logger.info(f"Cleaning up container %s:", self.test_container_name) self.remove_container(self.test_container_name) except Exception as e: - self.logger.error(f"Error during removing container %s: %s:", self.test_container_name,e) + self.logger.error( + f"Error during removing container %s: %s:", + self.test_container_name, + e, + ) if self.db_container_name is not None: try: - self.logger.info(f"Cleaning up container %s",self.db_container_name) + self.logger.info( + f"Cleaning up container %s", self.db_container_name + ) self.remove_container(self.db_container_name) except Exception as e: - self.logger.error(f"Error during removing container %s: %s: ",self.db_container_name,e) + self.logger.error( + f"Error during removing container %s: %s: ", + self.db_container_name, + e, + ) try: self.logger.info(f"Cleaning up dpcker network %s", self.network_name) self.remove_network(self.network_name) except Exception as e: - self.logger.error(f"Error during removing container %s: %s",self.network_name,e) + self.logger.error( + f"Error during removing container %s: %s", self.network_name, e + ) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py b/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py index 0140ffe02..9ee58e690 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/setup_external_database_host.py @@ -5,30 +5,44 @@ import luigi -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask -from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import JsonPickleParameter -from exasol_integration_test_docker_environment.lib.data.database_credentials import DatabaseCredentialsParameter -from exasol_integration_test_docker_environment.lib.data.database_info import DatabaseInfo -from exasol_integration_test_docker_environment.lib.data.docker_network_info import DockerNetworkInfo -from exasol_integration_test_docker_environment \ - .lib.test_environment.parameter.external_test_environment_parameter import \ - ExternalDatabaseXMLRPCParameter, ExternalDatabaseHostParameter -from exasol_integration_test_docker_environment \ - .lib.test_environment.ports import Ports +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import ( + JsonPickleParameter, +) +from exasol_integration_test_docker_environment.lib.data.database_credentials import ( + DatabaseCredentialsParameter, +) +from exasol_integration_test_docker_environment.lib.data.database_info import ( + DatabaseInfo, +) +from exasol_integration_test_docker_environment.lib.data.docker_network_info import ( + DockerNetworkInfo, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.external_test_environment_parameter import ( + ExternalDatabaseHostParameter, + ExternalDatabaseXMLRPCParameter, +) +from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports -class SetupExternalDatabaseHost(DependencyLoggerBaseTask, - ExternalDatabaseXMLRPCParameter, - ExternalDatabaseHostParameter, - DatabaseCredentialsParameter): - environment_name : str = luigi.Parameter() # type: ignore - network_info : DockerNetworkInfo = JsonPickleParameter(DockerNetworkInfo, significant=False) # type: ignore - attempt : int = luigi.IntParameter(1) # type: ignore +class SetupExternalDatabaseHost( + DependencyLoggerBaseTask, + ExternalDatabaseXMLRPCParameter, + ExternalDatabaseHostParameter, + DatabaseCredentialsParameter, +): + environment_name: str = luigi.Parameter() # type: ignore + network_info: DockerNetworkInfo = JsonPickleParameter(DockerNetworkInfo, significant=False) # type: ignore + attempt: int = luigi.IntParameter(1) # type: ignore def run_task(self): database_host = self.external_exasol_db_host - if self.external_exasol_db_host == "localhost" or \ - self.external_exasol_db_host == "127.0.01": + if ( + self.external_exasol_db_host == "localhost" + or self.external_exasol_db_host == "127.0.01" + ): database_host = self.network_info.gateway self.setup_database() ports = Ports( @@ -44,29 +58,41 @@ def setup_database(self): # TODO add option to use unverified ssl cluster = self.get_xml_rpc_object() self.start_database(cluster) - cluster.bfsdefault.editBucketFS({'http_port': int(self.external_exasol_bucketfs_port)}) + cluster.bfsdefault.editBucketFS( + {"http_port": int(self.external_exasol_bucketfs_port)} + ) try: - cluster.bfsdefault.addBucket({'bucket_name': 'myudfs', 'public_bucket': True, - 'read_password': self.bucketfs_write_password, - 'write_password': self.bucketfs_write_password}) + cluster.bfsdefault.addBucket( + { + "bucket_name": "myudfs", + "public_bucket": True, + "read_password": self.bucketfs_write_password, + "write_password": self.bucketfs_write_password, + } + ) except Exception as e: self.logger.info(e) try: - cluster.bfsdefault.addBucket({'bucket_name': 'jdbc_adapter', 'public_bucket': True, - 'read_password': self.bucketfs_write_password, - 'write_password': self.bucketfs_write_password}) + cluster.bfsdefault.addBucket( + { + "bucket_name": "jdbc_adapter", + "public_bucket": True, + "read_password": self.bucketfs_write_password, + "write_password": self.bucketfs_write_password, + } + ) except Exception as e: self.logger.info(e) def get_xml_rpc_object(self, object_name: str = ""): assert self.external_exasol_xmlrpc_user and self.external_exasol_xmlrpc_password - uri = 'https://{user}:{password}@{host}:{port}/{cluster_name}/{object_name}'.format( + uri = "https://{user}:{password}@{host}:{port}/{cluster_name}/{object_name}".format( user=quote_plus(self.external_exasol_xmlrpc_user), password=quote_plus(self.external_exasol_xmlrpc_password), host=self.external_exasol_xmlrpc_host, port=self.external_exasol_xmlrpc_port, cluster_name=self.external_exasol_xmlrpc_cluster_name, - object_name=object_name + object_name=object_name, ) server = ServerProxy(uri, context=ssl._create_unverified_context()) return server @@ -74,32 +100,34 @@ def get_xml_rpc_object(self, object_name: str = ""): def start_database(self, cluster: ServerProxy): storage = self.get_xml_rpc_object("storage") # wait until all nodes are online - self.logger.info('Waiting until all nodes are online') + self.logger.info("Waiting until all nodes are online") all_nodes_online = False while not all_nodes_online: all_nodes_online = True - for nodeName in cluster.getNodeList(): # type: ignore - node_state = self.get_xml_rpc_object(nodeName).getNodeState() # type: ignore - if node_state['status'] != 'Running': + for nodeName in cluster.getNodeList(): # type: ignore + node_state = self.get_xml_rpc_object(nodeName).getNodeState() # type: ignore + if node_state["status"] != "Running": all_nodes_online = False break sleep(5) - self.logger.info('All nodes are online now') + self.logger.info("All nodes are online now") # start EXAStorage if not storage.serviceIsOnline(): - if storage.startEXAStorage() != 'OK': - self.logger.info('EXAStorage has been started successfully') + if storage.startEXAStorage() != "OK": + self.logger.info("EXAStorage has been started successfully") else: - raise Exception('Not able startup EXAStorage!\n') + raise Exception("Not able startup EXAStorage!\n") elif storage.serviceIsOnline(): - self.logger.info('EXAStorage already online; continuing startup process') + self.logger.info("EXAStorage already online; continuing startup process") # triggering database startup - for databaseName in cluster.getDatabaseList(): # type: ignore - database = self.get_xml_rpc_object('/db_' + quote_plus(str(databaseName))) + for databaseName in cluster.getDatabaseList(): # type: ignore + database = self.get_xml_rpc_object("/db_" + quote_plus(str(databaseName))) if not database.runningDatabase(): - self.logger.info('Starting database instance %s' % str(databaseName)) + self.logger.info("Starting database instance %s" % str(databaseName)) database.startDatabase() else: - self.logger.info('Database instance %s already running' % str(databaseName)) + self.logger.info( + "Database instance %s already running" % str(databaseName) + ) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py b/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py index a94db5563..2c39633ee 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/shell_variables.py @@ -1,20 +1,27 @@ -from typing import Dict, Optional -from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo +from typing import ( + Dict, + Optional, +) + +from exasol_integration_test_docker_environment.lib.data.environment_info import ( + EnvironmentInfo, +) class ShellVariables: """ Represents a collection of unix shell environment variables. """ + def __init__(self, env: Dict[str, str]): self.env = env @classmethod def from_test_environment_info( - cls, - default_bridge_ip_address: Optional[str], - test_environment_info: EnvironmentInfo, - ) -> 'ShellVariables': + cls, + default_bridge_ip_address: Optional[str], + test_environment_info: EnvironmentInfo, + ) -> "ShellVariables": """ Create ShellVariables from the given test_container_name, default_bridge_ip_address and EnvironmentInfo. @@ -22,37 +29,47 @@ def from_test_environment_info( info = test_environment_info assert info.database_info.ports.database is not None assert info.database_info.ports.bucketfs is not None - env : Dict[str, str] = { + env: Dict[str, str] = { "NAME": info.name, "TYPE": info.type, "DATABASE_HOST": info.database_info.host, "DATABASE_DB_PORT": str(info.database_info.ports.database), "DATABASE_BUCKETFS_PORT": str(info.database_info.ports.bucketfs), - "DATABASE_SSH_PORT": str(info.database_info.ports.ssh) if info.database_info.ports.ssh is not None else "", + "DATABASE_SSH_PORT": ( + str(info.database_info.ports.ssh) + if info.database_info.ports.ssh is not None + else "" + ), } if info.database_info.container_info is not None: assert info.database_info.container_info.volume_name assert default_bridge_ip_address - network_aliases = " ".join(info.database_info.container_info.network_aliases) - env.update({ - "DATABASE_CONTAINER_NAME": info.database_info.container_info.container_name, - "DATABASE_CONTAINER_NETWORK_ALIASES": f'"{network_aliases}"', - "DATABASE_CONTAINER_IP_ADDRESS": info.database_info.container_info.ip_address, - "DATABASE_CONTAINER_VOLUMNE_NAME": info.database_info.container_info.volume_name, - "DATABASE_CONTAINER_DEFAULT_BRIDGE_IP_ADDRESS": default_bridge_ip_address, - }) + network_aliases = " ".join( + info.database_info.container_info.network_aliases + ) + env.update( + { + "DATABASE_CONTAINER_NAME": info.database_info.container_info.container_name, + "DATABASE_CONTAINER_NETWORK_ALIASES": f'"{network_aliases}"', + "DATABASE_CONTAINER_IP_ADDRESS": info.database_info.container_info.ip_address, + "DATABASE_CONTAINER_VOLUMNE_NAME": info.database_info.container_info.volume_name, + "DATABASE_CONTAINER_DEFAULT_BRIDGE_IP_ADDRESS": default_bridge_ip_address, + } + ) if info.test_container_info is not None: container_name = info.test_container_info.container_name network_aliases = " ".join(info.test_container_info.network_aliases) - env.update({ - "TEST_CONTAINER_NAME": container_name, - "TEST_CONTAINER_NETWORK_ALIASES": f'"{network_aliases}"', - "TEST_CONTAINER_IP_ADDRESS": info.test_container_info.ip_address, - }) + env.update( + { + "TEST_CONTAINER_NAME": container_name, + "TEST_CONTAINER_NETWORK_ALIASES": f'"{network_aliases}"', + "TEST_CONTAINER_IP_ADDRESS": info.test_container_info.ip_address, + } + ) return ShellVariables(env) def render(self, prefix: str = "") -> str: prefix += "ITDE" - aslist = [ f"{prefix}_{key}={value}" for key, value in self.env.items() ] + aslist = [f"{prefix}_{key}={value}" for key, value in self.env.items()] return "\n".join(aslist) + "\n" diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py index 217c362eb..fe4afa9dd 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_container.py @@ -1,65 +1,95 @@ from pathlib import Path -from typing import List, Optional, Dict, Union +from typing import ( + Dict, + List, + Optional, + Union, +) +import importlib_resources import luigi import netaddr -import importlib_resources - from docker.models.containers import Container from docker.transport import unixconn from exasol_integration_test_docker_environment.lib import PACKAGE_NAME -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import JsonPickleParameter -from exasol_integration_test_docker_environment.lib.data.container_info import ContainerInfo -from exasol_integration_test_docker_environment.lib.data.docker_network_info import DockerNetworkInfo -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageState, ImageInfo -from exasol_integration_test_docker_environment.lib.test_environment.analyze_test_container import \ - DockerTestContainerBuild -from exasol_integration_test_docker_environment.lib.test_environment.docker_container_copy import \ - copy_script_to_container -from exasol_integration_test_docker_environment.lib.test_environment.parameter.test_container_parameter import \ - TestContainerParameter +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import ( + JsonPickleParameter, +) +from exasol_integration_test_docker_environment.lib.data.container_info import ( + ContainerInfo, +) +from exasol_integration_test_docker_environment.lib.data.docker_network_info import ( + DockerNetworkInfo, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, + ImageState, +) +from exasol_integration_test_docker_environment.lib.test_environment.analyze_test_container import ( + DockerTestContainerBuild, +) +from exasol_integration_test_docker_environment.lib.test_environment.docker_container_copy import ( + copy_script_to_container, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.test_container_parameter import ( + TestContainerParameter, +) class SpawnTestContainer(DockerBaseTask, TestContainerParameter): - environment_name : str = luigi.Parameter() #type: ignore - test_container_name : str = luigi.Parameter() #type: ignore - network_info : DockerNetworkInfo = JsonPickleParameter( - DockerNetworkInfo, significant=False) # type: ignore - ip_address_index_in_subnet : int = luigi.IntParameter(significant=False) #type: ignore - attempt : int = luigi.IntParameter(1) #type: ignore - reuse_test_container : bool = luigi.BoolParameter(False, significant=False) #type: ignore - no_test_container_cleanup_after_success : bool = luigi.BoolParameter(False, significant=False) #type: ignore - no_test_container_cleanup_after_failure : bool = luigi.BoolParameter(False, significant=False) #type: ignore - docker_runtime : Optional[str] = luigi.OptionalParameter(None, significant=False) #type: ignore - certificate_volume_name : Optional[str] = luigi.OptionalParameter(None, significant=False) #type: ignore + environment_name: str = luigi.Parameter() # type: ignore + test_container_name: str = luigi.Parameter() # type: ignore + network_info: DockerNetworkInfo = JsonPickleParameter( + DockerNetworkInfo, significant=False + ) # type: ignore + ip_address_index_in_subnet: int = luigi.IntParameter(significant=False) # type: ignore + attempt: int = luigi.IntParameter(1) # type: ignore + reuse_test_container: bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_test_container_cleanup_after_success: bool = luigi.BoolParameter(False, significant=False) # type: ignore + no_test_container_cleanup_after_failure: bool = luigi.BoolParameter(False, significant=False) # type: ignore + docker_runtime: Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + certificate_volume_name: Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if self.ip_address_index_in_subnet < 0: raise Exception( "ip_address_index_in_subnet needs to be greater than 0 got %s" - % self.ip_address_index_in_subnet) + % self.ip_address_index_in_subnet + ) def register_required(self): - self.test_container_image_future = \ - self.register_dependency(self.create_child_task(task_class=DockerTestContainerBuild, - test_container_content=self.test_container_content)) + self.test_container_image_future = self.register_dependency( + self.create_child_task( + task_class=DockerTestContainerBuild, + test_container_content=self.test_container_content, + ) + ) def is_reuse_possible(self) -> bool: - test_container_image_info : ImageInfo = \ - self.get_values_from_futures(self.test_container_image_future)["test-container"] # type: ignore + test_container_image_info: ImageInfo = self.get_values_from_futures( + self.test_container_image_future + )[ + "test-container" + ] # type: ignore test_container = None with self._get_docker_client() as docker_client: try: test_container = docker_client.containers.get(self.test_container_name) except Exception as e: pass - ret_val : bool = bool(self.network_info.reused and self.reuse_test_container and \ - test_container is not None and \ - test_container_image_info.get_target_complete_name() in test_container.image.tags and \ - test_container_image_info.image_state == ImageState.USED_LOCAL.name) + ret_val: bool = bool( + self.network_info.reused + and self.reuse_test_container + and test_container is not None + and test_container_image_info.get_target_complete_name() + in test_container.image.tags + and test_container_image_info.image_state == ImageState.USED_LOCAL.name + ) return ret_val @@ -69,7 +99,9 @@ def run_task(self): container_info = None if self.is_reuse_possible(): - container_info = self._try_to_reuse_test_container(ip_address, self.network_info) + container_info = self._try_to_reuse_test_container( + ip_address, self.network_info + ) if container_info is None: container_info = self._create_test_container(ip_address, self.network_info) with self._get_docker_client() as docker_client: @@ -78,104 +110,137 @@ def run_task(self): self.return_object(container_info) def _copy_runtime_targets(self): - self.logger.info("Copy runtime targets in test container %s.", self.test_container_name) + self.logger.info( + "Copy runtime targets in test container %s.", self.test_container_name + ) with self._get_docker_client() as docker_client: test_container = docker_client.containers.get(self.test_container_name) for runtime_mapping in self.test_container_content.runtime_mappings: if runtime_mapping.deployment_target is not None: - self.logger.warning(f"Copy runtime target {runtime_mapping.target} " - f"in test container {self.test_container_name}.") + self.logger.warning( + f"Copy runtime target {runtime_mapping.target} " + f"in test container {self.test_container_name}." + ) try: - test_container.exec_run(cmd=f"rm -r {runtime_mapping.deployment_target}") + test_container.exec_run( + cmd=f"rm -r {runtime_mapping.deployment_target}" + ) except: pass - test_container.exec_run(cmd=f"cp -r {runtime_mapping.target} {runtime_mapping.deployment_target}") + test_container.exec_run( + cmd=f"cp -r {runtime_mapping.target} {runtime_mapping.deployment_target}" + ) - def _try_to_reuse_test_container(self, ip_address: str, - network_info: DockerNetworkInfo) -> Optional[ContainerInfo]: - self.logger.info("Try to reuse test container %s", - self.test_container_name) + def _try_to_reuse_test_container( + self, ip_address: str, network_info: DockerNetworkInfo + ) -> Optional[ContainerInfo]: + self.logger.info("Try to reuse test container %s", self.test_container_name) container_info = None try: network_aliases = self._get_network_aliases() - container_info = self.create_container_info(ip_address, network_aliases, network_info) + container_info = self.create_container_info( + ip_address, network_aliases, network_info + ) except Exception as e: - self.logger.warning("Tried to reuse test container %s, but got Exeception %s. " - "Fallback to create new database.", self.test_container_name, e) + self.logger.warning( + "Tried to reuse test container %s, but got Exeception %s. " + "Fallback to create new database.", + self.test_container_name, + e, + ) return container_info - def _create_test_container(self, ip_address, - network_info: DockerNetworkInfo) -> ContainerInfo: + def _create_test_container( + self, ip_address, network_info: DockerNetworkInfo + ) -> ContainerInfo: self._remove_container(self.test_container_name) self.logger.info(f"Creating new test container {self.test_container_name}") - test_container_image_info = \ - self.get_values_from_futures(self.test_container_image_future)["test-container"] + test_container_image_info = self.get_values_from_futures( + self.test_container_image_future + )["test-container"] - volumes : Dict[Union[str, Path], Dict[str, str]] = dict() + volumes: Dict[Union[str, Path], Dict[str, str]] = dict() for runtime_mapping in self.test_container_content.runtime_mappings: volumes[runtime_mapping.source.absolute()] = { "bind": runtime_mapping.target, - "mode": "rw" + "mode": "rw", } if self.certificate_volume_name is not None: volumes[self.certificate_volume_name] = { "bind": "/certificates", - "mode": "ro" + "mode": "ro", } with self._get_docker_client() as docker_client: - docker_unix_sockets = [i for i in docker_client.api.adapters.values() - if isinstance(i, unixconn.UnixHTTPAdapter)] + docker_unix_sockets = [ + i + for i in docker_client.api.adapters.values() + if isinstance(i, unixconn.UnixHTTPAdapter) + ] if len(docker_unix_sockets) > 0: host_docker_socker_path = docker_unix_sockets[0].socket_path volumes[host_docker_socker_path] = { "bind": "/var/run/docker.sock", - "mode": "rw" + "mode": "rw", } - test_container = \ - docker_client.containers.create( - image=test_container_image_info.get_target_complete_name(), - name=self.test_container_name, - network_mode=None, - command="sleep infinity", - detach=True, - volumes=volumes, - labels={"test_environment_name": self.environment_name, "container_type": "test_container"}, - runtime=self.docker_runtime - ) + test_container = docker_client.containers.create( + image=test_container_image_info.get_target_complete_name(), + name=self.test_container_name, + network_mode=None, + command="sleep infinity", + detach=True, + volumes=volumes, + labels={ + "test_environment_name": self.environment_name, + "container_type": "test_container", + }, + runtime=self.docker_runtime, + ) docker_network = docker_client.networks.get(network_info.network_name) network_aliases = self._get_network_aliases() - docker_network.connect(test_container, ipv4_address=ip_address, aliases=network_aliases) + docker_network.connect( + test_container, ipv4_address=ip_address, aliases=network_aliases + ) test_container.start() self.register_certificates(test_container) - container_info = self.create_container_info(ip_address, network_aliases, network_info) + container_info = self.create_container_info( + ip_address, network_aliases, network_info + ) return container_info def _get_network_aliases(self): network_aliases = ["test_container", self.test_container_name] return network_aliases - def create_container_info(self, ip_address: str, network_aliases: List[str], - network_info: DockerNetworkInfo) -> ContainerInfo: + def create_container_info( + self, + ip_address: str, + network_aliases: List[str], + network_info: DockerNetworkInfo, + ) -> ContainerInfo: with self._get_docker_client() as docker_client: test_container = docker_client.containers.get(self.test_container_name) if test_container.status != "running": raise Exception(f"Container {self.test_container_name} not running") - container_info = ContainerInfo(container_name=self.test_container_name, - ip_address=ip_address, - network_aliases=network_aliases, - network_info=network_info) + container_info = ContainerInfo( + container_name=self.test_container_name, + ip_address=ip_address, + network_aliases=network_aliases, + network_info=network_info, + ) return container_info def _get_export_directory(self): - return self.get_values_from_future(self.export_directory_future) # type: ignore + return self.get_values_from_future(self.export_directory_future) # type: ignore def _remove_container(self, container_name: str): try: with self._get_docker_client() as docker_client: container = docker_client.containers.get(container_name) container.remove(force=True) - self.logger.info(f"Removed container: name: '{container_name}', id: '{container.short_id}'") + self.logger.info( + f"Removed container: name: '{container_name}', id: '{container.short_id}'" + ) except Exception as e: pass @@ -194,15 +259,24 @@ def register_certificates(self, test_container: Container): test_container, ) - exit_code, output = test_container.exec_run(f"bash {script_location_in_container}") + exit_code, output = test_container.exec_run( + f"bash {script_location_in_container}" + ) if exit_code != 0: - raise RuntimeError(f"Error installing certificates:{output.decode('utf-8')}") + raise RuntimeError( + f"Error installing certificates:{output.decode('utf-8')}" + ) def cleanup_task(self, success: bool): - if (success and not self.no_test_container_cleanup_after_success) or \ - (not success and not self.no_test_container_cleanup_after_failure): + if (success and not self.no_test_container_cleanup_after_success) or ( + not success and not self.no_test_container_cleanup_after_failure + ): try: self.logger.info(f"Cleaning up container %s", self.test_container_name) self._remove_container(self.test_container_name) except Exception as e: - self.logger.error(f"Error during removing container %s: %s", self.test_container_name, e) + self.logger.error( + f"Error during removing container %s: %s", + self.test_container_name, + e, + ) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py index a54dfcce7..59ffb534c 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_database.py @@ -1,46 +1,67 @@ import math -from typing import Optional, Tuple, List, Union from pathlib import Path +from typing import ( + List, + Optional, + Tuple, + Union, +) import docker import humanfriendly +import importlib_resources import luigi import netaddr -import importlib_resources - +from docker.client import DockerClient from docker.models.containers import Container from docker.models.volumes import Volume -from docker.client import DockerClient from importlib_resources.abc import Traversable from jinja2 import Template from exasol_integration_test_docker_environment.lib import PACKAGE_NAME -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import JsonPickleParameter -from exasol_integration_test_docker_environment.lib.base.still_running_logger import StillRunningLogger -from exasol_integration_test_docker_environment.lib.data.container_info import ContainerInfo -from exasol_integration_test_docker_environment.lib.data.database_info import DatabaseInfo -from exasol_integration_test_docker_environment.lib.data.docker_network_info import DockerNetworkInfo -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.pull_log_handler import PullLogHandler -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageInfo -from exasol_integration_test_docker_environment.lib.test_environment.db_version import DbVersion -from exasol_integration_test_docker_environment.lib.data.ssh_info import SshInfo -from exasol_integration_test_docker_environment.lib.test_environment.ports import ( - find_free_ports, - Ports, +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, ) -from exasol_integration_test_docker_environment.lib.test_environment.docker_container_copy import DockerContainerCopy -from exasol_integration_test_docker_environment.lib \ - .test_environment.parameter \ - .docker_db_test_environment_parameter import ( - DbOsAccess, - DockerDBTestEnvironmentParameter, +from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import ( + JsonPickleParameter, ) from exasol_integration_test_docker_environment.lib.base.ssh_access import ( - SshKeyCache, SshKey, + SshKeyCache, +) +from exasol_integration_test_docker_environment.lib.base.still_running_logger import ( + StillRunningLogger, +) +from exasol_integration_test_docker_environment.lib.data.container_info import ( + ContainerInfo, +) +from exasol_integration_test_docker_environment.lib.data.database_info import ( + DatabaseInfo, +) +from exasol_integration_test_docker_environment.lib.data.docker_network_info import ( + DockerNetworkInfo, +) +from exasol_integration_test_docker_environment.lib.data.ssh_info import SshInfo +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.pull_log_handler import ( + PullLogHandler, +) +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageInfo, +) +from exasol_integration_test_docker_environment.lib.test_environment.db_version import ( + DbVersion, +) +from exasol_integration_test_docker_environment.lib.test_environment.docker_container_copy import ( + DockerContainerCopy, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter import ( + DbOsAccess, + DockerDBTestEnvironmentParameter, +) +from exasol_integration_test_docker_environment.lib.test_environment.ports import ( + Ports, + find_free_ports, ) - CERTIFICATES_MOUNT_DIR = "/certificates" CERTIFICATES_DEFAULT_DIR = "/exa/etc/ssl/" @@ -51,23 +72,24 @@ def int_or_none(value: str) -> Optional[int]: class SpawnTestDockerDatabase(DockerBaseTask, DockerDBTestEnvironmentParameter): - environment_name : str = luigi.Parameter() # type: ignore - db_container_name : str = luigi.Parameter() # type: ignore - attempt : int = luigi.IntParameter(1) # type: ignore - network_info : DockerNetworkInfo = JsonPickleParameter(DockerNetworkInfo, significant=False) # type: ignore - ip_address_index_in_subnet : int = luigi.IntParameter(significant=False) # type: ignore - docker_runtime : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore - certificate_volume_name : Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore - additional_db_parameter : List[str] = luigi.ListParameter() # type: ignore - ssh_user : str = luigi.Parameter("root") # type: ignore - ssh_key_file : Union[str, Path, None] = luigi.OptionalParameter(None, significant=False) # type: ignore + environment_name: str = luigi.Parameter() # type: ignore + db_container_name: str = luigi.Parameter() # type: ignore + attempt: int = luigi.IntParameter(1) # type: ignore + network_info: DockerNetworkInfo = JsonPickleParameter(DockerNetworkInfo, significant=False) # type: ignore + ip_address_index_in_subnet: int = luigi.IntParameter(significant=False) # type: ignore + docker_runtime: Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + certificate_volume_name: Optional[str] = luigi.OptionalParameter(None, significant=False) # type: ignore + additional_db_parameter: List[str] = luigi.ListParameter() # type: ignore + ssh_user: str = luigi.Parameter("root") # type: ignore + ssh_key_file: Union[str, Path, None] = luigi.OptionalParameter(None, significant=False) # type: ignore def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if self.ip_address_index_in_subnet < 0: raise Exception( "ip_address_index_in_subnet needs to be greater than 0 got %s" - % self.ip_address_index_in_subnet) + % self.ip_address_index_in_subnet + ) self.db_version = DbVersion.from_db_version_str(self.docker_db_image_version) self.docker_db_config_resource_name = f"docker_db_config/{self.db_version}" @@ -83,23 +105,30 @@ def __init__(self, *args, **kwargs): def run_task(self): subnet = netaddr.IPNetwork(self.network_info.subnet) db_ip_address = str(subnet[2 + self.ip_address_index_in_subnet]) - db_private_network = "{ip}/{prefix}".format(ip=db_ip_address, prefix=subnet.prefixlen) + db_private_network = f"{db_ip_address}/{subnet.prefixlen}" database_info = None if self.network_info.reused: database_info = self._try_to_reuse_database(db_ip_address) if database_info is None: - database_info = self._create_database_container(db_ip_address, db_private_network) + database_info = self._create_database_container( + db_ip_address, db_private_network + ) self.return_object(database_info) def _try_to_reuse_database(self, db_ip_address: str) -> Optional[DatabaseInfo]: - self.logger.info("Try to reuse database container %s", - self.db_container_name) + self.logger.info("Try to reuse database container %s", self.db_container_name) try: - database_info = self._create_database_info(db_ip_address=db_ip_address, reused=True) + database_info = self._create_database_info( + db_ip_address=db_ip_address, reused=True + ) return database_info except Exception as e: - self.logger.warning("Tried to reuse database container %s, but got Exeception %s. " - "Fallback to create new database.", self.db_container_name, e) + self.logger.warning( + "Tried to reuse database container %s, but got Exeception %s. " + "Fallback to create new database.", + self.db_container_name, + e, + ) return None def _get_ssh_key(self) -> SshKey: @@ -112,20 +141,25 @@ def _handle_output(self, output_generator, image_info: ImageInfo): log_file_path = self.get_log_path().joinpath("pull_docker_db_image.log") with PullLogHandler(log_file_path, self.logger, image_info) as log_handler: still_running_logger = StillRunningLogger( - self.logger, "pull image %s" % image_info.get_source_complete_name()) + self.logger, "pull image %s" % image_info.get_source_complete_name() + ) for log_line in output_generator: still_running_logger.log() log_handler.handle_log_lines(log_line) def _get_network_aliases(self): - network_aliases = ["exasol_test_database", "exasol-test-database", self.db_container_name] + network_aliases = [ + "exasol_test_database", + "exasol-test-database", + self.db_container_name, + ] return network_aliases def _connect_docker_network( - self, - docker_client: DockerClient, - container: Container, - ip_address: str, + self, + docker_client: DockerClient, + container: Container, + ip_address: str, ): network = docker_client.networks.get(self.network_info.network_name) aliases = self._get_network_aliases() @@ -136,7 +170,7 @@ def _port_mapping(self, internal_ports, forwarded_ports): for name, internal in internal_ports.__dict__.items(): forward = forwarded_ports.__getattribute__(name) if forward: - result[f"{internal}/tcp"] = ('0.0.0.0', forward) + result[f"{internal}/tcp"] = ("0.0.0.0", forward) return result def _create_database_container(self, db_ip_address: str, db_private_network: str): @@ -158,7 +192,9 @@ def enable_ssh_access(container: Container, authorized_keys: str): authorized_keys = get_authorized_keys(ssh_key) with self._get_docker_client() as docker_client: try: - docker_client.containers.get(self.db_container_name).remove(force=True, v=True) + docker_client.containers.get(self.db_container_name).remove( + force=True, v=True + ) except: pass docker_db_image_info = self._pull_docker_db_images_if_necessary() @@ -171,22 +207,26 @@ def enable_ssh_access(container: Container, authorized_keys: str): port_mapping = self._port_mapping(self.internal_ports, self.forwarded_ports) volumes = {db_volume.name: {"bind": "/exa", "mode": "rw"}} if self.certificate_volume_name is not None: - volumes[self.certificate_volume_name] = {"bind": CERTIFICATES_MOUNT_DIR, "mode": "ro"} - db_container = \ - docker_client.containers.create( - image="%s" % (docker_db_image_info.get_source_complete_name()), - name=self.db_container_name, - detach=True, - privileged=True, - volumes=volumes, - network_mode=None, - ports=port_mapping, - runtime=self.docker_runtime - ) + volumes[self.certificate_volume_name] = { + "bind": CERTIFICATES_MOUNT_DIR, + "mode": "ro", + } + db_container = docker_client.containers.create( + image="%s" % (docker_db_image_info.get_source_complete_name()), + name=self.db_container_name, + detach=True, + privileged=True, + volumes=volumes, + network_mode=None, + ports=port_mapping, + runtime=self.docker_runtime, + ) enable_ssh_access(db_container, authorized_keys) self._connect_docker_network(docker_client, db_container, db_ip_address) db_container.start() - database_info = self._create_database_info(db_ip_address=db_ip_address, reused=False) + database_info = self._create_database_info( + db_ip_address=db_ip_address, reused=False + ) return database_info def _create_database_info(self, db_ip_address: str, reused: bool) -> DatabaseInfo: @@ -220,27 +260,34 @@ def _pull_docker_db_images_if_necessary(self): source_repository_name=image_name, source_tag=self.docker_db_image_version, target_tag=self.docker_db_image_version, - hash_value="", commit="", - image_description=None) + hash_value="", + commit="", + image_description=None, + ) with self._get_docker_client() as docker_client: try: - docker_client.images.get(docker_db_image_info.get_source_complete_name()) + docker_client.images.get( + docker_db_image_info.get_source_complete_name() + ) except docker.errors.ImageNotFound as e: - self.logger.info("Pulling docker-db image %s", - docker_db_image_info.get_source_complete_name()) + self.logger.info( + "Pulling docker-db image %s", + docker_db_image_info.get_source_complete_name(), + ) output_generator = docker_client.api.pull( docker_db_image_info.source_repository_name, tag=docker_db_image_info.source_tag, - stream=True) + stream=True, + ) self._handle_output(output_generator, docker_db_image_info) return docker_db_image_info def _prepare_db_volume( - self, - docker_client, - db_private_network: str, - authorized_keys: str, - docker_db_image_info: ImageInfo + self, + docker_client, + db_private_network: str, + authorized_keys: str, + docker_db_image_info: ImageInfo, ) -> Volume: volume, container = self._prepare_volume( docker_client, @@ -261,7 +308,7 @@ def _get_db_volume_preparation_container_name(self): def _get_db_volume_name(self): return f"""{self.db_container_name}_volume""" - def _remove_container(self, container_name:str): + def _remove_container(self, container_name: str): try: with self._get_docker_client() as docker_client: docker_client.containers.get(container_name).remove(force=True) @@ -269,7 +316,7 @@ def _remove_container(self, container_name:str): except docker.errors.NotFound: pass - def _remove_volume(self, volume_name:str): + def _remove_volume(self, volume_name: str): try: with self._get_docker_client() as docker_client: docker_client.volumes.get(volume_name).remove(force=True) @@ -278,11 +325,11 @@ def _remove_volume(self, volume_name:str): pass def _prepare_volume( - self, - docker_client: docker.api.APIClient, - volume_name, - container_name, - remove_old_instances: bool = False, + self, + docker_client: docker.api.APIClient, + volume_name, + container_name, + remove_old_instances: bool = False, ) -> Tuple[Volume, Container]: """ Create an intermediate Docker Container containing a volume that @@ -293,16 +340,16 @@ def _prepare_volume( self._remove_volume(volume_name) volume = docker_client.volumes.create(volume_name) container = docker_client.containers.run( - image="ubuntu:18.04", - name=container_name, - auto_remove=True, - command="sleep infinity", - detach=True, - volumes={volume.name: {"bind": "/exa", "mode": "rw"}}, - labels={ - "test_environment_name": self.environment_name, - "container_type": "db_volume_preparation_container", - } + image="ubuntu:18.04", + name=container_name, + auto_remove=True, + command="sleep infinity", + detach=True, + volumes={volume.name: {"bind": "/exa", "mode": "rw"}}, + labels={ + "test_environment_name": self.environment_name, + "container_type": "db_volume_preparation_container", + }, ) return volume, container @@ -314,10 +361,10 @@ def _db_file(self, filename: str) -> Traversable: ) def _upload_init_db_files( - self, - container: Container, - db_private_network: str, - authorized_keys: str, + self, + container: Container, + db_private_network: str, + authorized_keys: str, ): copy = DockerContainerCopy(container) init_script = self._db_file("init_db.sh") @@ -326,67 +373,87 @@ def _upload_init_db_files( copy.copy("/") def _add_exa_conf( - self, - copy: DockerContainerCopy, - db_private_network: str, - authorized_keys: str, + self, + copy: DockerContainerCopy, + db_private_network: str, + authorized_keys: str, ): """ Multiple authorized_keys can be comma-separated. """ - certificate_dir = CERTIFICATES_MOUNT_DIR if self.certificate_volume_name is not None \ - else CERTIFICATES_DEFAULT_DIR + certificate_dir = ( + CERTIFICATES_MOUNT_DIR + if self.certificate_volume_name is not None + else CERTIFICATES_DEFAULT_DIR + ) template_file = self._db_file("EXAConf") template = Template(template_file.read_text()) additional_db_parameter_str = " ".join(self.additional_db_parameter) - rendered_template = template.render(private_network=db_private_network, - db_version=str(self.db_version), - db_port=self.internal_ports.database, - ssh_port=self.internal_ports.ssh, - bucketfs_port=self.internal_ports.bucketfs, - image_version=self.docker_db_image_version, - mem_size=self.mem_size, - disk_size=self.disk_size, - name_servers=",".join(self.nameservers), - certificate_dir=certificate_dir, - additional_db_parameters=additional_db_parameter_str, - authorized_keys=authorized_keys) + rendered_template = template.render( + private_network=db_private_network, + db_version=str(self.db_version), + db_port=self.internal_ports.database, + ssh_port=self.internal_ports.ssh, + bucketfs_port=self.internal_ports.bucketfs, + image_version=self.docker_db_image_version, + mem_size=self.mem_size, + disk_size=self.disk_size, + name_servers=",".join(self.nameservers), + certificate_dir=certificate_dir, + additional_db_parameters=additional_db_parameter_str, + authorized_keys=authorized_keys, + ) copy.add_string_to_file("EXAConf", rendered_template) def _execute_init_db(self, db_volume: Volume, preparation_container: Container): disk_size_in_bytes = humanfriendly.parse_size(self.disk_size) min_overhead_in_gigabyte = 2 # Exasol needs at least a 2 GB larger device than the configured disk size - overhead_factor = max(0.01, ( - min_overhead_in_gigabyte * 1024 * 1024 * 1024) / disk_size_in_bytes) # and in general 1% larger + overhead_factor = max( + 0.01, (min_overhead_in_gigabyte * 1024 * 1024 * 1024) / disk_size_in_bytes + ) # and in general 1% larger device_size_in_bytes = disk_size_in_bytes * (1 + overhead_factor) device_size_in_megabytes = math.ceil( - device_size_in_bytes / (1024 * 1024)) # The init_db.sh script works with MB, because its faster + device_size_in_bytes / (1024 * 1024) + ) # The init_db.sh script works with MB, because its faster self.logger.info( - f"Creating database volume of size {device_size_in_megabytes / 1024} GB using and overhead factor of {overhead_factor}") - (exit_code, output) = preparation_container.exec_run(cmd=f"bash /init_db.sh {device_size_in_megabytes}") + f"Creating database volume of size {device_size_in_megabytes / 1024} GB using and overhead factor of {overhead_factor}" + ) + (exit_code, output) = preparation_container.exec_run( + cmd=f"bash /init_db.sh {device_size_in_megabytes}" + ) if exit_code != 0: raise Exception( - "Error during preparation of docker-db volume %s got following output %s" % (db_volume.name, output)) + "Error during preparation of docker-db volume {} got following output {}".format( + db_volume.name, output + ) + ) def cleanup_task(self, success): - if (success and not self.no_database_cleanup_after_success) or \ - (not success and not self.no_database_cleanup_after_failure): + if (success and not self.no_database_cleanup_after_success) or ( + not success and not self.no_database_cleanup_after_failure + ): volume_container = self._get_db_volume_preparation_container_name() try: self.logger.info(f"Cleaning up container %s", volume_container) self._remove_container(volume_container) except Exception as e: - self.logger.error(f"Error during removing container %s: %s", volume_container, e) + self.logger.error( + f"Error during removing container %s: %s", volume_container, e + ) try: self.logger.info(f"Cleaning up container %s", self.db_container_name) self._remove_container(self.db_container_name) except Exception as e: - self.logger.error(f"Error during removing container %s: %s", self.db_container_name, e) + self.logger.error( + f"Error during removing container %s: %s", self.db_container_name, e + ) db_volume_name = self._get_db_volume_name() try: self.logger.info(f"Cleaning up docker volume %s", db_volume_name) self._remove_volume(db_volume_name) except Exception as e: - self.logger.error(f"Error during removing docker volume %s: %s", db_volume_name, e) + self.logger.error( + f"Error during removing docker volume %s: %s", db_volume_name, e + ) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment.py index 5a19520ee..e0af1cdcb 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment.py @@ -1,13 +1,20 @@ import luigi -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask -from exasol_integration_test_docker_environment.lib.data.environment_type import EnvironmentType -from exasol_integration_test_docker_environment.lib.test_environment.parameter.spawn_test_environment_parameter import \ - SpawnTestEnvironmentParameter -from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_environment_with_docker_db import \ - SpawnTestEnvironmentWithDockerDB -from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_environment_with_external_db import \ - SpawnTestEnvironmentWithExternalDB +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) +from exasol_integration_test_docker_environment.lib.data.environment_type import ( + EnvironmentType, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.spawn_test_environment_parameter import ( + SpawnTestEnvironmentParameter, +) +from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_environment_with_docker_db import ( + SpawnTestEnvironmentWithDockerDB, +) +from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_environment_with_external_db import ( + SpawnTestEnvironmentWithExternalDB, +) class SpawnTestEnvironment(DependencyLoggerBaseTask, SpawnTestEnvironmentParameter): @@ -33,23 +40,21 @@ def _create_external_db_environment(self): raise Exception("external_exasol_db_port not set") if self.external_exasol_bucketfs_port is None: raise Exception("external_exasol_bucketfs_port not set") - task = \ - self.create_child_task_with_common_params( - SpawnTestEnvironmentWithExternalDB, - db_user=self.external_exasol_db_user, - db_password=self.external_exasol_db_password, - bucketfs_write_password=self.external_exasol_bucketfs_write_password - ) + task = self.create_child_task_with_common_params( + SpawnTestEnvironmentWithExternalDB, + db_user=self.external_exasol_db_user, + db_password=self.external_exasol_db_password, + bucketfs_write_password=self.external_exasol_bucketfs_write_password, + ) return task def _create_docker_db_environment(self): - task = \ - self.create_child_task_with_common_params( - SpawnTestEnvironmentWithDockerDB, - db_user=self.DEFAULT_DB_USER, - db_password=self.DEFAULT_DATABASE_PASSWORD, - bucketfs_write_password=self.DEFAULT_BUCKETFS_WRITE_PASSWORD - ) + task = self.create_child_task_with_common_params( + SpawnTestEnvironmentWithDockerDB, + db_user=self.DEFAULT_DB_USER, + db_password=self.DEFAULT_DATABASE_PASSWORD, + bucketfs_write_password=self.DEFAULT_BUCKETFS_WRITE_PASSWORD, + ) return task def run_task(self): diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_docker_db.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_docker_db.py index b81287677..9284ccb4e 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_docker_db.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_docker_db.py @@ -1,45 +1,53 @@ from typing import Optional -from exasol_integration_test_docker_environment.lib.data.container_info import ContainerInfo -from exasol_integration_test_docker_environment.lib.data.database_info import DatabaseInfo -from exasol_integration_test_docker_environment.lib.data.docker_network_info import DockerNetworkInfo -from exasol_integration_test_docker_environment.lib.data.docker_volume_info import DockerVolumeInfo -from exasol_integration_test_docker_environment.lib.data.environment_type import EnvironmentType -from exasol_integration_test_docker_environment \ - .lib.test_environment.abstract_spawn_test_environment \ - import AbstractSpawnTestEnvironment -from exasol_integration_test_docker_environment \ - .lib.test_environment.create_certificates.create_ssl_certificates_task \ - import CreateSSLCertificatesTask -from exasol_integration_test_docker_environment \ - .lib.test_environment.database_waiters.wait_for_test_docker_database \ - import WaitForTestDockerDatabase -from exasol_integration_test_docker_environment \ - .lib.test_environment.db_version \ - import db_version_supports_custom_certificates -from exasol_integration_test_docker_environment \ - .lib.test_environment.parameter.docker_db_test_environment_parameter \ - import DockerDBTestEnvironmentParameter -from exasol_integration_test_docker_environment \ - .lib.test_environment.prepare_network_for_test_environment \ - import PrepareDockerNetworkForTestEnvironment -from exasol_integration_test_docker_environment \ - .lib.test_environment.spawn_test_database \ - import SpawnTestDockerDatabase -from exasol_integration_test_docker_environment \ - .lib.base.db_os_executor import ( - DockerClientFactory, - DbOsExecFactory, - SshExecFactory, - DockerExecFactory, - ) -from exasol_integration_test_docker_environment \ - .lib.test_environment.parameter.docker_db_test_environment_parameter \ - import DbOsAccess +from exasol_integration_test_docker_environment.lib.base.db_os_executor import ( + DbOsExecFactory, + DockerClientFactory, + DockerExecFactory, + SshExecFactory, +) +from exasol_integration_test_docker_environment.lib.data.container_info import ( + ContainerInfo, +) +from exasol_integration_test_docker_environment.lib.data.database_info import ( + DatabaseInfo, +) +from exasol_integration_test_docker_environment.lib.data.docker_network_info import ( + DockerNetworkInfo, +) +from exasol_integration_test_docker_environment.lib.data.docker_volume_info import ( + DockerVolumeInfo, +) +from exasol_integration_test_docker_environment.lib.data.environment_type import ( + EnvironmentType, +) +from exasol_integration_test_docker_environment.lib.test_environment.abstract_spawn_test_environment import ( + AbstractSpawnTestEnvironment, +) +from exasol_integration_test_docker_environment.lib.test_environment.create_certificates.create_ssl_certificates_task import ( + CreateSSLCertificatesTask, +) +from exasol_integration_test_docker_environment.lib.test_environment.database_waiters.wait_for_test_docker_database import ( + WaitForTestDockerDatabase, +) +from exasol_integration_test_docker_environment.lib.test_environment.db_version import ( + db_version_supports_custom_certificates, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter import ( + DbOsAccess, + DockerDBTestEnvironmentParameter, +) +from exasol_integration_test_docker_environment.lib.test_environment.prepare_network_for_test_environment import ( + PrepareDockerNetworkForTestEnvironment, +) +from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_database import ( + SpawnTestDockerDatabase, +) + class SpawnTestEnvironmentWithDockerDB( - AbstractSpawnTestEnvironment, - DockerDBTestEnvironmentParameter): + AbstractSpawnTestEnvironment, DockerDBTestEnvironmentParameter +): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -51,7 +59,9 @@ def get_environment_type(self): def create_ssl_certificates(self): if not db_version_supports_custom_certificates(self.docker_db_image_version): - raise ValueError("Minimal supported Database with custom certificates is '7.0.6'") + raise ValueError( + "Minimal supported Database with custom certificates is '7.0.6'" + ) return self.create_child_task_with_common_params( CreateSSLCertificatesTask, environment_name=self.environment_name, @@ -60,8 +70,10 @@ def create_ssl_certificates(self): docker_runtime=self.docker_runtime, volume_name=self.certificate_volume_name, reuse=self.reuse_database or self.reuse_test_container, - no_cleanup_after_success=self.no_database_cleanup_after_success or self.no_test_container_cleanup_after_success, - no_cleanup_after_failure=self.no_database_cleanup_after_failure or self.no_test_container_cleanup_after_failure, + no_cleanup_after_success=self.no_database_cleanup_after_success + or self.no_test_container_cleanup_after_success, + no_cleanup_after_failure=self.no_database_cleanup_after_failure + or self.no_test_container_cleanup_after_failure, ) def create_network_task(self, attempt: int): @@ -71,8 +83,10 @@ def create_network_task(self, attempt: int): network_name=self.network_name, db_container_name=self.db_container_name, reuse=self.reuse_database or self.reuse_test_container, - no_cleanup_after_success=self.no_database_cleanup_after_success or self.no_test_container_cleanup_after_success, - no_cleanup_after_failure=self.no_database_cleanup_after_failure or self.no_test_container_cleanup_after_failure, + no_cleanup_after_success=self.no_database_cleanup_after_success + or self.no_test_container_cleanup_after_success, + no_cleanup_after_failure=self.no_database_cleanup_after_failure + or self.no_test_container_cleanup_after_failure, attempt=attempt, ) @@ -83,13 +97,14 @@ def _executor_factory(self, database_info: DatabaseInfo) -> DbOsExecFactory: return DockerExecFactory(self.db_container_name, client_factory) def create_spawn_database_task( - self, - network_info: DockerNetworkInfo, - certificate_volume_info: Optional[DockerVolumeInfo], - attempt: int, + self, + network_info: DockerNetworkInfo, + certificate_volume_info: Optional[DockerVolumeInfo], + attempt: int, ): def volume_name(info): return None if info is None else info.volume_name + return self.create_child_task_with_common_params( SpawnTestDockerDatabase, db_container_name=self.db_container_name, @@ -97,7 +112,7 @@ def volume_name(info): certificate_volume_name=volume_name(certificate_volume_info), ip_address_index_in_subnet=0, attempt=attempt, - additional_db_parameter=self.additional_db_parameter + additional_db_parameter=self.additional_db_parameter, ) def create_wait_for_database_task(self, attempt: int, database_info: DatabaseInfo): @@ -106,5 +121,5 @@ def create_wait_for_database_task(self, attempt: int, database_info: DatabaseInf database_info=database_info, attempt=attempt, docker_db_image_version=self.docker_db_image_version, - executor_factory=self._executor_factory(database_info) + executor_factory=self._executor_factory(database_info), ) diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_external_db.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_external_db.py index b4ad5a261..fab2dc0af 100644 --- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_external_db.py +++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_external_db.py @@ -1,57 +1,71 @@ from typing import Optional -from exasol_integration_test_docker_environment.lib.data.database_info import DatabaseInfo -from exasol_integration_test_docker_environment.lib.data.docker_network_info import DockerNetworkInfo -from exasol_integration_test_docker_environment.lib.data.docker_volume_info import DockerVolumeInfo -from exasol_integration_test_docker_environment.lib.data.environment_type import EnvironmentType -from exasol_integration_test_docker_environment.lib.test_environment.abstract_spawn_test_environment import \ - AbstractSpawnTestEnvironment -from exasol_integration_test_docker_environment.lib.test_environment.database_waiters.wait_for_external_database import \ - WaitForTestExternalDatabase -from exasol_integration_test_docker_environment.lib.test_environment.parameter.external_test_environment_parameter import \ - ExternalDatabaseXMLRPCParameter, ExternalDatabaseHostParameter -from exasol_integration_test_docker_environment.lib.test_environment.prepare_network_for_test_environment import \ - PrepareDockerNetworkForTestEnvironment -from exasol_integration_test_docker_environment.lib.test_environment.setup_external_database_host import \ - SetupExternalDatabaseHost - - -class SpawnTestEnvironmentWithExternalDB(AbstractSpawnTestEnvironment, - ExternalDatabaseXMLRPCParameter, - ExternalDatabaseHostParameter): +from exasol_integration_test_docker_environment.lib.data.database_info import ( + DatabaseInfo, +) +from exasol_integration_test_docker_environment.lib.data.docker_network_info import ( + DockerNetworkInfo, +) +from exasol_integration_test_docker_environment.lib.data.docker_volume_info import ( + DockerVolumeInfo, +) +from exasol_integration_test_docker_environment.lib.data.environment_type import ( + EnvironmentType, +) +from exasol_integration_test_docker_environment.lib.test_environment.abstract_spawn_test_environment import ( + AbstractSpawnTestEnvironment, +) +from exasol_integration_test_docker_environment.lib.test_environment.database_waiters.wait_for_external_database import ( + WaitForTestExternalDatabase, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.external_test_environment_parameter import ( + ExternalDatabaseHostParameter, + ExternalDatabaseXMLRPCParameter, +) +from exasol_integration_test_docker_environment.lib.test_environment.prepare_network_for_test_environment import ( + PrepareDockerNetworkForTestEnvironment, +) +from exasol_integration_test_docker_environment.lib.test_environment.setup_external_database_host import ( + SetupExternalDatabaseHost, +) + + +class SpawnTestEnvironmentWithExternalDB( + AbstractSpawnTestEnvironment, + ExternalDatabaseXMLRPCParameter, + ExternalDatabaseHostParameter, +): def get_environment_type(self): return EnvironmentType.external_db def create_network_task(self, attempt: int): - return \ - self.create_child_task_with_common_params( - PrepareDockerNetworkForTestEnvironment, - reuse=self.reuse_test_container, - attempt=attempt, - test_container_name=self.test_container_name, - network_name=self.network_name, - no_cleanup_after_success=self.reuse_test_container, - no_cleanup_after_failure=self.reuse_test_container - ) + return self.create_child_task_with_common_params( + PrepareDockerNetworkForTestEnvironment, + reuse=self.reuse_test_container, + attempt=attempt, + test_container_name=self.test_container_name, + network_name=self.network_name, + no_cleanup_after_success=self.reuse_test_container, + no_cleanup_after_failure=self.reuse_test_container, + ) - def create_spawn_database_task(self, network_info: DockerNetworkInfo, - certificate_volume_info: Optional[DockerVolumeInfo], attempt: int): + def create_spawn_database_task( + self, + network_info: DockerNetworkInfo, + certificate_volume_info: Optional[DockerVolumeInfo], + attempt: int, + ): if certificate_volume_info is not None: - raise ValueError("Certificate volume must be None when using external database") - - return \ - self.create_child_task_with_common_params( - SetupExternalDatabaseHost, - network_info=network_info, - attempt=attempt + raise ValueError( + "Certificate volume must be None when using external database" ) - def create_wait_for_database_task(self, - attempt: int, - database_info: DatabaseInfo): - return \ - self.create_child_task_with_common_params( - WaitForTestExternalDatabase, - database_info=database_info, - attempt=attempt) + return self.create_child_task_with_common_params( + SetupExternalDatabaseHost, network_info=network_info, attempt=attempt + ) + + def create_wait_for_database_task(self, attempt: int, database_info: DatabaseInfo): + return self.create_child_task_with_common_params( + WaitForTestExternalDatabase, database_info=database_info, attempt=attempt + ) diff --git a/exasol_integration_test_docker_environment/lib/utils/resource_directory.py b/exasol_integration_test_docker_environment/lib/utils/resource_directory.py index 54333b504..72e75d7a0 100644 --- a/exasol_integration_test_docker_environment/lib/utils/resource_directory.py +++ b/exasol_integration_test_docker_environment/lib/utils/resource_directory.py @@ -10,7 +10,9 @@ LOG = logging.getLogger("resource_directory") -def _copy_importlib_resources_file(src_file: ir.abc.Traversable, target_file: Path) -> None: +def _copy_importlib_resources_file( + src_file: ir.abc.Traversable, target_file: Path +) -> None: """ Uses a given source path "src_file" given as an importlib_resources.abc.Traversable to copy the file it points to into the destination denoted by target_path. @@ -31,7 +33,9 @@ def _copy_importlib_resources_file(src_file: ir.abc.Traversable, target_file: Pa file.write(content) -def _copy_importlib_resources_dir_tree(src_path: ir.abc.Traversable, target_path: Path) -> None: +def _copy_importlib_resources_dir_tree( + src_path: ir.abc.Traversable, target_path: Path +) -> None: """ Uses a given source path "scr_path" given as an importlib_resources.abc.Traversable to copy all files/directories in the directory tree whose root is scr_path into target_path. @@ -61,7 +65,7 @@ def __init__(self, resource_package: ModuleType): # We need to transform the module to a string and later back to a module # because this class will be pickled by luigi and modules are not supported for serialization self._resource_package_str = resource_package.__name__ - self._tmp_directory : Optional[tempfile.TemporaryDirectory] = None + self._tmp_directory: Optional[tempfile.TemporaryDirectory] = None @property def tmp_directory(self): @@ -74,7 +78,9 @@ def create(self) -> str: self._tmp_directory = tempfile.TemporaryDirectory() assert self._tmp_directory source_path = ir.files(self._resource_package_str) - LOG.debug(f"Copying resource package: '{self._resource_package_str}' to '{self._tmp_directory.name}'") + LOG.debug( + f"Copying resource package: '{self._resource_package_str}' to '{self._tmp_directory.name}'" + ) _copy_importlib_resources_dir_tree(source_path, Path(self._tmp_directory.name)) return self._tmp_directory.name diff --git a/exasol_integration_test_docker_environment/main.py b/exasol_integration_test_docker_environment/main.py index 4a1b0b110..63514e6ec 100755 --- a/exasol_integration_test_docker_environment/main.py +++ b/exasol_integration_test_docker_environment/main.py @@ -7,8 +7,9 @@ def main(): # required so the cli will print the available subcommands from exasol_integration_test_docker_environment.cli.commands import ( health, - spawn_test_environment + spawn_test_environment, ) + cli() diff --git a/exasol_integration_test_docker_environment/test/get_test_container_content.py b/exasol_integration_test_docker_environment/test/get_test_container_content.py index 0f1cd4b72..36df73790 100644 --- a/exasol_integration_test_docker_environment/test/get_test_container_content.py +++ b/exasol_integration_test_docker_environment/test/get_test_container_content.py @@ -1,8 +1,11 @@ from pathlib import Path from typing import Tuple -from exasol_integration_test_docker_environment.lib.data.test_container_content_description import \ - TestContainerContentDescription, TestContainerBuildMapping, TestContainerRuntimeMapping +from exasol_integration_test_docker_environment.lib.data.test_container_content_description import ( + TestContainerBuildMapping, + TestContainerContentDescription, + TestContainerRuntimeMapping, +) TEST_CONTAINER_ROOT_PATH = Path(__file__).parent / "resources" / "test_container" @@ -10,12 +13,16 @@ MOCK_TEST_CONTAINER_PATH = TEST_CONTAINER_ROOT_PATH / "mock" -def get_test_container_content(test_container_path: Path = FULL_TEST_CONTAINER_PATH, - runtime_mapping: Tuple[TestContainerRuntimeMapping, ...] = tuple()) \ - -> TestContainerContentDescription: +def get_test_container_content( + test_container_path: Path = FULL_TEST_CONTAINER_PATH, + runtime_mapping: Tuple[TestContainerRuntimeMapping, ...] = tuple(), +) -> TestContainerContentDescription: return TestContainerContentDescription( docker_file=str(test_container_path / "Dockerfile"), - build_files_and_directories=[TestContainerBuildMapping(source=test_container_path / "test.txt", - target="test.text")], - runtime_mappings=list(runtime_mapping) + build_files_and_directories=[ + TestContainerBuildMapping( + source=test_container_path / "test.txt", target="test.text" + ) + ], + runtime_mappings=list(runtime_mapping), ) diff --git a/exasol_integration_test_docker_environment/test/test_api_build_test_container.py b/exasol_integration_test_docker_environment/test/test_api_build_test_container.py index 63b7c403a..22b50a9f1 100644 --- a/exasol_integration_test_docker_environment/test/test_api_build_test_container.py +++ b/exasol_integration_test_docker_environment/test/test_api_build_test_container.py @@ -3,12 +3,17 @@ from exasol_integration_test_docker_environment.lib import api from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.lib.docker.images.image_info import ImageState -from exasol_integration_test_docker_environment.test.get_test_container_content import get_test_container_content, \ - MOCK_TEST_CONTAINER_PATH - +from exasol_integration_test_docker_environment.lib.docker.images.image_info import ( + ImageState, +) +from exasol_integration_test_docker_environment.test.get_test_container_content import ( + MOCK_TEST_CONTAINER_PATH, + get_test_container_content, +) from exasol_integration_test_docker_environment.testing import luigi_utils -from exasol_integration_test_docker_environment.testing.api_test_environment import ApiTestEnvironment +from exasol_integration_test_docker_environment.testing.api_test_environment import ( + ApiTestEnvironment, +) TEST_CONTAINER_CONTENT = get_test_container_content(MOCK_TEST_CONTAINER_PATH) @@ -25,8 +30,10 @@ def tearDown(self): def test_build_test_container(self): docker_repository_name = self.test_environment.docker_repository_name - image_info = api.build_test_container(target_docker_repository_name=docker_repository_name, - test_container_content=TEST_CONTAINER_CONTENT) + image_info = api.build_test_container( + target_docker_repository_name=docker_repository_name, + test_container_content=TEST_CONTAINER_CONTENT, + ) with ContextDockerClient() as docker_client: image = docker_client.images.get(image_info.get_target_complete_name()) self.assertEqual(len(image.tags), 1) @@ -35,23 +42,31 @@ def test_build_test_container(self): def test_build_test_container_use_cached(self): docker_repository_name = self.test_environment.docker_repository_name - image_info1 = api.build_test_container(target_docker_repository_name=docker_repository_name, - test_container_content=TEST_CONTAINER_CONTENT) + image_info1 = api.build_test_container( + target_docker_repository_name=docker_repository_name, + test_container_content=TEST_CONTAINER_CONTENT, + ) self.assertEqual(image_info1.image_state, ImageState.WAS_BUILD.name) - image_info2 = api.build_test_container(target_docker_repository_name=docker_repository_name, - test_container_content=TEST_CONTAINER_CONTENT) + image_info2 = api.build_test_container( + target_docker_repository_name=docker_repository_name, + test_container_content=TEST_CONTAINER_CONTENT, + ) self.assertEqual(image_info2.image_state, ImageState.USED_LOCAL.name) def test_build_test_container_force_rebuild(self): docker_repository_name = self.test_environment.docker_repository_name - image_info1 = api.build_test_container(target_docker_repository_name=docker_repository_name, - test_container_content=TEST_CONTAINER_CONTENT) + image_info1 = api.build_test_container( + target_docker_repository_name=docker_repository_name, + test_container_content=TEST_CONTAINER_CONTENT, + ) self.assertEqual(image_info1.image_state, ImageState.WAS_BUILD.name) - image_info2 = api.build_test_container(target_docker_repository_name=docker_repository_name, - test_container_content=TEST_CONTAINER_CONTENT, - force_rebuild=True) + image_info2 = api.build_test_container( + target_docker_repository_name=docker_repository_name, + test_container_content=TEST_CONTAINER_CONTENT, + force_rebuild=True, + ) self.assertEqual(image_info2.image_state, ImageState.WAS_BUILD.name) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_api_external_test_environment.py b/exasol_integration_test_docker_environment/test/test_api_external_test_environment.py index e6bc32c7e..f52219315 100644 --- a/exasol_integration_test_docker_environment/test/test_api_external_test_environment.py +++ b/exasol_integration_test_docker_environment/test/test_api_external_test_environment.py @@ -1,15 +1,30 @@ import unittest from sys import stderr -from exasol_integration_test_docker_environment.lib.api.common import generate_root_task, run_task -from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo -from exasol_integration_test_docker_environment.lib.data.environment_type import EnvironmentType +from exasol_integration_test_docker_environment.lib.api.common import ( + generate_root_task, + run_task, +) +from exasol_integration_test_docker_environment.lib.data.environment_info import ( + EnvironmentInfo, +) +from exasol_integration_test_docker_environment.lib.data.environment_type import ( + EnvironmentType, +) from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.lib.docker.container.utils import remove_docker_container -from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_environment import SpawnTestEnvironment -from exasol_integration_test_docker_environment.test.get_test_container_content import get_test_container_content +from exasol_integration_test_docker_environment.lib.docker.container.utils import ( + remove_docker_container, +) +from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_environment import ( + SpawnTestEnvironment, +) +from exasol_integration_test_docker_environment.test.get_test_container_content import ( + get_test_container_content, +) from exasol_integration_test_docker_environment.testing import utils -from exasol_integration_test_docker_environment.testing.api_test_environment import ApiTestEnvironment +from exasol_integration_test_docker_environment.testing.api_test_environment import ( + ApiTestEnvironment, +) class APISpawnTestExternalEnvironmentTest(unittest.TestCase): @@ -19,66 +34,88 @@ def setUpClass(cls): print(f"SetUp {cls.__name__}", file=stderr) cls.test_environment = ApiTestEnvironment(cls) cls.docker_environment_name = cls.__name__ - cls.environment = \ - cls.test_environment.spawn_docker_test_environment( - name=cls.docker_environment_name) + cls.environment = cls.test_environment.spawn_docker_test_environment( + name=cls.docker_environment_name + ) cls.ext_environment_name = "APISpawnExternalTestExternalEnvironmentTest" - task_creator = \ - lambda: generate_root_task(task_class=SpawnTestEnvironment, - environment_type=EnvironmentType.external_db, - environment_name=cls.ext_environment_name, - external_exasol_db_host=cls.environment.database_host, - external_exasol_db_port=cls.environment.ports.database, - external_exasol_bucketfs_port=cls.environment.ports.bucketfs, - external_exasol_ssh_port=cls.environment.ports.ssh, - external_exasol_db_user=cls.environment.db_username, - external_exasol_db_password=cls.environment.db_password, - external_exasol_bucketfs_write_password=cls.environment.bucketfs_password, - external_exasol_xmlrpc_host=None, - external_exasol_xmlrpc_port=443, - external_exasol_xmlrpc_user="admin", - external_exasol_xmlrpc_password=None, - external_exasol_xmlrpc_cluster_name="cluster1", - no_test_container_cleanup_after_success=True, - no_test_container_cleanup_after_failure=False, - reuse_test_container=True, - test_container_content=get_test_container_content(), - additional_db_parameter=tuple()) + task_creator = lambda: generate_root_task( + task_class=SpawnTestEnvironment, + environment_type=EnvironmentType.external_db, + environment_name=cls.ext_environment_name, + external_exasol_db_host=cls.environment.database_host, + external_exasol_db_port=cls.environment.ports.database, + external_exasol_bucketfs_port=cls.environment.ports.bucketfs, + external_exasol_ssh_port=cls.environment.ports.ssh, + external_exasol_db_user=cls.environment.db_username, + external_exasol_db_password=cls.environment.db_password, + external_exasol_bucketfs_write_password=cls.environment.bucketfs_password, + external_exasol_xmlrpc_host=None, + external_exasol_xmlrpc_port=443, + external_exasol_xmlrpc_user="admin", + external_exasol_xmlrpc_password=None, + external_exasol_xmlrpc_cluster_name="cluster1", + no_test_container_cleanup_after_success=True, + no_test_container_cleanup_after_failure=False, + reuse_test_container=True, + test_container_content=get_test_container_content(), + additional_db_parameter=tuple(), + ) cls.ext_environment_info: EnvironmentInfo = run_task(task_creator, 1, None) @classmethod def tearDownClass(cls): utils.close_environments(cls.environment, cls.test_environment) with ContextDockerClient() as docker_client: - containers = [c.name for c in docker_client.containers.list() if cls.ext_environment_name in c.name] + containers = [ + c.name + for c in docker_client.containers.list() + if cls.ext_environment_name in c.name + ] remove_docker_container(containers) def test_external_db(self): with ContextDockerClient() as docker_client: - containers = [c.name for c in docker_client.containers.list() if self.docker_environment_name in c.name] - self.assertEqual(len(containers), 1, - f"Not exactly 1 containers in {containers}.") + containers = [ + c.name + for c in docker_client.containers.list() + if self.docker_environment_name in c.name + ] + self.assertEqual( + len(containers), 1, f"Not exactly 1 containers in {containers}." + ) db_container = [c for c in containers if "db_container" in c] - self.assertEqual(len(db_container), 1, - f"Found no db container in {containers}.") - containers = [c.name for c in docker_client.containers.list() if self.ext_environment_name in c.name] - self.assertEqual(len(containers), 1, - f"Not exactly 1 containers in {containers}.") + self.assertEqual( + len(db_container), 1, f"Found no db container in {containers}." + ) + containers = [ + c.name + for c in docker_client.containers.list() + if self.ext_environment_name in c.name + ] + self.assertEqual( + len(containers), 1, f"Not exactly 1 containers in {containers}." + ) test_container = [c for c in containers if "test_container" in c] - self.assertEqual(len(test_container), 1, - f"Found no test container in {containers}.") + self.assertEqual( + len(test_container), 1, f"Found no test container in {containers}." + ) def test_docker_available_in_test_container(self): environment_info = self.ext_environment_info with ContextDockerClient() as docker_client: - test_container = docker_client.containers.get(environment_info.test_container_info.container_name) + test_container = docker_client.containers.get( + environment_info.test_container_info.container_name + ) exit_result = test_container.exec_run("docker ps") exit_code = exit_result[0] output = exit_result[1] - self.assertEqual(exit_code, 0, - f"Error while executing 'docker ps' in test container got output\n {output}.") + self.assertEqual( + exit_code, + 0, + f"Error while executing 'docker ps' in test container got output\n {output}.", + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_api_logging.py b/exasol_integration_test_docker_environment/test/test_api_logging.py index 47064064a..ed4d50dfb 100644 --- a/exasol_integration_test_docker_environment/test/test_api_logging.py +++ b/exasol_integration_test_docker_environment/test/test_api_logging.py @@ -7,7 +7,11 @@ from contextlib import redirect_stderr from pathlib import Path from tempfile import TemporaryDirectory -from typing import Any, Dict, Optional +from typing import ( + Any, + Dict, + Optional, +) LOGGER_STR = "logger_str" @@ -39,10 +43,14 @@ TEST_FORMAT = "TEST_FORMAT" -from exasol_integration_test_docker_environment.lib.api.common import generate_root_task, run_task, \ - set_build_config -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import \ - DependencyLoggerBaseTask +from exasol_integration_test_docker_environment.lib.api.common import ( + generate_root_task, + run_task, + set_build_config, +) +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) @contextlib.contextmanager @@ -51,8 +59,7 @@ def ignore_resource_warning(): Ignore ResourceWarning to keep the captured output clean for the asserts """ with warnings.catch_warnings(): - warnings.filterwarnings(action="ignore", - category=ResourceWarning) + warnings.filterwarnings(action="ignore", category=ResourceWarning) yield @@ -92,35 +99,51 @@ def tearDown(self): unittest.util._MAX_LENGTH = self.old_max_length @ignore_resource_warning() - def dummy_api_command(self, log_level: Optional[str], use_job_specific_log_file: bool): - set_build_config(False, - tuple(), - False, - False, - self._build_output_temp_dir.name, - "/tmp", - "", - "test") + def dummy_api_command( + self, log_level: Optional[str], use_job_specific_log_file: bool + ): + set_build_config( + False, + tuple(), + False, + False, + self._build_output_temp_dir.name, + "/tmp", + "", + "test", + ) task_creator = lambda: generate_root_task(task_class=DummyTask) - result = run_task(task_creator=task_creator, - workers=2, - task_dependencies_dot_file=None, - log_level=log_level, - use_job_specific_log_file=use_job_specific_log_file) + result = run_task( + task_creator=task_creator, + workers=2, + task_dependencies_dot_file=None, + log_level=log_level, + use_job_specific_log_file=use_job_specific_log_file, + ) return result def configure_logging(self, log_level: int): logging.basicConfig( - format=f'{TEST_FORMAT} %(levelname)s %(message)s', + format=f"{TEST_FORMAT} %(levelname)s %(message)s", level=log_level, - force=True) + force=True, + ) def assert_loggers_are_equal(self, logger_infos_after, logger_infos_before): - self.assertEqual(logger_infos_before[ROOT_LOGGER], logger_infos_after[ROOT_LOGGER]) - self.assertEqual(logger_infos_before[API_CLIENT_LOGGING_TEST_LOGGER], - logger_infos_after[API_CLIENT_LOGGING_TEST_LOGGER]) - self.assertEqual(logger_infos_before[LUIGI_LOGGER], logger_infos_after[LUIGI_LOGGER]) - self.assertEqual(logger_infos_before[LUIGI_INTERFACE_LOGGER], logger_infos_after[LUIGI_INTERFACE_LOGGER]) + self.assertEqual( + logger_infos_before[ROOT_LOGGER], logger_infos_after[ROOT_LOGGER] + ) + self.assertEqual( + logger_infos_before[API_CLIENT_LOGGING_TEST_LOGGER], + logger_infos_after[API_CLIENT_LOGGING_TEST_LOGGER], + ) + self.assertEqual( + logger_infos_before[LUIGI_LOGGER], logger_infos_after[LUIGI_LOGGER] + ) + self.assertEqual( + logger_infos_before[LUIGI_INTERFACE_LOGGER], + logger_infos_after[LUIGI_INTERFACE_LOGGER], + ) def create_test_regex(self, log_level: int): level_name = logging.getLevelName(log_level) @@ -130,7 +153,9 @@ def test_luigi_log_level_info_and_basic_logging_error(self): with catch_stderr() as catched_stderr: self.configure_logging(log_level=logging.ERROR) logger_infos_before = self.create_logger_infos() - result = self.dummy_api_command(log_level="INFO", use_job_specific_log_file=False) + result = self.dummy_api_command( + log_level="INFO", use_job_specific_log_file=False + ) logger_infos_after = self.create_logger_infos() self.assert_loggers_are_equal(logger_infos_after, logger_infos_before) catched_stderr.seek(0) @@ -139,14 +164,18 @@ def test_luigi_log_level_info_and_basic_logging_error(self): self.assertRegex(stderr_output, self.create_test_regex(logging.ERROR)) self.assertRegex(stderr_output, self.create_test_regex(logging.INFO)) self.assertRegex(stderr_output, ".*===== Luigi Execution Summary =====.*") - main_log_glob = list(Path(self._build_output_temp_dir.name).glob("**/main.log")) + main_log_glob = list( + Path(self._build_output_temp_dir.name).glob("**/main.log") + ) self.assertEqual(main_log_glob, []) def test_luigi_log_level_error_and_basic_logging_info(self): with catch_stderr() as catched_stderr: self.configure_logging(log_level=logging.INFO) logger_infos_before = self.create_logger_infos() - result = self.dummy_api_command(log_level="ERROR", use_job_specific_log_file=False) + result = self.dummy_api_command( + log_level="ERROR", use_job_specific_log_file=False + ) logger_infos_after = self.create_logger_infos() self.assert_loggers_are_equal(logger_infos_after, logger_infos_before) catched_stderr.seek(0) @@ -154,15 +183,21 @@ def test_luigi_log_level_error_and_basic_logging_info(self): self.assertNotEqual(catched_stderr, "") self.assertRegex(stderr_output, self.create_test_regex(logging.ERROR)) self.assertNotRegex(stderr_output, self.create_test_regex(logging.INFO)) - main_log_glob = list(Path(self._build_output_temp_dir.name).glob("**/main.log")) + main_log_glob = list( + Path(self._build_output_temp_dir.name).glob("**/main.log") + ) self.assertEqual(main_log_glob, []) def test_luigi_log_level_error_multiple_calls_and_basic_logging_info(self): with catch_stderr() as catched_stderr: self.configure_logging(log_level=logging.INFO) logger_infos_before = self.create_logger_infos() - result = self.dummy_api_command(log_level="ERROR", use_job_specific_log_file=False) - result = self.dummy_api_command(log_level="ERROR", use_job_specific_log_file=False) + result = self.dummy_api_command( + log_level="ERROR", use_job_specific_log_file=False + ) + result = self.dummy_api_command( + log_level="ERROR", use_job_specific_log_file=False + ) logger_infos_after = self.create_logger_infos() self.assert_loggers_are_equal(logger_infos_after, logger_infos_before) catched_stderr.seek(0) @@ -171,28 +206,36 @@ def test_luigi_log_level_error_multiple_calls_and_basic_logging_info(self): self.assertRegex(stderr_output, self.create_test_regex(logging.ERROR)) self.assertNotRegex(stderr_output, self.create_test_regex(logging.INFO)) self.assertEqual(2, stderr_output.count("DUMMY LOGGER ERROR")) - main_log_glob = list(Path(self._build_output_temp_dir.name).glob("**/main.log")) + main_log_glob = list( + Path(self._build_output_temp_dir.name).glob("**/main.log") + ) self.assertEqual(main_log_glob, []) def test_luigi_use_job_specific_log_file_and_basic_logging_error(self): with catch_stderr() as catched_stderr: self.configure_logging(log_level=logging.ERROR) logger_infos_before = self.create_logger_infos() - result = self.dummy_api_command(log_level=None, use_job_specific_log_file=True) + result = self.dummy_api_command( + log_level=None, use_job_specific_log_file=True + ) logger_infos_after = self.create_logger_infos() self.assert_loggers_are_equal(logger_infos_after, logger_infos_before) catched_stderr.seek(0) stderr_output = catched_stderr.read() self.assert_loggers_are_equal(logger_infos_after, logger_infos_before) self.assertEqual(stderr_output, "") - main_log_glob = list(Path(self._build_output_temp_dir.name).glob("**/main.log")) + main_log_glob = list( + Path(self._build_output_temp_dir.name).glob("**/main.log") + ) self.assertNotEqual(main_log_glob, []) def test_luigi_no_log_config_and_basic_logging_info(self): with catch_stderr() as catched_stderr: self.configure_logging(log_level=logging.INFO) logger_infos_before = self.create_logger_infos() - result = self.dummy_api_command(log_level=None, use_job_specific_log_file=False) + result = self.dummy_api_command( + log_level=None, use_job_specific_log_file=False + ) logger_infos_after = self.create_logger_infos() self.assert_loggers_are_equal(logger_infos_after, logger_infos_before) catched_stderr.seek(0) @@ -200,14 +243,18 @@ def test_luigi_no_log_config_and_basic_logging_info(self): self.assertRegex(stderr_output, self.create_test_regex(logging.ERROR)) self.assertRegex(stderr_output, self.create_test_regex(logging.INFO)) self.assertRegex(stderr_output, ".*===== Luigi Execution Summary =====.*") - main_log_glob = list(Path(self._build_output_temp_dir.name).glob("**/main.log")) + main_log_glob = list( + Path(self._build_output_temp_dir.name).glob("**/main.log") + ) self.assertEqual(main_log_glob, []) def test_luigi_no_log_config_and_basic_logging_error(self): with catch_stderr() as catched_stderr: self.configure_logging(log_level=logging.ERROR) logger_infos_before = self.create_logger_infos() - result = self.dummy_api_command(log_level=None, use_job_specific_log_file=False) + result = self.dummy_api_command( + log_level=None, use_job_specific_log_file=False + ) logger_infos_after = self.create_logger_infos() self.assert_loggers_are_equal(logger_infos_after, logger_infos_before) catched_stderr.seek(0) @@ -215,15 +262,18 @@ def test_luigi_no_log_config_and_basic_logging_error(self): self.assertNotEqual(stderr_output, "") self.assertRegex(stderr_output, self.create_test_regex(logging.ERROR)) self.assertNotRegex(stderr_output, self.create_test_regex(logging.INFO)) - main_log_glob = list(Path(self._build_output_temp_dir.name).glob("**/main.log")) + main_log_glob = list( + Path(self._build_output_temp_dir.name).glob("**/main.log") + ) self.assertEqual(main_log_glob, []) def reset_logging(self): logging.basicConfig( - format='%(asctime)s %(levelname)-8s %(message)s', + format="%(asctime)s %(levelname)-8s %(message)s", level=logging.ERROR, - datefmt='%Y-%m-%d %H:%M:%S', - force=True) + datefmt="%Y-%m-%d %H:%M:%S", + force=True, + ) loggerDict = logging.root.manager.loggerDict for key in list(loggerDict.keys()): loggerDict[key].disabled = True @@ -232,14 +282,18 @@ def reset_logging(self): def create_logger_infos(self) -> Dict[str, Dict[str, Any]]: logger_infos = { ROOT_LOGGER: self.get_logger_info(logging.root), - API_CLIENT_LOGGING_TEST_LOGGER: self.get_logger_info(logging.getLogger("APIClientLoggingTest")), + API_CLIENT_LOGGING_TEST_LOGGER: self.get_logger_info( + logging.getLogger("APIClientLoggingTest") + ), LUIGI_LOGGER: self.get_logger_info(logging.getLogger("luigi")), - LUIGI_INTERFACE_LOGGER: self.get_logger_info(logging.getLogger("luigi-interface")) + LUIGI_INTERFACE_LOGGER: self.get_logger_info( + logging.getLogger("luigi-interface") + ), } return logger_infos def get_logger_info(self, logger: logging.Logger) -> Dict[str, Any]: - logger_info : Dict[str, Any] = dict() + logger_info: Dict[str, Any] = dict() logger_info[LOGGER_STR] = str(logger) logger_info[LEVEL] = logger.level logger_info[LEVEL_NAME] = logging.getLevelName(logger.level) @@ -252,5 +306,5 @@ def get_logger_info(self, logger: logging.Logger) -> Dict[str, Any]: return logger_info -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_api_push_test_container.py b/exasol_integration_test_docker_environment/test/test_api_push_test_container.py index 6a04b093c..c67ad68e2 100644 --- a/exasol_integration_test_docker_environment/test/test_api_push_test_container.py +++ b/exasol_integration_test_docker_environment/test/test_api_push_test_container.py @@ -2,10 +2,16 @@ from sys import stderr from exasol_integration_test_docker_environment.lib import api -from exasol_integration_test_docker_environment.test.get_test_container_content import get_test_container_content +from exasol_integration_test_docker_environment.test.get_test_container_content import ( + get_test_container_content, +) from exasol_integration_test_docker_environment.testing import luigi_utils -from exasol_integration_test_docker_environment.testing.api_test_environment import ApiTestEnvironment -from exasol_integration_test_docker_environment.testing.docker_registry import LocalDockerRegistryContextManager +from exasol_integration_test_docker_environment.testing.api_test_environment import ( + ApiTestEnvironment, +) +from exasol_integration_test_docker_environment.testing.docker_registry import ( + LocalDockerRegistryContextManager, +) class APIPushTestContainerTest(unittest.TestCase): @@ -18,22 +24,29 @@ def tearDown(self): self.test_environment.close() def test_docker_push(self): - with LocalDockerRegistryContextManager(self.test_environment.name) as docker_registry: + with LocalDockerRegistryContextManager( + self.test_environment.name + ) as docker_registry: print("registry:", docker_registry.repositories, file=stderr) docker_repository_name = docker_registry.name try: - image_info = api.push_test_container(source_docker_repository_name=docker_repository_name, - target_docker_repository_name=docker_repository_name, - test_container_content=get_test_container_content()) + image_info = api.push_test_container( + source_docker_repository_name=docker_repository_name, + target_docker_repository_name=docker_repository_name, + test_container_content=get_test_container_content(), + ) print("repos:", docker_registry.repositories, file=stderr) images = docker_registry.images print("images", images, file=stderr) - self.assertEqual(len(images["tags"]), 1, - f"{images} doesn't have the expected 1 tags, it has {len(images['tags'])}") + self.assertEqual( + len(images["tags"]), + 1, + f"{images} doesn't have the expected 1 tags, it has {len(images['tags'])}", + ) self.assertIn(image_info.get_target_complete_tag(), images["tags"][0]) finally: luigi_utils.clean(docker_repository_name) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_api_test_environment.py b/exasol_integration_test_docker_environment/test/test_api_test_environment.py index 7043ec82b..20b091c7e 100644 --- a/exasol_integration_test_docker_environment/test/test_api_test_environment.py +++ b/exasol_integration_test_docker_environment/test/test_api_test_environment.py @@ -4,13 +4,20 @@ from sys import stderr from typing import Optional -from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo -from exasol_integration_test_docker_environment.lib.data.test_container_content_description import \ - TestContainerRuntimeMapping +from exasol_integration_test_docker_environment.lib.data.environment_info import ( + EnvironmentInfo, +) +from exasol_integration_test_docker_environment.lib.data.test_container_content_description import ( + TestContainerRuntimeMapping, +) from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.test.get_test_container_content import get_test_container_content +from exasol_integration_test_docker_environment.test.get_test_container_content import ( + get_test_container_content, +) from exasol_integration_test_docker_environment.testing import utils -from exasol_integration_test_docker_environment.testing.api_test_environment import ApiTestEnvironment +from exasol_integration_test_docker_environment.testing.api_test_environment import ( + ApiTestEnvironment, +) class APISpawnTestEnvironmentTest(unittest.TestCase): @@ -20,11 +27,12 @@ def setUpClass(cls): print(f"SetUp {cls.__name__}", file=stderr) cls.test_environment = ApiTestEnvironment(cls) cls.docker_environment_name = cls.__name__ - cls.environment = \ + cls.environment = ( cls.test_environment.spawn_docker_test_environment_with_test_container( name=cls.docker_environment_name, - test_container_content=get_test_container_content() + test_container_content=get_test_container_content(), ) + ) @classmethod def tearDownClass(cls): @@ -32,52 +40,76 @@ def tearDownClass(cls): def test_all_containers_started(self): with ContextDockerClient() as docker_client: - containers = [c.name for c in docker_client.containers.list() if self.docker_environment_name in c.name] - self.assertEqual(len(containers), 2, - f"Not exactly 2 containers in {containers}.") + containers = [ + c.name + for c in docker_client.containers.list() + if self.docker_environment_name in c.name + ] + self.assertEqual( + len(containers), 2, f"Not exactly 2 containers in {containers}." + ) db_container = [c for c in containers if "db_container" in c] - self.assertEqual(len(db_container), 1, - f"Found no db container in {containers}.") + self.assertEqual( + len(db_container), 1, f"Found no db container in {containers}." + ) test_container = [c for c in containers if "test_container" in c] - self.assertEqual(len(test_container), 1, - f"Found no test container in {containers}.") + self.assertEqual( + len(test_container), 1, f"Found no test container in {containers}." + ) def test_docker_available_in_test_container(self): environment_info = self.environment.environment_info with ContextDockerClient() as docker_client: - test_container = docker_client.containers.get(environment_info.test_container_info.container_name) + test_container = docker_client.containers.get( + environment_info.test_container_info.container_name + ) exit_result = test_container.exec_run("docker ps") exit_code = exit_result[0] output = exit_result[1] - self.assertEqual(exit_code, 0, - f"Error while executing 'docker ps' in test container got output\n {output}.") + self.assertEqual( + exit_code, + 0, + f"Error while executing 'docker ps' in test container got output\n {output}.", + ) def test_db_container_available(self): environment_info = self.environment.environment_info with ContextDockerClient() as docker_client: - db_container = docker_client.containers.get(environment_info.database_info.container_info.container_name) + db_container = docker_client.containers.get( + environment_info.database_info.container_info.container_name + ) exit_result = db_container.exec_run("ls /exa") exit_code = exit_result[0] output = exit_result[1] - self.assertEqual(exit_code, 0, - f"Error while executing 'ls /exa' in db container got output\n {output}.") + self.assertEqual( + exit_code, + 0, + f"Error while executing 'ls /exa' in db container got output\n {output}.", + ) def test_db_available(self): environment_info = self.environment.environment_info with ContextDockerClient() as docker_client: - test_container = docker_client.containers.get(environment_info.test_container_info.container_name) + test_container = docker_client.containers.get( + environment_info.test_container_info.container_name + ) exit_result = test_container.exec_run(self.create_db_connection_command()) exit_code = exit_result[0] output = exit_result[1] - self.assertEqual(exit_code, 0, - f"Error while executing 'exaplus' in test container got output\n {output}.") + self.assertEqual( + exit_code, + 0, + f"Error while executing 'exaplus' in test container got output\n {output}.", + ) def create_db_connection_command(self): spawned_docker_test_environments = self.environment username = spawned_docker_test_environments.db_username password = spawned_docker_test_environments.db_password db_host = spawned_docker_test_environments.environment_info.database_info.host - db_port = spawned_docker_test_environments.environment_info.database_info.ports.database + db_port = ( + spawned_docker_test_environments.environment_info.database_info.ports.database + ) connection_options = f"-c '{db_host}:{db_port}' -u '{username}' -p '{password}'" cmd = f"""$EXAPLUS {connection_options} -sql 'select 1;' -jdbcparam 'validateservercertificate=0'""" bash_cmd = f"""bash -c "{cmd}" """ @@ -86,7 +118,9 @@ def create_db_connection_command(self): def test_build_mapping(self): environment_info = self.environment.environment_info with ContextDockerClient() as docker_client: - test_container = docker_client.containers.get(environment_info.test_container_info.container_name) + test_container = docker_client.containers.get( + environment_info.test_container_info.container_name + ) exit_code, output = test_container.exec_run("cat /test.text") self.assertEqual(exit_code, 0) self.assertEqual(output.decode("UTF-8"), "Empty File") @@ -107,34 +141,47 @@ def tearDownClass(cls): def _assert_deployment_available(self, environment_info: EnvironmentInfo) -> None: with ContextDockerClient() as docker_client: assert environment_info.test_container_info - test_container = docker_client.containers.get(environment_info.test_container_info.container_name) + test_container = docker_client.containers.get( + environment_info.test_container_info.container_name + ) exit_code, output = test_container.exec_run("cat /test/test.txt") self.assertEqual(exit_code, 0) self.assertEqual(output.decode("utf-8"), "test") - def _assert_deployment_not_shared(self, environment_info: EnvironmentInfo, temp_path: Path): + def _assert_deployment_not_shared( + self, environment_info: EnvironmentInfo, temp_path: Path + ): with ContextDockerClient() as docker_client: assert environment_info.test_container_info - test_container = docker_client.containers.get(environment_info.test_container_info.container_name) - exit_code, output = test_container.exec_run("touch /test_target/test_new.txt") + test_container = docker_client.containers.get( + environment_info.test_container_info.container_name + ) + exit_code, output = test_container.exec_run( + "touch /test_target/test_new.txt" + ) self.assertEqual(exit_code, 0) local_path = temp_path / "test_new.txt" self.assertFalse(local_path.exists()) - def _get_test_mapping(self, temp_path: Path, deployment_target: Optional[str] = None): + def _get_test_mapping( + self, temp_path: Path, deployment_target: Optional[str] = None + ): with open(temp_path / "test.txt", "w") as f: f.write("test") - return TestContainerRuntimeMapping(source=temp_path, target="/test", deployment_target=deployment_target) + return TestContainerRuntimeMapping( + source=temp_path, target="/test", deployment_target=deployment_target + ) def test_runtime_mapping_without_deployment(self): with tempfile.TemporaryDirectory() as temp_dir: mapping = self._get_test_mapping(Path(temp_dir)) try: - environment = \ - self.test_environment.spawn_docker_test_environment_with_test_container( - name=self.docker_environment_name, - test_container_content=get_test_container_content(runtime_mapping=(mapping,)) - ) + environment = self.test_environment.spawn_docker_test_environment_with_test_container( + name=self.docker_environment_name, + test_container_content=get_test_container_content( + runtime_mapping=(mapping,) + ), + ) environment_info = environment.environment_info self._assert_deployment_available(environment_info) finally: @@ -143,13 +190,16 @@ def test_runtime_mapping_without_deployment(self): def test_runtime_mapping_deployment(self): with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) - mapping = self._get_test_mapping(temp_path=temp_path, deployment_target="/test_target") + mapping = self._get_test_mapping( + temp_path=temp_path, deployment_target="/test_target" + ) try: - environment = \ - self.test_environment.spawn_docker_test_environment_with_test_container( - name=self.docker_environment_name, - test_container_content=get_test_container_content(runtime_mapping=(mapping,)) - ) + environment = self.test_environment.spawn_docker_test_environment_with_test_container( + name=self.docker_environment_name, + test_container_content=get_test_container_content( + runtime_mapping=(mapping,) + ), + ) environment_info = environment.environment_info self._assert_deployment_available(environment_info) self._assert_deployment_not_shared(environment_info, temp_path) @@ -158,5 +208,5 @@ def test_runtime_mapping_deployment(self): utils.close_environments(environment) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_api_test_environment_certificate.py b/exasol_integration_test_docker_environment/test/test_api_test_environment_certificate.py index 0b22dc1d7..18a4f19e1 100644 --- a/exasol_integration_test_docker_environment/test/test_api_test_environment_certificate.py +++ b/exasol_integration_test_docker_environment/test/test_api_test_environment_certificate.py @@ -1,13 +1,19 @@ import unittest - from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.lib.test_environment.db_version import \ - db_version_supports_custom_certificates -from exasol_integration_test_docker_environment.test.get_test_container_content import get_test_container_content +from exasol_integration_test_docker_environment.lib.test_environment.db_version import ( + db_version_supports_custom_certificates, +) +from exasol_integration_test_docker_environment.test.get_test_container_content import ( + get_test_container_content, +) from exasol_integration_test_docker_environment.testing import utils -from exasol_integration_test_docker_environment.testing.api_test_environment import ApiTestEnvironment -from exasol_integration_test_docker_environment.testing.utils import check_db_version_from_env +from exasol_integration_test_docker_environment.testing.api_test_environment import ( + ApiTestEnvironment, +) +from exasol_integration_test_docker_environment.testing.utils import ( + check_db_version_from_env, +) class CertificateTest(unittest.TestCase): @@ -27,26 +33,39 @@ def setUpClass(cls): cls.docker_environment_name = "" additional_parameter = {"create_certificates": True} - cls.on_host_docker_environment = \ + cls.on_host_docker_environment = ( cls.test_environment.spawn_docker_test_environment_with_test_container( cls.docker_environment_name, test_container_content=get_test_container_content(), - additional_parameter=additional_parameter) + additional_parameter=additional_parameter, + ) + ) @classmethod def tearDownClass(cls): - utils.close_environments(cls.spawned_docker_test_environments, cls.test_environment) + utils.close_environments( + cls.spawned_docker_test_environments, cls.test_environment + ) - @unittest.skipIf(not db_version_supports_custom_certificates(check_db_version_from_env()), "Database not supported") + @unittest.skipIf( + not db_version_supports_custom_certificates(check_db_version_from_env()), + "Database not supported", + ) def test_certificate(self): with ContextDockerClient() as docker_client: - test_container_name = self.on_host_docker_environment.environment_info.test_container_info.container_name + test_container_name = ( + self.on_host_docker_environment.environment_info.test_container_info.container_name + ) test_container = docker_client.containers.get(test_container_name) - database_container = \ + database_container = ( self.on_host_docker_environment.environment_info.database_info.container_info.container_name - database_network_name = \ + ) + database_network_name = ( self.on_host_docker_environment.environment_info.database_info.container_info.network_info.network_name - db_port = self.on_host_docker_environment.environment_info.database_info.ports.database + ) + db_port = ( + self.on_host_docker_environment.environment_info.database_info.ports.database + ) openssl_check_cmd = f"openssl s_client -connect {database_container}.{database_network_name}:{db_port}" print(f"OpenSSL cmd:{openssl_check_cmd}") @@ -54,10 +73,12 @@ def test_certificate(self): print(f"fOpenSSL out:{output}") self.assertEqual(exit_code, 0) log = output.decode("utf-8") - expected_subject = f"subject=C = XX, ST = N/A, L = N/A, O = Self-signed certificate, " \ - f"CN = {database_container}" + expected_subject = ( + f"subject=C = XX, ST = N/A, L = N/A, O = Self-signed certificate, " + f"CN = {database_container}" + ) self.assertIn(expected_subject, log, "Certificate check") -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_api_test_environment_docker_runtime.py b/exasol_integration_test_docker_environment/test/test_api_test_environment_docker_runtime.py index c13b7c4d6..7a97a7616 100644 --- a/exasol_integration_test_docker_environment/test/test_api_test_environment_docker_runtime.py +++ b/exasol_integration_test_docker_environment/test/test_api_test_environment_docker_runtime.py @@ -3,18 +3,25 @@ from exasol_integration_test_docker_environment.lib.api import api_errors from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.test.get_test_container_content import get_test_container_content +from exasol_integration_test_docker_environment.test.get_test_container_content import ( + get_test_container_content, +) from exasol_integration_test_docker_environment.testing import utils -from exasol_integration_test_docker_environment.testing.api_test_environment import ApiTestEnvironment +from exasol_integration_test_docker_environment.testing.api_test_environment import ( + ApiTestEnvironment, +) def assert_container_runtime(self, container_name, expected_runtime): with ContextDockerClient() as docker_client: container = docker_client.containers.get(container_name) container.reload() - actual_runtime = container.attrs['HostConfig']['Runtime'] - self.assertEqual(actual_runtime, expected_runtime, - f"{container_name} has the wrong runtime expected {expected_runtime} got {actual_runtime}.") + actual_runtime = container.attrs["HostConfig"]["Runtime"] + self.assertEqual( + actual_runtime, + expected_runtime, + f"{container_name} has the wrong runtime expected {expected_runtime} got {actual_runtime}.", + ) def get_default_docker_runtime(): @@ -22,7 +29,7 @@ def get_default_docker_runtime(): tmp_container = docker_client.containers.create("ubuntu:18.04", "echo") try: tmp_container.reload() - default_docker_runtime = tmp_container.attrs['HostConfig']['Runtime'] + default_docker_runtime = tmp_container.attrs["HostConfig"]["Runtime"] finally: tmp_container.remove(force=True) return default_docker_runtime @@ -35,11 +42,12 @@ def setUpClass(cls): print(f"SetUp {cls.__name__}") cls.test_environment = ApiTestEnvironment(cls) cls.docker_environment_name = "test_no_runtime_given" - cls.environment = \ + cls.environment = ( cls.test_environment.spawn_docker_test_environment_with_test_container( name=cls.docker_environment_name, - test_container_content=get_test_container_content() + test_container_content=get_test_container_content(), ) + ) cls.default_docker_runtime = get_default_docker_runtime() @classmethod @@ -53,8 +61,12 @@ def test_test_container_runtime(self): def test_database_container_runtime(self): environment_info = self.environment.environment_info - database_container_name = environment_info.database_info.container_info.container_name - assert_container_runtime(self, database_container_name, self.default_docker_runtime) + database_container_name = ( + environment_info.database_info.container_info.container_name + ) + assert_container_runtime( + self, database_container_name, self.default_docker_runtime + ) class DockerTestEnvironmentDockerRuntimeDefaultRuntimeGivenTest(unittest.TestCase): @@ -65,12 +77,15 @@ def setUpClass(cls): cls.test_environment = ApiTestEnvironment(cls) cls.default_docker_runtime = get_default_docker_runtime() cls.docker_environment_name = "test_default_runtime_given" - cls.environment = cls.test_environment.spawn_docker_test_environment_with_test_container( + cls.environment = ( + cls.test_environment.spawn_docker_test_environment_with_test_container( cls.docker_environment_name, test_container_content=get_test_container_content(), additional_parameter={ "docker_runtime": cls.default_docker_runtime, - }) + }, + ) + ) @classmethod def tearDownClass(cls): @@ -83,8 +98,12 @@ def test_test_container_runtime(self): def test_database_container_runtime(self): environment_info = self.environment.environment_info - database_container_name = environment_info.database_info.container_info.container_name - assert_container_runtime(self, database_container_name, self.default_docker_runtime) + database_container_name = ( + environment_info.database_info.container_info.container_name + ) + assert_container_runtime( + self, database_container_name, self.default_docker_runtime + ) class DockerTestEnvironmentDockerRuntimeInvalidRuntimeGivenTest(unittest.TestCase): @@ -103,12 +122,15 @@ def test_docker_environment_not_available(self): exception_thrown = False spawn_docker_test_environments_successful = False try: - environment = self.test_environment.spawn_docker_test_environment_with_test_container( + environment = ( + self.test_environment.spawn_docker_test_environment_with_test_container( self.docker_environment_name, test_container_content=get_test_container_content(), additional_parameter={ "docker_runtime": "AAAABBBBCCCC_INVALID_RUNTIME_111122223333" - }) + }, + ) + ) spawn_docker_test_environments_successful = True utils.close_environments(environment) except api_errors.TaskRuntimeError: @@ -117,5 +139,5 @@ def test_docker_environment_not_available(self): self.assertTrue(exception_thrown) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_base_task.py b/exasol_integration_test_docker_environment/test/test_base_task.py index 1f53706e7..5a6f9a0a4 100644 --- a/exasol_integration_test_docker_environment/test/test_base_task.py +++ b/exasol_integration_test_docker_environment/test/test_base_task.py @@ -3,26 +3,37 @@ import unittest import luigi -from luigi import Parameter, Config +from luigi import ( + Config, + Parameter, +) from exasol_integration_test_docker_environment.lib.api.common import generate_root_task -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask -from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import JsonPickleParameter +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import ( + JsonPickleParameter, +) TestBaseTask = DependencyLoggerBaseTask class TestTask1(TestBaseTask): def register_required(self): - self.task2 = self.register_dependency(self.create_child_task(task_class=TestTask2)) + self.task2 = self.register_dependency( + self.create_child_task(task_class=TestTask2) + ) def run_task(self): self.logger.info("RUN") self.logger.info(f"task2 {self.task2.get_output()}") - tasks_3 = yield from self.run_dependencies({ - "1": TestTask3(input_param="e"), - "2": TestTask3(input_param="d"), - }) + tasks_3 = yield from self.run_dependencies( + { + "1": TestTask3(input_param="e"), + "2": TestTask3(input_param="d"), + } + ) self.logger.info(f"""task3_1 {tasks_3["1"].get_output("output")}""") self.logger.info(f"""task3_2 {tasks_3["2"].get_output("output")}""") @@ -45,9 +56,7 @@ def run_task(self): class TestTask4(TestBaseTask): def run_task(self): - yield from self.run_dependencies([ - TestTask5(), - TestTask6()]) + yield from self.run_dependencies([TestTask5(), TestTask6()]) class TestTask5(TestBaseTask): @@ -69,7 +78,9 @@ class TestParameter(Config): class TestTask7(TestBaseTask, TestParameter): def register_required(self): - task8 = self.create_child_task_with_common_params(task_class=TestTask8, new_parameter="new") + task8 = self.create_child_task_with_common_params( + task_class=TestTask8, new_parameter="new" + ) self.task8_future = self.register_dependency(task8) def run_task(self): @@ -103,11 +114,16 @@ def __init__(self, *args, **kwargs): def register_required(self): inputs = [Data(i, f"{i}") for i in range(2)] - tasks = [self.create_child_task(task_class=TestTask10, parameter_1=input) for input in inputs] + tasks = [ + self.create_child_task(task_class=TestTask10, parameter_1=input) + for input in inputs + ] self.register_dependencies(tasks) def run_task(self): - yield from self.run_dependencies(self.create_child_task(task_class=TestTask10, parameter_1=Data(3, "3"))) + yield from self.run_dependencies( + self.create_child_task(task_class=TestTask10, parameter_1=Data(3, "3")) + ) class TestTask10(TestBaseTask): @@ -127,13 +143,17 @@ def register_required(self): pass def run_task(self): - yield from self.run_dependencies(self.create_child_task(task_class=TestTask10, parameter_1=Data1())) + yield from self.run_dependencies( + self.create_child_task(task_class=TestTask10, parameter_1=Data1()) + ) class TestTask11(TestBaseTask): def run_task(self): - tasks = [self.create_child_task(task_class=TestTask12, p=f"{i}") for i in range(10)] + tasks = [ + self.create_child_task(task_class=TestTask12, p=f"{i}") for i in range(10) + ] self.logger.info(tasks) yield from self.run_dependencies(tasks) @@ -144,6 +164,7 @@ class TestTask12(TestBaseTask): def run_task(self): self.logger.info("Start and wait") import time + time.sleep(5) self.logger.info("Finished wait and fail") raise Exception("%s" % self.task_id) @@ -152,7 +173,9 @@ def run_task(self): class TestTask13(TestBaseTask): def register_required(self): - tasks = [self.create_child_task(task_class=TestTask14, p=f"{i}") for i in range(10)] + tasks = [ + self.create_child_task(task_class=TestTask14, p=f"{i}") for i in range(10) + ] self.register_dependencies(tasks) def run_task(self): @@ -201,7 +224,7 @@ def test_json_pickle_parameter_success(self): finally: if task._get_tmp_path_for_job().exists(): shutil.rmtree(str(task._get_tmp_path_for_job())) - + def test_json_pickle_parameter_fail(self): task = generate_root_task(task_class=TestTask9_Fail) try: @@ -239,11 +262,11 @@ def test_collect_failures_one_task_fails_but_is_dependency_of_multiple(self): print(failure) print() print() - self.assertEquals(len(failures), 1) + self.assertEqual(len(failures), 1) finally: if task._get_tmp_path_for_job().exists(): shutil.rmtree(str(task._get_tmp_path_for_job())) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_base_task_cleanup.py b/exasol_integration_test_docker_environment/test/test_base_task_cleanup.py index bba3445c2..fb357f9aa 100644 --- a/exasol_integration_test_docker_environment/test/test_base_task_cleanup.py +++ b/exasol_integration_test_docker_environment/test/test_base_task_cleanup.py @@ -2,10 +2,15 @@ import unittest import luigi -from luigi import BoolParameter, IntParameter +from luigi import ( + BoolParameter, + IntParameter, +) from exasol_integration_test_docker_environment.lib.api.common import generate_root_task -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) TestBaseTask = DependencyLoggerBaseTask @@ -16,14 +21,18 @@ class TestTaskBase(TestBaseTask): def _build_child_task(self, use_dynamic_dependency): index_for_grandchild = 0 - child_a = self.create_child_task(task_class=TestTaskChildA, - index_for_grandchild=index_for_grandchild, - use_dynamic_dependency=use_dynamic_dependency) + child_a = self.create_child_task( + task_class=TestTaskChildA, + index_for_grandchild=index_for_grandchild, + use_dynamic_dependency=use_dynamic_dependency, + ) if self.different_grandchild: index_for_grandchild = 1 - child_b = self.create_child_task(task_class=TestTaskChildB, - index_for_grandchild=index_for_grandchild, - use_dynamic_dependency=use_dynamic_dependency) + child_b = self.create_child_task( + task_class=TestTaskChildB, + index_for_grandchild=index_for_grandchild, + use_dynamic_dependency=use_dynamic_dependency, + ) return child_a, child_b def register_required(self): @@ -43,14 +52,18 @@ class TestTaskChildA(TestBaseTask): def register_required(self): if not self.use_dynamic_dependency: - grandchild = self.create_child_task(task_class=TestTaskGrandchild, - index_for_grandchild=self.index_for_grandchild) + grandchild = self.create_child_task( + task_class=TestTaskGrandchild, + index_for_grandchild=self.index_for_grandchild, + ) self.register_dependency(grandchild) def run_task(self): if self.use_dynamic_dependency: - grandchild = self.create_child_task(task_class=TestTaskGrandchild, - index_for_grandchild=self.index_for_grandchild) + grandchild = self.create_child_task( + task_class=TestTaskGrandchild, + index_for_grandchild=self.index_for_grandchild, + ) yield from self.run_dependencies(grandchild) def cleanup_task(self, success: bool): @@ -63,14 +76,18 @@ class TestTaskChildB(TestBaseTask): def register_required(self): if not self.use_dynamic_dependency: - grandchild = self.create_child_task(task_class=TestTaskGrandchild, - index_for_grandchild=self.index_for_grandchild) + grandchild = self.create_child_task( + task_class=TestTaskGrandchild, + index_for_grandchild=self.index_for_grandchild, + ) self.register_dependency(grandchild) def run_task(self): if self.use_dynamic_dependency: - grandchild = self.create_child_task(task_class=TestTaskGrandchild, - index_for_grandchild=self.index_for_grandchild) + grandchild = self.create_child_task( + task_class=TestTaskGrandchild, + index_for_grandchild=self.index_for_grandchild, + ) yield from self.run_dependencies(grandchild) def cleanup_task(self, success: bool): @@ -96,30 +113,38 @@ class BaseTaskCleanupTest(unittest.TestCase): def _run_it(self, different_grandchild, use_dynamic_dependency, expected_result): global global_counter global_counter = 0 - task = generate_root_task(task_class=TestTaskBase, - different_grandchild=different_grandchild, - use_dynamic_dependency=use_dynamic_dependency) + task = generate_root_task( + task_class=TestTaskBase, + different_grandchild=different_grandchild, + use_dynamic_dependency=use_dynamic_dependency, + ) try: luigi.build([task], workers=3, local_scheduler=True, log_level="INFO") task.cleanup(success=True) finally: if task._get_tmp_path_for_job().exists(): shutil.rmtree(str(task._get_tmp_path_for_job())) - self.assertEqual(global_counter, expected_result, "number of Cleanups not matching") + self.assertEqual( + global_counter, expected_result, "number of Cleanups not matching" + ) def test_cleanup_of_grandchildren_called_only_once(self): """ Test that creating the same grandchild task by two different parent tasks, will invoke cleanup of grandchild task only once! Luigi takes care of invoking run only once, we take care to invoke cleanup() only once. """ - self._run_it(different_grandchild=False, use_dynamic_dependency=False, expected_result=1) + self._run_it( + different_grandchild=False, use_dynamic_dependency=False, expected_result=1 + ) def test_cleanup_of_grandchildren_called_twice(self): """ Test that creating grandchild task with different parameters by two different parent tasks, will invoke cleanup of grandchild twice. """ - self._run_it(different_grandchild=True, use_dynamic_dependency=False, expected_result=2) + self._run_it( + different_grandchild=True, use_dynamic_dependency=False, expected_result=2 + ) def test_cleanup_of_grandchildren_called_only_once_dynamic(self): """ @@ -127,7 +152,9 @@ def test_cleanup_of_grandchildren_called_only_once_dynamic(self): task only once! Luigi takes care of invoking run only once, we take care to invoke cleanup() only once. In this test all child tasks are created dynamically. """ - self._run_it(different_grandchild=False, use_dynamic_dependency=True, expected_result=1) + self._run_it( + different_grandchild=False, use_dynamic_dependency=True, expected_result=1 + ) def test_cleanup_of_grandchildren_called_twice_dynamic(self): """ @@ -135,8 +162,10 @@ def test_cleanup_of_grandchildren_called_twice_dynamic(self): will invoke cleanup of grandchild twice. In this test all child tasks are created dynamically. """ - self._run_it(different_grandchild=True, use_dynamic_dependency=True, expected_result=2) + self._run_it( + different_grandchild=True, use_dynamic_dependency=True, expected_result=2 + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_certificate_docker_build.py b/exasol_integration_test_docker_environment/test/test_certificate_docker_build.py index cadb16f91..ba0f4570a 100644 --- a/exasol_integration_test_docker_environment/test/test_certificate_docker_build.py +++ b/exasol_integration_test_docker_environment/test/test_certificate_docker_build.py @@ -3,21 +3,33 @@ import luigi -from exasol_integration_test_docker_environment.lib.api.common import generate_root_task, set_docker_repository_config -from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.lib.docker.images.clean.clean_images import CleanImagesStartingWith -from exasol_integration_test_docker_environment.lib.docker.images.utils import find_images_by_tag -from exasol_integration_test_docker_environment.lib.test_environment.create_certificates.analyze_certificate_container import \ - DockerCertificateContainerBuild -from exasol_integration_test_docker_environment.lib.utils.resource_directory import ResourceDirectory import exasol_integration_test_docker_environment.certificate_resources.container +from exasol_integration_test_docker_environment.lib.api.common import ( + generate_root_task, + set_docker_repository_config, +) +from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient +from exasol_integration_test_docker_environment.lib.docker.images.clean.clean_images import ( + CleanImagesStartingWith, +) +from exasol_integration_test_docker_environment.lib.docker.images.utils import ( + find_images_by_tag, +) +from exasol_integration_test_docker_environment.lib.test_environment.create_certificates.analyze_certificate_container import ( + DockerCertificateContainerBuild, +) +from exasol_integration_test_docker_environment.lib.utils.resource_directory import ( + ResourceDirectory, +) class DockerCertificateBuildTest(unittest.TestCase): def clean(self): - task = generate_root_task(task_class=CleanImagesStartingWith, - starts_with_pattern="exasol-certificate-docker-build") + task = generate_root_task( + task_class=CleanImagesStartingWith, + starts_with_pattern="exasol-certificate-docker-build", + ) success = luigi.build([task], workers=1, local_scheduler=True, log_level="INFO") task.cleanup(success) @@ -29,8 +41,12 @@ def tearDown(self): def assert_image_exists(self, prefix): with ContextDockerClient() as docker_client: - image_list = find_images_by_tag(docker_client, lambda x: x.startswith(prefix)) - self.assertEquals(len(image_list), 1, f"Image with prefix {prefix} not found") + image_list = find_images_by_tag( + docker_client, lambda x: x.startswith(prefix) + ) + self.assertEqual( + len(image_list), 1, f"Image with prefix {prefix} not found" + ) def test_build(self): try: @@ -39,17 +55,25 @@ def test_build(self): docker_repository_name="exasol-certificate-docker-build", docker_username=None, tag_prefix="", - kind="target" + kind="target", ) - with ResourceDirectory(exasol_integration_test_docker_environment.certificate_resources.container) as d: - task = generate_root_task(task_class=DockerCertificateContainerBuild, - certificate_container_root_directory=d) - success = luigi.build([task], workers=1, local_scheduler=True, log_level="INFO") - self.assert_image_exists("exasol-certificate-docker-build:certificate_resources") + with ResourceDirectory( + exasol_integration_test_docker_environment.certificate_resources.container + ) as d: + task = generate_root_task( + task_class=DockerCertificateContainerBuild, + certificate_container_root_directory=d, + ) + success = luigi.build( + [task], workers=1, local_scheduler=True, log_level="INFO" + ) + self.assert_image_exists( + "exasol-certificate-docker-build:certificate_resources" + ) finally: task.cleanup(success) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_cli_test_environment_additional_params.py b/exasol_integration_test_docker_environment/test/test_cli_test_environment_additional_params.py index f3e888044..58e4ac186 100644 --- a/exasol_integration_test_docker_environment/test/test_cli_test_environment_additional_params.py +++ b/exasol_integration_test_docker_environment/test/test_cli_test_environment_additional_params.py @@ -2,7 +2,9 @@ from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient from exasol_integration_test_docker_environment.testing import utils -from exasol_integration_test_docker_environment.testing.exaslct_test_environment import ExaslctTestEnvironment +from exasol_integration_test_docker_environment.testing.exaslct_test_environment import ( + ExaslctTestEnvironment, +) ADDITIONAL_DB_PARAMS = ["-disableIndexIteratorScan=1", "-disableIndexIteratorScan=1"] @@ -12,37 +14,58 @@ class DockerTestEnvironmentAdditionParamsTest(unittest.TestCase): @classmethod def setUpClass(cls): print(f"SetUp {cls.__name__}") - cls.test_environment = \ - ExaslctTestEnvironment( - cls, - utils.INTEGRATION_TEST_DOCKER_ENVIRONMENT_DEFAULT_BIN, - clean_images_at_close=False) + cls.test_environment = ExaslctTestEnvironment( + cls, + utils.INTEGRATION_TEST_DOCKER_ENVIRONMENT_DEFAULT_BIN, + clean_images_at_close=False, + ) # TODO cls.test_environment.clean_images() cls.docker_environment_name = cls.__name__ - additional_db_parameters = [f"--additional-db-parameter {add_param}" for add_param in ADDITIONAL_DB_PARAMS] - cls.spawned_docker_test_environments = \ - cls.test_environment.spawn_docker_test_environments(name=cls.docker_environment_name, - additional_parameter=additional_db_parameters) + additional_db_parameters = [ + f"--additional-db-parameter {add_param}" + for add_param in ADDITIONAL_DB_PARAMS + ] + cls.spawned_docker_test_environments = ( + cls.test_environment.spawn_docker_test_environments( + name=cls.docker_environment_name, + additional_parameter=additional_db_parameters, + ) + ) @classmethod def tearDownClass(cls): - utils.close_environments(cls.spawned_docker_test_environments, cls.test_environment) + utils.close_environments( + cls.spawned_docker_test_environments, cls.test_environment + ) def test_additional_params_are_used(self): - environment_info = self.spawned_docker_test_environments.on_host_docker_environment - db_container_name = environment_info.environment_info.database_info.container_info.container_name + environment_info = ( + self.spawned_docker_test_environments.on_host_docker_environment + ) + db_container_name = ( + environment_info.environment_info.database_info.container_info.container_name + ) with ContextDockerClient() as docker_client: db_container = docker_client.containers.get(db_container_name) exit_code, output = db_container.exec_run("dwad_client print-params DB1") - self.assertEqual(exit_code, 0, - f"Error while executing 'dwad_client' " - f"in db container got output\n {output.decode('UTF-8')}.") - params_lines = [line for line in output.decode("UTF-8").splitlines() if line.startswith("Parameters:")] - self.assertEqual(len(params_lines), 1, "Unexpected format of output of dwad_client") + self.assertEqual( + exit_code, + 0, + f"Error while executing 'dwad_client' " + f"in db container got output\n {output.decode('UTF-8')}.", + ) + params_lines = [ + line + for line in output.decode("UTF-8").splitlines() + if line.startswith("Parameters:") + ] + self.assertEqual( + len(params_lines), 1, "Unexpected format of output of dwad_client" + ) params_line = params_lines[0] for add_db_param in ADDITIONAL_DB_PARAMS: self.assertIn(add_db_param, params_line) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_cli_test_environment_db_disk_size.py b/exasol_integration_test_docker_environment/test/test_cli_test_environment_db_disk_size.py index 4c0518838..df4886a90 100644 --- a/exasol_integration_test_docker_environment/test/test_cli_test_environment_db_disk_size.py +++ b/exasol_integration_test_docker_environment/test/test_cli_test_environment_db_disk_size.py @@ -2,7 +2,9 @@ from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient from exasol_integration_test_docker_environment.testing import utils -from exasol_integration_test_docker_environment.testing.exaslct_test_environment import ExaslctTestEnvironment +from exasol_integration_test_docker_environment.testing.exaslct_test_environment import ( + ExaslctTestEnvironment, +) class DockerTestEnvironmentDBDiskSizeTest(unittest.TestCase): @@ -10,11 +12,11 @@ class DockerTestEnvironmentDBDiskSizeTest(unittest.TestCase): @classmethod def setUpClass(cls): print(f"SetUp {cls.__name__}") - cls.test_environment = \ - ExaslctTestEnvironment( - cls, - utils.INTEGRATION_TEST_DOCKER_ENVIRONMENT_DEFAULT_BIN, - clean_images_at_close=False) + cls.test_environment = ExaslctTestEnvironment( + cls, + utils.INTEGRATION_TEST_DOCKER_ENVIRONMENT_DEFAULT_BIN, + clean_images_at_close=False, + ) @classmethod def tearDownClass(cls): @@ -22,13 +24,21 @@ def tearDownClass(cls): def assert_disk_size(self, size: str): with ContextDockerClient() as docker_client: - containers = [c.name for c in docker_client.containers.list() if self.docker_environment_name in c.name] + containers = [ + c.name + for c in docker_client.containers.list() + if self.docker_environment_name in c.name + ] db_container = [c for c in containers if "db_container" in c] - exit_result = docker_client.containers.get(db_container[0]).exec_run("cat /exa/etc/EXAConf") + exit_result = docker_client.containers.get(db_container[0]).exec_run( + "cat /exa/etc/EXAConf" + ) output = exit_result[1].decode("UTF-8") return_code = exit_result[0] - if output == '': - exit_result = docker_client.containers.get(db_container[0]).exec_run("cat /exa/etc/EXAConf") + if output == "": + exit_result = docker_client.containers.get(db_container[0]).exec_run( + "cat /exa/etc/EXAConf" + ) output = exit_result[1].decode("UTF-8") return_code = exit_result[0] self.assertEqual(return_code, 0) @@ -36,26 +46,28 @@ def assert_disk_size(self, size: str): def test_default_db_disk_size(self): self.docker_environment_name = "test_default_db_disk_size" - with self.test_environment.spawn_docker_test_environments(name=self.docker_environment_name): + with self.test_environment.spawn_docker_test_environments( + name=self.docker_environment_name + ): self.assert_disk_size("2 GiB") def test_smallest_valid_db_disk_size(self): self.docker_environment_name = "test_smallest_valid_db_disk_size" - with self.test_environment.spawn_docker_test_environments(name=self.docker_environment_name, - additional_parameter=[ - "--db-disk-size", "'100 MiB'" - ]): + with self.test_environment.spawn_docker_test_environments( + name=self.docker_environment_name, + additional_parameter=["--db-disk-size", "'100 MiB'"], + ): self.assert_disk_size("100 MiB") def test_invalid_db_mem_size(self): self.docker_environment_name = "test_invalid_db_disk_size" with self.assertRaises(Exception) as context: - with self.test_environment.spawn_docker_test_environments(name=self.docker_environment_name, - additional_parameter=[ - "--db-disk-size", "'90 MiB'" - ]): + with self.test_environment.spawn_docker_test_environments( + name=self.docker_environment_name, + additional_parameter=["--db-disk-size", "'90 MiB'"], + ): pass -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_cli_test_environment_db_mem_size.py b/exasol_integration_test_docker_environment/test/test_cli_test_environment_db_mem_size.py index adb889fbc..9b4f9aae4 100644 --- a/exasol_integration_test_docker_environment/test/test_cli_test_environment_db_mem_size.py +++ b/exasol_integration_test_docker_environment/test/test_cli_test_environment_db_mem_size.py @@ -2,7 +2,9 @@ from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient from exasol_integration_test_docker_environment.testing import utils -from exasol_integration_test_docker_environment.testing.exaslct_test_environment import ExaslctTestEnvironment +from exasol_integration_test_docker_environment.testing.exaslct_test_environment import ( + ExaslctTestEnvironment, +) class DockerTestEnvironmentDBMemSizeTest(unittest.TestCase): @@ -10,11 +12,11 @@ class DockerTestEnvironmentDBMemSizeTest(unittest.TestCase): @classmethod def setUpClass(cls): print(f"SetUp {cls.__name__}") - cls.test_environment = \ - ExaslctTestEnvironment( - cls, - utils.INTEGRATION_TEST_DOCKER_ENVIRONMENT_DEFAULT_BIN, - clean_images_at_close=False) + cls.test_environment = ExaslctTestEnvironment( + cls, + utils.INTEGRATION_TEST_DOCKER_ENVIRONMENT_DEFAULT_BIN, + clean_images_at_close=False, + ) @classmethod def tearDownClass(cls): @@ -22,41 +24,51 @@ def tearDownClass(cls): def assert_mem_size(self, size: str): with ContextDockerClient() as docker_client: - containers = [c.name for c in docker_client.containers.list() if self.docker_environment_name in c.name] + containers = [ + c.name + for c in docker_client.containers.list() + if self.docker_environment_name in c.name + ] db_container = [c for c in containers if "db_container" in c] - exit_result = docker_client.containers.get(db_container[0]).exec_run("cat /exa/etc/EXAConf") + exit_result = docker_client.containers.get(db_container[0]).exec_run( + "cat /exa/etc/EXAConf" + ) output = exit_result[1].decode("UTF-8") - if output == '': - exit_result = docker_client.containers.get(db_container[0]).exec_run("cat /exa/etc/EXAConf") + if output == "": + exit_result = docker_client.containers.get(db_container[0]).exec_run( + "cat /exa/etc/EXAConf" + ) output = exit_result[1].decode("UTF-8") return_code = exit_result[0] return_code = exit_result[0] - self.assertEquals(return_code, 0) + self.assertEqual(return_code, 0) self.assertIn("MemSize = %s" % size, output) def test_default_db_mem_size(self): self.docker_environment_name = "test_default_db_mem_size" - with self.test_environment.spawn_docker_test_environments(name=self.docker_environment_name): + with self.test_environment.spawn_docker_test_environments( + name=self.docker_environment_name + ): self.assert_mem_size("2 GiB") def test_smallest_valid_db_mem_size(self): self.docker_environment_name = "test_smallest_valid_db_mem_size" - with self.test_environment.spawn_docker_test_environments(name=self.docker_environment_name, - additional_parameter=[ - "--db-mem-size", "'1 GiB'" - ]): + with self.test_environment.spawn_docker_test_environments( + name=self.docker_environment_name, + additional_parameter=["--db-mem-size", "'1 GiB'"], + ): self.assert_mem_size("1 GiB") def test_invalid_db_mem_size(self): self.docker_environment_name = "test_invalid_db_mem_size" with self.assertRaises(Exception) as context: - with self.test_environment.spawn_docker_test_environments(name=self.docker_environment_name, - additional_parameter=[ - "--db-mem-size", "'999 MiB'" - ]): + with self.test_environment.spawn_docker_test_environments( + name=self.docker_environment_name, + additional_parameter=["--db-mem-size", "'999 MiB'"], + ): pass -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_cli_test_environment_nameservers.py b/exasol_integration_test_docker_environment/test/test_cli_test_environment_nameservers.py index 325f92ef4..d9674d00b 100644 --- a/exasol_integration_test_docker_environment/test/test_cli_test_environment_nameservers.py +++ b/exasol_integration_test_docker_environment/test/test_cli_test_environment_nameservers.py @@ -1,9 +1,10 @@ import unittest - from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient from exasol_integration_test_docker_environment.testing import utils -from exasol_integration_test_docker_environment.testing.exaslct_test_environment import ExaslctTestEnvironment +from exasol_integration_test_docker_environment.testing.exaslct_test_environment import ( + ExaslctTestEnvironment, +) class DockerTestEnvironmentDBMemSizeTest(unittest.TestCase): @@ -11,11 +12,11 @@ class DockerTestEnvironmentDBMemSizeTest(unittest.TestCase): @classmethod def setUpClass(cls): print(f"SetUp {cls.__name__}") - cls.test_environment = \ - ExaslctTestEnvironment( - cls, - utils.INTEGRATION_TEST_DOCKER_ENVIRONMENT_DEFAULT_BIN, - clean_images_at_close=False) + cls.test_environment = ExaslctTestEnvironment( + cls, + utils.INTEGRATION_TEST_DOCKER_ENVIRONMENT_DEFAULT_BIN, + clean_images_at_close=False, + ) @classmethod def tearDownClass(cls): @@ -23,40 +24,53 @@ def tearDownClass(cls): def assert_nameserver(self, nameservers: str): with ContextDockerClient() as docker_client: - containers = [c.name for c in docker_client.containers.list() if self.docker_environment_name in c.name] + containers = [ + c.name + for c in docker_client.containers.list() + if self.docker_environment_name in c.name + ] db_container = [c for c in containers if "db_container" in c] - exit_result = docker_client.containers.get(db_container[0]).exec_run("cat /exa/etc/EXAConf") + exit_result = docker_client.containers.get(db_container[0]).exec_run( + "cat /exa/etc/EXAConf" + ) output = exit_result[1].decode("UTF-8") - if output == '': - exit_result = docker_client.containers.get(db_container[0]).exec_run("cat /exa/etc/EXAConf") + if output == "": + exit_result = docker_client.containers.get(db_container[0]).exec_run( + "cat /exa/etc/EXAConf" + ) output = exit_result[1].decode("UTF-8") return_code = exit_result[0] return_code = exit_result[0] - self.assertEquals(return_code, 0) + self.assertEqual(return_code, 0) self.assertIn("NameServers = %s" % nameservers, output) def test_no_nameserver(self): self.docker_environment_name = "test_no_nameserver" - with self.test_environment.spawn_docker_test_environments(self.docker_environment_name): + with self.test_environment.spawn_docker_test_environments( + self.docker_environment_name + ): self.assert_nameserver("") def test_single_nameserver(self): self.docker_environment_name = "test_single_nameserver" - with self.test_environment.spawn_docker_test_environments(self.docker_environment_name, - [ - "--nameserver", "'8.8.8.8'" - ]): + with self.test_environment.spawn_docker_test_environments( + self.docker_environment_name, ["--nameserver", "'8.8.8.8'"] + ): self.assert_nameserver("8.8.8.8") def test_multiple_nameserver(self): self.docker_environment_name = "test_multiple_nameserver" - with self.test_environment.spawn_docker_test_environments(self.docker_environment_name, - [ - "--nameserver", "'8.8.8.8'", - "--nameserver", "'8.8.8.9'", - ]): + with self.test_environment.spawn_docker_test_environments( + self.docker_environment_name, + [ + "--nameserver", + "'8.8.8.8'", + "--nameserver", + "'8.8.8.9'", + ], + ): self.assert_nameserver("8.8.8.8,8.8.8.9") -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_click_api_consistency.py b/exasol_integration_test_docker_environment/test/test_click_api_consistency.py index 3fb2d185d..18e768988 100644 --- a/exasol_integration_test_docker_environment/test/test_click_api_consistency.py +++ b/exasol_integration_test_docker_environment/test/test_click_api_consistency.py @@ -3,9 +3,12 @@ from exasol_integration_test_docker_environment.cli import commands from exasol_integration_test_docker_environment.lib import api - -from exasol_integration_test_docker_environment.testing.api_consistency_utils import \ - param_names_of_click_call, get_click_and_api_functions, defaults_of_click_call, get_click_and_api_function_names +from exasol_integration_test_docker_environment.testing.api_consistency_utils import ( + defaults_of_click_call, + get_click_and_api_function_names, + get_click_and_api_functions, + param_names_of_click_call, +) from exasol_integration_test_docker_environment.testing.utils import multiassert @@ -28,12 +31,23 @@ def test_api_arguments(self): del api_spec.annotations["return"] loop = f"cli: {cli_call}, api: {api_call}" + def cli_spec_names(): self.assertEqual(api_spec.args, cli_spec.args, msg=f"{loop} - cli spec") + def type_annotations(): - self.assertEqual(api_spec.annotations, cli_spec.annotations, msg=f"{loop} - type annotations") + self.assertEqual( + api_spec.annotations, + cli_spec.annotations, + msg=f"{loop} - type annotations", + ) + def cli_param_names(): - self.assertEqual(api_spec.args, param_names_of_click_call(cli_call), msg=f"{loop} - cli param names") + self.assertEqual( + api_spec.args, + param_names_of_click_call(cli_call), + msg=f"{loop} - cli param names", + ) multiassert([cli_spec_names, type_annotations, cli_param_names], self) @@ -52,11 +66,16 @@ def test_api_default_values(self): self.assertEqual(len(cli_defaults), len(api_spec_defaults)) for api_default_value, cli_default in zip(api_spec_defaults, cli_defaults): cli_param_name, cli_default_value = cli_default - if api_default_value != cli_default_value and cli_param_name != "use_job_specific_log_file": - self.fail(f"Default value for parameter '{cli_param_name}' " - f"for method '{api_call.__name__}' does not match. " - f"API method has default value '{api_default_value}' " - f"while CLI method has default value '{cli_default_value}'") + if ( + api_default_value != cli_default_value + and cli_param_name != "use_job_specific_log_file" + ): + self.fail( + f"Default value for parameter '{cli_param_name}' " + f"for method '{api_call.__name__}' does not match. " + f"API method has default value '{api_default_value}' " + f"while CLI method has default value '{cli_default_value}'" + ) def test_same_functions(self): """ @@ -64,9 +83,11 @@ def test_same_functions(self): For that we use inspect to get all classes of type click.Command in module 'commands', and on the other hand get all functions in module 'api'. The list of names from both most be identical. """ - click_command_names, api_function_names = get_click_and_api_function_names(commands, api) + click_command_names, api_function_names = get_click_and_api_function_names( + commands, api + ) self.assertEqual(click_command_names, api_function_names) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_common_run_task.py b/exasol_integration_test_docker_environment/test/test_common_run_task.py index 04422dd23..2b8f4edeb 100644 --- a/exasol_integration_test_docker_environment/test/test_common_run_task.py +++ b/exasol_integration_test_docker_environment/test/test_common_run_task.py @@ -2,8 +2,13 @@ import luigi -from exasol_integration_test_docker_environment.lib.api.common import generate_root_task, run_task -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask +from exasol_integration_test_docker_environment.lib.api.common import ( + generate_root_task, + run_task, +) +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) class TestTaskWithReturn(DependencyLoggerBaseTask): @@ -20,11 +25,15 @@ def test_return_value(self) -> None: Integration test which verifies that the return value processing in run_task works as expected. """ - task_creator = lambda: generate_root_task(task_class=TestTaskWithReturn, x="Test") + task_creator = lambda: generate_root_task( + task_class=TestTaskWithReturn, x="Test" + ) - return_value = run_task(task_creator, workers=5, task_dependencies_dot_file=None) + return_value = run_task( + task_creator, workers=5, task_dependencies_dot_file=None + ) assert return_value == "Test-123" -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_db_version_supports_custom_certificates.py b/exasol_integration_test_docker_environment/test/test_db_version_supports_custom_certificates.py index cf241ca76..3a7dbc40d 100644 --- a/exasol_integration_test_docker_environment/test/test_db_version_supports_custom_certificates.py +++ b/exasol_integration_test_docker_environment/test/test_db_version_supports_custom_certificates.py @@ -1,7 +1,8 @@ import unittest -from exasol_integration_test_docker_environment.lib.test_environment.db_version import \ - db_version_supports_custom_certificates +from exasol_integration_test_docker_environment.lib.test_environment.db_version import ( + db_version_supports_custom_certificates, +) class TestDbVersionSupportCustomCertificates(unittest.TestCase): @@ -34,5 +35,5 @@ def test_throw_error(self): self.assertRaises(ValueError, db_version_supports_custom_certificates, "7abc") -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_docker_access_method.py b/exasol_integration_test_docker_environment/test/test_docker_access_method.py index 259c8e7c9..238c5bb68 100644 --- a/exasol_integration_test_docker_environment/test/test_docker_access_method.py +++ b/exasol_integration_test_docker_environment/test/test_docker_access_method.py @@ -1,13 +1,19 @@ -from typing import Dict, Any +from typing import ( + Any, + Dict, +) import luigi import pytest from exasol_integration_test_docker_environment.lib.api.common import generate_root_task -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task \ - import DependencyLoggerBaseTask -from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter \ - import DbOsAccess, DockerDBTestEnvironmentParameter +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter import ( + DbOsAccess, + DockerDBTestEnvironmentParameter, +) class Testee(DependencyLoggerBaseTask, DockerDBTestEnvironmentParameter): @@ -15,11 +21,11 @@ def run_task(self): pass @classmethod - def make(cls, method: str) -> 'Testee': - kwargs : Dict[str, Any] = {"task_class": Testee} + def make(cls, method: str) -> "Testee": + kwargs: Dict[str, Any] = {"task_class": Testee} if method: kwargs["db_os_access"] = method - task : Testee = generate_root_task(**kwargs) # type: ignore + task: Testee = generate_root_task(**kwargs) # type: ignore luigi.build([task], workers=1, local_scheduler=True, log_level="INFO") return task diff --git a/exasol_integration_test_docker_environment/test/test_docker_build_base.py b/exasol_integration_test_docker_environment/test/test_docker_build_base.py index 6935eda24..7315b1951 100644 --- a/exasol_integration_test_docker_environment/test/test_docker_build_base.py +++ b/exasol_integration_test_docker_environment/test/test_docker_build_base.py @@ -1,19 +1,29 @@ import shutil import unittest from pathlib import Path +from typing import ( + Dict, + List, + Set, +) import luigi from luigi import Parameter -from typing import Set, Dict, List - from exasol_integration_test_docker_environment.lib.api.common import generate_root_task from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_build_base import DockerBuildBase -from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_analyze_task import \ - DockerAnalyzeImageTask -from exasol_integration_test_docker_environment.lib.docker.images.clean.clean_images import CleanImagesStartingWith -from exasol_integration_test_docker_environment.lib.docker.images.utils import find_images_by_tag +from exasol_integration_test_docker_environment.lib.docker.images.clean.clean_images import ( + CleanImagesStartingWith, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_build_base import ( + DockerBuildBase, +) +from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_analyze_task import ( + DockerAnalyzeImageTask, +) +from exasol_integration_test_docker_environment.lib.docker.images.utils import ( + find_images_by_tag, +) class TestDockerBuildBaseTestAnalyzeImage(DockerAnalyzeImageTask): @@ -36,7 +46,9 @@ def get_mapping_of_build_files_and_directories(self): def get_dockerfile(self): script_dir = Path(__file__).resolve().parent - dockerfile_path = Path(script_dir, "resources/test_docker_build_base/test_analyze_image/Dockerfile") + dockerfile_path = Path( + script_dir, "resources/test_docker_build_base/test_analyze_image/Dockerfile" + ) return dockerfile_path def is_rebuild_requested(self) -> bool: @@ -44,16 +56,17 @@ def is_rebuild_requested(self) -> bool: class TestDockerBuildBase(DockerBuildBase): - goals : List[str] = luigi.ListParameter([]) # type: ignore + goals: List[str] = luigi.ListParameter([]) # type: ignore def get_goal_class_map(self) -> Dict[str, DockerAnalyzeImageTask]: goal_class_map = { - "test-analyze-image-1": - self.create_child_task(task_class=TestDockerBuildBaseTestAnalyzeImage, - task_name="test-analyze-image-1"), - "test-analyze-image-2": - self.create_child_task(TestDockerBuildBaseTestAnalyzeImage, - task_name="test-analyze-image-2") + "test-analyze-image-1": self.create_child_task( + task_class=TestDockerBuildBaseTestAnalyzeImage, + task_name="test-analyze-image-1", + ), + "test-analyze-image-2": self.create_child_task( + TestDockerBuildBaseTestAnalyzeImage, task_name="test-analyze-image-2" + ), } return goal_class_map @@ -74,8 +87,10 @@ def run_task(self): class DockerBuildBaseTest(unittest.TestCase): def clean(self): - task = generate_root_task(task_class=CleanImagesStartingWith, - starts_with_pattern="exasol-test-docker-build-base") + task = generate_root_task( + task_class=CleanImagesStartingWith, + starts_with_pattern="exasol-test-docker-build-base", + ) luigi.build([task], workers=1, local_scheduler=True, log_level="INFO") if task._get_tmp_path_for_job().exists(): shutil.rmtree(str(task._get_tmp_path_for_job())) @@ -88,32 +103,44 @@ def tearDown(self): def assert_image_exists(self, prefix): with ContextDockerClient() as docker_client: - image_list = find_images_by_tag(docker_client, lambda x: x.startswith(prefix)) - self.assertEquals(len(image_list), 1, f"Image with prefix {prefix} not found") + image_list = find_images_by_tag( + docker_client, lambda x: x.startswith(prefix) + ) + self.assertEqual( + len(image_list), 1, f"Image with prefix {prefix} not found" + ) def test_default_parameter(self): task = generate_root_task(task_class=TestDockerBuildBase) try: luigi.build([task], workers=1, local_scheduler=True, log_level="INFO") - self.assert_image_exists("exasol-test-docker-build-base:test-analyze-image-1") + self.assert_image_exists( + "exasol-test-docker-build-base:test-analyze-image-1" + ) finally: if task._get_tmp_path_for_job().exists(): shutil.rmtree(str(task._get_tmp_path_for_job())) def test_valid_non_default_goal(self): - task = generate_root_task(task_class=TestDockerBuildBase, goals=["test-analyze-image-2"]) + task = generate_root_task( + task_class=TestDockerBuildBase, goals=["test-analyze-image-2"] + ) try: luigi.build([task], workers=1, local_scheduler=True, log_level="INFO") - self.assert_image_exists("exasol-test-docker-build-base:test-analyze-image-2") + self.assert_image_exists( + "exasol-test-docker-build-base:test-analyze-image-2" + ) finally: if task._get_tmp_path_for_job().exists(): shutil.rmtree(str(task._get_tmp_path_for_job())) def test_non_valid_non_default_goal(self): with self.assertRaises(Exception) as contex: - task = generate_root_task(task_class=TestDockerBuildBase, goals=["test-analyze-image-3"]) + task = generate_root_task( + task_class=TestDockerBuildBase, goals=["test-analyze-image-3"] + ) self.assertIn("Unknown goal(s)", str(contex.exception)) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_docker_registry_image_checker.py b/exasol_integration_test_docker_environment/test/test_docker_registry_image_checker.py index 30cf8fbd3..15b6be48c 100644 --- a/exasol_integration_test_docker_environment/test/test_docker_registry_image_checker.py +++ b/exasol_integration_test_docker_environment/test/test_docker_registry_image_checker.py @@ -2,8 +2,9 @@ from docker.errors import DockerException -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.docker_registry_image_checker import \ - DockerRegistryImageChecker +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.docker_registry_image_checker import ( + DockerRegistryImageChecker, +) class MyTestCase(unittest.TestCase): @@ -21,5 +22,5 @@ def test_pull_fail_with_DockerException(self): self.assertRaises(DockerException, exists) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_doctor.py b/exasol_integration_test_docker_environment/test/test_doctor.py index 7ed1767a4..98a71fc16 100644 --- a/exasol_integration_test_docker_environment/test/test_doctor.py +++ b/exasol_integration_test_docker_environment/test/test_doctor.py @@ -1,7 +1,6 @@ import os import unittest from contextlib import contextmanager - from typing import Generator from unittest.mock import patch diff --git a/exasol_integration_test_docker_environment/test/test_find_free_port.py b/exasol_integration_test_docker_environment/test/test_find_free_port.py index b33ddab2d..997705300 100644 --- a/exasol_integration_test_docker_environment/test/test_find_free_port.py +++ b/exasol_integration_test_docker_environment/test/test_find_free_port.py @@ -1,6 +1,8 @@ import unittest -from exasol_integration_test_docker_environment.lib.test_environment.ports import find_free_ports +from exasol_integration_test_docker_environment.lib.test_environment.ports import ( + find_free_ports, +) class FindFreePortTest(unittest.TestCase): @@ -9,7 +11,7 @@ def run_it(self, num_ports: int): ports = find_free_ports(num_ports) self.assertNotIn(0, ports) self.assertEqual(len(ports), num_ports) - #Check that there are no duplicates! + # Check that there are no duplicates! ports_set = set(ports) self.assertEqual(len(ports), len(ports_set)) @@ -19,5 +21,5 @@ def test_find_ports(self): self.run_it(num_port) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_generate_graph_plot.py b/exasol_integration_test_docker_environment/test/test_generate_graph_plot.py index a1fdb43de..25730efe1 100644 --- a/exasol_integration_test_docker_environment/test/test_generate_graph_plot.py +++ b/exasol_integration_test_docker_environment/test/test_generate_graph_plot.py @@ -4,15 +4,23 @@ import luigi -from exasol_integration_test_docker_environment.lib.api.common import generate_root_task, run_task, set_build_config -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask +from exasol_integration_test_docker_environment.lib.api.common import ( + generate_root_task, + run_task, + set_build_config, +) +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) class TestTask(DependencyLoggerBaseTask): x = luigi.Parameter() def register_required(self): - self.register_dependency(self.create_child_task(task_class=TestChildTask, y=["1", "2", "3"])) + self.register_dependency( + self.create_child_task(task_class=TestChildTask, y=["1", "2", "3"]) + ) def run_task(self): pass @@ -20,6 +28,7 @@ def run_task(self): class TestChildTask(DependencyLoggerBaseTask): y = luigi.ListParameter() + def run_task(self): pass @@ -31,14 +40,18 @@ def test_generate_dependency_dot_file(self): task_id_generator = (x for x in range(NUMBER_TASK)) def create_task(): - return generate_root_task(task_class=TestTask, x=f"{next(task_id_generator)}") + return generate_root_task( + task_class=TestTask, x=f"{next(task_id_generator)}" + ) with tempfile.TemporaryDirectory() as d: for i in range(NUMBER_TASK): dot_file = Path(d) / f"dot_file_{i}.dot" - run_task(create_task, workers=5, task_dependencies_dot_file=str(dot_file)) + run_task( + create_task, workers=5, task_dependencies_dot_file=str(dot_file) + ) assert dot_file.exists() -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_hash_temp_dir.py b/exasol_integration_test_docker_environment/test/test_hash_temp_dir.py index 002559da8..e2cf7ec6a 100644 --- a/exasol_integration_test_docker_environment/test/test_hash_temp_dir.py +++ b/exasol_integration_test_docker_environment/test/test_hash_temp_dir.py @@ -3,10 +3,15 @@ import shutil import tempfile import unittest -from pathlib import Path, PurePath +from pathlib import ( + Path, + PurePath, +) -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.file_directory_list_hasher import \ - FileDirectoryListHasher, PathMapping +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.file_directory_list_hasher import ( + FileDirectoryListHasher, + PathMapping, +) TEST_FILE = "/tmp/SEFQWEFWQEHDUWEFDGZWGDZWEFDUWESGRFUDWEGFUDWAFGWAZESGFDWZA" @@ -28,13 +33,13 @@ def setUpClass(self): self.generate_test_dir(self.test_dir1) self.test_dir2 = self.temp_dir + "/test2" self.generate_test_dir(self.test_dir2) - + self.test_dir3 = self.temp_dir + "/test3" - os.makedirs(self.test_dir3+"/d") - with open(self.test_dir3+"/f", "wt") as f: + os.makedirs(self.test_dir3 + "/d") + with open(self.test_dir3 + "/f", "w") as f: f.write("test") - - with open(TEST_FILE, "wt") as f: + + with open(TEST_FILE, "w") as f: f.write("test") @classmethod @@ -48,13 +53,19 @@ def generate_test_dir(self, test_dir): for i2 in range(level2): for i3 in range(level3): for i4 in range(level4): - path = "/level0/level1_%s/level2_%s/level3_%s/level4_%s/" \ - % (i1, i2, i3, i4) + path = ( + "/level0/level1_{}/level2_{}/level3_{}/level4_{}/".format( + i1, + i2, + i3, + i4, + ) + ) os.makedirs(test_dir + path) os.makedirs(test_dir + path + "test") for i5 in range(level5): - file = "%s/level5_file_%s" % (path, i5) - with open(test_dir + file, mode="wt") as f: + file = f"{path}/level5_file_{i5}" + with open(test_dir + file, mode="w") as f: f.write(file) @classmethod @@ -63,52 +74,62 @@ def tearDownClass(self): shutil.rmtree(self.temp_dir) def test_single_character_directory_name(self): - hasher = FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=True, - hash_file_names=True) + hasher = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=True, hash_file_names=True + ) old_pwd = os.getcwd() os.chdir(self.test_dir3) hash = hasher.hash([simple_path_mapping(".")]) ascii_hash = base64.b32encode(hash).decode("ASCII") - self.assertEqual("LVE2ZFQRMP6QLY43MKMZRHIEHE7KNSUS5LFWVJKPOWMI6JUPZHEQ====", ascii_hash) + self.assertEqual( + "LVE2ZFQRMP6QLY43MKMZRHIEHE7KNSUS5LFWVJKPOWMI6JUPZHEQ====", ascii_hash + ) os.chdir(old_pwd) def test_file_content_only_fixed_hash(self): - hasher = FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=False, - hash_file_names=False) + hasher = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=False, hash_file_names=False + ) hash = hasher.hash([simple_path_mapping(TEST_FILE)]) ascii_hash = base64.b32encode(hash).decode("ASCII") - self.assertEqual("SVGVUSP5ODM3RPG3GXJFEJTYFGKX67XX7JWHJ6EEDG64L2BCBH2A====", ascii_hash) + self.assertEqual( + "SVGVUSP5ODM3RPG3GXJFEJTYFGKX67XX7JWHJ6EEDG64L2BCBH2A====", ascii_hash + ) def test_file_with_path(self): - hasher = FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=True, - hash_file_names=True) + hasher = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=True, hash_file_names=True + ) hash = hasher.hash([simple_path_mapping(TEST_FILE)]) ascii_hash = base64.b32encode(hash).decode("ASCII") - self.assertEqual("7D34CBUU2SNSWF3UFM6A7BYFJVV5ZFEY5F6THIMGJY725WC45KEA====", ascii_hash) + self.assertEqual( + "7D34CBUU2SNSWF3UFM6A7BYFJVV5ZFEY5F6THIMGJY725WC45KEA====", ascii_hash + ) def test_directory_with_relative_paths_fixed_hash(self): - hasher = FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=True, - hash_file_names=True) + hasher = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=True, hash_file_names=True + ) hash = hasher.hash([simple_path_mapping(self.test_dir1)]) ascii_hash = base64.b32encode(hash).decode("ASCII") - self.assertEqual("EZ3ER6KZHAAYG4JNFGFLHUI7TVHZVIRVOV4QWJT4ERQ4XGI2GLUA====", ascii_hash) + self.assertEqual( + "EZ3ER6KZHAAYG4JNFGFLHUI7TVHZVIRVOV4QWJT4ERQ4XGI2GLUA====", ascii_hash + ) def test_directory_content_only_fixed_hash(self): - hasher = FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=False, - hash_file_names=False) + hasher = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=False, hash_file_names=False + ) hash = hasher.hash([simple_path_mapping(self.test_dir1)]) ascii_hash = base64.b32encode(hash).decode("ASCII") - self.assertEqual("TM2V22T326TCTLQ537BZAOR3I5NVHXE6IDJ4TXPCJPTUGDTI5WYQ====", ascii_hash) + self.assertEqual( + "TM2V22T326TCTLQ537BZAOR3I5NVHXE6IDJ4TXPCJPTUGDTI5WYQ====", ascii_hash + ) def test_directory_to_same_destination_equal(self): - hasher = FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=True, - hash_file_names=True) + hasher = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=True, hash_file_names=True + ) hash1 = hasher.hash([PathMapping(PurePath("test"), Path(self.test_dir1))]) hash2 = hasher.hash([PathMapping(PurePath("test"), Path(self.test_dir2))]) ascii_hash1 = base64.b32encode(hash1).decode("ASCII") @@ -116,9 +137,9 @@ def test_directory_to_same_destination_equal(self): self.assertEqual(ascii_hash1, ascii_hash2) def test_directory_without_relative_paths_not_equal(self): - hasher = FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=True, - hash_file_names=True) + hasher = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=True, hash_file_names=True + ) hash1 = hasher.hash([simple_path_mapping(self.test_dir1)]) hash2 = hasher.hash([simple_path_mapping(self.test_dir2)]) ascii_hash1 = base64.b32encode(hash1).decode("ASCII") @@ -126,9 +147,9 @@ def test_directory_without_relative_paths_not_equal(self): self.assertNotEqual(ascii_hash1, ascii_hash2) def test_directory_content_only_equal(self): - hasher = FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=False, - hash_file_names=False) + hasher = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=False, hash_file_names=False + ) hash1 = hasher.hash([simple_path_mapping(self.test_dir1)]) hash2 = hasher.hash([simple_path_mapping(self.test_dir2)]) ascii_hash1 = base64.b32encode(hash1).decode("ASCII") @@ -136,65 +157,65 @@ def test_directory_content_only_equal(self): self.assertEqual(ascii_hash1, ascii_hash2) def test_directory_content_only_not_equal_to_with_paths(self): - hasher_content_only = \ - FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=False, - hash_file_names=False) - hasher_with_paths = \ - FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=True, - hash_file_names=True) - hash1_content_only = hasher_content_only.hash([simple_path_mapping(self.test_dir1)]) + hasher_content_only = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=False, hash_file_names=False + ) + hasher_with_paths = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=True, hash_file_names=True + ) + hash1_content_only = hasher_content_only.hash( + [simple_path_mapping(self.test_dir1)] + ) hash2_with_paths = hasher_with_paths.hash([simple_path_mapping(self.test_dir2)]) ascii_hash1_content_only = base64.b32encode(hash1_content_only).decode("ASCII") ascii_hash2_with_paths = base64.b32encode(hash2_with_paths).decode("ASCII") self.assertNotEqual(ascii_hash1_content_only, ascii_hash2_with_paths) def test_directory_content_only_not_equal_to_dir_names(self): - hasher_content_only = \ - FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=False, - hash_file_names=False) - hasher_with_paths = \ - FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=True, - hash_file_names=False) - hash1_content_only = hasher_content_only.hash([simple_path_mapping(self.test_dir1)]) + hasher_content_only = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=False, hash_file_names=False + ) + hasher_with_paths = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=True, hash_file_names=False + ) + hash1_content_only = hasher_content_only.hash( + [simple_path_mapping(self.test_dir1)] + ) hash2_with_paths = hasher_with_paths.hash([simple_path_mapping(self.test_dir1)]) ascii_hash1_content_only = base64.b32encode(hash1_content_only).decode("ASCII") ascii_hash2_with_paths = base64.b32encode(hash2_with_paths).decode("ASCII") self.assertNotEqual(ascii_hash1_content_only, ascii_hash2_with_paths) def test_directory_content_only_not_equal_to_file_names(self): - hasher_content_only = \ - FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=False, - hash_file_names=False) - hasher_with_paths = \ - FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=False, - hash_file_names=True) - hash1_content_only = hasher_content_only.hash([simple_path_mapping(self.test_dir1)]) + hasher_content_only = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=False, hash_file_names=False + ) + hasher_with_paths = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=False, hash_file_names=True + ) + hash1_content_only = hasher_content_only.hash( + [simple_path_mapping(self.test_dir1)] + ) hash2_with_paths = hasher_with_paths.hash([simple_path_mapping(self.test_dir1)]) ascii_hash1_content_only = base64.b32encode(hash1_content_only).decode("ASCII") ascii_hash2_with_paths = base64.b32encode(hash2_with_paths).decode("ASCII") self.assertNotEqual(ascii_hash1_content_only, ascii_hash2_with_paths) def test_directory_file_names_not_equal_to_dir_names(self): - hasher_content_only = \ - FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=False, - hash_file_names=True) - hasher_with_paths = \ - FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=True, - hash_file_names=False) - hash1_content_only = hasher_content_only.hash([simple_path_mapping(self.test_dir1)]) + hasher_content_only = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=False, hash_file_names=True + ) + hasher_with_paths = FileDirectoryListHasher( + hashfunc="sha256", hash_directory_names=True, hash_file_names=False + ) + hash1_content_only = hasher_content_only.hash( + [simple_path_mapping(self.test_dir1)] + ) hash2_with_paths = hasher_with_paths.hash([simple_path_mapping(self.test_dir2)]) ascii_hash1_content_only = base64.b32encode(hash1_content_only).decode("ASCII") ascii_hash2_with_paths = base64.b32encode(hash2_with_paths).decode("ASCII") self.assertNotEqual(ascii_hash1_content_only, ascii_hash2_with_paths) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_hash_temp_dir_with_files.py b/exasol_integration_test_docker_environment/test/test_hash_temp_dir_with_files.py index 292064037..ac49111a7 100644 --- a/exasol_integration_test_docker_environment/test/test_hash_temp_dir_with_files.py +++ b/exasol_integration_test_docker_environment/test/test_hash_temp_dir_with_files.py @@ -2,14 +2,19 @@ import os import tempfile import unittest -from pathlib import Path, PurePath +from pathlib import ( + Path, + PurePath, +) -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.file_directory_list_hasher import \ - FileDirectoryListHasher, PathMapping +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.file_directory_list_hasher import ( + FileDirectoryListHasher, + PathMapping, +) class HashTempDirTest(unittest.TestCase): - + def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.temp_path = Path(self.temp_dir.name) @@ -27,12 +32,13 @@ def test_file_name_with_relative_path(self): 1. Mapping dest="test.txt", src="/tmp/.../$tmpA/test.txt" 2. Mapping dest="test.txt", src="/tmp/.../$tmpB/test.txt" """ - hasher_content_only = \ - FileDirectoryListHasher(followlinks=True, - hashfunc="sha256", - hash_file_names=True, - hash_directory_names=True, - hash_permissions=True) + hasher_content_only = FileDirectoryListHasher( + followlinks=True, + hashfunc="sha256", + hash_file_names=True, + hash_directory_names=True, + hash_permissions=True, + ) test_file1 = Path(f"{self.test_path1}/test.txt") with open(test_file1, "w") as f: f.write("test") @@ -55,12 +61,13 @@ def test_file_name_with_relative_path_in_same_sub_path(self): 1. Mapping dest="level0/test.txt", src="/tmp/.../$tmpA/level0/test.txt" 2. Mapping dest="level0/test.txt", src="/tmp/.../$tmpB/level0/test.txt" """ - hasher_content_only = \ - FileDirectoryListHasher(followlinks=True, - hashfunc="sha256", - hash_file_names=True, - hash_directory_names=True, - hash_permissions=True) + hasher_content_only = FileDirectoryListHasher( + followlinks=True, + hashfunc="sha256", + hash_file_names=True, + hash_directory_names=True, + hash_permissions=True, + ) p1 = self.test_path1 / "level0" p1.mkdir() test_file1 = p1 / "test.txt" @@ -87,18 +94,21 @@ def test_file_name_with_relative_path_in_different_sub_path(self): 1. Mapping dest="level0/test.txt", src="/tmp/.../level0/test.txt" 2. Mapping dest="level0/level1_0/test.txt", src="/tmp/.../level0/level1_0/test.txt" """ - hasher_content_only = \ - FileDirectoryListHasher(followlinks=True, - hashfunc="sha256", - hash_file_names=True, - hash_directory_names=True, - hash_permissions=True) + hasher_content_only = FileDirectoryListHasher( + followlinks=True, + hashfunc="sha256", + hash_file_names=True, + hash_directory_names=True, + hash_permissions=True, + ) p1 = self.test_path1 / "level0" p1.mkdir() test_file1 = p1 / "test.txt" with open(test_file1, "w") as f: f.write("test") - hash1_content_only = hasher_content_only.hash([PathMapping(PurePath("level0/test.txt"), test_file1)]) + hash1_content_only = hasher_content_only.hash( + [PathMapping(PurePath("level0/test.txt"), test_file1)] + ) p2 = self.test_path2 / "level0" / "level1_0" p2.mkdir(parents=True) @@ -106,8 +116,9 @@ def test_file_name_with_relative_path_in_different_sub_path(self): with open(test_file2, "w") as f: f.write("test") - hash2_content_only = hasher_content_only.hash([PathMapping(PurePath("level0/level1_0/test.txt"), - test_file2)]) + hash2_content_only = hasher_content_only.hash( + [PathMapping(PurePath("level0/level1_0/test.txt"), test_file2)] + ) ascii_hash1_content_only = base64.b32encode(hash1_content_only).decode("ASCII") ascii_hash2_content_only = base64.b32encode(hash2_content_only).decode("ASCII") @@ -121,12 +132,13 @@ def test_file_name_with_relative_path_in_relative_path_as_argument(self): 1. Mapping dest="test.txt", src="test.txt" 2. Mapping dest="test.txt", src="test.txt" """ - hasher_content_only = \ - FileDirectoryListHasher(followlinks=True, - hashfunc="sha256", - hash_file_names=True, - hash_directory_names=True, - hash_permissions=True) + hasher_content_only = FileDirectoryListHasher( + followlinks=True, + hashfunc="sha256", + hash_file_names=True, + hash_directory_names=True, + hash_permissions=True, + ) test_file = f"test.txt" old_pwd = os.getcwd() os.chdir(self.test_path1) @@ -151,12 +163,13 @@ def test_duplicated_file_mapping_raises_exception(self): 1. Mapping dest="test.txt", src="/tmp/.../$tmpB/level0/level1_0/test.txt" 2. Mapping dest="test.txt", src="/tmp/.../$tmpB/level0/level1_1/test.txt" """ - hasher_content_only = \ - FileDirectoryListHasher(followlinks=True, - hashfunc="sha256", - hash_file_names=True, - hash_directory_names=True, - hash_permissions=True) + hasher_content_only = FileDirectoryListHasher( + followlinks=True, + hashfunc="sha256", + hash_file_names=True, + hash_directory_names=True, + hash_permissions=True, + ) p1 = self.test_path1 / "level0" / "level1_0" p1.mkdir(parents=True) test_file1 = p1 / "test.txt" @@ -170,9 +183,13 @@ def test_duplicated_file_mapping_raises_exception(self): with open(test_file2, "w") as f: f.write("test") - path_mappings = [PathMapping(PurePath("test.txt"), test_file1), - PathMapping(PurePath("test.txt"), test_file2)] - self.assertRaises(AssertionError, lambda: hasher_content_only.hash(path_mappings)) + path_mappings = [ + PathMapping(PurePath("test.txt"), test_file1), + PathMapping(PurePath("test.txt"), test_file2), + ] + self.assertRaises( + AssertionError, lambda: hasher_content_only.hash(path_mappings) + ) def test_duplicated_path_mapping_raises_exception(self): """ @@ -180,12 +197,13 @@ def test_duplicated_path_mapping_raises_exception(self): 1. Mapping dest="test", src="/tmp/.../$tmpA/level0/level1_0", content under src="test/test.txt" 2. Mapping dest="test", src="/tmp/.../$tmpB/level0/level1_1", content under src="test/test.txt" """ - hasher_content_only = \ - FileDirectoryListHasher(followlinks=True, - hashfunc="sha256", - hash_file_names=True, - hash_directory_names=True, - hash_permissions=True) + hasher_content_only = FileDirectoryListHasher( + followlinks=True, + hashfunc="sha256", + hash_file_names=True, + hash_directory_names=True, + hash_permissions=True, + ) p1 = self.test_path1 / "level0" / "level1_0" / "test" test_file1 = p1 / "test.txt" @@ -200,9 +218,13 @@ def test_duplicated_path_mapping_raises_exception(self): path1 = p1.parent path2 = p2.parent - path_mappings = [PathMapping(PurePath("test"), path1), - PathMapping(PurePath("test"), path2)] - self.assertRaises(AssertionError, lambda: hasher_content_only.hash(path_mappings)) + path_mappings = [ + PathMapping(PurePath("test"), path1), + PathMapping(PurePath("test"), path2), + ] + self.assertRaises( + AssertionError, lambda: hasher_content_only.hash(path_mappings) + ) def test_duplicated_path_mapping_with_subpath_raises_exception(self): """ @@ -210,12 +232,13 @@ def test_duplicated_path_mapping_with_subpath_raises_exception(self): 1. Mapping dest="test/abc", src="/tmp/.../$tmpA/level0/level1_0", content under src="test/test.txt" 2. Mapping dest="test/abc", src="/tmp/.../$tmpB/level0/level1_1", content under src="test/test.txt" """ - hasher_content_only = \ - FileDirectoryListHasher(followlinks=True, - hashfunc="sha256", - hash_file_names=True, - hash_directory_names=True, - hash_permissions=True) + hasher_content_only = FileDirectoryListHasher( + followlinks=True, + hashfunc="sha256", + hash_file_names=True, + hash_directory_names=True, + hash_permissions=True, + ) p1 = self.test_path1 / "level0" / "level1_0" / "test" test_file1 = p1 / "test.txt" @@ -231,8 +254,13 @@ def test_duplicated_path_mapping_with_subpath_raises_exception(self): path1 = p1.parent path2 = p2.parent destination_path = PurePath("test/abc") - path_mappings = [PathMapping(destination_path, path1), PathMapping(destination_path, path2)] - self.assertRaises(AssertionError, lambda: hasher_content_only.hash(path_mappings)) + path_mappings = [ + PathMapping(destination_path, path1), + PathMapping(destination_path, path2), + ] + self.assertRaises( + AssertionError, lambda: hasher_content_only.hash(path_mappings) + ) def test_duplicated_path_mapping_with_destination_subpath_raises_exception(self): """ @@ -243,19 +271,20 @@ def test_duplicated_path_mapping_with_destination_subpath_raises_exception(self) 1. Mapping dest="test", src="/tmp/.../$tmpA", content under src="abc/level0/level1_0/test/test.txt" 2. Mapping dest="test/abc", src="/tmp/.../$tmpB", content under src="level0/level1_0/test/test.txt" """ - hasher_content_only = \ - FileDirectoryListHasher(followlinks=True, - hashfunc="sha256", - hash_file_names=True, - hash_directory_names=True, - hash_permissions=True) + hasher_content_only = FileDirectoryListHasher( + followlinks=True, + hashfunc="sha256", + hash_file_names=True, + hash_directory_names=True, + hash_permissions=True, + ) p1 = self.test_path1 / "level1_0" / "test" test_file1 = p1 / "test.txt" p1.mkdir(parents=True) with open(test_file1, "w") as f: f.write("test") - p2 = self.test_path2 / "abc" /"level1_0" / "test" + p2 = self.test_path2 / "abc" / "level1_0" / "test" p2.mkdir(parents=True) test_file2 = p2 / "test.txt" with open(test_file2, "w") as f: @@ -264,9 +293,14 @@ def test_duplicated_path_mapping_with_destination_subpath_raises_exception(self) path1 = self.test_path1 path2 = self.test_path2 - path_mappings = [PathMapping(PurePath("test/abc"), path1), PathMapping(PurePath("test"), path2)] - self.assertRaises(AssertionError, lambda: hasher_content_only.hash(path_mappings)) + path_mappings = [ + PathMapping(PurePath("test/abc"), path1), + PathMapping(PurePath("test"), path2), + ] + self.assertRaises( + AssertionError, lambda: hasher_content_only.hash(path_mappings) + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_populate_data.py b/exasol_integration_test_docker_environment/test/test_populate_data.py index 9aeb79c7e..3f4502123 100644 --- a/exasol_integration_test_docker_environment/test/test_populate_data.py +++ b/exasol_integration_test_docker_environment/test/test_populate_data.py @@ -1,18 +1,27 @@ import unittest -from pathlib import Path, PurePath +from pathlib import ( + Path, + PurePath, +) from sys import stderr import luigi from exasol_integration_test_docker_environment.lib.api.common import generate_root_task -from exasol_integration_test_docker_environment.lib.data.test_container_content_description import \ - TestContainerRuntimeMapping +from exasol_integration_test_docker_environment.lib.data.test_container_content_description import ( + TestContainerRuntimeMapping, +) from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.lib.test_environment.database_setup.populate_data import \ - PopulateTestDataToDatabase -from exasol_integration_test_docker_environment.test.get_test_container_content import get_test_container_content +from exasol_integration_test_docker_environment.lib.test_environment.database_setup.populate_data import ( + PopulateTestDataToDatabase, +) +from exasol_integration_test_docker_environment.test.get_test_container_content import ( + get_test_container_content, +) from exasol_integration_test_docker_environment.testing import utils -from exasol_integration_test_docker_environment.testing.api_test_environment import ApiTestEnvironment +from exasol_integration_test_docker_environment.testing.api_test_environment import ( + ApiTestEnvironment, +) class TestDataPopulateData(PopulateTestDataToDatabase): @@ -32,28 +41,36 @@ def setUpClass(cls): cls.test_environment = ApiTestEnvironment(cls) cls.docker_environment_name = cls.__name__ test_data_folder = Path(__file__).parent / "resources" / "test_data" - test_container_runtime_mapping = TestContainerRuntimeMapping(source=test_data_folder, target="/test_data") - test_container_content = get_test_container_content(runtime_mapping=(test_container_runtime_mapping,)) - cls.environment = \ + test_container_runtime_mapping = TestContainerRuntimeMapping( + source=test_data_folder, target="/test_data" + ) + test_container_content = get_test_container_content( + runtime_mapping=(test_container_runtime_mapping,) + ) + cls.environment = ( cls.test_environment.spawn_docker_test_environment_with_test_container( name=cls.docker_environment_name, - test_container_content=test_container_content + test_container_content=test_container_content, ) + ) @classmethod def tearDownClass(cls): utils.close_environments(cls.environment, cls.test_environment) def _populate_data(self, reuse=False): - task = generate_root_task(task_class=TestDataPopulateData, - environment_name=self.environment.name, - db_user=self.environment.db_username, - db_password=self.environment.db_password, - bucketfs_write_password=self.environment.bucketfs_password, - test_environment_info=self.environment.environment_info, - ) + task = generate_root_task( + task_class=TestDataPopulateData, + environment_name=self.environment.name, + db_user=self.environment.db_username, + db_password=self.environment.db_password, + bucketfs_write_password=self.environment.bucketfs_password, + test_environment_info=self.environment.environment_info, + ) try: - success = luigi.build([task], workers=1, local_scheduler=True, log_level="INFO") + success = luigi.build( + [task], workers=1, local_scheduler=True, log_level="INFO" + ) if not success: raise Exception("Task failed") except Exception as e: @@ -66,16 +83,19 @@ def setUp(self) -> None: def _execute_sql_on_db(self, sql: str) -> str: with ContextDockerClient() as docker_client: print(f"Executing sql on db: '{sql}'") - #environment is a class variable, need to suppress type check - environment = self.environment # type: ignore - test_container = docker_client.containers.get(environment.environment_info. - test_container_info.container_name) + # environment is a class variable, need to suppress type check + environment = self.environment # type: ignore + test_container = docker_client.containers.get( + environment.environment_info.test_container_info.container_name + ) db_info = environment.environment_info.database_info db_user_name = environment.db_username db_password = environment.db_password - cmd = f"$EXAPLUS -x -q -c '{db_info.host}:{db_info.ports.database}' " \ - f"-u '{db_user_name}' -p '{db_password}' -sql '{sql}' " \ - f"-jdbcparam 'validateservercertificate=0'" + cmd = ( + f"$EXAPLUS -x -q -c '{db_info.host}:{db_info.ports.database}' " + f"-u '{db_user_name}' -p '{db_password}' -sql '{sql}' " + f"-jdbcparam 'validateservercertificate=0'" + ) bash_cmd = f"""bash -c "{cmd}" """ exit_code, output = test_container.exec_run(cmd=bash_cmd) @@ -88,8 +108,10 @@ def _execute_sql_on_db(self, sql: str) -> str: def test_populate_data(self): self._populate_data() result = self._execute_sql_on_db("SELECT count(*) FROM TEST.ENGINETABLE;") - expected_result = '\nCOUNT(*) \n---------------------\n 100\n\n' - self.assertEquals(result, expected_result) + expected_result = ( + "\nCOUNT(*) \n---------------------\n 100\n\n" + ) + self.assertEqual(result, expected_result) def test_populate_twice_throws_exception(self): self._populate_data() @@ -101,5 +123,5 @@ def test_populate_twice_throws_exception(self): self.assertTrue(exception_thrown) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_symlink_loops.py b/exasol_integration_test_docker_environment/test/test_symlink_loops.py index d39325074..1c27ded79 100644 --- a/exasol_integration_test_docker_environment/test/test_symlink_loops.py +++ b/exasol_integration_test_docker_environment/test/test_symlink_loops.py @@ -1,10 +1,12 @@ import os import shutil -import unittest import tempfile +import unittest -from exasol_integration_test_docker_environment.lib.docker.images.create.utils.file_directory_list_hasher import \ - FileDirectoryListHasher, PathMapping +from exasol_integration_test_docker_environment.lib.docker.images.create.utils.file_directory_list_hasher import ( + FileDirectoryListHasher, + PathMapping, +) class TestSymlinkLoops(unittest.TestCase): @@ -22,22 +24,33 @@ def generate_test_dir(self, with_symlink_loop): for i2 in range(level2): for i3 in range(level3): for i4 in range(level4): - path = "/level0/level1_%s/level2_%s/level3_%s/level4_%s/" \ - % (i1, i2, i3, i4) + path = ( + "/level0/level1_{}/level2_{}/level3_{}/level4_{}/".format( + i1, + i2, + i3, + i4, + ) + ) os.makedirs(self.temp_dir + path) os.makedirs(self.temp_dir + path + "test") for i5 in range(level5): - file = "%s/level5_file_%s" % (path, i5) - with open(self.temp_dir + file, mode="wt") as f: + file = f"{path}/level5_file_{i5}" + with open(self.temp_dir + file, mode="w") as f: f.write(file) - path = "/level0/level1_%s/level2_%s/level3_%s/level4_%s/" \ - % (level1-1, level2-1, level3-1, level4-1) - dest_path = "/level0/level1_%s" \ - % (level1-1) + path = "/level0/level1_{}/level2_{}/level3_{}/level4_{}/".format( + level1 - 1, + level2 - 1, + level3 - 1, + level4 - 1, + ) + dest_path = "/level0/level1_%s" % (level1 - 1) if with_symlink_loop: - os.symlink(self.temp_dir + dest_path, self.temp_dir + f"{path}/evil_symlink") + os.symlink( + self.temp_dir + dest_path, self.temp_dir + f"{path}/evil_symlink" + ) return self.temp_dir + f"{path}/evil_symlink" def tearDown(self): @@ -46,13 +59,17 @@ def tearDown(self): def test_symlink_detection(self): self.generate_test_dir(True) - hasher = FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=True, - hash_file_names=True, - followlinks=True) + hasher = FileDirectoryListHasher( + hashfunc="sha256", + hash_directory_names=True, + hash_file_names=True, + followlinks=True, + ) exception_thrown = False try: - hash = hasher.hash([PathMapping(destination=self.__class__.__name__, source=self.temp_dir)]) + hash = hasher.hash( + [PathMapping(destination=self.__class__.__name__, source=self.temp_dir)] + ) except OSError as e: if "contains symlink loops" in str(e): exception_thrown = True @@ -60,14 +77,18 @@ def test_symlink_detection(self): def test_symlink_detection_size(self): self.generate_test_dir(True) - hasher = FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=True, - hash_file_names=True, - followlinks=True, - max_characters_paths=100) + hasher = FileDirectoryListHasher( + hashfunc="sha256", + hash_directory_names=True, + hash_file_names=True, + followlinks=True, + max_characters_paths=100, + ) exception_thrown = False try: - hash = hasher.hash([PathMapping(destination=self.__class__.__name__, source=self.temp_dir)]) + hash = hasher.hash( + [PathMapping(destination=self.__class__.__name__, source=self.temp_dir)] + ) except OSError as e: if "Walking through too many directories." in str(e): exception_thrown = True @@ -76,12 +97,16 @@ def test_symlink_detection_size(self): def test_symlink_no_loop(self): symlink_dest = self.generate_test_dir(False) os.symlink(self.temp_dir_dummy, symlink_dest) - hasher = FileDirectoryListHasher(hashfunc="sha256", - hash_directory_names=True, - hash_file_names=True, - followlinks=True) - hash = hasher.hash([PathMapping(destination=self.__class__.__name__, source=self.temp_dir)]) + hasher = FileDirectoryListHasher( + hashfunc="sha256", + hash_directory_names=True, + hash_file_names=True, + followlinks=True, + ) + hash = hasher.hash( + [PathMapping(destination=self.__class__.__name__, source=self.temp_dir)] + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_termination_handler.py b/exasol_integration_test_docker_environment/test/test_termination_handler.py index 4401f56dd..ef3626cb2 100644 --- a/exasol_integration_test_docker_environment/test/test_termination_handler.py +++ b/exasol_integration_test_docker_environment/test/test_termination_handler.py @@ -30,6 +30,7 @@ def run_positive(queue: Queue) -> None: with m.TerminationHandler(): pass + def run_with_unknown_error(queue: Queue) -> None: stdout_queue = StdoutQueue(queue) sys.stdout = stdout_queue @@ -55,8 +56,10 @@ def test_success(self): p.start() p.join() res = get_queue_content(q) - self.assertTrue(any(re.match(r"^The command took .+ s$", line) for line in res), - f"Result {res} doesn't contain 'The command took'") + self.assertTrue( + any(re.match(r"^The command took .+ s$", line) for line in res), + f"Result {res} doesn't contain 'The command took'", + ) self.assertEqual(p.exitcode, 0) def test_unknown_error(self): @@ -65,11 +68,17 @@ def test_unknown_error(self): p.start() p.join() res = get_queue_content(q) - self.assertTrue(any(re.match(r"^The command failed after .+ s with:$", line) for line in res)) + self.assertTrue( + any( + re.match(r"^The command failed after .+ s with:$", line) for line in res + ) + ) self.assertTrue(any("Caught exception:unknown error" == line for line in res)) - self.assertTrue(any('raise RuntimeError("unknown error")' in line for line in res)) + self.assertTrue( + any('raise RuntimeError("unknown error")' in line for line in res) + ) self.assertEqual(p.exitcode, 1) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_test_container_reuse.py b/exasol_integration_test_docker_environment/test/test_test_container_reuse.py index d2f850824..50b4d7ff1 100644 --- a/exasol_integration_test_docker_environment/test/test_test_container_reuse.py +++ b/exasol_integration_test_docker_environment/test/test_test_container_reuse.py @@ -5,18 +5,30 @@ import luigi -from exasol_integration_test_docker_environment.lib.api.common import set_build_config, set_docker_repository_config, \ - generate_root_task -from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask -from exasol_integration_test_docker_environment.lib.data.container_info import ContainerInfo -from exasol_integration_test_docker_environment.lib.data.test_container_content_description import \ - TestContainerContentDescription +from exasol_integration_test_docker_environment.lib.api.common import ( + generate_root_task, + set_build_config, + set_docker_repository_config, +) +from exasol_integration_test_docker_environment.lib.base.docker_base_task import ( + DockerBaseTask, +) +from exasol_integration_test_docker_environment.lib.data.container_info import ( + ContainerInfo, +) +from exasol_integration_test_docker_environment.lib.data.test_container_content_description import ( + TestContainerContentDescription, +) from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.lib.test_environment.parameter.test_container_parameter import \ - TestContainerParameter -from exasol_integration_test_docker_environment.lib.test_environment.prepare_network_for_test_environment import \ - PrepareDockerNetworkForTestEnvironment -from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_container import SpawnTestContainer +from exasol_integration_test_docker_environment.lib.test_environment.parameter.test_container_parameter import ( + TestContainerParameter, +) +from exasol_integration_test_docker_environment.lib.test_environment.prepare_network_for_test_environment import ( + PrepareDockerNetworkForTestEnvironment, +) +from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_container import ( + SpawnTestContainer, +) from exasol_integration_test_docker_environment.testing import luigi_utils @@ -25,45 +37,55 @@ class TestTask(DockerBaseTask, TestContainerParameter): attempt = luigi.IntParameter() def run_task(self): - docker_network_task_1 = self.create_child_task(task_class=PrepareDockerNetworkForTestEnvironment, - environment_name="test_environment_TestContainerReuseTest", - network_name="docker_network_TestContainerReuseTest", - test_container_name="test_container_TestContainerReuseTest", - db_container_name="db_container_TestContainerReuseTest", - reuse=self.reuse, - no_cleanup_after_success=True, - no_cleanup_after_failure=False, - attempt=self.attempt - ) - self.docker_network_future_1 = yield from self.run_dependencies(docker_network_task_1) - - test_container_task_1 = \ - self.create_child_task(task_class=SpawnTestContainer, - environment_name="test_environment_TestContainerReuseTest", - test_container_name="test_container_TestContainerReuseTest", - network_info=self.docker_network_future_1.get_output(), - ip_address_index_in_subnet=2, - attempt=self.attempt, - reuse_test_container=self.reuse, - no_test_container_cleanup_after_success=True, - no_test_container_cleanup_after_failure=False, - test_container_content=self.test_container_content - ) - test_container_future_1 = yield from self.run_dependencies(test_container_task_1) - container_info : ContainerInfo = test_container_future_1.get_output() # type: ignore + docker_network_task_1 = self.create_child_task( + task_class=PrepareDockerNetworkForTestEnvironment, + environment_name="test_environment_TestContainerReuseTest", + network_name="docker_network_TestContainerReuseTest", + test_container_name="test_container_TestContainerReuseTest", + db_container_name="db_container_TestContainerReuseTest", + reuse=self.reuse, + no_cleanup_after_success=True, + no_cleanup_after_failure=False, + attempt=self.attempt, + ) + self.docker_network_future_1 = yield from self.run_dependencies( + docker_network_task_1 + ) + + test_container_task_1 = self.create_child_task( + task_class=SpawnTestContainer, + environment_name="test_environment_TestContainerReuseTest", + test_container_name="test_container_TestContainerReuseTest", + network_info=self.docker_network_future_1.get_output(), + ip_address_index_in_subnet=2, + attempt=self.attempt, + reuse_test_container=self.reuse, + no_test_container_cleanup_after_success=True, + no_test_container_cleanup_after_failure=False, + test_container_content=self.test_container_content, + ) + test_container_future_1 = yield from self.run_dependencies( + test_container_task_1 + ) + container_info: ContainerInfo = test_container_future_1.get_output() # type: ignore with ContextDockerClient() as docker_client: container = docker_client.containers.get(container_info.container_name) - self.return_object({"container_id": container.id, "image_id": container.image.id}) + self.return_object( + {"container_id": container.id, "image_id": container.image.id} + ) class TestContainerReuseTest(unittest.TestCase): def setUp(self): - resource_directory = Path(Path(__file__).parent, "resources/test_test_container_reuse") + resource_directory = Path( + Path(__file__).parent, "resources/test_test_container_reuse" + ) print("resource_directory content", list(Path(resource_directory).iterdir())) self.temp_directory = tempfile.mkdtemp() - self.working_directory = shutil.copytree(resource_directory, - Path(self.temp_directory, "test_test_container_reuse")) + self.working_directory = shutil.copytree( + resource_directory, Path(self.temp_directory, "test_test_container_reuse") + ) print("working_directory", self.working_directory) print("working_directory content", list(Path(self.working_directory).iterdir())) self.docker_repository_name = self.__class__.__name__.lower() @@ -79,25 +101,26 @@ def get_test_container_content(self) -> TestContainerContentDescription: return TestContainerContentDescription( docker_file=str(self.dockerfile), build_files_and_directories=list(), - runtime_mappings=list() + runtime_mappings=list(), ) def setup_luigi_config(self): - set_build_config(force_rebuild=False, - force_pull=False, - force_rebuild_from=tuple(), - log_build_context_content=False, - output_directory=self.temp_directory, - cache_directory="", - build_name="", - temporary_base_directory="/tmp" - ) + set_build_config( + force_rebuild=False, + force_pull=False, + force_rebuild_from=tuple(), + log_build_context_content=False, + output_directory=self.temp_directory, + cache_directory="", + build_name="", + temporary_base_directory="/tmp", + ) set_docker_repository_config( docker_password=None, docker_repository_name=self.docker_repository_name, docker_username=None, tag_prefix="", - kind="target" + kind="target", ) @property @@ -105,10 +128,16 @@ def dockerfile(self): return Path(self.working_directory) / "tests" / "Dockerfile" def run1(self): - task = generate_root_task(task_class=TestTask, reuse=False, attempt=1, - test_container_content=self.get_test_container_content()) + task = generate_root_task( + task_class=TestTask, + reuse=False, + attempt=1, + test_container_content=self.get_test_container_content(), + ) try: - success = luigi.build([task], workers=1, local_scheduler=True, log_level="INFO") + success = luigi.build( + [task], workers=1, local_scheduler=True, log_level="INFO" + ) if success: result = task.get_result() task.cleanup(True) @@ -120,10 +149,16 @@ def run1(self): raise RuntimeError("Error spawning test environment") from e def run2(self): - task = generate_root_task(task_class=TestTask, reuse=True, attempt=2, - test_container_content=self.get_test_container_content()) + task = generate_root_task( + task_class=TestTask, + reuse=True, + attempt=2, + test_container_content=self.get_test_container_content(), + ) try: - success = luigi.build([task], workers=1, local_scheduler=True, log_level="INFO") + success = luigi.build( + [task], workers=1, local_scheduler=True, log_level="INFO" + ) if success: return task.get_result() @@ -156,5 +191,5 @@ def test_test_container_reuse(self): assert p1 == p2 -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/test/test_test_env_reuse.py b/exasol_integration_test_docker_environment/test/test_test_env_reuse.py index b03c80156..539441fc3 100644 --- a/exasol_integration_test_docker_environment/test/test_test_env_reuse.py +++ b/exasol_integration_test_docker_environment/test/test_test_env_reuse.py @@ -2,25 +2,38 @@ import luigi -from exasol_integration_test_docker_environment.lib.data.environment_type import EnvironmentType +from exasol_integration_test_docker_environment.cli.options import ( + test_environment_options, +) +from exasol_integration_test_docker_environment.lib.api.common import ( + generate_root_task, + set_docker_repository_config, +) +from exasol_integration_test_docker_environment.lib.data.environment_type import ( + EnvironmentType, +) from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_environment import SpawnTestEnvironment -from exasol_integration_test_docker_environment.lib.api.common import set_docker_repository_config, generate_root_task -from exasol_integration_test_docker_environment.test.get_test_container_content import get_test_container_content -from exasol_integration_test_docker_environment.testing import luigi_utils -from exasol_integration_test_docker_environment.cli.options import test_environment_options -from exasol_integration_test_docker_environment.testing.utils import check_db_version_from_env from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports +from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_environment import ( + SpawnTestEnvironment, +) +from exasol_integration_test_docker_environment.test.get_test_container_content import ( + get_test_container_content, +) +from exasol_integration_test_docker_environment.testing import luigi_utils +from exasol_integration_test_docker_environment.testing.utils import ( + check_db_version_from_env, +) class TestContainerReuseTest(unittest.TestCase): - ''' + """ This test spawns a new test environment and, with parameters: * reuse_database_setup=True, * reuse_database=True, * reuse_test_container=True and verifies if the test data was populated to the docker db. - ''' + """ def env_name(self): return self.__class__.__name__.lower() @@ -30,7 +43,9 @@ def setUp(self): print("docker_repository_name", self._docker_repository_name) luigi_utils.clean(self._docker_repository_name) - self.docker_db_version_parameter = check_db_version_from_env() or test_environment_options.LATEST_DB_VERSION + self.docker_db_version_parameter = ( + check_db_version_from_env() or test_environment_options.LATEST_DB_VERSION + ) self.setup_luigi_config() self.ports = Ports.random_free() @@ -44,38 +59,41 @@ def setup_luigi_config(self): docker_repository_name=self._docker_repository_name, docker_username=None, tag_prefix="", - kind="target" + kind="target", ) def run_spawn_test_env(self, cleanup: bool): result = None - task = generate_root_task(task_class=SpawnTestEnvironment, - reuse_database_setup=True, - reuse_database=True, - reuse_test_container=True, - no_test_container_cleanup_after_success=not cleanup, - no_database_cleanup_after_success=not cleanup, - external_exasol_db_port=self.ports.database, - external_exasol_bucketfs_port=self.ports.bucketfs, - external_exasol_ssh_port=self.ports.ssh, - external_exasol_xmlrpc_host="", - external_exasol_db_host="", - external_exasol_xmlrpc_port=0, - external_exasol_db_user="", - external_exasol_db_password="", - external_exasol_xmlrpc_user="", - external_exasol_xmlrpc_password="", - external_exasol_xmlrpc_cluster_name="", - external_exasol_bucketfs_write_password="", - environment_type=EnvironmentType.docker_db, - environment_name=self.env_name(), - docker_db_image_version=self.docker_db_version_parameter, - docker_db_image_name="exasol/docker-db", - test_container_content=get_test_container_content(), - additional_db_parameter=tuple() - ) + task = generate_root_task( + task_class=SpawnTestEnvironment, + reuse_database_setup=True, + reuse_database=True, + reuse_test_container=True, + no_test_container_cleanup_after_success=not cleanup, + no_database_cleanup_after_success=not cleanup, + external_exasol_db_port=self.ports.database, + external_exasol_bucketfs_port=self.ports.bucketfs, + external_exasol_ssh_port=self.ports.ssh, + external_exasol_xmlrpc_host="", + external_exasol_db_host="", + external_exasol_xmlrpc_port=0, + external_exasol_db_user="", + external_exasol_db_password="", + external_exasol_xmlrpc_user="", + external_exasol_xmlrpc_password="", + external_exasol_xmlrpc_cluster_name="", + external_exasol_bucketfs_write_password="", + environment_type=EnvironmentType.docker_db, + environment_name=self.env_name(), + docker_db_image_version=self.docker_db_version_parameter, + docker_db_image_name="exasol/docker-db", + test_container_content=get_test_container_content(), + additional_db_parameter=tuple(), + ) try: - success = luigi.build([task], workers=1, local_scheduler=True, log_level="INFO") + success = luigi.build( + [task], workers=1, local_scheduler=True, log_level="INFO" + ) if success: result = task else: @@ -91,24 +109,33 @@ def _create_exaplus_check_cmd(self, test_environment_info): database_host = test_environment_info.database_info.host database_port = test_environment_info.database_info.db_port q = "SELECT TABLE_NAME FROM SYS.EXA_ALL_TABLES WHERE TABLE_SCHEMA='TEST';" - return f"$EXAPLUS -c '{database_host}:{database_port}' -u '{username}' -p '{password}' " \ - f"-jdbcparam 'validateservercertificate=0' -sql \\\"{q}\\\"" + return ( + f"$EXAPLUS -c '{database_host}:{database_port}' -u '{username}' -p '{password}' " + f"-jdbcparam 'validateservercertificate=0' -sql \\\"{q}\\\"" + ) def _exec_cmd_in_test_container(self, test_environment_info, cmd): with ContextDockerClient() as docker_client: bash_cmd = f"""bash -c "{cmd}" """ - test_container = docker_client.containers.get(test_environment_info.test_container_info.container_name) + test_container = docker_client.containers.get( + test_environment_info.test_container_info.container_name + ) exit_code, output = test_container.exec_run(cmd=bash_cmd) - self.assertEquals(exit_code, 0) - return output.decode('utf-8') + self.assertEqual(exit_code, 0) + return output.decode("utf-8") def get_instance_ids(self, test_environment_info): with ContextDockerClient() as docker_client: - test_container = docker_client.containers.get(test_environment_info.test_container_info.container_name) - db_container = \ - docker_client.containers.get(test_environment_info.database_info.container_info.container_name) - network = docker_client.networks.get(test_environment_info.network_info.network_name) + test_container = docker_client.containers.get( + test_environment_info.test_container_info.container_name + ) + db_container = docker_client.containers.get( + test_environment_info.database_info.container_info.container_name + ) + network = docker_client.networks.get( + test_environment_info.network_info.network_name + ) return test_container.id, db_container.id, network.id def test_reuse_env_same_instances(self): @@ -121,10 +148,10 @@ def test_reuse_env_same_instances(self): task = self.run_spawn_test_env(cleanup=True) test_environment_info = task.get_result() new_instance_ids = self.get_instance_ids(test_environment_info) - self.assertEquals(old_instance_ids, new_instance_ids) + self.assertEqual(old_instance_ids, new_instance_ids) task.cleanup(True) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/exasol_integration_test_docker_environment/testing/api_consistency_utils.py b/exasol_integration_test_docker_environment/testing/api_consistency_utils.py index b6c809fa0..7dbd13cee 100644 --- a/exasol_integration_test_docker_environment/testing/api_consistency_utils.py +++ b/exasol_integration_test_docker_environment/testing/api_consistency_utils.py @@ -1,5 +1,10 @@ import inspect -from typing import Any, List, Tuple, Optional +from typing import ( + Any, + List, + Optional, + Tuple, +) import click @@ -29,11 +34,17 @@ def is_click_command(obj: Any) -> bool: return isinstance(obj, click.Command) -def defaults_of_click_call(click_call: click.Command) -> List[Tuple[Optional[str] , Any]]: +def defaults_of_click_call( + click_call: click.Command, +) -> List[Tuple[Optional[str], Any]]: """ Returns the default values of all None-required parameters of a click-command. """ - return [(o.name, adjust_default_value_for_multiple(o)) for o in click_call.params if not o.required] + return [ + (o.name, adjust_default_value_for_multiple(o)) + for o in click_call.params + if not o.required + ] def param_names_of_click_call(click_call: click.Command) -> List[Optional[str]]: @@ -43,19 +54,31 @@ def param_names_of_click_call(click_call: click.Command) -> List[Optional[str]]: return [o.name for o in click_call.params] -def get_click_and_api_functions(click_module, api_module) -> Tuple[List[Any], List[Any]]: +def get_click_and_api_functions( + click_module, api_module +) -> Tuple[List[Any], List[Any]]: # Get all click commands in module 'click_module' click_commands = [c[1] for c in inspect.getmembers(click_module, is_click_command)] # Get all functions in module 'api_module' - api_functions = [f[1] for f in inspect.getmembers(api_module, inspect.isfunction) - if f[1].__cli_function__] # type: ignore + api_functions = [ + f[1] + for f in inspect.getmembers(api_module, inspect.isfunction) + if f[1].__cli_function__ # type: ignore + ] return click_commands, api_functions -def get_click_and_api_function_names(click_module, api_module) -> Tuple[List[Any], List[Any]]: +def get_click_and_api_function_names( + click_module, api_module +) -> Tuple[List[Any], List[Any]]: # Get all click command names in module 'click_module' - click_command_names = [c[0] for c in inspect.getmembers(click_module, is_click_command)] + click_command_names = [ + c[0] for c in inspect.getmembers(click_module, is_click_command) + ] # Get all function names in module 'api_module' - api_function_names = [f[0] for f in inspect.getmembers(api_module, inspect.isfunction) - if f[1].__cli_function__] # type: ignore + api_function_names = [ + f[0] + for f in inspect.getmembers(api_module, inspect.isfunction) + if f[1].__cli_function__ # type: ignore + ] return click_command_names, api_function_names diff --git a/exasol_integration_test_docker_environment/testing/api_test_environment.py b/exasol_integration_test_docker_environment/testing/api_test_environment.py index fd4acd4df..c216522c7 100644 --- a/exasol_integration_test_docker_environment/testing/api_test_environment.py +++ b/exasol_integration_test_docker_environment/testing/api_test_environment.py @@ -4,23 +4,30 @@ import tempfile from pathlib import Path from sys import stderr -from typing import Dict, Any, Optional +from typing import ( + Any, + Dict, + Optional, +) -from exasol_integration_test_docker_environment.lib.api import spawn_test_environment -from exasol_integration_test_docker_environment.lib.api import spawn_test_environment_with_test_container -from exasol_integration_test_docker_environment.lib.data.test_container_content_description import \ - TestContainerContentDescription -from exasol_integration_test_docker_environment.testing.docker_registry import default_docker_repository_name -from exasol_integration_test_docker_environment \ - .testing.exaslct_docker_test_environment import \ - ExaslctDockerTestEnvironment -from exasol_integration_test_docker_environment \ - .testing.exaslct_test_environment import ( - get_class, - get_test_flavor, +from exasol_integration_test_docker_environment.lib.api import ( + spawn_test_environment, + spawn_test_environment_with_test_container, +) +from exasol_integration_test_docker_environment.lib.data.test_container_content_description import ( + TestContainerContentDescription, +) +from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports +from exasol_integration_test_docker_environment.testing.docker_registry import ( + default_docker_repository_name, +) +from exasol_integration_test_docker_environment.testing.exaslct_docker_test_environment import ( + ExaslctDockerTestEnvironment, +) +from exasol_integration_test_docker_environment.testing.exaslct_test_environment import ( + get_class, + get_test_flavor, ) -from exasol_integration_test_docker_environment \ - .lib.test_environment.ports import Ports class ApiTestEnvironment: @@ -59,17 +66,17 @@ def _get_default_test_environment(self, name: str, ports: Ports): ) def spawn_docker_test_environment_with_test_container( - self, - name: str, - test_container_content: TestContainerContentDescription, - additional_parameter: Optional[Dict[str, Any]] = None, + self, + name: str, + test_container_content: TestContainerContentDescription, + additional_parameter: Optional[Dict[str, Any]] = None, ) -> ExaslctDockerTestEnvironment: if additional_parameter is None: additional_parameter = dict() ports = Ports.random_free() on_host_parameter = self._get_default_test_environment(name, ports) docker_db_image_version = on_host_parameter.docker_db_image_version - on_host_parameter.environment_info, on_host_parameter.clean_up = \ + on_host_parameter.environment_info, on_host_parameter.clean_up = ( spawn_test_environment_with_test_container( environment_name=on_host_parameter.name, database_port_forward=ports.database, @@ -79,12 +86,13 @@ def spawn_docker_test_environment_with_test_container( test_container_content=test_container_content, **additional_parameter, ) + ) return on_host_parameter def spawn_docker_test_environment( - self, - name: str, - additional_parameter: Optional[Dict[str, Any]] = None, + self, + name: str, + additional_parameter: Optional[Dict[str, Any]] = None, ) -> ExaslctDockerTestEnvironment: if additional_parameter is None: additional_parameter = dict() diff --git a/exasol_integration_test_docker_environment/testing/docker_registry.py b/exasol_integration_test_docker_environment/testing/docker_registry.py index c86e360ed..b68760874 100644 --- a/exasol_integration_test_docker_environment/testing/docker_registry.py +++ b/exasol_integration_test_docker_environment/testing/docker_registry.py @@ -5,8 +5,12 @@ import requests from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.lib.docker.container.utils import remove_docker_container -from exasol_integration_test_docker_environment.lib.test_environment.ports import find_free_ports +from exasol_integration_test_docker_environment.lib.docker.container.utils import ( + remove_docker_container, +) +from exasol_integration_test_docker_environment.lib.test_environment.ports import ( + find_free_ports, +) def default_docker_repository_name(env_name: str) -> str: @@ -25,14 +29,18 @@ def name(self): @property def images(self): - url = f"http://localhost:{self._registry_port}/v2/{self._name.lower()}/tags/list" + url = ( + f"http://localhost:{self._registry_port}/v2/{self._name.lower()}/tags/list" + ) result = requests.request("GET", url) images = json.loads(result.content.decode("UTF-8")) return images @property def repositories(self): - result = requests.request("GET", f"http://localhost:{self._registry_port}/v2/_catalog/") + result = requests.request( + "GET", f"http://localhost:{self._registry_port}/v2/_catalog/" + ) repositories_ = json.loads(result.content.decode("UTF-8"))["repositories"] return repositories_ @@ -54,13 +62,16 @@ def __enter__(self): except: pass registry_container = docker_client.containers.run( - image="registry:2", name=registry_container_name, + image="registry:2", + name=registry_container_name, ports={5000: registry_port}, - detach=True + detach=True, ) time.sleep(10) logging.debug(f"Finished start container of {registry_container_name}") - self._local_docker_registry = LocalDockerRegistry(self._name, registry_container, registry_port) + self._local_docker_registry = LocalDockerRegistry( + self._name, registry_container, registry_port + ) return self._local_docker_registry def __exit__(self, exc_type, exc_val, exc_tb): @@ -68,4 +79,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): def close(self): if self._local_docker_registry is not None: - remove_docker_container([self._local_docker_registry._registry_container.id]) + remove_docker_container( + [self._local_docker_registry._registry_container.id] + ) diff --git a/exasol_integration_test_docker_environment/testing/exaslct_docker_test_environment.py b/exasol_integration_test_docker_environment/testing/exaslct_docker_test_environment.py index a473cc1a6..3a04ac3aa 100644 --- a/exasol_integration_test_docker_environment/testing/exaslct_docker_test_environment.py +++ b/exasol_integration_test_docker_environment/testing/exaslct_docker_test_environment.py @@ -1,24 +1,31 @@ import subprocess from typing import Optional -from exasol_integration_test_docker_environment \ - .cli.options.test_environment_options import LATEST_DB_VERSION -from exasol_integration_test_docker_environment \ - .lib.data.environment_info import EnvironmentInfo -from exasol_integration_test_docker_environment \ - .lib.test_environment.ports import Ports -from exasol_integration_test_docker_environment \ - .testing.utils import check_db_version_from_env +from exasol_integration_test_docker_environment.cli.options.test_environment_options import ( + LATEST_DB_VERSION, +) +from exasol_integration_test_docker_environment.lib.data.environment_info import ( + EnvironmentInfo, +) +from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports +from exasol_integration_test_docker_environment.testing.utils import ( + check_db_version_from_env, +) class ExaslctDockerTestEnvironment: - def __init__(self, name: str, database_host: str, - db_username: str, db_password: str, - bucketfs_username: str, - bucketfs_password: str, - ports: Ports, - environment_info: Optional[EnvironmentInfo] = None, - completed_process: Optional[subprocess.CompletedProcess] = None): + def __init__( + self, + name: str, + database_host: str, + db_username: str, + db_password: str, + bucketfs_username: str, + bucketfs_password: str, + ports: Ports, + environment_info: Optional[EnvironmentInfo] = None, + completed_process: Optional[subprocess.CompletedProcess] = None, + ): self.db_password = db_password self.db_username = db_username self.ports = ports diff --git a/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py b/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py index 3cbcab4a1..d5a0e26ed 100644 --- a/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py +++ b/exasol_integration_test_docker_environment/testing/exaslct_test_environment.py @@ -1,30 +1,44 @@ import functools import inspect import os +import shlex import shutil import subprocess import tempfile from pathlib import Path -import shlex from sys import stderr -from typing import List, Optional - -from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo +from typing import ( + List, + Optional, +) + +from exasol_integration_test_docker_environment.lib.data.environment_info import ( + EnvironmentInfo, +) from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.lib.docker.container.utils import remove_docker_container -from exasol_integration_test_docker_environment.lib.docker.volumes.utils import remove_docker_volumes -from exasol_integration_test_docker_environment.testing.docker_registry import default_docker_repository_name -from exasol_integration_test_docker_environment.testing.exaslct_docker_test_environment import \ - ExaslctDockerTestEnvironment -from exasol_integration_test_docker_environment.testing.spawned_test_environments import SpawnedTestEnvironments -from exasol_integration_test_docker_environment \ - .lib.test_environment.ports import Ports -from exasol_integration_test_docker_environment.testing.utils import check_db_version_from_env +from exasol_integration_test_docker_environment.lib.docker.container.utils import ( + remove_docker_container, +) +from exasol_integration_test_docker_environment.lib.docker.volumes.utils import ( + remove_docker_volumes, +) +from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports +from exasol_integration_test_docker_environment.testing.docker_registry import ( + default_docker_repository_name, +) +from exasol_integration_test_docker_environment.testing.exaslct_docker_test_environment import ( + ExaslctDockerTestEnvironment, +) +from exasol_integration_test_docker_environment.testing.spawned_test_environments import ( + SpawnedTestEnvironments, +) +from exasol_integration_test_docker_environment.testing.utils import ( + check_db_version_from_env, +) def _cleanup(env_name: str): - remove_docker_container([f"test_container_{env_name}", - f"db_container_{env_name}"]) + remove_docker_container([f"test_container_{env_name}", f"db_container_{env_name}"]) remove_docker_volumes([f"db_container_{env_name}_volume"]) @@ -42,12 +56,15 @@ def get_test_flavor(test_class): return None source_file_of_test_object = inspect.getsourcefile(test_class) return Path(os.path.realpath(source_file_of_test_object)).parent.joinpath( - "resources/test-flavor") + "resources/test-flavor" + ) class ExaslctTestEnvironment: - def __init__(self, test_object, executable="./exaslct", clean_images_at_close=True, name=None): + def __init__( + self, test_object, executable="./exaslct", clean_images_at_close=True, name=None + ): self.clean_images_at_close = clean_images_at_close self.executable = executable self.test_object = test_object @@ -79,22 +96,33 @@ def repository_name(self, value): def _update_attributes(self): self.flavor_path_argument = f"--flavor-path {self.flavor_path}" repository_name = self.repository_name - self.docker_repository_arguments = f"--source-docker-repository-name {repository_name} " \ - f"--target-docker-repository-name {repository_name}" - self.clean_docker_repository_arguments = f"--docker-repository-name {repository_name}" + self.docker_repository_arguments = ( + f"--source-docker-repository-name {repository_name} " + f"--target-docker-repository-name {repository_name}" + ) + self.clean_docker_repository_arguments = ( + f"--docker-repository-name {repository_name}" + ) self.output_directory_arguments = f"--output-directory {self.temp_dir}" - self.task_dependencies_argument = " ".join([f"--task-dependencies-dot-file {self.name}.dot", ]) + self.task_dependencies_argument = " ".join( + [ + f"--task-dependencies-dot-file {self.name}.dot", + ] + ) def clean_images(self): self.run_command(f"{self.executable} clean-flavor-images", clean=True) - def run_command(self, command: str, - use_output_directory: bool = True, - use_flavor_path: bool = True, - use_docker_repository: bool = True, - track_task_dependencies: bool = False, - clean: bool = False, - capture_output: bool = False): + def run_command( + self, + command: str, + use_output_directory: bool = True, + use_flavor_path: bool = True, + use_docker_repository: bool = True, + track_task_dependencies: bool = False, + clean: bool = False, + capture_output: bool = False, + ): if use_output_directory: command = f"{command} {self.output_directory_arguments}" if track_task_dependencies: @@ -108,7 +136,9 @@ def run_command(self, command: str, print(file=stderr) print(f"command: {command}", file=stderr) if capture_output: - completed_process = subprocess.run(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + completed_process = subprocess.run( + shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) else: completed_process = subprocess.run(shlex.split(command)) try: @@ -130,8 +160,9 @@ def close(self): except Exception as e: print(e, file=stderr) - def spawn_docker_test_environments(self, name: str, additional_parameter: Optional[List[str]] = None) \ - -> SpawnedTestEnvironments: + def spawn_docker_test_environments( + self, name: str, additional_parameter: Optional[List[str]] = None + ) -> SpawnedTestEnvironments: ports = Ports.random_free() on_host_parameter = ExaslctDockerTestEnvironment( name=self.name + "_" + name, @@ -143,7 +174,7 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: Option ports=ports, ) - arguments : List[str] = [ + arguments: List[str] = [ f"--environment-name {on_host_parameter.name}", f"--database-port-forward {on_host_parameter.ports.database}", f"--bucketfs-port-forward {on_host_parameter.ports.bucketfs}", @@ -157,8 +188,12 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: Option arguments_str = " ".join(arguments) command = f"{self.executable} spawn-test-environment {arguments_str}" - completed_process = self.run_command(command, use_flavor_path=False, use_docker_repository=False, - capture_output=True) + completed_process = self.run_command( + command, + use_flavor_path=False, + use_docker_repository=False, + capture_output=True, + ) on_host_parameter.completed_process = completed_process environment_info_json_path = Path( self.temp_dir, @@ -168,7 +203,7 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: Option with environment_info_json_path.open() as f: environment_info = EnvironmentInfo.from_json(f.read()) on_host_parameter.environment_info = environment_info - on_host_parameter.clean_up = functools.partial(_cleanup, on_host_parameter.name) #type: ignore + on_host_parameter.clean_up = functools.partial(_cleanup, on_host_parameter.name) # type: ignore if "RUN_SLC_TESTS_WITHIN_CONTAINER" in os.environ: slc_test_run_parameter = ExaslctDockerTestEnvironment( name=on_host_parameter.name, @@ -179,17 +214,21 @@ def spawn_docker_test_environments(self, name: str, additional_parameter: Option bucketfs_password=on_host_parameter.bucketfs_password, ports=Ports.default_ports, environment_info=on_host_parameter.environment_info, - completed_process=on_host_parameter.completed_process + completed_process=on_host_parameter.completed_process, ) with ContextDockerClient() as docker_client: - db_container = docker_client.containers.get(f"db_container_{slc_test_run_parameter.name}") + db_container = docker_client.containers.get( + f"db_container_{slc_test_run_parameter.name}" + ) cloudbuild_network = docker_client.networks.get("cloudbuild") cloudbuild_network.connect(db_container) db_container.reload() - slc_test_run_parameter.database_host = \ - db_container.attrs["NetworkSettings"]["Networks"][cloudbuild_network.name]["IPAddress"] - return SpawnedTestEnvironments(on_host_parameter, slc_test_run_parameter) + slc_test_run_parameter.database_host = db_container.attrs[ + "NetworkSettings" + ]["Networks"][cloudbuild_network.name]["IPAddress"] + return SpawnedTestEnvironments( + on_host_parameter, slc_test_run_parameter + ) else: return SpawnedTestEnvironments(on_host_parameter, None) - diff --git a/exasol_integration_test_docker_environment/testing/luigi_utils.py b/exasol_integration_test_docker_environment/testing/luigi_utils.py index aaa3d0293..9004c3b95 100644 --- a/exasol_integration_test_docker_environment/testing/luigi_utils.py +++ b/exasol_integration_test_docker_environment/testing/luigi_utils.py @@ -3,11 +3,15 @@ import luigi from exasol_integration_test_docker_environment.lib.api.common import generate_root_task -from exasol_integration_test_docker_environment.lib.docker.images.clean.clean_images import CleanImagesStartingWith +from exasol_integration_test_docker_environment.lib.docker.images.clean.clean_images import ( + CleanImagesStartingWith, +) def clean(docker_repository_name): - task = generate_root_task(task_class=CleanImagesStartingWith, starts_with_pattern=docker_repository_name) + task = generate_root_task( + task_class=CleanImagesStartingWith, starts_with_pattern=docker_repository_name + ) luigi.build([task], workers=1, local_scheduler=True, log_level="INFO") if task._get_tmp_path_for_job().exists(): shutil.rmtree(str(task._get_tmp_path_for_job())) diff --git a/exasol_integration_test_docker_environment/testing/spawned_test_environments.py b/exasol_integration_test_docker_environment/testing/spawned_test_environments.py index 5d2b22247..a23462317 100644 --- a/exasol_integration_test_docker_environment/testing/spawned_test_environments.py +++ b/exasol_integration_test_docker_environment/testing/spawned_test_environments.py @@ -1,12 +1,16 @@ from typing import Optional -from exasol_integration_test_docker_environment.testing.exaslct_docker_test_environment import \ - ExaslctDockerTestEnvironment +from exasol_integration_test_docker_environment.testing.exaslct_docker_test_environment import ( + ExaslctDockerTestEnvironment, +) class SpawnedTestEnvironments: - def __init__(self, on_host_environment: ExaslctDockerTestEnvironment, - slc_test_run_environment: Optional[ExaslctDockerTestEnvironment]): + def __init__( + self, + on_host_environment: ExaslctDockerTestEnvironment, + slc_test_run_environment: Optional[ExaslctDockerTestEnvironment], + ): self.on_host_docker_environment = on_host_environment self.slc_test_run_environment = slc_test_run_environment diff --git a/exasol_integration_test_docker_environment/testing/utils.py b/exasol_integration_test_docker_environment/testing/utils.py index f9e602641..c6e29bbfb 100644 --- a/exasol_integration_test_docker_environment/testing/utils.py +++ b/exasol_integration_test_docker_environment/testing/utils.py @@ -1,7 +1,11 @@ import json import os import unittest -from typing import Optional, List, Callable +from typing import ( + Callable, + List, + Optional, +) import requests @@ -35,5 +39,5 @@ def multiassert(assert_list: List[Callable], unit_test: unittest.TestCase): failure_log.append(f"\nFailure {len(failure_log)}: {str(e)}") if len(failure_log) != 0: - res_failure_log = '\n'.join(failure_log) + res_failure_log = "\n".join(failure_log) unit_test.fail(f"{len(failure_log)} failures within test.\n {res_failure_log}") diff --git a/noxfile.py b/noxfile.py index 29c59e176..e4cb40b31 100644 --- a/noxfile.py +++ b/noxfile.py @@ -98,30 +98,41 @@ def release(session: nox.Session): session.run("git", "tag", version) session.run("git", "push", "origin", version) + @nox.session(name="starter-scripts-checksums", python=False) def starter_scripts_checksums(session: nox.Session): start_script_dir = ROOT / "starter_scripts" with session.chdir(start_script_dir): for start_script_entry in start_script_dir.iterdir(): if start_script_entry.is_file(): - sha512 : str = session.run("sha512sum", start_script_entry.name, silent=True) # type: ignore - with open( start_script_dir /"checksums" / f"{start_script_entry.name}.sha512sum", "w") as f: + sha512: str = session.run("sha512sum", start_script_entry.name, silent=True) # type: ignore + with open( + start_script_dir + / "checksums" + / f"{start_script_entry.name}.sha512sum", + "w", + ) as f: f.write(sha512) session.run("git", "add", "starter_scripts/checksums") + @nox.session(name="copy-docker-db-config-templates", python=False) def copy_docker_db_config_templates(session: nox.Session): - target_path = ROOT / "exasol_integration_test_docker_environment" / "docker_db_config" + target_path = ( + ROOT / "exasol_integration_test_docker_environment" / "docker_db_config" + ) if target_path.is_dir(): shutil.rmtree(target_path) with session.chdir(ROOT): session.run("cp", "-rL", "docker_db_config_template", str(target_path)) session.run("git", "add", str(target_path)) + @nox.session(name="test:unit", python=False) def unit_tests(session: nox.Session) -> None: """Runs all unit tests""" from exasol.toolbox.nox._shared import _context from exasol.toolbox.nox._test import _unit_tests + context = _context(session, coverage=True) _unit_tests(session, PROJECT_CONFIG, context) diff --git a/pyproject.toml b/pyproject.toml index c2b170b6d..06d20cfd5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,3 +65,18 @@ ignore_missing_imports = true [tool.pylint.master] fail-under = 6.50 + +[tool.black] +line-length = 88 +verbose = false +include = "\\.pyi?$" + + +[tool.isort] +profile = "black" +force_grid_wrap = 2 + + +[tool.pylint.format] +max-line-length = 88 +max-module-lines = 800 \ No newline at end of file diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 3994bb89b..48273103d 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -1,19 +1,28 @@ import contextlib import io +from test.integration.helpers import normalize_request_name +from typing import ( + Any, + Callable, + ContextManager, + Dict, + Generator, + Iterator, + List, + NewType, + Optional, +) import pytest -from typing import Any, Callable, Dict, Iterator, List, NewType, Optional, Generator, ContextManager - -from test.integration.helpers import normalize_request_name from exasol_integration_test_docker_environment.testing import utils -from exasol_integration_test_docker_environment \ - .testing.api_test_environment import ApiTestEnvironment -from exasol_integration_test_docker_environment \ - .testing.exaslct_docker_test_environment import \ - ExaslctDockerTestEnvironment -from exasol_integration_test_docker_environment.testing \ - .exaslct_test_environment import ( +from exasol_integration_test_docker_environment.testing.api_test_environment import ( + ApiTestEnvironment, +) +from exasol_integration_test_docker_environment.testing.exaslct_docker_test_environment import ( + ExaslctDockerTestEnvironment, +) +from exasol_integration_test_docker_environment.testing.exaslct_test_environment import ( ExaslctTestEnvironment, SpawnedTestEnvironments, ) @@ -40,16 +49,18 @@ def api_isolation(request) -> Iterator[ApiTestEnvironment]: utils.close_environments(environment) -CliContextProvider = NewType( # type: ignore - "CliContextProvider", Callable[ - [Optional[str], Optional[List[str]]], - SpawnedTestEnvironments - ], +CliContextProvider = NewType( # type: ignore + "CliContextProvider", + Callable[[Optional[str], Optional[List[str]]], SpawnedTestEnvironments], ) @pytest.fixture -def cli_database(cli_isolation) -> Callable[[Optional[str], Optional[list[str]]], ContextManager[SpawnedTestEnvironments]]: +def cli_database( + cli_isolation, +) -> Callable[ + [Optional[str], Optional[list[str]]], ContextManager[SpawnedTestEnvironments] +]: """ Returns a method that test case implementations can use to create a context with a database. @@ -64,8 +75,8 @@ def test_case(database): @contextlib.contextmanager def create_context( - name: Optional[str] = None, - additional_parameters: Optional[List[str]] = None, + name: Optional[str] = None, + additional_parameters: Optional[List[str]] = None, ) -> Iterator[SpawnedTestEnvironments]: name = name if name else cli_isolation.name spawned = cli_isolation.spawn_docker_test_environments( @@ -78,12 +89,9 @@ def create_context( return create_context -ApiContextProvider = NewType( # type: ignore +ApiContextProvider = NewType( # type: ignore "ApiContextProvider", - Callable[ - [Optional[str], Optional[Dict[str, Any]]], - ExaslctDockerTestEnvironment - ], + Callable[[Optional[str], Optional[Dict[str, Any]]], ExaslctDockerTestEnvironment], ) @@ -91,8 +99,8 @@ def create_context( def api_database(api_isolation: ApiTestEnvironment) -> ApiContextProvider: @contextlib.contextmanager def create_context( - name: Optional[str] = None, - additional_parameters: Optional[Dict[str, Any]] = None, + name: Optional[str] = None, + additional_parameters: Optional[Dict[str, Any]] = None, ) -> Generator[ExaslctDockerTestEnvironment, None, None]: name = name if name else api_isolation.name spawned = api_isolation.spawn_docker_test_environment( @@ -102,7 +110,8 @@ def create_context( yield spawned utils.close_environments(spawned) - return create_context # type: ignore + return create_context # type: ignore + @pytest.fixture def fabric_stdin(monkeypatch): @@ -112,4 +121,4 @@ def fabric_stdin(monkeypatch): output is captured! Consider using ``-s``. See https://github.com/fabric/fabric/issues/2005 """ - monkeypatch.setattr('sys.stdin', io.StringIO('')) + monkeypatch.setattr("sys.stdin", io.StringIO("")) diff --git a/test/integration/helpers.py b/test/integration/helpers.py index 2baae5c4c..79c4edc83 100644 --- a/test/integration/helpers.py +++ b/test/integration/helpers.py @@ -1,20 +1,25 @@ import contextlib import re -from typing import Any, Union, cast +from typing import ( + Any, + Union, + cast, +) from unittest.mock import Mock -from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment \ - .lib.test_environment.parameter.docker_db_test_environment_parameter \ - import DbOsAccess from exasol_integration_test_docker_environment.lib.base.db_os_executor import ( - SshExecFactory, - DockerExecFactory, DbOsExecFactory, DockerClientFactory, + DockerExecFactory, + SshExecFactory, +) +from exasol_integration_test_docker_environment.lib.data.database_info import ( + DatabaseInfo, +) +from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient +from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter import ( + DbOsAccess, ) -from exasol_integration_test_docker_environment.lib.data.database_info \ - import DatabaseInfo def normalize_request_name(name: str): @@ -29,6 +34,7 @@ def exact_matcher(names): def superset_matcher(names): return lambda value: all(x in value for x in names) + @contextlib.contextmanager def container_named(*names, matcher=None): matcher = matcher if matcher else exact_matcher(names) diff --git a/test/integration/test_cli_environment.py b/test/integration/test_cli_environment.py index 29f28a067..ddc20bf74 100644 --- a/test/integration/test_cli_environment.py +++ b/test/integration/test_cli_environment.py @@ -1,21 +1,30 @@ +from inspect import cleandoc +from test.integration.helpers import get_executor_factory +from typing import ( + List, + Optional, +) + import docker import pytest -from inspect import cleandoc -from typing import List, Optional +from exasol_integration_test_docker_environment.lib.base.db_os_executor import ( + DbOsExecFactory, +) from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.testing.spawned_test_environments \ - import SpawnedTestEnvironments -from exasol_integration_test_docker_environment.lib.base.db_os_executor \ - import DbOsExecFactory -from exasol_integration_test_docker_environment.testing.exaslct_docker_test_environment \ - import ExaslctDockerTestEnvironment -from exasol_integration_test_docker_environment \ - .lib.test_environment.database_setup.find_exaplus_in_db_container \ - import find_exaplus -from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter \ - import DbOsAccess -from test.integration.helpers import get_executor_factory +from exasol_integration_test_docker_environment.lib.test_environment.database_setup.find_exaplus_in_db_container import ( + find_exaplus, +) +from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter import ( + DbOsAccess, +) +from exasol_integration_test_docker_environment.testing.exaslct_docker_test_environment import ( + ExaslctDockerTestEnvironment, +) +from exasol_integration_test_docker_environment.testing.spawned_test_environments import ( + SpawnedTestEnvironments, +) + class NumberCheck: def __init__(self, db: SpawnedTestEnvironments, all: List[str]): @@ -28,13 +37,8 @@ def count(self, selected: Optional[List[str]] = None): @property def log(self) -> str: assert self.db.on_host_docker_environment.completed_process - return ( - self - .db - .on_host_docker_environment - .completed_process - .stdout - .decode('utf8') + return self.db.on_host_docker_environment.completed_process.stdout.decode( + "utf8" ) def fail(self, prefix) -> str: @@ -53,11 +57,14 @@ def quote(s): assert env.environment_info db_info = env.environment_info.database_info - command : List[str] = [ + command: List[str] = [ str(exaplus_path), - "-c", quote(f"{db_info.host}:{db_info.ports.database}"), - "-u", quote(env.db_username), - "-p", quote(env.db_password), + "-c", + quote(f"{db_info.host}:{db_info.ports.database}"), + "-u", + quote(env.db_username), + "-p", + quote(env.db_password), ] command += [ "-sql", @@ -73,9 +80,11 @@ def test_db_container_started(cli_database): with cli_database() as db: with ContextDockerClient() as docker_client: name = db.on_host_docker_environment.name - containers = [c.name for c in docker_client.containers.list() if name in c.name] + containers = [ + c.name for c in docker_client.containers.list() if name in c.name + ] check = NumberCheck(db, containers) - assert check.count() ==1, check.fail("Not exactly 1 container") + assert check.count() == 1, check.fail("Not exactly 1 container") db_containers = [c for c in containers if "db_container" in c] check = NumberCheck(db, containers) @@ -84,7 +93,7 @@ def test_db_container_started(cli_database): @pytest.mark.parametrize("db_os_access", [DbOsAccess.DOCKER_EXEC, DbOsAccess.SSH]) def test_db_available(cli_database, fabric_stdin, db_os_access): - params = [ "--db-os-access", db_os_access.name ] + params = ["--db-os-access", db_os_access.name] with cli_database(additional_parameters=params) as db: with ContextDockerClient() as docker_client: dbinfo = db.on_host_docker_environment.environment_info.database_info @@ -96,5 +105,6 @@ def test_db_available(cli_database, fabric_stdin, db_os_access): exaplus = find_exaplus(db_container, executor) command = smoke_test_sql(exaplus, db.on_host_docker_environment) exit_code, output = db_container.exec_run(command) - assert exit_code == 0, \ - f"Error while executing 'exaplus' in test container. Got output:\n {output}" + assert ( + exit_code == 0 + ), f"Error while executing 'exaplus' in test container. Got output:\n {output}" diff --git a/test/integration/test_db_container_log_thread.py b/test/integration/test_db_container_log_thread.py index f398b9fb7..8f61a755f 100644 --- a/test/integration/test_db_container_log_thread.py +++ b/test/integration/test_db_container_log_thread.py @@ -2,13 +2,18 @@ import tempfile import time from pathlib import Path -from typing import List, Optional +from typing import ( + List, + Optional, +) import pytest from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol_integration_test_docker_environment.lib.test_environment.database_waiters.db_container_log_thread import \ - DBContainerLogThread +from exasol_integration_test_docker_environment.lib.test_environment.database_waiters.db_container_log_thread import ( + DBContainerLogThread, +) + def _build_docker_command(logs: List[str]): """ @@ -24,6 +29,7 @@ def _build_docker_command(logs: List[str]): bash_command = f"while true; do {echo_commdands_str}; sleep 0.1; done" return ["bash", "-c", bash_command] + def _run_container_log_thread(logger, logs: List[str]) -> Optional[str]: """ Starts a dummy docker container which prints logs in an endless loop, and calls DBContainerLogThread on that container. @@ -33,8 +39,12 @@ def _run_container_log_thread(logger, logs: List[str]) -> Optional[str]: with tempfile.TemporaryDirectory() as tmpDir: with ContextDockerClient(timeout=3600) as client: try: - container = client.containers.run("ubuntu", _build_docker_command(logs), detach=True) - thread = DBContainerLogThread(container, logger, Path(tmpDir) / "log.txt", "test") + container = client.containers.run( + "ubuntu", _build_docker_command(logs), detach=True + ) + thread = DBContainerLogThread( + container, logger, Path(tmpDir) / "log.txt", "test" + ) thread.start() time.sleep(2) thread.stop() @@ -48,20 +58,27 @@ def _run_container_log_thread(logger, logs: List[str]) -> Optional[str]: def test_logger(): return logging.Logger(__name__) + def test_container_log_thread_no_error(test_logger) -> None: """ Integration test which verifies that the DBContainerLogThread returns no error message if no error is logged. """ - error_message = _run_container_log_thread(test_logger, ["test", "something", "db started"]) + error_message = _run_container_log_thread( + test_logger, ["test", "something", "db started"] + ) assert error_message is None + def test_container_log_thread_error(test_logger) -> None: """ Integration test which verifies that the DBContainerLogThread returns error message if error is logged. """ - error_message = _run_container_log_thread(test_logger, ["confd returned with state 1"]) + error_message = _run_container_log_thread( + test_logger, ["confd returned with state 1"] + ) assert error_message and "confd returned with state 1\n" in error_message + def test_container_log_thread_ignore_rsyslogd(test_logger) -> None: """ Integration test which verifies that the DBContainerLogThread returns no error message if rsyslogd crashes. @@ -69,11 +86,12 @@ def test_container_log_thread_ignore_rsyslogd(test_logger) -> None: rsys_logd_logs = [ "[2024-09-17 14:12:20.335085 +00:00] child 58687 (Part:9 Node:0 rsyslogd) returned with state 1.", "[2024-09-17 14:12:20.336886 +00:00] Started /sbin/rsyslogd with PID:58688 UID:0 GID:0 Part:9 Node:0", - "[2024-09-17 14:12:20.336967 +00:00] 30 auto-restarted processes exited in the last 0 seconds. Starting to delay process death handling." + "[2024-09-17 14:12:20.336967 +00:00] 30 auto-restarted processes exited in the last 0 seconds. Starting to delay process death handling.", ] error_message = _run_container_log_thread(test_logger, rsys_logd_logs) assert error_message is None + def test_container_log_thread_ignore_sshd(test_logger) -> None: """ Integration test which verifies that the DBContainerLogThread returns no error message if sshd crashes. @@ -84,6 +102,7 @@ def test_container_log_thread_ignore_sshd(test_logger) -> None: error_message = _run_container_log_thread(test_logger, sshd_logs) assert error_message is None + def test_container_log_thread_exception(test_logger) -> None: """ Integration test which verifies that the DBContainerLogThread returns an error message if an exception was thrown. @@ -91,9 +110,9 @@ def test_container_log_thread_exception(test_logger) -> None: sshd_logs = [ "Traceback (most recent call last):", 'File "/opt/cos/something.py", line 364, in runcode', - ' coro = func()', + " coro = func()", ' File "", line 1, in ', - 'Exception: bad thing happend' + "Exception: bad thing happend", ] error_message = _run_container_log_thread(test_logger, sshd_logs) assert error_message and "exception: bad thing happend\n" in error_message diff --git a/test/integration/test_ssh_access.py b/test/integration/test_ssh_access.py index a3a20b11a..fdef54ce1 100644 --- a/test/integration/test_ssh_access.py +++ b/test/integration/test_ssh_access.py @@ -1,35 +1,39 @@ import contextlib -import docker -from docker.models.containers import Container as DockerContainer -import fabric import os -import pytest import time - -from exasol_integration_test_docker_environment.lib.test_environment.ports import ( - find_free_ports, - Ports, -) -from exasol_integration_test_docker_environment.lib.data.ssh_info \ - import SshInfo -from exasol_integration_test_docker_environment.lib.data.database_info \ - import DatabaseInfo -from exasol_integration_test_docker_environment \ - .lib.test_environment.parameter.docker_db_test_environment_parameter \ - import DbOsAccess -from exasol_integration_test_docker_environment.lib.data.container_info \ - import ContainerInfo - -from exasol_integration_test_docker_environment.lib.base.ssh_access import SshKey, SshKeyCache from test.integration.helpers import ( container_named, get_executor_factory, normalize_request_name, ) +import docker +import fabric +import pytest +from docker.models.containers import Container as DockerContainer + +from exasol_integration_test_docker_environment.lib.base.ssh_access import ( + SshKey, + SshKeyCache, +) +from exasol_integration_test_docker_environment.lib.data.container_info import ( + ContainerInfo, +) +from exasol_integration_test_docker_environment.lib.data.database_info import ( + DatabaseInfo, +) +from exasol_integration_test_docker_environment.lib.data.ssh_info import SshInfo +from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter import ( + DbOsAccess, +) +from exasol_integration_test_docker_environment.lib.test_environment.ports import ( + Ports, + find_free_ports, +) + def test_generate_ssh_key_file(api_database): - params = { "db_os_access": "SSH" } + params = {"db_os_access": "SSH"} with api_database(additional_parameters=params) as db: cache = SshKeyCache() container_name = db.environment_info.database_info.container_info.container_name @@ -40,7 +44,7 @@ def test_generate_ssh_key_file(api_database): def test_ssh_access(api_database, fabric_stdin): - params = { "db_os_access": "SSH" } + params = {"db_os_access": "SSH"} with api_database(additional_parameters=params) as db: container_name = db.environment_info.database_info.container_info.container_name with container_named(container_name) as container: @@ -48,26 +52,24 @@ def test_ssh_access(api_database, fabric_stdin): key = SshKey.from_cache() result = fabric.Connection( f"root@localhost:{db.ports.ssh}", - connect_kwargs={ "pkey": key.private }, - ).run('ls /exa/etc/EXAConf') + connect_kwargs={"pkey": key.private}, + ).run("ls /exa/etc/EXAConf") assert result.stdout == "/exa/etc/EXAConf\n" @pytest.fixture def sshd_container(request): testname = normalize_request_name(request.node.name) + @contextlib.contextmanager - def create_context( - ssh_port_forward: int, - public_key: str - ) -> DockerContainer: + def create_context(ssh_port_forward: int, public_key: str) -> DockerContainer: client = docker.from_env() container = client.containers.run( name=testname, image="linuxserver/openssh-server:9.3_p2-r0-ls123", detach=True, - ports={ '2222/tcp': ssh_port_forward }, - environment={ "PUBLIC_KEY": public_key }, + ports={"2222/tcp": ssh_port_forward}, + environment={"PUBLIC_KEY": public_key}, ) try: yield container @@ -105,5 +107,5 @@ def database_info(container_name, ssh_port_forward): factory = get_executor_factory(dbinfo, ssh_port_forward) with factory.executor() as executor: exit_code, output = executor.exec("ls /keygen.sh") - output = output.decode('utf-8').strip() + output = output.decode("utf-8").strip() assert (exit_code, output) == (0, "/keygen.sh") diff --git a/test/integration/test_udf_execution.py b/test/integration/test_udf_execution.py index 875ae639a..e0a202095 100644 --- a/test/integration/test_udf_execution.py +++ b/test/integration/test_udf_execution.py @@ -1,9 +1,11 @@ -import pyexasol - from inspect import cleandoc from time import sleep -from exasol_integration_test_docker_environment.lib.test_environment.db_version import DbVersion +import pyexasol + +from exasol_integration_test_docker_environment.lib.test_environment.db_version import ( + DbVersion, +) def test_udf_execution(api_database): diff --git a/test/unit/test_base_ssh_keys.py b/test/unit/test_base_ssh_keys.py index 8cd9804b7..d7a80adbc 100644 --- a/test/unit/test_base_ssh_keys.py +++ b/test/unit/test_base_ssh_keys.py @@ -1,9 +1,13 @@ import os import platform +from pathlib import Path + import pytest -from pathlib import Path -from exasol_integration_test_docker_environment.lib.base.ssh_access import SshKeyCache, SshKey +from exasol_integration_test_docker_environment.lib.base.ssh_access import ( + SshKey, + SshKeyCache, +) def test_create_file_and_permissions(tmp_path): diff --git a/test/unit/test_db_os_executor.py b/test/unit/test_db_os_executor.py index 3327c9e3a..1b30d4d60 100644 --- a/test/unit/test_db_os_executor.py +++ b/test/unit/test_db_os_executor.py @@ -1,31 +1,32 @@ +from test.integration.helpers import mock_cast from unittest.mock import ( MagicMock, - create_autospec, call, + create_autospec, ) + from docker import DockerClient from docker.models.containers import Container as DockerContainer -from exasol_integration_test_docker_environment \ - .lib.base.db_os_executor import ( - DockerClientFactory, - DbOsExecutor, - DockerExecutor, - SshExecutor, - DockerExecFactory, - SshExecFactory, +from exasol_integration_test_docker_environment.lib.base.db_os_executor import ( + DbOsExecutor, + DockerClientFactory, + DockerExecFactory, + DockerExecutor, + SshExecFactory, + SshExecutor, +) +from exasol_integration_test_docker_environment.lib.data.database_info import ( + DatabaseInfo, ) -from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports from exasol_integration_test_docker_environment.lib.data.ssh_info import SshInfo -from exasol_integration_test_docker_environment.lib.data.database_info import DatabaseInfo -from exasol_integration_test_docker_environment.lib.docker \ - import ContextDockerClient -from test.integration.helpers import mock_cast +from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient +from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports def test_executor_closes_client(): container = create_autospec(DockerContainer) - client:Union[MagicMock, DockerClient] = create_autospec(DockerClient) + client: Union[MagicMock, DockerClient] = create_autospec(DockerClient) client.containers.get = MagicMock(return_value=container) with DockerExecutor(client, "container_name") as executor: executor.exec("sample command") @@ -37,16 +38,14 @@ def test_executor_closes_client(): def test_ssh_exec_factory(): factory = SshExecFactory("connect_string", "ssh_key_file") executor = factory.executor() - assert isinstance(executor, DbOsExecutor) \ - and type(executor) is SshExecutor + assert isinstance(executor, DbOsExecutor) and type(executor) is SshExecutor def test_docker_exec_factory(): client_factory = create_autospec(DockerClientFactory) factory = DockerExecFactory("container_name", client_factory) executor = factory.executor() - assert isinstance(executor, DbOsExecutor) \ - and type(executor) is DockerExecutor + assert isinstance(executor, DbOsExecutor) and type(executor) is DockerExecutor def test_docker_client_factory_usage(): @@ -55,12 +54,13 @@ def test_docker_client_factory_usage(): factory.client = MagicMock(return_value=client) testee = DockerExecFactory("container_name", factory) executor = testee.executor() - assert executor._client == client \ - and mock_cast(factory.client).mock_calls == [call()] + assert executor._client == client and mock_cast(factory.client).mock_calls == [ + call() + ] def test_ssh_exec_factory_from_database_info(): - ports = Ports(1,2,3) + ports = Ports(1, 2, 3) ssh_info = SshInfo("my_user", "my_key_file") dbinfo = DatabaseInfo( "my_host", diff --git a/test/unit/test_env_variable.py b/test/unit/test_env_variable.py index d536123dc..9ec686e92 100644 --- a/test/unit/test_env_variable.py +++ b/test/unit/test_env_variable.py @@ -1,14 +1,24 @@ -import pytest import os.path from pathlib import Path from unittest import mock import luigi +import pytest -from exasol_integration_test_docker_environment.lib.base.luigi_log_config import LOG_ENV_VARIABLE_NAME, get_log_path -from exasol_integration_test_docker_environment.lib.api.common import generate_root_task, run_task -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask -from exasol_integration_test_docker_environment.lib.config.build_config import build_config +from exasol_integration_test_docker_environment.lib.api.common import ( + generate_root_task, + run_task, +) +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) +from exasol_integration_test_docker_environment.lib.base.luigi_log_config import ( + LOG_ENV_VARIABLE_NAME, + get_log_path, +) +from exasol_integration_test_docker_environment.lib.config.build_config import ( + build_config, +) @pytest.fixture @@ -31,8 +41,10 @@ def __init__(self, task): self.task_input_parameter = task["in_parameter"] def __repr__(self): - return f"log path: {str(self.log_path)}" \ - f"\nlog content: '{self.log_path.read_text()}'" + return ( + f"log path: {str(self.log_path)}" + f"\nlog content: '{self.log_path.read_text()}'" + ) class LogPathCorrectnessMatcher: @@ -43,19 +55,23 @@ def __init__(self, expected_log_path: Path): def __eq__(self, used_log_path: object): if not isinstance(used_log_path, UsedLogPath): - return False + return False log_path = used_log_path.log_path - if not (log_path == self.expected_log_path - and log_path.exists() - and log_path.is_file()): + if not ( + log_path == self.expected_log_path + and log_path.exists() + and log_path.is_file() + ): return False log_content = log_path.read_text() return f"Logging: {used_log_path.task_input_parameter}" in log_content def __repr__(self): - return f"log path: {str(self.expected_log_path)}" \ - f"\nlog content: '.*Logging: {self.task_input_parameter}.*'" + return ( + f"log path: {str(self.expected_log_path)}" + f"\nlog content: '.*Logging: {self.task_input_parameter}.*'" + ) def default_log_path(job_id): @@ -79,10 +95,17 @@ def create_task(): return generate_root_task(task_class=TestTask, x=f"{next(task_id_generator)}") for j in range(NUMBER_TASK): - output = run_task(create_task, workers=5, task_dependencies_dot_file=None, use_job_specific_log_file=True) + output = run_task( + create_task, + workers=5, + task_dependencies_dot_file=None, + use_job_specific_log_file=True, + ) jobid = output["job_id"] log_path = get_log_path(jobid) - tasks.append({"jobid": jobid, "log_path": log_path, "in_parameter": output["parameter"]}) + tasks.append( + {"jobid": jobid, "log_path": log_path, "in_parameter": output["parameter"]} + ) return tasks @@ -90,7 +113,12 @@ def create_task(): def use_specific_log_file(task_creator, temp_dir, test_name): log_path = Path(temp_dir) / (test_name + ".log") os.environ[LOG_ENV_VARIABLE_NAME] = str(log_path) - run_task(task_creator, workers=5, task_dependencies_dot_file=None, use_job_specific_log_file=True) + run_task( + task_creator, + workers=5, + task_dependencies_dot_file=None, + use_job_specific_log_file=True, + ) return log_path @@ -150,7 +178,7 @@ def test_preexisting_custom_log_file(set_tempdir, mock_settings_env_vars): log_path = UsedLogPath(tasks[0]) assert log_path == log_path_matcher - with open(custom_log_path, "r") as f: + with open(custom_log_path) as f: log_content = f.read() assert file_content in log_content @@ -183,7 +211,7 @@ def test_different_custom_logging_file(set_tempdir, mock_settings_env_vars): for log_path in [log_path_1, log_path_2]: assert log_path.exists() - with open(log_path, "r") as f: + with open(log_path) as f: log_content = f.read() assert f"Logging: Test" in log_content diff --git a/test/unit/test_shell_variables.py b/test/unit/test_shell_variables.py index 80649eb2c..92b9eaa3a 100644 --- a/test/unit/test_shell_variables.py +++ b/test/unit/test_shell_variables.py @@ -1,12 +1,13 @@ import contextlib -import pytest - from inspect import cleandoc from unittest.mock import Mock -from exasol_integration_test_docker_environment \ - .lib.test_environment.shell_variables import ShellVariables -from exasol_integration_test_docker_environment \ - .lib.test_environment.ports import Ports + +import pytest + +from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports +from exasol_integration_test_docker_environment.lib.test_environment.shell_variables import ( + ShellVariables, +) def test_render_with_prefix(): @@ -16,25 +17,25 @@ def test_render_with_prefix(): def test_from_test_environment_info(): container_info = Mock( - network_aliases = ["cna-1", "cna-2"], - container_name = "container-name", - ip_address = "container-ip", - volume_name = "container-volume", + network_aliases=["cna-1", "cna-2"], + container_name="container-name", + ip_address="container-ip", + volume_name="container-volume", ) database_info = Mock( - host = "db-host", - ports = Ports(1,2,3), - container_info = container_info, + host="db-host", + ports=Ports(1, 2, 3), + container_info=container_info, ) test_container_info = Mock( - container_name = "test-container-name", - network_aliases = ["tcna-1", "tcna-2"], - ip_address = "tc-ip", + container_name="test-container-name", + network_aliases=["tcna-1", "tcna-2"], + ip_address="tc-ip", ) test_environment = Mock( - type = "type", - database_info = database_info, - test_container_info = test_container_info, + type="type", + database_info=database_info, + test_container_info=test_container_info, ) test_environment.name = "name" @@ -42,7 +43,8 @@ def test_from_test_environment_info(): "ip-address", test_environment, ) - assert actual.render().strip() == cleandoc(""" + assert actual.render().strip() == cleandoc( + """ ITDE_NAME=name ITDE_TYPE=type ITDE_DATABASE_HOST=db-host @@ -57,4 +59,5 @@ def test_from_test_environment_info(): ITDE_TEST_CONTAINER_NAME=test-container-name ITDE_TEST_CONTAINER_NETWORK_ALIASES="tcna-1 tcna-2" ITDE_TEST_CONTAINER_IP_ADDRESS=tc-ip - """) + """ + ) diff --git a/test/unit/test_task_runtime_error.py b/test/unit/test_task_runtime_error.py index 1433833dc..75dff0381 100644 --- a/test/unit/test_task_runtime_error.py +++ b/test/unit/test_task_runtime_error.py @@ -1,23 +1,30 @@ import re import time import traceback +from test.matchers import regex_matcher import pytest from _pytest._code import ExceptionInfo from joblib.testing import fixture -from exasol_integration_test_docker_environment.lib.api.api_errors import TaskRuntimeError, TaskFailures -from exasol_integration_test_docker_environment.lib.api.common import run_task, generate_root_task -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask -from test.matchers import regex_matcher +from exasol_integration_test_docker_environment.lib.api.api_errors import ( + TaskFailures, + TaskRuntimeError, +) +from exasol_integration_test_docker_environment.lib.api.common import ( + generate_root_task, + run_task, +) +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) class CompositeFailingTask(DependencyLoggerBaseTask): def register_required(self): self.dependencies = self.register_dependencies( - [self.create_child_task(FailingTask1), - self.create_child_task(FailingTask2)] + [self.create_child_task(FailingTask1), self.create_child_task(FailingTask2)] ) return self.dependencies @@ -56,15 +63,19 @@ def exception_from_sut(self) -> TaskRuntimeError: def task_creator(): return generate_root_task(task_class=SingleFailingTask) - with pytest.raises(TaskRuntimeError, - match=r"Task SingleFailingTask.* \(or any of it's subtasks\) failed\.") as raises: + with pytest.raises( + TaskRuntimeError, + match=r"Task SingleFailingTask.* \(or any of it's subtasks\) failed\.", + ) as raises: run_task(task_creator=task_creator, workers=3) return raises.value @fixture def formatted_cause(self, exception_from_sut): cause = exception_from_sut.__cause__ - formatted_cause = "".join(traceback.format_exception(type(cause), cause, cause.__traceback__)) + formatted_cause = "".join( + traceback.format_exception(type(cause), cause, cause.__traceback__) + ) return formatted_cause def test_cause_is_task_failures_instance(self, exception_from_sut): @@ -73,13 +84,16 @@ def test_cause_is_task_failures_instance(self, exception_from_sut): def test_tasks_failures(self, formatted_cause): assert formatted_cause == regex_matcher( - ".*Following task failures were caught during the execution:") + ".*Following task failures were caught during the execution:" + ) def test_task_in_list_of_failures(self, formatted_cause): assert formatted_cause == regex_matcher(r".*- SingleFailingTask.*:", re.DOTALL) def test_task_error(self, formatted_cause): - assert formatted_cause == regex_matcher(r".*RuntimeError: Error in SingleFailingTask occurred\..*", re.DOTALL) + assert formatted_cause == regex_matcher( + r".*RuntimeError: Error in SingleFailingTask occurred\..*", re.DOTALL + ) class TestMultipleTaskFailure: @@ -89,15 +103,19 @@ def exception_from_sut(self) -> TaskRuntimeError: def task_creator(): return generate_root_task(task_class=CompositeFailingTask) - with pytest.raises(TaskRuntimeError, - match=r"Task CompositeFailingTask.* \(or any of it's subtasks\) failed\.") as raises: + with pytest.raises( + TaskRuntimeError, + match=r"Task CompositeFailingTask.* \(or any of it's subtasks\) failed\.", + ) as raises: run_task(task_creator=task_creator, workers=3) return raises.value @fixture def formatted_cause(self, exception_from_sut): cause = exception_from_sut.__cause__ - formatted_cause = "".join(traceback.format_exception(type(cause), cause, cause.__traceback__)) + formatted_cause = "".join( + traceback.format_exception(type(cause), cause, cause.__traceback__) + ) return formatted_cause def test_cause_is_task_failures_instance(self, exception_from_sut): @@ -106,7 +124,8 @@ def test_cause_is_task_failures_instance(self, exception_from_sut): def test_tasks_failures(self, formatted_cause): assert formatted_cause == regex_matcher( - ".*Following task failures were caught during the execution:") + ".*Following task failures were caught during the execution:" + ) def test_sub_task1_in_list_of_failures(self, formatted_cause): assert formatted_cause == regex_matcher(r".*- FailingTask1.*:", re.DOTALL) @@ -115,7 +134,11 @@ def test_sub_task2_in_list_of_failures(self, formatted_cause): assert formatted_cause == regex_matcher(r".*- FailingTask2.*:", re.DOTALL) def test_sub_task1_error(self, formatted_cause): - assert formatted_cause == regex_matcher(r".*RuntimeError: Error in FailingTask1 occurred\..*", re.DOTALL) + assert formatted_cause == regex_matcher( + r".*RuntimeError: Error in FailingTask1 occurred\..*", re.DOTALL + ) def test_sub_task2_error(self, formatted_cause): - assert formatted_cause == regex_matcher(r".*RuntimeError: Error in FailingTask2 occurred\..*", re.DOTALL) + assert formatted_cause == regex_matcher( + r".*RuntimeError: Error in FailingTask2 occurred\..*", re.DOTALL + ) diff --git a/test/unit/test_termination_handler.py b/test/unit/test_termination_handler.py index 2b87ca4c3..e1fd9db81 100644 --- a/test/unit/test_termination_handler.py +++ b/test/unit/test_termination_handler.py @@ -2,22 +2,32 @@ import io import re import time -from contextlib import redirect_stdout, redirect_stderr, ExitStack +from contextlib import ( + ExitStack, + redirect_stderr, + redirect_stdout, +) +from test.matchers import regex_matcher import pytest -from exasol_integration_test_docker_environment.cli.termination_handler import TerminationHandler -from exasol_integration_test_docker_environment.lib.api.common import generate_root_task, run_task -from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask -from test.matchers import regex_matcher +from exasol_integration_test_docker_environment.cli.termination_handler import ( + TerminationHandler, +) +from exasol_integration_test_docker_environment.lib.api.common import ( + generate_root_task, + run_task, +) +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) class CompositeFailingTask(DependencyLoggerBaseTask): def register_required(self): self.dependencies = self.register_dependencies( - [self.create_child_task(FailingTask1), - self.create_child_task(FailingTask2)] + [self.create_child_task(FailingTask1), self.create_child_task(FailingTask2)] ) return self.dependencies @@ -69,27 +79,34 @@ def task_creator(): def test_command_runtime(capture_output_of_test): - assert capture_output_of_test.err == regex_matcher("The command failed after .* s with:") + assert capture_output_of_test.err == regex_matcher( + "The command failed after .* s with:" + ) def test_composite_task_failed(capture_output_of_test): assert capture_output_of_test.err == regex_matcher( - r".*Task failure message: Task CompositeFailingTask.* \(or any of it's subtasks\) failed\.", re.DOTALL) + r".*Task failure message: Task CompositeFailingTask.* \(or any of it's subtasks\) failed\.", + re.DOTALL, + ) def test_sub_tasks_failed(capture_output_of_test): assert capture_output_of_test.err == regex_matcher( - r".*Following task failures were caught during the execution:", re.DOTALL) + r".*Following task failures were caught during the execution:", re.DOTALL + ) def test_sub_task1_error(capture_output_of_test): assert capture_output_of_test.err == regex_matcher( - r".*RuntimeError: Error in FailingTask1 occurred.", re.DOTALL) + r".*RuntimeError: Error in FailingTask1 occurred.", re.DOTALL + ) def test_sub_task2_error(capture_output_of_test): assert capture_output_of_test.err == regex_matcher( - r".*RuntimeError: Error in FailingTask2 occurred.", re.DOTALL) + r".*RuntimeError: Error in FailingTask2 occurred.", re.DOTALL + ) def test_out_empty(capture_output_of_test):