From 31e86c60687870ea3a825640fbd9e511d5c9f00f Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 12 Jun 2023 09:41:41 +0200 Subject: [PATCH 1/8] Added shallow abstraction layer for boto which only implements what is needed by this project - Moved AwcAccess under module aws --- .../cli/commands/deploy_ci_build.py | 2 +- .../cli/commands/deploy_release_build.py | 2 +- .../cli/commands/deploy_source_credentials.py | 2 +- .../cli/commands/start_ci_build.py | 3 +- .../cli/commands/start_release_build.py | 2 +- .../cli/commands/start_test_release_build.py | 2 +- .../cli/commands/validate_ci_build.py | 5 +- .../cli/commands/validate_release_build.py | 2 +- .../commands/validate_source_credentials.py | 2 +- .../lib/aws/__init__.py | 0 .../lib/{ => aws}/aws_access.py | 37 +++---- .../lib/{ => aws}/deployer.py | 0 .../lib/aws/wrapper/__init__.py | 0 .../lib/aws/wrapper/aws_client.py | 50 ++++++++++ .../lib/aws/wrapper/cloudformation_service.py | 35 +++++++ .../lib/aws/wrapper/codebuild_service.py | 36 +++++++ .../lib/aws/wrapper/datamodels/__init__.py | 0 .../aws/wrapper/datamodels/cloudformation.py | 61 ++++++++++++ .../lib/aws/wrapper/datamodels/codebuild.py | 20 ++++ .../lib/aws/wrapper/datamodels/common.py | 12 +++ .../aws/wrapper/datamodels/secretsmanager.py | 20 ++++ .../lib/aws/wrapper/secretsmanager_service.py | 22 +++++ .../lib/ci_build.py | 2 +- .../lib/release_build.py | 2 +- .../lib/run_start_build.py | 2 +- .../lib/source_credentials.py | 2 +- test/test_deploy_ci.py | 1 - test/unit_tests/aws/__init__.py | 0 test/unit_tests/aws/wrapper/__init__.py | 0 .../aws/wrapper/datamodels/__init__.py | 0 .../datamodels/cloudformation/__init__.py | 0 .../test_list_stack_resources_result.py | 94 ++++++++++++++++++ .../test_stack_resource_summary.py | 46 +++++++++ .../cloudformation/test_validation_result.py | 16 +++ .../wrapper/datamodels/codebuild/__init__.py | 0 .../datamodels/codebuild/test_build_batch.py | 34 +++++++ .../datamodels/secretsmanager/__init__.py | 0 .../datamodels/secretsmanager/test_secret.py | 36 +++++++ .../unit_tests/aws/wrapper/test_aws_client.py | 98 +++++++++++++++++++ .../wrapper/test_cloudformation_service.py | 83 ++++++++++++++++ .../aws/wrapper/test_codebuild_service.py | 79 +++++++++++++++ .../wrapper/test_secretsmanager_service.py | 40 ++++++++ 42 files changed, 816 insertions(+), 34 deletions(-) create mode 100644 exasol_script_languages_container_ci_setup/lib/aws/__init__.py rename exasol_script_languages_container_ci_setup/lib/{ => aws}/aws_access.py (86%) rename exasol_script_languages_container_ci_setup/lib/{ => aws}/deployer.py (100%) create mode 100644 exasol_script_languages_container_ci_setup/lib/aws/wrapper/__init__.py create mode 100644 exasol_script_languages_container_ci_setup/lib/aws/wrapper/aws_client.py create mode 100644 exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py create mode 100644 exasol_script_languages_container_ci_setup/lib/aws/wrapper/codebuild_service.py create mode 100644 exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/__init__.py create mode 100644 exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/cloudformation.py create mode 100644 exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/codebuild.py create mode 100644 exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/common.py create mode 100644 exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/secretsmanager.py create mode 100644 exasol_script_languages_container_ci_setup/lib/aws/wrapper/secretsmanager_service.py create mode 100644 test/unit_tests/aws/__init__.py create mode 100644 test/unit_tests/aws/wrapper/__init__.py create mode 100644 test/unit_tests/aws/wrapper/datamodels/__init__.py create mode 100644 test/unit_tests/aws/wrapper/datamodels/cloudformation/__init__.py create mode 100644 test/unit_tests/aws/wrapper/datamodels/cloudformation/test_list_stack_resources_result.py create mode 100644 test/unit_tests/aws/wrapper/datamodels/cloudformation/test_stack_resource_summary.py create mode 100644 test/unit_tests/aws/wrapper/datamodels/cloudformation/test_validation_result.py create mode 100644 test/unit_tests/aws/wrapper/datamodels/codebuild/__init__.py create mode 100644 test/unit_tests/aws/wrapper/datamodels/codebuild/test_build_batch.py create mode 100644 test/unit_tests/aws/wrapper/datamodels/secretsmanager/__init__.py create mode 100644 test/unit_tests/aws/wrapper/datamodels/secretsmanager/test_secret.py create mode 100644 test/unit_tests/aws/wrapper/test_aws_client.py create mode 100644 test/unit_tests/aws/wrapper/test_cloudformation_service.py create mode 100644 test/unit_tests/aws/wrapper/test_codebuild_service.py create mode 100644 test/unit_tests/aws/wrapper/test_secretsmanager_service.py diff --git a/exasol_script_languages_container_ci_setup/cli/commands/deploy_ci_build.py b/exasol_script_languages_container_ci_setup/cli/commands/deploy_ci_build.py index 0e12895..4bbb0c0 100644 --- a/exasol_script_languages_container_ci_setup/cli/commands/deploy_ci_build.py +++ b/exasol_script_languages_container_ci_setup/cli/commands/deploy_ci_build.py @@ -7,7 +7,7 @@ from exasol_script_languages_container_ci_setup.cli.cli import cli from exasol_script_languages_container_ci_setup.cli.common import add_options from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.lib.ci_build import run_deploy_ci_build from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options diff --git a/exasol_script_languages_container_ci_setup/cli/commands/deploy_release_build.py b/exasol_script_languages_container_ci_setup/cli/commands/deploy_release_build.py index 51d1d51..da34260 100644 --- a/exasol_script_languages_container_ci_setup/cli/commands/deploy_release_build.py +++ b/exasol_script_languages_container_ci_setup/cli/commands/deploy_release_build.py @@ -7,7 +7,7 @@ from exasol_script_languages_container_ci_setup.cli.cli import cli from exasol_script_languages_container_ci_setup.cli.common import add_options from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options from exasol_script_languages_container_ci_setup.lib.release_build import run_deploy_release_build diff --git a/exasol_script_languages_container_ci_setup/cli/commands/deploy_source_credentials.py b/exasol_script_languages_container_ci_setup/cli/commands/deploy_source_credentials.py index 280af23..faec467 100644 --- a/exasol_script_languages_container_ci_setup/cli/commands/deploy_source_credentials.py +++ b/exasol_script_languages_container_ci_setup/cli/commands/deploy_source_credentials.py @@ -7,7 +7,7 @@ from exasol_script_languages_container_ci_setup.cli.cli import cli from exasol_script_languages_container_ci_setup.cli.common import add_options from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.lib.source_credentials import run_deploy_source_credentials from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options diff --git a/exasol_script_languages_container_ci_setup/cli/commands/start_ci_build.py b/exasol_script_languages_container_ci_setup/cli/commands/start_ci_build.py index 404fd71..5909d38 100644 --- a/exasol_script_languages_container_ci_setup/cli/commands/start_ci_build.py +++ b/exasol_script_languages_container_ci_setup/cli/commands/start_ci_build.py @@ -1,4 +1,3 @@ -import os from typing import Optional import click @@ -6,7 +5,7 @@ from exasol_script_languages_container_ci_setup.cli.cli import cli from exasol_script_languages_container_ci_setup.cli.common import add_options from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options from exasol_script_languages_container_ci_setup.lib.run_start_build import run_start_ci_build diff --git a/exasol_script_languages_container_ci_setup/cli/commands/start_release_build.py b/exasol_script_languages_container_ci_setup/cli/commands/start_release_build.py index 2903f56..1f0d434 100644 --- a/exasol_script_languages_container_ci_setup/cli/commands/start_release_build.py +++ b/exasol_script_languages_container_ci_setup/cli/commands/start_release_build.py @@ -6,7 +6,7 @@ from exasol_script_languages_container_ci_setup.cli.cli import cli from exasol_script_languages_container_ci_setup.cli.common import add_options from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options from exasol_script_languages_container_ci_setup.lib.run_start_build import run_start_release_build diff --git a/exasol_script_languages_container_ci_setup/cli/commands/start_test_release_build.py b/exasol_script_languages_container_ci_setup/cli/commands/start_test_release_build.py index 6903a3d..571004d 100644 --- a/exasol_script_languages_container_ci_setup/cli/commands/start_test_release_build.py +++ b/exasol_script_languages_container_ci_setup/cli/commands/start_test_release_build.py @@ -6,7 +6,7 @@ from exasol_script_languages_container_ci_setup.cli.cli import cli from exasol_script_languages_container_ci_setup.cli.common import add_options from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options from exasol_script_languages_container_ci_setup.lib.github_draft_release_creator import GithubDraftReleaseCreator from exasol_script_languages_container_ci_setup.lib.run_start_build import run_start_test_release_build diff --git a/exasol_script_languages_container_ci_setup/cli/commands/validate_ci_build.py b/exasol_script_languages_container_ci_setup/cli/commands/validate_ci_build.py index cb77a3b..3b2f17e 100644 --- a/exasol_script_languages_container_ci_setup/cli/commands/validate_ci_build.py +++ b/exasol_script_languages_container_ci_setup/cli/commands/validate_ci_build.py @@ -1,4 +1,3 @@ -import logging from typing import Optional import click @@ -6,8 +5,8 @@ from exasol_script_languages_container_ci_setup.cli.cli import cli from exasol_script_languages_container_ci_setup.cli.common import add_options from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess -from exasol_script_languages_container_ci_setup.lib.ci_build import run_deploy_ci_build, run_validate_ci_build +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.ci_build import run_validate_ci_build from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options diff --git a/exasol_script_languages_container_ci_setup/cli/commands/validate_release_build.py b/exasol_script_languages_container_ci_setup/cli/commands/validate_release_build.py index 6a429d2..f7320bf 100644 --- a/exasol_script_languages_container_ci_setup/cli/commands/validate_release_build.py +++ b/exasol_script_languages_container_ci_setup/cli/commands/validate_release_build.py @@ -5,7 +5,7 @@ from exasol_script_languages_container_ci_setup.cli.cli import cli from exasol_script_languages_container_ci_setup.cli.common import add_options from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options from exasol_script_languages_container_ci_setup.lib.release_build import run_validate_release_build diff --git a/exasol_script_languages_container_ci_setup/cli/commands/validate_source_credentials.py b/exasol_script_languages_container_ci_setup/cli/commands/validate_source_credentials.py index f9cc819..95d8775 100644 --- a/exasol_script_languages_container_ci_setup/cli/commands/validate_source_credentials.py +++ b/exasol_script_languages_container_ci_setup/cli/commands/validate_source_credentials.py @@ -5,7 +5,7 @@ from exasol_script_languages_container_ci_setup.cli.cli import cli from exasol_script_languages_container_ci_setup.cli.common import add_options from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.lib.source_credentials import run_validate_source_credentials from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options diff --git a/exasol_script_languages_container_ci_setup/lib/aws/__init__.py b/exasol_script_languages_container_ci_setup/lib/aws/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/exasol_script_languages_container_ci_setup/lib/aws_access.py b/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py similarity index 86% rename from exasol_script_languages_container_ci_setup/lib/aws_access.py rename to exasol_script_languages_container_ci_setup/lib/aws/aws_access.py index 80464d2..39fa9a4 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws_access.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py @@ -2,14 +2,16 @@ import time from typing import Optional, List, Dict, Any, Iterable -import boto3 from botocore.exceptions import ClientError -from exasol_script_languages_container_ci_setup.lib.deployer import Deployer +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.aws_client import AwsClientFactory +from exasol_script_languages_container_ci_setup.lib.aws.deployer import Deployer -class AwsAccess(object): - def __init__(self, aws_profile: Optional[str]): +class AwsAccess: + def __init__(self, aws_profile: Optional[str], + aws_client_wrapper_factory: AwsClientFactory): + self._aws_client_wrapper_factory = aws_client_wrapper_factory self._aws_profile = aws_profile @property @@ -23,6 +25,9 @@ def aws_profile_for_logging(self) -> str: def aws_profile(self) -> Optional[str]: return self._aws_profile + def _get_aws_client(self, service_name: str) -> Any: + return self._aws_client_wrapper_factory.create(profile=self._aws_profile) + def upload_cloudformation_stack(self, yml: str, stack_name: str): """ Deploy the cloudformation stack. @@ -55,7 +60,7 @@ def read_secret_arn(self, secret_name: str): client = self._get_aws_client(service_name='secretsmanager') try: - get_secret_value_response = client.get_secret_value(SecretId=secret_name) + get_secret_value_response = client.get_secret_value(secret_id=secret_name) return get_secret_value_response["ARN"] except ClientError as e: logging.error("Unable to read secret") @@ -73,13 +78,7 @@ def validate_cloudformation_template(self, cloudformation_yml) -> None: """ logging.debug(f"Running validate_cloudformation_template for aws profile {self.aws_profile_for_logging}") cloud_client = self._get_aws_client("cloudformation") - cloud_client.validate_template(TemplateBody=cloudformation_yml) - - def _get_aws_client(self, service_name: str) -> Any: - if self._aws_profile is None: - return boto3.client(service_name) - aws_session = boto3.session.Session(profile_name=self._aws_profile) - return aws_session.client(service_name) + cloud_client.validate_template(template_body=cloudformation_yml) def get_all_stack_resources(self, stack_name: str) -> List[Dict[str, str]]: """ @@ -90,7 +89,7 @@ def get_all_stack_resources(self, stack_name: str) -> List[Dict[str, str]]: """ logging.debug(f"Running get_all_codebuild_projects for aws profile {self.aws_profile_for_logging}") cf_client = self._get_aws_client('cloudformation') - current_result = cf_client.list_stack_resources(StackName=stack_name) + current_result = cf_client.list_stack_resources(stack_name=stack_name) result = current_result["StackResourceSummaries"] while "nextToken" in current_result: @@ -98,7 +97,11 @@ def get_all_stack_resources(self, stack_name: str) -> List[Dict[str, str]]: result.extend(current_result["StackResourceSummaries"]) return result - def start_codebuild(self, project: str, environment_variables_overrides: List[Dict[str, str]], branch: str) -> None: + def start_codebuild(self, + project: str, + environment_variables_overrides: List[Dict[str, str]], + branch: str, + poll_interval_seconds: int = 30) -> None: """ This functions uses Boto3 to start a batch build. It forwards all variables from parameter env_variables as environment variables to the CodeBuild project. @@ -110,9 +113,9 @@ def start_codebuild(self, project: str, environment_variables_overrides: List[Di codebuild_client = self._get_aws_client("codebuild") logging.info(f"Trigger codebuild for project {project} with branch {branch} " f"and env_variables ({environment_variables_overrides})") - ret_val = codebuild_client.start_build_batch(projectName=project, - sourceVersion=branch, - environmentVariablesOverride=list( + ret_val = codebuild_client.start_build_batch(project_name=project, + source_version=branch, + environment_variables_override=list( environment_variables_overrides)) def wait_for(seconds: int, interval: int) -> Iterable[int]: diff --git a/exasol_script_languages_container_ci_setup/lib/deployer.py b/exasol_script_languages_container_ci_setup/lib/aws/deployer.py similarity index 100% rename from exasol_script_languages_container_ci_setup/lib/deployer.py rename to exasol_script_languages_container_ci_setup/lib/aws/deployer.py diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/__init__.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/aws_client.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/aws_client.py new file mode 100644 index 0000000..825e642 --- /dev/null +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/aws_client.py @@ -0,0 +1,50 @@ +import dataclasses +from abc import abstractmethod +from typing import List, Dict, Optional, Any, Protocol + +import boto3 +from boto3 import Session + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.cloudformation_service import CloudFormationService +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.codebuild_service import CodeBuildService +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.secretsmanager_service import SecretsManagerService + + +class BotoSessionFactory(Protocol): + + @abstractmethod + def __call__(self, profile_name: Optional[str] = None, region_name: Optional[str] = None) -> Session: + ... + + +class AwsClient: + def __init__(self, + profile: Optional[str] = None, + region: Optional[str] = None, + boto_session_factory: BotoSessionFactory = boto3.session.Session): + self._boto_session_factory = boto_session_factory + self._region = region + self._profile = profile + + def _create_aws_session(self) -> Session: + return self._boto_session_factory(profile_name=self._profile, region_name=self._region) + + def create_codebuild_service(self) -> CodeBuildService: + session = self._create_aws_session() + boto_client = session.client("codebuild") + return CodeBuildService(boto_client=boto_client) + + def create_secretsmanager_service(self) -> SecretsManagerService: + session = self._create_aws_session() + client = session.client("secretsmanager") + return SecretsManagerService(boto_client=client) + + def create_cloudformation_service(self) -> CloudFormationService: + session = self._create_aws_session() + client = session.client("cloudformation") + return CloudFormationService(boto_client=client) + + +class AwsClientFactory: + def create(self, profile: Optional[str] = None, region: Optional[str] = None) -> AwsClient: + return AwsClient(profile=profile, region=region) diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py new file mode 100644 index 0000000..26b9977 --- /dev/null +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py @@ -0,0 +1,35 @@ +from typing import Any, Dict, Optional, Callable + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.cloudformation import ValidationResult, \ + ListStackResourcesResult, NextToken +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId + + +class CloudFormationService: + def __init__(self, boto_client): + self._boto_client = boto_client + + @property + def boto_client(self) -> Any: + return self._boto_client + + def validate_template(self, + template_body: str, + from_boto: Callable[[Dict[str, Any]], ValidationResult] = + ValidationResult.from_boto) \ + -> ValidationResult: + boto_validation_result = self._boto_client.validate_template(TemplateBody=template_body) + validation_result = from_boto(boto_validation_result) + return validation_result + + def list_stack_resources(self, + stack_name: PhysicalResourceId, + next_token: Optional[NextToken], + from_boto: Callable[[Dict[str, Any]], ListStackResourcesResult] = + ListStackResourcesResult.from_boto) \ + -> ListStackResourcesResult: + aws_next_token = None if next_token is None else next_token.aws_next_token + boto_list_stack_resources_result = self._boto_client.list_stack_resources( + StackName=stack_name.aws_physical_resource_id, NextToken=aws_next_token) + list_stack_resources_result = from_boto(boto_list_stack_resources_result) + return list_stack_resources_result diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/codebuild_service.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/codebuild_service.py new file mode 100644 index 0000000..939dff0 --- /dev/null +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/codebuild_service.py @@ -0,0 +1,36 @@ +from typing import Dict, Any, List, Callable + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.codebuild import BuildBatch +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId + + +class CodeBuildService: + def __init__(self, boto_client): + self._boto_client = boto_client + + @property + def boto_client(self) -> Any: + return self._boto_client + + def start_build_batch( + self, + project_name: PhysicalResourceId, + source_version: str, + environment_variables_override: List[Dict[str, str]], + from_boto: Callable[[Dict[str, Any]], BuildBatch] = BuildBatch.from_boto) \ + -> BuildBatch: + boto_build_batch = self._boto_client.start_build_batch( + projectName=project_name.aws_physical_resource_id, + sourceVersion=source_version, + environmentVariablesOverride=environment_variables_override) + build_batch = from_boto(boto_build_batch['buildBatch']) + return build_batch + + def batch_get_build_batches(self, + build_batch_ids: List[PhysicalResourceId], + from_boto: Callable[[Dict[str, Any]], BuildBatch] = BuildBatch.from_boto) \ + -> List[BuildBatch]: + aws_ids = [build_batch_id.aws_physical_resource_id for build_batch_id in build_batch_ids] + boto_build_batches = self._boto_client.batch_get_build_batches(ids=aws_ids) + build_batches = [from_boto(boto_build_batch) for boto_build_batch in boto_build_batches] + return build_batches diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/__init__.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/cloudformation.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/cloudformation.py new file mode 100644 index 0000000..d395b40 --- /dev/null +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/cloudformation.py @@ -0,0 +1,61 @@ +import dataclasses +from typing import Any, Optional, List, Dict + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId + +RESOURCE_TYPE = "ResourceType" + +PHYSICAL_RESOURCE_ID = "PhysicalResourceId" + +STACK_RESOURCE_SUMMARIES = "StackResourceSummaries" + +NEXT_TOKEN = "NextToken" + + +@dataclasses.dataclass(frozen=True) +class NextToken: + aws_next_token: Any + + +@dataclasses.dataclass(frozen=True) +class StackResourceSummary: + physical_resource_id: PhysicalResourceId + resource_type: str + + @classmethod + def from_boto(cls, boto_stack_resource_summary: Dict[str, Any]) -> "StackResourceSummary": + physical_resource_id = PhysicalResourceId( + aws_physical_resource_id=boto_stack_resource_summary[PHYSICAL_RESOURCE_ID]) + resource_type = boto_stack_resource_summary[RESOURCE_TYPE] + stack_resource_summary = StackResourceSummary(physical_resource_id=physical_resource_id, + resource_type=resource_type) + return stack_resource_summary + + +@dataclasses.dataclass(frozen=True) +class ListStackResourcesResult: + next_token: Optional[NextToken] + stack_resource_summaries: List[StackResourceSummary] + + @classmethod + def from_boto(cls, boto_list_stack_resources_result: Dict[str, Any]) -> "ListStackResourcesResult": + next_token = None + if cls._has_next_token(boto_list_stack_resources_result): + next_token = NextToken(boto_list_stack_resources_result[NEXT_TOKEN]) + stack_resource_summaries = [StackResourceSummary.from_boto(boto_stack_resource_summary) + for boto_stack_resource_summary + in boto_list_stack_resources_result[STACK_RESOURCE_SUMMARIES]] + return ListStackResourcesResult(next_token=next_token, stack_resource_summaries=stack_resource_summaries) + + @classmethod + def _has_next_token(cls, boto_list_stack_resources_result): + return NEXT_TOKEN in boto_list_stack_resources_result \ + and boto_list_stack_resources_result[NEXT_TOKEN] is not None + + +@dataclasses.dataclass(frozen=True) +class ValidationResult: + + @classmethod + def from_boto(cls, boto_validation_result: Dict[str, Any]) -> "ValidationResult": + return ValidationResult() diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/codebuild.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/codebuild.py new file mode 100644 index 0000000..a6f2a56 --- /dev/null +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/codebuild.py @@ -0,0 +1,20 @@ +import dataclasses +from typing import Dict, Any + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId + +ID = "id" + + +@dataclasses.dataclass(frozen=True) +class BuildBatch: + id: PhysicalResourceId + + @classmethod + def from_boto(self, boto_buildbatch: Dict[str, Any]) -> "BuildBatch": + id = boto_buildbatch[ID] + if id is None: + raise ValueError("id was None") + batch_id = PhysicalResourceId(aws_physical_resource_id=id) + build_batch = BuildBatch(id=batch_id) + return build_batch diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/common.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/common.py new file mode 100644 index 0000000..9a2b3ff --- /dev/null +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/common.py @@ -0,0 +1,12 @@ +import dataclasses +from typing import Any + + +@dataclasses.dataclass(frozen=True) +class ARN: + aws_arn: Any + + +@dataclasses.dataclass(frozen=True) +class PhysicalResourceId: + aws_physical_resource_id: Any diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/secretsmanager.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/secretsmanager.py new file mode 100644 index 0000000..8497b10 --- /dev/null +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/secretsmanager.py @@ -0,0 +1,20 @@ +import dataclasses +from typing import Dict, Any + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import ARN + +ARN_KEY = "ARN" + + +@dataclasses.dataclass(frozen=True) +class Secret: + arn: ARN + + @classmethod + def from_boto(cls, boto_secret: Dict[str, Any]) -> "Secret": + aws_arn = boto_secret[ARN_KEY] + if aws_arn is None: + raise ValueError("ARN was None") + arn = ARN(aws_arn=aws_arn) + secret = Secret(arn=arn) + return secret diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/secretsmanager_service.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/secretsmanager_service.py new file mode 100644 index 0000000..f888b66 --- /dev/null +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/secretsmanager_service.py @@ -0,0 +1,22 @@ +import dataclasses +from typing import Any, Dict, Callable + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import ARN, PhysicalResourceId +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.secretsmanager import Secret + + +class SecretsManagerService: + def __init__(self, boto_client): + self._boto_client = boto_client + + @property + def boto_client(self) -> Any: + return self._boto_client + + def get_secret_value(self, + secret_id: PhysicalResourceId, + from_boto: Callable[[Dict[str, Any]], Secret]) \ + -> Secret: + boto_secret = self._boto_client.get_secret_value(SecretId=secret_id.aws_physical_resource_id) + secret = from_boto(boto_secret) + return secret diff --git a/exasol_script_languages_container_ci_setup/lib/ci_build.py b/exasol_script_languages_container_ci_setup/lib/ci_build.py index cda4dc3..587f801 100644 --- a/exasol_script_languages_container_ci_setup/lib/ci_build.py +++ b/exasol_script_languages_container_ci_setup/lib/ci_build.py @@ -1,6 +1,6 @@ import logging -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.lib.render_template import render_template CODE_BUILD_STACK_NAME = "CIBuild" diff --git a/exasol_script_languages_container_ci_setup/lib/release_build.py b/exasol_script_languages_container_ci_setup/lib/release_build.py index e290422..721edcc 100644 --- a/exasol_script_languages_container_ci_setup/lib/release_build.py +++ b/exasol_script_languages_container_ci_setup/lib/release_build.py @@ -1,6 +1,6 @@ import logging -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.lib.render_template import render_template CODE_BUILD_STACK_NAME = "ReleaseBuild" diff --git a/exasol_script_languages_container_ci_setup/lib/run_start_build.py b/exasol_script_languages_container_ci_setup/lib/run_start_build.py index 9112d0a..a3ed6bd 100644 --- a/exasol_script_languages_container_ci_setup/lib/run_start_build.py +++ b/exasol_script_languages_container_ci_setup/lib/run_start_build.py @@ -2,7 +2,7 @@ import re from typing import Tuple, Dict, List -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.lib.ci_build import ci_stack_name from exasol_script_languages_container_ci_setup.lib.github_draft_release_creator import GithubDraftReleaseCreator from exasol_script_languages_container_ci_setup.lib.release_build import release_stack_name diff --git a/exasol_script_languages_container_ci_setup/lib/source_credentials.py b/exasol_script_languages_container_ci_setup/lib/source_credentials.py index 9503a8c..b5e2c9d 100644 --- a/exasol_script_languages_container_ci_setup/lib/source_credentials.py +++ b/exasol_script_languages_container_ci_setup/lib/source_credentials.py @@ -1,6 +1,6 @@ import logging -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.lib.render_template import render_template SOURCE_CREDENTIALS_STACK_NAME = "SLCSourceCredentials" diff --git a/test/test_deploy_ci.py b/test/test_deploy_ci.py index 21c4831..c42e94b 100644 --- a/test/test_deploy_ci.py +++ b/test/test_deploy_ci.py @@ -2,7 +2,6 @@ import pytest -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.lib.ci_build import run_deploy_ci_build, ci_stack_name, \ CI_BUILD_WEBHOOK_FILTER_PATTERN from exasol_script_languages_container_ci_setup.lib.release_build import run_deploy_release_build, release_stack_name diff --git a/test/unit_tests/aws/__init__.py b/test/unit_tests/aws/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit_tests/aws/wrapper/__init__.py b/test/unit_tests/aws/wrapper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit_tests/aws/wrapper/datamodels/__init__.py b/test/unit_tests/aws/wrapper/datamodels/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit_tests/aws/wrapper/datamodels/cloudformation/__init__.py b/test/unit_tests/aws/wrapper/datamodels/cloudformation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit_tests/aws/wrapper/datamodels/cloudformation/test_list_stack_resources_result.py b/test/unit_tests/aws/wrapper/datamodels/cloudformation/test_list_stack_resources_result.py new file mode 100644 index 0000000..fc4ead8 --- /dev/null +++ b/test/unit_tests/aws/wrapper/datamodels/cloudformation/test_list_stack_resources_result.py @@ -0,0 +1,94 @@ +import pytest + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.cloudformation import \ + ListStackResourcesResult, NextToken, StackResourceSummary +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId + + +def test_with_next_token(): + expected_list_stack_resources_result = \ + ListStackResourcesResult( + next_token=NextToken(aws_next_token="aws_next_token"), + stack_resource_summaries=[] + ) + boto_list_stack_resources_result = { + "NextToken": expected_list_stack_resources_result.next_token.aws_next_token, + "StackResourceSummaries": [] + } + list_stack_resources_result = ListStackResourcesResult.from_boto(boto_list_stack_resources_result) + assert list_stack_resources_result == expected_list_stack_resources_result + + +def test_with_next_token_none(): + expected_list_stack_resources_result = \ + ListStackResourcesResult( + next_token=None, + stack_resource_summaries=[] + ) + boto_list_stack_resources_result = { + "NextToken": None, + "StackResourceSummaries": [] + } + list_stack_resources_result = ListStackResourcesResult.from_boto(boto_list_stack_resources_result) + assert list_stack_resources_result == expected_list_stack_resources_result + + +def test_without_next_token_none(): + expected_list_stack_resources_result = \ + ListStackResourcesResult( + next_token=None, + stack_resource_summaries=[] + ) + boto_list_stack_resources_result = { + "StackResourceSummaries": [] + } + list_stack_resources_result = ListStackResourcesResult.from_boto(boto_list_stack_resources_result) + assert list_stack_resources_result == expected_list_stack_resources_result + + +@pytest.mark.parametrize("count", range(3)) +def test_with_multiple_stack_resource_summary(count: int): + expected_list_stack_resources_result = \ + ListStackResourcesResult( + next_token=None, + stack_resource_summaries=[ + StackResourceSummary( + physical_resource_id=PhysicalResourceId(f"physical_resource_id_{i}"), + resource_type=f"resource_type_{i}" + ) + for i in range(count) + ] + ) + boto_list_stack_resources_result = { + "StackResourceSummaries": [ + { + "PhysicalResourceId": summary.physical_resource_id.aws_physical_resource_id, + "ResourceType": summary.resource_type + } + for summary in expected_list_stack_resources_result.stack_resource_summaries + ] + } + list_stack_resources_result = ListStackResourcesResult.from_boto(boto_list_stack_resources_result) + assert list_stack_resources_result == expected_list_stack_resources_result + + +def test_without_stack_resource_summaries(): + boto_list_stack_resources_result = {} + with pytest.raises(KeyError): + list_stack_resources_result = ListStackResourcesResult.from_boto(boto_list_stack_resources_result) + + +def test_with_extra_keys(): + expected_list_stack_resources_result = \ + ListStackResourcesResult( + next_token=None, + stack_resource_summaries=[] + ) + boto_list_stack_resources_result = { + "NextToken": None, + "StackResourceSummaries": [], + "extra1": None, + "extra2": 1 + } + list_stack_resources_result = ListStackResourcesResult.from_boto(boto_list_stack_resources_result) + assert list_stack_resources_result == expected_list_stack_resources_result diff --git a/test/unit_tests/aws/wrapper/datamodels/cloudformation/test_stack_resource_summary.py b/test/unit_tests/aws/wrapper/datamodels/cloudformation/test_stack_resource_summary.py new file mode 100644 index 0000000..c862f29 --- /dev/null +++ b/test/unit_tests/aws/wrapper/datamodels/cloudformation/test_stack_resource_summary.py @@ -0,0 +1,46 @@ +import pytest + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.cloudformation import StackResourceSummary +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId + + +def test_valid(): + expected_stack_resource_summary = StackResourceSummary( + physical_resource_id=PhysicalResourceId("physical_resource_id"), + resource_type="resource_type") + boto_stack_resource_summary = { + "PhysicalResourceId": expected_stack_resource_summary.physical_resource_id.aws_physical_resource_id, + "ResourceType": expected_stack_resource_summary.resource_type + } + stack_resource_summary = StackResourceSummary.from_boto(boto_stack_resource_summary) + assert stack_resource_summary == expected_stack_resource_summary + + +def test_without_physical_resource_id(): + with pytest.raises(KeyError): + boto_stack_resource_summary = { + "ResourceType": "resource_type" + } + stack_resource_summary = StackResourceSummary.from_boto(boto_stack_resource_summary) + + +def test_without_resource_type(): + with pytest.raises(KeyError): + boto_stack_resource_summary = { + "PhysicalResourceId": "physical_resource_id", + } + stack_resource_summary = StackResourceSummary.from_boto(boto_stack_resource_summary) + + +def test_with_extra_keys(): + expected_stack_resource_summary = StackResourceSummary( + physical_resource_id=PhysicalResourceId("physical_resource_id"), + resource_type="resource_type") + boto_stack_resource_summary = { + "PhysicalResourceId": expected_stack_resource_summary.physical_resource_id.aws_physical_resource_id, + "ResourceType": expected_stack_resource_summary.resource_type, + "extra1": None, + "extra2": 1 + } + stack_resource_summary = StackResourceSummary.from_boto(boto_stack_resource_summary) + assert stack_resource_summary == expected_stack_resource_summary diff --git a/test/unit_tests/aws/wrapper/datamodels/cloudformation/test_validation_result.py b/test/unit_tests/aws/wrapper/datamodels/cloudformation/test_validation_result.py new file mode 100644 index 0000000..4b037aa --- /dev/null +++ b/test/unit_tests/aws/wrapper/datamodels/cloudformation/test_validation_result.py @@ -0,0 +1,16 @@ +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.cloudformation import ValidationResult + + +def test_empyt_dict(): + boto_validation_result = {} + validation_result = ValidationResult.from_boto(boto_validation_result) + assert validation_result == ValidationResult() + + +def test_with_extra_keys(): + boto_validation_result = { + "extra1": None, + "extra2": 1 + } + validation_result = ValidationResult.from_boto(boto_validation_result) + assert validation_result == ValidationResult() diff --git a/test/unit_tests/aws/wrapper/datamodels/codebuild/__init__.py b/test/unit_tests/aws/wrapper/datamodels/codebuild/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit_tests/aws/wrapper/datamodels/codebuild/test_build_batch.py b/test/unit_tests/aws/wrapper/datamodels/codebuild/test_build_batch.py new file mode 100644 index 0000000..da7b642 --- /dev/null +++ b/test/unit_tests/aws/wrapper/datamodels/codebuild/test_build_batch.py @@ -0,0 +1,34 @@ +import pytest + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.codebuild import BuildBatch +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId + + +def test_id_exists(): + expected_id = PhysicalResourceId("expected_id") + boto_buildbatch = {"id": expected_id.aws_physical_resource_id} + build_batch = BuildBatch.from_boto(boto_buildbatch) + assert build_batch.id == expected_id + + +def test_id_not_exists(): + boto_buildbatch = {} + with pytest.raises(KeyError): + build_batch = BuildBatch.from_boto(boto_buildbatch) + + +def test_with_extra_keys(): + expected_id = PhysicalResourceId("expected_id") + boto_buildbatch = { + "id": expected_id.aws_physical_resource_id, + "extra1": None, + "extra2": 1 + } + build_batch = BuildBatch.from_boto(boto_buildbatch) + assert build_batch.id == expected_id + + +def test_empty_id(): + with pytest.raises(ValueError, match="id was None"): + boto_buildbatch = {"id": None} + build_batch = BuildBatch.from_boto(boto_buildbatch) diff --git a/test/unit_tests/aws/wrapper/datamodels/secretsmanager/__init__.py b/test/unit_tests/aws/wrapper/datamodels/secretsmanager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit_tests/aws/wrapper/datamodels/secretsmanager/test_secret.py b/test/unit_tests/aws/wrapper/datamodels/secretsmanager/test_secret.py new file mode 100644 index 0000000..04907df --- /dev/null +++ b/test/unit_tests/aws/wrapper/datamodels/secretsmanager/test_secret.py @@ -0,0 +1,36 @@ +import pytest + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import ARN +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.secretsmanager import Secret + + +def test_arn_exists(): + expected_arn = ARN(aws_arn="EXPECTED_ARN") + boto_secret = {"ARN": expected_arn.aws_arn} + secret = Secret.from_boto(boto_secret) + assert secret.arn == expected_arn + + +def test_arn_not_exists(): + with pytest.raises(KeyError): + boto_secret = {} + secret = Secret.from_boto(boto_secret) + + +def test_with_extra_keys(): + expected_arn = ARN(aws_arn="EXPECTED_ARN") + boto_secret = { + "ARN": expected_arn.aws_arn, + "extra1": None, + "extra2": 1 + } + secret = Secret.from_boto(boto_secret) + assert secret.arn == expected_arn + + +def test_empty_arn(): + with pytest.raises(ValueError, match="ARN was None"): + boto_secret = { + "ARN": None + } + secret = Secret.from_boto(boto_secret) diff --git a/test/unit_tests/aws/wrapper/test_aws_client.py b/test/unit_tests/aws/wrapper/test_aws_client.py new file mode 100644 index 0000000..d2e089d --- /dev/null +++ b/test/unit_tests/aws/wrapper/test_aws_client.py @@ -0,0 +1,98 @@ +import dataclasses +from typing import Union +from unittest.mock import MagicMock, Mock, call, create_autospec + +import pytest +from boto3 import Session + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.aws_client import AwsClient +from test.mock_cast import mock_cast + + +def test_init(): + boto_session_factory: MagicMock = Mock() + aws_client = AwsClient(profile="profile", region="region", + boto_session_factory=boto_session_factory) + assert boto_session_factory.mock_calls == [] + + +@dataclasses.dataclass +class CodeBuildServiceTestSetup: + boto_session: Union[MagicMock, Session] = create_autospec(Session) + boto_session_factory = Mock(return_value=boto_session) + aws_client = AwsClient(profile="profile", region="region", + boto_session_factory=boto_session_factory) + codebuild_service = aws_client.create_codebuild_service() + + +def test_create_codebuild_service_boto_session_factory(): + setup = CodeBuildServiceTestSetup() + assert setup.boto_session_factory.mock_calls == [ + call(profile_name='profile', region_name='region'), + ] + + +def test_create_codebuild_service_boto_session_client(): + setup = CodeBuildServiceTestSetup() + assert setup.boto_session.mock_calls == [ + call.client('codebuild'), + ] + +def test_create_codebuild_service_boto_client(): + setup = CodeBuildServiceTestSetup() + assert setup.codebuild_service.boto_client == mock_cast(setup.boto_session.client).return_value + + +@dataclasses.dataclass +class CloudFormationServiceTestSetup: + boto_session: Union[MagicMock, Session] = create_autospec(Session) + boto_session_factory = Mock(return_value=boto_session) + aws_client = AwsClient(profile="profile", region="region", + boto_session_factory=boto_session_factory) + cloudformation_service = aws_client.create_cloudformation_service() + + +def test_create_cloudformation_service_boto_session_factory(): + setup = CloudFormationServiceTestSetup() + assert setup.boto_session_factory.mock_calls == [ + call(profile_name='profile', region_name='region'), + ] + + +def test_create_cloudformation_service_boto_session_client(): + setup = CloudFormationServiceTestSetup() + assert setup.boto_session.mock_calls == [ + call.client('cloudformation'), + ] + +def test_create_cloudformation_service_boto_client(): + setup = CloudFormationServiceTestSetup() + assert setup.cloudformation_service.boto_client == mock_cast(setup.boto_session.client).return_value + + +@dataclasses.dataclass +class SecreteManagerServiceTestSetup: + boto_session: Union[MagicMock, Session] = create_autospec(Session) + boto_session_factory = Mock(return_value=boto_session) + aws_client = AwsClient(profile="profile", region="region", + boto_session_factory=boto_session_factory) + secretsmanager_service = aws_client.create_secretsmanager_service() + + +def test_create_secretsmanager_service_boto_session_factory(): + setup = SecreteManagerServiceTestSetup() + assert setup.boto_session_factory.mock_calls == [ + call(profile_name='profile', region_name='region'), + ] + + +def test_create_secretsmanager_service_boto_session_client(): + setup = SecreteManagerServiceTestSetup() + assert setup.boto_session.mock_calls == [ + call.client('secretsmanager'), + ] + + +def test_create_secretsmanager_service_boto_client(): + setup = SecreteManagerServiceTestSetup() + assert setup.secretsmanager_service.boto_client == mock_cast(setup.boto_session.client).return_value diff --git a/test/unit_tests/aws/wrapper/test_cloudformation_service.py b/test/unit_tests/aws/wrapper/test_cloudformation_service.py new file mode 100644 index 0000000..bfd175f --- /dev/null +++ b/test/unit_tests/aws/wrapper/test_cloudformation_service.py @@ -0,0 +1,83 @@ +import dataclasses +from typing import Optional +from unittest.mock import Mock, call + +import pytest + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.cloudformation import NextToken +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.cloudformation_service import CloudFormationService +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId +from test.mock_cast import mock_cast + + +def test_init(): + boto_client = Mock() + service = CloudFormationService(boto_client=boto_client) + assert service.boto_client == boto_client + + +@dataclasses.dataclass(frozen=True) +class ValidateTemplateSetup: + boto_client = Mock() + from_boto = Mock() + service = CloudFormationService(boto_client=boto_client) + template_body = "TemplateBody" + validation_result = service.validate_template(template_body=template_body, from_boto=from_boto) + boto_client_validate_template_return_value = mock_cast(boto_client.validate_template).return_value + + +def test_validate_template_boto_client(): + setup = ValidateTemplateSetup() + assert setup.boto_client.mock_calls == [call.validate_template(TemplateBody=setup.template_body)] \ + and setup.from_boto.mock_calls == [call(setup.boto_client_validate_template_return_value)] \ + and setup.validation_result == setup.from_boto.return_value + + +def test_validate_template_from_boto(): + setup = ValidateTemplateSetup() + assert setup.from_boto.mock_calls == [call(setup.boto_client_validate_template_return_value)] + + +def test_validate_template_result(): + setup = ValidateTemplateSetup() + assert setup.validation_result == setup.from_boto.return_value + + +class ListStackResourcesTestSetup: + def __init__(self, next_token: Optional[NextToken]): + self.next_token = next_token + self.boto_client = Mock() + self.from_boto = Mock() + self.service = CloudFormationService(boto_client=self.boto_client) + self.physical_resource_id = PhysicalResourceId(aws_physical_resource_id="stack_name") + self.list_stack_resources_result = self.service.list_stack_resources( + stack_name=self.physical_resource_id, next_token=next_token, from_boto=self.from_boto) + self.aws_next_token = None if next_token is None else next_token.aws_next_token + self.boto_client_list_stack_resources_return_value = mock_cast( + self.boto_client.list_stack_resources).return_value + + +list_stack_resources_parameters = pytest.mark.parametrize("next_token", [ + NextToken(aws_next_token="next_token"), + None +]) + + +@list_stack_resources_parameters +def test_list_stack_resources_boto_client(next_token): + setup = ListStackResourcesTestSetup(next_token) + assert setup.boto_client.mock_calls == [ + call.list_stack_resources(StackName=setup.physical_resource_id.aws_physical_resource_id, + NextToken=setup.aws_next_token)] + + +@list_stack_resources_parameters +def test_list_stack_resources_from_boto(next_token): + setup = ListStackResourcesTestSetup(next_token) + assert setup.from_boto.mock_calls == [call(setup.boto_client_list_stack_resources_return_value)] + + +@list_stack_resources_parameters +def test_list_stack_resources_result(next_token): + setup = ListStackResourcesTestSetup(next_token) + assert setup.list_stack_resources_result == setup.from_boto.return_value diff --git a/test/unit_tests/aws/wrapper/test_codebuild_service.py b/test/unit_tests/aws/wrapper/test_codebuild_service.py new file mode 100644 index 0000000..eb291bd --- /dev/null +++ b/test/unit_tests/aws/wrapper/test_codebuild_service.py @@ -0,0 +1,79 @@ +import dataclasses +from unittest.mock import Mock, call + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.codebuild_service import CodeBuildService +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId +from test.mock_cast import mock_cast + + +def test_init(): + boto_client = Mock() + service = CodeBuildService(boto_client=boto_client) + assert service.boto_client == boto_client + + +@dataclasses.dataclass(frozen=True) +class BatchGetBuildBatchesTestSetup: + boto_client = Mock() + batch_get_build_batches_return_values = [Mock(), Mock()] + mock_cast(boto_client.batch_get_build_batches).return_value = batch_get_build_batches_return_values + from_boto = Mock() + from_boto_return_values = [Mock(), Mock()] + from_boto.side_effect = from_boto_return_values + service = CodeBuildService(boto_client=boto_client) + build_batch_ids = [PhysicalResourceId(aws_physical_resource_id="1"), + PhysicalResourceId(aws_physical_resource_id="2")] + build_batches = service.batch_get_build_batches(build_batch_ids=build_batch_ids, from_boto=from_boto) + ids = [id.aws_physical_resource_id for id in build_batch_ids] + + +def test_batch_get_build_batches_boto_client(): + setup = BatchGetBuildBatchesTestSetup() + assert setup.boto_client.mock_calls == [call.batch_get_build_batches(ids=setup.ids)] + + +def test_batch_get_build_batches_from_boto(): + setup = BatchGetBuildBatchesTestSetup() + assert setup.from_boto.mock_calls == [call(return_value) for return_value in + setup.batch_get_build_batches_return_values] + + +def test_batch_get_build_batches_result(): + setup = BatchGetBuildBatchesTestSetup() + assert setup.build_batches == setup.from_boto_return_values + + +@dataclasses.dataclass(frozen=True) +class StartBuildBatchTestSetup: + boto_client = Mock() + from_boto_input = Mock() + start_build_batch_return_values = {"buildBatch": from_boto_input} + mock_cast(boto_client.start_build_batch).return_value = start_build_batch_return_values + from_boto = Mock() + from_boto.return_value = Mock() + service = CodeBuildService(boto_client=boto_client) + projectName = PhysicalResourceId(aws_physical_resource_id="id") + sourceVersion = Mock() + environmentVariablesOverride = Mock() + build_batch = service.start_build_batch(project_name=projectName, + source_version=sourceVersion, + environment_variables_override=environmentVariablesOverride, + from_boto=from_boto) + + +def test_start_build_batch_boto_client(): + setup = StartBuildBatchTestSetup() + assert setup.boto_client.mock_calls == [ + call.start_build_batch(projectName=setup.projectName.aws_physical_resource_id, + sourceVersion=setup.sourceVersion, + environmentVariablesOverride=setup.environmentVariablesOverride)] + + +def test_start_build_batch_from_boto(): + setup = StartBuildBatchTestSetup() + assert setup.from_boto.mock_calls == [call(setup.from_boto_input)] + + +def test_start_build_batch_result(): + setup = StartBuildBatchTestSetup() + assert setup.build_batch == setup.from_boto.return_value diff --git a/test/unit_tests/aws/wrapper/test_secretsmanager_service.py b/test/unit_tests/aws/wrapper/test_secretsmanager_service.py new file mode 100644 index 0000000..af625fb --- /dev/null +++ b/test/unit_tests/aws/wrapper/test_secretsmanager_service.py @@ -0,0 +1,40 @@ +import dataclasses +from unittest.mock import Mock, call + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.secretsmanager_service import SecretsManagerService +from test.mock_cast import mock_cast + + +def test_init(): + boto_client = Mock() + service = SecretsManagerService(boto_client=boto_client) + assert service.boto_client == boto_client + + +@dataclasses.dataclass(frozen=True) +class GetSecretValueTestSetup: + boto_client = Mock() + get_secret_value_return_values = Mock() + mock_cast(boto_client.get_secret_value).return_value = get_secret_value_return_values + from_boto = Mock() + from_boto_return_values = Mock() + from_boto.return_value = from_boto_return_values + service = SecretsManagerService(boto_client=boto_client) + secret_id = PhysicalResourceId(aws_physical_resource_id="id") + secret = service.get_secret_value(secret_id=secret_id, from_boto=from_boto) + + +def test_get_secret_value_boto_client(): + setup = GetSecretValueTestSetup() + assert setup.boto_client.mock_calls == [call.get_secret_value(SecretId=setup.secret_id.aws_physical_resource_id)] + + +def test_get_secret_value_from_boto(): + setup = GetSecretValueTestSetup() + assert setup.from_boto.mock_calls == [call(setup.get_secret_value_return_values)] + + +def test_get_secret_value_result(): + setup = GetSecretValueTestSetup() + assert setup.secret == setup.from_boto_return_values From 3ca1a893f0da0177304fc4ce6c37e530c4c9f479 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 12 Jun 2023 09:41:41 +0200 Subject: [PATCH 2/8] Refactored AwsAccess to us shallow abstraction layer for boto --- .../lib/aws/aws_access.py | 75 +++++++++--------- .../lib/aws/wrapper/cloudformation_service.py | 2 +- .../aws/wrapper/datamodels/cloudformation.py | 18 ++++- .../lib/aws/wrapper/datamodels/codebuild.py | 47 +++++++++-- .../lib/aws/wrapper/secretsmanager_service.py | 2 +- .../test_stack_resource_summary.py | 27 +++++-- .../datamodels/codebuild/test_build_batch.py | 77 +++++++++++++++---- 7 files changed, 182 insertions(+), 66 deletions(-) diff --git a/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py b/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py index 39fa9a4..74412cd 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py @@ -1,17 +1,23 @@ import logging import time -from typing import Optional, List, Dict, Any, Iterable +from typing import Optional, List, Dict, Iterable, Callable from botocore.exceptions import ClientError -from exasol_script_languages_container_ci_setup.lib.aws.wrapper.aws_client import AwsClientFactory from exasol_script_languages_container_ci_setup.lib.aws.deployer import Deployer +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.aws_client import AwsClientFactory, AwsClient +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.cloudformation import StackResourceSummary +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.codebuild import BuildBatchStatus +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId + +BUILD_STATUS_FAILURES = [BuildBatchStatus.FAILED, BuildBatchStatus.FAULT, + BuildBatchStatus.STOPPED, BuildBatchStatus.TIMED_OUT] class AwsAccess: def __init__(self, aws_profile: Optional[str], - aws_client_wrapper_factory: AwsClientFactory): - self._aws_client_wrapper_factory = aws_client_wrapper_factory + aws_client_factory: AwsClientFactory): + self._aws_client_factory = aws_client_factory self._aws_profile = aws_profile @property @@ -25,17 +31,17 @@ def aws_profile_for_logging(self) -> str: def aws_profile(self) -> Optional[str]: return self._aws_profile - def _get_aws_client(self, service_name: str) -> Any: - return self._aws_client_wrapper_factory.create(profile=self._aws_profile) + def _get_aws_client(self) -> AwsClient: + return self._aws_client_factory.create(profile=self._aws_profile) def upload_cloudformation_stack(self, yml: str, stack_name: str): """ Deploy the cloudformation stack. """ logging.debug(f"Running upload_cloudformation_stack for aws profile {self.aws_profile_for_logging}") - cloud_client = self._get_aws_client("cloudformation") + client = self._get_aws_client().create_cloudformation_service() try: - cfn_deployer = Deployer(cloudformation_client=cloud_client) + cfn_deployer = Deployer(cloudformation_client=client.boto_client) result = cfn_deployer.create_and_wait_for_changeset(stack_name=stack_name, cfn_template=yml, parameter_values=[], capabilities=("CAPABILITY_IAM",), role_arn=None, @@ -57,11 +63,11 @@ def read_secret_arn(self, secret_name: str): """ logging.debug(f"Reading secret for getting ARN, secret name = {secret_name}, " f"for aws profile {self.aws_profile_for_logging}") - client = self._get_aws_client(service_name='secretsmanager') + client = self._get_aws_client().create_secretsmanager_service() try: - get_secret_value_response = client.get_secret_value(secret_id=secret_name) - return get_secret_value_response["ARN"] + secret = client.get_secret_value(secret_id=PhysicalResourceId(secret_name)) + return secret.arn except ClientError as e: logging.error("Unable to read secret") raise e @@ -77,10 +83,10 @@ def validate_cloudformation_template(self, cloudformation_yml) -> None: It requires to have the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env variables set correctly. """ logging.debug(f"Running validate_cloudformation_template for aws profile {self.aws_profile_for_logging}") - cloud_client = self._get_aws_client("cloudformation") - cloud_client.validate_template(template_body=cloudformation_yml) + client = self._get_aws_client().create_cloudformation_service() + client.validate_template(template_body=cloudformation_yml) - def get_all_stack_resources(self, stack_name: str) -> List[Dict[str, str]]: + def get_all_stack_resources(self, stack_name: str) -> List[StackResourceSummary]: """ This functions uses Boto3 to get all AWS Cloudformation resources for a specific Cloudformation stack, identified by parameter `stack_name`. @@ -88,13 +94,14 @@ def get_all_stack_resources(self, stack_name: str) -> List[Dict[str, str]]: passing the previous retrieved token until no token is returned. """ logging.debug(f"Running get_all_codebuild_projects for aws profile {self.aws_profile_for_logging}") - cf_client = self._get_aws_client('cloudformation') - current_result = cf_client.list_stack_resources(stack_name=stack_name) - result = current_result["StackResourceSummaries"] + client = self._get_aws_client().create_cloudformation_service() + stack_name_id = PhysicalResourceId(stack_name) + current_result = client.list_stack_resources(stack_name=stack_name_id) + result = current_result.stack_resource_summaries while "nextToken" in current_result: - current_result = cf_client.list_projects(StackName=stack_name, nextToken=current_result["nextToken"]) - result.extend(current_result["StackResourceSummaries"]) + current_result = client.list_stack_resources(stack_name=stack_name_id, next_token=current_result.next_token) + result.extend(current_result.stack_resource_summaries) return result def start_codebuild(self, @@ -110,37 +117,35 @@ def start_codebuild(self, :raises `RuntimeError` if build fails or AWS Batch build returns unknown status """ - codebuild_client = self._get_aws_client("codebuild") + client = self._get_aws_client().create_codebuild_service() logging.info(f"Trigger codebuild for project {project} with branch {branch} " f"and env_variables ({environment_variables_overrides})") - ret_val = codebuild_client.start_build_batch(project_name=project, - source_version=branch, - environment_variables_override=list( - environment_variables_overrides)) + build_batch = client.start_build_batch(project_name=PhysicalResourceId(project), + source_version=branch, + environment_variables_override=list( + environment_variables_overrides)) def wait_for(seconds: int, interval: int) -> Iterable[int]: for _ in range(int(seconds / interval)): yield interval - build_id = ret_val['buildBatch']['id'] + build_id = build_batch.id logging.debug(f"Codebuild for project {project} with branch {branch} triggered. Id is {build_id}.") interval = 30 timeout_time_in_seconds = 60 * 60 * 2 # We wait for maximal 2h + (something) for seconds_to_wait in wait_for(seconds=timeout_time_in_seconds, interval=interval): time.sleep(seconds_to_wait) logging.debug(f"Checking status of codebuild id {build_id}.") - build_response = codebuild_client.batch_get_build_batches(ids=[build_id]) - logging.debug(f"Build response of codebuild id {build_id} is {build_response}") - if len(build_response['buildBatches']) != 1: - logging.error(f"Unexpected return value from 'batch_get_build_batches': {build_response}") - build_status = build_response['buildBatches'][0]['buildBatchStatus'] + build_batches = client.batch_get_build_batches(build_batch_ids=[build_id]) + logging.debug(f"Build response of codebuild id {build_id} is {build_batches}") + if len(build_batches) != 1: + logging.error(f"Unexpected return value from 'batch_get_build_batches': {build_batches}") + build_status = build_batches[0].build_batch_status logging.info(f"Build status of codebuild id {build_id} is {build_status}") - if build_status == 'SUCCEEDED': + if build_status == BuildBatchStatus.SUCCEEDED: break - elif build_status in ['FAILED', 'FAULT', 'STOPPED', 'TIMED_OUT']: - raise RuntimeError(f"Build ({build_id}) failed with status: {build_status}") - elif build_status != "IN_PROGRESS": - raise RuntimeError(f"Batch build {build_id} has unknown build status: {build_status}") + elif build_status in BUILD_STATUS_FAILURES: + raise RuntimeError(f"Build ({build_id}) failed with status: {build_status.name}") # if loop does not break early, build wasn't successful else: raise RuntimeError(f"Batch build {build_id} ran into timeout.") diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py index 26b9977..608db46 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py @@ -24,7 +24,7 @@ def validate_template(self, def list_stack_resources(self, stack_name: PhysicalResourceId, - next_token: Optional[NextToken], + next_token: Optional[NextToken] = None, from_boto: Callable[[Dict[str, Any]], ListStackResourcesResult] = ListStackResourcesResult.from_boto) \ -> ListStackResourcesResult: diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/cloudformation.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/cloudformation.py index d395b40..419a7d3 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/cloudformation.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/cloudformation.py @@ -19,18 +19,30 @@ class NextToken: @dataclasses.dataclass(frozen=True) class StackResourceSummary: - physical_resource_id: PhysicalResourceId + physical_resource_id: Optional[PhysicalResourceId] resource_type: str @classmethod def from_boto(cls, boto_stack_resource_summary: Dict[str, Any]) -> "StackResourceSummary": - physical_resource_id = PhysicalResourceId( - aws_physical_resource_id=boto_stack_resource_summary[PHYSICAL_RESOURCE_ID]) + physical_resource_id = cls._extract_physcial_resource_id(boto_stack_resource_summary) resource_type = boto_stack_resource_summary[RESOURCE_TYPE] stack_resource_summary = StackResourceSummary(physical_resource_id=physical_resource_id, resource_type=resource_type) return stack_resource_summary + @classmethod + def _extract_physcial_resource_id(cls, boto_stack_resource_summary: Dict[str, Any]) -> PhysicalResourceId: + physical_resource_id = None + if cls._has_physical_resource_id(boto_stack_resource_summary): + physical_resource_id = PhysicalResourceId( + aws_physical_resource_id=boto_stack_resource_summary[PHYSICAL_RESOURCE_ID]) + return physical_resource_id + + @classmethod + def _has_physical_resource_id(cls, boto_stack_resource_summary: Dict[str, Any]): + return PHYSICAL_RESOURCE_ID in boto_stack_resource_summary \ + and boto_stack_resource_summary[PHYSICAL_RESOURCE_ID] is not None + @dataclasses.dataclass(frozen=True) class ListStackResourcesResult: diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/codebuild.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/codebuild.py index a6f2a56..a5ef394 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/codebuild.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/codebuild.py @@ -1,20 +1,53 @@ import dataclasses -from typing import Dict, Any +from enum import Enum, auto +from typing import Dict, Any, Optional from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId +BUILD_BATCH_STATUS = "buildBatchStatus" + ID = "id" +class BuildBatchStatus(Enum): + SUCCEEDED = auto() + FAILED = auto() + FAULT = auto() + TIMED_OUT = auto() + IN_PROGRESS = auto() + STOPPED = auto() + + @dataclasses.dataclass(frozen=True) class BuildBatch: - id: PhysicalResourceId + id: Optional[PhysicalResourceId] + build_batch_status: Optional[BuildBatchStatus] @classmethod def from_boto(self, boto_buildbatch: Dict[str, Any]) -> "BuildBatch": - id = boto_buildbatch[ID] - if id is None: - raise ValueError("id was None") - batch_id = PhysicalResourceId(aws_physical_resource_id=id) - build_batch = BuildBatch(id=batch_id) + batch_id = self._extract_build_id(boto_buildbatch) + build_batch_status = self._extract_build_batch_status(boto_buildbatch) + build_batch = BuildBatch(id=batch_id, build_batch_status=build_batch_status) return build_batch + + @classmethod + def _extract_build_id(cls, boto_buildbatch: Dict[str, Any]) -> Optional[PhysicalResourceId]: + batch_id = None + if BuildBatch._has_id(boto_buildbatch): + batch_id = PhysicalResourceId(aws_physical_resource_id=boto_buildbatch[ID]) + return batch_id + + @classmethod + def _has_id(cls, boto_buildbatch: Dict[str, Any]) -> bool: + return ID in boto_buildbatch and boto_buildbatch[ID] is not None + + @classmethod + def _extract_build_batch_status(cls, boto_buildbatch: Dict[str, Any]) -> Optional[BuildBatchStatus]: + build_batch_status = None + if BuildBatch._has_build_status(boto_buildbatch): + build_batch_status = BuildBatchStatus[boto_buildbatch[BUILD_BATCH_STATUS]] + return build_batch_status + + @classmethod + def _has_build_status(cls, boto_buildbatch: Dict[str, Any]) -> bool: + return BUILD_BATCH_STATUS in boto_buildbatch and boto_buildbatch[BUILD_BATCH_STATUS] is not None diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/secretsmanager_service.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/secretsmanager_service.py index f888b66..3f79b34 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/secretsmanager_service.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/secretsmanager_service.py @@ -15,7 +15,7 @@ def boto_client(self) -> Any: def get_secret_value(self, secret_id: PhysicalResourceId, - from_boto: Callable[[Dict[str, Any]], Secret]) \ + from_boto: Callable[[Dict[str, Any]], Secret] = Secret.from_boto) \ -> Secret: boto_secret = self._boto_client.get_secret_value(SecretId=secret_id.aws_physical_resource_id) secret = from_boto(boto_secret) diff --git a/test/unit_tests/aws/wrapper/datamodels/cloudformation/test_stack_resource_summary.py b/test/unit_tests/aws/wrapper/datamodels/cloudformation/test_stack_resource_summary.py index c862f29..8c7fe2e 100644 --- a/test/unit_tests/aws/wrapper/datamodels/cloudformation/test_stack_resource_summary.py +++ b/test/unit_tests/aws/wrapper/datamodels/cloudformation/test_stack_resource_summary.py @@ -17,11 +17,28 @@ def test_valid(): def test_without_physical_resource_id(): - with pytest.raises(KeyError): - boto_stack_resource_summary = { - "ResourceType": "resource_type" - } - stack_resource_summary = StackResourceSummary.from_boto(boto_stack_resource_summary) + expected_stack_resource_summary = StackResourceSummary( + physical_resource_id=None, + resource_type="resource_type") + + boto_stack_resource_summary = { + "ResourceType": "resource_type" + } + stack_resource_summary = StackResourceSummary.from_boto(boto_stack_resource_summary) + assert stack_resource_summary == expected_stack_resource_summary + + +def test_physical_resource_id_none(): + expected_stack_resource_summary = StackResourceSummary( + physical_resource_id=None, + resource_type="resource_type") + + boto_stack_resource_summary = { + "physical_resource_id": None, + "ResourceType": "resource_type" + } + stack_resource_summary = StackResourceSummary.from_boto(boto_stack_resource_summary) + assert stack_resource_summary == expected_stack_resource_summary def test_without_resource_type(): diff --git a/test/unit_tests/aws/wrapper/datamodels/codebuild/test_build_batch.py b/test/unit_tests/aws/wrapper/datamodels/codebuild/test_build_batch.py index da7b642..7372442 100644 --- a/test/unit_tests/aws/wrapper/datamodels/codebuild/test_build_batch.py +++ b/test/unit_tests/aws/wrapper/datamodels/codebuild/test_build_batch.py @@ -1,34 +1,83 @@ import pytest -from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.codebuild import BuildBatch +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.codebuild import BuildBatch, BuildBatchStatus from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId -def test_id_exists(): - expected_id = PhysicalResourceId("expected_id") - boto_buildbatch = {"id": expected_id.aws_physical_resource_id} +def test_with_id(): + expected_build_batch = BuildBatch( + id=PhysicalResourceId("expected_id"), + build_batch_status=None + ) + boto_buildbatch = { + "id": expected_build_batch.id.aws_physical_resource_id, + } build_batch = BuildBatch.from_boto(boto_buildbatch) - assert build_batch.id == expected_id + assert build_batch == expected_build_batch +@pytest.mark.parametrize("status", BuildBatchStatus) +def test_with_build_batch_status(status): + expected_build_batch = BuildBatch( + id=None, + build_batch_status=BuildBatchStatus.SUCCEEDED + ) + boto_buildbatch = { + "buildBatchStatus": expected_build_batch.build_batch_status.name + } + build_batch = BuildBatch.from_boto(boto_buildbatch) + assert build_batch == expected_build_batch -def test_id_not_exists(): - boto_buildbatch = {} +def test_with_unknown_build_batch_status(): + boto_buildbatch = { + "buildBatchStatus": "MY_UNKNOWN_BUILD_STATUS" + } with pytest.raises(KeyError): build_batch = BuildBatch.from_boto(boto_buildbatch) +def test_empty(): + expected_build_batch = BuildBatch( + id=None, + build_batch_status=None + ) + boto_buildbatch = {} + build_batch = BuildBatch.from_boto(boto_buildbatch) + assert build_batch == expected_build_batch + + def test_with_extra_keys(): - expected_id = PhysicalResourceId("expected_id") + expected_build_batch = BuildBatch( + id=PhysicalResourceId("expected_id"), + build_batch_status=BuildBatchStatus.SUCCEEDED + ) boto_buildbatch = { - "id": expected_id.aws_physical_resource_id, + "id": expected_build_batch.id.aws_physical_resource_id, + "buildBatchStatus": expected_build_batch.build_batch_status.name, "extra1": None, "extra2": 1 } build_batch = BuildBatch.from_boto(boto_buildbatch) - assert build_batch.id == expected_id + assert build_batch == expected_build_batch -def test_empty_id(): - with pytest.raises(ValueError, match="id was None"): - boto_buildbatch = {"id": None} - build_batch = BuildBatch.from_boto(boto_buildbatch) +def test_id_is_None(): + expected_build_batch = BuildBatch( + id=None, + build_batch_status=None + ) + boto_buildbatch = { + "id": None + } + build_batch = BuildBatch.from_boto(boto_buildbatch) + assert build_batch == expected_build_batch + +def test_build_batch_status_is_None(): + expected_build_batch = BuildBatch( + id=None, + build_batch_status=None + ) + boto_buildbatch = { + "buildBatchStatus": None + } + build_batch = BuildBatch.from_boto(boto_buildbatch) + assert build_batch == expected_build_batch From a1c2be5735599380199a4e2394fb0ea8ef9429a1 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 12 Jun 2023 09:41:41 +0200 Subject: [PATCH 3/8] Replaced direct MagicMock instantiation with create_autospec to catch wrong usage of methods of the mocked class --- .../lib/aws/aws_access.py | 4 +- .../lib/run_start_build.py | 44 ++++++++--- test/test_deploy_ci.py | 14 ++-- test/test_deploy_source_credentials.py | 16 ++-- test/test_start_ci_build.py | 67 ++++++---------- test/test_start_release_build.py | 74 ++++++----------- test/test_start_test_release_build.py | 79 ++++++++----------- .../unit_tests/aws/wrapper/test_aws_client.py | 5 +- 8 files changed, 130 insertions(+), 173 deletions(-) diff --git a/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py b/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py index 74412cd..f27fc2e 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py @@ -105,7 +105,7 @@ def get_all_stack_resources(self, stack_name: str) -> List[StackResourceSummary] return result def start_codebuild(self, - project: str, + project: PhysicalResourceId, environment_variables_overrides: List[Dict[str, str]], branch: str, poll_interval_seconds: int = 30) -> None: @@ -120,7 +120,7 @@ def start_codebuild(self, client = self._get_aws_client().create_codebuild_service() logging.info(f"Trigger codebuild for project {project} with branch {branch} " f"and env_variables ({environment_variables_overrides})") - build_batch = client.start_build_batch(project_name=PhysicalResourceId(project), + build_batch = client.start_build_batch(project_name=project, source_version=branch, environment_variables_override=list( environment_variables_overrides)) diff --git a/exasol_script_languages_container_ci_setup/lib/run_start_build.py b/exasol_script_languages_container_ci_setup/lib/run_start_build.py index a3ed6bd..36415f8 100644 --- a/exasol_script_languages_container_ci_setup/lib/run_start_build.py +++ b/exasol_script_languages_container_ci_setup/lib/run_start_build.py @@ -3,17 +3,21 @@ from typing import Tuple, Dict, List from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.cloudformation import StackResourceSummary from exasol_script_languages_container_ci_setup.lib.ci_build import ci_stack_name from exasol_script_languages_container_ci_setup.lib.github_draft_release_creator import GithubDraftReleaseCreator from exasol_script_languages_container_ci_setup.lib.release_build import release_stack_name +AWS_CODE_BUILD_PROJECT_RESOURCE_TYPE = "AWS::CodeBuild::Project" + def get_environment_variable_override(env_variable: Tuple[str, str]) -> Dict[str, str]: return {"name": env_variable[0], "value": env_variable[1], "type": "PLAINTEXT"} -def get_aws_codebuild_project(resources: List[Dict[str, str]], project: str) -> Dict[str, str]: - matching_project = [resource for resource in resources if resource["ResourceType"] == "AWS::CodeBuild::Project"] +def get_aws_codebuild_project(resources: List[StackResourceSummary], project: str) -> StackResourceSummary: + matching_project = [resource for resource in resources + if resource.resource_type == AWS_CODE_BUILD_PROJECT_RESOURCE_TYPE] if len(matching_project) == 0: raise ValueError(f"No project deployed for {project}. Found following resources: {resources}") if len(matching_project) > 1: @@ -34,7 +38,8 @@ def _parse_upload_url(upload_url: str) -> int: def _execute_release_build(aws_access: AwsAccess, project: str, branch: str, - release_id: int, is_dry_run: bool, gh_token: str) -> None: + release_id: int, is_dry_run: bool, gh_token: str, + timeout_time_in_seconds: int) -> None: """ This function: 1. Retrieve resources for the release codebuild stack for that given project @@ -61,26 +66,41 @@ def _execute_release_build(aws_access: AwsAccess, project: str, branch: str, ("DRY_RUN", dry_run_value), ("GITHUB_TOKEN", gh_token)] environment_variables_overrides = list(map(get_environment_variable_override, env_variables)) - aws_access.start_codebuild(matching_project["PhysicalResourceId"], + aws_access.start_codebuild(matching_project.physical_resource_id, environment_variables_overrides=environment_variables_overrides, - branch=branch) + branch=branch, timeout_time_in_seconds=timeout_time_in_seconds) -def run_start_release_build(aws_access: AwsAccess, project: str, upload_url: str, branch: str, gh_token: str) -> None: +def run_start_release_build(project: str, upload_url: str, + branch: str, gh_token: str, + timeout_time_in_seconds: int, + aws_access: AwsAccess) -> None: logging.info(f"run_start_release_build for aws profile {aws_access.aws_profile_for_logging} for project {project} " f"with upload url: {upload_url}") - _execute_release_build(aws_access, project, branch, _parse_upload_url(upload_url=upload_url), False, gh_token) + _execute_release_build(aws_access=aws_access, + project=project, + branch=branch, + release_id=_parse_upload_url(upload_url=upload_url), + is_dry_run=False, + gh_token=gh_token, + timeout_time_in_seconds=timeout_time_in_seconds) def run_start_test_release_build(aws_access: AwsAccess, gh_release_creator: GithubDraftReleaseCreator, repo_name: str, - project: str, branch: str, release_title: str, gh_token: str) -> None: + project: str, branch: str, release_title: str, gh_token: str, + timeout_time_in_seconds: int) -> None: logging.info(f"run_start_test_release_build for aws profile {aws_access.aws_profile_for_logging} " f"for project {project} for branch: {branch} with title: {release_title}") release_id = gh_release_creator.create_release(repo_name, branch, release_title, gh_token) - _execute_release_build(aws_access, project, branch, release_id, True, gh_token) + _execute_release_build( + aws_access=aws_access, + project=project, branch=branch, + release_id=release_id, is_dry_run=True, gh_token=gh_token, + timeout_time_in_seconds=timeout_time_in_seconds + ) -def run_start_ci_build(aws_access: AwsAccess, project: str, branch: str) -> None: +def run_start_ci_build(aws_access: AwsAccess, project: str, branch: str, timeout_time_in_seconds: int) -> None: logging.info(f"run_start_ci_build for aws profile {aws_access.aws_profile_for_logging} for project {project} " f"on branch {branch}") """ @@ -103,6 +123,6 @@ def run_start_ci_build(aws_access: AwsAccess, project: str, branch: str) -> None # (which itself uses the parameter to evaluate build strategies) env_variables = [("CUSTOM_BRANCH", branch)] environment_variables_overrides = list(map(get_environment_variable_override, env_variables)) - aws_access.start_codebuild(matching_project["PhysicalResourceId"], + aws_access.start_codebuild(matching_project.physical_resource_id, environment_variables_overrides=environment_variables_overrides, - branch=branch) + branch=branch, timeout_time_in_seconds=timeout_time_in_seconds) diff --git a/test/test_deploy_ci.py b/test/test_deploy_ci.py index c42e94b..1f11ac4 100644 --- a/test/test_deploy_ci.py +++ b/test/test_deploy_ci.py @@ -1,7 +1,9 @@ -from unittest.mock import MagicMock +from typing import Union +from unittest.mock import MagicMock, create_autospec, call import pytest +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.lib.ci_build import run_deploy_ci_build, ci_stack_name, \ CI_BUILD_WEBHOOK_FILTER_PATTERN from exasol_script_languages_container_ci_setup.lib.release_build import run_deploy_release_build, release_stack_name @@ -25,11 +27,11 @@ def test_deploy_ci_upload_invoked(ci_code_build_yml): Test if function upload_cloudformation_stack() will be invoked with expected values when we run run_deploy_ci_build() """ - aws_access_mock = MagicMock() + aws_access_mock: Union[MagicMock, AwsAccess] = create_autospec(AwsAccess) aws_access_mock.read_dockerhub_secret_arn.return_value = DOCKERHUB_SECRET_ARN run_deploy_ci_build(aws_access=aws_access_mock, project=PROJECT, github_url=GH_URL) - aws_access_mock.upload_cloudformation_stack.assert_called_once_with(ci_code_build_yml, ci_stack_name(PROJECT)) + assert call.upload_cloudformation_stack(ci_code_build_yml, ci_stack_name(PROJECT)) in aws_access_mock.mock_calls def test_deploy_ci_template(ci_code_build_yml): @@ -52,12 +54,12 @@ def test_deploy_release_upload_invoked(release_code_build_yml): Test if function upload_cloudformation_stack() will be invoked with expected values when we run run_deploy_release_build() """ - aws_access_mock = MagicMock() + aws_access_mock: Union[MagicMock, AwsAccess] = create_autospec(AwsAccess) aws_access_mock.read_dockerhub_secret_arn.return_value = DOCKERHUB_SECRET_ARN run_deploy_release_build(aws_access=aws_access_mock, project=PROJECT, github_url=GH_URL) - aws_access_mock.upload_cloudformation_stack.assert_called_once_with(release_code_build_yml, - release_stack_name(PROJECT)) + assert call.upload_cloudformation_stack(release_code_build_yml, release_stack_name(PROJECT)) \ + in aws_access_mock.mock_calls def test_deploy_release_template(release_code_build_yml): diff --git a/test/test_deploy_source_credentials.py b/test/test_deploy_source_credentials.py index cfbe2d2..643b7a6 100644 --- a/test/test_deploy_source_credentials.py +++ b/test/test_deploy_source_credentials.py @@ -1,16 +1,16 @@ from unittest.mock import MagicMock +from typing import Union +from unittest.mock import MagicMock, create_autospec, call import pytest +from test.unit_tests.cloudformation_validation import validate_using_cfn_lint -from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.lib.render_template import render_template from exasol_script_languages_container_ci_setup.lib.source_credentials import ( run_deploy_source_credentials, SOURCE_CREDENTIALS_STACK_NAME ) -from test.cloudformation_validation import validate_using_cfn_lint - SECRET_NAME = "test_secret" SECRET_USER_KEY = "test_secret_user_key" SECRET_TOKEN_KEY = "test_secret_token_key" @@ -29,15 +29,11 @@ def test_deploy_source_credentials_upload_invoked(source_credentials_yml): Test if function upload_cloudformation_stack() will be invoked with expected values when we run run_deploy_source_credentials() """ - aws_access_mock = MagicMock() + aws_access_mock: Union[MagicMock, AwsAccess] = create_autospec(AwsAccess) run_deploy_source_credentials(aws_access=aws_access_mock, secret_name=SECRET_NAME, secret_user_key=SECRET_USER_KEY, secret_token_key=SECRET_TOKEN_KEY) - aws_access_mock.upload_cloudformation_stack.assert_called_once_with(source_credentials_yml, SOURCE_CREDENTIALS_STACK_NAME) - - -def test_deploy_source_credentials_template(source_credentials_yml): - aws_access = AwsAccess(aws_profile=None) - aws_access.validate_cloudformation_template(source_credentials_yml) + assert call.upload_cloudformation_stack(source_credentials_yml, SOURCE_CREDENTIALS_STACK_NAME) \ + in aws_access_mock.mock_calls def test_deploy_source_credentials_template_with_cnf_lint(tmp_path, source_credentials_yml): diff --git a/test/test_start_ci_build.py b/test/test_start_ci_build.py index 21099f6..e400766 100644 --- a/test/test_start_ci_build.py +++ b/test/test_start_ci_build.py @@ -1,60 +1,39 @@ import datetime -from unittest.mock import MagicMock +from typing import Union +from unittest.mock import MagicMock, create_autospec, call from dateutil.tz import tzutc +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.cloudformation import StackResourceSummary +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId from exasol_script_languages_container_ci_setup.lib.run_start_build import run_start_ci_build REPO_NAME = "script-languages-repo" BRANCH = "feature-branch" -#Original resources extracted from a ScriptLanguage cloudformation stack -DUMMY_RESOURCES = [ - {'LogicalResourceId': 'ArtifactsBucket', - 'PhysicalResourceId': 'scriptlanguagesreleasebuil-artifactsbucket-6ikq8b0bojhj', - 'ResourceType': 'AWS::S3::Bucket', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 38, 36, 391000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', - 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - }, - {'LogicalResourceId': 'BatchBuildRole', - 'PhysicalResourceId': 'ScriptLanguagesReleaseBuild-BatchBuildRole-18RVZAPWKW3ZB', - 'ResourceType': 'AWS::IAM::Role', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 38, 35, 103000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', - 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - }, - {'LogicalResourceId': 'BuildLogGroup', - 'PhysicalResourceId': '/aws/codebuild/ScriptLanguagesCodeB-FTGeeZLjmjX7', - 'ResourceType': 'AWS::Logs::LogGroup', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 39, 11, 935000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - }, - {'LogicalResourceId': 'CodeBuildRole', - 'PhysicalResourceId': 'ScriptLanguagesBuild-CodeBuildRole-1WPN324U80IRE', - 'ResourceType': 'AWS::IAM::Role', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 39, 1, 806000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - }, - {'LogicalResourceId': 'ScriptLanguagesCodeBuild', - 'PhysicalResourceId': 'ScriptLanguagesCodeB-FTGeeZLjmjX7', - 'ResourceType': 'AWS::CodeBuild::Project', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 39, 7, 850000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - } -] - def test_run_ci_build(): """ Test if invocation of run_start_ci_build calls AwsAccess with expected arguments. """ - aws_access_mock = MagicMock() - aws_access_mock.get_all_stack_resources.return_value = DUMMY_RESOURCES - run_start_ci_build(aws_access=aws_access_mock, project="slc", branch=BRANCH) + aws_access_mock: Union[MagicMock, AwsAccess] = create_autospec(AwsAccess) + physical_resource_id = PhysicalResourceId(aws_physical_resource_id="id") + aws_access_mock.get_all_stack_resources.return_value = [ + StackResourceSummary(physical_resource_id=None, + resource_type="SomethingElse"), + StackResourceSummary(physical_resource_id=physical_resource_id, + resource_type="AWS::CodeBuild::Project"), + StackResourceSummary(physical_resource_id=None, + resource_type="SomethingElse") + ] + timeout_time_in_seconds = 30 expected_env_variable_overrides = [{"name": "CUSTOM_BRANCH", "value": BRANCH, "type": "PLAINTEXT"}] + run_start_ci_build(aws_access=aws_access_mock, project="slc", branch=BRANCH, + timeout_time_in_seconds=timeout_time_in_seconds) - aws_access_mock. \ - start_codebuild.assert_called_once_with("ScriptLanguagesCodeB-FTGeeZLjmjX7", - environment_variables_overrides=expected_env_variable_overrides, - branch=BRANCH) + assert call.start_codebuild(physical_resource_id, + environment_variables_overrides=expected_env_variable_overrides, + branch=BRANCH, + timeout_time_in_seconds=timeout_time_in_seconds) \ + in aws_access_mock.mock_calls diff --git a/test/test_start_release_build.py b/test/test_start_release_build.py index 9ecdb51..3715bc5 100644 --- a/test/test_start_release_build.py +++ b/test/test_start_release_build.py @@ -1,67 +1,43 @@ -import datetime -import os -from unittest.mock import MagicMock - -from dateutil.tz import tzutc +from typing import Union +from unittest.mock import MagicMock, create_autospec, call +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.cloudformation import StackResourceSummary +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId from exasol_script_languages_container_ci_setup.lib.run_start_build import run_start_release_build UPLOAD_URL = "https://uploads.github.com/repos/exasol/script-languages-repo/releases/123/assets{?name,label}" BRANCH = "main" GITHUB_TOKEN = "gh_secret" -#Original resources extracted from a ScriptLanguage cloudformation stack -DUMMY_RESOURCES = [ - {'LogicalResourceId': 'ReleaseArtifactsBucket', - 'PhysicalResourceId': 'scriptlanguagesreleasebuil-releaseartifactsbucket-6ikq8b0bojhj', - 'ResourceType': 'AWS::S3::Bucket', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 38, 36, 391000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', - 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - }, - {'LogicalResourceId': 'ReleaseBatchBuildRole', - 'PhysicalResourceId': 'ScriptLanguagesReleaseBuild-ReleaseBatchBuildRole-18RVZAPWKW3ZB', - 'ResourceType': 'AWS::IAM::Role', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 38, 35, 103000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', - 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - }, - {'LogicalResourceId': 'ReleaseCodeBuildLogGroup', - 'PhysicalResourceId': '/aws/codebuild/ScriptLanguagesReleaseCodeB-FTGeeZLjmjX7', - 'ResourceType': 'AWS::Logs::LogGroup', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 39, 11, 935000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - }, - {'LogicalResourceId': 'ReleaseCodeBuildRole', - 'PhysicalResourceId': 'ScriptLanguagesReleaseBuild-ReleaseCodeBuildRole-1WPN324U80IRE', - 'ResourceType': 'AWS::IAM::Role', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 39, 1, 806000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - }, - {'LogicalResourceId': 'ScriptLanguagesReleaseCodeBuild', - 'PhysicalResourceId': 'ScriptLanguagesReleaseCodeB-FTGeeZLjmjX7', - 'ResourceType': 'AWS::CodeBuild::Project', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 39, 7, 850000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - } -] - def test_run_release_build(): """ Test if invocation of run_start_release_build calls AwsAccess with expected arguments. """ - aws_access_mock = MagicMock() - aws_access_mock.get_all_stack_resources.return_value = DUMMY_RESOURCES - run_start_release_build(aws_access=aws_access_mock, project="slc", - upload_url=UPLOAD_URL, branch=BRANCH, gh_token=GITHUB_TOKEN) + aws_access_mock: Union[MagicMock, AwsAccess] = create_autospec(AwsAccess) + physical_resource_id = PhysicalResourceId(aws_physical_resource_id="id") + aws_access_mock.get_all_stack_resources.return_value = [ + StackResourceSummary(physical_resource_id=None, + resource_type="SomethingElse"), + StackResourceSummary(physical_resource_id=physical_resource_id, + resource_type="AWS::CodeBuild::Project"), + StackResourceSummary(physical_resource_id=None, + resource_type="SomethingElse") + ] + timeout_time_in_seconds = 30 expected_env_variable_overrides = [ {"name": "RELEASE_ID", "value": "123", "type": "PLAINTEXT"}, {"name": "DRY_RUN", "value": "--no-dry-run", "type": "PLAINTEXT"}, {"name": "GITHUB_TOKEN", "value": GITHUB_TOKEN, "type": "PLAINTEXT"} ] - aws_access_mock. \ - start_codebuild.assert_called_once_with("ScriptLanguagesReleaseCodeB-FTGeeZLjmjX7", - environment_variables_overrides=expected_env_variable_overrides, - branch=BRANCH) + run_start_release_build(aws_access=aws_access_mock, project="slc", + upload_url=UPLOAD_URL, branch=BRANCH, gh_token=GITHUB_TOKEN, + timeout_time_in_seconds=timeout_time_in_seconds) + + assert call.start_codebuild(physical_resource_id, + environment_variables_overrides=expected_env_variable_overrides, + branch=BRANCH, + timeout_time_in_seconds=timeout_time_in_seconds) \ + in aws_access_mock.mock_calls diff --git a/test/test_start_test_release_build.py b/test/test_start_test_release_build.py index 7eafa45..ae4d982 100644 --- a/test/test_start_test_release_build.py +++ b/test/test_start_test_release_build.py @@ -1,70 +1,53 @@ import datetime -from unittest.mock import MagicMock +from typing import Union +from unittest.mock import MagicMock, create_autospec, call from dateutil.tz import tzutc +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.cloudformation import StackResourceSummary +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId +from exasol_script_languages_container_ci_setup.lib.github_draft_release_creator import GithubDraftReleaseCreator from exasol_script_languages_container_ci_setup.lib.run_start_build import run_start_test_release_build +from test.mock_cast import mock_cast REPO_NAME = "script-languages-repo" BRANCH = "main" GITHUB_TOKEN = "gh_secret" RELEASE_TITLE = "test-release" -#Original resources extracted from a ScriptLanguage cloudformation stack -DUMMY_RESOURCES = [ - {'LogicalResourceId': 'ReleaseArtifactsBucket', - 'PhysicalResourceId': 'scriptlanguagesreleasebuil-releaseartifactsbucket-6ikq8b0bojhj', - 'ResourceType': 'AWS::S3::Bucket', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 38, 36, 391000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', - 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - }, - {'LogicalResourceId': 'ReleaseBatchBuildRole', - 'PhysicalResourceId': 'ScriptLanguagesReleaseBuild-ReleaseBatchBuildRole-18RVZAPWKW3ZB', - 'ResourceType': 'AWS::IAM::Role', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 38, 35, 103000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', - 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - }, - {'LogicalResourceId': 'ReleaseCodeBuildLogGroup', - 'PhysicalResourceId': '/aws/codebuild/ScriptLanguagesReleaseCodeB-FTGeeZLjmjX7', - 'ResourceType': 'AWS::Logs::LogGroup', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 39, 11, 935000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - }, - {'LogicalResourceId': 'ReleaseCodeBuildRole', - 'PhysicalResourceId': 'ScriptLanguagesReleaseBuild-ReleaseCodeBuildRole-1WPN324U80IRE', - 'ResourceType': 'AWS::IAM::Role', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 39, 1, 806000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - }, - {'LogicalResourceId': 'ScriptLanguagesReleaseCodeBuild', - 'PhysicalResourceId': 'ScriptLanguagesReleaseCodeB-FTGeeZLjmjX7', - 'ResourceType': 'AWS::CodeBuild::Project', - 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 39, 7, 850000, tzinfo=tzutc()), - 'ResourceStatus': 'CREATE_COMPLETE', 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} - } -] - def test_run_test_release_build(): """ Test if invocation of run_start_test_release_build calls AwsAccess with expected arguments. """ - aws_access_mock = MagicMock() - github_release_creator_mock = MagicMock() - aws_access_mock.get_all_stack_resources.return_value = DUMMY_RESOURCES - github_release_creator_mock.create_release.return_value = 123 - run_start_test_release_build(aws_access=aws_access_mock, gh_release_creator=github_release_creator_mock, - project="slc", repo_name=REPO_NAME, branch=BRANCH, - release_title=RELEASE_TITLE, gh_token=GITHUB_TOKEN) + aws_access_mock: Union[MagicMock, AwsAccess] = create_autospec(AwsAccess) + physical_resource_id = PhysicalResourceId(aws_physical_resource_id="id") + aws_access_mock.get_all_stack_resources.return_value = [ + StackResourceSummary(physical_resource_id=None, + resource_type="SomethingElse"), + StackResourceSummary(physical_resource_id=physical_resource_id, + resource_type="AWS::CodeBuild::Project"), + StackResourceSummary(physical_resource_id=None, + resource_type="SomethingElse") + ] + timeout_time_in_seconds = 30 + github_release_creator_mock: Union[MagicMock, GithubDraftReleaseCreator] = \ + create_autospec(GithubDraftReleaseCreator) + mock_cast(github_release_creator_mock.create_release).return_value = 123 expected_env_variable_overrides = [ {"name": "RELEASE_ID", "value": "123", "type": "PLAINTEXT"}, {"name": "DRY_RUN", "value": "--dry-run", "type": "PLAINTEXT"}, {"name": "GITHUB_TOKEN", "value": GITHUB_TOKEN, "type": "PLAINTEXT"} ] - aws_access_mock. \ - start_codebuild.assert_called_once_with("ScriptLanguagesReleaseCodeB-FTGeeZLjmjX7", - environment_variables_overrides=expected_env_variable_overrides, - branch=BRANCH) + run_start_test_release_build(aws_access=aws_access_mock, gh_release_creator=github_release_creator_mock, + project="slc", repo_name=REPO_NAME, branch=BRANCH, + release_title=RELEASE_TITLE, gh_token=GITHUB_TOKEN, + timeout_time_in_seconds=timeout_time_in_seconds) + + assert call.start_codebuild(physical_resource_id, + environment_variables_overrides=expected_env_variable_overrides, + branch=BRANCH, + timeout_time_in_seconds=timeout_time_in_seconds) \ + in aws_access_mock.mock_calls diff --git a/test/unit_tests/aws/wrapper/test_aws_client.py b/test/unit_tests/aws/wrapper/test_aws_client.py index d2e089d..38d4f06 100644 --- a/test/unit_tests/aws/wrapper/test_aws_client.py +++ b/test/unit_tests/aws/wrapper/test_aws_client.py @@ -2,7 +2,6 @@ from typing import Union from unittest.mock import MagicMock, Mock, call, create_autospec -import pytest from boto3 import Session from exasol_script_languages_container_ci_setup.lib.aws.wrapper.aws_client import AwsClient @@ -10,7 +9,7 @@ def test_init(): - boto_session_factory: MagicMock = Mock() + boto_session_factory = Mock() aws_client = AwsClient(profile="profile", region="region", boto_session_factory=boto_session_factory) assert boto_session_factory.mock_calls == [] @@ -38,6 +37,7 @@ def test_create_codebuild_service_boto_session_client(): call.client('codebuild'), ] + def test_create_codebuild_service_boto_client(): setup = CodeBuildServiceTestSetup() assert setup.codebuild_service.boto_client == mock_cast(setup.boto_session.client).return_value @@ -65,6 +65,7 @@ def test_create_cloudformation_service_boto_session_client(): call.client('cloudformation'), ] + def test_create_cloudformation_service_boto_client(): setup = CloudFormationServiceTestSetup() assert setup.cloudformation_service.boto_client == mock_cast(setup.boto_session.client).return_value From 5d65ffd0a6ba2a2d3194159c18656b1ed07e24fa Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 12 Jun 2023 09:41:41 +0200 Subject: [PATCH 4/8] Sorted tests into integration_tests and unit_tests and fixed some things left by the split from the timeout PR --- .../lib/aws/aws_access.py | 4 ++-- .../lib/aws/wrapper/cloudformation_service.py | 13 +++++++--- .../lib/aws/wrapper/codebuild_service.py | 3 ++- .../lib/run_start_build.py | 17 +++++-------- test/integration_tests/__init__.py | 0 .../{ => integration_tests}/test_deploy_ci.py | 2 +- test/mock_cast.py | 6 +++++ test/unit_tests/__init__.py | 0 .../wrapper/test_cloudformation_service.py | 24 ++++++++++++------- .../aws/wrapper/test_codebuild_service.py | 4 +++- .../cloudformation_validation.py | 0 .../test_deploy_source_credentials.py | 4 ++-- .../test_generate_buildspec.py | 0 test/{ => unit_tests}/test_start_ci_build.py | 9 ++----- .../test_start_release_build.py | 6 ++--- .../test_start_test_release_build.py | 6 ++--- .../test_webhook_filter_pattern.py | 0 17 files changed, 54 insertions(+), 44 deletions(-) create mode 100644 test/integration_tests/__init__.py rename test/{ => integration_tests}/test_deploy_ci.py (97%) create mode 100644 test/mock_cast.py create mode 100644 test/unit_tests/__init__.py rename test/{ => unit_tests}/cloudformation_validation.py (100%) rename test/{ => unit_tests}/test_deploy_source_credentials.py (95%) rename test/{ => unit_tests}/test_generate_buildspec.py (100%) rename test/{ => unit_tests}/test_start_ci_build.py (86%) rename test/{ => unit_tests}/test_start_release_build.py (89%) rename test/{ => unit_tests}/test_start_test_release_build.py (91%) rename test/{ => unit_tests}/test_webhook_filter_pattern.py (100%) diff --git a/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py b/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py index f27fc2e..dbbfa64 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py @@ -16,7 +16,7 @@ class AwsAccess: def __init__(self, aws_profile: Optional[str], - aws_client_factory: AwsClientFactory): + aws_client_factory: AwsClientFactory = AwsClientFactory()): self._aws_client_factory = aws_client_factory self._aws_profile = aws_profile @@ -99,7 +99,7 @@ def get_all_stack_resources(self, stack_name: str) -> List[StackResourceSummary] current_result = client.list_stack_resources(stack_name=stack_name_id) result = current_result.stack_resource_summaries - while "nextToken" in current_result: + while current_result.next_token is not None: current_result = client.list_stack_resources(stack_name=stack_name_id, next_token=current_result.next_token) result.extend(current_result.stack_resource_summaries) return result diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py index 608db46..3767187 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py @@ -28,8 +28,15 @@ def list_stack_resources(self, from_boto: Callable[[Dict[str, Any]], ListStackResourcesResult] = ListStackResourcesResult.from_boto) \ -> ListStackResourcesResult: - aws_next_token = None if next_token is None else next_token.aws_next_token - boto_list_stack_resources_result = self._boto_client.list_stack_resources( - StackName=stack_name.aws_physical_resource_id, NextToken=aws_next_token) + if next_token is not None: + boto_list_stack_resources_result = \ + self._boto_client.list_stack_resources( + StackName=stack_name.aws_physical_resource_id, + NextToken=next_token.aws_next_token) + else: + boto_list_stack_resources_result = \ + self._boto_client.list_stack_resources( + StackName=stack_name.aws_physical_resource_id) + list_stack_resources_result = from_boto(boto_list_stack_resources_result) return list_stack_resources_result diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/codebuild_service.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/codebuild_service.py index 939dff0..0e33a08 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/codebuild_service.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/codebuild_service.py @@ -32,5 +32,6 @@ def batch_get_build_batches(self, -> List[BuildBatch]: aws_ids = [build_batch_id.aws_physical_resource_id for build_batch_id in build_batch_ids] boto_build_batches = self._boto_client.batch_get_build_batches(ids=aws_ids) - build_batches = [from_boto(boto_build_batch) for boto_build_batch in boto_build_batches] + print(boto_build_batches) + build_batches = [from_boto(boto_build_batch) for boto_build_batch in boto_build_batches['buildBatches']] return build_batches diff --git a/exasol_script_languages_container_ci_setup/lib/run_start_build.py b/exasol_script_languages_container_ci_setup/lib/run_start_build.py index 36415f8..2b8f26e 100644 --- a/exasol_script_languages_container_ci_setup/lib/run_start_build.py +++ b/exasol_script_languages_container_ci_setup/lib/run_start_build.py @@ -38,8 +38,7 @@ def _parse_upload_url(upload_url: str) -> int: def _execute_release_build(aws_access: AwsAccess, project: str, branch: str, - release_id: int, is_dry_run: bool, gh_token: str, - timeout_time_in_seconds: int) -> None: + release_id: int, is_dry_run: bool, gh_token: str) -> None: """ This function: 1. Retrieve resources for the release codebuild stack for that given project @@ -68,12 +67,11 @@ def _execute_release_build(aws_access: AwsAccess, project: str, branch: str, environment_variables_overrides = list(map(get_environment_variable_override, env_variables)) aws_access.start_codebuild(matching_project.physical_resource_id, environment_variables_overrides=environment_variables_overrides, - branch=branch, timeout_time_in_seconds=timeout_time_in_seconds) + branch=branch) def run_start_release_build(project: str, upload_url: str, branch: str, gh_token: str, - timeout_time_in_seconds: int, aws_access: AwsAccess) -> None: logging.info(f"run_start_release_build for aws profile {aws_access.aws_profile_for_logging} for project {project} " f"with upload url: {upload_url}") @@ -82,13 +80,11 @@ def run_start_release_build(project: str, upload_url: str, branch=branch, release_id=_parse_upload_url(upload_url=upload_url), is_dry_run=False, - gh_token=gh_token, - timeout_time_in_seconds=timeout_time_in_seconds) + gh_token=gh_token) def run_start_test_release_build(aws_access: AwsAccess, gh_release_creator: GithubDraftReleaseCreator, repo_name: str, - project: str, branch: str, release_title: str, gh_token: str, - timeout_time_in_seconds: int) -> None: + project: str, branch: str, release_title: str, gh_token: str) -> None: logging.info(f"run_start_test_release_build for aws profile {aws_access.aws_profile_for_logging} " f"for project {project} for branch: {branch} with title: {release_title}") release_id = gh_release_creator.create_release(repo_name, branch, release_title, gh_token) @@ -96,11 +92,10 @@ def run_start_test_release_build(aws_access: AwsAccess, gh_release_creator: Gith aws_access=aws_access, project=project, branch=branch, release_id=release_id, is_dry_run=True, gh_token=gh_token, - timeout_time_in_seconds=timeout_time_in_seconds ) -def run_start_ci_build(aws_access: AwsAccess, project: str, branch: str, timeout_time_in_seconds: int) -> None: +def run_start_ci_build(aws_access: AwsAccess, project: str, branch: str) -> None: logging.info(f"run_start_ci_build for aws profile {aws_access.aws_profile_for_logging} for project {project} " f"on branch {branch}") """ @@ -125,4 +120,4 @@ def run_start_ci_build(aws_access: AwsAccess, project: str, branch: str, timeout environment_variables_overrides = list(map(get_environment_variable_override, env_variables)) aws_access.start_codebuild(matching_project.physical_resource_id, environment_variables_overrides=environment_variables_overrides, - branch=branch, timeout_time_in_seconds=timeout_time_in_seconds) + branch=branch) diff --git a/test/integration_tests/__init__.py b/test/integration_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_deploy_ci.py b/test/integration_tests/test_deploy_ci.py similarity index 97% rename from test/test_deploy_ci.py rename to test/integration_tests/test_deploy_ci.py index 1f11ac4..a082fd0 100644 --- a/test/test_deploy_ci.py +++ b/test/integration_tests/test_deploy_ci.py @@ -8,7 +8,7 @@ CI_BUILD_WEBHOOK_FILTER_PATTERN from exasol_script_languages_container_ci_setup.lib.release_build import run_deploy_release_build, release_stack_name from exasol_script_languages_container_ci_setup.lib.render_template import render_template -from test.cloudformation_validation import validate_using_cfn_lint +from test.unit_tests.cloudformation_validation import validate_using_cfn_lint PROJECT = "slc" GH_URL = "https://github.com/slc" diff --git a/test/mock_cast.py b/test/mock_cast.py new file mode 100644 index 0000000..3a3866e --- /dev/null +++ b/test/mock_cast.py @@ -0,0 +1,6 @@ +from typing import Any, cast +from unittest.mock import Mock + + +def mock_cast(obj: Any) -> Mock: + return cast(Mock, obj) diff --git a/test/unit_tests/__init__.py b/test/unit_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit_tests/aws/wrapper/test_cloudformation_service.py b/test/unit_tests/aws/wrapper/test_cloudformation_service.py index bfd175f..094cf0e 100644 --- a/test/unit_tests/aws/wrapper/test_cloudformation_service.py +++ b/test/unit_tests/aws/wrapper/test_cloudformation_service.py @@ -57,18 +57,26 @@ def __init__(self, next_token: Optional[NextToken]): self.boto_client.list_stack_resources).return_value -list_stack_resources_parameters = pytest.mark.parametrize("next_token", [ - NextToken(aws_next_token="next_token"), - None -]) +def test_list_stack_resources_boto_client_next_token_is_none(): + setup = ListStackResourcesTestSetup(None) + assert setup.boto_client.mock_calls == [ + call.list_stack_resources(StackName=setup.physical_resource_id.aws_physical_resource_id)] -@list_stack_resources_parameters -def test_list_stack_resources_boto_client(next_token): +def test_list_stack_resources_boto_client_next_token_is_not_none(): + next_token = NextToken("next_token") setup = ListStackResourcesTestSetup(next_token) assert setup.boto_client.mock_calls == [ - call.list_stack_resources(StackName=setup.physical_resource_id.aws_physical_resource_id, - NextToken=setup.aws_next_token)] + call.list_stack_resources( + StackName=setup.physical_resource_id.aws_physical_resource_id, + NextToken=next_token.aws_next_token) + ] + + +list_stack_resources_parameters = pytest.mark.parametrize("next_token", [ + NextToken(aws_next_token="next_token"), + None +]) @list_stack_resources_parameters diff --git a/test/unit_tests/aws/wrapper/test_codebuild_service.py b/test/unit_tests/aws/wrapper/test_codebuild_service.py index eb291bd..5482285 100644 --- a/test/unit_tests/aws/wrapper/test_codebuild_service.py +++ b/test/unit_tests/aws/wrapper/test_codebuild_service.py @@ -16,7 +16,9 @@ def test_init(): class BatchGetBuildBatchesTestSetup: boto_client = Mock() batch_get_build_batches_return_values = [Mock(), Mock()] - mock_cast(boto_client.batch_get_build_batches).return_value = batch_get_build_batches_return_values + mock_cast(boto_client.batch_get_build_batches).return_value = { + 'buildBatches': batch_get_build_batches_return_values + } from_boto = Mock() from_boto_return_values = [Mock(), Mock()] from_boto.side_effect = from_boto_return_values diff --git a/test/cloudformation_validation.py b/test/unit_tests/cloudformation_validation.py similarity index 100% rename from test/cloudformation_validation.py rename to test/unit_tests/cloudformation_validation.py diff --git a/test/test_deploy_source_credentials.py b/test/unit_tests/test_deploy_source_credentials.py similarity index 95% rename from test/test_deploy_source_credentials.py rename to test/unit_tests/test_deploy_source_credentials.py index 643b7a6..5e21133 100644 --- a/test/test_deploy_source_credentials.py +++ b/test/unit_tests/test_deploy_source_credentials.py @@ -1,15 +1,15 @@ -from unittest.mock import MagicMock from typing import Union from unittest.mock import MagicMock, create_autospec, call import pytest -from test.unit_tests.cloudformation_validation import validate_using_cfn_lint +from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.lib.render_template import render_template from exasol_script_languages_container_ci_setup.lib.source_credentials import ( run_deploy_source_credentials, SOURCE_CREDENTIALS_STACK_NAME ) +from test.unit_tests.cloudformation_validation import validate_using_cfn_lint SECRET_NAME = "test_secret" SECRET_USER_KEY = "test_secret_user_key" diff --git a/test/test_generate_buildspec.py b/test/unit_tests/test_generate_buildspec.py similarity index 100% rename from test/test_generate_buildspec.py rename to test/unit_tests/test_generate_buildspec.py diff --git a/test/test_start_ci_build.py b/test/unit_tests/test_start_ci_build.py similarity index 86% rename from test/test_start_ci_build.py rename to test/unit_tests/test_start_ci_build.py index e400766..ac4ce24 100644 --- a/test/test_start_ci_build.py +++ b/test/unit_tests/test_start_ci_build.py @@ -1,9 +1,6 @@ -import datetime from typing import Union from unittest.mock import MagicMock, create_autospec, call -from dateutil.tz import tzutc - from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.cloudformation import StackResourceSummary from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId @@ -29,11 +26,9 @@ def test_run_ci_build(): ] timeout_time_in_seconds = 30 expected_env_variable_overrides = [{"name": "CUSTOM_BRANCH", "value": BRANCH, "type": "PLAINTEXT"}] - run_start_ci_build(aws_access=aws_access_mock, project="slc", branch=BRANCH, - timeout_time_in_seconds=timeout_time_in_seconds) + run_start_ci_build(aws_access=aws_access_mock, project="slc", branch=BRANCH) assert call.start_codebuild(physical_resource_id, environment_variables_overrides=expected_env_variable_overrides, - branch=BRANCH, - timeout_time_in_seconds=timeout_time_in_seconds) \ + branch=BRANCH) \ in aws_access_mock.mock_calls diff --git a/test/test_start_release_build.py b/test/unit_tests/test_start_release_build.py similarity index 89% rename from test/test_start_release_build.py rename to test/unit_tests/test_start_release_build.py index 3715bc5..98b028a 100644 --- a/test/test_start_release_build.py +++ b/test/unit_tests/test_start_release_build.py @@ -33,11 +33,9 @@ def test_run_release_build(): ] run_start_release_build(aws_access=aws_access_mock, project="slc", - upload_url=UPLOAD_URL, branch=BRANCH, gh_token=GITHUB_TOKEN, - timeout_time_in_seconds=timeout_time_in_seconds) + upload_url=UPLOAD_URL, branch=BRANCH, gh_token=GITHUB_TOKEN) assert call.start_codebuild(physical_resource_id, environment_variables_overrides=expected_env_variable_overrides, - branch=BRANCH, - timeout_time_in_seconds=timeout_time_in_seconds) \ + branch=BRANCH) \ in aws_access_mock.mock_calls diff --git a/test/test_start_test_release_build.py b/test/unit_tests/test_start_test_release_build.py similarity index 91% rename from test/test_start_test_release_build.py rename to test/unit_tests/test_start_test_release_build.py index ae4d982..1520102 100644 --- a/test/test_start_test_release_build.py +++ b/test/unit_tests/test_start_test_release_build.py @@ -43,11 +43,9 @@ def test_run_test_release_build(): run_start_test_release_build(aws_access=aws_access_mock, gh_release_creator=github_release_creator_mock, project="slc", repo_name=REPO_NAME, branch=BRANCH, - release_title=RELEASE_TITLE, gh_token=GITHUB_TOKEN, - timeout_time_in_seconds=timeout_time_in_seconds) + release_title=RELEASE_TITLE, gh_token=GITHUB_TOKEN) assert call.start_codebuild(physical_resource_id, environment_variables_overrides=expected_env_variable_overrides, - branch=BRANCH, - timeout_time_in_seconds=timeout_time_in_seconds) \ + branch=BRANCH) \ in aws_access_mock.mock_calls diff --git a/test/test_webhook_filter_pattern.py b/test/unit_tests/test_webhook_filter_pattern.py similarity index 100% rename from test/test_webhook_filter_pattern.py rename to test/unit_tests/test_webhook_filter_pattern.py From d19a4bc788b5e2041b6867f29c1c0b22fcdcc193 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 12 Jun 2023 19:34:32 +0200 Subject: [PATCH 5/8] Add contract tests for code_build_service and secret_manager --- test/contract_tests/__init__.py | 0 test/contract_tests/aws_access/__init__.py | 0 .../aws_access/code_build_service.py | 129 ++++++++++++++++++ .../aws_access/secret_manager.py | 28 ++++ 4 files changed, 157 insertions(+) create mode 100644 test/contract_tests/__init__.py create mode 100644 test/contract_tests/aws_access/__init__.py create mode 100644 test/contract_tests/aws_access/code_build_service.py create mode 100644 test/contract_tests/aws_access/secret_manager.py diff --git a/test/contract_tests/__init__.py b/test/contract_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/contract_tests/aws_access/__init__.py b/test/contract_tests/aws_access/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/contract_tests/aws_access/code_build_service.py b/test/contract_tests/aws_access/code_build_service.py new file mode 100644 index 0000000..ff031e1 --- /dev/null +++ b/test/contract_tests/aws_access/code_build_service.py @@ -0,0 +1,129 @@ +from typing import List, Dict, Tuple + +import pytest + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.codebuild_service import CodeBuildService +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.codebuild import BuildBatchStatus, BuildBatch +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId + + +class StartCodeBuildContract: + + @pytest.fixture(scope="class") + def codebuild_service(self) -> CodeBuildService: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def project_name(self) -> PhysicalResourceId: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def source_version(self) -> str: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def environment_variables_override(self) -> List[Dict[str, str]]: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def build_batch(self, codebuild_service, project_name, source_version, + environment_variables_override) -> BuildBatch: + build_batch = codebuild_service.start_build_batch( + project_name=project_name, + source_version=source_version, + environment_variables_override=environment_variables_override) + return build_batch + + def test_build_batch_status(self, build_batch): + assert build_batch.build_batch_status in set(BuildBatchStatus) + + +class BatchGetBuildBatchesWithBatchIdFromStartBuildCode: + + @pytest.fixture(scope="class") + def codebuild_service(self) -> CodeBuildService: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def project_name(self) -> PhysicalResourceId: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def source_version(self) -> str: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def environment_variables_override(self) -> List[Dict[str, str]]: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def build_batches(self, codebuild_service, project_name, source_version, + environment_variables_override) -> Tuple[BuildBatch, BuildBatch]: + build_batch_from_start = codebuild_service.start_build_batch( + project_name=project_name, + source_version=source_version, + environment_variables_override=environment_variables_override) + build_batches_from_get = codebuild_service.batch_get_build_batches(build_batch_ids=[build_batch_from_start.id]) + return build_batch_from_start, build_batches_from_get[0] + + def test_build_batch_id(self, build_batches): + assert build_batches[0].id == build_batches[1].id + + def test_build_batch_status_from_start(self, build_batches): + assert build_batches[0].build_batch_status in set(BuildBatchStatus) + + def test_build_batch_status_from_get(self, build_batches): + assert build_batches[1].build_batch_status in set(BuildBatchStatus) + + +class BatchGetBuildBatchesSingleBuildIdContract: + + @pytest.fixture(scope="class") + def codebuild_service(self) -> CodeBuildService: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def build_batch_id(self, codebuild_service) -> PhysicalResourceId: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def build_batches(self, codebuild_service, build_batch_id) -> List[BuildBatch]: + build_batches = codebuild_service.batch_get_build_batches(build_batch_ids=[build_batch_id]) + return build_batches + + @pytest.fixture + def expected_build_batch_status(self) -> BuildBatchStatus: + raise NotImplementedError() + + def test_length(self, build_batches): + assert len(build_batches) == 1 + + def test_same_id(self, build_batch_id, build_batches): + assert build_batches[0].id == build_batch_id + + def test_build_batch_status(self, build_batches, expected_build_batch_status): + assert build_batches[0].build_batch_status == expected_build_batch_status + + +class BatchGetBuildBatchesSingleBuildIdInProgressContract(BatchGetBuildBatchesSingleBuildIdContract): + + def expected_build_batch_status(self) -> BuildBatchStatus: + return BuildBatchStatus.IN_PROGRESS + + +class BatchGetBuildBatchesSingleBuildIdSucceededContract(BatchGetBuildBatchesSingleBuildIdContract): + + def expected_build_batch_status(self) -> BuildBatchStatus: + return BuildBatchStatus.SUCCEEDED + + +class BatchGetBuildBatchesSingleBuildIdFailedContract(BatchGetBuildBatchesSingleBuildIdContract): + + def expected_build_batch_status(self) -> BuildBatchStatus: + return BuildBatchStatus.FAILED + + +class BatchGetBuildBatchesSingleBuildIdStoppedContract(BatchGetBuildBatchesSingleBuildIdContract): + + def expected_build_batch_status(self) -> BuildBatchStatus: + return BuildBatchStatus.STOPPED diff --git a/test/contract_tests/aws_access/secret_manager.py b/test/contract_tests/aws_access/secret_manager.py new file mode 100644 index 0000000..50f8859 --- /dev/null +++ b/test/contract_tests/aws_access/secret_manager.py @@ -0,0 +1,28 @@ +import pytest + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId, ARN +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.secretsmanager import Secret +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.secretsmanager_service import SecretsManagerService + + +class GetSecretValueArnEqualContract: + + @pytest.fixture(scope="class") + def secretmanager_service(self) -> SecretsManagerService: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def secret_id(self) -> PhysicalResourceId: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def expected_secret_arn(self) -> ARN: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def secret(self, secretmanager_service, secret_id) -> Secret: + secret = secretmanager_service.get_secret_value(secret_id) + return secret + + def test_id(self, secret, expected_secret_arn): + assert secret.arn == expected_secret_arn From df030d00e846e13aa8904780c7de9eff4605d50c Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Tue, 13 Jun 2023 10:23:04 +0200 Subject: [PATCH 6/8] Add contract tests for cloudformation_service --- .../aws_access/cloudformation_service.py | 69 +++++++++++++++++++ ...et_manager.py => secretmanager_service.py} | 0 2 files changed, 69 insertions(+) create mode 100644 test/contract_tests/aws_access/cloudformation_service.py rename test/contract_tests/aws_access/{secret_manager.py => secretmanager_service.py} (100%) diff --git a/test/contract_tests/aws_access/cloudformation_service.py b/test/contract_tests/aws_access/cloudformation_service.py new file mode 100644 index 0000000..93b6af3 --- /dev/null +++ b/test/contract_tests/aws_access/cloudformation_service.py @@ -0,0 +1,69 @@ +import pytest + +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.cloudformation_service import CloudFormationService +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.cloudformation import ValidationResult, \ + ListStackResourcesResult +from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId + + +class ValidateContractContract: + + @pytest.fixture(scope="class") + def cloudformation_service(self) -> CloudFormationService: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def template_body(self) -> str: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def validation_result(self, cloudformation_service, template_body) -> ValidationResult: + validation_result = cloudformation_service.validate_template(template_body=template_body) + return validation_result + + def test_validation_result_is_not_none(self, validation_result): + assert validation_result is not None + + +class ListStackResourcesContractContract: + + @pytest.fixture(scope="class") + def cloudformation_service(self) -> CloudFormationService: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def stack_name(self) -> PhysicalResourceId: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def resource_type(self) -> str: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def number_of_resources(self) -> int: + raise NotImplementedError() + + @pytest.fixture(scope="class") + def list_stack_resources_result(self, cloudformation_service, stack_name) -> ListStackResourcesResult: + list_stack_resources_result = cloudformation_service.list_stack_resources( + stack_name=stack_name, + next_token=None + ) + return list_stack_resources_result + + def test_stack_resource_summaries_count(self, list_stack_resources_result, number_of_resources): + assert len(list_stack_resources_result.stack_resource_summaries) == number_of_resources + + def test_physical_resource_ids_is_not_none(self, list_stack_resources_result): + resources_with_physical_resource_id = [stack_resource_summary + for stack_resource_summary + in list_stack_resources_result.stack_resource_summaries + if stack_resource_summary.physical_resource_id is not None] + assert len(resources_with_physical_resource_id) > 0 + + def test_resource_type(self, list_stack_resources_result, resource_type): + resources_with_given_resource_type = [stack_resource_summary.physical_resource_id + for stack_resource_summary + in list_stack_resources_result.stack_resource_summaries + if stack_resource_summary.resource_type == resource_type] + assert len(resources_with_given_resource_type) > 0 diff --git a/test/contract_tests/aws_access/secret_manager.py b/test/contract_tests/aws_access/secretmanager_service.py similarity index 100% rename from test/contract_tests/aws_access/secret_manager.py rename to test/contract_tests/aws_access/secretmanager_service.py From 0f4f359a1ea6ffdc03c40b5b61ccdbaf9b27c8eb Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Tue, 13 Jun 2023 19:25:09 +0200 Subject: [PATCH 7/8] Apply suggestions from code review Co-authored-by: Nicola Coretti --- .../aws/wrapper/datamodels/secretsmanager/test_secret.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/unit_tests/aws/wrapper/datamodels/secretsmanager/test_secret.py b/test/unit_tests/aws/wrapper/datamodels/secretsmanager/test_secret.py index 04907df..60279c9 100644 --- a/test/unit_tests/aws/wrapper/datamodels/secretsmanager/test_secret.py +++ b/test/unit_tests/aws/wrapper/datamodels/secretsmanager/test_secret.py @@ -5,10 +5,11 @@ def test_arn_exists(): - expected_arn = ARN(aws_arn="EXPECTED_ARN") - boto_secret = {"ARN": expected_arn.aws_arn} - secret = Secret.from_boto(boto_secret) - assert secret.arn == expected_arn + expected = ARN(aws_arn="EXPECTED_ARN") + boto_secret = {"ARN": expected.aws_arn} + actual = Secret.from_boto(boto_secret) + assert actual.arn == expected.arn + def test_arn_not_exists(): From 06af19a9be7580d9db5ac8183d0dc9848db8cdc6 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Thu, 15 Jun 2023 15:37:05 +0200 Subject: [PATCH 8/8] Apply review findings --- .../lib/aws/aws_access.py | 2 +- .../lib/aws/wrapper/aws_client.py | 8 ++-- .../lib/aws/wrapper/cloudformation_service.py | 14 +++---- .../lib/aws/wrapper/codebuild_service.py | 12 +++--- .../lib/aws/wrapper/datamodels/common.py | 10 +++++ .../lib/aws/wrapper/secretsmanager_service.py | 10 ++--- .../datamodels/secretsmanager/test_secret.py | 13 +++---- .../unit_tests/aws/wrapper/test_aws_client.py | 12 +++--- .../wrapper/test_cloudformation_service.py | 38 +++++++++---------- .../aws/wrapper/test_codebuild_service.py | 26 ++++++------- .../wrapper/test_secretsmanager_service.py | 16 ++++---- 11 files changed, 85 insertions(+), 76 deletions(-) diff --git a/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py b/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py index dbbfa64..6ad749a 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/aws_access.py @@ -41,7 +41,7 @@ def upload_cloudformation_stack(self, yml: str, stack_name: str): logging.debug(f"Running upload_cloudformation_stack for aws profile {self.aws_profile_for_logging}") client = self._get_aws_client().create_cloudformation_service() try: - cfn_deployer = Deployer(cloudformation_client=client.boto_client) + cfn_deployer = Deployer(cloudformation_client=client.internal_aws_client) result = cfn_deployer.create_and_wait_for_changeset(stack_name=stack_name, cfn_template=yml, parameter_values=[], capabilities=("CAPABILITY_IAM",), role_arn=None, diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/aws_client.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/aws_client.py index 825e642..02784ce 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/aws_client.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/aws_client.py @@ -31,18 +31,18 @@ def _create_aws_session(self) -> Session: def create_codebuild_service(self) -> CodeBuildService: session = self._create_aws_session() - boto_client = session.client("codebuild") - return CodeBuildService(boto_client=boto_client) + internal_aws_client = session.client("codebuild") + return CodeBuildService(internal_aws_client=internal_aws_client) def create_secretsmanager_service(self) -> SecretsManagerService: session = self._create_aws_session() client = session.client("secretsmanager") - return SecretsManagerService(boto_client=client) + return SecretsManagerService(internal_aws_client=client) def create_cloudformation_service(self) -> CloudFormationService: session = self._create_aws_session() client = session.client("cloudformation") - return CloudFormationService(boto_client=client) + return CloudFormationService(internal_aws_client=client) class AwsClientFactory: diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py index 3767187..b738510 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py @@ -6,19 +6,19 @@ class CloudFormationService: - def __init__(self, boto_client): - self._boto_client = boto_client + def __init__(self, internal_aws_client): + self._internal_aws_client = internal_aws_client @property - def boto_client(self) -> Any: - return self._boto_client + def internal_aws_client(self) -> Any: + return self._internal_aws_client def validate_template(self, template_body: str, from_boto: Callable[[Dict[str, Any]], ValidationResult] = ValidationResult.from_boto) \ -> ValidationResult: - boto_validation_result = self._boto_client.validate_template(TemplateBody=template_body) + boto_validation_result = self._internal_aws_client.validate_template(TemplateBody=template_body) validation_result = from_boto(boto_validation_result) return validation_result @@ -30,12 +30,12 @@ def list_stack_resources(self, -> ListStackResourcesResult: if next_token is not None: boto_list_stack_resources_result = \ - self._boto_client.list_stack_resources( + self._internal_aws_client.list_stack_resources( StackName=stack_name.aws_physical_resource_id, NextToken=next_token.aws_next_token) else: boto_list_stack_resources_result = \ - self._boto_client.list_stack_resources( + self._internal_aws_client.list_stack_resources( StackName=stack_name.aws_physical_resource_id) list_stack_resources_result = from_boto(boto_list_stack_resources_result) diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/codebuild_service.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/codebuild_service.py index 0e33a08..fcf9bf1 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/codebuild_service.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/codebuild_service.py @@ -5,12 +5,12 @@ class CodeBuildService: - def __init__(self, boto_client): - self._boto_client = boto_client + def __init__(self, internal_aws_client): + self._internal_aws_client = internal_aws_client @property - def boto_client(self) -> Any: - return self._boto_client + def internal_aws_client(self) -> Any: + return self._internal_aws_client def start_build_batch( self, @@ -19,7 +19,7 @@ def start_build_batch( environment_variables_override: List[Dict[str, str]], from_boto: Callable[[Dict[str, Any]], BuildBatch] = BuildBatch.from_boto) \ -> BuildBatch: - boto_build_batch = self._boto_client.start_build_batch( + boto_build_batch = self._internal_aws_client.start_build_batch( projectName=project_name.aws_physical_resource_id, sourceVersion=source_version, environmentVariablesOverride=environment_variables_override) @@ -31,7 +31,7 @@ def batch_get_build_batches(self, from_boto: Callable[[Dict[str, Any]], BuildBatch] = BuildBatch.from_boto) \ -> List[BuildBatch]: aws_ids = [build_batch_id.aws_physical_resource_id for build_batch_id in build_batch_ids] - boto_build_batches = self._boto_client.batch_get_build_batches(ids=aws_ids) + boto_build_batches = self._internal_aws_client.batch_get_build_batches(ids=aws_ids) print(boto_build_batches) build_batches = [from_boto(boto_build_batch) for boto_build_batch in boto_build_batches['buildBatches']] return build_batches diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/common.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/common.py index 9a2b3ff..ad59767 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/common.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/datamodels/common.py @@ -4,9 +4,19 @@ @dataclasses.dataclass(frozen=True) class ARN: + """ + This class represents a AWS ARN + We use a dataclass to encapsulate the original value of the ARN, to decouple our abstraction from AWS. + This way we don't care when AWS should change its datatype. + """ aws_arn: Any @dataclasses.dataclass(frozen=True) class PhysicalResourceId: + """ + This class represents a AWS PhysicalResourceId. + We use a dataclass to encapsulate the original value of the PhysicalResourceId, + to decouple our abstraction from AWS. This way we don't care when AWS should change its datatype. + """ aws_physical_resource_id: Any diff --git a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/secretsmanager_service.py b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/secretsmanager_service.py index 3f79b34..1be5818 100644 --- a/exasol_script_languages_container_ci_setup/lib/aws/wrapper/secretsmanager_service.py +++ b/exasol_script_languages_container_ci_setup/lib/aws/wrapper/secretsmanager_service.py @@ -6,17 +6,17 @@ class SecretsManagerService: - def __init__(self, boto_client): - self._boto_client = boto_client + def __init__(self, internal_aws_client): + self._internal_aws_client = internal_aws_client @property - def boto_client(self) -> Any: - return self._boto_client + def internal_aws_client(self) -> Any: + return self._internal_aws_client def get_secret_value(self, secret_id: PhysicalResourceId, from_boto: Callable[[Dict[str, Any]], Secret] = Secret.from_boto) \ -> Secret: - boto_secret = self._boto_client.get_secret_value(SecretId=secret_id.aws_physical_resource_id) + boto_secret = self._internal_aws_client.get_secret_value(SecretId=secret_id.aws_physical_resource_id) secret = from_boto(boto_secret) return secret diff --git a/test/unit_tests/aws/wrapper/datamodels/secretsmanager/test_secret.py b/test/unit_tests/aws/wrapper/datamodels/secretsmanager/test_secret.py index 60279c9..c255546 100644 --- a/test/unit_tests/aws/wrapper/datamodels/secretsmanager/test_secret.py +++ b/test/unit_tests/aws/wrapper/datamodels/secretsmanager/test_secret.py @@ -5,13 +5,12 @@ def test_arn_exists(): - expected = ARN(aws_arn="EXPECTED_ARN") - boto_secret = {"ARN": expected.aws_arn} + expected = Secret(arn=ARN(aws_arn="EXPECTED_ARN")) + boto_secret = {"ARN": expected.arn.aws_arn} actual = Secret.from_boto(boto_secret) assert actual.arn == expected.arn - def test_arn_not_exists(): with pytest.raises(KeyError): boto_secret = {} @@ -19,14 +18,14 @@ def test_arn_not_exists(): def test_with_extra_keys(): - expected_arn = ARN(aws_arn="EXPECTED_ARN") + expected = Secret(arn=ARN(aws_arn="EXPECTED_ARN")) boto_secret = { - "ARN": expected_arn.aws_arn, + "ARN": expected.arn.aws_arn, "extra1": None, "extra2": 1 } - secret = Secret.from_boto(boto_secret) - assert secret.arn == expected_arn + actual = Secret.from_boto(boto_secret) + assert actual == expected def test_empty_arn(): diff --git a/test/unit_tests/aws/wrapper/test_aws_client.py b/test/unit_tests/aws/wrapper/test_aws_client.py index 38d4f06..4d5b7cf 100644 --- a/test/unit_tests/aws/wrapper/test_aws_client.py +++ b/test/unit_tests/aws/wrapper/test_aws_client.py @@ -38,9 +38,9 @@ def test_create_codebuild_service_boto_session_client(): ] -def test_create_codebuild_service_boto_client(): +def test_create_codebuild_service_internal_aws_client(): setup = CodeBuildServiceTestSetup() - assert setup.codebuild_service.boto_client == mock_cast(setup.boto_session.client).return_value + assert setup.codebuild_service.internal_aws_client == mock_cast(setup.boto_session.client).return_value @dataclasses.dataclass @@ -66,9 +66,9 @@ def test_create_cloudformation_service_boto_session_client(): ] -def test_create_cloudformation_service_boto_client(): +def test_create_cloudformation_service_internal_aws_client(): setup = CloudFormationServiceTestSetup() - assert setup.cloudformation_service.boto_client == mock_cast(setup.boto_session.client).return_value + assert setup.cloudformation_service.internal_aws_client == mock_cast(setup.boto_session.client).return_value @dataclasses.dataclass @@ -94,6 +94,6 @@ def test_create_secretsmanager_service_boto_session_client(): ] -def test_create_secretsmanager_service_boto_client(): +def test_create_secretsmanager_service_internal_aws_client(): setup = SecreteManagerServiceTestSetup() - assert setup.secretsmanager_service.boto_client == mock_cast(setup.boto_session.client).return_value + assert setup.secretsmanager_service.internal_aws_client == mock_cast(setup.boto_session.client).return_value diff --git a/test/unit_tests/aws/wrapper/test_cloudformation_service.py b/test/unit_tests/aws/wrapper/test_cloudformation_service.py index 094cf0e..a0644ce 100644 --- a/test/unit_tests/aws/wrapper/test_cloudformation_service.py +++ b/test/unit_tests/aws/wrapper/test_cloudformation_service.py @@ -11,31 +11,31 @@ def test_init(): - boto_client = Mock() - service = CloudFormationService(boto_client=boto_client) - assert service.boto_client == boto_client + internal_aws_client = Mock() + service = CloudFormationService(internal_aws_client=internal_aws_client) + assert service.internal_aws_client == internal_aws_client @dataclasses.dataclass(frozen=True) class ValidateTemplateSetup: - boto_client = Mock() + internal_aws_client = Mock() from_boto = Mock() - service = CloudFormationService(boto_client=boto_client) + service = CloudFormationService(internal_aws_client=internal_aws_client) template_body = "TemplateBody" validation_result = service.validate_template(template_body=template_body, from_boto=from_boto) - boto_client_validate_template_return_value = mock_cast(boto_client.validate_template).return_value + internal_aws_client_validate_template_return_value = mock_cast(internal_aws_client.validate_template).return_value -def test_validate_template_boto_client(): +def test_validate_template_internal_aws_client(): setup = ValidateTemplateSetup() - assert setup.boto_client.mock_calls == [call.validate_template(TemplateBody=setup.template_body)] \ - and setup.from_boto.mock_calls == [call(setup.boto_client_validate_template_return_value)] \ + assert setup.internal_aws_client.mock_calls == [call.validate_template(TemplateBody=setup.template_body)] \ + and setup.from_boto.mock_calls == [call(setup.internal_aws_client_validate_template_return_value)] \ and setup.validation_result == setup.from_boto.return_value def test_validate_template_from_boto(): setup = ValidateTemplateSetup() - assert setup.from_boto.mock_calls == [call(setup.boto_client_validate_template_return_value)] + assert setup.from_boto.mock_calls == [call(setup.internal_aws_client_validate_template_return_value)] def test_validate_template_result(): @@ -46,27 +46,27 @@ def test_validate_template_result(): class ListStackResourcesTestSetup: def __init__(self, next_token: Optional[NextToken]): self.next_token = next_token - self.boto_client = Mock() + self.internal_aws_client = Mock() self.from_boto = Mock() - self.service = CloudFormationService(boto_client=self.boto_client) + self.service = CloudFormationService(internal_aws_client=self.internal_aws_client) self.physical_resource_id = PhysicalResourceId(aws_physical_resource_id="stack_name") self.list_stack_resources_result = self.service.list_stack_resources( stack_name=self.physical_resource_id, next_token=next_token, from_boto=self.from_boto) self.aws_next_token = None if next_token is None else next_token.aws_next_token - self.boto_client_list_stack_resources_return_value = mock_cast( - self.boto_client.list_stack_resources).return_value + self.internal_aws_client_list_stack_resources_return_value = mock_cast( + self.internal_aws_client.list_stack_resources).return_value -def test_list_stack_resources_boto_client_next_token_is_none(): +def test_list_stack_resources_internal_aws_client_next_token_is_none(): setup = ListStackResourcesTestSetup(None) - assert setup.boto_client.mock_calls == [ + assert setup.internal_aws_client.mock_calls == [ call.list_stack_resources(StackName=setup.physical_resource_id.aws_physical_resource_id)] -def test_list_stack_resources_boto_client_next_token_is_not_none(): +def test_list_stack_resources_internal_aws_client_next_token_is_not_none(): next_token = NextToken("next_token") setup = ListStackResourcesTestSetup(next_token) - assert setup.boto_client.mock_calls == [ + assert setup.internal_aws_client.mock_calls == [ call.list_stack_resources( StackName=setup.physical_resource_id.aws_physical_resource_id, NextToken=next_token.aws_next_token) @@ -82,7 +82,7 @@ def test_list_stack_resources_boto_client_next_token_is_not_none(): @list_stack_resources_parameters def test_list_stack_resources_from_boto(next_token): setup = ListStackResourcesTestSetup(next_token) - assert setup.from_boto.mock_calls == [call(setup.boto_client_list_stack_resources_return_value)] + assert setup.from_boto.mock_calls == [call(setup.internal_aws_client_list_stack_resources_return_value)] @list_stack_resources_parameters diff --git a/test/unit_tests/aws/wrapper/test_codebuild_service.py b/test/unit_tests/aws/wrapper/test_codebuild_service.py index 5482285..e07011f 100644 --- a/test/unit_tests/aws/wrapper/test_codebuild_service.py +++ b/test/unit_tests/aws/wrapper/test_codebuild_service.py @@ -7,31 +7,31 @@ def test_init(): - boto_client = Mock() - service = CodeBuildService(boto_client=boto_client) - assert service.boto_client == boto_client + internal_aws_client = Mock() + service = CodeBuildService(internal_aws_client=internal_aws_client) + assert service.internal_aws_client == internal_aws_client @dataclasses.dataclass(frozen=True) class BatchGetBuildBatchesTestSetup: - boto_client = Mock() + internal_aws_client = Mock() batch_get_build_batches_return_values = [Mock(), Mock()] - mock_cast(boto_client.batch_get_build_batches).return_value = { + mock_cast(internal_aws_client.batch_get_build_batches).return_value = { 'buildBatches': batch_get_build_batches_return_values } from_boto = Mock() from_boto_return_values = [Mock(), Mock()] from_boto.side_effect = from_boto_return_values - service = CodeBuildService(boto_client=boto_client) + service = CodeBuildService(internal_aws_client=internal_aws_client) build_batch_ids = [PhysicalResourceId(aws_physical_resource_id="1"), PhysicalResourceId(aws_physical_resource_id="2")] build_batches = service.batch_get_build_batches(build_batch_ids=build_batch_ids, from_boto=from_boto) ids = [id.aws_physical_resource_id for id in build_batch_ids] -def test_batch_get_build_batches_boto_client(): +def test_batch_get_build_batches_internal_aws_client(): setup = BatchGetBuildBatchesTestSetup() - assert setup.boto_client.mock_calls == [call.batch_get_build_batches(ids=setup.ids)] + assert setup.internal_aws_client.mock_calls == [call.batch_get_build_batches(ids=setup.ids)] def test_batch_get_build_batches_from_boto(): @@ -47,13 +47,13 @@ def test_batch_get_build_batches_result(): @dataclasses.dataclass(frozen=True) class StartBuildBatchTestSetup: - boto_client = Mock() + internal_aws_client = Mock() from_boto_input = Mock() start_build_batch_return_values = {"buildBatch": from_boto_input} - mock_cast(boto_client.start_build_batch).return_value = start_build_batch_return_values + mock_cast(internal_aws_client.start_build_batch).return_value = start_build_batch_return_values from_boto = Mock() from_boto.return_value = Mock() - service = CodeBuildService(boto_client=boto_client) + service = CodeBuildService(internal_aws_client=internal_aws_client) projectName = PhysicalResourceId(aws_physical_resource_id="id") sourceVersion = Mock() environmentVariablesOverride = Mock() @@ -63,9 +63,9 @@ class StartBuildBatchTestSetup: from_boto=from_boto) -def test_start_build_batch_boto_client(): +def test_start_build_batch_internal_aws_client(): setup = StartBuildBatchTestSetup() - assert setup.boto_client.mock_calls == [ + assert setup.internal_aws_client.mock_calls == [ call.start_build_batch(projectName=setup.projectName.aws_physical_resource_id, sourceVersion=setup.sourceVersion, environmentVariablesOverride=setup.environmentVariablesOverride)] diff --git a/test/unit_tests/aws/wrapper/test_secretsmanager_service.py b/test/unit_tests/aws/wrapper/test_secretsmanager_service.py index af625fb..d97272f 100644 --- a/test/unit_tests/aws/wrapper/test_secretsmanager_service.py +++ b/test/unit_tests/aws/wrapper/test_secretsmanager_service.py @@ -7,27 +7,27 @@ def test_init(): - boto_client = Mock() - service = SecretsManagerService(boto_client=boto_client) - assert service.boto_client == boto_client + internal_aws_client = Mock() + service = SecretsManagerService(internal_aws_client=internal_aws_client) + assert service.internal_aws_client == internal_aws_client @dataclasses.dataclass(frozen=True) class GetSecretValueTestSetup: - boto_client = Mock() + internal_aws_client = Mock() get_secret_value_return_values = Mock() - mock_cast(boto_client.get_secret_value).return_value = get_secret_value_return_values + mock_cast(internal_aws_client.get_secret_value).return_value = get_secret_value_return_values from_boto = Mock() from_boto_return_values = Mock() from_boto.return_value = from_boto_return_values - service = SecretsManagerService(boto_client=boto_client) + service = SecretsManagerService(internal_aws_client=internal_aws_client) secret_id = PhysicalResourceId(aws_physical_resource_id="id") secret = service.get_secret_value(secret_id=secret_id, from_boto=from_boto) -def test_get_secret_value_boto_client(): +def test_get_secret_value_internal_aws_client(): setup = GetSecretValueTestSetup() - assert setup.boto_client.mock_calls == [call.get_secret_value(SecretId=setup.secret_id.aws_physical_resource_id)] + assert setup.internal_aws_client.mock_calls == [call.get_secret_value(SecretId=setup.secret_id.aws_physical_resource_id)] def test_get_secret_value_from_boto():