#50: Added shallow abstraction for boto3

exasol_script_languages_container_ci_setup/cli/commands/deploy_ci_build.py

@@ -7,7 +7,7 @@
 from exasol_script_languages_container_ci_setup.cli.cli import cli
 from exasol_script_languages_container_ci_setup.cli.common import add_options
 from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
-from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess
 from exasol_script_languages_container_ci_setup.lib.ci_build import run_deploy_ci_build
 from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
 

exasol_script_languages_container_ci_setup/cli/commands/deploy_release_build.py

@@ -7,7 +7,7 @@
 from exasol_script_languages_container_ci_setup.cli.cli import cli
 from exasol_script_languages_container_ci_setup.cli.common import add_options
 from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
-from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess
 from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
 from exasol_script_languages_container_ci_setup.lib.release_build import run_deploy_release_build
 

exasol_script_languages_container_ci_setup/cli/commands/deploy_source_credentials.py

@@ -7,7 +7,7 @@
 from exasol_script_languages_container_ci_setup.cli.cli import cli
 from exasol_script_languages_container_ci_setup.cli.common import add_options
 from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
-from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess
 from exasol_script_languages_container_ci_setup.lib.source_credentials import run_deploy_source_credentials
 from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
 

exasol_script_languages_container_ci_setup/cli/commands/start_ci_build.py

@@ -1,12 +1,11 @@
-import os
 from typing import Optional
 
 import click
 
 from exasol_script_languages_container_ci_setup.cli.cli import cli
 from exasol_script_languages_container_ci_setup.cli.common import add_options
 from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
-from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess
 from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
 from exasol_script_languages_container_ci_setup.lib.run_start_build import run_start_ci_build
 

exasol_script_languages_container_ci_setup/cli/commands/start_release_build.py

@@ -6,7 +6,7 @@
 from exasol_script_languages_container_ci_setup.cli.cli import cli
 from exasol_script_languages_container_ci_setup.cli.common import add_options
 from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
-from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess
 from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
 from exasol_script_languages_container_ci_setup.lib.run_start_build import run_start_release_build
 

exasol_script_languages_container_ci_setup/cli/commands/start_test_release_build.py

@@ -6,7 +6,7 @@
 from exasol_script_languages_container_ci_setup.cli.cli import cli
 from exasol_script_languages_container_ci_setup.cli.common import add_options
 from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
-from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess
 from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
 from exasol_script_languages_container_ci_setup.lib.github_draft_release_creator import GithubDraftReleaseCreator
 from exasol_script_languages_container_ci_setup.lib.run_start_build import run_start_test_release_build

exasol_script_languages_container_ci_setup/cli/commands/validate_ci_build.py

@@ -1,13 +1,12 @@
-import logging
 from typing import Optional
 
 import click
 
 from exasol_script_languages_container_ci_setup.cli.cli import cli
 from exasol_script_languages_container_ci_setup.cli.common import add_options
 from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
-from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
-from exasol_script_languages_container_ci_setup.lib.ci_build import run_deploy_ci_build, run_validate_ci_build
+from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.lib.ci_build import run_validate_ci_build
 from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
 
 

exasol_script_languages_container_ci_setup/cli/commands/validate_release_build.py

@@ -5,7 +5,7 @@
 from exasol_script_languages_container_ci_setup.cli.cli import cli
 from exasol_script_languages_container_ci_setup.cli.common import add_options
 from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
-from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess
 from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
 from exasol_script_languages_container_ci_setup.lib.release_build import run_validate_release_build
 

exasol_script_languages_container_ci_setup/cli/commands/validate_source_credentials.py

@@ -5,7 +5,7 @@
 from exasol_script_languages_container_ci_setup.cli.cli import cli
 from exasol_script_languages_container_ci_setup.cli.common import add_options
 from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
-from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess
 from exasol_script_languages_container_ci_setup.lib.source_credentials import run_validate_source_credentials
 from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
 

exasol_script_languages_container_ci_setup/lib/aws_access.py → exasol_script_languages_container_ci_setup/lib/aws/aws_access.py

@@ -1,15 +1,23 @@
 import logging
 import time
-from typing import Optional, List, Dict, Any, Iterable
+from typing import Optional, List, Dict, Iterable, Callable
 
-import boto3
 from botocore.exceptions import ClientError
 
-from exasol_script_languages_container_ci_setup.lib.deployer import Deployer
+from exasol_script_languages_container_ci_setup.lib.aws.deployer import Deployer
+from exasol_script_languages_container_ci_setup.lib.aws.wrapper.aws_client import AwsClientFactory, AwsClient
+from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.cloudformation import StackResourceSummary
+from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.codebuild import BuildBatchStatus
+from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId
 
+BUILD_STATUS_FAILURES = [BuildBatchStatus.FAILED, BuildBatchStatus.FAULT,
+                         BuildBatchStatus.STOPPED, BuildBatchStatus.TIMED_OUT]
 
-class AwsAccess(object):
-    def __init__(self, aws_profile: Optional[str]):
+
+class AwsAccess:
+    def __init__(self, aws_profile: Optional[str],
+                 aws_client_factory: AwsClientFactory = AwsClientFactory()):
+        self._aws_client_factory = aws_client_factory
         self._aws_profile = aws_profile
 
     @property
@@ -23,14 +31,17 @@ def aws_profile_for_logging(self) -> str:
     def aws_profile(self) -> Optional[str]:
         return self._aws_profile
 
+    def _get_aws_client(self) -> AwsClient:
+        return self._aws_client_factory.create(profile=self._aws_profile)
+
     def upload_cloudformation_stack(self, yml: str, stack_name: str):
         """
         Deploy the cloudformation stack.
         """
         logging.debug(f"Running upload_cloudformation_stack for aws profile {self.aws_profile_for_logging}")
-        cloud_client = self._get_aws_client("cloudformation")
+        client = self._get_aws_client().create_cloudformation_service()
         try:
-            cfn_deployer = Deployer(cloudformation_client=cloud_client)
+            cfn_deployer = Deployer(cloudformation_client=client.internal_aws_client)
             result = cfn_deployer.create_and_wait_for_changeset(stack_name=stack_name, cfn_template=yml,
                                                                 parameter_values=[],
                                                                 capabilities=("CAPABILITY_IAM",), role_arn=None,
@@ -52,11 +63,11 @@ def read_secret_arn(self, secret_name: str):
         """
         logging.debug(f"Reading secret for getting ARN, secret name = {secret_name}, "
                       f"for aws profile {self.aws_profile_for_logging}")
-        client = self._get_aws_client(service_name='secretsmanager')
+        client = self._get_aws_client().create_secretsmanager_service()
 
         try:
-            get_secret_value_response = client.get_secret_value(SecretId=secret_name)
-            return get_secret_value_response["ARN"]
+            secret = client.get_secret_value(secret_id=PhysicalResourceId(secret_name))
+            return secret.arn
         except ClientError as e:
             logging.error("Unable to read secret")
             raise e
@@ -72,33 +83,32 @@ def validate_cloudformation_template(self, cloudformation_yml) -> None:
         It requires to have the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env variables set correctly.
         """
         logging.debug(f"Running validate_cloudformation_template for aws profile {self.aws_profile_for_logging}")
-        cloud_client = self._get_aws_client("cloudformation")
-        cloud_client.validate_template(TemplateBody=cloudformation_yml)
+        client = self._get_aws_client().create_cloudformation_service()
+        client.validate_template(template_body=cloudformation_yml)
 
-    def _get_aws_client(self, service_name: str) -> Any:
-        if self._aws_profile is None:
-            return boto3.client(service_name)
-        aws_session = boto3.session.Session(profile_name=self._aws_profile)
-        return aws_session.client(service_name)
-
-    def get_all_stack_resources(self, stack_name: str) -> List[Dict[str, str]]:
+    def get_all_stack_resources(self, stack_name: str) -> List[StackResourceSummary]:
         """
         This functions uses Boto3 to get all AWS Cloudformation resources for a specific Cloudformation stack,
         identified by parameter `stack_name`.
         The AWS API truncates at a size of 1MB, and in order to get all chunks the method must be called
         passing the previous retrieved token until no token is returned.
         """
         logging.debug(f"Running get_all_codebuild_projects for aws profile {self.aws_profile_for_logging}")
-        cf_client = self._get_aws_client('cloudformation')
-        current_result = cf_client.list_stack_resources(StackName=stack_name)
-        result = current_result["StackResourceSummaries"]
-
-        while "nextToken" in current_result:
-            current_result = cf_client.list_projects(StackName=stack_name, nextToken=current_result["nextToken"])
-            result.extend(current_result["StackResourceSummaries"])
+        client = self._get_aws_client().create_cloudformation_service()
+        stack_name_id = PhysicalResourceId(stack_name)
+        current_result = client.list_stack_resources(stack_name=stack_name_id)
+        result = current_result.stack_resource_summaries
+
+        while current_result.next_token is not None:
+            current_result = client.list_stack_resources(stack_name=stack_name_id, next_token=current_result.next_token)
+            result.extend(current_result.stack_resource_summaries)
         return result
 
-    def start_codebuild(self, project: str, environment_variables_overrides: List[Dict[str, str]], branch: str) -> None:
+    def start_codebuild(self,
+                        project: PhysicalResourceId,
+                        environment_variables_overrides: List[Dict[str, str]],
+                        branch: str,
+                        poll_interval_seconds: int = 30) -> None:
         """
         This functions uses Boto3 to start a batch build.
         It forwards all variables from parameter env_variables as environment variables to the CodeBuild project.
@@ -107,37 +117,35 @@ def start_codebuild(self, project: str, environment_variables_overrides: List[Di
         :raises
             `RuntimeError` if build fails or AWS Batch build returns unknown status
         """
-        codebuild_client = self._get_aws_client("codebuild")
+        client = self._get_aws_client().create_codebuild_service()
         logging.info(f"Trigger codebuild for project {project} with branch {branch} "
                      f"and env_variables ({environment_variables_overrides})")
-        ret_val = codebuild_client.start_build_batch(projectName=project,
-                                                     sourceVersion=branch,
-                                                     environmentVariablesOverride=list(
-                                                         environment_variables_overrides))
+        build_batch = client.start_build_batch(project_name=project,
+                                               source_version=branch,
+                                               environment_variables_override=list(
+                                                   environment_variables_overrides))
 
         def wait_for(seconds: int, interval: int) -> Iterable[int]:
             for _ in range(int(seconds / interval)):
                 yield interval
 
-        build_id = ret_val['buildBatch']['id']
+        build_id = build_batch.id
         logging.debug(f"Codebuild for project {project} with branch {branch} triggered. Id is {build_id}.")
         interval = 30
         timeout_time_in_seconds = 60 * 60 * 2  # We wait for maximal 2h + (something)
         for seconds_to_wait in wait_for(seconds=timeout_time_in_seconds, interval=interval):
             time.sleep(seconds_to_wait)
             logging.debug(f"Checking status of codebuild id {build_id}.")
-            build_response = codebuild_client.batch_get_build_batches(ids=[build_id])
-            logging.debug(f"Build response of codebuild id {build_id} is {build_response}")
-            if len(build_response['buildBatches']) != 1:
-                logging.error(f"Unexpected return value from 'batch_get_build_batches': {build_response}")
-            build_status = build_response['buildBatches'][0]['buildBatchStatus']
+            build_batches = client.batch_get_build_batches(build_batch_ids=[build_id])
+            logging.debug(f"Build response of codebuild id {build_id} is {build_batches}")
+            if len(build_batches) != 1:
+                logging.error(f"Unexpected return value from 'batch_get_build_batches': {build_batches}")
+            build_status = build_batches[0].build_batch_status
             logging.info(f"Build status of codebuild id {build_id} is {build_status}")
-            if build_status == 'SUCCEEDED':
+            if build_status == BuildBatchStatus.SUCCEEDED:
                 break
-            elif build_status in ['FAILED', 'FAULT', 'STOPPED', 'TIMED_OUT']:
-                raise RuntimeError(f"Build ({build_id}) failed with status: {build_status}")
-            elif build_status != "IN_PROGRESS":
-                raise RuntimeError(f"Batch build {build_id} has unknown build status: {build_status}")
+            elif build_status in BUILD_STATUS_FAILURES:
+                raise RuntimeError(f"Build ({build_id}) failed with status: {build_status.name}")
         # if loop does not break early, build wasn't successful
         else:
             raise RuntimeError(f"Batch build {build_id} ran into timeout.")

exasol_script_languages_container_ci_setup/lib/aws/wrapper/aws_client.py

@@ -0,0 +1,50 @@
+import dataclasses
+from abc import abstractmethod
+from typing import List, Dict, Optional, Any, Protocol
+
+import boto3
+from boto3 import Session
+
+from exasol_script_languages_container_ci_setup.lib.aws.wrapper.cloudformation_service import CloudFormationService
+from exasol_script_languages_container_ci_setup.lib.aws.wrapper.codebuild_service import CodeBuildService
+from exasol_script_languages_container_ci_setup.lib.aws.wrapper.secretsmanager_service import SecretsManagerService
+
+
+class BotoSessionFactory(Protocol):
+
+    @abstractmethod
+    def __call__(self, profile_name: Optional[str] = None, region_name: Optional[str] = None) -> Session:
+        ...
+
+
+class AwsClient:
+    def __init__(self,
+                 profile: Optional[str] = None,
+                 region: Optional[str] = None,
+                 boto_session_factory: BotoSessionFactory = boto3.session.Session):
+        self._boto_session_factory = boto_session_factory
+        self._region = region
+        self._profile = profile
+
+    def _create_aws_session(self) -> Session:
+        return self._boto_session_factory(profile_name=self._profile, region_name=self._region)
+
+    def create_codebuild_service(self) -> CodeBuildService:
+        session = self._create_aws_session()
+        internal_aws_client = session.client("codebuild")
+        return CodeBuildService(internal_aws_client=internal_aws_client)
+
+    def create_secretsmanager_service(self) -> SecretsManagerService:
+        session = self._create_aws_session()
+        client = session.client("secretsmanager")
+        return SecretsManagerService(internal_aws_client=client)
+
+    def create_cloudformation_service(self) -> CloudFormationService:
+        session = self._create_aws_session()
+        client = session.client("cloudformation")
+        return CloudFormationService(internal_aws_client=client)
+
+
+class AwsClientFactory:
+    def create(self, profile: Optional[str] = None, region: Optional[str] = None) -> AwsClient:
+        return AwsClient(profile=profile, region=region)

exasol_script_languages_container_ci_setup/lib/aws/wrapper/cloudformation_service.py

@@ -0,0 +1,42 @@
+from typing import Any, Dict, Optional, Callable
+
+from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.cloudformation import ValidationResult, \
+    ListStackResourcesResult, NextToken
+from exasol_script_languages_container_ci_setup.lib.aws.wrapper.datamodels.common import PhysicalResourceId
+
+
+class CloudFormationService:
+    def __init__(self, internal_aws_client):
+        self._internal_aws_client = internal_aws_client
+
+    @property
+    def internal_aws_client(self) -> Any:
+        return self._internal_aws_client
+
+    def validate_template(self,
+                          template_body: str,
+                          from_boto: Callable[[Dict[str, Any]], ValidationResult] =
+                          ValidationResult.from_boto) \
+            -> ValidationResult:
+        boto_validation_result = self._internal_aws_client.validate_template(TemplateBody=template_body)
+        validation_result = from_boto(boto_validation_result)
+        return validation_result
+
+    def list_stack_resources(self,
+                             stack_name: PhysicalResourceId,
+                             next_token: Optional[NextToken] = None,
+                             from_boto: Callable[[Dict[str, Any]], ListStackResourcesResult] =
+                             ListStackResourcesResult.from_boto) \
+            -> ListStackResourcesResult:
+        if next_token is not None:
+            boto_list_stack_resources_result = \
+                self._internal_aws_client.list_stack_resources(
+                    StackName=stack_name.aws_physical_resource_id,
+                    NextToken=next_token.aws_next_token)
+        else:
+            boto_list_stack_resources_result = \
+                self._internal_aws_client.list_stack_resources(
+                    StackName=stack_name.aws_physical_resource_id)
+
+        list_stack_resources_result = from_boto(boto_list_stack_resources_result)
+        return list_stack_resources_result