#3: Added release code build

doc/changes/changes_0.1.0.md

@@ -14,6 +14,7 @@ n/a
 ## Features / Enhancements
 
  - #1: Implement codebuild deployment and buildspec generation
+ - #3: Added release code build
 
 ## Documentation
 

exasol_script_languages_container_ci_setup/__init__.py

@@ -1,8 +1,12 @@
 from exasol_script_languages_container_ci_setup.cli.commands import (
     health,
     generate_buildspec,
+    generate_release_buildspec,
     deploy_source_credentials,
     deploy_ci_build,
+    deploy_release_build,
     validate_ci_build,
-    validate_source_credentials
+    validate_release_build,
+    validate_source_credentials,
+    start_release_build
 )

exasol_script_languages_container_ci_setup/cli/commands/deploy_release_build.py

@@ -0,0 +1,31 @@
+import logging
+import sys
+
+import click
+
+from exasol_script_languages_container_ci_setup.cli.cli import cli
+from exasol_script_languages_container_ci_setup.cli.common import add_options
+from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
+from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
+from exasol_script_languages_container_ci_setup.lib.release_build import run_deploy_release_build
+
+
+@cli.command()
+@add_options(aws_options)
+@add_options(logging_options)
+@click.option('--project', type=str, required=True,
+              help="""The project for which the stack will be created.""")
+@click.option('--project-url', type=str, required=True,
+              help="""The URL of the project on Github.""")
+def deploy_release_build(
+        aws_profile: str,
+        log_level: str,
+        project: str,
+        project_url: str):
+    set_log_level(log_level)
+    try:
+        run_deploy_release_build(AwsAccess(aws_profile), project, project_url)
+    except Exception:
+        logging.error("run_deploy_release_build failed.")
+        sys.exit(1)

exasol_script_languages_container_ci_setup/cli/commands/generate_release_buildspec.py

@@ -0,0 +1,33 @@
+from typing import Tuple, Optional
+
+import click
+
+from exasol_script_languages_container_ci_setup.cli.cli import cli
+from exasol_script_languages_container_ci_setup.cli.common import add_options
+from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
+from exasol_script_languages_container_ci_setup.lib.run_generate_release_buildspec import run_generate_release_buildspec
+
+
+@cli.command()
+@add_options(logging_options)
+@click.option('--flavor-root-path', required=True, multiple=True,
+              type=click.Path(file_okay=False, dir_okay=True, exists=True),
+              help="Path where script language container flavors are located.")
+@click.option('--output-path', type=click.Path(file_okay=False, dir_okay=True, exists=True, writable=True),
+              default="./aws-code-build/release", show_default=True,
+              help="Path where buildspec files will be deployed.")
+@click.option('--config-file', type=click.Path(file_okay=True, dir_okay=False, exists=True),
+              help="Configuration file for build (project specific).")
+def generate_release_buildspecs(
+        flavor_root_path: Tuple[str, ...],
+        log_level: str,
+        output_path: str,
+        config_file: Optional[str]
+        ):
+    """
+    This command generates the buildspec file(s) for the AWS CodeBuild Release build based
+    on the flavors located in path "flavor_root_path".
+    """
+    set_log_level(log_level)
+    run_generate_release_buildspec(flavor_root_path, output_path, config_file)
+

exasol_script_languages_container_ci_setup/cli/commands/start_release_build.py

@@ -0,0 +1,31 @@
+import click
+
+from exasol_script_languages_container_ci_setup.cli.cli import cli
+from exasol_script_languages_container_ci_setup.cli.common import add_options
+from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
+from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
+from exasol_script_languages_container_ci_setup.lib.run_start_release_build import run_start_release_build
+
+
+@cli.command()
+@add_options(aws_options)
+@add_options(logging_options)
+@click.option('--project', type=str, required=True,
+              help="""The project name. Must be same name as used for the AWS CodeBuild release stack creation.""")
+@click.option('--upload-url', type=str, required=True,
+              help="""The URL of the Github release where artifacts will be stored.""")
+@click.option('--branch', type=str, required=True,
+              help="""The branch of the repository which will be used.""")
+@click.option('--dry-run/--no-dry-run', default=False,
+              help="If true, runs release without pushing the container to the docker release repository."
+                   "If false, also pushes the container to the docker release repository.")
+def start_release_build(
+        aws_profile: str,
+        log_level: str,
+        project: str,
+        upload_url: str,
+        branch: str,
+        dry_run: bool):
+    set_log_level(log_level)
+    run_start_release_build(AwsAccess(aws_profile), project, upload_url, branch, dry_run)

exasol_script_languages_container_ci_setup/cli/commands/validate_release_build.py

@@ -0,0 +1,24 @@
+import click
+
+from exasol_script_languages_container_ci_setup.cli.cli import cli
+from exasol_script_languages_container_ci_setup.cli.common import add_options
+from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
+from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
+from exasol_script_languages_container_ci_setup.lib.release_build import run_validate_release_build
+
+
+@cli.command()
+@add_options(aws_options)
+@add_options(logging_options)
+@click.option('--project', type=str, required=True,
+              help="""The project for which the stack will be created.""")
+@click.option('--project-url', type=str, required=True,
+              help="""The URL of the project on Github.""")
+def validate_release_build(
+        aws_profile: str,
+        log_level: str,
+        project: str,
+        project_url: str):
+    set_log_level(log_level)
+    run_validate_release_build(AwsAccess(aws_profile), project, project_url)

exasol_script_languages_container_ci_setup/lib/aws_access.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Optional
+import time
+from typing import Optional, List, Dict, Tuple
 
 import boto3
 from botocore.exceptions import ClientError
@@ -78,3 +79,63 @@ def validate_cloudformation_template(self, cloudformation_yml) -> None:
             logging.debug(f"Running validate_cloudformation_template for default aws profile.")
             cloud_client = boto3.client('cloudformation')
             cloud_client.validate_template(TemplateBody=cloudformation_yml)
+
+    def _get_codebuild_client(self):
+        if self._aws_profile is not None:
+            aws_session = boto3.session.Session(profile_name=self._aws_profile)
+            codebuild_client = aws_session.client('codebuild')
+        else:
+            codebuild_client = boto3.client('codebuild')
+        return codebuild_client
+
+    def get_all_codebuild_projects(self) -> List[str]:
+        """
+        This functions uses Boto3 to get all CodeBuild projects. The AWS API truncates at a size of 100, and
+        in order to get all chunks the method must be called passing the previous retrieved token until no token
+        is returned.
+        """
+        logging.debug(f"Running get_all_codebuild_projects for aws profile {self._aws_profile}")
+        codebuild_client = self._get_codebuild_client()
+        current_result = codebuild_client.list_projects()
+        result = current_result["projects"]
+
+        while "nextToken" in current_result:
+            current_result = codebuild_client.list_projects(nextToken=current_result["nextToken"])
+            result.extend(current_result["projects"])
+        return result
+
+    def start_codebuild(self, project: str, environment_variables_overrides: List[Dict[str, str]], branch: str) -> None:
+        """
+        This functions uses Boto3 to start a batch build.
+        It forwards all variabkes from parameter env_variables as environment variables to the CodeBuild project.
+        If a branch is given, it starts the codebuild for the given branch.
+        After the build has triggered it waits until the batch build finished
+        :raises
+            `RuntimeError` if build fails or AWS Batch build returns unknown status
+        """
+        codebuild_client = self._get_codebuild_client()
+        logging.info(f"Trigger codebuild for project {project} with branch {branch} "
+                     f"and env_variables ({environment_variables_overrides})")
+        ret_val = codebuild_client.start_build_batch(projectName=project,
+                                                     sourceVersion=branch,
+                                                     environmentVariablesOverride=list(
+                                                         environment_variables_overrides))
+        build_id = ret_val['buildBatch']['id']
+        logging.debug(f"Codebuild for project {project} with branch {branch} triggered. Id is {build_id}.")
+
+        for counter in range(120):  #We wait for maximal 1h + (something)
+            time.sleep(30)
+            logging.debug(f"Checking status of codebuild id {build_id}.")
+            build_response = codebuild_client.batch_get_build_batches(ids=[build_id])
+            logging.debug(f"Build response of codebuild id {build_id} is {build_response}")
+            if len(build_response['buildBatches']) != 1:
+                logging.error(f"Unexpected return value from 'batch_get_build_batches': {build_response}")
+            build_status = build_response['buildBatches'][0]['buildBatchStatus']
+            logging.info(f"Build status of codebuild id {build_id} is {build_status}")
+            if build_status == 'SUCCEEDED':
+                break
+            elif build_status in ['FAILED', 'FAULT', 'STOPPED', 'TIMED_OUT']:
+                raise RuntimeError(f"Build ({build_id}) failed with status: {build_status}")
+            elif build_status is not "IN_PROGRESS":
+                raise RuntimeError(f"Batch build {build_id} has unknown build status: {build_status}")
+

exasol_script_languages_container_ci_setup/lib/ci_build.py

@@ -4,9 +4,10 @@
 from exasol_script_languages_container_ci_setup.lib.render_template import render_template
 
 CODE_BUILD_STACK_NAME = "CIBuild"
+CI_CODE_BUILD_TEMPLATE = "slc_code_build.yaml"
 
 
-def stack_name(project: str):
+def ci_stack_name(project: str):
     return f"{project}{CODE_BUILD_STACK_NAME}"
 
 
@@ -18,9 +19,9 @@ def run_deploy_ci_build(aws_access: AwsAccess, project: str, github_url: str):
     """
     logging.info(f"run_deploy_ci_build for aws profile {aws_access.aws_profile} for project {project} at {github_url}")
     dockerhub_secret_arn = aws_access.read_dockerhub_secret_arn()
-    yml = render_template("slc_code_build.yaml", project=project,
+    yml = render_template(CI_CODE_BUILD_TEMPLATE, project=project,
                           dockerhub_secret_arn=dockerhub_secret_arn, github_url=github_url)
-    aws_access.upload_cloudformation_stack(yml, stack_name(project))
+    aws_access.upload_cloudformation_stack(yml, ci_stack_name(project))
 
 
 def run_validate_ci_build(aws_access: AwsAccess, project: str, github_url: str):
@@ -30,6 +31,6 @@ def run_validate_ci_build(aws_access: AwsAccess, project: str, github_url: str):
     logging.info(f"run_validate_ci_build for aws profile {aws_access.aws_profile} "
                  f"for project {project} at {github_url}")
     dockerhub_secret_arn = "dummy_arn"
-    yml = render_template("slc_code_build.yaml", project=project,
+    yml = render_template(CI_CODE_BUILD_TEMPLATE, project=project,
                           dockerhub_secret_arn=dockerhub_secret_arn, github_url=github_url)
     aws_access.validate_cloudformation_template(yml)

exasol_script_languages_container_ci_setup/lib/generate_buildspec_common.py

@@ -0,0 +1,75 @@
+import json
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional, Tuple, Set
+
+import jsonschema
+
+from exasol_script_languages_container_ci_setup.lib.render_template import render_template
+
+
+@dataclass(eq=True, frozen=True)
+class Flavor(object):
+    """"
+    Holds the name and the formatted name used for generating the buildspec.
+    """
+    flavor_original: str
+
+    @property
+    def flavor_formatted(self) -> str:
+        return self.flavor_original.replace(".", "").replace("-", "_")
+
+
+def validate_config_file(config_file: Optional[str]):
+    """
+    Validates config file, path given by parameter config_file.
+    :raises:
+
+        `jsonschema.exceptions.ValidationError` if the config file has invalid JSON format.
+        `jsonschema.exceptions.SchemaError` if the config file is not in accordance with the the schema.
+        `ValueError` if the ignored path given in the config file does not exist.
+    """
+    if config_file is not None:
+        with open(config_file, "r") as config_file_:
+            config = json.load(config_file_)
+            config_schema = json.loads(render_template("config_schema.json"))
+            jsonschema.validate(instance=config, schema=config_schema)
+            ignored_paths = config["build_ignore"]["ignored_paths"]
+            for ignored_path in ignored_paths:
+                folder_path = Path(ignored_path)
+                if not folder_path.exists():
+                    raise ValueError(f"Ignored folder '{ignored_path}' does not exist.")
+
+
+def get_config_file_parameter(config_file: Optional[str]):
+    if config_file is None:
+        return ""
+    return f"--config-file {config_file}"
+
+
+def _find_flavors(flavor_root_paths: Tuple[str, ...]) -> Set[Flavor]:
+    flavors = set()
+    for flavor_root_path in [Path(f).resolve() for f in flavor_root_paths]:
+        assert flavor_root_path.is_dir()
+        assert flavor_root_path.exists()
+        assert flavor_root_path.name == "flavors"
+        dirs = (d for d in flavor_root_path.iterdir() if d.is_dir())
+        flavors.update(map(lambda directory: Flavor(directory.name), dirs))
+    logging.info(f"Found flavors: {flavors}")
+    return flavors
+
+
+def write_batch_build_spec(flavor_root_paths: Tuple[str, ...], output_pathname: str) -> None:
+    buildspec_body = []
+    flavors = _find_flavors(flavor_root_paths)
+    for flavor in flavors:
+        buildspec_body.append(render_template("buildspec_batch_entry.yaml",
+                                              flavor_original=flavor.flavor_original,
+                                              flavor_formatted=flavor.flavor_formatted,
+                                              out_path=output_pathname))
+
+    result_yaml = render_template("buildspec_hull.yaml", batch_entries="\n".join(buildspec_body))
+
+    with open(Path(output_pathname) / "buildspec.yaml", "w") as output_file:
+        output_file.write(result_yaml)

exasol_script_languages_container_ci_setup/lib/release_build.py

@@ -0,0 +1,37 @@
+import logging
+
+from exasol_script_languages_container_ci_setup.lib.aws_access import AwsAccess
+from exasol_script_languages_container_ci_setup.lib.render_template import render_template
+
+CODE_BUILD_STACK_NAME = "ReleaseBuild"
+RELEASE_CODEBUILD_TEMPLATE = "slc_code_release_build.yaml"
+
+
+def release_stack_name(project: str):
+    return f"{project}{CODE_BUILD_STACK_NAME}"
+
+
+def run_deploy_release_build(aws_access: AwsAccess, project: str, github_url: str):
+    """
+    This command deploys the ci build cloudformation stack
+    1. It get's the dockerhub secret ARN from AWS via Boto3
+    2. Then it renders the template and uploads the resulting cloudformation YAML file.
+    """
+    logging.info(f"run_deploy_release_build for aws profile "
+                 f"{aws_access.aws_profile} for project {project} at {github_url}")
+    dockerhub_secret_arn = aws_access.read_dockerhub_secret_arn()
+    yml = render_template(RELEASE_CODEBUILD_TEMPLATE, project=project,
+                          dockerhub_secret_arn=dockerhub_secret_arn, github_url=github_url)
+    aws_access.upload_cloudformation_stack(yml, release_stack_name(project))
+
+
+def run_validate_release_build(aws_access: AwsAccess, project: str, github_url: str):
+    """
+    This command validates the release build cloudformation stack
+    """
+    logging.info(f"run_validate_release_build for aws profile {aws_access.aws_profile} "
+                 f"for project {project} at {github_url}")
+    dockerhub_secret_arn = "dummy_arn"
+    yml = render_template(RELEASE_CODEBUILD_TEMPLATE, project=project,
+                          dockerhub_secret_arn=dockerhub_secret_arn, github_url=github_url)
+    aws_access.validate_cloudformation_template(yml)