#52 Created CI build workflow to build and push Docker Image

.github/workflows/check_ci.yaml

@@ -24,7 +24,7 @@ jobs:
          --override-ini=log_cli=true
          --override-ini=log_cli_level=INFO
          test/unit
-         test/integration/test_create_dss_docker_image.py
+         test/integration
        env: # Set the secret as an env variable
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}

aws-code-build/ci/buildspec.yaml

@@ -25,4 +25,4 @@ phases:
       commands: >
         poetry run python3 -m pytest
         -s -o log_cli=true -o log_cli_level=INFO
-        test/ci/test_ci*.py
+        test/codebuild/test_ci*.py

aws-code-build/ci/buildspec_release.yaml

@@ -7,6 +7,10 @@ env:
     ASSET_ID: ""
     AWS_USER_NAME: "release_user"
     MAKE_AMI_PUBLIC_OPTION: "--no-make-ami-public"
+  secrets-manager:
+    DOCKER_REGISTRY_USER: "Dockerhub:User"
+    DOCKER_REGISTRY_PASSWORD: "Dockerhub:AccessToken"
+
 phases:
 
   install:
@@ -28,4 +32,5 @@ phases:
   build:
       commands:
         - poetry run python3 -m exasol.ds.sandbox.main create-vm --default-password "$DEFAULT_PASSWORD" --asset-id "$ASSET_ID" $MAKE_AMI_PUBLIC_OPTION
+        - poetry run python3 -m exasol.ds.sandbox.main create-docker-image --version "$ASSET_ID" --publish --log-level info
         - poetry run python3 -m exasol.ds.sandbox.main update-release --release-id "$RELEASE_ID" --asset-id "$ASSET_ID"

doc/changes/changes_0.1.0.md

@@ -37,6 +37,7 @@ Version: 0.1.0
 * #56: Moved jupyter notebook files again
 * #63: Improved logging of Ansible tasks
 * #46: Enabled to suppress ansible output
+* #52: Created CI build workflow to build and push Docker Image
 
 ## Documentation
 

doc/developer_guide/developer_guide.md

@@ -260,4 +260,18 @@ The project has two types of CI tests:
 * A system test which runs on a AWS Codebuild
 
 Both ci tests need to pass before the approval of a Github PR.
-The Github workflow will run on each push to a branch in the Github repository. However, the AWS Codebuild will only run after you push a commit containing the string "[CodeBuild]" in the commit message.
+The Github workflow will run on each push to a branch in the Github repository. However, the AWS Codebuild will only run after you push a commit containing the string "[CodeBuild]" in the commit message, see [Executing AWS CodeBuild](#executing-aws-codebuild).
+
+## Executing AWS CodeBuild
+
+Use the following git commands to execute the AWS CodeBuild script:
+```shell
+git commit -m "[CodeBuild]" --allow-empty && git push
+```
+
+This will trigger a webhook that was installed by an AWS template into the git-Repository.
+* The webhook is defined in file `exasol/ds/sandbox/templates/ci_code_build.jinja.yaml`
+* and calls `aws-code-build/ci/buildspec.yaml`
+* which then executes `test/codebuild/test_ci*.py`
+
+The CodeBuild will take about 20 minutes to complete.

exasol/ds/sandbox/cli/commands/create_docker_image.py

@@ -4,7 +4,11 @@
 from exasol.ds.sandbox.cli.cli import cli, option_with_env_default
 from exasol.ds.sandbox.cli.options.logging import logging_options
 from exasol.ds.sandbox.cli.common import add_options
-from exasol.ds.sandbox.lib.dss_docker import DssDockerImage, DockerRegistry
+from exasol.ds.sandbox.lib.dss_docker import (
+    DEFAULT_ORG_AND_REPOSITORY,
+    DssDockerImage,
+    DockerRegistry,
+)
 from exasol.ds.sandbox.lib.logging import SUPPORTED_LOG_LEVELS
 from exasol.ds.sandbox.lib.logging import set_log_level
 
@@ -18,7 +22,7 @@
     click.option(
         "--repository", type=str, metavar="[HOST[:PORT]/]ORG/REPO",
         show_default=True,
-        default="exasol/data-science-sandbox",
+        default=DEFAULT_ORG_AND_REPOSITORY,
         help="""
         Organization and repository on hub.docker.com to publish the
         docker image to. Optionally prefixed by a host and a port,

exasol/ds/sandbox/cli/commands/show_aws_assets.py

@@ -1,26 +1,39 @@
 from typing import Tuple, Optional
 
 import click
+import contextlib
 
 from exasol.ds.sandbox.cli.cli import cli
 from exasol.ds.sandbox.cli.common import add_options
 from exasol.ds.sandbox.cli.options.aws_options import aws_options
 from exasol.ds.sandbox.cli.options.logging import logging_options
 from exasol.ds.sandbox.lib.asset_id import AssetId
 from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess
-from exasol.ds.sandbox.lib.asset_printing.print_assets import all_asset_types, print_assets
+from exasol.ds.sandbox.lib.asset_printing.print_assets import (
+    aws_asset_type_names,
+    print_assets,
+    AssetTypes,
+)
 from exasol.ds.sandbox.lib.logging import set_log_level
 
 
+@contextlib.contextmanager
+def optional_write_to(path: str):
+    if path is None:
+        yield None
+    else:
+        yield open(path, "w")
+
+
 @cli.command()
 @add_options(aws_options)
 @add_options(logging_options)
 @click.option('--asset-id', type=str, default=None,
               help="The asset-id used to create the AWS resources during the other commands. "
                    "If not set, all resources will be printed. "
                    "The value might contain wildcards.")
-@click.option('--asset-type', default=all_asset_types(),
-              type=click.Choice(list(all_asset_types())), multiple=True,
+@click.option('--asset-type', default=aws_asset_type_names(),
+              type=click.Choice(list(aws_asset_type_names())), multiple=True,
               help="The asset types to print. Can be declared multiple times.")
 @click.option('--out-file', default=None, type=click.Path(exists=False, file_okay=True, dir_okay=False),
               help="If given, writes the AWS assets to this file in markdown format.")
@@ -35,5 +48,11 @@ def show_aws_assets(
     """
     set_log_level(log_level)
     _asset_id = AssetId(asset_id) if asset_id is not None else None
-    print_assets(AwsAccess(aws_profile=aws_profile), asset_id=_asset_id,
-                 outfile=out_file, asset_types=asset_type)
+    asset_types = tuple(AssetTypes.from_name(n) for n in asset_type)
+    with optional_write_to(out_file) as handle:
+        print_assets(
+            AwsAccess(aws_profile=aws_profile),
+            asset_id=_asset_id,
+            out_file_obj=handle,
+            asset_types=asset_types,
+        )

exasol/ds/sandbox/cli/commands/start_test_release_build.py

@@ -1,4 +1,3 @@
-import os
 from typing import Optional
 
 import click
@@ -9,7 +8,10 @@
 from exasol.ds.sandbox.cli.options.logging import logging_options
 from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess
 from exasol.ds.sandbox.lib.logging import set_log_level
-from exasol.ds.sandbox.lib.github_release_access import GithubReleaseAccess
+from exasol.ds.sandbox.lib.github_release_access import (
+    github_token_or_exit,
+    GithubReleaseAccess,
+)
 from exasol.ds.sandbox.lib.release_build.run_release_build import run_start_test_release_build
 
 
@@ -27,9 +29,17 @@ def start_test_release_build(
         release_title: str
 ):
     """
-    This command  triggers the AWS release Codebuild to generate a new sandbox test version.
+    This command triggers the AWS release Codebuild to generate a new
+    sandbox test version. GitHub token is expected to be found in environment
+    variable GITHUB_TOKEN.
     """
     set_log_level(log_level)
-    gh_token = os.getenv("GITHUB_TOKEN")
-    run_start_test_release_build(AwsAccess(aws_profile), GithubReleaseAccess(gh_token),
-                                 branch, release_title, gh_token)
+    gh_token = github_token_or_exit()
+    run_start_test_release_build(
+        AwsAccess(aws_profile),
+        GithubReleaseAccess(gh_token),
+        branch,
+        release_title,
+        gh_token,
+    )
+

exasol/ds/sandbox/cli/commands/update_release.py

@@ -1,4 +1,3 @@
-import os
 from typing import Optional
 
 import click
@@ -11,7 +10,10 @@
 from exasol.ds.sandbox.lib.asset_id import AssetId
 from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess
 from exasol.ds.sandbox.lib.logging import set_log_level
-from exasol.ds.sandbox.lib.github_release_access import GithubReleaseAccess
+from exasol.ds.sandbox.lib.github_release_access import (
+    github_token_or_exit,
+    GithubReleaseAccess,
+)
 from exasol.ds.sandbox.lib.update_release.run_update_release import run_update_release
 
 
@@ -27,9 +29,15 @@ def update_release(
         asset_id: str,
         log_level: str):
     """
-    This command attaches the links of the release assets (AMI, VM images) to the Github release,
-    indicated by parameter 'release-id'.
+    This command attaches the links of the release assets (AMI, VM images)
+    to the Github release, indicated by command line option '--release-id'.
+    GitHub token is expected to be found in environment variable GITHUB_TOKEN.
     """
     set_log_level(log_level)
-    run_update_release(AwsAccess(aws_profile), GithubReleaseAccess(os.getenv("GITHUB_TOKEN")),
-                       release_id, AssetId(asset_id))
+    gh_token = github_token_or_exit()
+    run_update_release(
+        AwsAccess(aws_profile),
+        GithubReleaseAccess(gh_token),
+        release_id,
+        AssetId(asset_id),
+    )

exasol/ds/sandbox/lib/asset_printing/mark_down_printer.py

@@ -1,11 +1,19 @@
 import io
-from abc import ABC
-from typing import Tuple
-
 import pandas as pd
 
-from exasol.ds.sandbox.lib.asset_printing.printing_factory import PrintingFactory, TablePrinter, \
-    TextPrinter, TextObject
+from abc import ABC
+from typing import Optional, Tuple
+from inspect import cleandoc
+
+from exasol.ds.sandbox.lib.asset_printing.printing_factory import (
+    CodeBlockTextObject,
+    HighlightedTextObject,
+    TitleTextObject,
+    PrintingFactory,
+    TablePrinter,
+    TextObject,
+    TextPrinter,
+)
 
 
 class MarkdownTablePrinter(TablePrinter, ABC):
@@ -28,18 +36,38 @@ def finish(self):
 
 
 class MarkdownTextPrinter(TextPrinter, ABC):
+    def __init__(self, target: io.TextIOBase):
+        self.target = target
+
+    def _format(self, obj: TextObject) -> str:
+        if isinstance(obj, TitleTextObject):
+            return f'#### {obj.text}\n'
+        if isinstance(obj, CodeBlockTextObject):
+            return cleandoc(f"""
+            ```shell
+            {obj.text}
+            ```
+            """)
+        return f'{obj.text}\n'
 
+    def print(self, text_objects: Tuple[TextObject, ...]):
+        for text_object in text_objects:
+            print(self._format(text_object), file=self.target)
+
+
+class NullPrinter(TextPrinter):
     def print(self, text_objects: Tuple[TextObject, ...]):
         pass
 
 
 class MarkdownPrintingFactory(PrintingFactory, ABC):
-
     def __init__(self, target: io.TextIOBase):
         self.target = target
 
     def create_table_printer(self, title: str):
         return MarkdownTablePrinter(title, self.target)
 
-    def create_text_printer(self):
-        return MarkdownTextPrinter()
+    def create_text_printer(self, console_only: bool = False):
+        if console_only:
+            return NullPrinter()
+        return MarkdownTextPrinter(self.target)