Supported pushing Docker image to a Docker registry

doc/changes/changes_0.1.0.md

@@ -24,6 +24,7 @@ Version: 0.1.0
 * #23: Fixed AWS Code build
 * #69: Added entry point to start Jupyter server
 * #84: Fixed retrieval and display of jupyter password
+* #36: Supported pushing Docker image to a Docker registry
 
 ## Bug Fixes
 

exasol/ds/sandbox/cli/cli.py

@@ -1,6 +1,15 @@
 import click
+import os
 
 
 @click.group()
 def cli():
     pass
+
+
+def option_with_env_default(envvar: str, *args, **kwargs):
+    kwargs["help"] = f"{kwargs['help']} [defaults to environment variable '{envvar}']"
+    return click.option(
+        *args, **kwargs,
+        default=lambda: os.environ.get(envvar, ""),
+    )

exasol/ds/sandbox/cli/commands/create_docker_image.py

@@ -1,9 +1,9 @@
 import click
 
-from exasol.ds.sandbox.cli.cli import cli
+from exasol.ds.sandbox.cli.cli import cli, option_with_env_default
 from exasol.ds.sandbox.cli.options.logging import logging_options
 from exasol.ds.sandbox.cli.common import add_options
-from exasol.ds.sandbox.lib.dss_docker import DssDockerImage
+from exasol.ds.sandbox.lib.dss_docker import DssDockerImage, DockerRegistry
 from exasol.ds.sandbox.lib.logging import SUPPORTED_LOG_LEVELS
 from exasol.ds.sandbox.lib.logging import set_log_level
 
@@ -14,10 +14,15 @@
         '--repository', type=str, metavar="ORG/REPO", show_default=True,
         default="exasol/data-science-sandbox",
         help="Organization and repository on hub.docker.com to publish the docker image to"),
-    click.option('--version', type=str, help="Docker image version tag"),
     click.option(
-        '--publish', type=bool, is_flag=True,
-        help="Whether to publish the created Docker image"),
+        '--version', type=str,  metavar="VERSION",
+        help="Docker image version tag"),
+    option_with_env_default("DOCKER_REGISTRY_USER",
+        '--registry-user', type=str, metavar="USER",
+        help="Username for publication to Docker registry"),
+    option_with_env_default("DOCKER_REGISTRY_PASSWORD",
+        '--registry-password', type=str, metavar="PASSWORD",
+        help="Password for publication to Docker registry"),
     click.option(
         '--keep-container', type=bool, is_flag=True,
         help="""Keep the Docker Container running after creating the image.
@@ -27,18 +32,26 @@
 def create_docker_image(
         repository: str,
         version: str,
-        publish: bool,
+        registry_user: str,
+        registry_password: str,
         keep_container: bool,
         log_level: str,
 ):
     """
-    Create a Docker image for data-science-sandbox and deploy
-    it to a Docker repository.
+    Create a Docker image for data-science-sandbox.  If username and password
+    for the Docker registry are specified then deploy the image to the registry.
     """
     set_log_level(log_level)
-    DssDockerImage(
+    creator = DssDockerImage(
         repository=repository,
         version=version,
-        publish=publish,
         keep_container=keep_container,
-    ).create()
+    )
+    if registry_user and registry_password:
+        creator.registry = DockerRegistry(
+            creator.repository,
+            registry_user,
+            registry_password,
+        )
+    print(f'user {creator.registry.username} password {registry_password}')
+    # creator.create()

exasol/ds/sandbox/lib/dss_docker/__init__.py

@@ -1 +1 @@
-from .create_image import DssDockerImage
+from .create_image import DssDockerImage, DockerRegistry

exasol/ds/sandbox/lib/dss_docker/create_image.py

@@ -19,7 +19,7 @@
 from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess, AnsibleFacts
 from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies
 from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo
-
+from exasol.ds.sandbox.lib.dss_docker.push_image import DockerRegistry
 
 DSS_VERSION = version("exasol-data-science-sandbox")
 _logger = get_status_logger(LogType.DOCKER_IMAGE)
@@ -63,15 +63,19 @@ def __init__(
             self,
             repository: str,
             version: str = None,
-            publish: bool = False,
             keep_container: bool = False,
     ):
         version = version if version else DSS_VERSION
         self.container_name = f"ds-sandbox-{DssDockerImage.timestamp()}"
-        self.image_name = f"{repository}:{version}"
-        self.publish = publish
+        self.repository = repository
+        self.version = version
         self.keep_container = keep_container
         self._start = None
+        self.registry = None
+
+    @property
+    def image_name(self):
+        return f"{self.repository}:{self.version}"
 
     def _ansible_run_context(self) -> AnsibleRunContext:
         extra_vars = {
@@ -150,11 +154,16 @@ def _cleanup(self, container: DockerContainer):
         _logger.info("Removing container")
         container.remove()
 
+    def _push(self):
+        if self.registry is not None:
+            self.registry.push(self.version)
+
     def create(self):
         try:
             container = self._start_container()
             facts = self._install_dependencies()
             image = self._commit_container(container, facts)
+            self._push()
         except Exception as ex:
             raise ex
         finally:

exasol/ds/sandbox/lib/dss_docker/push_image.py

@@ -0,0 +1,79 @@
+import docker
+import json
+import logging
+import requests
+
+from docker.client import DockerClient
+
+from typing import Callable, Dict, Optional
+from exasol.ds.sandbox.lib.logging import get_status_logger, LogType
+
+
+_logger = get_status_logger(LogType.DOCKER_IMAGE)
+
+
+def get_from_dict(d: Dict[str, any], *keys: str) -> str:
+    for key in keys:
+        if not key in d:
+            return None
+        d = d[key]
+    return d
+
+
+class ProgressReporter:
+    def __init__(self, verbose: bool):
+        self.last_status = None
+        self.verbose = verbose
+        self.need_linefeed = False
+
+    def _report(self, printer: Callable, msg: Optional[str], **kwargs):
+        if msg is not None:
+            printer(msg, **kwargs)
+
+    def _linefeed(self):
+        if self.need_linefeed:
+            self.need_linefeed = False
+            print()
+
+    def report(self, status: Optional[str], progress: Optional[str]):
+        if not self.verbose:
+            return
+        if status == self.last_status:
+            self._report(print, progress, end="\r")
+            self.need_linefeed = progress
+        else:
+            self.last_status = status
+            self._linefeed()
+            self._report(_logger.info, status)
+
+
+class DockerRegistry:
+    def __init__(self, repository: str, username: str, password: str):
+        self.repository = repository
+        self.username = username
+        self.password = password
+        self._client = None
+
+    def client(self):
+        if self._client is None:
+            self._client = docker.from_env()
+        return self._client
+
+    def push(self, tag: str):
+        auth_config = {
+            "username": self.username,
+            "password": self.password,
+        }
+        resp = self.client().images.push(
+            repository=self.repository,
+            tag=tag,
+            auth_config=auth_config,
+            stream=True,
+            decode=True,
+        )
+        reporter = ProgressReporter(_logger.isEnabledFor(logging.INFO))
+        for el in resp:
+            reporter.report(
+                el.get("status", None),
+                el.get("progress", None),
+            )

exasol/ds/sandbox/runtime/ansible/dss_docker_playbook.yml

@@ -21,5 +21,5 @@
       need_sudo: false
       docker_integration_test: true
   tasks:
-    - import_tasks: general_setup_tasks.yml
-    - import_tasks: cleanup_tasks.yml
+#     - import_tasks: general_setup_tasks.yml
+#     - import_tasks: cleanup_tasks.yml

test/integration/conftest.py

@@ -1,6 +1,41 @@
+import docker
+import pytest
+
+from exasol.ds.sandbox.lib.dss_docker import DssDockerImage
+
 
 def pytest_addoption(parser):
     parser.addoption(
         "--dss-docker-image", default=None,
         help="Name and version of existing Docker image to use for tests",
     )
+    parser.addoption(
+        "--docker-registry", default=None, metavar="HOST:PORT",
+        help="Docker registry for pushing Docker images to",
+    )
+
+
+@pytest.fixture(scope="session")
+def dss_docker_image(request):
+    """
+    If dss_docker_image_name is provided then don't create an image but
+    reuse the existing image as specified by cli option
+    --ds-docker-image-name.
+    """
+    existing = request.config.getoption("--dss-docker-image")
+    if existing and ":" in existing:
+        name, version = existing.split(":")
+        yield DssDockerImage(name, version)
+        return
+
+    testee = DssDockerImage(
+        "my-repo/dss-test-image",
+        version=f"{DssDockerImage.timestamp()}",
+        publish=False,
+        keep_container=False,
+    )
+    testee.create()
+    try:
+        yield testee
+    finally:
+        docker.from_env().images.remove(testee.image_name)

test/integration/test_create_dss_docker_image.py

@@ -18,32 +18,6 @@
 from exasol.ds.sandbox.lib import pretty_print
 
 
-@pytest.fixture(scope="session")
-def dss_docker_image(request):
-    """
-    If dss_docker_image_name is provided then don't create an image but
-    reuse the existing image as specified by cli option
-    --ds-docker-image-name, see file conftest.py.
-    """
-    existing = request.config.getoption("--dss-docker-image")
-    if existing and ":" in existing:
-        name, version = existing.split(":")
-        yield DssDockerImage(name, version)
-        return
-
-    testee = DssDockerImage(
-        "my-repo/dss-test-image",
-        version=f"{DssDockerImage.timestamp()}",
-        publish=False,
-        keep_container=False,
-    )
-    testee.create()
-    try:
-        yield testee
-    finally:
-        docker.from_env().images.remove(testee.image_name)
-
-
 @pytest.fixture
 def dss_docker_container(dss_docker_image):
     client = docker.from_env()