Supported pushing Docker image to a Docker registry

doc/changes/changes_0.1.0.md

@@ -24,6 +24,7 @@ Version: 0.1.0
 * #23: Fixed AWS Code build
 * #69: Added entry point to start Jupyter server
 * #84: Fixed retrieval and display of jupyter password
+* #36: Supported pushing Docker image to a Docker registry
 
 ## Bug Fixes
 

exasol/ds/sandbox/cli/cli.py

@@ -1,6 +1,15 @@
 import click
+import os
 
 
 @click.group()
 def cli():
     pass
+
+
+def option_with_env_default(envvar: str, *args, **kwargs):
+    kwargs["help"] = f"{kwargs['help']} [defaults to environment variable '{envvar}']"
+    return click.option(
+        *args, **kwargs,
+        default=lambda: os.environ.get(envvar, ""),
+    )

exasol/ds/sandbox/cli/commands/create_docker_image.py

@@ -1,25 +1,46 @@
 import click
+import os
 
-from exasol.ds.sandbox.cli.cli import cli
+from exasol.ds.sandbox.cli.cli import cli, option_with_env_default
 from exasol.ds.sandbox.cli.options.logging import logging_options
 from exasol.ds.sandbox.cli.common import add_options
-from exasol.ds.sandbox.lib.dss_docker import DssDockerImage
+from exasol.ds.sandbox.lib.dss_docker import DssDockerImage, DockerRegistry
 from exasol.ds.sandbox.lib.logging import SUPPORTED_LOG_LEVELS
 from exasol.ds.sandbox.lib.logging import set_log_level
 
 
+USER_ENV = "DOCKER_REGISTRY_USER"
+PASSWORD_ENV = "DOCKER_REGISTRY_PASSWORD"
+
+
 @cli.command()
 @add_options([
     click.option(
-        '--repository', type=str, metavar="ORG/REPO", show_default=True,
+        "--repository", type=str, metavar="[HOST[:PORT]/]ORG/REPO",
+        show_default=True,
         default="exasol/data-science-sandbox",
-        help="Organization and repository on hub.docker.com to publish the docker image to"),
-    click.option('--version', type=str, help="Docker image version tag"),
+        help="""
+        Organization and repository on hub.docker.com to publish the
+        docker image to. Optionally prefixed by a host and a port,
+        see https://docs.docker.com/engine/reference/commandline/tag.
+        """),
+    click.option(
+        "--version", type=str,  metavar="VERSION",
+        help="Docker image version tag"),
     click.option(
-        '--publish', type=bool, is_flag=True,
+        "--publish", type=bool, is_flag=True,
         help="Whether to publish the created Docker image"),
     click.option(
-        '--keep-container', type=bool, is_flag=True,
+        "--registry-user", type=str, metavar="USER",
+        default=lambda: os.environ.get(USER_ENV, None),
+        help=f"""
+        Username for Docker registry [defaults to environment
+        variable {USER_ENV}]. If specified then password is read
+        from environment variable {PASSWORD_ENV}.
+        """
+        ),
+    click.option(
+        "--keep-container", type=bool, is_flag=True,
         help="""Keep the Docker Container running after creating the image.
         Otherwise stop and remove the container."""),
 ])
@@ -28,17 +49,23 @@ def create_docker_image(
         repository: str,
         version: str,
         publish: bool,
+        registry_user: str,
         keep_container: bool,
         log_level: str,
 ):
     """
-    Create a Docker image for data-science-sandbox and deploy
-    it to a Docker repository.
+    Create a Docker image for data-science-sandbox.  If option
+    ``--publish`` is specified then deploy the image to the Docker registry
+    using the specified user name and reading the password from environment
+    variable ``PASSWORD_ENV``.
     """
+    def registry_password():
+        if registry_user is None:
+            return None
+        return os.environ.get(PASSWORD_ENV, None)
+
     set_log_level(log_level)
-    DssDockerImage(
-        repository=repository,
-        version=version,
-        publish=publish,
-        keep_container=keep_container,
-    ).create()
+    creator = DssDockerImage(repository, version, keep_container)
+    if publish:
+        creator.registry = DockerRegistry(registry_user, registry_password())
+    creator.create()

exasol/ds/sandbox/lib/dss_docker/__init__.py

@@ -1 +1 @@
-from .create_image import DssDockerImage
+from .create_image import DssDockerImage, DockerRegistry

exasol/ds/sandbox/lib/dss_docker/create_image.py

@@ -2,12 +2,13 @@
 import humanfriendly
 import importlib_resources
 
+from functools import reduce
 from datetime import datetime
 from docker.types import Mount
 from exasol.ds.sandbox.lib import pretty_print
 from importlib_metadata import version
 from pathlib import Path
-from typing import List
+from typing import Dict, List, Optional
 
 from docker.models.containers import Container as DockerContainer
 from docker.models.images import Image as DockerImage
@@ -20,19 +21,20 @@
 from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies
 from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo
 
-
 DSS_VERSION = version("exasol-data-science-sandbox")
 _logger = get_status_logger(LogType.DOCKER_IMAGE)
 
 
-def get_fact(facts: AnsibleFacts, *keys: str) -> str:
-    keys = list(keys)
-    keys.insert(0, "dss_facts")
-    for key in keys:
-        if not key in facts:
-            return None
-        facts = facts[key]
-    return facts
+def get_fact(facts: AnsibleFacts, *keys: str) -> Optional[str]:
+    return get_nested_value(facts, "dss_facts", *keys)
+
+
+def get_nested_value(mapping: Dict[str, any], *keys: str) -> Optional[str]:
+    def nested_item(current, key):
+        valid = current is not None and key in current
+        return current[key] if valid else None
+
+    return reduce(nested_item, keys, mapping)
 
 
 def entrypoint(facts: AnsibleFacts) -> List[str]:
@@ -63,15 +65,19 @@ def __init__(
             self,
             repository: str,
             version: str = None,
-            publish: bool = False,
             keep_container: bool = False,
     ):
         version = version if version else DSS_VERSION
         self.container_name = f"ds-sandbox-{DssDockerImage.timestamp()}"
-        self.image_name = f"{repository}:{version}"
-        self.publish = publish
+        self.repository = repository
+        self.version = version
         self.keep_container = keep_container
         self._start = None
+        self.registry = None
+
+    @property
+    def image_name(self):
+        return f"{self.repository}:{self.version}"
 
     def _ansible_run_context(self) -> AnsibleRunContext:
         extra_vars = {
@@ -150,11 +156,16 @@ def _cleanup(self, container: DockerContainer):
         _logger.info("Removing container")
         container.remove()
 
+    def _push(self):
+        if self.registry is not None:
+            self.registry.push(self.repository, self.version)
+
     def create(self):
         try:
             container = self._start_container()
             facts = self._install_dependencies()
             image = self._commit_container(container, facts)
+            self._push()
         except Exception as ex:
             raise ex
         finally:

exasol/ds/sandbox/lib/dss_docker/push_image.py

@@ -0,0 +1,65 @@
+import docker
+import json
+import logging
+import requests
+
+from docker.client import DockerClient
+
+from typing import Callable, Optional
+from exasol.ds.sandbox.lib.logging import get_status_logger, LogType
+
+
+_logger = get_status_logger(LogType.DOCKER_IMAGE)
+
+
+class ProgressReporter:
+    def __init__(self, verbose: bool):
+        self.last_status = None
+        self.verbose = verbose
+        self.need_linefeed = False
+
+    def _report(self, printer: Callable, msg: Optional[str], **kwargs):
+        if msg is not None:
+            printer(msg, **kwargs)
+
+    def _linefeed(self):
+        if self.need_linefeed:
+            self.need_linefeed = False
+            print()
+
+    def report(self, status: Optional[str], progress: Optional[str]):
+        if not self.verbose:
+            return
+        if status == self.last_status:
+            self._report(print, progress, end="\r")
+            self.need_linefeed = progress
+        else:
+            self.last_status = status
+            self._linefeed()
+            self._report(_logger.info, status)
+
+
+class DockerRegistry:
+    def __init__(self, username: str, password: str):
+        self.username = username
+        self.password = password
+
+    def push(self, repository: str, tag: str):
+        auth_config = {
+            "username": self.username,
+            "password": self.password,
+        }
+        client = docker.from_env()
+        resp = client.images.push(
+            repository=repository,
+            tag=tag,
+            auth_config=auth_config,
+            stream=True,
+            decode=True,
+        )
+        reporter = ProgressReporter(_logger.isEnabledFor(logging.INFO))
+        for el in resp:
+            reporter.report(
+                el.get("status", None),
+                el.get("progress", None),
+            )

pyproject.toml

@@ -61,3 +61,8 @@ testpaths = [
     "test"
 ]
 
+# ignore warnings caused by docker lib
+filterwarnings = [
+    "ignore:the imp module is deprecated:DeprecationWarning",
+    "ignore:distutils Version classes are deprecated:DeprecationWarning",
+]

test/integration/conftest.py

@@ -1,6 +1,40 @@
+import docker
+import pytest
+
+from exasol.ds.sandbox.lib.dss_docker import DssDockerImage
+
 
 def pytest_addoption(parser):
     parser.addoption(
         "--dss-docker-image", default=None,
         help="Name and version of existing Docker image to use for tests",
     )
+    parser.addoption(
+        "--docker-registry", default=None, metavar="HOST:PORT",
+        help="Docker registry for pushing Docker images to",
+    )
+
+
+@pytest.fixture(scope="session")
+def dss_docker_image(request):
+    """
+    If dss_docker_image_name is provided then don't create an image but
+    reuse the existing image as specified by cli option
+    --ds-docker-image-name.
+    """
+    existing = request.config.getoption("--dss-docker-image")
+    if existing and ":" in existing:
+        name, version = existing.split(":")
+        yield DssDockerImage(name, version)
+        return
+
+    testee = DssDockerImage(
+        "my-repo/dss-test-image",
+        version=f"{DssDockerImage.timestamp()}",
+        keep_container=False,
+    )
+    testee.create()
+    try:
+        yield testee
+    finally:
+        docker.from_env().images.remove(testee.image_name)

test/integration/local_docker_registry.py

@@ -0,0 +1,47 @@
+import docker
+import contextlib
+import json
+import logging
+import requests
+import time
+
+from typing import Dict, List
+from test.ports import find_free_port
+from exasol.ds.sandbox.lib.dss_docker import DssDockerImage, DockerRegistry
+
+
+_logger = logging.getLogger(__name__)
+_logger.setLevel(logging.DEBUG)
+
+
+class LocalDockerRegistry(DockerRegistry):
+    """
+    Simulate Docker registry by running a local Docker container and using
+    the registry inside.
+
+    Please note for pushing images to a Docker registry with host or port
+    differing from the official address requires to tag images in advance.
+
+    So host and port must be prepended to property ``repository`` of the
+    image.
+    """
+    def __init__(self, host_and_port: str):
+        super().__init__(username=None, password=None)
+        self.host_and_port = host_and_port
+
+    @property
+    def url(self):
+        return f'http://{self.host_and_port}'
+
+    def images(self, repo_name: str) -> Dict[str, any]:
+        url = f"{self.url}/v2/{repo_name}/tags/list"
+        result = requests.request("GET", url)
+        images = json.loads(result.content.decode("UTF-8"))
+        return images
+
+    @property
+    def repositories(self) -> List[str]:
+        url = f"{self.url}/v2/_catalog/"
+        result = requests.request("GET", url)
+        repos = json.loads(result.content.decode("UTF-8"))["repositories"]
+        return repos

test/integration/test_create_dss_docker_image.py

@@ -18,32 +18,6 @@
 from exasol.ds.sandbox.lib import pretty_print
 
 
-@pytest.fixture(scope="session")
-def dss_docker_image(request):
-    """
-    If dss_docker_image_name is provided then don't create an image but
-    reuse the existing image as specified by cli option
-    --ds-docker-image-name, see file conftest.py.
-    """
-    existing = request.config.getoption("--dss-docker-image")
-    if existing and ":" in existing:
-        name, version = existing.split(":")
-        yield DssDockerImage(name, version)
-        return
-
-    testee = DssDockerImage(
-        "my-repo/dss-test-image",
-        version=f"{DssDockerImage.timestamp()}",
-        publish=False,
-        keep_container=False,
-    )
-    testee.create()
-    try:
-        yield testee
-    finally:
-        docker.from_env().images.remove(testee.image_name)
-
-
 @pytest.fixture
 def dss_docker_container(dss_docker_image):
     client = docker.from_env()