diff --git a/doc/changes/changes_0.1.0.md b/doc/changes/changes_0.1.0.md index b805652b..73611329 100644 --- a/doc/changes/changes_0.1.0.md +++ b/doc/changes/changes_0.1.0.md @@ -24,6 +24,7 @@ Version: 0.1.0 * #23: Fixed AWS Code build * #69: Added entry point to start Jupyter server * #84: Fixed retrieval and display of jupyter password +* #36: Supported pushing Docker image to a Docker registry ## Bug Fixes diff --git a/exasol/ds/sandbox/cli/cli.py b/exasol/ds/sandbox/cli/cli.py index 21966a2e..8f5d9e1e 100644 --- a/exasol/ds/sandbox/cli/cli.py +++ b/exasol/ds/sandbox/cli/cli.py @@ -1,6 +1,15 @@ import click +import os @click.group() def cli(): pass + + +def option_with_env_default(envvar: str, *args, **kwargs): + kwargs["help"] = f"{kwargs['help']} [defaults to environment variable '{envvar}']" + return click.option( + *args, **kwargs, + default=lambda: os.environ.get(envvar, ""), + ) diff --git a/exasol/ds/sandbox/cli/commands/create_docker_image.py b/exasol/ds/sandbox/cli/commands/create_docker_image.py index 16ffb587..b4d86f3b 100644 --- a/exasol/ds/sandbox/cli/commands/create_docker_image.py +++ b/exasol/ds/sandbox/cli/commands/create_docker_image.py @@ -1,25 +1,46 @@ import click +import os -from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.cli import cli, option_with_env_default from exasol.ds.sandbox.cli.options.logging import logging_options from exasol.ds.sandbox.cli.common import add_options -from exasol.ds.sandbox.lib.dss_docker import DssDockerImage +from exasol.ds.sandbox.lib.dss_docker import DssDockerImage, DockerRegistry from exasol.ds.sandbox.lib.logging import SUPPORTED_LOG_LEVELS from exasol.ds.sandbox.lib.logging import set_log_level +USER_ENV = "DOCKER_REGISTRY_USER" +PASSWORD_ENV = "DOCKER_REGISTRY_PASSWORD" + + @cli.command() @add_options([ click.option( - '--repository', type=str, metavar="ORG/REPO", show_default=True, + "--repository", type=str, metavar="[HOST[:PORT]/]ORG/REPO", + show_default=True, default="exasol/data-science-sandbox", - help="Organization and repository on hub.docker.com to publish the docker image to"), - click.option('--version', type=str, help="Docker image version tag"), + help=""" + Organization and repository on hub.docker.com to publish the + docker image to. Optionally prefixed by a host and a port, + see https://docs.docker.com/engine/reference/commandline/tag. + """), + click.option( + "--version", type=str, metavar="VERSION", + help="Docker image version tag"), click.option( - '--publish', type=bool, is_flag=True, + "--publish", type=bool, is_flag=True, help="Whether to publish the created Docker image"), click.option( - '--keep-container', type=bool, is_flag=True, + "--registry-user", type=str, metavar="USER", + default=lambda: os.environ.get(USER_ENV, None), + help=f""" + Username for Docker registry [defaults to environment + variable {USER_ENV}]. If specified then password is read + from environment variable {PASSWORD_ENV}. + """ + ), + click.option( + "--keep-container", type=bool, is_flag=True, help="""Keep the Docker Container running after creating the image. Otherwise stop and remove the container."""), ]) @@ -28,17 +49,23 @@ def create_docker_image( repository: str, version: str, publish: bool, + registry_user: str, keep_container: bool, log_level: str, ): """ - Create a Docker image for data-science-sandbox and deploy - it to a Docker repository. + Create a Docker image for data-science-sandbox. If option + ``--publish`` is specified then deploy the image to the Docker registry + using the specified user name and reading the password from environment + variable ``PASSWORD_ENV``. """ + def registry_password(): + if registry_user is None: + return None + return os.environ.get(PASSWORD_ENV, None) + set_log_level(log_level) - DssDockerImage( - repository=repository, - version=version, - publish=publish, - keep_container=keep_container, - ).create() + creator = DssDockerImage(repository, version, keep_container) + if publish: + creator.registry = DockerRegistry(registry_user, registry_password()) + creator.create() diff --git a/exasol/ds/sandbox/lib/dss_docker/__init__.py b/exasol/ds/sandbox/lib/dss_docker/__init__.py index f62d46be..30bd24a4 100644 --- a/exasol/ds/sandbox/lib/dss_docker/__init__.py +++ b/exasol/ds/sandbox/lib/dss_docker/__init__.py @@ -1 +1,2 @@ from .create_image import DssDockerImage +from .push_image import DockerRegistry diff --git a/exasol/ds/sandbox/lib/dss_docker/create_image.py b/exasol/ds/sandbox/lib/dss_docker/create_image.py index fba83070..eb0f5b11 100644 --- a/exasol/ds/sandbox/lib/dss_docker/create_image.py +++ b/exasol/ds/sandbox/lib/dss_docker/create_image.py @@ -2,12 +2,13 @@ import humanfriendly import importlib_resources +from functools import reduce from datetime import datetime from docker.types import Mount from exasol.ds.sandbox.lib import pretty_print from importlib_metadata import version from pathlib import Path -from typing import List +from typing import Dict, List, Optional from docker.models.containers import Container as DockerContainer from docker.models.images import Image as DockerImage @@ -20,19 +21,20 @@ from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo - DSS_VERSION = version("exasol-data-science-sandbox") _logger = get_status_logger(LogType.DOCKER_IMAGE) -def get_fact(facts: AnsibleFacts, *keys: str) -> str: - keys = list(keys) - keys.insert(0, "dss_facts") - for key in keys: - if not key in facts: - return None - facts = facts[key] - return facts +def get_fact(facts: AnsibleFacts, *keys: str) -> Optional[str]: + return get_nested_value(facts, "dss_facts", *keys) + + +def get_nested_value(mapping: Dict[str, any], *keys: str) -> Optional[str]: + def nested_item(current, key): + valid = current is not None and key in current + return current[key] if valid else None + + return reduce(nested_item, keys, mapping) def entrypoint(facts: AnsibleFacts) -> List[str]: @@ -63,15 +65,19 @@ def __init__( self, repository: str, version: str = None, - publish: bool = False, keep_container: bool = False, ): version = version if version else DSS_VERSION self.container_name = f"ds-sandbox-{DssDockerImage.timestamp()}" - self.image_name = f"{repository}:{version}" - self.publish = publish + self.repository = repository + self.version = version self.keep_container = keep_container self._start = None + self.registry = None + + @property + def image_name(self): + return f"{self.repository}:{self.version}" def _ansible_run_context(self) -> AnsibleRunContext: extra_vars = { @@ -150,11 +156,16 @@ def _cleanup(self, container: DockerContainer): _logger.info("Removing container") container.remove() + def _push(self): + if self.registry is not None: + self.registry.push(self.repository, self.version) + def create(self): try: container = self._start_container() facts = self._install_dependencies() image = self._commit_container(container, facts) + self._push() except Exception as ex: raise ex finally: diff --git a/exasol/ds/sandbox/lib/dss_docker/push_image.py b/exasol/ds/sandbox/lib/dss_docker/push_image.py new file mode 100644 index 00000000..5061ed4c --- /dev/null +++ b/exasol/ds/sandbox/lib/dss_docker/push_image.py @@ -0,0 +1,65 @@ +import docker +import json +import logging +import requests + +from docker.client import DockerClient + +from typing import Callable, Optional +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType + + +_logger = get_status_logger(LogType.DOCKER_IMAGE) + + +class ProgressReporter: + def __init__(self, verbose: bool): + self.last_status = None + self.verbose = verbose + self.need_linefeed = False + + def _report(self, printer: Callable, msg: Optional[str], **kwargs): + if msg is not None: + printer(msg, **kwargs) + + def _linefeed(self): + if self.need_linefeed: + self.need_linefeed = False + print() + + def report(self, status: Optional[str], progress: Optional[str]): + if not self.verbose: + return + if status == self.last_status: + self._report(print, progress, end="\r") + self.need_linefeed = progress + else: + self.last_status = status + self._linefeed() + self._report(_logger.info, status) + + +class DockerRegistry: + def __init__(self, username: str, password: str): + self.username = username + self.password = password + + def push(self, repository: str, tag: str): + auth_config = { + "username": self.username, + "password": self.password, + } + client = docker.from_env() + resp = client.images.push( + repository=repository, + tag=tag, + auth_config=auth_config, + stream=True, + decode=True, + ) + reporter = ProgressReporter(_logger.isEnabledFor(logging.INFO)) + for el in resp: + reporter.report( + el.get("status", None), + el.get("progress", None), + ) diff --git a/exasol/ds/sandbox/runtime/ansible/roles/poetry/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/poetry/defaults/main.yml index 55cb510d..e60a2225 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/poetry/defaults/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/poetry/defaults/main.yml @@ -1,6 +1,6 @@ --- apt_dependencies: - - curl=7.68.0-1ubuntu2.20 + - curl=7.68.0-1ubuntu2.21 - python3.8-venv=3.8.10-0ubuntu1~20.04.9 - python3-pip=20.0.2-5ubuntu1.10 diff --git a/pyproject.toml b/pyproject.toml index 5dc17a2c..058608ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,3 +61,8 @@ testpaths = [ "test" ] +# ignore warnings caused by docker lib +filterwarnings = [ + "ignore:the imp module is deprecated:DeprecationWarning", + "ignore:distutils Version classes are deprecated:DeprecationWarning", +] diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 628acf87..6fd05472 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -1,6 +1,40 @@ +import docker +import pytest + +from exasol.ds.sandbox.lib.dss_docker import DssDockerImage + def pytest_addoption(parser): parser.addoption( "--dss-docker-image", default=None, help="Name and version of existing Docker image to use for tests", ) + parser.addoption( + "--docker-registry", default=None, metavar="HOST:PORT", + help="Docker registry for pushing Docker images to", + ) + + +@pytest.fixture(scope="session") +def dss_docker_image(request): + """ + If dss_docker_image_name is provided then don't create an image but + reuse the existing image as specified by cli option + --ds-docker-image-name. + """ + existing = request.config.getoption("--dss-docker-image") + if existing and ":" in existing: + name, version = existing.split(":") + yield DssDockerImage(name, version) + return + + testee = DssDockerImage( + "my-repo/dss-test-image", + version=f"{DssDockerImage.timestamp()}", + keep_container=False, + ) + testee.create() + try: + yield testee + finally: + docker.from_env().images.remove(testee.image_name) diff --git a/test/integration/local_docker_registry.py b/test/integration/local_docker_registry.py new file mode 100644 index 00000000..285385be --- /dev/null +++ b/test/integration/local_docker_registry.py @@ -0,0 +1,47 @@ +import docker +import contextlib +import json +import logging +import requests +import time + +from typing import Dict, List +from test.ports import find_free_port +from exasol.ds.sandbox.lib.dss_docker import DssDockerImage, DockerRegistry + + +_logger = logging.getLogger(__name__) +_logger.setLevel(logging.DEBUG) + + +class LocalDockerRegistry(DockerRegistry): + """ + Simulate Docker registry by running a local Docker container and using + the registry inside. + + Please note for pushing images to a Docker registry with host or port + differing from the official address requires to tag images in advance. + + So host and port must be prepended to property ``repository`` of the + image. + """ + def __init__(self, host_and_port: str): + super().__init__(username=None, password=None) + self.host_and_port = host_and_port + + @property + def url(self): + return f'http://{self.host_and_port}' + + def images(self, repo_name: str) -> Dict[str, any]: + url = f"{self.url}/v2/{repo_name}/tags/list" + result = requests.request("GET", url) + images = json.loads(result.content.decode("UTF-8")) + return images + + @property + def repositories(self) -> List[str]: + url = f"{self.url}/v2/_catalog/" + result = requests.request("GET", url) + repos = json.loads(result.content.decode("UTF-8"))["repositories"] + return repos diff --git a/test/integration/test_create_dss_docker_image.py b/test/integration/test_create_dss_docker_image.py index c0f16d04..ee2c60f9 100644 --- a/test/integration/test_create_dss_docker_image.py +++ b/test/integration/test_create_dss_docker_image.py @@ -18,32 +18,6 @@ from exasol.ds.sandbox.lib import pretty_print -@pytest.fixture(scope="session") -def dss_docker_image(request): - """ - If dss_docker_image_name is provided then don't create an image but - reuse the existing image as specified by cli option - --ds-docker-image-name, see file conftest.py. - """ - existing = request.config.getoption("--dss-docker-image") - if existing and ":" in existing: - name, version = existing.split(":") - yield DssDockerImage(name, version) - return - - testee = DssDockerImage( - "my-repo/dss-test-image", - version=f"{DssDockerImage.timestamp()}", - publish=False, - keep_container=False, - ) - testee.create() - try: - yield testee - finally: - docker.from_env().images.remove(testee.image_name) - - @pytest.fixture def dss_docker_container(dss_docker_image): client = docker.from_env() diff --git a/test/integration/test_push_docker_image.py b/test/integration/test_push_docker_image.py new file mode 100644 index 00000000..bd1db5ab --- /dev/null +++ b/test/integration/test_push_docker_image.py @@ -0,0 +1,118 @@ +import contextlib +import docker +import json +import logging +import pytest +import re +import requests +import time + +from exasol.ds.sandbox.lib.dss_docker import DssDockerImage +from test.ports import find_free_port +from dataclasses import dataclass + +from test.integration.local_docker_registry import ( + LocalDockerRegistry, +) + +_logger = logging.getLogger(__name__) +_logger.setLevel(logging.DEBUG) + + +def normalize_request_name(name: str): + name = re.sub(r"[\[\]._]+", "_", name) + return re.sub(r"^_+|_+$", "", name) + + +@dataclass +class DockerImageSpec: + repository: str + tag: str + + @property + def name(self) -> str: + return f"{self.repository}:{self.tag}" + + +@pytest.fixture(scope="session") +def registry_image(): + spec = DockerImageSpec("registry", "2") + client = docker.from_env() + if not client.images.list(spec.name): + _logger.debug("Pulling Docker image with Docker registry") + client.images.pull(spec.repository, spec.tag) + return spec + + +@pytest.fixture(scope="session") +def sample_docker_image(registry_image): + return registry_image + + +@contextlib.contextmanager +def tagged_image(old: DockerImageSpec, new: DockerImageSpec): + """ + Tag the Docker image with spec ``old`` to ``new`` to enable pushing + the image to a local registry. + """ + client = docker.from_env() + _logger.debug(f'tagging image from {old.name} to {new.name}') + image = client.images.get(old.name) + image.tag(new.repository, new.tag) + tagged = DssDockerImage(new.repository, new.tag) + try: + yield tagged + finally: + docker.from_env().images.remove(new.name) + + +@pytest.fixture(scope="session") +def docker_registry(request, registry_image): + """ + Create a LocalDockerRegistry for executing integration tests. You can + provide cli option ``--docker-registry HOST:PORT`` to pytest in order + reuse an already running Docker container as registry, see file + ``conftest.py``. + """ + existing = request.config.getoption("--docker-registry") + if existing is not None: + yield LocalDockerRegistry(existing) + return + + test_name = normalize_request_name(request.node.name) + container_name = f"{test_name}_registry" + + port = find_free_port() + client = docker.from_env() + _logger.debug(f"Starting container {container_name}") + try: + client.containers.get(container_name).remove(force=True) + except: + pass + container = client.containers.run( + image="registry:2", + name=container_name, + ports={5000: port}, + detach=True, + ) + time.sleep(10) + _logger.debug(f"Finished starting container {container_name}") + try: + yield LocalDockerRegistry(f"localhost:{port}") + finally: + _logger.debug("Stopping container") + container.stop() + _logger.debug("Removing container") + container.remove() + + +def test_push_tag(sample_docker_image, docker_registry): + repo = "org/sample_repo" + spec = DockerImageSpec( + docker_registry.host_and_port + "/" + repo, + "999.9.9", + ) + with tagged_image(sample_docker_image, spec) as tagged: + docker_registry.push(tagged.repository, spec.tag) + assert repo in docker_registry.repositories + assert spec.tag in docker_registry.images(repo)["tags"] diff --git a/test/ports.py b/test/ports.py new file mode 100644 index 00000000..26bc33f2 --- /dev/null +++ b/test/ports.py @@ -0,0 +1,30 @@ +import socket +from contextlib import ExitStack +from typing import List + + +# copied from https://github.com/exasol/integration-test-docker-environment +def find_free_ports(num_ports: int) -> List[int]: + def new_socket(): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + def bind(sock: socket.socket, port: int): + sock.bind(('', port)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + def acquire_port_numbers(num_ports: int) -> List[int]: + with ExitStack() as stack: + sockets = [stack.enter_context(new_socket()) for dummy in range(num_ports)] + for sock in sockets: + bind(sock, 0) + yield sock.getsockname()[1] + def check_port_numbers(ports): + with ExitStack() as stack: + sockets_and_ports = [(stack.enter_context(new_socket()), port) for port in ports] + for sock, port in sockets_and_ports: + bind(sock, port) + ports = list(acquire_port_numbers(num_ports)) + check_port_numbers(ports) + return ports + + +def find_free_port(): + return find_free_ports(1)[0] diff --git a/test/unit/test_dss_docker_image.py b/test/unit/test_dss_docker_image.py index 8a346670..9bc5e05b 100644 --- a/test/unit/test_dss_docker_image.py +++ b/test/unit/test_dss_docker_image.py @@ -1,6 +1,8 @@ import pytest -from exasol.ds.sandbox.lib.dss_docker import create_image +from unittest.mock import MagicMock, Mock, create_autospec +from datetime import datetime +from exasol.ds.sandbox.lib.dss_docker import create_image, DockerRegistry from exasol.ds.sandbox.lib.dss_docker.create_image import ( DssDockerImage, DSS_VERSION, @@ -15,7 +17,6 @@ def sample_repo(): def test_constructor_defaults(sample_repo): testee = DssDockerImage(sample_repo) assert testee.image_name == f"{sample_repo}:{DSS_VERSION}" - assert testee.publish == False assert testee.keep_container == False @@ -24,14 +25,28 @@ def test_constructor(sample_repo): testee = DssDockerImage( repository=sample_repo, version=version, - publish=True, keep_container=True, ) assert testee.image_name == f"{sample_repo}:{version}" - assert testee.publish == True assert testee.keep_container == True +@pytest.mark.parametrize( + "testee", + [ + {}, + {"key": "sample value"}, + ]) +def test_nested_value_missing_entry(testee): + assert create_image.get_nested_value(testee, "missing_entry") == None + assert create_image.get_nested_value(testee, "key", "key-2") == None + + +def test_nested_value_level_2(): + testee = {"key": {"key-2": "value"}} + assert create_image.get_nested_value(testee, "key", "key-2") == "value" + + @pytest.mark.parametrize( "facts", [ @@ -83,3 +98,23 @@ def test_entrypoint_with_copy_args(): "--notebooks", final, "--jupyter-server", jupyter, ] + +@pytest.fixture +def mocked_docker_image(): + testee = DssDockerImage("org/sample_repo", "version") + testee._start = datetime.now() + testee._start_container = MagicMock() + testee._install_dependencies = MagicMock() + testee._cleanup = MagicMock() + image = Mock(attrs={"Size": 1025}) + testee._commit_container = MagicMock(return_value=image) + return testee + + +def test_push_called(mocker, mocked_docker_image): + testee = mocked_docker_image + testee.registry = create_autospec(DockerRegistry) + testee.create() + assert testee.registry.push.called + expected = mocker.call(testee.repository, testee.version) + assert testee.registry.push.call_args == expected