Added initial version of cli.

.github/workflows/check_ci.yaml

@@ -18,8 +18,11 @@ jobs:
        uses: ./.github/actions/prepare_poetry_env
 
      - name: Run pytest
-       run: poetry run pytest test/ci/test_install_dependencies.py
+       run: >
+         poetry run pytest
+         test/integration/test_create_dss_docker_image.py
+         test/unit
        env: # Set the secret as an env variable
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
-          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
+         AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}

doc/changes/changes_0.1.0.md

@@ -14,6 +14,7 @@ Version: 0.1.0
 
  - #11: Created a notebook to show training with scikit-learn in the notebook
  - #15: Installed exasol-notebook-connector via ansible
+ - #30: Added script to build the Data Science Sandbox as Docker Image
 
 ## Bug Fixes
 

doc/developer_guide/developer_guide.md

@@ -39,11 +39,11 @@ A CLI command has normally a respective function in the `lib` submodule. Hence,
 
 There are generally three types of commands:
 
-| Type | Explanation |
-| ----- | --------- |
-| Release Commands | used during the release |
-| Deployment Commands | used to deploy infrastructure onto AWS cloud |
-| Development Commands | used to identify problems or for testing |
+| Type                 | Explanation                                  |
+|----------------------|----------------------------------------------|
+| Release Commands     | used during the release                      |
+| Deployment Commands  | used to deploy infrastructure onto AWS cloud |
+| Development Commands | used to identify problems or for testing     |
 
 ### Release commands
 
@@ -71,6 +71,7 @@ The following commands can be used to deploy the infrastructure onto a given AWS
 - `setup-vm-bucket` - deploys the AWS Bucket cloudformation stack which will be used to deploy the VM images
 - `setup-release-codebuild` - deploys the AWS Codebuild cloudformation stack which will be used for the release-build
 - `setup-vm-bucket-waf` - deploys the AWS Codebuild cloudformation stack which contains the WAF Acl configuration for the Cloudfront distribution of the VM Bucket
+- `create-dss-docker-image` - creates a Docker image for data-science-sandbox and deploys it to hub.docker.com/exasol/data-science-sandbox
 
 ## Flow
 

exasol/ds/sandbox/cli/commands/__init__.py

@@ -13,3 +13,4 @@
 from .update_release import update_release
 from .make_ami_public import make_ami_public
 from .setup_vm_bucket_waf import setup_vm_bucket_waf
+from .create_dss_docker_image import create_dss_docker_image

exasol/ds/sandbox/cli/commands/create_dss_docker_image.py

@@ -0,0 +1,11 @@
+from exasol.ds.sandbox.cli.cli import cli
+from exasol.ds.sandbox.lib.dss_docker import DssDockerImage
+
+@cli.command()
+def create_dss_docker_image():
+    """
+    Create a Docker image for data-science-sandbox and deploy
+    it to https://hub.docker.com/exasol/data-science-sandbox.
+    """
+    print("Hello this is create_dss_docker_image")
+    # DssDockerImage.for_production().create()

exasol/ds/sandbox/lib/dss_docker/__init__.py

@@ -0,0 +1 @@
+from .create_image import DssDockerImage

exasol/ds/sandbox/lib/dss_docker/create_image.py

@@ -1,6 +1,7 @@
 import docker
 import logging
 from datetime import datetime
+from docker.types import Mount
 from exasol.ds.sandbox.lib import pretty_print
 from importlib_metadata import version
 from pathlib import Path
@@ -15,73 +16,90 @@
 from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess
 from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies
 
-CONTAINER_NAME = "ds-sandbox-docker"
+
 DSS_VERSION = version("exasol-data-science-sandbox")
-DOCKER_IMAGE = f"exasol/ds-sandbox:{DSS_VERSION}"
 
 
-_logger = logging.getLogger(__name__)
-_logger.setLevel(logging.INFO)
-logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s")
+class DssDockerImage:
+    DEFAULT_CONTAINER_NAME = "ds-sandbox-docker"
+    DEFAULT_IMAGE_NAME = f"exasol/data-science-sandbox:{DSS_VERSION}"
+
+    @classmethod
+    def for_production(cls) -> "DssDockerImage":
+        return DssDockerImage(
+            container_name=DssDockerImage.DEFAULT_CONTAINER_NAME,
+            image_name=DssDockerImage.DEFAULT_IMAGE_NAME,
+            log_level=logging.INFO,
+        )
 
+    def __init__(self, container_name: str, image_name: str, log_level: str):
+        self.container_name = container_name
+        self.image_name = image_name
+        self.log_level = log_level
 
-def create_image():
-    def _ansible_run_context() -> AnsibleRunContext:
+    def _ansible_run_context(self) -> AnsibleRunContext:
         extra_vars = {
-            "docker_container": CONTAINER_NAME,
+            "docker_container": self.container_name,
         }
         return AnsibleRunContext(
             playbook="dss_docker_playbook.yml",
             extra_vars=extra_vars,
         )
 
-    def _ansible_config() -> ConfigObject:
+    def _ansible_config(self) -> ConfigObject:
         return ConfigObject(
             time_to_wait_for_polling=0.1,
             slc_version=SLC_VERSION,
         )
 
-    try:
-        start = datetime.now()
-        docker_client = docker.from_env()
-        path = Path(__file__).parent
-        _logger.info(
-            f"Creating docker image {DOCKER_IMAGE}"
-            f" from {path / 'Dockerfile'}"
-        )
-        docker_client.images.build(path=str(path), tag=DOCKER_IMAGE)
-        container = docker_client.containers.create(
-            image=DOCKER_IMAGE,
-            name=CONTAINER_NAME,
-            command="sleep infinity",
-            detach=True,
-        )
-        _logger.info("Starting container")
-        container.start()
-        _logger.info("Installing dependencies")
-        run_install_dependencies(
-            AnsibleAccess(),
-            configuration=_ansible_config(),
-            host_infos=tuple(),
-            ansible_run_context=_ansible_run_context(),
-            ansible_repositories=ansible_repository.default_repositories,
-        )
-        _logger.info("Committing changes to docker container")
-        image = container.commit(
-            repository=DOCKER_IMAGE,
-        )
-    except Exception as ex:
-        raise ex
-    finally:
-        _logger.info("Stopping container")
-        container.stop()
-        _logger.info("Removing container")
-        container.remove()
-    size = pretty_print.size(image.attrs["Size"])
-    elapsed = pretty_print.elapsed(start)
-    _logger.info(f"Built Docker image {DOCKER_IMAGE} size {size} in {elapsed}.")
-    # TODO: Publish image to hub.docker.com
+    def create(self):
+        logger = logging.getLogger(__name__)
+        logger.setLevel(self.log_level)
+        logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s")
+        try:
+            start = datetime.now()
+            docker_client = docker.from_env()
+            path = Path(__file__).parent
+            logger.info(
+                f"Creating docker image {self.image_name}"
+                f" from {path / 'Dockerfile'}"
+            )
+            docker_client.images.build(path=str(path), tag=self.image_name)
+            socket_mount = Mount("/var/run/docker.sock", "/var/run/docker.sock", type="bind")
+            mapped_ports = {'8888/tcp': 8888}
+            container = docker_client.containers.create(
+                image=self.image_name,
+                name=self.container_name,
+                mounts=[socket_mount],
+                command="sleep infinity",
+                detach=True,
+                ports=mapped_ports,
+            )
+            logger.info("Starting container")
+            container.start()
+            logger.info("Installing dependencies")
+            run_install_dependencies(
+                AnsibleAccess(),
+                configuration=self._ansible_config(),
+                host_infos=tuple(),
+                ansible_run_context=self._ansible_run_context(),
+                ansible_repositories=ansible_repository.default_repositories,
+            )
+            logger.info("Committing changes to docker container")
+            image = container.commit(
+                repository=self.image_name,
+            )
+        except Exception as ex:
+            raise ex
+        finally:
+            logger.info("Stopping container")
+            container.stop()
+            logger.info("Removing container")
+            container.remove()
+        size = pretty_print.size(image.attrs["Size"])
+        elapsed = pretty_print.elapsed(start)
+        logger.info(f"Built Docker image {self.image_name} size {size} in {elapsed}.")
 
 
 if __name__ == "__main__":
-    create_image()
+    DssDockerImage.for_production().create()

exasol/ds/sandbox/runtime/ansible/ec2_setup_tasks.yml

@@ -1,6 +1,3 @@
-- name: Install Docker
-  include_role:
-    name: docker
 - name: Install Script_languages
   include_role:
     name: script_languages

exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml

@@ -13,3 +13,6 @@
   ansible.builtin.file:
     path: /root/.cache/pip
     state: absent
+- name: Install Docker
+  include_role:
+    name: docker

exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt

@@ -1,4 +1,4 @@
 uncertainties==3.1.7
 numpy==1.23.1
 pandas==1.4.3
-exasol-notebook-connector==0.1.0
+exasol-notebook-connector==0.2.0

test/integration/test_create_dss_docker_image.py

@@ -0,0 +1,65 @@
+import docker
+import logging
+import pytest
+import requests
+import time
+
+from exasol.ds.sandbox.lib.dss_docker import DssDockerImage
+from datetime import datetime
+
+
+@pytest.fixture(scope="session")
+def dss_docker_container():
+    timestamp = f'{datetime.now().timestamp():.0f}'
+    # testee = DssDockerImage(f"dss_container_{timestamp}", f"dss_image_{timestamp}", logging.INFO)
+    testee = DssDockerImage("ds-sandbox-docker", f"dss_image_{timestamp}", logging.INFO)
+    print(
+        "\n- Using"
+        f' Docker container {testee.container_name}'
+        f' with image {testee.image_name}'
+    )
+    testee.create()
+    client = docker.from_env()
+    mapped_ports = {'8888/tcp': 8888}
+    container = client.containers.create(
+        image=testee.image_name,
+        name=testee.container_name,
+        command="sleep infinity",
+        detach=True,
+        ports=mapped_ports,
+    )
+    container.start()
+    try:
+        yield container
+    finally:
+        pass
+        container.stop()
+        container.remove()
+        client.images.remove(testee.image_name)
+
+
+def test_jupyterlab(dss_docker_container):
+    """"
+    Test that jupyterlab is configured properly
+    """
+    jupyter_command = (
+        "/root/jupyterenv/bin/jupyter-lab"
+        " --notebook-dir=/root/notebooks"
+        " --no-browser"
+        " --allow-root"
+    )
+    container = dss_docker_container
+    container.exec_run(jupyter_command, detach=True)
+    time.sleep(5.0)
+    container.reload()
+    ip_address = container.attrs['NetworkSettings']['IPAddress']
+    http_conn = requests.get(f"http://{ip_address}:8888/lab")
+    assert http_conn.status_code == 200
+
+
+def test_install_notebook_connector(dss_docker_container):
+    container = dss_docker_container
+    command = '/root/jupyterenv/bin/python -c "import exasol.secret_store"'
+    exit_code, output = container.exec_run(command)
+    output = output.decode('utf-8').strip()
+    assert exit_code == 0, f'Got output "{output}".'

test/unit/test_dss_docker_image.py

@@ -0,0 +1,17 @@
+import logging
+
+from exasol.ds.sandbox.lib.dss_docker import DssDockerImage
+
+
+def test_for_production():
+    testee = DssDockerImage.for_production()
+    assert testee.container_name == DssDockerImage.DEFAULT_CONTAINER_NAME
+    assert testee.image_name == DssDockerImage.DEFAULT_IMAGE_NAME
+    assert testee.log_level == logging.INFO
+
+
+def test_constructor():
+    testee = DssDockerImage("cont", "img", logging.ERROR)
+    assert testee.container_name == "cont"
+    assert testee.image_name == "img"
+    assert testee.log_level == logging.ERROR