Refactoring/#258 used a non root user to run jupyter in the docker image ai lab

doc/changes/changes_1.1.0.md

@@ -33,4 +33,4 @@ n/a
 * #217: Changed notebook-connector dependency, now installing it from PyPi.
 * #220: Changed default ports in the external database configuration.
 * #221: Changed wording in the main configuration notebook, as suggested by PM.
-
+* #66: Used a non-root user to run Jupyter in the Docker Image ai-lab

exasol/ds/sandbox/lib/dss_docker/create_image.py

@@ -113,6 +113,10 @@ def _docker_file(self) -> importlib_resources.abc.Traversable:
     def _start_container(self) -> DockerContainer:
         self._start = datetime.now()
         docker_client = docker.from_env()
+        try:
+            return docker_client.containers.get(self.container_name)
+        except:
+            pass
         docker_file = self._docker_file()
         _logger.info(f"Creating docker image {self.image_name} from {docker_file}")
         if self.registry is not None:
@@ -157,6 +161,7 @@ def _commit_container(
         notebook_folder_initial = get_fact(facts, "notebook_folder", "initial")
         conf = {
             "Entrypoint": entrypoint(facts),
+            "User": get_fact(facts, "jupyter", "user"),
             "Cmd": [],
             "Volumes": {notebook_folder_final: {}, },
             "ExposedPorts": {f"{port}/tcp": {}},
@@ -200,3 +205,15 @@ def create(self):
         size = humanfriendly.format_size(image.attrs["Size"])
         elapsed = pretty_print.elapsed(self._start)
         _logger.info(f"Built Docker image {self.image_name} size {size} in {elapsed}.")
+
+
+import sys
+from exasol.ds.sandbox.lib.logging import set_log_level
+if __name__ == "__main__":
+    set_log_level("info")
+    di = DssDockerImage("exasol/ai-lab", "9.9.9", True)
+    if len(sys.argv) > 1:
+        di.container_name = sys.argv[1]
+        di._install_dependencies()
+    else:
+        di.create()

exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml

@@ -4,22 +4,39 @@
   vars:
       ansible_python_interpreter: python3
   tasks:
-    - name: Add docker container to inventory
+    - name: Add Docker Container to Ansible Inventory
       add_host:
         name: "{{docker_container}}"
         groups: docker_container_group
         ansible_connection: docker
 
+- name: Prepare Ansible Access
+  hosts: docker_container_group
+  gather_facts: true
+  vars:
+    my_ansible_user: "ansible"
+    jupyter_user: "jupyter"
+    jupyter_group: "jupyter"
+    jupyter_user_home: "/home/jupyter"
+    need_sudo: yes
+  tasks:
+    - import_tasks: apt_update.yml
+    - name: Ansible Access
+      include_role:
+        name: ansible_access
+
 - name: Setup AI-Lab Docker Container
   hosts: docker_container_group
+  remote_user: "ansible"
   gather_facts: true
   vars:
-      ansible_python_interpreter: python3
-      user_name: "{{ ansible_user_id }}"
-      user_home: "{{ ansible_env.HOME }}"
-      initial_notebook_folder: "{{ user_home }}/notebook-defaults"
-      need_sudo: false
-      docker_integration_test: true
+    ansible_python_interpreter: python3
+    user_name: "jupyter"
+    user_group: "jupyter"
+    user_home: "/home/jupyter"
+    initial_notebook_folder: "{{ user_home }}/notebook-defaults"
+    need_sudo: yes
+    docker_integration_test: true
   tasks:
     - import_tasks: general_setup_tasks.yml
     - import_tasks: cleanup_tasks.yml

exasol/ds/sandbox/runtime/ansible/apt_update.yml

@@ -0,0 +1,4 @@
+- name: Update and upgrade apt packages
+  apt:
+    upgrade: yes
+    update_cache: yes

exasol/ds/sandbox/runtime/ansible/create_jupyter_user.yml

@@ -0,0 +1,4 @@
+- name: Add user 'jupyter' for running jupyter
+  ansible.builtin.user:
+    name: "{{ user_name }}"
+    home: "{{ user_home }}"

exasol/ds/sandbox/runtime/ansible/ec2_playbook.yml

@@ -8,6 +8,7 @@
       need_sudo: yes
   remote_user: ubuntu
   tasks:
+    - import_tasks: apt_update.yml
     - import_tasks: general_setup_tasks.yml
     - import_tasks: ec2_setup_tasks.yml
     - import_tasks: cleanup_tasks.yml

exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml

@@ -15,14 +15,12 @@
         initial: "{{ initial_notebook_folder }}"
         final: "{{user_home}}/notebooks"
     cacheable: yes
-- name: Update and upgrade apt packages
-  apt:
-    upgrade: yes
-    update_cache: yes
-  become: "{{need_sudo}}"
 - name: Install rsync
   include_role:
     name: rsync
+- name: Create Group and User for Running Jupyter Server
+  include_role:
+    name: jupyter_user
 - name: Copy Entry Point Script
   include_role:
     name: entrypoint
@@ -36,7 +34,16 @@
     jupyterlab_password: "{{ dss_facts.jupyter.password }}"
     jupyterlab_notebook_folder_initial: "{{ dss_facts.notebook_folder.initial }}"
     jupyterlab_notebook_folder: "{{ dss_facts.notebook_folder.final }}"
+- name: Change owner of all files and dirs in home directory
+  ansible.builtin.file:
+    path: "{{ user_home }}"
+    owner: "{{ user_name }}"
+    group: "{{ user_group }}"
+    recurse: true
+  become: "{{ need_sudo }}"
 - name: Clear pip cache
+  # TBD: do we need become: "{{ need_sudo }}" here?
+  # should folder /root be replaced by {{ user_home }} ?
   ansible.builtin.file:
     path: /root/.cache/pip
     state: absent

exasol/ds/sandbox/runtime/ansible/roles/ansible_access/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+
+apt_dependencies:
+  - sudo=1.8.31-1ubuntu1.5

exasol/ds/sandbox/runtime/ansible/roles/ansible_access/tasks/main.yml

@@ -0,0 +1,16 @@
+- name: Install sudo
+  apt:
+    name: "{{apt_dependencies}}"
+    state: latest
+    install_recommends: false
+
+- name: Add dedicated user for executing ansible tasks
+  ansible.builtin.user:
+    name: "{{ my_ansible_user }}"
+- name: Add to sudoers
+  community.general.sudoers:
+    name: Add user ansible
+    user: "{{ my_ansible_user }}"
+    state: present
+    nopassword: true
+    commands: ALL

exasol/ds/sandbox/runtime/ansible/roles/coredumps/tasks/main.yml

@@ -4,3 +4,4 @@
     regexp: ^[^#].*core
     line: '*        hard       core      0'
     create: true
+  become: "{{ need_sudo }}"

exasol/ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml

@@ -2,5 +2,6 @@
 
 apt_dependencies:
   - apt-transport-https=2.0.10
+  - gpg-agent=2.2.19-3ubuntu2.2
   - ca-certificates=20230311ubuntu0.20.04.1
   - software-properties-common=0.99.9.12

exasol/ds/sandbox/runtime/ansible/roles/docker/tasks/main.yml

@@ -3,6 +3,7 @@
     name: "{{apt_dependencies}}"
     state: latest
     update_cache: true
+    install_recommends: false
   become: "{{need_sudo}}"
 
 - name: Add Docker GPG apt Key
@@ -22,11 +23,12 @@
     name: docker-ce
     state: latest
     update_cache: true
+    install_recommends: false
   become: "{{need_sudo}}"
 
 - name: Adding docker users (for use without sudo)
   user:
     name: "{{user_name}}"
     append: yes
     groups: docker
-  become: "{{need_sudo}}"
+  become: "{{need_sudo}}"

exasol/ds/sandbox/runtime/ansible/roles/entrypoint/tasks/main.yml

@@ -3,3 +3,6 @@
     src: "entrypoint.py"
     dest: "{{ dss_facts.entrypoint }}"
     mode: 0644
+    group: "{{ user_group }}"
+    owner: "{{ user_name }}"
+  become: "{{need_sudo}}"

exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml

@@ -1,7 +1,11 @@
 ---
 
 jupyterlab_ip: '*'
+# Setting environment variable $HOME to {{user_home}} forces the directory for
+# creating the config file.  Unfortunately this does not affect the json file.
 jupyterlab_config: "{{user_home}}/.jupyter/jupyter_lab_config.py"
+# jupyterlab_config_json: "{{ansible_env.HOME}}/.jupyter/jupyter_server_config.json"
+jupyterlab_config_json: "{{user_home}}/.jupyter/jupyter_server_config.json"
 jupyterlab_default_url: "/lab/tree/start.ipynb"
 
 # overridden in include_role

exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt → exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt

@@ -3,7 +3,7 @@ numpy==1.23.1
 pandas==1.4.3
 scikit-learn==1.0.2
 matplotlib==3.7.4
-jupysql==0.10.7
+jupysql==0.10.10
 sqlalchemy_exasol==4.6.3
 stopwatch.py==2.0.1
 boto3==1.26.163

exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml

@@ -5,12 +5,20 @@
   ansible.builtin.file:
     path: "{{ jupyterlab_config }}"
     state: absent
+  become: "{{ need_sudo }}"
 
-# this generates /root/.jupyter/jupyter_lab_config.py (it has all rows commented out)
-# this config file is very large (>30K)
+# it generates file ~/.jupyter/jupyter_lab_config.py
+# - the config file is very large (>30K)
+# - it has all rows commented out
+#
+# Set environment variable $HOME to {{user_home}} to force the directory for
+# creating the config file.  Unfortunately this does not affect the file
+# jupyter_server_config.json.
 - name: Generate JupyterLab config
   ansible.builtin.command: "{{jupyterlab_virtualenv}}/bin/jupyter lab --generate-config"
-  # this command automatically outputs to ~/.jupyter/
+  environment:
+    HOME: "{{ user_home }}"
+  become: "{{ need_sudo }}"
 
 - name: Edit JupyterLab config with our settings
   ansible.builtin.lineinfile:
@@ -26,16 +34,21 @@
       line: "c.LabApp.default_url = '{{ jupyterlab_default_url }}'"
     - regexp: "c.ServerApp.port"
       line: "c.ServerApp.port = {{ jupyterlab_port }}"
+  become: "{{ need_sudo }}"
 
 - name: Generate a new password
   expect:
     command: "{{jupyterlab_virtualenv}}/bin/jupyter server password"
     responses:
       Enter password: "{{jupyterlab_password}}"
       Verify password: "{{jupyterlab_password}}"
+  environment:
+    HOME: "{{ user_home }}"
+  become: "{{ need_sudo }}"
 
 - name: Create the notebook directory
   ansible.builtin.file:
     path: "{{ dss_facts.notebook_folder.initial }}"
     state: directory
     mode: '0755'
+  become: "{{ need_sudo }}"

exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml

@@ -7,33 +7,42 @@
       apt:
         name: "{{apt_dependencies}}"
         state: present
+        install_recommends: false
       become: "{{need_sudo}}"
 
     - name: Update pip version
       include_tasks:
         file: update-pip.yml
 
-    - name: Install JupyterLab and its dependencies
+    - name: Install JupyterLab and required packages
       include_tasks:
-        file: install-pip-packages.yml
+        file: pip-install.yml
       vars:
-        - pip_requirements_file: "requirements_jupyter.txt"
+        requirements_file: "jupyter_requirements.txt"
 
     - name: Setup Jupyterlab
       ansible.builtin.import_tasks: jupyterlab.yml
 
-    - name: Install dependencies used in Jupyterlab
+    - name: Install packages to be used inside Jupyter notebooks
       include_tasks:
-        file: install-pip-packages.yml
+        file: pip-install.yml
       vars:
-        - pip_requirements_file: "requirements_dependencies.txt"
+        requirements_file: "notebook_requirements.txt"
 
     - name: Setup Systemd service
       ansible.builtin.import_tasks: systemd.yml
 
     - name: Copy tutorial notebook
       ansible.builtin.import_tasks: tutorial.yml
 
+    - name: Change owner of all files and dirs in home directory
+      ansible.builtin.file:
+        path: "{{ user_home }}"
+        owner: "{{ user_name }}"
+        group: "{{ user_group }}"
+        recurse: true
+      become: "{{ need_sudo }}"
+
     - name: Setup motd
       ansible.builtin.import_tasks: motd.yml
 

exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/motd.yml

@@ -1,8 +1,9 @@
 ---
 
 - name: read current server config file
-  shell: "cat {{user_home}}/.jupyter/jupyter_server_config.json"
+  shell: "cat {{jupyterlab_config_json}}"
   register: jupyter_server_config
+  become: "{{ need_sudo }}"
 
 - name: save the Json data to a Variable as a Fact
   set_fact:
@@ -25,5 +26,4 @@
   with_items:
     - 'etc/update-motd.d/999-jupyter'
   vars:
-    jupyter_server_config_file: "{{user_home}}/.jupyter/jupyter_server_config.json"
     heading_jupyter_update_password: "{{lookup('ansible.builtin.file', 'heading_jupyter_update_password.txt') }}"

exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/pip-install.yml

@@ -0,0 +1,21 @@
+---
+
+- name: Copy requirements file {{ requirements_file }}
+  ansible.builtin.copy:
+    src: "{{requirements_file}}"
+    dest: "/tmp/{{requirements_file}}"
+    mode: 0644
+  register: copy_req_file
+
+- name: Call pip install {{ requirements_file }}
+  ansible.builtin.pip:
+    requirements: "{{ copy_req_file.dest }}"
+    state: latest
+    virtualenv: "{{jupyterlab_virtualenv}}"
+    virtualenv_python: python3.8
+  become: "{{need_sudo}}"
+
+- name: Remove requirements file {{ requirements_file }}
+  ansible.builtin.file:
+    path: "{{ copy_req_file.dest }}"
+    state: absent

exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml

@@ -4,3 +4,18 @@
   ansible.builtin.synchronize:
     src: "notebook/"
     dest: "{{ jupyterlab_notebook_folder_initial }}"
+  become: "{{ need_sudo }}"
+
+# 1. create_image.py adds this directory as volume mount point to the Docker
+# image which makes root the owner. 2. entrypoint.py running as separate user
+# fails to copy the notebook files to it.  3. Mitigation: Ansible creates the
+# directory in advance with appropriate owner.
+
+- name: Create directory for final notebooks
+  ansible.builtin.file:
+    path: "{{ jupyterlab_notebook_folder }}"
+    state: directory
+    owner: "{{ user_name }}"
+    group: "{{ user_group }}"
+    mode: '0755'
+  become: "{{ need_sudo }}"