From 7bacff937171a2aa56078e033f69a6e6bbda59be Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 23 Feb 2024 13:56:20 +0100 Subject: [PATCH 01/37] Use separate user for running Jupyter --- doc/changes/changes_1.1.0.md | 2 +- exasol/ds/sandbox/lib/dss_docker/create_image.py | 1 + .../ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml | 7 +++++-- exasol/ds/sandbox/runtime/ansible/create_jupyter_user.yml | 4 ++++ .../runtime/ansible/roles/jupyter/tasks/tutorial.yml | 7 +++++++ 5 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 exasol/ds/sandbox/runtime/ansible/create_jupyter_user.yml diff --git a/doc/changes/changes_1.1.0.md b/doc/changes/changes_1.1.0.md index 813ef8d3..0846601a 100644 --- a/doc/changes/changes_1.1.0.md +++ b/doc/changes/changes_1.1.0.md @@ -33,4 +33,4 @@ n/a * #217: Changed notebook-connector dependency, now installing it from PyPi. * #220: Changed default ports in the external database configuration. * #221: Changed wording in the main configuration notebook, as suggested by PM. - +* #258: Used a non-root user to run Jupyter in the Docker Image ai-lab diff --git a/exasol/ds/sandbox/lib/dss_docker/create_image.py b/exasol/ds/sandbox/lib/dss_docker/create_image.py index 951f0e79..51865049 100644 --- a/exasol/ds/sandbox/lib/dss_docker/create_image.py +++ b/exasol/ds/sandbox/lib/dss_docker/create_image.py @@ -157,6 +157,7 @@ def _commit_container( notebook_folder_initial = get_fact(facts, "notebook_folder", "initial") conf = { "Entrypoint": entrypoint(facts), + "User": get_fact(facts, "jupyter", "user"), "Cmd": [], "Volumes": {notebook_folder_final: {}, }, "ExposedPorts": {f"{port}/tcp": {}}, diff --git a/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml b/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml index b1f82d09..8f1a2ac9 100644 --- a/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml +++ b/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml @@ -15,11 +15,14 @@ gather_facts: true vars: ansible_python_interpreter: python3 - user_name: "{{ ansible_user_id }}" - user_home: "{{ ansible_env.HOME }}" + # user_name: "{{ ansible_user_id }}" + # user_home: "{{ ansible_env.HOME }}" + user_name: "jupyter" + user_home: "/home/jupyter" initial_notebook_folder: "{{ user_home }}/notebook-defaults" need_sudo: false docker_integration_test: true tasks: + - import_tasks: create_jupyter_user.yml - import_tasks: general_setup_tasks.yml - import_tasks: cleanup_tasks.yml diff --git a/exasol/ds/sandbox/runtime/ansible/create_jupyter_user.yml b/exasol/ds/sandbox/runtime/ansible/create_jupyter_user.yml new file mode 100644 index 00000000..6a1b0ab9 --- /dev/null +++ b/exasol/ds/sandbox/runtime/ansible/create_jupyter_user.yml @@ -0,0 +1,4 @@ +- name: Add user 'jupyter' for running jupyter + ansible.builtin.user: + name: "{{ user_name }}" + home: "{{ user_home }}" diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml index e2095e6a..7b931975 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml @@ -4,3 +4,10 @@ ansible.builtin.synchronize: src: "notebook/" dest: "{{ jupyterlab_notebook_folder_initial }}" + +- name: Create directory for final notebooks + ansible.builtin.file: + path: "{{ jupyterlab_notebook_folder }}" + state: directory + owner: "{{ user_name }}" + mode: '0755' From 66635623e5ccf5a1d07505af3d3c5f43fa18e203 Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 26 Feb 2024 11:00:30 +0100 Subject: [PATCH 02/37] Used a non-root user to run Jupyter in the Docker Image ai-lab Additionally Same as AMI and VM Editions also the Docker Edition uses a non-root user for executing the ansible installation. --- .../ansible/ai_lab_docker_playbook.yml | 32 +++++++++++++------ .../ds/sandbox/runtime/ansible/apt_update.yml | 4 +++ .../sandbox/runtime/ansible/ec2_playbook.yml | 1 + .../runtime/ansible/general_setup_tasks.yml | 7 ++-- .../roles/ansible_access/defaults/main.yml | 4 +++ .../roles/ansible_access/tasks/main.yml | 28 ++++++++++++++++ .../ansible/roles/docker/tasks/main.yml | 4 ++- .../ansible/roles/entrypoint/tasks/main.yml | 3 ++ .../ansible/roles/jupyter/tasks/main.yml | 1 + .../ansible/roles/jupyter/tasks/tutorial.yml | 10 ++++++ .../ansible/roles/netplan/tasks/main.yml | 3 +- .../ansible/roles/rsync/tasks/main.yml | 1 + 12 files changed, 82 insertions(+), 16 deletions(-) create mode 100644 exasol/ds/sandbox/runtime/ansible/apt_update.yml create mode 100644 exasol/ds/sandbox/runtime/ansible/roles/ansible_access/defaults/main.yml create mode 100644 exasol/ds/sandbox/runtime/ansible/roles/ansible_access/tasks/main.yml diff --git a/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml b/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml index 8f1a2ac9..dbb031a3 100644 --- a/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml +++ b/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml @@ -10,19 +10,33 @@ groups: docker_container_group ansible_connection: docker +- name: Prepare Ansible Access + hosts: docker_container_group + gather_facts: true + vars: + my_ansible_user: "ansible" + jupyter_user: "jupyter" + jupyter_group: "jupyter" + jupyter_user_home: "/home/jupyter" + need_sudo: yes + tasks: + - import_tasks: apt_update.yml + - name: Ansible Access + include_role: + name: ansible_access + - name: Setup AI-Lab Docker Container hosts: docker_container_group + remote_user: "ansible" gather_facts: true vars: - ansible_python_interpreter: python3 - # user_name: "{{ ansible_user_id }}" - # user_home: "{{ ansible_env.HOME }}" - user_name: "jupyter" - user_home: "/home/jupyter" - initial_notebook_folder: "{{ user_home }}/notebook-defaults" - need_sudo: false - docker_integration_test: true + ansible_python_interpreter: python3 + user_name: "jupyter" + user_group: "jupyter" + user_home: "/home/jupyter" + initial_notebook_folder: "{{ user_home }}/notebook-defaults" + need_sudo: yes + docker_integration_test: true tasks: - - import_tasks: create_jupyter_user.yml - import_tasks: general_setup_tasks.yml - import_tasks: cleanup_tasks.yml diff --git a/exasol/ds/sandbox/runtime/ansible/apt_update.yml b/exasol/ds/sandbox/runtime/ansible/apt_update.yml new file mode 100644 index 00000000..292bce0e --- /dev/null +++ b/exasol/ds/sandbox/runtime/ansible/apt_update.yml @@ -0,0 +1,4 @@ +- name: Update and upgrade apt packages + apt: + upgrade: yes + update_cache: yes diff --git a/exasol/ds/sandbox/runtime/ansible/ec2_playbook.yml b/exasol/ds/sandbox/runtime/ansible/ec2_playbook.yml index 7414bf40..087533a4 100644 --- a/exasol/ds/sandbox/runtime/ansible/ec2_playbook.yml +++ b/exasol/ds/sandbox/runtime/ansible/ec2_playbook.yml @@ -8,6 +8,7 @@ need_sudo: yes remote_user: ubuntu tasks: + - import_tasks: apt_update.yml - import_tasks: general_setup_tasks.yml - import_tasks: ec2_setup_tasks.yml - import_tasks: cleanup_tasks.yml diff --git a/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml b/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml index 3bc070d5..48f1e6a6 100644 --- a/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml +++ b/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml @@ -15,11 +15,6 @@ initial: "{{ initial_notebook_folder }}" final: "{{user_home}}/notebooks" cacheable: yes -- name: Update and upgrade apt packages - apt: - upgrade: yes - update_cache: yes - become: "{{need_sudo}}" - name: Install rsync include_role: name: rsync @@ -37,6 +32,8 @@ jupyterlab_notebook_folder_initial: "{{ dss_facts.notebook_folder.initial }}" jupyterlab_notebook_folder: "{{ dss_facts.notebook_folder.final }}" - name: Clear pip cache + # TBD: do we need become: "{{ need_sudo }}" here? + # should folder /root be replaced by {{ my_ansible_user_home }} ? ansible.builtin.file: path: /root/.cache/pip state: absent diff --git a/exasol/ds/sandbox/runtime/ansible/roles/ansible_access/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/ansible_access/defaults/main.yml new file mode 100644 index 00000000..aa8d10cf --- /dev/null +++ b/exasol/ds/sandbox/runtime/ansible/roles/ansible_access/defaults/main.yml @@ -0,0 +1,4 @@ +--- + +apt_dependencies: + - sudo=1.8.31-1ubuntu1.5 diff --git a/exasol/ds/sandbox/runtime/ansible/roles/ansible_access/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/ansible_access/tasks/main.yml new file mode 100644 index 00000000..58a8da21 --- /dev/null +++ b/exasol/ds/sandbox/runtime/ansible/roles/ansible_access/tasks/main.yml @@ -0,0 +1,28 @@ +- name: Install sudo + apt: + name: "{{apt_dependencies}}" + state: latest + install_recommends: false + +- name: Add to sudoers + community.general.sudoers: + name: Add-Ansible + user: "{{ my_ansible_user }}" + state: present + nopassword: true + commands: ALL + +- name: Create group for command su and ansible_common_remote_group + ansible.builtin.group: + name: jupyter + state: present + +- name: Add dedicated user for executing ansible tasks + ansible.builtin.user: + name: "{{ my_ansible_user }}" + group: jupyter +- name: Add user for running jupyter + ansible.builtin.user: + name: "{{ jupyter_user }}" + home: "{{ jupyter_user_home }}" + group: "{{ jupyter_group }}" diff --git a/exasol/ds/sandbox/runtime/ansible/roles/docker/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/docker/tasks/main.yml index c79bc4af..3975e411 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/docker/tasks/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/docker/tasks/main.yml @@ -3,6 +3,7 @@ name: "{{apt_dependencies}}" state: latest update_cache: true + install_recommends: false become: "{{need_sudo}}" - name: Add Docker GPG apt Key @@ -22,6 +23,7 @@ name: docker-ce state: latest update_cache: true + install_recommends: false become: "{{need_sudo}}" - name: Adding docker users (for use without sudo) @@ -29,4 +31,4 @@ name: "{{user_name}}" append: yes groups: docker - become: "{{need_sudo}}" \ No newline at end of file + become: "{{need_sudo}}" diff --git a/exasol/ds/sandbox/runtime/ansible/roles/entrypoint/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/entrypoint/tasks/main.yml index 1fd43c28..1e955355 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/entrypoint/tasks/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/entrypoint/tasks/main.yml @@ -3,3 +3,6 @@ src: "entrypoint.py" dest: "{{ dss_facts.entrypoint }}" mode: 0644 + group: "{{ user_group }}" + owner: "{{ user_name }}" + become: "{{need_sudo}}" diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml index 81680907..1f2fd07b 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml @@ -7,6 +7,7 @@ apt: name: "{{apt_dependencies}}" state: present + install_recommends: false become: "{{need_sudo}}" - name: Update pip version diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml index 7b931975..260710af 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml @@ -4,10 +4,20 @@ ansible.builtin.synchronize: src: "notebook/" dest: "{{ jupyterlab_notebook_folder_initial }}" + group: "{{ user_group }}" + owner: "{{ user_name }}" + become: "{{ need_sudo }}" + +# 1. create_image.py adds this directory as volume mount point to the Docker +# image which makes root the owner. 2. entrypoint.py running as separate user +# fails to copy the notebook files to it. 3. Mitigation: Ansible creates the +# directory in advance with appropriate owner. - name: Create directory for final notebooks ansible.builtin.file: path: "{{ jupyterlab_notebook_folder }}" state: directory owner: "{{ user_name }}" + group: "{{ user_group }}" mode: '0755' + become: "{{ need_sudo }}" diff --git a/exasol/ds/sandbox/runtime/ansible/roles/netplan/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/netplan/tasks/main.yml index 85e0d591..fcfc03a1 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/netplan/tasks/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/netplan/tasks/main.yml @@ -3,6 +3,7 @@ name: "{{apt_dependencies}}" state: latest update_cache: true + install_recommends: false become: "{{need_sudo}}" - name: Copy 01-network-manager.yaml file @@ -20,4 +21,4 @@ - name: Run netplan apply shell: "netplan apply" - become: "{{need_sudo}}" \ No newline at end of file + become: "{{need_sudo}}" diff --git a/exasol/ds/sandbox/runtime/ansible/roles/rsync/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/rsync/tasks/main.yml index ad88d557..b0e78dc6 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/rsync/tasks/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/rsync/tasks/main.yml @@ -3,4 +3,5 @@ name: "{{apt_dependencies}}" state: latest update_cache: true + install_recommends: false become: "{{need_sudo}}" From 194e9d1825695a33bffec87d9e1e303439b04974 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 09:48:33 +0100 Subject: [PATCH 03/37] Additional changes based on experiments --- .../ds/sandbox/lib/dss_docker/create_image.py | 16 +++++++++++++++ .../runtime/ansible/general_setup_tasks.yml | 12 ++++++++++- .../roles/ansible_access/tasks/main.yml | 20 ++++--------------- .../ansible/roles/jupyter/defaults/main.yml | 4 ++++ .../jupyter/files/requirements_jupyter.txt | 3 ++- .../jupyter/tasks/install-pip-packages.yml | 1 + .../roles/jupyter/tasks/jupyterlab.yml | 18 ++++++++++++++--- .../ansible/roles/jupyter/tasks/main.yml | 8 ++++++++ .../ansible/roles/jupyter/tasks/motd.yml | 4 ++-- .../ansible/roles/jupyter/tasks/tutorial.yml | 2 -- .../roles/jupyter/tasks/update-pip.yml | 1 + .../templates/etc/update-motd.d/999-jupyter | 2 +- .../roles/jupyter_user/defaults/main.yml | 1 + .../ansible/roles/jupyter_user/tasks/main.yml | 11 ++++++++++ 14 files changed, 77 insertions(+), 26 deletions(-) create mode 100644 exasol/ds/sandbox/runtime/ansible/roles/jupyter_user/defaults/main.yml create mode 100644 exasol/ds/sandbox/runtime/ansible/roles/jupyter_user/tasks/main.yml diff --git a/exasol/ds/sandbox/lib/dss_docker/create_image.py b/exasol/ds/sandbox/lib/dss_docker/create_image.py index 51865049..933f870d 100644 --- a/exasol/ds/sandbox/lib/dss_docker/create_image.py +++ b/exasol/ds/sandbox/lib/dss_docker/create_image.py @@ -113,6 +113,10 @@ def _docker_file(self) -> importlib_resources.abc.Traversable: def _start_container(self) -> DockerContainer: self._start = datetime.now() docker_client = docker.from_env() + try: + return docker_client.containers.get(self.container_name) + except: + pass docker_file = self._docker_file() _logger.info(f"Creating docker image {self.image_name} from {docker_file}") if self.registry is not None: @@ -201,3 +205,15 @@ def create(self): size = humanfriendly.format_size(image.attrs["Size"]) elapsed = pretty_print.elapsed(self._start) _logger.info(f"Built Docker image {self.image_name} size {size} in {elapsed}.") + + +import sys +from exasol.ds.sandbox.lib.logging import set_log_level +if __name__ == "__main__": + set_log_level("info") + di = DssDockerImage("exasol/ai-lab", "9.9.9", True) + if len(sys.argv) > 1: + di.container_name = sys.argv[1] + di._install_dependencies() + else: + di.create() diff --git a/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml b/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml index 48f1e6a6..a0f7a101 100644 --- a/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml +++ b/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml @@ -18,6 +18,9 @@ - name: Install rsync include_role: name: rsync +- name: Create Group and User for Running Jupyter Server + include_role: + name: jupyter_user - name: Copy Entry Point Script include_role: name: entrypoint @@ -31,9 +34,16 @@ jupyterlab_password: "{{ dss_facts.jupyter.password }}" jupyterlab_notebook_folder_initial: "{{ dss_facts.notebook_folder.initial }}" jupyterlab_notebook_folder: "{{ dss_facts.notebook_folder.final }}" +- name: Change owner of all files and dirs in home directory + ansible.builtin.file: + path: "{{ user_home }}" + owner: "{{ user_name }}" + group: "{{ user_group }}" + recurse: true + become: "{{ need_sudo }}" - name: Clear pip cache # TBD: do we need become: "{{ need_sudo }}" here? - # should folder /root be replaced by {{ my_ansible_user_home }} ? + # should folder /root be replaced by {{ user_home }} ? ansible.builtin.file: path: /root/.cache/pip state: absent diff --git a/exasol/ds/sandbox/runtime/ansible/roles/ansible_access/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/ansible_access/tasks/main.yml index 58a8da21..19b1890b 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/ansible_access/tasks/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/ansible_access/tasks/main.yml @@ -4,25 +4,13 @@ state: latest install_recommends: false +- name: Add dedicated user for executing ansible tasks + ansible.builtin.user: + name: "{{ my_ansible_user }}" - name: Add to sudoers community.general.sudoers: - name: Add-Ansible + name: Add user ansible user: "{{ my_ansible_user }}" state: present nopassword: true commands: ALL - -- name: Create group for command su and ansible_common_remote_group - ansible.builtin.group: - name: jupyter - state: present - -- name: Add dedicated user for executing ansible tasks - ansible.builtin.user: - name: "{{ my_ansible_user }}" - group: jupyter -- name: Add user for running jupyter - ansible.builtin.user: - name: "{{ jupyter_user }}" - home: "{{ jupyter_user_home }}" - group: "{{ jupyter_group }}" diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml index 3f96602a..dd6af94f 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml @@ -1,7 +1,11 @@ --- jupyterlab_ip: '*' +# Setting environment variable $HOME to {{user_home}} forces the directory for +# creating the config file. Unfortunately this does not affect the json file. jupyterlab_config: "{{user_home}}/.jupyter/jupyter_lab_config.py" +# jupyterlab_config_json: "{{ansible_env.HOME}}/.jupyter/jupyter_server_config.json" +jupyterlab_config_json: "{{user_home}}/.jupyter/jupyter_server_config.json" jupyterlab_default_url: "/lab/tree/start.ipynb" # overridden in include_role diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt index df24fc7b..06d107cd 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt @@ -1,4 +1,5 @@ -jupyterlab==4.1.1 +jupyterlab==4.0.6 +# jupyterlab==4.1.1 # enable interactive Javascript widgets in the notebooks ipywidgets==8.1.1 pexpect==4.8.0 diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml index bd6e65ca..0ffae114 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml @@ -13,6 +13,7 @@ state: latest virtualenv: "{{jupyterlab_virtualenv}}" virtualenv_python: python3.8 + become: "{{need_sudo}}" - name: Remove the requirements file (delete file) ansible.builtin.file: diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml index e42a7e27..08029c42 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml @@ -5,12 +5,20 @@ ansible.builtin.file: path: "{{ jupyterlab_config }}" state: absent + become: "{{ need_sudo }}" -# this generates /root/.jupyter/jupyter_lab_config.py (it has all rows commented out) -# this config file is very large (>30K) +# it generates file ~/.jupyter/jupyter_lab_config.py +# - the config file is very large (>30K) +# - it has all rows commented out +# +# Set environment variable $HOME to {{user_home}} to force the directory for +# creating the config file. Unfortunately this does not affect the file +# jupyter_server_config.json. - name: Generate JupyterLab config ansible.builtin.command: "{{jupyterlab_virtualenv}}/bin/jupyter lab --generate-config" - # this command automatically outputs to ~/.jupyter/ + environment: + HOME: "{{ user_home }}" + become: "{{ need_sudo }}" - name: Edit JupyterLab config with our settings ansible.builtin.lineinfile: @@ -26,6 +34,7 @@ line: "c.LabApp.default_url = '{{ jupyterlab_default_url }}'" - regexp: "c.ServerApp.port" line: "c.ServerApp.port = {{ jupyterlab_port }}" + become: "{{ need_sudo }}" - name: Generate a new password expect: @@ -33,9 +42,12 @@ responses: Enter password: "{{jupyterlab_password}}" Verify password: "{{jupyterlab_password}}" + environment: + HOME: "{{ user_home }}" - name: Create the notebook directory ansible.builtin.file: path: "{{ dss_facts.notebook_folder.initial }}" state: directory mode: '0755' + become: "{{ need_sudo }}" diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml index 1f2fd07b..5aaa7730 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml @@ -35,6 +35,14 @@ - name: Copy tutorial notebook ansible.builtin.import_tasks: tutorial.yml + - name: Change owner of all files and dirs in home directory + ansible.builtin.file: + path: "{{ user_home }}" + owner: "{{ user_name }}" + group: "{{ user_group }}" + recurse: true + become: "{{ need_sudo }}" + - name: Setup motd ansible.builtin.import_tasks: motd.yml diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/motd.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/motd.yml index e2ce0155..ad352efb 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/motd.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/motd.yml @@ -1,8 +1,9 @@ --- - name: read current server config file - shell: "cat {{user_home}}/.jupyter/jupyter_server_config.json" + shell: "cat {{jupyterlab_config_json}}" register: jupyter_server_config + become: "{{ need_sudo }}" - name: save the Json data to a Variable as a Fact set_fact: @@ -25,5 +26,4 @@ with_items: - 'etc/update-motd.d/999-jupyter' vars: - jupyter_server_config_file: "{{user_home}}/.jupyter/jupyter_server_config.json" heading_jupyter_update_password: "{{lookup('ansible.builtin.file', 'heading_jupyter_update_password.txt') }}" diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml index 260710af..7ab9f4cf 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml @@ -4,8 +4,6 @@ ansible.builtin.synchronize: src: "notebook/" dest: "{{ jupyterlab_notebook_folder_initial }}" - group: "{{ user_group }}" - owner: "{{ user_name }}" become: "{{ need_sudo }}" # 1. create_image.py adds this directory as volume mount point to the Docker diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/update-pip.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/update-pip.yml index 13e08607..eca24c2a 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/update-pip.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/update-pip.yml @@ -7,3 +7,4 @@ virtualenv: "{{jupyterlab_virtualenv}}" virtualenv_python: python3.8 extra_args: "--upgrade" + become: "{{need_sudo}}" diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter index 731e687b..40cb01c2 100755 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter @@ -15,7 +15,7 @@ To update the password as user {{user_name}} run {{jupyterlab_virtualenv}}/bin/jupyter server password """ -server_config_file = pathlib.Path("{{jupyter_server_config_file}}") +server_config_file = pathlib.Path("{{jupyterlab_config_json}}") if server_config_file.exists(): with open(server_config_file) as f: diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter_user/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter_user/defaults/main.yml new file mode 100644 index 00000000..ed97d539 --- /dev/null +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter_user/defaults/main.yml @@ -0,0 +1 @@ +--- diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter_user/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter_user/tasks/main.yml new file mode 100644 index 00000000..bfa1f664 --- /dev/null +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter_user/tasks/main.yml @@ -0,0 +1,11 @@ +- name: Create group for jupyter user + ansible.builtin.group: + name: "{{ user_group }}" + state: present + become: "{{ need_sudo }}" +- name: Add user for running jupyter + ansible.builtin.user: + name: "{{ user_name }}" + home: "{{ user_home }}" + group: "{{ user_group }}" + become: "{{ need_sudo }}" From 5befe3e0b2f793b8f7903bdbea88b827b6ff67e3 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 11:27:45 +0100 Subject: [PATCH 04/37] Fixed ansible tasks added dependency to gpg-agent --- exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml | 2 +- .../ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml | 1 + .../ansible/roles/jupyter/files/requirements_dependencies.txt | 2 +- .../ansible/roles/jupyter/files/requirements_jupyter.txt | 4 ++-- .../runtime/ansible/roles/jupyter/tasks/jupyterlab.yml | 1 + 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml b/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml index dbb031a3..12a44219 100644 --- a/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml +++ b/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml @@ -4,7 +4,7 @@ vars: ansible_python_interpreter: python3 tasks: - - name: Add docker container to inventory + - name: Add Docker Container to Ansible Inventory add_host: name: "{{docker_container}}" groups: docker_container_group diff --git a/exasol/ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml index 0386a05c..f78e595e 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml @@ -2,5 +2,6 @@ apt_dependencies: - apt-transport-https=2.0.10 + - gpg-agent=2.2.19-3ubuntu2.2 - ca-certificates=20230311ubuntu0.20.04.1 - software-properties-common=0.99.9.12 diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt index e5fb6c3a..211f857d 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt @@ -3,7 +3,7 @@ numpy==1.23.1 pandas==1.4.3 scikit-learn==1.0.2 matplotlib==3.7.4 -jupysql==0.10.7 +jupysql==0.10.10 sqlalchemy_exasol==4.6.3 stopwatch.py==2.0.1 boto3==1.26.163 diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt index 06d107cd..f80b9317 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt @@ -1,5 +1,5 @@ -jupyterlab==4.0.6 -# jupyterlab==4.1.1 +# jupyterlab==4.0.6 +jupyterlab==4.1.1 # enable interactive Javascript widgets in the notebooks ipywidgets==8.1.1 pexpect==4.8.0 diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml index 08029c42..809d4d78 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml @@ -44,6 +44,7 @@ Verify password: "{{jupyterlab_password}}" environment: HOME: "{{ user_home }}" + become: "{{ need_sudo }}" - name: Create the notebook directory ansible.builtin.file: From 1e510fc2bdc63bf054248e7c54c463d2c3f273c1 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 11:28:19 +0100 Subject: [PATCH 05/37] remove commented line --- .../runtime/ansible/roles/jupyter/files/requirements_jupyter.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt index f80b9317..df24fc7b 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt @@ -1,4 +1,3 @@ -# jupyterlab==4.0.6 jupyterlab==4.1.1 # enable interactive Javascript widgets in the notebooks ipywidgets==8.1.1 From 69de9d81dad5956a26fc8c829b11db75aabf1d5a Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 12:20:29 +0100 Subject: [PATCH 06/37] Added become for disabling core dumps --- exasol/ds/sandbox/runtime/ansible/roles/coredumps/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/exasol/ds/sandbox/runtime/ansible/roles/coredumps/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/coredumps/tasks/main.yml index 112aa17b..84f0555d 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/coredumps/tasks/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/coredumps/tasks/main.yml @@ -4,3 +4,4 @@ regexp: ^[^#].*core line: '* hard core 0' create: true + become: "{{ need_sudo }}" From d9eed178f9c5a571527346af4ba13f250712923e Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 12:21:02 +0100 Subject: [PATCH 07/37] poetry lock --- poetry.lock | 226 ++++++++++++++++++++++++++-------------------------- 1 file changed, 113 insertions(+), 113 deletions(-) diff --git a/poetry.lock b/poetry.lock index 09ae7b65..87604ebb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -27,13 +27,13 @@ ansible-core = ">=2.16.3,<2.17.0" [[package]] name = "ansible-core" -version = "2.16.3" +version = "2.16.4" description = "Radically simple IT automation" optional = false python-versions = ">=3.10" files = [ - {file = "ansible-core-2.16.3.tar.gz", hash = "sha256:76a8765a8586064ef073a299562e308fa2c180a75b5f7569bbd0f61d4171cdb3"}, - {file = "ansible_core-2.16.3-py3-none-any.whl", hash = "sha256:50c9f33a5b2ee645470a77f4bf99cf35d1ffdefef60388910020b0c58534bec1"}, + {file = "ansible-core-2.16.4.tar.gz", hash = "sha256:2cd208b0915948c88bffad331e5d07097b6edca1872cb53375e51b6719e6a060"}, + {file = "ansible_core-2.16.4-py3-none-any.whl", hash = "sha256:c55d9a5f55651eb6c7f004ca9a9ed854d8cc310e6b438d96cea051cf3d2b2710"}, ] [package.dependencies] @@ -181,17 +181,17 @@ css = ["tinycss2 (>=1.1.0,<1.3)"] [[package]] name = "boto3" -version = "1.34.47" +version = "1.34.50" description = "The AWS SDK for Python" optional = false python-versions = ">= 3.8" files = [ - {file = "boto3-1.34.47-py3-none-any.whl", hash = "sha256:1de708665cbf156e76ca67e376d6cabc5f9a06d7213f925a510cb818a15340a6"}, - {file = "boto3-1.34.47.tar.gz", hash = "sha256:7574afd70c767fdbb19726565a67b47291e1e35ec792c9fbb8ee63cb3f630d45"}, + {file = "boto3-1.34.50-py3-none-any.whl", hash = "sha256:8d709365231234bc4f0ca98fdf33a25eeebf78072853c6aa3d259f0f5cf09877"}, + {file = "boto3-1.34.50.tar.gz", hash = "sha256:290952be7899560039cb0042e8a2354f61a7dead0d0ca8bea6ba901930df0468"}, ] [package.dependencies] -botocore = ">=1.34.47,<1.35.0" +botocore = ">=1.34.50,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -200,13 +200,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.47" +version = "1.34.50" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">= 3.8" files = [ - {file = "botocore-1.34.47-py3-none-any.whl", hash = "sha256:8f0c989d12cfceb06b005808492ec1ff6ae90fab1fc4bf7ac8e825ac86bc8a0b"}, - {file = "botocore-1.34.47.tar.gz", hash = "sha256:29f1d6659602c5d79749eca7430857f7a48dd02e597d0ea4a95a83c47847993e"}, + {file = "botocore-1.34.50-py3-none-any.whl", hash = "sha256:fda510559dbe796eefdb59561cc81be1b99afba3dee53fd23db9a3d587adc0ab"}, + {file = "botocore-1.34.50.tar.gz", hash = "sha256:33ab82cb96c4bb684f0dbafb071808e4817d83debc88b223e7d988256370c6d7"}, ] [package.dependencies] @@ -219,13 +219,13 @@ crt = ["awscrt (==0.19.19)"] [[package]] name = "cachetools" -version = "5.3.2" +version = "5.3.3" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, - {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, + {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, + {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, ] [[package]] @@ -776,18 +776,18 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs [[package]] name = "importlib-resources" -version = "6.1.1" +version = "6.1.2" description = "Read resources from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_resources-6.1.1-py3-none-any.whl", hash = "sha256:e8bf90d8213b486f428c9c39714b920041cb02c184686a3dee24905aaa8105d6"}, - {file = "importlib_resources-6.1.1.tar.gz", hash = "sha256:3893a00122eafde6894c59914446a512f728a0c1a45f9bb9b63721b6bacf0b4a"}, + {file = "importlib_resources-6.1.2-py3-none-any.whl", hash = "sha256:9a0a862501dc38b68adebc82970140c9e4209fc99601782925178f8386339938"}, + {file = "importlib_resources-6.1.2.tar.gz", hash = "sha256:308abf8474e2dba5f867d279237cd4076482c3de7104a40b41426370e891549b"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-ruff", "zipp (>=3.17)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)", "zipp (>=3.17)"] [[package]] name = "iniconfig" @@ -1564,18 +1564,18 @@ files = [ [[package]] name = "pydantic" -version = "2.6.1" +version = "2.6.2" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.6.1-py3-none-any.whl", hash = "sha256:0b6a909df3192245cb736509a92ff69e4fef76116feffec68e93a567347bae6f"}, - {file = "pydantic-2.6.1.tar.gz", hash = "sha256:4fd5c182a2488dc63e6d32737ff19937888001e2a6d86e94b3f233104a5d1fa9"}, + {file = "pydantic-2.6.2-py3-none-any.whl", hash = "sha256:37a5432e54b12fecaa1049c5195f3d860a10e01bdfd24f1840ef14bd0d3aeab3"}, + {file = "pydantic-2.6.2.tar.gz", hash = "sha256:a09be1c3d28f3abe37f8a78af58284b236a92ce520105ddc91a6d29ea1176ba7"}, ] [package.dependencies] annotated-types = ">=0.4.0" -pydantic-core = "2.16.2" +pydantic-core = "2.16.3" typing-extensions = ">=4.6.1" [package.extras] @@ -1583,90 +1583,90 @@ email = ["email-validator (>=2.0.0)"] [[package]] name = "pydantic-core" -version = "2.16.2" +version = "2.16.3" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.16.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3fab4e75b8c525a4776e7630b9ee48aea50107fea6ca9f593c98da3f4d11bf7c"}, - {file = "pydantic_core-2.16.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8bde5b48c65b8e807409e6f20baee5d2cd880e0fad00b1a811ebc43e39a00ab2"}, - {file = "pydantic_core-2.16.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2924b89b16420712e9bb8192396026a8fbd6d8726224f918353ac19c4c043d2a"}, - {file = "pydantic_core-2.16.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:16aa02e7a0f539098e215fc193c8926c897175d64c7926d00a36188917717a05"}, - {file = "pydantic_core-2.16.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:936a787f83db1f2115ee829dd615c4f684ee48ac4de5779ab4300994d8af325b"}, - {file = "pydantic_core-2.16.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:459d6be6134ce3b38e0ef76f8a672924460c455d45f1ad8fdade36796df1ddc8"}, - {file = "pydantic_core-2.16.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9ee4febb249c591d07b2d4dd36ebcad0ccd128962aaa1801508320896575ef"}, - {file = "pydantic_core-2.16.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:40a0bd0bed96dae5712dab2aba7d334a6c67cbcac2ddfca7dbcc4a8176445990"}, - {file = "pydantic_core-2.16.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:870dbfa94de9b8866b37b867a2cb37a60c401d9deb4a9ea392abf11a1f98037b"}, - {file = "pydantic_core-2.16.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:308974fdf98046db28440eb3377abba274808bf66262e042c412eb2adf852731"}, - {file = "pydantic_core-2.16.2-cp310-none-win32.whl", hash = "sha256:a477932664d9611d7a0816cc3c0eb1f8856f8a42435488280dfbf4395e141485"}, - {file = "pydantic_core-2.16.2-cp310-none-win_amd64.whl", hash = "sha256:8f9142a6ed83d90c94a3efd7af8873bf7cefed2d3d44387bf848888482e2d25f"}, - {file = "pydantic_core-2.16.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:406fac1d09edc613020ce9cf3f2ccf1a1b2f57ab00552b4c18e3d5276c67eb11"}, - {file = "pydantic_core-2.16.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ce232a6170dd6532096cadbf6185271e4e8c70fc9217ebe105923ac105da9978"}, - {file = "pydantic_core-2.16.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a90fec23b4b05a09ad988e7a4f4e081711a90eb2a55b9c984d8b74597599180f"}, - {file = "pydantic_core-2.16.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8aafeedb6597a163a9c9727d8a8bd363a93277701b7bfd2749fbefee2396469e"}, - {file = "pydantic_core-2.16.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9957433c3a1b67bdd4c63717eaf174ebb749510d5ea612cd4e83f2d9142f3fc8"}, - {file = "pydantic_core-2.16.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0d7a9165167269758145756db43a133608a531b1e5bb6a626b9ee24bc38a8f7"}, - {file = "pydantic_core-2.16.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dffaf740fe2e147fedcb6b561353a16243e654f7fe8e701b1b9db148242e1272"}, - {file = "pydantic_core-2.16.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8ed79883b4328b7f0bd142733d99c8e6b22703e908ec63d930b06be3a0e7113"}, - {file = "pydantic_core-2.16.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:cf903310a34e14651c9de056fcc12ce090560864d5a2bb0174b971685684e1d8"}, - {file = "pydantic_core-2.16.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:46b0d5520dbcafea9a8645a8164658777686c5c524d381d983317d29687cce97"}, - {file = "pydantic_core-2.16.2-cp311-none-win32.whl", hash = "sha256:70651ff6e663428cea902dac297066d5c6e5423fda345a4ca62430575364d62b"}, - {file = "pydantic_core-2.16.2-cp311-none-win_amd64.whl", hash = "sha256:98dc6f4f2095fc7ad277782a7c2c88296badcad92316b5a6e530930b1d475ebc"}, - {file = "pydantic_core-2.16.2-cp311-none-win_arm64.whl", hash = "sha256:ef6113cd31411eaf9b39fc5a8848e71c72656fd418882488598758b2c8c6dfa0"}, - {file = "pydantic_core-2.16.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:88646cae28eb1dd5cd1e09605680c2b043b64d7481cdad7f5003ebef401a3039"}, - {file = "pydantic_core-2.16.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7b883af50eaa6bb3299780651e5be921e88050ccf00e3e583b1e92020333304b"}, - {file = "pydantic_core-2.16.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bf26c2e2ea59d32807081ad51968133af3025c4ba5753e6a794683d2c91bf6e"}, - {file = "pydantic_core-2.16.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:99af961d72ac731aae2a1b55ccbdae0733d816f8bfb97b41909e143de735f522"}, - {file = "pydantic_core-2.16.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02906e7306cb8c5901a1feb61f9ab5e5c690dbbeaa04d84c1b9ae2a01ebe9379"}, - {file = "pydantic_core-2.16.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5362d099c244a2d2f9659fb3c9db7c735f0004765bbe06b99be69fbd87c3f15"}, - {file = "pydantic_core-2.16.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ac426704840877a285d03a445e162eb258924f014e2f074e209d9b4ff7bf380"}, - {file = "pydantic_core-2.16.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b94cbda27267423411c928208e89adddf2ea5dd5f74b9528513f0358bba019cb"}, - {file = "pydantic_core-2.16.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:6db58c22ac6c81aeac33912fb1af0e930bc9774166cdd56eade913d5f2fff35e"}, - {file = "pydantic_core-2.16.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:396fdf88b1b503c9c59c84a08b6833ec0c3b5ad1a83230252a9e17b7dfb4cffc"}, - {file = "pydantic_core-2.16.2-cp312-none-win32.whl", hash = "sha256:7c31669e0c8cc68400ef0c730c3a1e11317ba76b892deeefaf52dcb41d56ed5d"}, - {file = "pydantic_core-2.16.2-cp312-none-win_amd64.whl", hash = "sha256:a3b7352b48fbc8b446b75f3069124e87f599d25afb8baa96a550256c031bb890"}, - {file = "pydantic_core-2.16.2-cp312-none-win_arm64.whl", hash = "sha256:a9e523474998fb33f7c1a4d55f5504c908d57add624599e095c20fa575b8d943"}, - {file = "pydantic_core-2.16.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:ae34418b6b389d601b31153b84dce480351a352e0bb763684a1b993d6be30f17"}, - {file = "pydantic_core-2.16.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:732bd062c9e5d9582a30e8751461c1917dd1ccbdd6cafb032f02c86b20d2e7ec"}, - {file = "pydantic_core-2.16.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b52776a2e3230f4854907a1e0946eec04d41b1fc64069ee774876bbe0eab55"}, - {file = "pydantic_core-2.16.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ef551c053692b1e39e3f7950ce2296536728871110e7d75c4e7753fb30ca87f4"}, - {file = "pydantic_core-2.16.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ebb892ed8599b23fa8f1799e13a12c87a97a6c9d0f497525ce9858564c4575a4"}, - {file = "pydantic_core-2.16.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa6c8c582036275997a733427b88031a32ffa5dfc3124dc25a730658c47a572f"}, - {file = "pydantic_core-2.16.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4ba0884a91f1aecce75202473ab138724aa4fb26d7707f2e1fa6c3e68c84fbf"}, - {file = "pydantic_core-2.16.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7924e54f7ce5d253d6160090ddc6df25ed2feea25bfb3339b424a9dd591688bc"}, - {file = "pydantic_core-2.16.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69a7b96b59322a81c2203be537957313b07dd333105b73db0b69212c7d867b4b"}, - {file = "pydantic_core-2.16.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7e6231aa5bdacda78e96ad7b07d0c312f34ba35d717115f4b4bff6cb87224f0f"}, - {file = "pydantic_core-2.16.2-cp38-none-win32.whl", hash = "sha256:41dac3b9fce187a25c6253ec79a3f9e2a7e761eb08690e90415069ea4a68ff7a"}, - {file = "pydantic_core-2.16.2-cp38-none-win_amd64.whl", hash = "sha256:f685dbc1fdadb1dcd5b5e51e0a378d4685a891b2ddaf8e2bba89bd3a7144e44a"}, - {file = "pydantic_core-2.16.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:55749f745ebf154c0d63d46c8c58594d8894b161928aa41adbb0709c1fe78b77"}, - {file = "pydantic_core-2.16.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b30b0dd58a4509c3bd7eefddf6338565c4905406aee0c6e4a5293841411a1286"}, - {file = "pydantic_core-2.16.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18de31781cdc7e7b28678df7c2d7882f9692ad060bc6ee3c94eb15a5d733f8f7"}, - {file = "pydantic_core-2.16.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5864b0242f74b9dd0b78fd39db1768bc3f00d1ffc14e596fd3e3f2ce43436a33"}, - {file = "pydantic_core-2.16.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8f9186ca45aee030dc8234118b9c0784ad91a0bb27fc4e7d9d6608a5e3d386c"}, - {file = "pydantic_core-2.16.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc6f6c9be0ab6da37bc77c2dda5f14b1d532d5dbef00311ee6e13357a418e646"}, - {file = "pydantic_core-2.16.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa057095f621dad24a1e906747179a69780ef45cc8f69e97463692adbcdae878"}, - {file = "pydantic_core-2.16.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6ad84731a26bcfb299f9eab56c7932d46f9cad51c52768cace09e92a19e4cf55"}, - {file = "pydantic_core-2.16.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3b052c753c4babf2d1edc034c97851f867c87d6f3ea63a12e2700f159f5c41c3"}, - {file = "pydantic_core-2.16.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e0f686549e32ccdb02ae6f25eee40cc33900910085de6aa3790effd391ae10c2"}, - {file = "pydantic_core-2.16.2-cp39-none-win32.whl", hash = "sha256:7afb844041e707ac9ad9acad2188a90bffce2c770e6dc2318be0c9916aef1469"}, - {file = "pydantic_core-2.16.2-cp39-none-win_amd64.whl", hash = "sha256:9da90d393a8227d717c19f5397688a38635afec89f2e2d7af0df037f3249c39a"}, - {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5f60f920691a620b03082692c378661947d09415743e437a7478c309eb0e4f82"}, - {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:47924039e785a04d4a4fa49455e51b4eb3422d6eaacfde9fc9abf8fdef164e8a"}, - {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6294e76b0380bb7a61eb8a39273c40b20beb35e8c87ee101062834ced19c545"}, - {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe56851c3f1d6f5384b3051c536cc81b3a93a73faf931f404fef95217cf1e10d"}, - {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9d776d30cde7e541b8180103c3f294ef7c1862fd45d81738d156d00551005784"}, - {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:72f7919af5de5ecfaf1eba47bf9a5d8aa089a3340277276e5636d16ee97614d7"}, - {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:4bfcbde6e06c56b30668a0c872d75a7ef3025dc3c1823a13cf29a0e9b33f67e8"}, - {file = "pydantic_core-2.16.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ff7c97eb7a29aba230389a2661edf2e9e06ce616c7e35aa764879b6894a44b25"}, - {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9b5f13857da99325dcabe1cc4e9e6a3d7b2e2c726248ba5dd4be3e8e4a0b6d0e"}, - {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a7e41e3ada4cca5f22b478c08e973c930e5e6c7ba3588fb8e35f2398cdcc1545"}, - {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60eb8ceaa40a41540b9acae6ae7c1f0a67d233c40dc4359c256ad2ad85bdf5e5"}, - {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7beec26729d496a12fd23cf8da9944ee338c8b8a17035a560b585c36fe81af20"}, - {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:22c5f022799f3cd6741e24f0443ead92ef42be93ffda0d29b2597208c94c3753"}, - {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:eca58e319f4fd6df004762419612122b2c7e7d95ffafc37e890252f869f3fb2a"}, - {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ed957db4c33bc99895f3a1672eca7e80e8cda8bd1e29a80536b4ec2153fa9804"}, - {file = "pydantic_core-2.16.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:459c0d338cc55d099798618f714b21b7ece17eb1a87879f2da20a3ff4c7628e2"}, - {file = "pydantic_core-2.16.2.tar.gz", hash = "sha256:0ba503850d8b8dcc18391f10de896ae51d37fe5fe43dbfb6a35c5c5cad271a06"}, + {file = "pydantic_core-2.16.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:75b81e678d1c1ede0785c7f46690621e4c6e63ccd9192af1f0bd9d504bbb6bf4"}, + {file = "pydantic_core-2.16.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9c865a7ee6f93783bd5d781af5a4c43dadc37053a5b42f7d18dc019f8c9d2bd1"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:162e498303d2b1c036b957a1278fa0899d02b2842f1ff901b6395104c5554a45"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2f583bd01bbfbff4eaee0868e6fc607efdfcc2b03c1c766b06a707abbc856187"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b926dd38db1519ed3043a4de50214e0d600d404099c3392f098a7f9d75029ff8"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:716b542728d4c742353448765aa7cdaa519a7b82f9564130e2b3f6766018c9ec"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4ad7f7ee1a13d9cb49d8198cd7d7e3aa93e425f371a68235f784e99741561f"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bd87f48924f360e5d1c5f770d6155ce0e7d83f7b4e10c2f9ec001c73cf475c99"}, + {file = "pydantic_core-2.16.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0df446663464884297c793874573549229f9eca73b59360878f382a0fc085979"}, + {file = "pydantic_core-2.16.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4df8a199d9f6afc5ae9a65f8f95ee52cae389a8c6b20163762bde0426275b7db"}, + {file = "pydantic_core-2.16.3-cp310-none-win32.whl", hash = "sha256:456855f57b413f077dff513a5a28ed838dbbb15082ba00f80750377eed23d132"}, + {file = "pydantic_core-2.16.3-cp310-none-win_amd64.whl", hash = "sha256:732da3243e1b8d3eab8c6ae23ae6a58548849d2e4a4e03a1924c8ddf71a387cb"}, + {file = "pydantic_core-2.16.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:519ae0312616026bf4cedc0fe459e982734f3ca82ee8c7246c19b650b60a5ee4"}, + {file = "pydantic_core-2.16.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b3992a322a5617ded0a9f23fd06dbc1e4bd7cf39bc4ccf344b10f80af58beacd"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d62da299c6ecb04df729e4b5c52dc0d53f4f8430b4492b93aa8de1f541c4aac"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2acca2be4bb2f2147ada8cac612f8a98fc09f41c89f87add7256ad27332c2fda"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b662180108c55dfbf1280d865b2d116633d436cfc0bba82323554873967b340"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e7c6ed0dc9d8e65f24f5824291550139fe6f37fac03788d4580da0d33bc00c97"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6b1bb0827f56654b4437955555dc3aeeebeddc47c2d7ed575477f082622c49e"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e56f8186d6210ac7ece503193ec84104da7ceb98f68ce18c07282fcc2452e76f"}, + {file = "pydantic_core-2.16.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:936e5db01dd49476fa8f4383c259b8b1303d5dd5fb34c97de194560698cc2c5e"}, + {file = "pydantic_core-2.16.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:33809aebac276089b78db106ee692bdc9044710e26f24a9a2eaa35a0f9fa70ba"}, + {file = "pydantic_core-2.16.3-cp311-none-win32.whl", hash = "sha256:ded1c35f15c9dea16ead9bffcde9bb5c7c031bff076355dc58dcb1cb436c4721"}, + {file = "pydantic_core-2.16.3-cp311-none-win_amd64.whl", hash = "sha256:d89ca19cdd0dd5f31606a9329e309d4fcbb3df860960acec32630297d61820df"}, + {file = "pydantic_core-2.16.3-cp311-none-win_arm64.whl", hash = "sha256:6162f8d2dc27ba21027f261e4fa26f8bcb3cf9784b7f9499466a311ac284b5b9"}, + {file = "pydantic_core-2.16.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0f56ae86b60ea987ae8bcd6654a887238fd53d1384f9b222ac457070b7ac4cff"}, + {file = "pydantic_core-2.16.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9bd22a2a639e26171068f8ebb5400ce2c1bc7d17959f60a3b753ae13c632975"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4204e773b4b408062960e65468d5346bdfe139247ee5f1ca2a378983e11388a2"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f651dd19363c632f4abe3480a7c87a9773be27cfe1341aef06e8759599454120"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aaf09e615a0bf98d406657e0008e4a8701b11481840be7d31755dc9f97c44053"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8e47755d8152c1ab5b55928ab422a76e2e7b22b5ed8e90a7d584268dd49e9c6b"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:500960cb3a0543a724a81ba859da816e8cf01b0e6aaeedf2c3775d12ee49cade"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cf6204fe865da605285c34cf1172879d0314ff267b1c35ff59de7154f35fdc2e"}, + {file = "pydantic_core-2.16.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d33dd21f572545649f90c38c227cc8631268ba25c460b5569abebdd0ec5974ca"}, + {file = "pydantic_core-2.16.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:49d5d58abd4b83fb8ce763be7794d09b2f50f10aa65c0f0c1696c677edeb7cbf"}, + {file = "pydantic_core-2.16.3-cp312-none-win32.whl", hash = "sha256:f53aace168a2a10582e570b7736cc5bef12cae9cf21775e3eafac597e8551fbe"}, + {file = "pydantic_core-2.16.3-cp312-none-win_amd64.whl", hash = "sha256:0d32576b1de5a30d9a97f300cc6a3f4694c428d956adbc7e6e2f9cad279e45ed"}, + {file = "pydantic_core-2.16.3-cp312-none-win_arm64.whl", hash = "sha256:ec08be75bb268473677edb83ba71e7e74b43c008e4a7b1907c6d57e940bf34b6"}, + {file = "pydantic_core-2.16.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:b1f6f5938d63c6139860f044e2538baeee6f0b251a1816e7adb6cbce106a1f01"}, + {file = "pydantic_core-2.16.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2a1ef6a36fdbf71538142ed604ad19b82f67b05749512e47f247a6ddd06afdc7"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:704d35ecc7e9c31d48926150afada60401c55efa3b46cd1ded5a01bdffaf1d48"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d937653a696465677ed583124b94a4b2d79f5e30b2c46115a68e482c6a591c8a"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9803edf8e29bd825f43481f19c37f50d2b01899448273b3a7758441b512acf8"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:72282ad4892a9fb2da25defeac8c2e84352c108705c972db82ab121d15f14e6d"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f752826b5b8361193df55afcdf8ca6a57d0232653494ba473630a83ba50d8c9"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4384a8f68ddb31a0b0c3deae88765f5868a1b9148939c3f4121233314ad5532c"}, + {file = "pydantic_core-2.16.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4b2bf78342c40b3dc830880106f54328928ff03e357935ad26c7128bbd66ce8"}, + {file = "pydantic_core-2.16.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:13dcc4802961b5f843a9385fc821a0b0135e8c07fc3d9949fd49627c1a5e6ae5"}, + {file = "pydantic_core-2.16.3-cp38-none-win32.whl", hash = "sha256:e3e70c94a0c3841e6aa831edab1619ad5c511199be94d0c11ba75fe06efe107a"}, + {file = "pydantic_core-2.16.3-cp38-none-win_amd64.whl", hash = "sha256:ecdf6bf5f578615f2e985a5e1f6572e23aa632c4bd1dc67f8f406d445ac115ed"}, + {file = "pydantic_core-2.16.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:bda1ee3e08252b8d41fa5537413ffdddd58fa73107171a126d3b9ff001b9b820"}, + {file = "pydantic_core-2.16.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:21b888c973e4f26b7a96491c0965a8a312e13be108022ee510248fe379a5fa23"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be0ec334369316fa73448cc8c982c01e5d2a81c95969d58b8f6e272884df0074"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5b6079cc452a7c53dd378c6f881ac528246b3ac9aae0f8eef98498a75657805"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ee8d5f878dccb6d499ba4d30d757111847b6849ae07acdd1205fffa1fc1253c"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7233d65d9d651242a68801159763d09e9ec96e8a158dbf118dc090cd77a104c9"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6119dc90483a5cb50a1306adb8d52c66e447da88ea44f323e0ae1a5fcb14256"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:578114bc803a4c1ff9946d977c221e4376620a46cf78da267d946397dc9514a8"}, + {file = "pydantic_core-2.16.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d8f99b147ff3fcf6b3cc60cb0c39ea443884d5559a30b1481e92495f2310ff2b"}, + {file = "pydantic_core-2.16.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4ac6b4ce1e7283d715c4b729d8f9dab9627586dafce81d9eaa009dd7f25dd972"}, + {file = "pydantic_core-2.16.3-cp39-none-win32.whl", hash = "sha256:e7774b570e61cb998490c5235740d475413a1f6de823169b4cf94e2fe9e9f6b2"}, + {file = "pydantic_core-2.16.3-cp39-none-win_amd64.whl", hash = "sha256:9091632a25b8b87b9a605ec0e61f241c456e9248bfdcf7abdf344fdb169c81cf"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:36fa178aacbc277bc6b62a2c3da95226520da4f4e9e206fdf076484363895d2c"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:dcca5d2bf65c6fb591fff92da03f94cd4f315972f97c21975398bd4bd046854a"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a72fb9963cba4cd5793854fd12f4cfee731e86df140f59ff52a49b3552db241"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b60cc1a081f80a2105a59385b92d82278b15d80ebb3adb200542ae165cd7d183"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cbcc558401de90a746d02ef330c528f2e668c83350f045833543cd57ecead1ad"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fee427241c2d9fb7192b658190f9f5fd6dfe41e02f3c1489d2ec1e6a5ab1e04a"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f4cb85f693044e0f71f394ff76c98ddc1bc0953e48c061725e540396d5c8a2e1"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b29eeb887aa931c2fcef5aa515d9d176d25006794610c264ddc114c053bf96fe"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a425479ee40ff021f8216c9d07a6a3b54b31c8267c6e17aa88b70d7ebd0e5e5b"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:5c5cbc703168d1b7a838668998308018a2718c2130595e8e190220238addc96f"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99b6add4c0b39a513d323d3b93bc173dac663c27b99860dd5bf491b240d26137"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f76ee558751746d6a38f89d60b6228fa174e5172d143886af0f85aa306fd89"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:00ee1c97b5364b84cb0bd82e9bbf645d5e2871fb8c58059d158412fee2d33d8a"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:287073c66748f624be4cef893ef9174e3eb88fe0b8a78dc22e88eca4bc357ca6"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ed25e1835c00a332cb10c683cd39da96a719ab1dfc08427d476bce41b92531fc"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:86b3d0033580bd6bbe07590152007275bd7af95f98eaa5bd36f3da219dcd93da"}, + {file = "pydantic_core-2.16.3.tar.gz", hash = "sha256:1cac689f80a3abab2d3c0048b29eea5751114054f032a941a32de4c852c59cad"}, ] [package.dependencies] @@ -2331,19 +2331,19 @@ files = [ [[package]] name = "setuptools" -version = "69.1.0" +version = "69.1.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, - {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, + {file = "setuptools-69.1.1-py3-none-any.whl", hash = "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56"}, + {file = "setuptools-69.1.1.tar.gz", hash = "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -2380,13 +2380,13 @@ files = [ [[package]] name = "stevedore" -version = "5.1.0" +version = "5.2.0" description = "Manage dynamic plugins for Python applications" optional = false python-versions = ">=3.8" files = [ - {file = "stevedore-5.1.0-py3-none-any.whl", hash = "sha256:8cc040628f3cea5d7128f2e76cf486b2251a4e543c7b938f58d9a377f6694a2d"}, - {file = "stevedore-5.1.0.tar.gz", hash = "sha256:a54534acf9b89bc7ed264807013b505bf07f74dbe4bcfa37d32bd063870b087c"}, + {file = "stevedore-5.2.0-py3-none-any.whl", hash = "sha256:1c15d95766ca0569cad14cb6272d4d31dae66b011a929d7c18219c176ea1b5c9"}, + {file = "stevedore-5.2.0.tar.gz", hash = "sha256:46b93ca40e1114cea93d738a6c1e365396981bb6bb78c27045b7587c9473544d"}, ] [package.dependencies] @@ -2507,13 +2507,13 @@ test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0, [[package]] name = "typing-extensions" -version = "4.9.0" +version = "4.10.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, - {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, + {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, + {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, ] [[package]] @@ -2680,4 +2680,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10.0,<4.0" -content-hash = "4229969d58cbf7b7f259153ed58e4dcb35a34764d7743f52a5f53abf97dd8529" +content-hash = "305a8b50cfcef1fdfa760acfa608afd99ab0166e88e2fcd10fb3a38b07ba82f6" From af1a9294d29b5885c05db929148e317357c93c52 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 13:42:38 +0100 Subject: [PATCH 08/37] Fixed test --- doc/changes/changes_1.1.0.md | 2 +- test/integration/test_motd_jupyter_template.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/doc/changes/changes_1.1.0.md b/doc/changes/changes_1.1.0.md index 0846601a..043c01ac 100644 --- a/doc/changes/changes_1.1.0.md +++ b/doc/changes/changes_1.1.0.md @@ -33,4 +33,4 @@ n/a * #217: Changed notebook-connector dependency, now installing it from PyPi. * #220: Changed default ports in the external database configuration. * #221: Changed wording in the main configuration notebook, as suggested by PM. -* #258: Used a non-root user to run Jupyter in the Docker Image ai-lab +* #66: Used a non-root user to run Jupyter in the Docker Image ai-lab diff --git a/test/integration/test_motd_jupyter_template.py b/test/integration/test_motd_jupyter_template.py index c743d6ca..591da688 100644 --- a/test/integration/test_motd_jupyter_template.py +++ b/test/integration/test_motd_jupyter_template.py @@ -11,7 +11,8 @@ @pytest.fixture() def motd_file(tmp_path): - jupyter_server_config_file = tmp_path / "jupyter_server_config.json" + # jupyterlab_config_json + config_file = tmp_path / "jupyter_server_config.json" python_file = tmp_path / "999_jupyter.py" src_path = Path(__file__).parent.parent.parent \ @@ -27,12 +28,12 @@ class jupyterlab(NamedTuple): password = "dummy_password" python_code = python_template.render(user_name="test_user", jupyterlab=jupyterlab(), - jupyter_server_config_file=str(jupyter_server_config_file), + jupyterlab_config_json=str(config_file), jupyter_server_hashed_password="dummy_password_hash", heading_jupyter_update_password=jupyter_update_msg_heading) with open(python_file, "w") as f: f.write(python_code) - yield python_file, jupyter_server_config_file + yield python_file, config_file def test_motd_jupyter_template_prints_password_message(motd_file): @@ -40,13 +41,13 @@ def test_motd_jupyter_template_prints_password_message(motd_file): Test which runs the motd jupyter template and validates that the message is printed because the password matches. """ - python_file, jupyter_server_config_file = motd_file + python_file, config_file = motd_file mock_data = { "IdentityProvider": { "hashed_password": "dummy_password_hash" } } - with open(jupyter_server_config_file, "w") as f: + with open(config_file, "w") as f: json.dump(mock_data, f) result = subprocess.run(["python3", python_file], capture_output=True) @@ -58,13 +59,13 @@ def test_motd_jupyter_template_prints_password_message_not_if_passward_was_chang Test which runs the motd jupyter template and validates that the message is not printed because the password does not match. """ - python_file, jupyter_server_config_file = motd_file + python_file, config_file = motd_file mock_data = { "IdentityProvider": { "hashed_password": "NOT_MATCHING_PASSWORD" } } - with open(jupyter_server_config_file, "w") as f: + with open(config_file, "w") as f: json.dump(mock_data, f) result = subprocess.run(["python3", python_file], capture_output=True) From c776849c7cf1dfee5d7dd8c2d37641ccb16bdd1b Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 14:09:07 +0100 Subject: [PATCH 09/37] Improved Ansible output for pip install fixed Ansible warning about providing a list of dictionaries to vars: See https://docs.ansible.com/ansible/latest/porting_guides/porting_guide_8.html#deprecated --- ...{requirements_jupyter.txt => jupyter_requirements.txt} | 0 ...rements_dependencies.txt => notebook_requirements.txt} | 0 .../ansible/roles/jupyter/tasks/install-pip-packages.yml | 6 +++--- .../sandbox/runtime/ansible/roles/jupyter/tasks/main.yml | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) rename exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/{requirements_jupyter.txt => jupyter_requirements.txt} (100%) rename exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/{requirements_dependencies.txt => notebook_requirements.txt} (100%) diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/jupyter_requirements.txt similarity index 100% rename from exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/jupyter_requirements.txt diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt similarity index 100% rename from exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml index 0ffae114..5f1c115e 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml @@ -1,13 +1,13 @@ --- -- name: Copy pip requirements file +- name: Copy requirements file {{ pip_requirements_file }} ansible.builtin.copy: src: "{{pip_requirements_file}}" dest: "/tmp/{{pip_requirements_file}}" mode: 0644 register: pip_requirements_file_loc -- name: Install Python packages via pip +- name: Call pip install {{ pip_requirements_file }} ansible.builtin.pip: requirements: "{{ pip_requirements_file_loc.dest }}" state: latest @@ -15,7 +15,7 @@ virtualenv_python: python3.8 become: "{{need_sudo}}" -- name: Remove the requirements file (delete file) +- name: Remove requirements file {{ pip_requirements_file }} ansible.builtin.file: path: "{{ pip_requirements_file_loc.dest }}" state: absent diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml index 5aaa7730..23a6e3ee 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml @@ -14,20 +14,20 @@ include_tasks: file: update-pip.yml - - name: Install JupyterLab and its dependencies + - name: Install JupyterLab and required packages include_tasks: file: install-pip-packages.yml vars: - - pip_requirements_file: "requirements_jupyter.txt" + pip_requirements_file: "jupyter_requirements.txt" - name: Setup Jupyterlab ansible.builtin.import_tasks: jupyterlab.yml - - name: Install dependencies used in Jupyterlab + - name: Install packages to be used inside Jupyter notebooks include_tasks: file: install-pip-packages.yml vars: - - pip_requirements_file: "requirements_dependencies.txt" + pip_requirements_file: "notebook_requirements.txt" - name: Setup Systemd service ansible.builtin.import_tasks: systemd.yml From 36117ead779e908fb533972e2a9daeff7a54e42f Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 14:16:36 +0100 Subject: [PATCH 10/37] Improved Ansible output for pip install fixed Ansible warning about providing a list of dictionaries to vars: See https://docs.ansible.com/ansible/latest/porting_guides/porting_guide_8.html#deprecated --- .../jupyter/tasks/install-pip-packages.yml | 21 ------------------- .../ansible/roles/jupyter/tasks/main.yml | 8 +++---- .../roles/jupyter/tasks/pip-install.yml | 21 +++++++++++++++++++ 3 files changed, 25 insertions(+), 25 deletions(-) delete mode 100644 exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml create mode 100644 exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/pip-install.yml diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml deleted file mode 100644 index 5f1c115e..00000000 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- - -- name: Copy requirements file {{ pip_requirements_file }} - ansible.builtin.copy: - src: "{{pip_requirements_file}}" - dest: "/tmp/{{pip_requirements_file}}" - mode: 0644 - register: pip_requirements_file_loc - -- name: Call pip install {{ pip_requirements_file }} - ansible.builtin.pip: - requirements: "{{ pip_requirements_file_loc.dest }}" - state: latest - virtualenv: "{{jupyterlab_virtualenv}}" - virtualenv_python: python3.8 - become: "{{need_sudo}}" - -- name: Remove requirements file {{ pip_requirements_file }} - ansible.builtin.file: - path: "{{ pip_requirements_file_loc.dest }}" - state: absent diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml index 23a6e3ee..d7823afb 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml @@ -16,18 +16,18 @@ - name: Install JupyterLab and required packages include_tasks: - file: install-pip-packages.yml + file: pip-install.yml vars: - pip_requirements_file: "jupyter_requirements.txt" + requirements_file: "jupyter_requirements.txt" - name: Setup Jupyterlab ansible.builtin.import_tasks: jupyterlab.yml - name: Install packages to be used inside Jupyter notebooks include_tasks: - file: install-pip-packages.yml + file: pip-install.yml vars: - pip_requirements_file: "notebook_requirements.txt" + requirements_file: "notebook_requirements.txt" - name: Setup Systemd service ansible.builtin.import_tasks: systemd.yml diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/pip-install.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/pip-install.yml new file mode 100644 index 00000000..011a0dbd --- /dev/null +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/pip-install.yml @@ -0,0 +1,21 @@ +--- + +- name: Copy requirements file {{ requirements_file }} + ansible.builtin.copy: + src: "{{requirements_file}}" + dest: "/tmp/{{requirements_file}}" + mode: 0644 + register: copy_req_file + +- name: Call pip install {{ requirements_file }} + ansible.builtin.pip: + requirements: "{{ copy_req_file.dest }}" + state: latest + virtualenv: "{{jupyterlab_virtualenv}}" + virtualenv_python: python3.8 + become: "{{need_sudo}}" + +- name: Remove requirements file {{ requirements_file }} + ansible.builtin.file: + path: "{{ copy_req_file.dest }}" + state: absent From 230474169a92eff418218918b8e479f08c6158c2 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 14:44:42 +0100 Subject: [PATCH 11/37] Removed test scaffolds --- exasol/ds/sandbox/lib/dss_docker/create_image.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/exasol/ds/sandbox/lib/dss_docker/create_image.py b/exasol/ds/sandbox/lib/dss_docker/create_image.py index 933f870d..99484568 100644 --- a/exasol/ds/sandbox/lib/dss_docker/create_image.py +++ b/exasol/ds/sandbox/lib/dss_docker/create_image.py @@ -205,15 +205,3 @@ def create(self): size = humanfriendly.format_size(image.attrs["Size"]) elapsed = pretty_print.elapsed(self._start) _logger.info(f"Built Docker image {self.image_name} size {size} in {elapsed}.") - - -import sys -from exasol.ds.sandbox.lib.logging import set_log_level -if __name__ == "__main__": - set_log_level("info") - di = DssDockerImage("exasol/ai-lab", "9.9.9", True) - if len(sys.argv) > 1: - di.container_name = sys.argv[1] - di._install_dependencies() - else: - di.create() From b030baf5edb821705865800c718c4931cb69b1ef Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 14:46:30 +0100 Subject: [PATCH 12/37] Capitalized name of Ansible task --- exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml b/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml index 11423d0f..a3cd0223 100644 --- a/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml +++ b/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml @@ -34,7 +34,7 @@ jupyterlab_password: "{{ dss_facts.jupyter.password }}" jupyterlab_notebook_folder_initial: "{{ dss_facts.notebook_folder.initial }}" jupyterlab_notebook_folder: "{{ dss_facts.notebook_folder.final }}" -- name: Change owner of all files and dirs in home directory +- name: Change Owner of All Files and Dirs in Home Directory ansible.builtin.file: path: "{{ user_home }}" owner: "{{ user_name }}" From a66235dd5b0063689c9a200d8a25e372a3ee9a71 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 14:50:24 +0100 Subject: [PATCH 13/37] Updated comments in Ansible tasks --- .../sandbox/runtime/ansible/roles/jupyter/defaults/main.yml | 3 --- .../runtime/ansible/roles/jupyter/tasks/jupyterlab.yml | 5 +++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml index c070626b..bca77c7d 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml @@ -1,10 +1,7 @@ --- jupyterlab_ip: '*' -# Setting environment variable $HOME to {{user_home}} forces the directory for -# creating the config file. Unfortunately this does not affect the json file. jupyterlab_config: "{{user_home}}/.jupyter/jupyter_lab_config.py" -# jupyterlab_config_json: "{{ansible_env.HOME}}/.jupyter/jupyter_server_config.json" jupyterlab_config_json: "{{user_home}}/.jupyter/jupyter_server_config.json" jupyterlab_default_url: "/lab/tree/start.ipynb" diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml index 809d4d78..96cdbcfa 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml @@ -12,8 +12,7 @@ # - it has all rows commented out # # Set environment variable $HOME to {{user_home}} to force the directory for -# creating the config file. Unfortunately this does not affect the file -# jupyter_server_config.json. +# creating the config file. - name: Generate JupyterLab config ansible.builtin.command: "{{jupyterlab_virtualenv}}/bin/jupyter lab --generate-config" environment: @@ -36,6 +35,8 @@ line: "c.ServerApp.port = {{ jupyterlab_port }}" become: "{{ need_sudo }}" +# Set environment variable $HOME to {{user_home}} to force the directory for +# creating the file jupyter_server_config.json. - name: Generate a new password expect: command: "{{jupyterlab_virtualenv}}/bin/jupyter server password" From 71ecf25973ed41b1335501ed6c7694cf32448832 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 17:33:54 +0100 Subject: [PATCH 14/37] Updated user guide for Docker Edition --- doc/user_guide/docker/docker-usage.md | 8 ++++---- doc/user_guide/docker/managing-user-data.md | 4 ++-- doc/user_guide/jupyter.md | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/user_guide/docker/docker-usage.md b/doc/user_guide/docker/docker-usage.md index 285263db..349dda51 100644 --- a/doc/user_guide/docker/docker-usage.md +++ b/doc/user_guide/docker/docker-usage.md @@ -39,14 +39,14 @@ In this scenario the AI-Lab Docker container does not need access to the Docker The following command will * Download the Docker image for the specified version `$VERSION` of the AI-Lab if the image of the specified version is not yet available in your Docker service * Run a Docker container using this image -* Mount the volume `$VOLUME` to the directory `/root/notebooks` inside the container +* Mount the volume `$VOLUME` to the directory `/home/jupyter/notebooks` inside the container * If the volume does not exist yet, then it will be created automatically. * Forward port `49494` on the [daemon machine](prerequisites.md) to allow connections from all IP addresses matched by `$LISTEN_IP` ```shell docker run \ --name ${CONTAINER_NAME} \ - --volume ${VOLUME}:/root/notebooks \ + --volume ${VOLUME}:/home/jupyter/notebooks \ --publish ${LISTEN_IP}:49494:49494 \ exasol/ai-lab:${VERSION} ``` @@ -68,7 +68,7 @@ In this scenario you must enable the AI-Lab Docker container to access the Docke ```shell docker run \ --name ${CONTAINER_NAME} \ - --volume ${VOLUME}:/root/notebooks \ + --volume ${VOLUME}:/home/jupyter/notebooks \ --volume /var/run/docker.sock:/var/run/docker.sock \ --publish ${LISTEN_IP}:49494:49494 \ exasol/ai-lab:${VERSION} @@ -123,7 +123,7 @@ port to the same port then you can connect with http://localhost:49494. The default password is "ailab". To update the password, log in to the Docker container as the user root and run - /root/jupyterenv/bin/jupyter-lab server password + /home/jupyter/jupyterenv/bin/jupyter-lab server password ``` Using an internet browser you then can connect to the Jupyter server running in the Docker container in order to follow the tutorials presented by a set of Jupyter notebooks, see [Connecting to Jupyter Service](../jupyter.md#open-jupyter-in-your-browser). diff --git a/doc/user_guide/docker/managing-user-data.md b/doc/user_guide/docker/managing-user-data.md index 6d4f33d6..c222d52e 100644 --- a/doc/user_guide/docker/managing-user-data.md +++ b/doc/user_guide/docker/managing-user-data.md @@ -16,8 +16,8 @@ The Docker Edition of Exasol AI-Lab allows you to ## Basics In order to save your changes persistently, to reuse, backup, and restore them, the AI-Lab -* Keeps the notebook files as well as the SCS in the directory `/root/notebooks` inside the Docker container -* Uses a so-called [_Docker Volume_](https://docs.docker.com/storage/volumes) mounted to `/root/notebooks` to store the content independent of the Docker container +* Keeps the notebook files as well as the SCS in the directory `/home/jupyter/notebooks` inside the Docker container +* Uses a so-called [_Docker Volume_](https://docs.docker.com/storage/volumes) mounted to `/home/jupyter/notebooks` to store the content independent of the Docker container ## Additional Environment Variables diff --git a/doc/user_guide/jupyter.md b/doc/user_guide/jupyter.md index 3063354c..f1f9fec7 100644 --- a/doc/user_guide/jupyter.md +++ b/doc/user_guide/jupyter.md @@ -2,7 +2,7 @@ Root location * For Exasol AI-Lab's VM and AMI editions the root location is `$ROOT=/home/ubuntu`. -* For the Docker edition the root location is `$ROOT=/root`. +* For the Docker edition the root location is `$ROOT=/home/jupyter`. | Item | Location or value | |---------------------|-------------------------------------------| From 514cdd08beb690fe7b30cbd1753c079a22bec553 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 17:34:13 +0100 Subject: [PATCH 15/37] Updated tests for docker Edition --- test/integration/test_create_dss_docker_image.py | 4 ++-- test/unit/test_dss_docker_image.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/test_create_dss_docker_image.py b/test/integration/test_create_dss_docker_image.py index e2fa90ff..4e190915 100644 --- a/test/integration/test_create_dss_docker_image.py +++ b/test/integration/test_create_dss_docker_image.py @@ -66,7 +66,7 @@ def request_with_retry(url: str) -> requests.Response: def test_install_notebook_connector(dss_docker_container): container = dss_docker_container - command = '/root/jupyterenv/bin/python -c "import exasol.nb_connector.secret_store"' + command = '/home/jupyter/jupyterenv/bin/python -c "import exasol.nb_connector.secret_store"' exit_code, output = container.exec_run(command) output = output.decode('utf-8').strip() assert exit_code == 0, f'Got output "{output}".' @@ -77,7 +77,7 @@ def filename_set(string: str) -> Set[str]: return set(re.split(r'\s+', string.strip())) exit_code, output = dss_docker_container.exec_run( - "ls --indicator-style=slash /root/notebooks" + "ls --indicator-style=slash /home/jupyter/notebooks" ) output = output.decode('utf-8').strip() assert exit_code == 0, f'Got output "{output}".' diff --git a/test/unit/test_dss_docker_image.py b/test/unit/test_dss_docker_image.py index a361d97f..e9cf9dcc 100644 --- a/test/unit/test_dss_docker_image.py +++ b/test/unit/test_dss_docker_image.py @@ -75,7 +75,7 @@ def test_entrypoint_default(facts): def test_entrypoint_with_copy_args(): - jupyter = "/root/jupyterenv/bin/jupyter-lab" + jupyter = "/home/jupyter/jupyterenv/bin/jupyter-lab" port = "port" entrypoint = "/path/to/entrypoint.py" initial = "/path/to/initial" From 50b443220ea9edb817872ceb4fee3ad0df399788 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 17:34:35 +0100 Subject: [PATCH 16/37] Increased time out for commiting the Docker container As integration tests in test_create_dss_docker_image.py failed with timeout error in test_install_notebook_connector during docker.commit(). --- exasol/ds/sandbox/lib/dss_docker/create_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/ds/sandbox/lib/dss_docker/create_image.py b/exasol/ds/sandbox/lib/dss_docker/create_image.py index 99484568..5179be11 100644 --- a/exasol/ds/sandbox/lib/dss_docker/create_image.py +++ b/exasol/ds/sandbox/lib/dss_docker/create_image.py @@ -112,7 +112,7 @@ def _docker_file(self) -> importlib_resources.abc.Traversable: def _start_container(self) -> DockerContainer: self._start = datetime.now() - docker_client = docker.from_env() + docker_client = docker.from_env(timeout=600) try: return docker_client.containers.get(self.container_name) except: From b595ae663bcc1a23d569d3cd134df6f6179f2e39 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 27 Feb 2024 17:39:03 +0100 Subject: [PATCH 17/37] Added comment for timeout --- exasol/ds/sandbox/lib/dss_docker/create_image.py | 1 + 1 file changed, 1 insertion(+) diff --git a/exasol/ds/sandbox/lib/dss_docker/create_image.py b/exasol/ds/sandbox/lib/dss_docker/create_image.py index 5179be11..387aa87b 100644 --- a/exasol/ds/sandbox/lib/dss_docker/create_image.py +++ b/exasol/ds/sandbox/lib/dss_docker/create_image.py @@ -112,6 +112,7 @@ def _docker_file(self) -> importlib_resources.abc.Traversable: def _start_container(self) -> DockerContainer: self._start = datetime.now() + # default timeout is 60 seconds docker_client = docker.from_env(timeout=600) try: return docker_client.containers.get(self.container_name) From 97b56359b6d9807c13cd7c96b50156b8fe199a95 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 09:11:32 +0100 Subject: [PATCH 18/37] Show name of docker container when keeping it alive --- exasol/ds/sandbox/lib/dss_docker/create_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/ds/sandbox/lib/dss_docker/create_image.py b/exasol/ds/sandbox/lib/dss_docker/create_image.py index 387aa87b..be22e5f7 100644 --- a/exasol/ds/sandbox/lib/dss_docker/create_image.py +++ b/exasol/ds/sandbox/lib/dss_docker/create_image.py @@ -180,7 +180,7 @@ def _cleanup(self, container: DockerContainer): if container is None: return if self.keep_container: - _logger.info("Keeping container running") + _logger.info(f"Keeping container {self.container_name} running") return _logger.info("Stopping container") container.stop() From b8229da0b7a162faf27f0e80339a966cf077cada Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 09:24:46 +0100 Subject: [PATCH 19/37] Moved integration/test_create_dss_docker_image to codebuild as Github runner reported no space left on device. --- .../test_ci_create_dss_docker_image.py} | 6 ++++++ 1 file changed, 6 insertions(+) rename test/{integration/test_create_dss_docker_image.py => codebuild/test_ci_create_dss_docker_image.py} (85%) diff --git a/test/integration/test_create_dss_docker_image.py b/test/codebuild/test_ci_create_dss_docker_image.py similarity index 85% rename from test/integration/test_create_dss_docker_image.py rename to test/codebuild/test_ci_create_dss_docker_image.py index 4e190915..0ca55b47 100644 --- a/test/integration/test_create_dss_docker_image.py +++ b/test/codebuild/test_ci_create_dss_docker_image.py @@ -44,6 +44,8 @@ def retry(exception: typing.Type[BaseException], timeout: timedelta): ) +@pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', + reason="CI test need to be activated by env variable DSS_RUN_CI_TEST") def test_jupyterlab(dss_docker_container, jupyter_port): """" Test that jupyterlab is configured properly @@ -64,6 +66,8 @@ def request_with_retry(url: str) -> requests.Response: assert response.status_code == 200 +@pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', + reason="CI test need to be activated by env variable DSS_RUN_CI_TEST") def test_install_notebook_connector(dss_docker_container): container = dss_docker_container command = '/home/jupyter/jupyterenv/bin/python -c "import exasol.nb_connector.secret_store"' @@ -72,6 +76,8 @@ def test_install_notebook_connector(dss_docker_container): assert exit_code == 0, f'Got output "{output}".' +@pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', + reason="CI test need to be activated by env variable DSS_RUN_CI_TEST") def test_install_notebooks(dss_docker_container): def filename_set(string: str) -> Set[str]: return set(re.split(r'\s+', string.strip())) From f347df91a7ff3adb10d43ac01b4d6c7a6ff64aac Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 09:30:37 +0100 Subject: [PATCH 20/37] [CodeBuild] From e0a0caefbf6dd6b42183669715205270caece87c Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 09:43:37 +0100 Subject: [PATCH 21/37] Fixed import for ci tests --- test/codebuild/test_ci_create_dss_docker_image.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/codebuild/test_ci_create_dss_docker_image.py b/test/codebuild/test_ci_create_dss_docker_image.py index 0ca55b47..d085d890 100644 --- a/test/codebuild/test_ci_create_dss_docker_image.py +++ b/test/codebuild/test_ci_create_dss_docker_image.py @@ -1,6 +1,7 @@ import docker import io import pytest +import os import re import requests import sys @@ -45,7 +46,7 @@ def retry(exception: typing.Type[BaseException], timeout: timedelta): @pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', - reason="CI test need to be activated by env variable DSS_RUN_CI_TEST") + reason="CI tests need to be activated by env variable DSS_RUN_CI_TEST") def test_jupyterlab(dss_docker_container, jupyter_port): """" Test that jupyterlab is configured properly @@ -67,7 +68,7 @@ def request_with_retry(url: str) -> requests.Response: @pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', - reason="CI test need to be activated by env variable DSS_RUN_CI_TEST") + reason="CI tests need to be activated by env variable DSS_RUN_CI_TEST") def test_install_notebook_connector(dss_docker_container): container = dss_docker_container command = '/home/jupyter/jupyterenv/bin/python -c "import exasol.nb_connector.secret_store"' @@ -77,7 +78,7 @@ def test_install_notebook_connector(dss_docker_container): @pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', - reason="CI test need to be activated by env variable DSS_RUN_CI_TEST") + reason="CI tests need to be activated by env variable DSS_RUN_CI_TEST") def test_install_notebooks(dss_docker_container): def filename_set(string: str) -> Set[str]: return set(re.split(r'\s+', string.strip())) From dc426ad4e910f168b7baafc968a674b04d8422f1 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 09:44:05 +0100 Subject: [PATCH 22/37] [CodeBuild] From c254cee4e867c085da7bac61844aa78e98d50178 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 10:04:10 +0100 Subject: [PATCH 23/37] Updated dependencies to fix vulnerabilities Additionally added become directive to ansible task for Updating and upgrading apt packages. --- .../ds/sandbox/runtime/ansible/apt_update.yml | 3 +- poetry.lock | 140 +++++++++--------- 2 files changed, 76 insertions(+), 67 deletions(-) diff --git a/exasol/ds/sandbox/runtime/ansible/apt_update.yml b/exasol/ds/sandbox/runtime/ansible/apt_update.yml index 292bce0e..2eb35b7f 100644 --- a/exasol/ds/sandbox/runtime/ansible/apt_update.yml +++ b/exasol/ds/sandbox/runtime/ansible/apt_update.yml @@ -1,4 +1,5 @@ -- name: Update and upgrade apt packages +- name: Update and Upgrade apt Packages apt: upgrade: yes update_cache: yes + become: "{{need_sudo}}" diff --git a/poetry.lock b/poetry.lock index 87604ebb..08e81e24 100644 --- a/poetry.lock +++ b/poetry.lock @@ -13,17 +13,17 @@ files = [ [[package]] name = "ansible" -version = "9.2.0" +version = "9.3.0" description = "Radically simple IT automation" optional = false python-versions = ">=3.10" files = [ - {file = "ansible-9.2.0-py3-none-any.whl", hash = "sha256:39b19c252800aeed531413a626ccd07473b79615a3cea77568a1624c1aefaf7c"}, - {file = "ansible-9.2.0.tar.gz", hash = "sha256:a207a4a00a45e5cd178a7f94ca42afe26f23c9d27be49901ea8c45d18a07b7c6"}, + {file = "ansible-9.3.0-py3-none-any.whl", hash = "sha256:471993dd239611b4b6134e46911612f85639035f10d82b6c888528b5ffb3b16a"}, + {file = "ansible-9.3.0.tar.gz", hash = "sha256:7f4ea0e4d065538879b3e11e81e85eed4d802d1940f6564ad950e9d11a31b03c"}, ] [package.dependencies] -ansible-core = ">=2.16.3,<2.17.0" +ansible-core = ">=2.16.4,<2.17.0" [[package]] name = "ansible-core" @@ -181,17 +181,17 @@ css = ["tinycss2 (>=1.1.0,<1.3)"] [[package]] name = "boto3" -version = "1.34.50" +version = "1.34.51" description = "The AWS SDK for Python" optional = false python-versions = ">= 3.8" files = [ - {file = "boto3-1.34.50-py3-none-any.whl", hash = "sha256:8d709365231234bc4f0ca98fdf33a25eeebf78072853c6aa3d259f0f5cf09877"}, - {file = "boto3-1.34.50.tar.gz", hash = "sha256:290952be7899560039cb0042e8a2354f61a7dead0d0ca8bea6ba901930df0468"}, + {file = "boto3-1.34.51-py3-none-any.whl", hash = "sha256:67732634dc7d0afda879bd9a5e2d0818a2c14a98bef766b95a3e253ea5104cb9"}, + {file = "boto3-1.34.51.tar.gz", hash = "sha256:2cd9463e738a184cbce8a6824027c22163c5f73e277a35ff5aa0fb0e845b4301"}, ] [package.dependencies] -botocore = ">=1.34.50,<1.35.0" +botocore = ">=1.34.51,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -200,13 +200,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.50" +version = "1.34.51" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">= 3.8" files = [ - {file = "botocore-1.34.50-py3-none-any.whl", hash = "sha256:fda510559dbe796eefdb59561cc81be1b99afba3dee53fd23db9a3d587adc0ab"}, - {file = "botocore-1.34.50.tar.gz", hash = "sha256:33ab82cb96c4bb684f0dbafb071808e4817d83debc88b223e7d988256370c6d7"}, + {file = "botocore-1.34.51-py3-none-any.whl", hash = "sha256:01d5156247f991b3466a8404e3d7460a9ecbd9b214f9992d6ba797d9ddc6f120"}, + {file = "botocore-1.34.51.tar.gz", hash = "sha256:5086217442e67dd9de36ec7e87a0c663f76b7790d5fb6a12de565af95e87e319"}, ] [package.dependencies] @@ -464,47 +464,56 @@ test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] [[package]] name = "cryptography" -version = "41.0.7" +version = "42.0.5" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf"}, - {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d"}, - {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a"}, - {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15"}, - {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a"}, - {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1"}, - {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157"}, - {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406"}, - {file = "cryptography-41.0.7-cp37-abi3-win32.whl", hash = "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"}, - {file = "cryptography-41.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2"}, - {file = "cryptography-41.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960"}, - {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"}, - {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"}, - {file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"}, - {file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"}, - {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"}, - {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"}, - {file = "cryptography-41.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309"}, - {file = "cryptography-41.0.7.tar.gz", hash = "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc"}, + {file = "cryptography-42.0.5-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16"}, + {file = "cryptography-42.0.5-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da"}, + {file = "cryptography-42.0.5-cp37-abi3-win32.whl", hash = "sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74"}, + {file = "cryptography-42.0.5-cp37-abi3-win_amd64.whl", hash = "sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940"}, + {file = "cryptography-42.0.5-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30"}, + {file = "cryptography-42.0.5-cp39-abi3-win32.whl", hash = "sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413"}, + {file = "cryptography-42.0.5-cp39-abi3-win_amd64.whl", hash = "sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd"}, + {file = "cryptography-42.0.5.tar.gz", hash = "sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1"}, ] [package.dependencies] -cffi = ">=1.12" +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} [package.extras] docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] -docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] nox = ["nox"] -pep8test = ["black", "check-sdist", "mypy", "ruff"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] sdist = ["build"] ssh = ["bcrypt (>=3.1.5)"] -test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] [[package]] @@ -999,42 +1008,42 @@ files = [ [[package]] name = "localstack" -version = "3.1.0" +version = "3.2.0" description = "LocalStack - A fully functional local Cloud stack" optional = false python-versions = "*" files = [ - {file = "localstack-3.1.0.tar.gz", hash = "sha256:927487cccce7e6f50c8c49eff700ed70678724568df171b7961225f7fac363f5"}, + {file = "localstack-3.2.0.tar.gz", hash = "sha256:df50d82f5fa6ceae7e78d35bc9a46e00f4c5ae0c5ecc3563be9030a1199c7124"}, ] [package.dependencies] localstack-core = "*" -localstack-ext = "3.1.0" +localstack-ext = "3.2.0" [package.extras] -full = ["localstack-core[runtime]", "localstack-ext[runtime] (==3.1.0)"] -runtime = ["localstack-core[runtime]", "localstack-ext[runtime] (==3.1.0)"] +full = ["localstack-core[runtime]", "localstack-ext[runtime] (==3.2.0)"] +runtime = ["localstack-core[runtime]", "localstack-ext[runtime] (==3.2.0)"] [[package]] name = "localstack-core" -version = "3.1.0" +version = "3.2.0" description = "The core library and runtime of LocalStack" optional = false python-versions = ">=3.8" files = [ - {file = "localstack-core-3.1.0.tar.gz", hash = "sha256:0934f6b51115b9324c8601d91f202f740b57de7231dbc242ba8a6d90616cb94b"}, - {file = "localstack_core-3.1.0-py3-none-any.whl", hash = "sha256:2b6c0fb667181eea7da667e4a4ee8b9c50bf83ba42eb4ffff77f1e37799f1398"}, + {file = "localstack-core-3.2.0.tar.gz", hash = "sha256:2e22c4984ec37bf116b9ce88ccb528ba02e68d5c0e065ffa8ed5b79de2ddc036"}, + {file = "localstack_core-3.2.0-py3-none-any.whl", hash = "sha256:1427258bbd64a1ed473cf6b78ae9b1483322a083aa02e3df2b95746b274a0817"}, ] [package.dependencies] cachetools = ">=5.0" click = ">=7.1" -cryptography = "<42.0.0" +cryptography = "*" dill = "0.3.6" dnslib = ">=0.9.10" dnspython = ">=1.16.0" -plux = ">=1.3.1" -psutil = ">=5.4.8,<6.0.0" +plux = ">=1.7" +psutil = ">=5.4.8" python-dotenv = ">=0.19.1" pyyaml = ">=5.1" requests = ">=2.20.0" @@ -1044,42 +1053,41 @@ stevedore = ">=3.4.0" tailer = ">=0.4.1" [package.extras] -base-runtime = ["Quart (>=0.19.2)", "Werkzeug (>=3.0.0)", "awscrt (>=0.13.14)", "boto3 (>=1.26.121)", "botocore (>=1.34.12)", "cbor2 (>=5.2.0)", "dnspython (>=1.16.0)", "docker (>=6.1.1,<7.0.0)", "flask (>=3.0.0)", "hypercorn (>=0.14.4)", "jsonpatch (>=1.24,<2.0)", "pyopenssl (>=23.0.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "rolo (>=0.2,<0.3)", "urllib3 (>=2.0.7)", "xmltodict (>=0.13.0)"] -dev = ["Cython", "black (==23.10.0)", "coveralls (>=3.3.1)", "networkx (>=2.8.4)", "pandoc", "pre-commit (==3.5.0)", "pypandoc", "rstr (>=3.2.0)", "ruff (==0.1.0)"] -full = ["Quart (>=0.19.2)", "Werkzeug (>=3.0.0)", "airspeed-ext (>=0.6.3)", "amazon-kclpy (>=2.0.6,!=2.1.0)", "antlr4-python3-runtime (==4.13.1)", "apispec (>=5.1.1)", "aws-sam-translator (>=1.15.1)", "awscli (>=1.22.90)", "awscrt (>=0.13.14)", "boto3 (>=1.26.121)", "botocore (>=1.34.12)", "cbor2 (>=5.2.0)", "crontab (>=0.22.6)", "cryptography (>=41.0.5,<42.0.0)", "dnspython (>=1.16.0)", "docker (>=6.1.1,<7.0.0)", "flask (>=3.0.0)", "hypercorn (>=0.14.4)", "json5 (==0.9.11)", "jsonpatch (>=1.24,<2.0)", "jsonpath-ng (==1.5.3)", "jsonpath-rw (>=1.4.0,<2.0.0)", "jsonschema (<=4.19.0)", "moto-ext[all] (==5.0.0a1.post1)", "opensearch-py (>=2.4.1)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "rolo (>=0.2,<0.3)", "urllib3 (>=2.0.7)", "xmltodict (>=0.13.0)"] -runtime = ["Quart (>=0.19.2)", "Werkzeug (>=3.0.0)", "airspeed-ext (>=0.6.3)", "amazon-kclpy (>=2.0.6,!=2.1.0)", "antlr4-python3-runtime (==4.13.1)", "apispec (>=5.1.1)", "aws-sam-translator (>=1.15.1)", "awscli (>=1.22.90)", "awscrt (>=0.13.14)", "boto3 (>=1.26.121)", "botocore (>=1.34.12)", "cbor2 (>=5.2.0)", "crontab (>=0.22.6)", "cryptography (>=41.0.5,<42.0.0)", "dnspython (>=1.16.0)", "docker (>=6.1.1,<7.0.0)", "flask (>=3.0.0)", "hypercorn (>=0.14.4)", "json5 (==0.9.11)", "jsonpatch (>=1.24,<2.0)", "jsonpath-ng (==1.5.3)", "jsonpath-rw (>=1.4.0,<2.0.0)", "jsonschema (<=4.19.0)", "moto-ext[all] (==5.0.0a1.post1)", "opensearch-py (>=2.4.1)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "rolo (>=0.2,<0.3)", "urllib3 (>=2.0.7)", "xmltodict (>=0.13.0)"] -test = ["aws-cdk-lib (>=2.88.0)", "coverage[toml] (>=5.5)", "deepdiff (>=6.4.1)", "jsonpath-ng (>=1.5.3)", "pluggy (>=1.3.0)", "pytest (==7.4.2)", "pytest-httpserver (>=1.0.1)", "pytest-rerunfailures (==12.0)", "pytest-split (>=0.8.0)", "pytest-tinybird (>=0.2.0)", "websocket-client (>=1.7.0)"] -typehint = ["boto3-stubs[acm,acm-pca,amplify,apigateway,apigatewayv2,appconfig,appconfigdata,appsync,athena,autoscaling,backup,batch,ce,cloudcontrol,cloudformation,cloudfront,cloudtrail,cloudwatch,codecommit,cognito-identity,cognito-idp,dms,docdb,dynamodb,dynamodbstreams,ec2,ecr,ecs,efs,eks,elasticache,elasticbeanstalk,elbv2,emr,emr-serverless,es,events,firehose,fis,glacier,glue,iam,identitystore,iot,iot-data,iotanalytics,iotwireless,kafka,kinesis,kinesisanalytics,kinesisanalyticsv2,kms,lakeformation,lambda,logs,managedblockchain,mediaconvert,mediastore,mq,mwaa,neptune,opensearch,organizations,pi,qldb,qldb-session,rds,rds-data,redshift,redshift-data,resource-groups,resourcegroupstaggingapi,route53,route53resolver,s3,s3control,sagemaker,sagemaker-runtime,secretsmanager,serverlessrepo,servicediscovery,ses,sesv2,sns,sqs,ssm,stepfunctions,sts,timestream-query,timestream-write,transcribe,wafv2,xray]"] +base-runtime = ["Quart (>=0.19.2)", "Werkzeug (>=3.0.0)", "awscrt (>=0.13.14)", "boto3 (>=1.26.121)", "botocore (==1.34.49)", "cbor2 (>=5.2.0)", "dnspython (>=1.16.0)", "docker (>=6.1.1,<7.0.0)", "flask (>=3.0.0)", "hypercorn (>=0.14.4)", "jsonpatch (>=1.24)", "pyopenssl (>=23.0.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "rolo (>=0.3)", "urllib3 (>=2.0.7)", "xmltodict (>=0.13.0)"] +dev = ["Cython", "Quart (>=0.19.2)", "Werkzeug (>=3.0.0)", "airspeed-ext (>=0.6.3)", "amazon-kclpy (>=2.0.6,!=2.1.0)", "antlr4-python3-runtime (==4.13.1)", "apispec (>=5.1.1)", "aws-cdk-lib (>=2.88.0)", "aws-sam-translator (>=1.15.1)", "awscli (>=1.22.90)", "awscrt (>=0.13.14)", "black (>=23.10.0,<24)", "boto3 (>=1.26.121)", "botocore (==1.34.49)", "cbor2 (>=5.2.0)", "coverage[toml] (>=5.5)", "coveralls (>=3.3.1)", "crontab (>=0.22.6)", "cryptography (>=41.0.5)", "deepdiff (>=6.4.1)", "dnspython (>=1.16.0)", "docker (>=6.1.1,<7.0.0)", "flask (>=3.0.0)", "hypercorn (>=0.14.4)", "json5 (>=0.9.11)", "jsonpatch (>=1.24)", "jsonpath-ng (>=1.6.1)", "jsonpath-rw (>=1.4.0)", "localstack-snapshot (>=0.1.1)", "moto-ext[all] (==5.0.2.post1)", "networkx (>=2.8.4)", "opensearch-py (>=2.4.1)", "pandoc", "pluggy (>=1.3.0)", "pre-commit (>=3.5.0)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "pypandoc", "pytest (>=7.4.2)", "pytest-httpserver (>=1.0.1)", "pytest-rerunfailures (>=12.0)", "pytest-split (>=0.8.0)", "pytest-tinybird (>=0.2.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "rolo (>=0.3)", "rstr (>=3.2.0)", "ruff (>=0.1.0)", "urllib3 (>=2.0.7)", "websocket-client (>=1.7.0)", "xmltodict (>=0.13.0)"] +runtime = ["Quart (>=0.19.2)", "Werkzeug (>=3.0.0)", "airspeed-ext (>=0.6.3)", "amazon-kclpy (>=2.0.6,!=2.1.0)", "antlr4-python3-runtime (==4.13.1)", "apispec (>=5.1.1)", "aws-sam-translator (>=1.15.1)", "awscli (>=1.22.90)", "awscrt (>=0.13.14)", "boto3 (>=1.26.121)", "botocore (==1.34.49)", "cbor2 (>=5.2.0)", "crontab (>=0.22.6)", "cryptography (>=41.0.5)", "dnspython (>=1.16.0)", "docker (>=6.1.1,<7.0.0)", "flask (>=3.0.0)", "hypercorn (>=0.14.4)", "json5 (>=0.9.11)", "jsonpatch (>=1.24)", "jsonpath-ng (>=1.6.1)", "jsonpath-rw (>=1.4.0)", "moto-ext[all] (==5.0.2.post1)", "opensearch-py (>=2.4.1)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "rolo (>=0.3)", "urllib3 (>=2.0.7)", "xmltodict (>=0.13.0)"] +test = ["Quart (>=0.19.2)", "Werkzeug (>=3.0.0)", "airspeed-ext (>=0.6.3)", "amazon-kclpy (>=2.0.6,!=2.1.0)", "antlr4-python3-runtime (==4.13.1)", "apispec (>=5.1.1)", "aws-cdk-lib (>=2.88.0)", "aws-sam-translator (>=1.15.1)", "awscli (>=1.22.90)", "awscrt (>=0.13.14)", "boto3 (>=1.26.121)", "botocore (==1.34.49)", "cbor2 (>=5.2.0)", "coverage[toml] (>=5.5)", "crontab (>=0.22.6)", "cryptography (>=41.0.5)", "deepdiff (>=6.4.1)", "dnspython (>=1.16.0)", "docker (>=6.1.1,<7.0.0)", "flask (>=3.0.0)", "hypercorn (>=0.14.4)", "json5 (>=0.9.11)", "jsonpatch (>=1.24)", "jsonpath-ng (>=1.6.1)", "jsonpath-rw (>=1.4.0)", "localstack-snapshot (>=0.1.1)", "moto-ext[all] (==5.0.2.post1)", "opensearch-py (>=2.4.1)", "pluggy (>=1.3.0)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "pytest (>=7.4.2)", "pytest-httpserver (>=1.0.1)", "pytest-rerunfailures (>=12.0)", "pytest-split (>=0.8.0)", "pytest-tinybird (>=0.2.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "rolo (>=0.3)", "urllib3 (>=2.0.7)", "websocket-client (>=1.7.0)", "xmltodict (>=0.13.0)"] +typehint = ["Cython", "Quart (>=0.19.2)", "Werkzeug (>=3.0.0)", "airspeed-ext (>=0.6.3)", "amazon-kclpy (>=2.0.6,!=2.1.0)", "antlr4-python3-runtime (==4.13.1)", "apispec (>=5.1.1)", "aws-cdk-lib (>=2.88.0)", "aws-sam-translator (>=1.15.1)", "awscli (>=1.22.90)", "awscrt (>=0.13.14)", "black (>=23.10.0,<24)", "boto3 (>=1.26.121)", "boto3-stubs[acm,acm-pca,amplify,apigateway,apigatewayv2,appconfig,appconfigdata,application-autoscaling,appsync,athena,autoscaling,backup,batch,ce,cloudcontrol,cloudformation,cloudfront,cloudtrail,cloudwatch,codecommit,cognito-identity,cognito-idp,dms,docdb,dynamodb,dynamodbstreams,ec2,ecr,ecs,efs,eks,elasticache,elasticbeanstalk,elbv2,emr,emr-serverless,es,events,firehose,fis,glacier,glue,iam,identitystore,iot,iot-data,iotanalytics,iotwireless,kafka,kinesis,kinesisanalytics,kinesisanalyticsv2,kms,lakeformation,lambda,logs,managedblockchain,mediaconvert,mediastore,mq,mwaa,neptune,opensearch,organizations,pi,pipes,qldb,qldb-session,rds,rds-data,redshift,redshift-data,resource-groups,resourcegroupstaggingapi,route53,route53resolver,s3,s3control,sagemaker,sagemaker-runtime,secretsmanager,serverlessrepo,servicediscovery,ses,sesv2,sns,sqs,ssm,sso-admin,stepfunctions,sts,timestream-query,timestream-write,transcribe,wafv2,xray]", "botocore (==1.34.49)", "cbor2 (>=5.2.0)", "coverage[toml] (>=5.5)", "coveralls (>=3.3.1)", "crontab (>=0.22.6)", "cryptography (>=41.0.5)", "deepdiff (>=6.4.1)", "dnspython (>=1.16.0)", "docker (>=6.1.1,<7.0.0)", "flask (>=3.0.0)", "hypercorn (>=0.14.4)", "json5 (>=0.9.11)", "jsonpatch (>=1.24)", "jsonpath-ng (>=1.6.1)", "jsonpath-rw (>=1.4.0)", "localstack-snapshot (>=0.1.1)", "moto-ext[all] (==5.0.2.post1)", "networkx (>=2.8.4)", "opensearch-py (>=2.4.1)", "pandoc", "pluggy (>=1.3.0)", "pre-commit (>=3.5.0)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "pypandoc", "pytest (>=7.4.2)", "pytest-httpserver (>=1.0.1)", "pytest-rerunfailures (>=12.0)", "pytest-split (>=0.8.0)", "pytest-tinybird (>=0.2.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "rolo (>=0.3)", "rstr (>=3.2.0)", "ruff (>=0.1.0)", "urllib3 (>=2.0.7)", "websocket-client (>=1.7.0)", "xmltodict (>=0.13.0)"] [[package]] name = "localstack-ext" -version = "3.1.0" +version = "3.2.0" description = "Extensions for LocalStack" optional = false python-versions = ">=3.8" files = [ - {file = "localstack-ext-3.1.0.tar.gz", hash = "sha256:144f46f2b4fbc6e0f2f112184b4c6ce3cf340ab5dfdf8d835b9690ff7b2b66a2"}, + {file = "localstack-ext-3.2.0.tar.gz", hash = "sha256:e77a5bb7b90d698411b0b6fe388f202f0477701135f1928b6661b868fd7ff771"}, ] [package.dependencies] dill = ">=0.3.2" dnslib = ">=0.9.10" dnspython = ">=1.16.0" -localstack-core = "3.1.0" +localstack-core = "3.2.0" packaging = "*" plux = ">=1.5.0" pyaes = ">=1.6.0" python-dateutil = ">=2.8" -python-jose = {version = ">=3.1.0,<4.0.0", extras = ["cryptography"]} +python-jose = {version = ">=3.1.0", extras = ["cryptography"]} requests = ">=2.20.0" tabulate = "*" windows-curses = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] package = ["python-minifier"] -runtime = ["Js2Py (>=0.71)", "PyJWT (>=1.7.0)", "Whoosh (>=2.7.4)", "amazon.ion (>=0.9.3,<0.10)", "avro (>=1.11.0)", "aws-encryption-sdk (>=3.1.0)", "dirtyjson (==1.0.7)", "distro", "docker-registry-client (>=0.5.2)", "dulwich (>=0.19.16)", "graphql-core (>=3.0.3)", "janus (>=0.5.0)", "jsonpatch (>=1.32)", "kafka-python", "kubernetes (==21.7.0)", "localstack-core[runtime] (==3.1.0)", "paho-mqtt (>=1.5)", "parquet[snappy] (>=1.3.1)", "parse (==1.19.0)", "pg8000 (>=1.10)", "postgres (>=2.2.2)", "postgresql-proxy (>=0.2.0)", "pproxy-ext (>=2.7.9)", "presto-python-client (>=0.7.0)", "pure-sasl (>=0.6.2)", "pyftpdlib (>=1.5.6)", "pyhive[hive-pure-sasl] (>=0.7.0)", "pyion2json (>=0.0.2)", "pymysql", "pyqldb (>=3.2,<4.0)", "python-snappy (>=0.6)", "readerwriterlock (>=1.0.7)", "redis (>=5.0)", "rsa (>=4.0)", "sql-metadata (>=2.6.0)", "sqlglot", "srp-ext (==1.0.7.1)", "testing.common.database (>=1.1.0)", "thrift (>=0.10.0)", "thrift_sasl (>=0.1.0)", "tornado (>=6.0)", "warrant-ext (>=0.6.2)", "websockets (>=8.1)"] -test = ["Js2Py (>=0.71)", "PyAthena[pandas]", "aiohttp", "async-timeout", "aws-cdk-lib (>=2.88.0)", "aws_cdk.aws_neptune_alpha", "aws_cdk.aws_redshift_alpha", "aws_xray_sdk (>=2.4.2)", "awswrangler (>=2.19.0)", "black (==23.10.0)", "coverage[toml] (>=5.0.0)", "coveralls (>=3.3.1)", "deepdiff (>=5.5.0)", "gremlinpython", "jws (>=0.1.3)", "localstack-core[test] (==3.1.0)", "lxml (>=4.6.5,<5.0)", "msal", "msal-extensions", "msrest", "mysql-connector-python", "neo4j", "nest-asyncio (>=1.4.1)", "paramiko (>=2.11.0,<2.12.0)", "portalocker", "pre-commit (==3.5.0)", "pyarrow", "pymongo", "pymssql (>=2.2.8)", "pytest-instafail (>=0.4.2)", "python-terraform", "redshift_connector", "ruff (==0.1.0)", "stomp.py (==8.0.1)"] -typehint = ["boto3-stubs[acm,amplify,apigateway,apigatewayv2,appconfig,appsync,athena,autoscaling,backup,batch,ce,cloudcontrol,cloudformation,cloudfront,cloudtrail,cloudwatch,codecommit,cognito-identity,cognito-idp,dms,docdb,dynamodb,dynamodbstreams,ec2,ecr,ecs,efs,eks,elasticache,elasticbeanstalk,elbv2,emr,emr-serverless,es,events,firehose,fis,glacier,glue,iam,iot,iot-data,iotanalytics,iotwireless,kafka,kinesis,kinesisanalytics,kinesisanalyticsv2,kms,lakeformation,lambda,logs,mediaconvert,mediastore,mq,mwaa,neptune,opensearch,organizations,pi,qldb,qldb-session,rds,rds-data,redshift,redshift-data,resource-groups,resourcegroupstaggingapi,route53,route53resolver,s3,s3control,sagemaker,sagemaker-runtime,secretsmanager,serverlessrepo,servicediscovery,ses,sesv2,sns,sqs,ssm,sso-admin,stepfunctions,sts,timestream-query,timestream-write,transcribe,xray]"] +runtime = ["Js2Py (>=0.71)", "PyJWT (>=1.7.0)", "Whoosh (>=2.7.4)", "amazon.ion (>=0.9.3)", "avro (>=1.11.0)", "aws-encryption-sdk (>=3.1.0)", "dirtyjson (>=1.0.7)", "distro", "docker-registry-client (>=0.5.2)", "dulwich (>=0.19.16)", "graphql-core (>=3.0.3)", "janus (>=0.5.0)", "jsonpatch (>=1.32)", "kafka-python", "kubernetes (>=21.7.0)", "localstack-core[runtime] (==3.2.0)", "paho-mqtt (>=1.5)", "parquet[snappy] (>=1.3.1)", "parse (>=1.19.0)", "pg8000 (>=1.10)", "postgres (>=2.2.2)", "postgresql-proxy (>=0.2.0)", "pproxy-ext (>=2.7.9)", "presto-python-client (>=0.7.0)", "pure-sasl (>=0.6.2)", "pyftpdlib (>=1.5.6)", "pyhive[hive-pure-sasl] (>=0.7.0)", "pyion2json (>=0.0.2)", "pymysql", "pyqldb (>=3.2,<4.0)", "python-snappy (>=0.6)", "readerwriterlock (>=1.0.7)", "redis (>=5.0)", "rsa (>=4.0)", "sql-metadata (>=2.6.0)", "sqlglot", "srp-ext (>=1.0.7.1)", "testing.common.database (>=1.1.0)", "thrift (>=0.10.0)", "thrift_sasl (>=0.1.0)", "tornado (>=6.0)", "warrant-ext (>=0.6.2)", "websockets (>=8.1)"] +test = ["Js2Py (>=0.71)", "Js2Py (>=0.71)", "PyAthena[pandas]", "PyJWT (>=1.7.0)", "Whoosh (>=2.7.4)", "aiohttp", "amazon.ion (>=0.9.3)", "async-timeout", "avro (>=1.11.0)", "aws-cdk-lib (>=2.88.0)", "aws-encryption-sdk (>=3.1.0)", "aws_cdk.aws_neptune_alpha", "aws_cdk.aws_redshift_alpha", "aws_xray_sdk (>=2.4.2)", "awswrangler (>=3.5.2)", "black (>=23.10.0,<24)", "coverage[toml] (>=5.0.0)", "coveralls (>=3.3.1)", "deepdiff (>=5.5.0)", "dirtyjson (>=1.0.7)", "distro", "docker-registry-client (>=0.5.2)", "dulwich (>=0.19.16)", "graphql-core (>=3.0.3)", "gremlinpython", "janus (>=0.5.0)", "jsonpatch (>=1.32)", "jws (>=0.1.3)", "kafka-python", "kubernetes (>=21.7.0)", "localstack-core[runtime] (==3.2.0)", "localstack-core[test] (==3.2.0)", "msal", "msal-extensions", "msrest", "mysql-connector-python", "neo4j", "nest-asyncio (>=1.4.1)", "paho-mqtt (>=1.5)", "paramiko (>=2.11.0,<2.12.0)", "parquet[snappy] (>=1.3.1)", "parse (>=1.19.0)", "pg8000 (>=1.10)", "portalocker", "postgres (>=2.2.2)", "postgresql-proxy (>=0.2.0)", "pproxy-ext (>=2.7.9)", "pre-commit (>=3.5.0)", "presto-python-client (>=0.7.0)", "pure-sasl (>=0.6.2)", "pyarrow", "pyftpdlib (>=1.5.6)", "pyhive[hive-pure-sasl] (>=0.7.0)", "pyion2json (>=0.0.2)", "pymongo", "pymssql (>=2.2.8)", "pymysql", "pyqldb (>=3.2,<4.0)", "pytest-instafail (>=0.4.2)", "python-snappy (>=0.6)", "python-terraform", "readerwriterlock (>=1.0.7)", "redis (>=5.0)", "redshift_connector", "rsa (>=4.0)", "ruff (>=0.1.0)", "sql-metadata (>=2.6.0)", "sqlglot", "srp-ext (>=1.0.7.1)", "stomp.py (>=8.0.1)", "testing.common.database (>=1.1.0)", "thrift (>=0.10.0)", "thrift_sasl (>=0.1.0)", "tornado (>=6.0)", "warrant-ext (>=0.6.2)", "websockets (>=8.1)"] +typehint = ["Js2Py (>=0.71)", "Js2Py (>=0.71)", "PyAthena[pandas]", "PyJWT (>=1.7.0)", "Whoosh (>=2.7.4)", "aiohttp", "amazon.ion (>=0.9.3)", "async-timeout", "avro (>=1.11.0)", "aws-cdk-lib (>=2.88.0)", "aws-encryption-sdk (>=3.1.0)", "aws_cdk.aws_neptune_alpha", "aws_cdk.aws_redshift_alpha", "aws_xray_sdk (>=2.4.2)", "awswrangler (>=3.5.2)", "black (>=23.10.0,<24)", "boto3-stubs[acm,amplify,apigateway,apigatewayv2,appconfig,appsync,athena,autoscaling,backup,batch,ce,cloudcontrol,cloudformation,cloudfront,cloudtrail,cloudwatch,codecommit,cognito-identity,cognito-idp,dms,docdb,dynamodb,dynamodbstreams,ec2,ecr,ecs,efs,eks,elasticache,elasticbeanstalk,elbv2,emr,emr-serverless,es,events,firehose,fis,glacier,glue,iam,iot,iot-data,iotanalytics,iotwireless,kafka,kinesis,kinesisanalytics,kinesisanalyticsv2,kms,lakeformation,lambda,logs,mediaconvert,mediastore,mq,mwaa,neptune,opensearch,organizations,pi,qldb,qldb-session,rds,rds-data,redshift,redshift-data,resource-groups,resourcegroupstaggingapi,route53,route53resolver,s3,s3control,sagemaker,sagemaker-runtime,secretsmanager,serverlessrepo,servicediscovery,ses,sesv2,sns,sqs,ssm,sso-admin,stepfunctions,sts,timestream-query,timestream-write,transcribe,xray]", "coverage[toml] (>=5.0.0)", "coveralls (>=3.3.1)", "deepdiff (>=5.5.0)", "dirtyjson (>=1.0.7)", "distro", "docker-registry-client (>=0.5.2)", "dulwich (>=0.19.16)", "graphql-core (>=3.0.3)", "gremlinpython", "janus (>=0.5.0)", "jsonpatch (>=1.32)", "jws (>=0.1.3)", "kafka-python", "kubernetes (>=21.7.0)", "localstack-core[runtime] (==3.2.0)", "localstack-core[test] (==3.2.0)", "msal", "msal-extensions", "msrest", "mysql-connector-python", "neo4j", "nest-asyncio (>=1.4.1)", "paho-mqtt (>=1.5)", "paramiko (>=2.11.0,<2.12.0)", "parquet[snappy] (>=1.3.1)", "parse (>=1.19.0)", "pg8000 (>=1.10)", "portalocker", "postgres (>=2.2.2)", "postgresql-proxy (>=0.2.0)", "pproxy-ext (>=2.7.9)", "pre-commit (>=3.5.0)", "presto-python-client (>=0.7.0)", "pure-sasl (>=0.6.2)", "pyarrow", "pyftpdlib (>=1.5.6)", "pyhive[hive-pure-sasl] (>=0.7.0)", "pyion2json (>=0.0.2)", "pymongo", "pymssql (>=2.2.8)", "pymysql", "pyqldb (>=3.2,<4.0)", "pytest-instafail (>=0.4.2)", "python-snappy (>=0.6)", "python-terraform", "readerwriterlock (>=1.0.7)", "redis (>=5.0)", "redshift_connector", "rsa (>=4.0)", "ruff (>=0.1.0)", "sql-metadata (>=2.6.0)", "sqlglot", "srp-ext (>=1.0.7.1)", "stomp.py (>=8.0.1)", "testing.common.database (>=1.1.0)", "thrift (>=0.10.0)", "thrift_sasl (>=0.1.0)", "tornado (>=6.0)", "warrant-ext (>=0.6.2)", "websockets (>=8.1)"] [[package]] name = "lockfile" @@ -1564,13 +1572,13 @@ files = [ [[package]] name = "pydantic" -version = "2.6.2" +version = "2.6.3" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.6.2-py3-none-any.whl", hash = "sha256:37a5432e54b12fecaa1049c5195f3d860a10e01bdfd24f1840ef14bd0d3aeab3"}, - {file = "pydantic-2.6.2.tar.gz", hash = "sha256:a09be1c3d28f3abe37f8a78af58284b236a92ce520105ddc91a6d29ea1176ba7"}, + {file = "pydantic-2.6.3-py3-none-any.whl", hash = "sha256:72c6034df47f46ccdf81869fddb81aade68056003900a8724a4f160700016a2a"}, + {file = "pydantic-2.6.3.tar.gz", hash = "sha256:e07805c4c7f5c6826e33a1d4c9d47950d7eaf34868e2690f8594d2e30241f11f"}, ] [package.dependencies] From 0235d4dc26672f39608abed9e1e9ad630a9a3f33 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 10:04:37 +0100 Subject: [PATCH 24/37] [CodeBuild] From 92781ad8efe0b025ae17e28c92e29663a752fb37 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 10:23:53 +0100 Subject: [PATCH 25/37] Added free disk space to workflows/check_ci.yaml moved codebuild/test_ci_create_dss_docker_image.py back to normal ci tests --- .github/workflows/check_ci.yaml | 20 +++++++++++++++++++ .../test_create_dss_docker_image.py} | 0 2 files changed, 20 insertions(+) rename test/{codebuild/test_ci_create_dss_docker_image.py => integration/test_create_dss_docker_image.py} (100%) diff --git a/.github/workflows/check_ci.yaml b/.github/workflows/check_ci.yaml index 97619e8b..36c60a88 100644 --- a/.github/workflows/check_ci.yaml +++ b/.github/workflows/check_ci.yaml @@ -14,6 +14,25 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + large-packages: false + + - name: Free disk space by removing large directories + run: | + sudo rm -rf /usr/local/graalvm/ + sudo rm -rf /usr/local/.ghcup/ + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /usr/local/lib/node_modules + sudo rm -rf /opt/ghc + + - name: Show available disk space + run: df -h + - name: Setup Python & Poetry Environment uses: ./.github/actions/prepare_poetry_env @@ -39,6 +58,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Free disk space uses: jlumbroso/free-disk-space@main with: diff --git a/test/codebuild/test_ci_create_dss_docker_image.py b/test/integration/test_create_dss_docker_image.py similarity index 100% rename from test/codebuild/test_ci_create_dss_docker_image.py rename to test/integration/test_create_dss_docker_image.py From 62004a38b334cf5b14b0ac832f4d0f28f657d629 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 12:39:22 +0100 Subject: [PATCH 26/37] Use user "jupyter" for EC2, too --- exasol/ds/sandbox/lib/dss_docker/create_image.py | 3 +-- .../ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml | 6 +++--- exasol/ds/sandbox/runtime/ansible/ec2_playbook.yml | 5 +++-- .../jupyter/templates/etc/systemd/system/jupyter.service | 2 +- .../roles/jupyter/templates/etc/update-motd.d/999-jupyter | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/exasol/ds/sandbox/lib/dss_docker/create_image.py b/exasol/ds/sandbox/lib/dss_docker/create_image.py index be22e5f7..a5d11e21 100644 --- a/exasol/ds/sandbox/lib/dss_docker/create_image.py +++ b/exasol/ds/sandbox/lib/dss_docker/create_image.py @@ -112,8 +112,7 @@ def _docker_file(self) -> importlib_resources.abc.Traversable: def _start_container(self) -> DockerContainer: self._start = datetime.now() - # default timeout is 60 seconds - docker_client = docker.from_env(timeout=600) + docker_client = docker.from_env() try: return docker_client.containers.get(self.container_name) except: diff --git a/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml b/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml index 12a44219..6ea0d3c7 100644 --- a/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml +++ b/exasol/ds/sandbox/runtime/ansible/ai_lab_docker_playbook.yml @@ -14,11 +14,11 @@ hosts: docker_container_group gather_facts: true vars: - my_ansible_user: "ansible" + my_ansible_user: "ubuntu" jupyter_user: "jupyter" jupyter_group: "jupyter" jupyter_user_home: "/home/jupyter" - need_sudo: yes + need_sudo: no tasks: - import_tasks: apt_update.yml - name: Ansible Access @@ -27,7 +27,7 @@ - name: Setup AI-Lab Docker Container hosts: docker_container_group - remote_user: "ansible" + remote_user: "ubuntu" gather_facts: true vars: ansible_python_interpreter: python3 diff --git a/exasol/ds/sandbox/runtime/ansible/ec2_playbook.yml b/exasol/ds/sandbox/runtime/ansible/ec2_playbook.yml index 087533a4..fa3cd2a0 100644 --- a/exasol/ds/sandbox/runtime/ansible/ec2_playbook.yml +++ b/exasol/ds/sandbox/runtime/ansible/ec2_playbook.yml @@ -2,8 +2,9 @@ hosts: ec2 vars: ansible_python_interpreter: /usr/bin/python3 - user_name: ubuntu - user_home: /home/ubuntu + user_name: jupyter + user_home: /home/jupyter + user_group: jupyter initial_notebook_folder: "{{ user_home }}/notebooks" need_sudo: yes remote_user: ubuntu diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/systemd/system/jupyter.service b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/systemd/system/jupyter.service index 6d225971..ee235e3a 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/systemd/system/jupyter.service +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/systemd/system/jupyter.service @@ -5,7 +5,7 @@ Description=Jupyter Notebook Type=simple PIDFile=/run/jupyter.pid ExecStart="{{jupyterlab_command}}" --notebook-dir="{{jupyterlab_notebook_folder}}" --no-browser -User={{user_name}} +User={{ user_name }} Restart=always RestartSec=10 diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter index 40cb01c2..c24eaa7b 100755 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter @@ -11,7 +11,7 @@ You can connect with https://:{{jupyterlab_port}}. {{heading_jupyter_update_password}} The default password is "{{jupyterlab_password}}". -To update the password as user {{user_name}} run +To update the password as user {{ user_name }} run {{jupyterlab_virtualenv}}/bin/jupyter server password """ From 3c1938833ce2e2e4b3d9373aa6760a60b3cb6971 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 12:47:13 +0100 Subject: [PATCH 27/37] Updated documentation --- doc/developer_guide/aws.md | 2 +- doc/user_guide/ami-usage.md | 1 + doc/user_guide/jupyter.md | 8 ++------ doc/user_guide/login-vm-and-ami.md | 2 ++ 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/doc/developer_guide/aws.md b/doc/developer_guide/aws.md index 5079cf69..ed65a6a9 100644 --- a/doc/developer_guide/aws.md +++ b/doc/developer_guide/aws.md @@ -32,7 +32,7 @@ After the export has finished, the cloudformation stack and the keypair is remov Installs all dependencies via Ansible: * installs and configures Jupyter -* installs Docker and adds the user `ubuntu` to the docker group +* installs Docker and adds the user `jupyter` to the docker group * changes the netplan configuration. This is necessary to have proper network configuration when running the VM image Finally, the default password will be set, and also the password will be marked as expired, such that the user will be forced to enter a new password during initial login. diff --git a/doc/user_guide/ami-usage.md b/doc/user_guide/ami-usage.md index 3db67679..00489489 100644 --- a/doc/user_guide/ami-usage.md +++ b/doc/user_guide/ami-usage.md @@ -37,6 +37,7 @@ Check the [AWS documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuid - For the storage we recommend to keep the pre-selected 100GB volume - Click button "Launch instance" 6. As soon as the machine becomes available you can connect per ssh with user `ubuntu`: `ssh -i your_key.pem ubuntu@the_new_ec_instance` + ## Login diff --git a/doc/user_guide/jupyter.md b/doc/user_guide/jupyter.md index f1f9fec7..277aac76 100644 --- a/doc/user_guide/jupyter.md +++ b/doc/user_guide/jupyter.md @@ -1,13 +1,9 @@ ## Open Jupyter In Your Browser -Root location -* For Exasol AI-Lab's VM and AMI editions the root location is `$ROOT=/home/ubuntu`. -* For the Docker edition the root location is `$ROOT=/home/jupyter`. - | Item | Location or value | |---------------------|-------------------------------------------| -| Virtual environment | location `/$ROOT/jupyterenv` | -| Location notebooks | location `/$ROOT/notebooks` | +| Virtual environment | location `/home/jupyter/jupyterenv` | +| Location notebooks | location `/home/jupyter/notebooks` | | Password | `ailab` | | HTTP Port | `49494` (or the port you forwarded it to) | diff --git a/doc/user_guide/login-vm-and-ami.md b/doc/user_guide/login-vm-and-ami.md index 18ab807f..ff5930a7 100644 --- a/doc/user_guide/login-vm-and-ami.md +++ b/doc/user_guide/login-vm-and-ami.md @@ -1,5 +1,7 @@ ## Login to AMI and VM Editions + + Username: **ubuntu** At the first login to the AI-Lab (image or AMI) you will be prompted to change your password. From d9d62058505e4d0412bb10d7c7730d4e21dea4b4 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 12:47:30 +0100 Subject: [PATCH 28/37] [CodeBuild] From 7ee7abf7892c080b280be6cab5a68242c23ae06f Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 13:10:22 +0100 Subject: [PATCH 29/37] Updated dependency network-manager [run-notebook-tests] --- .../ds/sandbox/runtime/ansible/roles/netplan/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/ds/sandbox/runtime/ansible/roles/netplan/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/netplan/defaults/main.yml index dd67904a..106d9cb6 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/netplan/defaults/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/netplan/defaults/main.yml @@ -1,4 +1,4 @@ --- apt_dependencies: - - network-manager=1.22.10-1ubuntu2.3 + - network-manager=1.22.10-1ubuntu2.4 From 887d558c45af780a45c6c7f10caaba7ca79b3c84 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 13:14:16 +0100 Subject: [PATCH 30/37] [CodeBuild] From 8ace780daa2813b435ce83abe4b8c1df78d20ef3 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 14:05:27 +0100 Subject: [PATCH 31/37] Fixed test test_jupyter_password_message_shown [CodeBuild] --- test/codebuild/test_ci.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/codebuild/test_ci.py b/test/codebuild/test_ci.py index 32788233..ae6d732e 100644 --- a/test/codebuild/test_ci.py +++ b/test/codebuild/test_ci.py @@ -94,6 +94,7 @@ def new_ec2_from_ami(): status = aws_access.get_instance_status(ec2_instance_description.id) assert status.ok time.sleep(10) + # probably we need to set the password for user "jupyter" here. change_password(host=ec2_instance_description.public_dns_name, user='ubuntu', curr_pass=default_password, new_password=new_password) yield ec2_instance_description.public_dns_name, new_password, default_password @@ -162,7 +163,7 @@ def test_jupyter_password_message_shown(new_ec2_from_ami): prompts = ((r"Enter password: ", f"{random_jupyter_password}\n"), (r"Verify password: ", f"{random_jupyter_password}\n")) responders = [Responder(pattern=prompt, response=response) for prompt, response in prompts] - res = con.run("./jupyterenv/bin/jupyter server password", + res = con.run("sudo --login --user=jupyter ./jupyterenv/bin/jupyter server password", watchers=responders, pty=True) assert res.ok From 427696be7b3dc2aa59187efe532d620e8ac0fa7e Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 14:42:35 +0100 Subject: [PATCH 32/37] Removed comment --- doc/user_guide/login-vm-and-ami.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/user_guide/login-vm-and-ami.md b/doc/user_guide/login-vm-and-ami.md index ff5930a7..41762913 100644 --- a/doc/user_guide/login-vm-and-ami.md +++ b/doc/user_guide/login-vm-and-ami.md @@ -1,7 +1,6 @@ ## Login to AMI and VM Editions - Username: **ubuntu** At the first login to the AI-Lab (image or AMI) you will be prompted to change your password. From 524656a0b47bfb3ae89d8688fc5645fc4674f5f5 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 15:41:36 +0100 Subject: [PATCH 33/37] Fixed review findings --- doc/user_guide/ami-usage.md | 3 ++- doc/user_guide/docker/docker-usage.md | 2 +- test/codebuild/test_ci.py | 1 - test/integration/test_create_dss_docker_image.py | 7 ------- test/integration/test_motd_jupyter_template.py | 1 - 5 files changed, 3 insertions(+), 11 deletions(-) diff --git a/doc/user_guide/ami-usage.md b/doc/user_guide/ami-usage.md index 00489489..70151363 100644 --- a/doc/user_guide/ami-usage.md +++ b/doc/user_guide/ami-usage.md @@ -37,7 +37,8 @@ Check the [AWS documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuid - For the storage we recommend to keep the pre-selected 100GB volume - Click button "Launch instance" 6. As soon as the machine becomes available you can connect per ssh with user `ubuntu`: `ssh -i your_key.pem ubuntu@the_new_ec_instance` - + +**Please note**: When you want to modify the settings of jupyterlab or to install additional pip packages into the virtual env `jupyterenv`, you need to prefix your commands with `sudo --login --user jupyter`. ## Login diff --git a/doc/user_guide/docker/docker-usage.md b/doc/user_guide/docker/docker-usage.md index 349dda51..5682b556 100644 --- a/doc/user_guide/docker/docker-usage.md +++ b/doc/user_guide/docker/docker-usage.md @@ -122,7 +122,7 @@ port to the same port then you can connect with http://localhost:49494. └─┘┴ ─┴┘┴ ┴ ┴ └─┘ ┴ └─┘└─┘┴└─ └┘└─┘┴ ┴ ┴ └─┘┴└─ ┴ ┴ ┴└─┘└─┘└┴┘└─┘┴└──┴┘ o The default password is "ailab". -To update the password, log in to the Docker container as the user root and run +To update the password, log in to the Docker container as the user jupyter and run /home/jupyter/jupyterenv/bin/jupyter-lab server password ``` diff --git a/test/codebuild/test_ci.py b/test/codebuild/test_ci.py index ae6d732e..ef1f3e3e 100644 --- a/test/codebuild/test_ci.py +++ b/test/codebuild/test_ci.py @@ -94,7 +94,6 @@ def new_ec2_from_ami(): status = aws_access.get_instance_status(ec2_instance_description.id) assert status.ok time.sleep(10) - # probably we need to set the password for user "jupyter" here. change_password(host=ec2_instance_description.public_dns_name, user='ubuntu', curr_pass=default_password, new_password=new_password) yield ec2_instance_description.public_dns_name, new_password, default_password diff --git a/test/integration/test_create_dss_docker_image.py b/test/integration/test_create_dss_docker_image.py index d085d890..4e190915 100644 --- a/test/integration/test_create_dss_docker_image.py +++ b/test/integration/test_create_dss_docker_image.py @@ -1,7 +1,6 @@ import docker import io import pytest -import os import re import requests import sys @@ -45,8 +44,6 @@ def retry(exception: typing.Type[BaseException], timeout: timedelta): ) -@pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', - reason="CI tests need to be activated by env variable DSS_RUN_CI_TEST") def test_jupyterlab(dss_docker_container, jupyter_port): """" Test that jupyterlab is configured properly @@ -67,8 +64,6 @@ def request_with_retry(url: str) -> requests.Response: assert response.status_code == 200 -@pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', - reason="CI tests need to be activated by env variable DSS_RUN_CI_TEST") def test_install_notebook_connector(dss_docker_container): container = dss_docker_container command = '/home/jupyter/jupyterenv/bin/python -c "import exasol.nb_connector.secret_store"' @@ -77,8 +72,6 @@ def test_install_notebook_connector(dss_docker_container): assert exit_code == 0, f'Got output "{output}".' -@pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', - reason="CI tests need to be activated by env variable DSS_RUN_CI_TEST") def test_install_notebooks(dss_docker_container): def filename_set(string: str) -> Set[str]: return set(re.split(r'\s+', string.strip())) diff --git a/test/integration/test_motd_jupyter_template.py b/test/integration/test_motd_jupyter_template.py index 591da688..a646c91d 100644 --- a/test/integration/test_motd_jupyter_template.py +++ b/test/integration/test_motd_jupyter_template.py @@ -11,7 +11,6 @@ @pytest.fixture() def motd_file(tmp_path): - # jupyterlab_config_json config_file = tmp_path / "jupyter_server_config.json" python_file = tmp_path / "999_jupyter.py" From dfb6b5ee1c87505c1bc648db025478d5ea02bef9 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 28 Feb 2024 16:03:26 +0100 Subject: [PATCH 34/37] Fixed review findings --- exasol/ds/sandbox/lib/dss_docker/create_image.py | 3 ++- exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/exasol/ds/sandbox/lib/dss_docker/create_image.py b/exasol/ds/sandbox/lib/dss_docker/create_image.py index a5d11e21..2ff6619a 100644 --- a/exasol/ds/sandbox/lib/dss_docker/create_image.py +++ b/exasol/ds/sandbox/lib/dss_docker/create_image.py @@ -112,7 +112,8 @@ def _docker_file(self) -> importlib_resources.abc.Traversable: def _start_container(self) -> DockerContainer: self._start = datetime.now() - docker_client = docker.from_env() + # default timeout is 60 seconds. + docker_client = docker.from_env(600) try: return docker_client.containers.get(self.container_name) except: diff --git a/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml b/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml index a3cd0223..88380567 100644 --- a/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml +++ b/exasol/ds/sandbox/runtime/ansible/general_setup_tasks.yml @@ -1,4 +1,4 @@ -- name: Set facts for entry point +- name: Set facts for Entry Point vars: jupyter_virtualenv: "{{user_home}}/jupyterenv" ansible.builtin.set_fact: @@ -41,9 +41,7 @@ group: "{{ user_group }}" recurse: true become: "{{ need_sudo }}" -- name: Clear pip cache - # TBD: do we need become: "{{ need_sudo }}" here? - # should folder /root be replaced by {{ user_home }} ? +- name: Clear pip Cache ansible.builtin.file: path: /root/.cache/pip state: absent From 0574c9595548d76454f4dde74a17fcbf8285a405 Mon Sep 17 00:00:00 2001 From: Christoph Kuhnke Date: Thu, 29 Feb 2024 11:01:24 +0100 Subject: [PATCH 35/37] Update exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter Co-authored-by: Torsten Kilias --- .../roles/jupyter/templates/etc/update-motd.d/999-jupyter | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter index c24eaa7b..b0fb571b 100755 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter @@ -11,7 +11,7 @@ You can connect with https://:{{jupyterlab_port}}. {{heading_jupyter_update_password}} The default password is "{{jupyterlab_password}}". -To update the password as user {{ user_name }} run +To update the password run as user {{ user_name }} the following command {{jupyterlab_virtualenv}}/bin/jupyter server password """ From 1778e812c9399104879ca65abc07d121d51c4e9d Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 29 Feb 2024 11:03:18 +0100 Subject: [PATCH 36/37] Fixed review finding --- .../roles/jupyter/templates/etc/update-motd.d/999-jupyter | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter index b0fb571b..34f3dc71 100755 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter @@ -11,8 +11,8 @@ You can connect with https://:{{jupyterlab_port}}. {{heading_jupyter_update_password}} The default password is "{{jupyterlab_password}}". -To update the password run as user {{ user_name }} the following command - {{jupyterlab_virtualenv}}/bin/jupyter server password +To update the password run the following command + sudo --login --user {{ user_name }} {{jupyterlab_virtualenv}}/bin/jupyter server password """ server_config_file = pathlib.Path("{{jupyterlab_config_json}}") From b10fa2ecfee7ea35a6997079798be5626205b167 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 29 Feb 2024 11:19:34 +0100 Subject: [PATCH 37/37] Fixed failing test --- exasol/ds/sandbox/lib/dss_docker/create_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/ds/sandbox/lib/dss_docker/create_image.py b/exasol/ds/sandbox/lib/dss_docker/create_image.py index 2ff6619a..2b5d2cd8 100644 --- a/exasol/ds/sandbox/lib/dss_docker/create_image.py +++ b/exasol/ds/sandbox/lib/dss_docker/create_image.py @@ -113,7 +113,7 @@ def _docker_file(self) -> importlib_resources.abc.Traversable: def _start_container(self) -> DockerContainer: self._start = datetime.now() # default timeout is 60 seconds. - docker_client = docker.from_env(600) + docker_client = docker.from_env(timeout=600) try: return docker_client.containers.get(self.container_name) except: