From 5c0b489cb550868db21218cf768d7c60da6f4fb1 Mon Sep 17 00:00:00 2001 From: Christoph Kuhnke Date: Mon, 23 Oct 2023 14:10:08 +0200 Subject: [PATCH] Renamed all occurrences of script-languages-developer-sandbox by data-science-sandbox (#13) * Applied first round of renamings: only names related to AWS resources and build artifacts. * Fixed version number in pyproject.toml * Updated curl from ubuntu2.19 to 2.20 * Renamed python packages from exasol_script_languages_developer_sandbox to exasol.ds.sandbox --- .github/workflows/check_ci.yaml | 6 +- ...ase_droid_upload_github_release_assets.yml | 2 +- .github/workflows/test_release.yml | 4 +- README.md | 7 +- aws-code-build/ci/buildspec.yaml | 6 +- aws-code-build/ci/buildspec_release.yaml | 6 +- doc/changes/changelog.md | 1 - doc/changes/changes_0.1.0.md | 46 +++------ doc/changes/changes_0.2.0.md | 32 ------ doc/developer_guide/developer_guide.md | 99 ++++++++++++++----- doc/user_guide/user_guide.md | 36 +++---- error_code_config.yml | 4 +- .../ds/sandbox}/__init__.py | 0 .../ds/sandbox}/cli/__init__.py | 0 .../ds/sandbox}/cli/cli.py | 0 .../ds/sandbox}/cli/commands/__init__.py | 0 .../ds/sandbox}/cli/commands/create_vm.py | 26 ++--- exasol/ds/sandbox/cli/commands/export_vm.py | 39 ++++++++ .../cli/commands/install_dependencies.py | 23 +++++ .../sandbox/cli/commands/make_ami_public.py | 24 +++++ .../ds/sandbox/cli/commands/reset_password.py | 27 +++++ .../cli/commands/setup_ci_codebuild.py | 20 ++++ exasol/ds/sandbox/cli/commands/setup_ec2.py | 31 ++++++ .../setup_ec2_and_install_dependencies.py | 34 +++++++ .../cli/commands/setup_release_codebuild.py | 21 ++++ .../sandbox/cli/commands/setup_vm_bucket.py | 21 ++++ .../cli/commands/setup_vm_bucket_waf.py | 26 +++++ .../sandbox}/cli/commands/show_aws_assets.py | 16 +-- .../cli/commands/start_release_build.py | 18 ++-- .../cli/commands/start_test_release_build.py | 18 ++-- .../ds/sandbox/cli/commands/update_release.py | 35 +++++++ .../ds/sandbox}/cli/common.py | 0 .../ds/sandbox}/cli/options/__init__.py | 0 .../ds/sandbox}/cli/options/aws_options.py | 0 .../ds/sandbox}/cli/options/ec2_options.py | 0 .../ds/sandbox}/cli/options/id_options.py | 2 +- .../ds/sandbox}/cli/options/logging.py | 2 +- .../ds/sandbox}/cli/options/vm_options.py | 2 +- .../ds/sandbox}/lib/__init__.py | 0 .../ds/sandbox}/lib/ansible/__init__.py | 0 .../ds/sandbox}/lib/ansible/ansible_access.py | 2 +- .../lib/ansible/ansible_context_manager.py | 6 +- .../lib/ansible/ansible_repository.py | 6 +- .../lib/ansible/ansible_run_context.py | 0 .../ds/sandbox}/lib/ansible/ansible_runner.py | 10 +- .../ds/sandbox}/lib/asset_id.py | 4 +- .../sandbox}/lib/asset_printing/__init__.py | 0 .../lib/asset_printing/mark_down_printer.py | 2 +- .../lib/asset_printing/print_assets.py | 22 ++--- .../lib/asset_printing/printing_factory.py | 0 .../asset_printing/rich_console_printer.py | 2 +- .../ds/sandbox}/lib/aws_access/__init__.py | 0 .../ds/sandbox}/lib/aws_access/ami.py | 0 .../ds/sandbox}/lib/aws_access/aws_access.py | 28 +++--- .../lib/aws_access/cloudformation_stack.py | 2 +- .../ds/sandbox}/lib/aws_access/common.py | 0 .../ds/sandbox}/lib/aws_access/deployer.py | 6 +- .../sandbox}/lib/aws_access/ec2_instance.py | 0 .../lib/aws_access/ec2_instance_status.py | 0 .../lib/aws_access/export_image_task.py | 2 +- .../ds/sandbox}/lib/aws_access/key_pair.py | 0 .../ds/sandbox}/lib/aws_access/s3_object.py | 0 .../ds/sandbox}/lib/aws_access/snapshot.py | 2 +- .../sandbox}/lib/aws_access/stack_resource.py | 0 .../lib/aws_access/waiter/__init__.py | 0 .../lib/aws_access/waiter/codebuild_waiter.py | 2 +- .../ds/sandbox}/lib/config.py | 0 .../ds/sandbox}/lib/export_vm/__init__.py | 0 .../lib/export_vm/rename_s3_objects.py | 12 +-- .../sandbox}/lib/export_vm/run_export_vm.py | 20 ++-- .../lib/export_vm/run_make_ami_public.py | 8 +- .../lib/export_vm/vm_disk_image_format.py | 0 .../ds/sandbox}/lib/github_release_access.py | 4 +- .../ds/sandbox}/lib/logging.py | 2 +- .../ds/sandbox}/lib/release_build/__init__.py | 0 .../lib/release_build/run_release_build.py | 12 +-- .../ds/sandbox}/lib/render_template.py | 2 +- .../ds/sandbox}/lib/run_create_vm.py | 31 +++--- .../lib/setup_ci_codebuild/__init__.py | 0 .../lib/setup_ci_codebuild/ci_codebuild.py | 15 +++ .../ds/sandbox}/lib/setup_ec2/__init__.py | 0 .../ds/sandbox}/lib/setup_ec2/cf_stack.py | 12 +-- .../ds/sandbox}/lib/setup_ec2/host_info.py | 0 .../lib/setup_ec2/key_file_manager.py | 6 +- .../lib/setup_ec2/random_string_generator.py | 0 .../lib/setup_ec2/run_install_dependencies.py | 12 +-- .../lib/setup_ec2/run_reset_password.py | 10 +- .../sandbox}/lib/setup_ec2/run_setup_ec2.py | 16 +-- .../run_setup_ec2_and_install_dependencies.py | 22 ++--- .../ds/sandbox}/lib/setup_ec2/source_ami.py | 4 +- .../lib/setup_release_codebuild/__init__.py | 0 .../release_codebuild.py | 15 +++ .../ds/sandbox}/lib/tags.py | 2 +- .../sandbox}/lib/update_release/__init__.py | 0 .../lib/update_release/run_update_release.py | 10 +- .../ds/sandbox}/lib/vm_bucket/__init__.py | 0 .../ds/sandbox/lib/vm_bucket/vm_dss_bucket.py | 12 +-- .../lib/vm_bucket/vm_dss_bucket_waf.py | 12 +-- exasol/ds/sandbox/main.py | 7 ++ .../ds/sandbox}/runtime/__init__.py | 0 .../runtime/ansible/reset_password.yml | 0 .../runtime/ansible/reset_password_tasks.yml | 0 .../ansible/roles/docker/defaults/main.yml | 0 .../ansible/roles/docker/tasks/main.yml | 0 .../ansible/roles/jupyter/defaults/main.yml | 0 .../files/heading_jupyter_update_password.txt | 0 .../jupyter/files/notebook/bash_runner.py | 0 .../files/notebook/script-languages.ipynb | 0 .../files/notebook/slc_main_build_steps.svg | 0 .../files/requirements_dependencies.txt | 0 .../jupyter/files/requirements_jupyter.txt | 0 .../jupyter/tasks/install-pip-packages.yml | 0 .../roles/jupyter/tasks/jupyterlab.yml | 0 .../ansible/roles/jupyter/tasks/main.yml | 0 .../ansible/roles/jupyter/tasks/motd.yml | 0 .../ansible/roles/jupyter/tasks/systemd.yml | 0 .../ansible/roles/jupyter/tasks/tutorial.yml | 0 .../etc/systemd/system/jupyter.service | 0 .../templates/etc/update-motd.d/999-jupyter | 0 .../ansible/roles/netplan/defaults/main.yml | 0 .../files/etc/netplan/01-network-manager.yaml | 0 .../ansible/roles/netplan/tasks/main.yml | 0 .../ansible/roles/poetry/defaults/main.yml | 2 +- .../ansible/roles/poetry/tasks/main.yml | 0 .../roles/script_languages/defaults/main.yml | 0 .../roles/script_languages/tasks/main.yml | 0 .../ds/sandbox}/runtime/ansible/slc_setup.yml | 0 .../runtime/ansible/slc_setup_tasks.yml | 0 .../templates/additional_release_notes.jinja | 0 .../templates/ci_code_build.jinja.yaml | 10 +- .../templates/ec2_cloudformation.jinja.yaml | 4 +- .../ds/sandbox}/templates/inventory.jinja | 0 .../templates/release_code_build.jinja.yaml | 14 +-- .../vm_bucket_cloudformation.jinja.yaml | 8 +- .../templates/waf_cloudformation.jinja.yaml | 16 +-- .../cli/commands/export_vm.py | 39 -------- .../cli/commands/install_dependencies.py | 23 ----- .../cli/commands/make_ami_public.py | 24 ----- .../cli/commands/reset_password.py | 27 ----- .../cli/commands/setup_ci_codebuild.py | 20 ---- .../cli/commands/setup_ec2.py | 31 ------ .../setup_ec2_and_install_dependencies.py | 34 ------- .../cli/commands/setup_release_codebuild.py | 21 ---- .../cli/commands/setup_vm_bucket.py | 21 ---- .../cli/commands/setup_vm_bucket_waf.py | 26 ----- .../cli/commands/update_release.py | 35 ------- .../lib/setup_ci_codebuild/ci_codebuild.py | 15 --- .../release_codebuild.py | 15 --- .../main.py | 7 -- pyproject.toml | 28 ++++-- test/aws/__init__.py | 5 + test/{ => aws}/cloudformation_validation.py | 2 +- test/{ => aws}/test_deploy_codebuild.py | 6 +- test/{ => aws}/test_deploy_ec2.py | 8 +- test/{ => aws}/test_deploy_vm_bucket.py | 8 +- test/{ => aws}/test_deploy_vm_bucket_waf.py | 6 +- test/{ => aws}/test_export_vm.py | 10 +- test/aws_local_stack_access.py | 2 +- test/aws_mock_data.py | 38 +++---- test/{ => ci}/test_ci.py | 30 +++--- test/{ => ci}/test_install_dependencies.py | 50 +++++++--- test/conftest.py | 8 +- test/integration/__init__.py | 4 + test/{ => integration}/localstack_test.py | 10 +- .../test_aws_codebuild_waiter.py | 2 +- .../test_motd_jupyter_template.py | 6 +- test/{ => integration}/test_serialization.py | 6 +- test/test_container/Dockerfile | 4 - test/unit/__init__.py | 4 + test/{ => unit}/ansible_conflict/__init__.py | 0 .../{ => unit}/ansible_conflict/slc_setup.yml | 0 test/{ => unit}/test_ansible.py | 12 +-- test/{ => unit}/test_key_file_manager.py | 2 +- test/{ => unit}/test_make_ami_public.py | 6 +- test/{ => unit}/test_printing.py | 6 +- test/{ => unit}/test_printing_factories.py | 10 +- test/{ => unit}/test_release_build.py | 14 +-- test/{ => unit}/test_run_lifecycle_for_ec2.py | 6 +- test/{ => unit}/test_source_ami.py | 4 +- 179 files changed, 856 insertions(+), 809 deletions(-) delete mode 100644 doc/changes/changes_0.2.0.md rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/cli.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/commands/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/commands/create_vm.py (52%) create mode 100644 exasol/ds/sandbox/cli/commands/export_vm.py create mode 100644 exasol/ds/sandbox/cli/commands/install_dependencies.py create mode 100644 exasol/ds/sandbox/cli/commands/make_ami_public.py create mode 100644 exasol/ds/sandbox/cli/commands/reset_password.py create mode 100644 exasol/ds/sandbox/cli/commands/setup_ci_codebuild.py create mode 100644 exasol/ds/sandbox/cli/commands/setup_ec2.py create mode 100644 exasol/ds/sandbox/cli/commands/setup_ec2_and_install_dependencies.py create mode 100644 exasol/ds/sandbox/cli/commands/setup_release_codebuild.py create mode 100644 exasol/ds/sandbox/cli/commands/setup_vm_bucket.py create mode 100644 exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/commands/show_aws_assets.py (64%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/commands/start_release_build.py (51%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/commands/start_test_release_build.py (53%) create mode 100644 exasol/ds/sandbox/cli/commands/update_release.py rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/common.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/options/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/options/aws_options.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/options/ec2_options.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/options/id_options.py (78%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/options/logging.py (82%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/options/vm_options.py (85%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/ansible/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/ansible/ansible_access.py (86%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/ansible/ansible_context_manager.py (76%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/ansible/ansible_repository.py (91%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/ansible/ansible_run_context.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/ansible/ansible_runner.py (63%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/asset_id.py (78%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/asset_printing/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/asset_printing/mark_down_printer.py (89%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/asset_printing/print_assets.py (92%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/asset_printing/printing_factory.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/asset_printing/rich_console_printer.py (90%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/ami.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/aws_access.py (93%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/cloudformation_stack.py (94%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/common.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/deployer.py (98%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/ec2_instance.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/ec2_instance_status.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/export_image_task.py (93%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/key_pair.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/s3_object.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/snapshot.py (92%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/stack_resource.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/waiter/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/waiter/codebuild_waiter.py (96%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/config.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/export_vm/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/export_vm/rename_s3_objects.py (76%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/export_vm/run_export_vm.py (87%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/export_vm/run_make_ami_public.py (72%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/export_vm/vm_disk_image_format.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/github_release_access.py (93%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/logging.py (95%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/release_build/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/release_build/run_release_build.py (88%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/render_template.py (90%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/run_create_vm.py (65%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ci_codebuild/__init__.py (100%) create mode 100644 exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/cf_stack.py (85%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/host_info.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/key_file_manager.py (89%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/random_string_generator.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/run_install_dependencies.py (71%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/run_reset_password.py (77%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/run_setup_ec2.py (89%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/run_setup_ec2_and_install_dependencies.py (73%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/source_ami.py (64%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_release_codebuild/__init__.py (100%) create mode 100644 exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/tags.py (86%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/update_release/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/update_release/run_update_release.py (65%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/vm_bucket/__init__.py (100%) rename exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket.py => exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket.py (80%) rename exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket_waf.py => exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py (77%) create mode 100755 exasol/ds/sandbox/main.py rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/reset_password.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/reset_password_tasks.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/docker/defaults/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/docker/tasks/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/defaults/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/files/heading_jupyter_update_password.txt (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/files/notebook/bash_runner.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/files/notebook/script-languages.ipynb (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/files/notebook/slc_main_build_steps.svg (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/tasks/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/tasks/motd.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/tasks/systemd.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/tasks/tutorial.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/templates/etc/systemd/system/jupyter.service (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/netplan/defaults/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/netplan/files/etc/netplan/01-network-manager.yaml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/netplan/tasks/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/poetry/defaults/main.yml (78%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/poetry/tasks/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/script_languages/defaults/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/script_languages/tasks/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/slc_setup.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/slc_setup_tasks.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/templates/additional_release_notes.jinja (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/templates/ci_code_build.jinja.yaml (91%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/templates/ec2_cloudformation.jinja.yaml (93%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/templates/inventory.jinja (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/templates/release_code_build.jinja.yaml (88%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/templates/vm_bucket_cloudformation.jinja.yaml (94%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/templates/waf_cloudformation.jinja.yaml (79%) delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/export_vm.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/install_dependencies.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/make_ami_public.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/reset_password.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/setup_ci_codebuild.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/setup_ec2.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/setup_ec2_and_install_dependencies.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/setup_release_codebuild.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket_waf.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/update_release.py delete mode 100644 exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/ci_codebuild.py delete mode 100644 exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/release_codebuild.py delete mode 100755 exasol_script_languages_developer_sandbox/main.py create mode 100644 test/aws/__init__.py rename test/{ => aws}/cloudformation_validation.py (90%) rename test/{ => aws}/test_deploy_codebuild.py (74%) rename test/{ => aws}/test_deploy_ec2.py (84%) rename test/{ => aws}/test_deploy_vm_bucket.py (86%) rename test/{ => aws}/test_deploy_vm_bucket_waf.py (83%) rename test/{ => aws}/test_export_vm.py (90%) rename test/{ => ci}/test_ci.py (87%) rename test/{ => ci}/test_install_dependencies.py (69%) create mode 100644 test/integration/__init__.py rename test/{ => integration}/localstack_test.py (92%) rename test/{ => integration}/test_aws_codebuild_waiter.py (95%) rename test/{ => integration}/test_motd_jupyter_template.py (91%) rename test/{ => integration}/test_serialization.py (93%) delete mode 100644 test/test_container/Dockerfile create mode 100644 test/unit/__init__.py rename test/{ => unit}/ansible_conflict/__init__.py (100%) rename test/{ => unit}/ansible_conflict/slc_setup.yml (100%) rename test/{ => unit}/test_ansible.py (92%) rename test/{ => unit}/test_key_file_manager.py (93%) rename test/{ => unit}/test_make_ami_public.py (87%) rename test/{ => unit}/test_printing.py (96%) rename test/{ => unit}/test_printing_factories.py (81%) rename test/{ => unit}/test_release_build.py (84%) rename test/{ => unit}/test_run_lifecycle_for_ec2.py (94%) rename test/{ => unit}/test_source_ami.py (99%) diff --git a/.github/workflows/check_ci.yaml b/.github/workflows/check_ci.yaml index dbe9ecb5..d74bea25 100644 --- a/.github/workflows/check_ci.yaml +++ b/.github/workflows/check_ci.yaml @@ -1,4 +1,4 @@ -name: Run Unit Tests +name: Run Tests for CI Build on: push: @@ -6,7 +6,7 @@ on: - "main" jobs: - run_unit_tests: + run_ci_tests: environment: AWS_CI_TESTS runs-on: ubuntu-latest @@ -18,7 +18,7 @@ jobs: uses: ./.github/actions/prepare_poetry_env - name: Run pytest - run: poetry run pytest test/test_install_dependencies.py + run: poetry run pytest test/ci/test_install_dependencies.py env: # Set the secret as an env variable AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }} diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml index 1a098118..4b548454 100644 --- a/.github/workflows/release_droid_upload_github_release_assets.yml +++ b/.github/workflows/release_droid_upload_github_release_assets.yml @@ -29,7 +29,7 @@ jobs: - name: Setup Python & Poetry Environment uses: ./.github/actions/prepare_poetry_env - name: Build Release - run: poetry run python3 -m exasol_script_languages_developer_sandbox.main start-release-build --upload-url "${{ github.event.inputs.upload_url }}" --branch "$GITHUB_REF" + run: poetry run python3 -m exasol.ds.sandbox.main start-release-build --upload-url "${{ github.event.inputs.upload_url }}" --branch "$GITHUB_REF" env: # Set the secret as an env variable AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }} diff --git a/.github/workflows/test_release.yml b/.github/workflows/test_release.yml index dcfda8cb..dec14637 100644 --- a/.github/workflows/test_release.yml +++ b/.github/workflows/test_release.yml @@ -21,9 +21,9 @@ jobs: uses: ./.github/actions/prepare_poetry_env - name: Start test release - run: poetry run python3 -m exasol_script_languages_developer_sandbox.main start-test-release-build --release-title "${{ github.event.inputs.release_title }}" --branch "$GITHUB_REF" + run: poetry run python3 -m exasol.ds.sandbox.main start-test-release-build --release-title "${{ github.event.inputs.release_title }}" --branch "$GITHUB_REF" env: # Set the secret as an env variable AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }} AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} - GITHUB_TOKEN: ${{ github.token }} \ No newline at end of file + GITHUB_TOKEN: ${{ github.token }} diff --git a/README.md b/README.md index 71800e0e..94edc287 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ +# Exasol Data Science Sandbox -# Overview +## Overview This project provides an automated mechanism to build and export virtual machines images (AWS AMI, VMDK,...) -which can be used to develop new script-languages container for the Exasol DB. +enabling users to try out data science algorithms in Jupyter notebooks connected to the Exasol database. ## Where to find the VM images -The release process will automatically store the links to the images in the [release notes](https://github.com/exasol/script-languages-developer-sandbox/releases/latest), as there will be a specific AMI per release. +The release process will automatically store the links to the images in the [release notes](https://github.com/exasol/data-science-sandbox/releases/latest), as there will be a specific AMI per release. Please check the user guide about details of the image. ## Links diff --git a/aws-code-build/ci/buildspec.yaml b/aws-code-build/ci/buildspec.yaml index 6669259a..1b7793a1 100644 --- a/aws-code-build/ci/buildspec.yaml +++ b/aws-code-build/ci/buildspec.yaml @@ -3,7 +3,7 @@ version: 0.2 env: shell: bash variables: - RUN_DEVELOPER_SANDBOX_CI_TEST: "true" + DSS_RUN_CI_TEST: "true" AWS_USER_NAME: "ci_user" phases: @@ -20,7 +20,7 @@ phases: pre_build: commands: - - echo RUN_DEVELOPER_SANDBOX_CI_TEST is "$RUN_DEVELOPER_SANDBOX_CI_TEST" #supposed to be true + - echo DSS_RUN_CI_TEST is "$DSS_RUN_CI_TEST" #supposed to be true build: commands: - - poetry run python3 -m pytest -s test/test_ci*.py + - poetry run python3 -m pytest -s test/ci/test_ci*.py diff --git a/aws-code-build/ci/buildspec_release.yaml b/aws-code-build/ci/buildspec_release.yaml index 58ede603..20d07264 100644 --- a/aws-code-build/ci/buildspec_release.yaml +++ b/aws-code-build/ci/buildspec_release.yaml @@ -3,7 +3,7 @@ version: 0.2 env: shell: bash variables: - DEFAULT_PASSWORD: "scriptlanguages" + DEFAULT_PASSWORD: "dss" ASSET_ID: "" AWS_USER_NAME: "release_user" MAKE_AMI_PUBLIC_OPTION: "--no-make-ami-public" @@ -27,5 +27,5 @@ phases: - echo MAKE_AMI_PUBLIC_OPTION is "$MAKE_AMI_PUBLIC_OPTION" build: commands: - - poetry run python3 -m exasol_script_languages_developer_sandbox.main create-vm --default-password "$DEFAULT_PASSWORD" --asset-id "$ASSET_ID" $MAKE_AMI_PUBLIC_OPTION - - poetry run python3 -m exasol_script_languages_developer_sandbox.main update-release --release-id "$RELEASE_ID" --asset-id "$ASSET_ID" + - poetry run python3 -m exasol.ds.sandbox.main create-vm --default-password "$DEFAULT_PASSWORD" --asset-id "$ASSET_ID" $MAKE_AMI_PUBLIC_OPTION + - poetry run python3 -m exasol.ds.sandbox.main update-release --release-id "$RELEASE_ID" --asset-id "$ASSET_ID" diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 39f13d8b..9e62d079 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,4 +1,3 @@ # Changes -* [0.2.0](changes_0.2.0.md) * [0.1.0](changes_0.1.0.md) diff --git a/doc/changes/changes_0.1.0.md b/doc/changes/changes_0.1.0.md index 754ca23a..9ffe8876 100644 --- a/doc/changes/changes_0.1.0.md +++ b/doc/changes/changes_0.1.0.md @@ -1,47 +1,23 @@ -# script-languages-developer-sandbox 0.1.0, released 2022-10-06 +# data-science-sandbox 0.1.0, released t.b.d. Code name: Initial release ## Summary -Initial release of the script-languages-developer-sandbox. It provides the creation of a developer sandbox AMI and virtual machine images for a specific version of the script-languages-release project. +Initial release of the data-science-sandbox. It provides the creation of an Amazon Machone Image (AMI) and virtual machine images for a specific version of the data-science-sanbox-release project. -## Script-Languages-Release +## Data-Science-Sandbox-Release -Version: 5.0.0 +Version: 0.1.0 + +## Features + + - #11: Created a notebook to show training with scikit-learn in the notebook ## Bug Fixes - - - #18: Fixed network connection - - #51: Fixed network connection - - #57: Fixed release build - - #60: Fixed release build command - - #64: Fixed Netplan file name - -## Features / Enhancements - - - #2: Implemented launch of an EC2 instance - - #3: Installed SLC dependencies via Ansible - - #4: Implemented deployment and access of S3 Bucket for VM's - - #5: Implemented export of VM's - - #24: Move CI test to AWS Codebuild - - #25: Implemented motd message about Jupyter password change - - #8: Implemented a release workflow - - #36: Added make-ami-public option - - #43: Added CDN to the S3 VM Bucket - - #45: Protected cloudfront access - - #47: Renamed virtual images - - #38: Included tutorial Jupyterlab notebook - -## Documentation - - - #19: Added user guide and developer guide - - #49: Added tutorial about how to start the VM/AMI + + - #1: Fixed CI build ## Refactoring - - #22: Improved logging - - #26: Implemented search for latest AMI - - #12: Updated the script-languages-release tag with the correct version - - #21: Minor refactoring tasks - - #41: Renamed cloudformation stack \ No newline at end of file + - #5: Renamed all occurrences of "script language developer" by "data science" diff --git a/doc/changes/changes_0.2.0.md b/doc/changes/changes_0.2.0.md deleted file mode 100644 index 3baa421d..00000000 --- a/doc/changes/changes_0.2.0.md +++ /dev/null @@ -1,32 +0,0 @@ -# script-languages-developer-sandbox 0.2.0, released t.b.d. - -Code name: t.b.d. - -## Summary - -t.b.d. - -## Script-Languages-Release - -Version: 5.0.0 - -## Bug Fixes - - - #78: Fix test_install_dependencies docker cleanup, fix pytest github action - -## Features / Enhancements - -n/a - -## Documentation - - - #67: Mentioned the AMI copy function in tutorial and release notes - -## Refactoring - - - #73: Removed setup.py - -## Security - - - #77: Update Ubuntu packages and added `cloudformation:ListStacks` to the roles for the CI runs - diff --git a/doc/developer_guide/developer_guide.md b/doc/developer_guide/developer_guide.md index c6caa75b..012c5e3b 100644 --- a/doc/developer_guide/developer_guide.md +++ b/doc/developer_guide/developer_guide.md @@ -1,4 +1,4 @@ -# Script-Languages-Developer-Sandbox Developer Guide +# Data Science Sandbox Developer Guide ## Overview @@ -25,13 +25,13 @@ bash install.sh ## Design Goals -script-languages-developer-sandbox uses AWS as backend, because it provides the possibility to run the whole workflow during a ci-test. +The Data Science Sandbox (DSS) uses AWS as backend, because it provides the possibility to run the whole workflow during a ci-test. -This project uses +This project uses - `boto3` to interact with AWS - `pygithub` to interact with the Github releases -- `ansible-runner` to interact with Ansible. -Proxy classes to those projects are injected at the CLI layer. This allows to inject mock classes in the unit tests. +- `ansible-runner` to interact with Ansible. +Proxy classes to those projects are injected at the CLI layer. This allows to inject mock classes in the unit tests. A CLI command has normally a respective function in the `lib` submodule. Hence, the CLI layer should not contain any logic, but invoke the respective library function only. Also, the proxy classes which abstract the dependant packages shall not contain too much logic. Ideally they should invoke only one function to the respective package. @@ -39,18 +39,18 @@ A CLI command has normally a respective function in the `lib` submodule. Hence, There are generally three types of commands: -| Type | Explanation | +| Type | Explanation | | ----- | --------- | -| Release Commands | used during the release | -| Deployment Commands | used to deploy infrastructure onto AWS cloud | -| Development Commands | used to identify problems or for testing | +| Release Commands | used during the release | +| Deployment Commands | used to deploy infrastructure onto AWS cloud | +| Development Commands | used to identify problems or for testing | ### Release commands The following commands are used during the release AWS Codebuild job: - `create-vm` - creates a new AMI and VM images - `update-release` - updates release notes of an existing Github release -- `start-release-build` - starts the release on AWS codebuild +- `start-release-build` - starts the release on AWS codebuild ### Developer commands @@ -62,7 +62,7 @@ All other commands provide a subset of the features of the release commands, and - `setup-ec2-and-install-dependencies` - starts a new EC2 instance and install dependencies via Ansible - `show-aws-assets` - shows AWS entities associated with a specific keyword (called __asset-id__) - `start-test-release` - starts a Test Release flow -- `make-ami-public` - Changes permissions of an existing AMI such that it becomes public +- `make-ami-public` - Changes permissions of an existing AMI such that it becomes public ### Deployment commands @@ -70,11 +70,11 @@ The following commands can be used to deploy the infrastructure onto a given AWS - `setup-ci-codebuild` - deploys the AWS Codebuild cloudformation stack which will run the ci-test - `setup-vm-bucket` - deploys the AWS Bucket cloudformation stack which will be used to deploy the VM images - `setup-release-codebuild` - deploys the AWS Codebuild cloudformation stack which will be used for the release-build -- `setup-vm-bucket-waf` - deploys the AWS Codebuild cloudformation stack which contains the WAF Acl configuration for the Cloudfront distribution of the VM Bucket +- `setup-vm-bucket-waf` - deploys the AWS Codebuild cloudformation stack which contains the WAF Acl configuration for the Cloudfront distribution of the VM Bucket ## Flow -The following diagram shows the high-level steps to generate the images: +The following diagram shows the high-level steps to generate the images: ![image info](./img/create-vm-overview.drawio.png) ### Setup EC2 @@ -94,6 +94,49 @@ Installs all dependencies via Ansible: Finally, the default password will be set, and also the password will be marked as expired, such that the user will be forced to enter a new password during initial login. Also, the ssh password authentication will be enabled, and for security reasons the folder "~/.ssh" will be removed. +### Tests + +DSS comes with a number of tests in directory `test`. +There are subdirectories clustering tests with common scope and prerequisites e.g. external resources. + +| Directory | Content | +|---------------------|---------| +| `test/unit` | Simple unit tests requiring no additional setup or external resources. | +| `test/integration` | Integration tests with longer runtime and some requiring additional resources. | +| `test/aws` | Tests involving AWS resources. In order to execute these tests you need an AWS account, a user with permissions in this account, and an access key. | + +To run the tests in file `test/integration/test_ci.py` please use +```shell +export DSS_RUN_CI_TEST=true +poetry run test/integration/test_ci.py +``` + +#### Executing tests involving AWS resources + +In AWS web interface, IAM create an access key for CLI usage and save or download the *access key id* and the *secret access key*. + +In file `~/.aws/config` add lines +``` +[profile dss_aws_tests] +region = eu-central-1 +``` + +In file `~/.aws/credentials` add +``` +[dss_aws_tests] +aws_access_key_id=... +aws_secret_access_key=... +``` + +In case your are using MFA authentication please allocate a temporary token. + +After that you can set an environment variable and execute the tests involving AWS resources: + +```shell +export AWS_PROFILE=dss_aws_tests_mfa +poetry run pytest test/test_deploy_codebuild.py +``` + ### Export The export creates an AMI based on the running EC2 instance and exports the AMI as VM image in the default formats to a S3 bucket. @@ -107,10 +150,10 @@ The release is executed in a AWS Codebuild job, the following diagram shows the The bucket has private access. In order to control access, the Bucket cloudformation stack also contains a Cloudfront distribution. Public Https access is only possibly through Cloudfront. Another stack contains a Web application firewall (WAF), which will be used by the Cloudfront distribution. Due to restrictions in AWS, the WAF stack needs to be deployed in region "us-east-1". The WAF stack provides two rules which aim to minimize a possible bot attack: -| Name | Explanation | Priority | +| Name | Explanation | Priority | |----------------------|-----------------------------------------------------------------------------------------|----------| -| VMBucketRateLimit | Declares the minimum possible rate limit for access: 100 requests in a 5 min interval. | 0 | -| CAPTCHA | Forces a captcha action for any IP which does not matcha predefined set of IP-addresses | 1 | +| VMBucketRateLimit | Declares the minimum possible rate limit for access: 100 requests in a 5 min interval. | 0 | +| CAPTCHA | Forces a captcha action for any IP which does not matcha predefined set of IP-addresses | 1 | @@ -119,18 +162,26 @@ The bucket has private access. In order to control access, the Bucket cloudforma The following diagram shows the involved cloudformation stacks: ![image info](./img/cloudformation-stacks.drawio.png) -"DEVELOPER-SANDBOX-VM-SLC-BUCKET", "DEVELOPER-SANDBOX-CI-TEST-CODEBUILD" & "DEVELOPER-SANDBOX-RELEASE-CODEBUILD" are permanent and need to be deployed using the "deploy" commands (see [commands](#deployment-commands)). -The EC2-stack lives only during the creation of a new developer sandbox image. +The following resources are permanent and need to be deployed using the "deploy" [commands](#deployment-commands): +* `DATA-SCIENCE-SANDBOX-VM-Bucket` +* `DATA-SCIENCE-SANDBOX-CI-TEST-CODEBUILD` +* `DATA-SCIENCE-SANDBOX-RELEASE-CODEBUILD` + +The EC2-stack lives only during the creation of a new sandbox image. ## Tagging Each of the involved resources might cause costs: cloudformation stacks, AMI, EC2 key-pairs. -To enable you to keep track of all these resources, the implementation tags them after creation with a specific keyword (called __asset-id__). -The S3 objects are identified by the prefix in the S3 bucket. The command tags only the dynamically created entities with the asset-id but not the permanent cloudformation stacks. -You can use the command `show-aws-assets` to get a list of all assets which were created during the execution. -This is very useful if an error occured. -If the creation of a sandbox finished normally it is expected to have only the AMI, images (S3 objects) and the export tasks (one for each image) listed. +To enable keeping track of all these resources, the implementation tags them after creation with a specific keyword (called __asset-id__). + +The S3 objects are identified by the prefix in the S3 bucket. + +The command tags only the dynamically created entities with the *asset-id* but not the permanent cloudformation stacks. + +The command `show-aws-assets` lists all assets which were created during the execution. +* This is very useful if an error occured. +* If the creation of a sandbox finished normally the list should contain only the AMI, images (S3 objects) and the export tasks (one for each image). ## How to contribute @@ -139,4 +190,4 @@ The project has two types of CI tests: - A system test which runs on a AWS Codebuild Both ci tests need to pass before the approval of a Github PR. -The Github workflow will run on each push to a branch in the Github repository. However, the AWS Codebuild will only run after you push a commit containing the string "[CodeBuild]" in the commit message. \ No newline at end of file +The Github workflow will run on each push to a branch in the Github repository. However, the AWS Codebuild will only run after you push a commit containing the string "[CodeBuild]" in the commit message. \ No newline at end of file diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index 28c10814..fccd564d 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -1,4 +1,4 @@ -# Script-Languages-Developer-Sandbox User Guide +# Data Science Sandbox User Guide ## Overview @@ -18,8 +18,8 @@ The disk of the VM is per default large enough to build and test script-language ### AMI -The AMI id is linked in the [release notes](https://github.com/exasol/script-languages-developer-sandbox/releases/latest) and can be used to start an EC2-instance in your AWS account. -Following name format is used for the AMI: "_Exasol-SLC-Developer-Sandbox-**version**_", e.g. "_Exasol-SLC-Developer-Sandbox-5.0.0_" +The AMI id is linked in the [release notes](https://github.com/exasol/data-science-sandbox/releases/latest) and can be used to start an EC2-instance in your AWS account. +Following name format is used for the AMI: "_Exasol-Data-Science-Sandbox-**version**_", e.g. "_Exasol-Data-Science-Sandbox-5.0.0_" ### VM image @@ -30,7 +30,7 @@ Currently two VM formats are supported: | VMDK | VMware Virtual Machine Disk | | VHD | Virtual Hard Disk by Microsoft | -The links to the images are stored in the release notes. +The links to the images are stored in the release notes. ## Usage @@ -61,15 +61,15 @@ __Important__: The AMI is currently only available in the AWS region `eu-central - Click button "Create Security Group" - Choose name and VPC - For outbound rules keep the default - - Create inbound rules: + - Create inbound rules: - One of type `ssh` - - If you plan to connect to the Jupyter lab, add another rule of type "Custom TCP" with port 8888. + - If you plan to connect to the Jupyter lab, add another rule of type "Custom TCP" with port 8888. - **Important**: With this rule you expose the Jupyter lab to the internet; anybody who has access to the password will be able to execute commands. For a minimum of security you should change the default Jupyter password. Details about how to do that will be shown in the login screen. However, you should consider to use only `ssh` with port forwarding. 4. Go back to the EC2 console 5. Launch the EC2 instance: - In the navigation bar on the left select "Instances" - Click button "Launch instances" - - At field "Application and OS Images" select the AMI id of the developer sandbox (found in the [release notes](https://github.com/exasol/script-languages-developer-sandbox/releases/latest)) + - At field "Application and OS Images" select the AMI id of the sandbox (found in the [release notes](https://github.com/exasol/data-science-sandbox/releases/latest)) - Select an appropriate instance type (at least "t2.small" or similar) - Choose your key pair - Choose the security group which your created in step 3. @@ -90,7 +90,7 @@ __Important__: The AMI is currently only available in the AWS region `eu-central ##### Step-by-step -1. Download the `VMDK` file from the [release notes](https://github.com/exasol/script-languages-developer-sandbox/releases/latest). +1. Download the `VMDK` file from the [release notes](https://github.com/exasol/data-science-sandbox/releases/latest). 2. Open Boxes 3. Create a new VM: Click the + Button 4. Choose: "Create virtual machine from file" @@ -106,13 +106,13 @@ __Important__: The AMI is currently only available in the AWS region `eu-central 10. If you want to connect to the Jupyterlab, you need to find out the IP address of the VM: - Click the 3 dots in Boxes, then preferences![image info](./img/tutorial-screenshot-open-preferences.png) - Then the IP address will be shown: ![image info](./img/tutorial-screenshot-show-ip.png) -- Use _IP-Address_:8888 to connect to the [Jupterlab](#Jupyter) +- Use _IP-Address_:8888 to connect to the [Jupterlab](#Jupyter) ### Login Username: **ubuntu** -At the first login to the sandbox (image or AMI) you will be prompted to change your password. +At the first login to the sandbox (image or AMI) you will be prompted to change your password. The default password is: **scriptlanguages** However, we suggest to use ssh-keys for the connection. When you use the AWS AMI, this will work automatically. When you use the VM images, you need to deploy your ssh-keys. After you enabled ssh-keys, we recommend to disable ssh password authentication: @@ -127,10 +127,10 @@ Also, we strongly recommend to change the Jupyter password as soon as possible. ### script-languages-release -**Location**: `/home/ubuntu/script-languages-release` -**Source**: [Github repo](https://github.com/exasol/script-languages-release) -The images are tightly coupled to the releases of script-languages-release; for each release of the script-languages-release there will be a release of the developer sandbox. -By default, the images contain a cloned repository of script-languages-container, including all dependencies to run it (Python, Poetry, Docker, etc.). The checked out version will be the tag of the respective release, e.g. a Developer Sandbox image for release 5.0.0 will have the tag 5.0.0 checked out for script-languages-release. +**Location**: `/home/ubuntu/script-languages-release` +**Source**: [Github repo](https://github.com/exasol/script-languages-release) +The images are tightly coupled to the releases of script-languages-release; for each release of the script-languages-release there will be a release of the data-science sandbox. +By default, the images contain a cloned repository of script-languages-container, including all dependencies to run it (Python, Poetry, Docker, etc.). The checked out version will be the tag of the respective release, e.g. a Sandbox image for release 5.0.0 will have the tag 5.0.0 checked out for script-languages-release. If you aim to customize existing containers, this should be fine. However, if you want to rebuild a container, you might encounter problems as dependant packages might have changed. In that case, we suggest to check out the master branch ```shell git checkout --recurse-submodules master @@ -142,10 +142,10 @@ For information about how to build script-languages-container please check: ### Jupyter -**Location virtual environment**: `/home/ubuntu/jupyterenv` -**Location notebooks**: `/home/ubuntu/notebooks` -**Password**: `script-languages` -**Http Port**: 8888 +**Location virtual environment**: `/home/ubuntu/jupyterenv` +**Location notebooks**: `/home/ubuntu/notebooks` +**Password**: `dss` +**Http Port**: 8888 There is a pre-loaded tutorial notebook available which guides you through the usage of `exaslct`: __script-languages.ipynb__. diff --git a/error_code_config.yml b/error_code_config.yml index 00da1e08..f2db5e1b 100644 --- a/error_code_config.yml +++ b/error_code_config.yml @@ -1,3 +1,3 @@ error-tags: - SLDS: - highest-index: 0 \ No newline at end of file + DSS: + highest-index: 0 diff --git a/exasol_script_languages_developer_sandbox/__init__.py b/exasol/ds/sandbox/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/__init__.py rename to exasol/ds/sandbox/__init__.py diff --git a/exasol_script_languages_developer_sandbox/cli/__init__.py b/exasol/ds/sandbox/cli/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/cli/__init__.py rename to exasol/ds/sandbox/cli/__init__.py diff --git a/exasol_script_languages_developer_sandbox/cli/cli.py b/exasol/ds/sandbox/cli/cli.py similarity index 100% rename from exasol_script_languages_developer_sandbox/cli/cli.py rename to exasol/ds/sandbox/cli/cli.py diff --git a/exasol_script_languages_developer_sandbox/cli/commands/__init__.py b/exasol/ds/sandbox/cli/commands/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/cli/commands/__init__.py rename to exasol/ds/sandbox/cli/commands/__init__.py diff --git a/exasol_script_languages_developer_sandbox/cli/commands/create_vm.py b/exasol/ds/sandbox/cli/commands/create_vm.py similarity index 52% rename from exasol_script_languages_developer_sandbox/cli/commands/create_vm.py rename to exasol/ds/sandbox/cli/commands/create_vm.py index 0e8b0b5e..b00c572d 100644 --- a/exasol_script_languages_developer_sandbox/cli/commands/create_vm.py +++ b/exasol/ds/sandbox/cli/commands/create_vm.py @@ -3,19 +3,19 @@ import click -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.ec2_options import ec2_key_options -from exasol_script_languages_developer_sandbox.cli.options.id_options import id_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.cli.options.vm_options import vm_options -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.run_create_vm import run_create_vm +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.ec2_options import ec2_key_options +from exasol.ds.sandbox.cli.options.id_options import id_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.cli.options.vm_options import vm_options +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.run_create_vm import run_create_vm @cli.command() diff --git a/exasol/ds/sandbox/cli/commands/export_vm.py b/exasol/ds/sandbox/cli/commands/export_vm.py new file mode 100644 index 00000000..424d4657 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/export_vm.py @@ -0,0 +1,39 @@ +from typing import Tuple + +import click + +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.id_options import id_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.cli.options.vm_options import vm_options +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.export_vm.run_export_vm import run_export_vm +from exasol.ds.sandbox.lib.logging import set_log_level + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +@click.option('--stack-name', required=True, + type=str, + help="Existing cloudformation stack containing the EC2 instance.") +@add_options(vm_options) +@add_options(id_options) +def export_vm( + aws_profile: str, + stack_name: str, + vm_image_format: Tuple[str, ...], + no_vm: bool, + asset_id: str, + log_level: str): + """ + Debug command which creates a new VM image from a running EC2-Instance. + """ + current_vm_image_formats = tuple() if no_vm else vm_image_format + set_log_level(log_level) + run_export_vm(AwsAccess(aws_profile), stack_name, current_vm_image_formats, + AssetId(asset_id), default_config_object) diff --git a/exasol/ds/sandbox/cli/commands/install_dependencies.py b/exasol/ds/sandbox/cli/commands/install_dependencies.py new file mode 100644 index 00000000..82115f76 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/install_dependencies.py @@ -0,0 +1,23 @@ +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.ec2_options import ec2_host_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies + + +@cli.command() +@add_options(logging_options) +@add_options(ec2_host_options) +def install_dependencies( + host_name: str, + ssh_private_key: str, + log_level: str): + """ + Debug command to ansible-installation onto an EC-2 instance. + """ + set_log_level(log_level) + run_install_dependencies(AnsibleAccess(), default_config_object, (HostInfo(host_name, ssh_private_key),)) diff --git a/exasol/ds/sandbox/cli/commands/make_ami_public.py b/exasol/ds/sandbox/cli/commands/make_ami_public.py new file mode 100644 index 00000000..bdb8c608 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/make_ami_public.py @@ -0,0 +1,24 @@ +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.id_options import id_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.export_vm.run_make_ami_public import run_make_ami_public +from exasol.ds.sandbox.lib.logging import set_log_level + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +@add_options(id_options) +def make_ami_public( + aws_profile: str, + asset_id: str, + log_level: str): + """ + Debug command which makes an existing AMI public. + """ + set_log_level(log_level) + run_make_ami_public(AwsAccess(aws_profile), AssetId(asset_id)) diff --git a/exasol/ds/sandbox/cli/commands/reset_password.py b/exasol/ds/sandbox/cli/commands/reset_password.py new file mode 100644 index 00000000..05411d6c --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/reset_password.py @@ -0,0 +1,27 @@ +import click + +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.ec2_options import ec2_host_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.setup_ec2.run_reset_password import run_reset_password + + +@cli.command() +@add_options(logging_options) +@add_options(ec2_host_options) +@click.option('--default-password', required=True, type=str, + help="The new (temporary) default password.") +def reset_password( + host_name: str, + ssh_private_key: str, + default_password: str, + log_level: str): + """ + Debug command to reset password on a remote EC-2-instance via ansible. + """ + set_log_level(log_level) + run_reset_password(AnsibleAccess(), default_password, (HostInfo(host_name, ssh_private_key),)) diff --git a/exasol/ds/sandbox/cli/commands/setup_ci_codebuild.py b/exasol/ds/sandbox/cli/commands/setup_ci_codebuild.py new file mode 100644 index 00000000..ce1c91d4 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/setup_ci_codebuild.py @@ -0,0 +1,20 @@ +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.setup_ci_codebuild.ci_codebuild import run_setup_ci_codebuild +from exasol.ds.sandbox.lib.logging import set_log_level + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +def setup_ci_codebuild( + aws_profile: str, + log_level: str): + """ + Command to deploy the CI CodeBuild stack + """ + set_log_level(log_level) + run_setup_ci_codebuild(AwsAccess(aws_profile)) diff --git a/exasol/ds/sandbox/cli/commands/setup_ec2.py b/exasol/ds/sandbox/cli/commands/setup_ec2.py new file mode 100644 index 00000000..48adfbe5 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/setup_ec2.py @@ -0,0 +1,31 @@ +from typing import Optional + +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.ec2_options import ec2_key_options +from exasol.ds.sandbox.cli.options.id_options import id_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_setup_ec2 + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +@add_options(ec2_key_options) +@add_options(id_options) +def setup_ec2( + aws_profile: str, + ec2_key_file: Optional[str], + ec2_key_name: Optional[str], + asset_id: str, + log_level: str): + """ + Debug command to test setup of an EC-2 instance. + """ + set_log_level(log_level) + run_setup_ec2(AwsAccess(aws_profile), ec2_key_file, ec2_key_name, AssetId(asset_id), default_config_object) diff --git a/exasol/ds/sandbox/cli/commands/setup_ec2_and_install_dependencies.py b/exasol/ds/sandbox/cli/commands/setup_ec2_and_install_dependencies.py new file mode 100644 index 00000000..5f2467f6 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/setup_ec2_and_install_dependencies.py @@ -0,0 +1,34 @@ +from typing import Optional + +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.ec2_options import ec2_key_options +from exasol.ds.sandbox.cli.options.id_options import id_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2_and_install_dependencies import \ + run_setup_ec2_and_install_dependencies + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +@add_options(ec2_key_options) +@add_options(id_options) +def setup_ec2_and_install_dependencies( + aws_profile: str, + ec2_key_file: Optional[str], + ec2_key_name: Optional[str], + asset_id: str, + log_level: str): + """ + Debug command to check setup and installation of an EC-2 instance. + """ + set_log_level(log_level) + run_setup_ec2_and_install_dependencies(AwsAccess(aws_profile), ec2_key_file, ec2_key_name, + AssetId(asset_id), AnsibleAccess(), default_config_object) diff --git a/exasol/ds/sandbox/cli/commands/setup_release_codebuild.py b/exasol/ds/sandbox/cli/commands/setup_release_codebuild.py new file mode 100644 index 00000000..40986442 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/setup_release_codebuild.py @@ -0,0 +1,21 @@ +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.setup_release_codebuild.release_codebuild import \ + run_setup_release_codebuild + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +def setup_release_codebuild( + aws_profile: str, + log_level: str): + """ + Command to deploy the Release CodeBuild stack + """ + set_log_level(log_level) + run_setup_release_codebuild(AwsAccess(aws_profile)) diff --git a/exasol/ds/sandbox/cli/commands/setup_vm_bucket.py b/exasol/ds/sandbox/cli/commands/setup_vm_bucket.py new file mode 100644 index 00000000..11ccfe4e --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/setup_vm_bucket.py @@ -0,0 +1,21 @@ +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import run_setup_vm_bucket + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +def setup_vm_bucket( + aws_profile: str, + log_level: str): + """ + Command to deploy the VM S3-Bucket + """ + set_log_level(log_level) + run_setup_vm_bucket(AwsAccess(aws_profile), default_config_object) diff --git a/exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py b/exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py new file mode 100644 index 00000000..7426297c --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py @@ -0,0 +1,26 @@ +import click + +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket_waf import run_setup_vm_bucket_waf + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +@click.option('--allowed-ip', type=str, + help="The allowed IP address for which CAPTCHA will not be applied.") +def setup_vm_bucket_waf( + aws_profile: str, + allowed_ip: str, + log_level: str): + """ + Command to deploy the VM S3-Bucket Web Application Firewall. Needs to run before deploying the VM Bucket itself. + """ + set_log_level(log_level) + run_setup_vm_bucket_waf(AwsAccess(aws_profile), allowed_ip, default_config_object) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/show_aws_assets.py b/exasol/ds/sandbox/cli/commands/show_aws_assets.py similarity index 64% rename from exasol_script_languages_developer_sandbox/cli/commands/show_aws_assets.py rename to exasol/ds/sandbox/cli/commands/show_aws_assets.py index 542ce8f9..63604237 100644 --- a/exasol_script_languages_developer_sandbox/cli/commands/show_aws_assets.py +++ b/exasol/ds/sandbox/cli/commands/show_aws_assets.py @@ -2,14 +2,14 @@ import click -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.asset_printing.print_assets import all_asset_types, print_assets -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.asset_printing.print_assets import all_asset_types, print_assets +from exasol.ds.sandbox.lib.logging import set_log_level @cli.command() diff --git a/exasol_script_languages_developer_sandbox/cli/commands/start_release_build.py b/exasol/ds/sandbox/cli/commands/start_release_build.py similarity index 51% rename from exasol_script_languages_developer_sandbox/cli/commands/start_release_build.py rename to exasol/ds/sandbox/cli/commands/start_release_build.py index 3457b2f9..9c1d80ee 100644 --- a/exasol_script_languages_developer_sandbox/cli/commands/start_release_build.py +++ b/exasol/ds/sandbox/cli/commands/start_release_build.py @@ -3,14 +3,14 @@ import click -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.release_build.run_release_build import run_start_release_build +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.release_build.run_release_build import run_start_release_build @cli.command() @@ -26,7 +26,7 @@ def start_release_build( upload_url: str, branch: str): """ - This command triggers the AWS release Codebuild to generate a new developer sandbox version. + This command triggers the AWS release Codebuild to generate a new sandbox version. """ set_log_level(log_level) run_start_release_build(AwsAccess(aws_profile), default_config_object, diff --git a/exasol_script_languages_developer_sandbox/cli/commands/start_test_release_build.py b/exasol/ds/sandbox/cli/commands/start_test_release_build.py similarity index 53% rename from exasol_script_languages_developer_sandbox/cli/commands/start_test_release_build.py rename to exasol/ds/sandbox/cli/commands/start_test_release_build.py index d12c2102..850dc8c3 100644 --- a/exasol_script_languages_developer_sandbox/cli/commands/start_test_release_build.py +++ b/exasol/ds/sandbox/cli/commands/start_test_release_build.py @@ -3,14 +3,14 @@ import click -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.github_release_access import GithubReleaseAccess -from exasol_script_languages_developer_sandbox.lib.release_build.run_release_build import run_start_test_release_build +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.github_release_access import GithubReleaseAccess +from exasol.ds.sandbox.lib.release_build.run_release_build import run_start_test_release_build @cli.command() @@ -27,7 +27,7 @@ def start_test_release_build( release_title: str ): """ - This command triggers the AWS release Codebuild to generate a new developer sandbox test version. + This command triggers the AWS release Codebuild to generate a new sandbox test version. """ set_log_level(log_level) gh_token = os.getenv("GITHUB_TOKEN") diff --git a/exasol/ds/sandbox/cli/commands/update_release.py b/exasol/ds/sandbox/cli/commands/update_release.py new file mode 100644 index 00000000..f8346eec --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/update_release.py @@ -0,0 +1,35 @@ +import os +from typing import Optional + +import click + +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.id_options import id_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.github_release_access import GithubReleaseAccess +from exasol.ds.sandbox.lib.update_release.run_update_release import run_update_release + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +@click.option('--release-id', type=int, required=True, + help="""The Github release id which will be updated.""") +@add_options(id_options) +def update_release( + aws_profile: Optional[str], + release_id: int, + asset_id: str, + log_level: str): + """ + This command attaches the links of the release assets (AMI, VM images) to the Github release, + indicated by parameter 'release-id'. + """ + set_log_level(log_level) + run_update_release(AwsAccess(aws_profile), GithubReleaseAccess(os.getenv("GITHUB_TOKEN")), + release_id, AssetId(asset_id)) diff --git a/exasol_script_languages_developer_sandbox/cli/common.py b/exasol/ds/sandbox/cli/common.py similarity index 100% rename from exasol_script_languages_developer_sandbox/cli/common.py rename to exasol/ds/sandbox/cli/common.py diff --git a/exasol_script_languages_developer_sandbox/cli/options/__init__.py b/exasol/ds/sandbox/cli/options/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/cli/options/__init__.py rename to exasol/ds/sandbox/cli/options/__init__.py diff --git a/exasol_script_languages_developer_sandbox/cli/options/aws_options.py b/exasol/ds/sandbox/cli/options/aws_options.py similarity index 100% rename from exasol_script_languages_developer_sandbox/cli/options/aws_options.py rename to exasol/ds/sandbox/cli/options/aws_options.py diff --git a/exasol_script_languages_developer_sandbox/cli/options/ec2_options.py b/exasol/ds/sandbox/cli/options/ec2_options.py similarity index 100% rename from exasol_script_languages_developer_sandbox/cli/options/ec2_options.py rename to exasol/ds/sandbox/cli/options/ec2_options.py diff --git a/exasol_script_languages_developer_sandbox/cli/options/id_options.py b/exasol/ds/sandbox/cli/options/id_options.py similarity index 78% rename from exasol_script_languages_developer_sandbox/cli/options/id_options.py rename to exasol/ds/sandbox/cli/options/id_options.py index 7b216d1d..06ad4f54 100644 --- a/exasol_script_languages_developer_sandbox/cli/options/id_options.py +++ b/exasol/ds/sandbox/cli/options/id_options.py @@ -1,6 +1,6 @@ import click -from exasol_script_languages_developer_sandbox.lib.config import SLC_VERSION +from exasol.ds.sandbox.lib.config import SLC_VERSION id_options = [ click.option('--asset-id', type=str, default=SLC_VERSION, diff --git a/exasol_script_languages_developer_sandbox/cli/options/logging.py b/exasol/ds/sandbox/cli/options/logging.py similarity index 82% rename from exasol_script_languages_developer_sandbox/cli/options/logging.py rename to exasol/ds/sandbox/cli/options/logging.py index f703521a..9cd31077 100644 --- a/exasol_script_languages_developer_sandbox/cli/options/logging.py +++ b/exasol/ds/sandbox/cli/options/logging.py @@ -1,6 +1,6 @@ import click -from exasol_script_languages_developer_sandbox.lib.logging import SUPPORTED_LOG_LEVELS +from exasol.ds.sandbox.lib.logging import SUPPORTED_LOG_LEVELS logging_options = [ click.option('--log-level', type=click.Choice(list(SUPPORTED_LOG_LEVELS.keys())), default="normal", diff --git a/exasol_script_languages_developer_sandbox/cli/options/vm_options.py b/exasol/ds/sandbox/cli/options/vm_options.py similarity index 85% rename from exasol_script_languages_developer_sandbox/cli/options/vm_options.py rename to exasol/ds/sandbox/cli/options/vm_options.py index d1b6a5b1..59fa2a6d 100644 --- a/exasol_script_languages_developer_sandbox/cli/options/vm_options.py +++ b/exasol/ds/sandbox/cli/options/vm_options.py @@ -1,6 +1,6 @@ import click -from exasol_script_languages_developer_sandbox.lib.export_vm.vm_disk_image_format \ +from exasol.ds.sandbox.lib.export_vm.vm_disk_image_format \ import VmDiskImageFormat vm_options = [ diff --git a/exasol_script_languages_developer_sandbox/lib/__init__.py b/exasol/ds/sandbox/lib/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/__init__.py rename to exasol/ds/sandbox/lib/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/ansible/__init__.py b/exasol/ds/sandbox/lib/ansible/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/ansible/__init__.py rename to exasol/ds/sandbox/lib/ansible/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_access.py b/exasol/ds/sandbox/lib/ansible/ansible_access.py similarity index 86% rename from exasol_script_languages_developer_sandbox/lib/ansible/ansible_access.py rename to exasol/ds/sandbox/lib/ansible/ansible_access.py index 3d2f7160..9a3770e0 100644 --- a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_access.py +++ b/exasol/ds/sandbox/lib/ansible/ansible_access.py @@ -2,7 +2,7 @@ import ansible_runner -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import AnsibleRunContext +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext class AnsibleException(RuntimeError): diff --git a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_context_manager.py b/exasol/ds/sandbox/lib/ansible/ansible_context_manager.py similarity index 76% rename from exasol_script_languages_developer_sandbox/lib/ansible/ansible_context_manager.py rename to exasol/ds/sandbox/lib/ansible/ansible_context_manager.py index a2b5ef0a..ffc876a1 100644 --- a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_context_manager.py +++ b/exasol/ds/sandbox/lib/ansible/ansible_context_manager.py @@ -2,9 +2,9 @@ from pathlib import Path from typing import Tuple -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_repository import AnsibleRepository -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_runner import AnsibleRunner +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.ansible.ansible_repository import AnsibleRepository +from exasol.ds.sandbox.lib.ansible.ansible_runner import AnsibleRunner class AnsibleContextManager: diff --git a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_repository.py b/exasol/ds/sandbox/lib/ansible/ansible_repository.py similarity index 91% rename from exasol_script_languages_developer_sandbox/lib/ansible/ansible_repository.py rename to exasol/ds/sandbox/lib/ansible/ansible_repository.py index 8e9fd658..c53329a3 100644 --- a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_repository.py +++ b/exasol/ds/sandbox/lib/ansible/ansible_repository.py @@ -1,7 +1,7 @@ from pathlib import Path import importlib_resources as ir -import exasol_script_languages_developer_sandbox.runtime.ansible -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType +import exasol.ds.sandbox.runtime.ansible +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType LOG = get_status_logger(LogType.ANSIBLE) @@ -71,4 +71,4 @@ def copy_to(self, target: Path) -> None: self.copy_importlib_resources_dir_tree(source_path, target) -default_repositories = (AnsibleResourceRepository(exasol_script_languages_developer_sandbox.runtime.ansible),) +default_repositories = (AnsibleResourceRepository(exasol.ds.sandbox.runtime.ansible),) diff --git a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_run_context.py b/exasol/ds/sandbox/lib/ansible/ansible_run_context.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/ansible/ansible_run_context.py rename to exasol/ds/sandbox/lib/ansible/ansible_run_context.py diff --git a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_runner.py b/exasol/ds/sandbox/lib/ansible/ansible_runner.py similarity index 63% rename from exasol_script_languages_developer_sandbox/lib/ansible/ansible_runner.py rename to exasol/ds/sandbox/lib/ansible/ansible_runner.py index e2a16b29..dd9c53dc 100644 --- a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_runner.py +++ b/exasol/ds/sandbox/lib/ansible/ansible_runner.py @@ -1,11 +1,11 @@ from pathlib import Path from typing import Tuple -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import AnsibleRunContext -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo -from exasol_script_languages_developer_sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.render_template import render_template LOG = get_status_logger(LogType.ANSIBLE) diff --git a/exasol_script_languages_developer_sandbox/lib/asset_id.py b/exasol/ds/sandbox/lib/asset_id.py similarity index 78% rename from exasol_script_languages_developer_sandbox/lib/asset_id.py rename to exasol/ds/sandbox/lib/asset_id.py index 4df71477..3bb65ffc 100644 --- a/exasol_script_languages_developer_sandbox/lib/asset_id.py +++ b/exasol/ds/sandbox/lib/asset_id.py @@ -1,5 +1,5 @@ class AssetId: - def __init__(self, asset_id: str, stack_prefix="EC2-SLC-DEV-SANDBOX-", ami_prefix="Exasol-SLC-Developer-Sandbox"): + def __init__(self, asset_id: str, stack_prefix="EC2-DATA-SCIENCE-SANDBOX-", ami_prefix="Exasol-Data-Science-Sandbox"): self._asset_id = asset_id self._stack_prefix = stack_prefix self._ami_prefix = ami_prefix @@ -23,4 +23,4 @@ def stack_prefix(self): def __repr__(self): return self._asset_id - BUCKET_PREFIX = "slc_developer_sandbox" + BUCKET_PREFIX = "data_science_sandbox" diff --git a/exasol_script_languages_developer_sandbox/lib/asset_printing/__init__.py b/exasol/ds/sandbox/lib/asset_printing/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/asset_printing/__init__.py rename to exasol/ds/sandbox/lib/asset_printing/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/asset_printing/mark_down_printer.py b/exasol/ds/sandbox/lib/asset_printing/mark_down_printer.py similarity index 89% rename from exasol_script_languages_developer_sandbox/lib/asset_printing/mark_down_printer.py rename to exasol/ds/sandbox/lib/asset_printing/mark_down_printer.py index 6ba324cb..b5dcd4bd 100644 --- a/exasol_script_languages_developer_sandbox/lib/asset_printing/mark_down_printer.py +++ b/exasol/ds/sandbox/lib/asset_printing/mark_down_printer.py @@ -4,7 +4,7 @@ import pandas as pd -from exasol_script_languages_developer_sandbox.lib.asset_printing.printing_factory import PrintingFactory, TablePrinter, \ +from exasol.ds.sandbox.lib.asset_printing.printing_factory import PrintingFactory, TablePrinter, \ TextPrinter, TextObject diff --git a/exasol_script_languages_developer_sandbox/lib/asset_printing/print_assets.py b/exasol/ds/sandbox/lib/asset_printing/print_assets.py similarity index 92% rename from exasol_script_languages_developer_sandbox/lib/asset_printing/print_assets.py rename to exasol/ds/sandbox/lib/asset_printing/print_assets.py index 4c722ae0..bb0362a6 100644 --- a/exasol_script_languages_developer_sandbox/lib/asset_printing/print_assets.py +++ b/exasol/ds/sandbox/lib/asset_printing/print_assets.py @@ -4,17 +4,17 @@ import humanfriendly -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.asset_printing.mark_down_printer import MarkdownPrintingFactory -from exasol_script_languages_developer_sandbox.lib.asset_printing.printing_factory import PrintingFactory, TextObject, \ +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.asset_printing.mark_down_printer import MarkdownPrintingFactory +from exasol.ds.sandbox.lib.asset_printing.printing_factory import PrintingFactory, TextObject, \ HighlightedTextObject -from exasol_script_languages_developer_sandbox.lib.asset_printing.rich_console_printer import RichConsolePrintingFactory -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.asset_printing.rich_console_printer import RichConsolePrintingFactory +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from enum import Enum -from exasol_script_languages_developer_sandbox.lib.aws_access.cloudformation_stack import CloudformationStack -from exasol_script_languages_developer_sandbox.lib.tags import DEFAULT_TAG_KEY -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket, find_url_for_bucket +from exasol.ds.sandbox.lib.aws_access.cloudformation_stack import CloudformationStack +from exasol.ds.sandbox.lib.tags import DEFAULT_TAG_KEY +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import find_vm_bucket, find_url_for_bucket class AssetTypes(Enum): @@ -144,15 +144,15 @@ def print_s3_objects(aws_access: AwsAccess, asset_id: Optional[AssetId], printin # How the filtering works: # 1. The VM are stored under following location in the S3 Bucket: $BUCKET_PREFIX/$AssetId/name.$VM_FORMAT - # For example "slc_developer_sandbox/5.0.0/export-ami-01be860e6a6a98bf8.vhd" + # For example "data_science_sandbox/5.0.0/export-ami-01be860e6a6a98bf8.vhd" # 2. Because S3 list_s3_object does not support wildcards, # we need to implement our own wildcard implementation here. - # We call list_s3_object with the standard prefix (e.g. "slc_developer_sandbox"), + # We call list_s3_object with the standard prefix (e.g. "data_science_sandbox"), # which returns ALL stored vm objects. # 3. If no filter is given (asset_id == None), "prefix" will be empty, and we return all s3 objects # 4. If the variable "prefix" is not empty, we need to ensure that it ends with a wildcard, so that the matching # works correctly. - # => Assume that a filter is given "5.0.0". Variable prefix would be "slc_developer_sandbox/5.0.0". + # => Assume that a filter is given "5.0.0". Variable prefix would be "data_science_sandbox/5.0.0". s3_objects = aws_access.list_s3_objects(bucket=vm_bucket, prefix=AssetId.BUCKET_PREFIX) diff --git a/exasol_script_languages_developer_sandbox/lib/asset_printing/printing_factory.py b/exasol/ds/sandbox/lib/asset_printing/printing_factory.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/asset_printing/printing_factory.py rename to exasol/ds/sandbox/lib/asset_printing/printing_factory.py diff --git a/exasol_script_languages_developer_sandbox/lib/asset_printing/rich_console_printer.py b/exasol/ds/sandbox/lib/asset_printing/rich_console_printer.py similarity index 90% rename from exasol_script_languages_developer_sandbox/lib/asset_printing/rich_console_printer.py rename to exasol/ds/sandbox/lib/asset_printing/rich_console_printer.py index 71370366..58a5e06f 100644 --- a/exasol_script_languages_developer_sandbox/lib/asset_printing/rich_console_printer.py +++ b/exasol/ds/sandbox/lib/asset_printing/rich_console_printer.py @@ -5,7 +5,7 @@ from rich.console import Console from rich.text import Text -from exasol_script_languages_developer_sandbox.lib.asset_printing.printing_factory import PrintingFactory, TablePrinter, \ +from exasol.ds.sandbox.lib.asset_printing.printing_factory import PrintingFactory, TablePrinter, \ TextPrinter, TextObject diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/__init__.py b/exasol/ds/sandbox/lib/aws_access/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/__init__.py rename to exasol/ds/sandbox/lib/aws_access/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/ami.py b/exasol/ds/sandbox/lib/aws_access/ami.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/ami.py rename to exasol/ds/sandbox/lib/aws_access/ami.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/aws_access.py b/exasol/ds/sandbox/lib/aws_access/aws_access.py similarity index 93% rename from exasol_script_languages_developer_sandbox/lib/aws_access/aws_access.py rename to exasol/ds/sandbox/lib/aws_access/aws_access.py index 90280786..847f5b5f 100644 --- a/exasol_script_languages_developer_sandbox/lib/aws_access/aws_access.py +++ b/exasol/ds/sandbox/lib/aws_access/aws_access.py @@ -4,20 +4,20 @@ import boto3 import botocore -from exasol_script_languages_developer_sandbox.lib.aws_access.ami import Ami -from exasol_script_languages_developer_sandbox.lib.aws_access.cloudformation_stack import CloudformationStack -from exasol_script_languages_developer_sandbox.lib.aws_access.deployer import Deployer -from exasol_script_languages_developer_sandbox.lib.aws_access.ec2_instance import EC2Instance -from exasol_script_languages_developer_sandbox.lib.aws_access.ec2_instance_status import EC2InstanceStatus -from exasol_script_languages_developer_sandbox.lib.aws_access.export_image_task import ExportImageTask -from exasol_script_languages_developer_sandbox.lib.aws_access.key_pair import KeyPair -from exasol_script_languages_developer_sandbox.lib.aws_access.s3_object import S3Object -from exasol_script_languages_developer_sandbox.lib.aws_access.snapshot import Snapshot -from exasol_script_languages_developer_sandbox.lib.aws_access.stack_resource import StackResource -from exasol_script_languages_developer_sandbox.lib.aws_access.waiter.codebuild_waiter import CodeBuildWaiter -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.tags import create_default_asset_tag -from exasol_script_languages_developer_sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat +from exasol.ds.sandbox.lib.aws_access.ami import Ami +from exasol.ds.sandbox.lib.aws_access.cloudformation_stack import CloudformationStack +from exasol.ds.sandbox.lib.aws_access.deployer import Deployer +from exasol.ds.sandbox.lib.aws_access.ec2_instance import EC2Instance +from exasol.ds.sandbox.lib.aws_access.ec2_instance_status import EC2InstanceStatus +from exasol.ds.sandbox.lib.aws_access.export_image_task import ExportImageTask +from exasol.ds.sandbox.lib.aws_access.key_pair import KeyPair +from exasol.ds.sandbox.lib.aws_access.s3_object import S3Object +from exasol.ds.sandbox.lib.aws_access.snapshot import Snapshot +from exasol.ds.sandbox.lib.aws_access.stack_resource import StackResource +from exasol.ds.sandbox.lib.aws_access.waiter.codebuild_waiter import CodeBuildWaiter +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.tags import create_default_asset_tag +from exasol.ds.sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat LOG = get_status_logger(LogType.AWS_ACCESS) diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/cloudformation_stack.py b/exasol/ds/sandbox/lib/aws_access/cloudformation_stack.py similarity index 94% rename from exasol_script_languages_developer_sandbox/lib/aws_access/cloudformation_stack.py rename to exasol/ds/sandbox/lib/aws_access/cloudformation_stack.py index 12e6caf4..a53bcf26 100644 --- a/exasol_script_languages_developer_sandbox/lib/aws_access/cloudformation_stack.py +++ b/exasol/ds/sandbox/lib/aws_access/cloudformation_stack.py @@ -2,7 +2,7 @@ from datetime import datetime from typing import List, Dict, Optional -from exasol_script_languages_developer_sandbox.lib.aws_access.common import get_value_safe +from exasol.ds.sandbox.lib.aws_access.common import get_value_safe class CloudformationStack: diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/common.py b/exasol/ds/sandbox/lib/aws_access/common.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/common.py rename to exasol/ds/sandbox/lib/aws_access/common.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/deployer.py b/exasol/ds/sandbox/lib/aws_access/deployer.py similarity index 98% rename from exasol_script_languages_developer_sandbox/lib/aws_access/deployer.py rename to exasol/ds/sandbox/lib/aws_access/deployer.py index ef643943..6df0903f 100644 --- a/exasol_script_languages_developer_sandbox/lib/aws_access/deployer.py +++ b/exasol/ds/sandbox/lib/aws_access/deployer.py @@ -28,10 +28,12 @@ "ChangeSetResult", ["changeset_id", "changeset_type"]) +DEFAULT_CHANGE_SET_PREFIX="dss-ci-setup-deploy-" + + class Deployer(object): - def __init__(self, cloudformation_client, - changeset_prefix="slc-ci-setup-deploy-"): + def __init__(self, cloudformation_client, changeset_prefix=DEFAULT_CHANGE_SET_PREFIX): self._client = cloudformation_client self.changeset_prefix = changeset_prefix diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/ec2_instance.py b/exasol/ds/sandbox/lib/aws_access/ec2_instance.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/ec2_instance.py rename to exasol/ds/sandbox/lib/aws_access/ec2_instance.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/ec2_instance_status.py b/exasol/ds/sandbox/lib/aws_access/ec2_instance_status.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/ec2_instance_status.py rename to exasol/ds/sandbox/lib/aws_access/ec2_instance_status.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/export_image_task.py b/exasol/ds/sandbox/lib/aws_access/export_image_task.py similarity index 93% rename from exasol_script_languages_developer_sandbox/lib/aws_access/export_image_task.py rename to exasol/ds/sandbox/lib/aws_access/export_image_task.py index 36d9b08d..bb6cd4e7 100644 --- a/exasol_script_languages_developer_sandbox/lib/aws_access/export_image_task.py +++ b/exasol/ds/sandbox/lib/aws_access/export_image_task.py @@ -1,6 +1,6 @@ from typing import Optional, List, Dict -from exasol_script_languages_developer_sandbox.lib.aws_access.common import get_value_safe +from exasol.ds.sandbox.lib.aws_access.common import get_value_safe class ExportImageTask: diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/key_pair.py b/exasol/ds/sandbox/lib/aws_access/key_pair.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/key_pair.py rename to exasol/ds/sandbox/lib/aws_access/key_pair.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/s3_object.py b/exasol/ds/sandbox/lib/aws_access/s3_object.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/s3_object.py rename to exasol/ds/sandbox/lib/aws_access/s3_object.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/snapshot.py b/exasol/ds/sandbox/lib/aws_access/snapshot.py similarity index 92% rename from exasol_script_languages_developer_sandbox/lib/aws_access/snapshot.py rename to exasol/ds/sandbox/lib/aws_access/snapshot.py index a0ddd4b8..e725c02c 100644 --- a/exasol_script_languages_developer_sandbox/lib/aws_access/snapshot.py +++ b/exasol/ds/sandbox/lib/aws_access/snapshot.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Optional, List, Dict -from exasol_script_languages_developer_sandbox.lib.aws_access.common import get_value_safe +from exasol.ds.sandbox.lib.aws_access.common import get_value_safe class Snapshot: diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/stack_resource.py b/exasol/ds/sandbox/lib/aws_access/stack_resource.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/stack_resource.py rename to exasol/ds/sandbox/lib/aws_access/stack_resource.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/waiter/__init__.py b/exasol/ds/sandbox/lib/aws_access/waiter/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/waiter/__init__.py rename to exasol/ds/sandbox/lib/aws_access/waiter/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/waiter/codebuild_waiter.py b/exasol/ds/sandbox/lib/aws_access/waiter/codebuild_waiter.py similarity index 96% rename from exasol_script_languages_developer_sandbox/lib/aws_access/waiter/codebuild_waiter.py rename to exasol/ds/sandbox/lib/aws_access/waiter/codebuild_waiter.py index 6ba9472b..52e774a8 100644 --- a/exasol_script_languages_developer_sandbox/lib/aws_access/waiter/codebuild_waiter.py +++ b/exasol/ds/sandbox/lib/aws_access/waiter/codebuild_waiter.py @@ -1,7 +1,7 @@ import time from typing import Iterable, Any -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType LOG = get_status_logger(LogType.AWS_ACCESS) diff --git a/exasol_script_languages_developer_sandbox/lib/config.py b/exasol/ds/sandbox/lib/config.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/config.py rename to exasol/ds/sandbox/lib/config.py diff --git a/exasol_script_languages_developer_sandbox/lib/export_vm/__init__.py b/exasol/ds/sandbox/lib/export_vm/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/export_vm/__init__.py rename to exasol/ds/sandbox/lib/export_vm/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/export_vm/rename_s3_objects.py b/exasol/ds/sandbox/lib/export_vm/rename_s3_objects.py similarity index 76% rename from exasol_script_languages_developer_sandbox/lib/export_vm/rename_s3_objects.py rename to exasol/ds/sandbox/lib/export_vm/rename_s3_objects.py index db634c6a..9acb2c95 100644 --- a/exasol_script_languages_developer_sandbox/lib/export_vm/rename_s3_objects.py +++ b/exasol/ds/sandbox/lib/export_vm/rename_s3_objects.py @@ -1,7 +1,7 @@ -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.aws_access.export_image_task import ExportImageTask -from exasol_script_languages_developer_sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.aws_access.export_image_task import ExportImageTask +from exasol.ds.sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat def build_image_source(prefix: str, export_image_task_id: str, vm_image_format: VmDiskImageFormat) -> str: @@ -14,7 +14,7 @@ def build_image_source(prefix: str, export_image_task_id: str, vm_image_format: def build_image_destination(prefix: str, asset_id: AssetId, vm_image_format: VmDiskImageFormat) -> str: img_format = vm_image_format.value.lower() - return "{bucket_prefix}exasol-script-languages-developer-sandbox-{asset_id}.{img_format}".format( + return "{bucket_prefix}exasol-data-science-sandbox-{asset_id}.{img_format}".format( bucket_prefix=prefix, asset_id=str(asset_id), img_format=img_format) @@ -26,7 +26,7 @@ def rename_image_in_s3(aws_access: AwsAccess, export_image_task: ExportImageTask """ Renames the resulting S3 object of an export-image-task. The source objects always have the format "$export-image-task-id.$format". - The destination objects always have the format "exasol-script-languages-developer-sandbox-{asset_id}.{img_format}" + The destination objects always have the format "exasol-data-science-sandbox-{asset_id}.{img_format}" The bucket and prefix in bucket do not change. :param aws_access: Access proxy to Aws :param export_image_task: The export image task which is expected to be completed successfully. diff --git a/exasol_script_languages_developer_sandbox/lib/export_vm/run_export_vm.py b/exasol/ds/sandbox/lib/export_vm/run_export_vm.py similarity index 87% rename from exasol_script_languages_developer_sandbox/lib/export_vm/run_export_vm.py rename to exasol/ds/sandbox/lib/export_vm/run_export_vm.py index f5875928..cdd49181 100644 --- a/exasol_script_languages_developer_sandbox/lib/export_vm/run_export_vm.py +++ b/exasol/ds/sandbox/lib/export_vm/run_export_vm.py @@ -2,16 +2,16 @@ from dataclasses import dataclass from typing import Tuple -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.aws_access.export_image_task import ExportImageTask -from exasol_script_languages_developer_sandbox.lib.export_vm.rename_s3_objects import rename_image_in_s3 -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.setup_ec2.cf_stack import find_ec2_instance_in_cf_stack -from exasol_script_languages_developer_sandbox.lib.asset_printing.print_assets import print_assets -from exasol_script_languages_developer_sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket, find_vm_import_role +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.aws_access.export_image_task import ExportImageTask +from exasol.ds.sandbox.lib.export_vm.rename_s3_objects import rename_image_in_s3 +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.setup_ec2.cf_stack import find_ec2_instance_in_cf_stack +from exasol.ds.sandbox.lib.asset_printing.print_assets import print_assets +from exasol.ds.sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import find_vm_bucket, find_vm_import_role LOG = get_status_logger(LogType.EXPORT) diff --git a/exasol_script_languages_developer_sandbox/lib/export_vm/run_make_ami_public.py b/exasol/ds/sandbox/lib/export_vm/run_make_ami_public.py similarity index 72% rename from exasol_script_languages_developer_sandbox/lib/export_vm/run_make_ami_public.py rename to exasol/ds/sandbox/lib/export_vm/run_make_ami_public.py index eec0beb2..e70f18fa 100644 --- a/exasol_script_languages_developer_sandbox/lib/export_vm/run_make_ami_public.py +++ b/exasol/ds/sandbox/lib/export_vm/run_make_ami_public.py @@ -1,7 +1,7 @@ -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.tags import DEFAULT_TAG_KEY +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.tags import DEFAULT_TAG_KEY LOG = get_status_logger(LogType.EXPORT) diff --git a/exasol_script_languages_developer_sandbox/lib/export_vm/vm_disk_image_format.py b/exasol/ds/sandbox/lib/export_vm/vm_disk_image_format.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/export_vm/vm_disk_image_format.py rename to exasol/ds/sandbox/lib/export_vm/vm_disk_image_format.py diff --git a/exasol_script_languages_developer_sandbox/lib/github_release_access.py b/exasol/ds/sandbox/lib/github_release_access.py similarity index 93% rename from exasol_script_languages_developer_sandbox/lib/github_release_access.py rename to exasol/ds/sandbox/lib/github_release_access.py index 2708abf7..842d9ccf 100644 --- a/exasol_script_languages_developer_sandbox/lib/github_release_access.py +++ b/exasol/ds/sandbox/lib/github_release_access.py @@ -3,7 +3,7 @@ from github import Github, GithubException from github.Repository import Repository -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType LOG = get_status_logger(LogType.RELEASE_ACCESS) @@ -56,5 +56,5 @@ def upload(self, archive_path: str, label: str, release_id: int, content_type: s @property def _get_repo(self) -> Repository: gh = Github(self._gh_token) - gh_repo = gh.get_repo("exasol/script-languages-developer-sandbox") + gh_repo = gh.get_repo("exasol/data-science-sandbox") return gh_repo diff --git a/exasol_script_languages_developer_sandbox/lib/logging.py b/exasol/ds/sandbox/lib/logging.py similarity index 95% rename from exasol_script_languages_developer_sandbox/lib/logging.py rename to exasol/ds/sandbox/lib/logging.py index e6b0132d..5d54d2a5 100644 --- a/exasol_script_languages_developer_sandbox/lib/logging.py +++ b/exasol/ds/sandbox/lib/logging.py @@ -20,7 +20,7 @@ class LogType(Enum): def get_status_logger(log_type: LogType) -> logging.Logger: - return logging.getLogger(f"eslds-{log_type.value}") + return logging.getLogger(f"edss-{log_type.value}") def set_log_level(level: str): diff --git a/exasol_script_languages_developer_sandbox/lib/release_build/__init__.py b/exasol/ds/sandbox/lib/release_build/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/release_build/__init__.py rename to exasol/ds/sandbox/lib/release_build/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/release_build/run_release_build.py b/exasol/ds/sandbox/lib/release_build/run_release_build.py similarity index 88% rename from exasol_script_languages_developer_sandbox/lib/release_build/run_release_build.py rename to exasol/ds/sandbox/lib/release_build/run_release_build.py index 9281dfa6..94627525 100644 --- a/exasol_script_languages_developer_sandbox/lib/release_build/run_release_build.py +++ b/exasol/ds/sandbox/lib/release_build/run_release_build.py @@ -2,12 +2,12 @@ import re from typing import Tuple, Dict, List -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.aws_access.stack_resource import StackResource -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.github_release_access import GithubReleaseAccess -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.setup_release_codebuild.release_codebuild import \ +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.aws_access.stack_resource import StackResource +from exasol.ds.sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.github_release_access import GithubReleaseAccess +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.setup_release_codebuild.release_codebuild import \ RELEASE_CODE_BUILD_STACK_NAME LOG = get_status_logger(LogType.RELEASE_BUILD) diff --git a/exasol_script_languages_developer_sandbox/lib/render_template.py b/exasol/ds/sandbox/lib/render_template.py similarity index 90% rename from exasol_script_languages_developer_sandbox/lib/render_template.py rename to exasol/ds/sandbox/lib/render_template.py index 6504321e..b3b2fabb 100644 --- a/exasol_script_languages_developer_sandbox/lib/render_template.py +++ b/exasol/ds/sandbox/lib/render_template.py @@ -3,7 +3,7 @@ def render_template(template: str, **kwargs): - env = jinja2.Environment(loader=jinja2.PackageLoader("exasol_script_languages_developer_sandbox"), + env = jinja2.Environment(loader=jinja2.PackageLoader("exasol.ds.sandbox"), autoescape=jinja2.select_autoescape(), keep_trailing_newline=True, undefined=StrictUndefined) t = env.get_template(template) diff --git a/exasol_script_languages_developer_sandbox/lib/run_create_vm.py b/exasol/ds/sandbox/lib/run_create_vm.py similarity index 65% rename from exasol_script_languages_developer_sandbox/lib/run_create_vm.py rename to exasol/ds/sandbox/lib/run_create_vm.py index 7f53bdb4..6ee5e4e5 100644 --- a/exasol_script_languages_developer_sandbox/lib/run_create_vm.py +++ b/exasol/ds/sandbox/lib/run_create_vm.py @@ -1,24 +1,24 @@ import time from typing import Tuple, Optional -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ default_repositories -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import \ +from exasol.ds.sandbox.lib.ansible.ansible_run_context import \ reset_password_ansible_run_context, default_ansible_run_context -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.export_vm.run_make_ami_public import run_make_ami_public -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.export_vm.run_make_ami_public import run_make_ami_public +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo -from exasol_script_languages_developer_sandbox.lib.export_vm.run_export_vm import export_vm -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_reset_password import run_reset_password -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.export_vm.run_export_vm import export_vm +from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies +from exasol.ds.sandbox.lib.setup_ec2.run_reset_password import run_reset_password +from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ EC2StackLifecycleContextManager -from exasol_script_languages_developer_sandbox.lib.setup_ec2.source_ami import find_source_ami +from exasol.ds.sandbox.lib.setup_ec2.source_ami import find_source_ami LOG = get_status_logger(LogType.CREATE_VM) @@ -36,7 +36,8 @@ def run_create_vm(aws_access: AwsAccess, ec2_key_file: Optional[str], ec2_key_na ansible_repositories: Tuple[AnsibleRepository, ...] = default_repositories) -> None: """ Runs setup of an EC2 instance and then installs all dependencies via Ansible, - and finally exports the VM to the S3 Bucket (which must be already created by the stack ("VM-SLC-Bucket"). + and finally exports the VM to the S3 Bucket (which must be already created + by the stack "DATA-SCIENCE-SANDBOX-VM-Bucket"). If anything goes wrong the cloudformation stack of the EC-2 instance will be removed. For debuging you can use the available debug commands. """ diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/__init__.py b/exasol/ds/sandbox/lib/setup_ci_codebuild/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/__init__.py rename to exasol/ds/sandbox/lib/setup_ci_codebuild/__init__.py diff --git a/exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py b/exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py new file mode 100644 index 00000000..d8112607 --- /dev/null +++ b/exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py @@ -0,0 +1,15 @@ +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import find_vm_bucket + +STACK_NAME = "DATA-SCIENCE-SANDBOX-CI-TEST-CODEBUILD" + +LOG = get_status_logger(LogType.SETUP_CI_CODEBUILD) + + +def run_setup_ci_codebuild(aws_access: AwsAccess) -> None: + yml = render_template("ci_code_build.jinja.yaml", vm_bucket=find_vm_bucket(aws_access)) + aws_access.upload_cloudformation_stack(yml, STACK_NAME) + LOG.info(f"Deployed cloudformation stack {STACK_NAME}") + diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/__init__.py b/exasol/ds/sandbox/lib/setup_ec2/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/__init__.py rename to exasol/ds/sandbox/lib/setup_ec2/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/cf_stack.py b/exasol/ds/sandbox/lib/setup_ec2/cf_stack.py similarity index 85% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/cf_stack.py rename to exasol/ds/sandbox/lib/setup_ec2/cf_stack.py index 4beec59b..ec275463 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ec2/cf_stack.py +++ b/exasol/ds/sandbox/lib/setup_ec2/cf_stack.py @@ -1,12 +1,12 @@ from __future__ import annotations from typing import Optional -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.setup_ec2.random_string_generator import get_random_str_of_length_n -from exasol_script_languages_developer_sandbox.lib.render_template import render_template -from exasol_script_languages_developer_sandbox.lib.tags import DEFAULT_TAG_KEY, create_default_asset_tag +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.setup_ec2.random_string_generator import get_random_str_of_length_n +from exasol.ds.sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.tags import DEFAULT_TAG_KEY, create_default_asset_tag _MAX_ATTEMPTS_TO_FIND_STACK_NAME = 3 diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/host_info.py b/exasol/ds/sandbox/lib/setup_ec2/host_info.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/host_info.py rename to exasol/ds/sandbox/lib/setup_ec2/host_info.py diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/key_file_manager.py b/exasol/ds/sandbox/lib/setup_ec2/key_file_manager.py similarity index 89% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/key_file_manager.py rename to exasol/ds/sandbox/lib/setup_ec2/key_file_manager.py index 680880e5..7b9e3c5c 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ec2/key_file_manager.py +++ b/exasol/ds/sandbox/lib/setup_ec2/key_file_manager.py @@ -2,9 +2,9 @@ from tempfile import mkstemp from typing import Optional -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.setup_ec2.random_string_generator import get_random_str +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.setup_ec2.random_string_generator import get_random_str LOG = get_status_logger(LogType.SETUP) diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/random_string_generator.py b/exasol/ds/sandbox/lib/setup_ec2/random_string_generator.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/random_string_generator.py rename to exasol/ds/sandbox/lib/setup_ec2/random_string_generator.py diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_install_dependencies.py b/exasol/ds/sandbox/lib/setup_ec2/run_install_dependencies.py similarity index 71% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/run_install_dependencies.py rename to exasol/ds/sandbox/lib/setup_ec2/run_install_dependencies.py index cefc5e36..c3c092ec 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_install_dependencies.py +++ b/exasol/ds/sandbox/lib/setup_ec2/run_install_dependencies.py @@ -1,15 +1,15 @@ from importlib.metadata import version from typing import Tuple -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_context_manager import AnsibleContextManager -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.ansible.ansible_context_manager import AnsibleContextManager +from exasol.ds.sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ default_repositories -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ default_ansible_run_context -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo def run_install_dependencies(ansible_access: AnsibleAccess, diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_reset_password.py b/exasol/ds/sandbox/lib/setup_ec2/run_reset_password.py similarity index 77% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/run_reset_password.py rename to exasol/ds/sandbox/lib/setup_ec2/run_reset_password.py index 17787087..021446db 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_reset_password.py +++ b/exasol/ds/sandbox/lib/setup_ec2/run_reset_password.py @@ -1,13 +1,13 @@ from typing import Tuple -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_context_manager import AnsibleContextManager -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.ansible.ansible_context_manager import AnsibleContextManager +from exasol.ds.sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ default_repositories -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ reset_password_ansible_run_context -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo def run_reset_password(ansible_access: AnsibleAccess, default_password: str, diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_setup_ec2.py b/exasol/ds/sandbox/lib/setup_ec2/run_setup_ec2.py similarity index 89% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/run_setup_ec2.py rename to exasol/ds/sandbox/lib/setup_ec2/run_setup_ec2.py index c1436087..7f5b15aa 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_setup_ec2.py +++ b/exasol/ds/sandbox/lib/setup_ec2/run_setup_ec2.py @@ -2,16 +2,16 @@ import time from typing import Optional, Tuple, Iterator -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.aws_access.ec2_instance import EC2Instance -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.setup_ec2.cf_stack import CloudformationStack, \ +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.aws_access.ec2_instance import EC2Instance +from exasol.ds.sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.setup_ec2.cf_stack import CloudformationStack, \ CloudformationStackContextManager -from exasol_script_languages_developer_sandbox.lib.setup_ec2.key_file_manager import KeyFileManager, \ +from exasol.ds.sandbox.lib.setup_ec2.key_file_manager import KeyFileManager, \ KeyFileManagerContextManager -from exasol_script_languages_developer_sandbox.lib.setup_ec2.source_ami import find_source_ami +from exasol.ds.sandbox.lib.setup_ec2.source_ami import find_source_ami LOG = get_status_logger(LogType.SETUP) diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_setup_ec2_and_install_dependencies.py b/exasol/ds/sandbox/lib/setup_ec2/run_setup_ec2_and_install_dependencies.py similarity index 73% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/run_setup_ec2_and_install_dependencies.py rename to exasol/ds/sandbox/lib/setup_ec2/run_setup_ec2_and_install_dependencies.py index 3676f9a7..e38a0275 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_setup_ec2_and_install_dependencies.py +++ b/exasol/ds/sandbox/lib/setup_ec2/run_setup_ec2_and_install_dependencies.py @@ -1,22 +1,22 @@ import signal import time from typing import Tuple, Optional -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ default_repositories -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ default_ansible_run_context -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ +from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies +from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ EC2StackLifecycleContextManager -from exasol_script_languages_developer_sandbox.lib.setup_ec2.source_ami import find_source_ami +from exasol.ds.sandbox.lib.setup_ec2.source_ami import find_source_ami LOG = get_status_logger(LogType.SETUP) diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/source_ami.py b/exasol/ds/sandbox/lib/setup_ec2/source_ami.py similarity index 64% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/source_ami.py rename to exasol/ds/sandbox/lib/setup_ec2/source_ami.py index d17d629c..0c21de64 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ec2/source_ami.py +++ b/exasol/ds/sandbox/lib/setup_ec2/source_ami.py @@ -1,7 +1,7 @@ from typing import Dict -from exasol_script_languages_developer_sandbox.lib.aws_access.ami import Ami -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.aws_access.ami import Ami +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess def find_source_ami(aws_access: AwsAccess, filters: Dict[str, str]) -> Ami: diff --git a/exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/__init__.py b/exasol/ds/sandbox/lib/setup_release_codebuild/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/__init__.py rename to exasol/ds/sandbox/lib/setup_release_codebuild/__init__.py diff --git a/exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py b/exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py new file mode 100644 index 00000000..5fb3beba --- /dev/null +++ b/exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py @@ -0,0 +1,15 @@ +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import find_vm_bucket + +RELEASE_CODE_BUILD_STACK_NAME = "DATA-SCIENCE-SANDBOX-RELEASE-CODEBUILD" + +LOG = get_status_logger(LogType.SETUP_RELEASE_BUILD) + + +def run_setup_release_codebuild(aws_access: AwsAccess) -> None: + yml = render_template("release_code_build.jinja.yaml", vm_bucket=find_vm_bucket(aws_access)) + aws_access.upload_cloudformation_stack(yml, RELEASE_CODE_BUILD_STACK_NAME) + LOG.info(f"Deployed cloudformation stack {RELEASE_CODE_BUILD_STACK_NAME}") + diff --git a/exasol_script_languages_developer_sandbox/lib/tags.py b/exasol/ds/sandbox/lib/tags.py similarity index 86% rename from exasol_script_languages_developer_sandbox/lib/tags.py rename to exasol/ds/sandbox/lib/tags.py index 129c6d89..0b4100d0 100644 --- a/exasol_script_languages_developer_sandbox/lib/tags.py +++ b/exasol/ds/sandbox/lib/tags.py @@ -1,4 +1,4 @@ -DEFAULT_TAG_KEY = "exa_slc_id" +DEFAULT_TAG_KEY = "exa_dss_id" def create_default_asset_tag(value: str) -> list: diff --git a/exasol_script_languages_developer_sandbox/lib/update_release/__init__.py b/exasol/ds/sandbox/lib/update_release/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/update_release/__init__.py rename to exasol/ds/sandbox/lib/update_release/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/update_release/run_update_release.py b/exasol/ds/sandbox/lib/update_release/run_update_release.py similarity index 65% rename from exasol_script_languages_developer_sandbox/lib/update_release/run_update_release.py rename to exasol/ds/sandbox/lib/update_release/run_update_release.py index 51a7329c..5f507415 100644 --- a/exasol_script_languages_developer_sandbox/lib/update_release/run_update_release.py +++ b/exasol/ds/sandbox/lib/update_release/run_update_release.py @@ -1,11 +1,11 @@ import logging import tempfile -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.asset_printing.print_assets import print_assets, AssetTypes -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.github_release_access import GithubReleaseAccess -from exasol_script_languages_developer_sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.asset_printing.print_assets import print_assets, AssetTypes +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.github_release_access import GithubReleaseAccess +from exasol.ds.sandbox.lib.render_template import render_template def run_update_release(aws_access: AwsAccess, gh_access: GithubReleaseAccess, diff --git a/exasol_script_languages_developer_sandbox/lib/vm_bucket/__init__.py b/exasol/ds/sandbox/lib/vm_bucket/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/vm_bucket/__init__.py rename to exasol/ds/sandbox/lib/vm_bucket/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket.py b/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket.py similarity index 80% rename from exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket.py rename to exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket.py index 4cb5d4be..8bd5864f 100644 --- a/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket.py +++ b/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket.py @@ -1,12 +1,12 @@ -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.render_template import render_template from enum import Enum -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket_waf import find_acl_arn +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket_waf import find_acl_arn -STACK_NAME = "DEVELOPER-SANDBOX-VM-SLC-Bucket" +STACK_NAME = "DATA-SCIENCE-SANDBOX-VM-Bucket" class OutputKey(Enum): diff --git a/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket_waf.py b/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py similarity index 77% rename from exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket_waf.py rename to exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py index f6b77fa7..59f7e703 100644 --- a/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket_waf.py +++ b/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py @@ -1,9 +1,9 @@ -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.render_template import render_template -STACK_NAME = "DEVELOPER-SANDBOX-VM-SLC-Bucket-WAF" +STACK_NAME = "DATA-SCIENCE-SANDBOX-VM-Bucket-WAF" LOG = get_status_logger(LogType.VM_BUCKET) @@ -18,7 +18,7 @@ def get_cloudformation_template(allowed_ip: str) -> str: def run_setup_vm_bucket_waf(aws_access: AwsAccess, allowed_ip: str, config: ConfigObject) -> None: """ Deploys the WAF Cloudformation stack. - It automatically deploys to AWS region indicated by config paramaer "waf_region". + It automatically deploys to AWS region indicated by configuration parameter "waf_region". """ yml = get_cloudformation_template(allowed_ip=allowed_ip) diff --git a/exasol/ds/sandbox/main.py b/exasol/ds/sandbox/main.py new file mode 100755 index 00000000..926cba03 --- /dev/null +++ b/exasol/ds/sandbox/main.py @@ -0,0 +1,7 @@ +#! /usr/bin/env python3 +# +import exasol.ds.sandbox.cli.commands +from exasol.ds.sandbox.cli.cli import cli + +if __name__ == '__main__': + cli() diff --git a/exasol_script_languages_developer_sandbox/runtime/__init__.py b/exasol/ds/sandbox/runtime/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/__init__.py rename to exasol/ds/sandbox/runtime/__init__.py diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/reset_password.yml b/exasol/ds/sandbox/runtime/ansible/reset_password.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/reset_password.yml rename to exasol/ds/sandbox/runtime/ansible/reset_password.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/reset_password_tasks.yml b/exasol/ds/sandbox/runtime/ansible/reset_password_tasks.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/reset_password_tasks.yml rename to exasol/ds/sandbox/runtime/ansible/reset_password_tasks.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/docker/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/docker/defaults/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/docker/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/docker/tasks/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/docker/tasks/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/docker/tasks/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/defaults/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/heading_jupyter_update_password.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/heading_jupyter_update_password.txt similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/heading_jupyter_update_password.txt rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/heading_jupyter_update_password.txt diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/notebook/bash_runner.py b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/bash_runner.py similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/notebook/bash_runner.py rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/bash_runner.py diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/notebook/script-languages.ipynb b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script-languages.ipynb similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/notebook/script-languages.ipynb rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script-languages.ipynb diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/notebook/slc_main_build_steps.svg b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/slc_main_build_steps.svg similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/notebook/slc_main_build_steps.svg rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/slc_main_build_steps.svg diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/motd.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/motd.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/motd.yml rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/motd.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/systemd.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/systemd.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/systemd.yml rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/systemd.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/templates/etc/systemd/system/jupyter.service b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/systemd/system/jupyter.service similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/templates/etc/systemd/system/jupyter.service rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/systemd/system/jupyter.service diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/netplan/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/netplan/defaults/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/netplan/defaults/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/netplan/defaults/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/netplan/files/etc/netplan/01-network-manager.yaml b/exasol/ds/sandbox/runtime/ansible/roles/netplan/files/etc/netplan/01-network-manager.yaml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/netplan/files/etc/netplan/01-network-manager.yaml rename to exasol/ds/sandbox/runtime/ansible/roles/netplan/files/etc/netplan/01-network-manager.yaml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/netplan/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/netplan/tasks/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/netplan/tasks/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/netplan/tasks/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/poetry/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/poetry/defaults/main.yml similarity index 78% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/poetry/defaults/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/poetry/defaults/main.yml index 4570f7e5..14e06296 100644 --- a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/poetry/defaults/main.yml +++ b/exasol/ds/sandbox/runtime/ansible/roles/poetry/defaults/main.yml @@ -1,6 +1,6 @@ --- apt_dependencies: - - curl=7.68.0-1ubuntu2.19 + - curl=7.68.0-1ubuntu2.20 - python3.8-venv=3.8.10-0ubuntu1~20.04.8 - python3-pip=20.0.2-5ubuntu1.9 diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/poetry/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/poetry/tasks/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/poetry/tasks/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/poetry/tasks/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/script_languages/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/script_languages/defaults/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/script_languages/defaults/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/script_languages/defaults/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/script_languages/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/script_languages/tasks/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/script_languages/tasks/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/script_languages/tasks/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/slc_setup.yml b/exasol/ds/sandbox/runtime/ansible/slc_setup.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/slc_setup.yml rename to exasol/ds/sandbox/runtime/ansible/slc_setup.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/slc_setup_tasks.yml b/exasol/ds/sandbox/runtime/ansible/slc_setup_tasks.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/slc_setup_tasks.yml rename to exasol/ds/sandbox/runtime/ansible/slc_setup_tasks.yml diff --git a/exasol_script_languages_developer_sandbox/templates/additional_release_notes.jinja b/exasol/ds/sandbox/templates/additional_release_notes.jinja similarity index 100% rename from exasol_script_languages_developer_sandbox/templates/additional_release_notes.jinja rename to exasol/ds/sandbox/templates/additional_release_notes.jinja diff --git a/exasol_script_languages_developer_sandbox/templates/ci_code_build.jinja.yaml b/exasol/ds/sandbox/templates/ci_code_build.jinja.yaml similarity index 91% rename from exasol_script_languages_developer_sandbox/templates/ci_code_build.jinja.yaml rename to exasol/ds/sandbox/templates/ci_code_build.jinja.yaml index 308cdc50..41299154 100644 --- a/exasol_script_languages_developer_sandbox/templates/ci_code_build.jinja.yaml +++ b/exasol/ds/sandbox/templates/ci_code_build.jinja.yaml @@ -66,10 +66,10 @@ Resources: - s3:ListBucket Resource: arn:aws:s3:::{{vm_bucket}} Effect: Allow - DeveloperSandboxCICodeBuild: + DataScienceSandboxCICodeBuild: Type: AWS::CodeBuild::Project Properties: - Description: CI Build which runs CI test if script-languages-developer-sandbox + Description: CI Build which runs CI test of data-science-sandbox Triggers: Webhook: True FilterGroups: @@ -89,7 +89,7 @@ Resources: Image: aws/codebuild/standard:5.0 Source: Type: GITHUB - Location: "https://github.com/exasol/script-languages-developer-sandbox" + Location: "https://github.com/exasol/data-science-sandbox" BuildSpec: "aws-code-build/ci/buildspec.yaml" TimeoutInMinutes: 240 @@ -97,10 +97,10 @@ Resources: CodeBuildLogGroup: Type: AWS::Logs::LogGroup Properties: - LogGroupName: !Sub "/aws/codebuild/${ DeveloperSandboxCICodeBuild }" + LogGroupName: !Sub "/aws/codebuild/${ DataScienceSandboxCICodeBuild }" RetentionInDays: 14 Outputs: ProjectName: - Value: !Ref DeveloperSandboxCICodeBuild + Value: !Ref DataScienceSandboxCICodeBuild Description: Developer Sandbox CI Build project name diff --git a/exasol_script_languages_developer_sandbox/templates/ec2_cloudformation.jinja.yaml b/exasol/ds/sandbox/templates/ec2_cloudformation.jinja.yaml similarity index 93% rename from exasol_script_languages_developer_sandbox/templates/ec2_cloudformation.jinja.yaml rename to exasol/ds/sandbox/templates/ec2_cloudformation.jinja.yaml index 88979f7d..cce42d74 100644 --- a/exasol_script_languages_developer_sandbox/templates/ec2_cloudformation.jinja.yaml +++ b/exasol/ds/sandbox/templates/ec2_cloudformation.jinja.yaml @@ -14,7 +14,7 @@ Resources: ToPort: 8888 Tags: - Key: "exasol:project" - Value: "ScriptLanguages" + Value: "DataScienceSandbox" - Key: "exasol:owner" Value: {{user_name}} - Key: {{trace_tag}} @@ -35,7 +35,7 @@ Resources: VolumeSize: 100 Tags: - Key: "exasol:project" - Value: "ScriptLanguages" + Value: "DataScienceSandbox" - Key: "exasol:owner" Value: {{user_name}} - Key: {{trace_tag}} diff --git a/exasol_script_languages_developer_sandbox/templates/inventory.jinja b/exasol/ds/sandbox/templates/inventory.jinja similarity index 100% rename from exasol_script_languages_developer_sandbox/templates/inventory.jinja rename to exasol/ds/sandbox/templates/inventory.jinja diff --git a/exasol_script_languages_developer_sandbox/templates/release_code_build.jinja.yaml b/exasol/ds/sandbox/templates/release_code_build.jinja.yaml similarity index 88% rename from exasol_script_languages_developer_sandbox/templates/release_code_build.jinja.yaml rename to exasol/ds/sandbox/templates/release_code_build.jinja.yaml index 59e58180..43e17b5c 100644 --- a/exasol_script_languages_developer_sandbox/templates/release_code_build.jinja.yaml +++ b/exasol/ds/sandbox/templates/release_code_build.jinja.yaml @@ -33,7 +33,7 @@ Resources: - cloudformation:DescribeChangeSet - cloudformation:ExecuteChangeSet - cloudformation:DeleteStack - Resource: !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/EC2-SLC-DEV-SANDBOX-*" + Resource: !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/EC2-DATA-SCIENCE-SANDBOX-*" Effect: Allow - Action: - ec2:CreateSecurityGroup @@ -70,12 +70,12 @@ Resources: - s3:PutObject - s3:GetObject - s3:DeleteObject - Resource: arn:aws:s3:::{{vm_bucket}}/slc_developer_sandbox/* + Resource: arn:aws:s3:::{{vm_bucket}}/data_science_sandbox/* Effect: Allow - DeveloperSandboxReleaseCodeBuild: + DataScienceSandboxReleaseCodeBuild: Type: AWS::CodeBuild::Project Properties: - Description: Release Build which creates a new AMI and VM iso images on script-languages-developer-sandbox + Description: Release Build which creates a new AMI and VM iso images on data-science-sandbox ServiceRole: !GetAtt CodeBuildRole.Arn Artifacts: Type: NO_ARTIFACTS @@ -85,7 +85,7 @@ Resources: Image: aws/codebuild/standard:5.0 Source: Type: GITHUB - Location: "https://github.com/exasol/script-languages-developer-sandbox" + Location: "https://github.com/exasol/data-science-sandbox" BuildSpec: "aws-code-build/ci/buildspec_release.yaml" TimeoutInMinutes: 240 @@ -93,10 +93,10 @@ Resources: CodeBuildLogGroup: Type: AWS::Logs::LogGroup Properties: - LogGroupName: !Sub "/aws/codebuild/${ DeveloperSandboxReleaseCodeBuild }" + LogGroupName: !Sub "/aws/codebuild/${ DataScienceSandboxReleaseCodeBuild }" RetentionInDays: 14 Outputs: ProjectName: - Value: !Ref DeveloperSandboxReleaseCodeBuild + Value: !Ref DataScienceSandboxReleaseCodeBuild Description: Developer Sandbox Release Build project name diff --git a/exasol_script_languages_developer_sandbox/templates/vm_bucket_cloudformation.jinja.yaml b/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml similarity index 94% rename from exasol_script_languages_developer_sandbox/templates/vm_bucket_cloudformation.jinja.yaml rename to exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml index 1775a6ef..d55a9d8b 100644 --- a/exasol_script_languages_developer_sandbox/templates/vm_bucket_cloudformation.jinja.yaml +++ b/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml @@ -4,7 +4,7 @@ Resources: Properties: AccessControl: 'Private' Metadata: - Comment: 'Bucket to store script-language developer sandbox VM images' + Comment: 'Bucket to store data-science sandbox VM images' DeletionPolicy: Retain UpdateReplacePolicy : Retain VMExportRole: @@ -57,7 +57,7 @@ Resources: Principal: Service: "cloudfront.amazonaws.com" Resource: - - !Sub "arn:aws:s3:::${ VMBucket }/slc_developer_sandbox/*" + - !Sub "arn:aws:s3:::${ VMBucket }/data_science_sandbox/*" Condition: StringEquals: 'AWS:SourceArn': !Sub "arn:aws:cloudfront::${AWS::AccountId}:distribution/${CfDistribution}" @@ -111,7 +111,7 @@ Resources: Properties: OriginAccessControlConfig: Description: Origin Access Control config for the VM S3 Bucket - Name: VM-Bucket-OAC + Name: DssVMBucketOAC OriginAccessControlOriginType: s3 SigningBehavior: always SigningProtocol: sigv4 @@ -128,4 +128,4 @@ Outputs: Value: !Ref CfDistribution {{CfDistributionDomainName}}: Description: 'Domain name for our cloudfront distribution' - Value: !GetAtt CfDistribution.DomainName \ No newline at end of file + Value: !GetAtt CfDistribution.DomainName diff --git a/exasol_script_languages_developer_sandbox/templates/waf_cloudformation.jinja.yaml b/exasol/ds/sandbox/templates/waf_cloudformation.jinja.yaml similarity index 79% rename from exasol_script_languages_developer_sandbox/templates/waf_cloudformation.jinja.yaml rename to exasol/ds/sandbox/templates/waf_cloudformation.jinja.yaml index 43148b3c..5b03725b 100644 --- a/exasol_script_languages_developer_sandbox/templates/waf_cloudformation.jinja.yaml +++ b/exasol/ds/sandbox/templates/waf_cloudformation.jinja.yaml @@ -2,7 +2,7 @@ Resources: VMDownloadACL: Type: AWS::WAFv2::WebACL Properties: - Name: VMDownloadLimitationWebACL + Name: DssVMDownloadLimitationWebACL Scope: CLOUDFRONT Description: This Web ACL contains rule to control access to the VM Bucket cloudfront distribution. DefaultAction: @@ -10,29 +10,29 @@ Resources: VisibilityConfig: SampledRequestsEnabled: true CloudWatchMetricsEnabled: true - MetricName: MetricForVMDownloads + MetricName: DssMetricForVMDownloads Rules: - - Name: Captcha + - Name: DssVmDownloadCaptcha Priority: 1 Action: Captcha: {} VisibilityConfig: SampledRequestsEnabled: true CloudWatchMetricsEnabled: true - MetricName: CaptchaMetricForVMDownloads + MetricName: DssCaptchaMetricForVMDownloads Statement: NotStatement: Statement: IPSetReferenceStatement: Arn: !GetAtt AllowedIpSet.Arn - - Name: VMBucketRateLimit + - Name: DssVMBucketRateLimit Priority: 0 Action: Block: {} VisibilityConfig: SampledRequestsEnabled: true CloudWatchMetricsEnabled: true - MetricName: RateLimitMetricForVMDownloads + MetricName: DssRateLimitMetricForVMDownloads Statement: RateBasedStatement: AggregateKeyType: IP @@ -40,8 +40,8 @@ Resources: AllowedIpSet: Type: 'AWS::WAFv2::IPSet' Properties: - Description: AllowedIp - Name: AllowedIp + Description: DssAllowedIp + Name: DssAllowedIp Scope: CLOUDFRONT IPAddressVersion: IPV4 Addresses: diff --git a/exasol_script_languages_developer_sandbox/cli/commands/export_vm.py b/exasol_script_languages_developer_sandbox/cli/commands/export_vm.py deleted file mode 100644 index 0bf15218..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/export_vm.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Tuple - -import click - -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.id_options import id_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.cli.options.vm_options import vm_options -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.export_vm.run_export_vm import run_export_vm -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -@click.option('--stack-name', required=True, - type=str, - help="Existing cloudformation stack containing the EC2 instance.") -@add_options(vm_options) -@add_options(id_options) -def export_vm( - aws_profile: str, - stack_name: str, - vm_image_format: Tuple[str, ...], - no_vm: bool, - asset_id: str, - log_level: str): - """ - Debug command which creates a new VM image from a running EC2-Instance. - """ - current_vm_image_formats = tuple() if no_vm else vm_image_format - set_log_level(log_level) - run_export_vm(AwsAccess(aws_profile), stack_name, current_vm_image_formats, - AssetId(asset_id), default_config_object) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/install_dependencies.py b/exasol_script_languages_developer_sandbox/cli/commands/install_dependencies.py deleted file mode 100644 index 3b975ca3..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/install_dependencies.py +++ /dev/null @@ -1,23 +0,0 @@ -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.ec2_options import ec2_host_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies - - -@cli.command() -@add_options(logging_options) -@add_options(ec2_host_options) -def install_dependencies( - host_name: str, - ssh_private_key: str, - log_level: str): - """ - Debug command to ansible-installation onto an EC-2 instance. - """ - set_log_level(log_level) - run_install_dependencies(AnsibleAccess(), default_config_object, (HostInfo(host_name, ssh_private_key),)) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/make_ami_public.py b/exasol_script_languages_developer_sandbox/cli/commands/make_ami_public.py deleted file mode 100644 index 331d47cf..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/make_ami_public.py +++ /dev/null @@ -1,24 +0,0 @@ -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.id_options import id_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.export_vm.run_make_ami_public import run_make_ami_public -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -@add_options(id_options) -def make_ami_public( - aws_profile: str, - asset_id: str, - log_level: str): - """ - Debug command which makes an existing AMI public. - """ - set_log_level(log_level) - run_make_ami_public(AwsAccess(aws_profile), AssetId(asset_id)) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/reset_password.py b/exasol_script_languages_developer_sandbox/cli/commands/reset_password.py deleted file mode 100644 index 3ebb2d37..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/reset_password.py +++ /dev/null @@ -1,27 +0,0 @@ -import click - -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.ec2_options import ec2_host_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_reset_password import run_reset_password - - -@cli.command() -@add_options(logging_options) -@add_options(ec2_host_options) -@click.option('--default-password', required=True, type=str, - help="The new (temporary) default password.") -def reset_password( - host_name: str, - ssh_private_key: str, - default_password: str, - log_level: str): - """ - Debug command to reset password on a remote EC-2-instance via ansible. - """ - set_log_level(log_level) - run_reset_password(AnsibleAccess(), default_password, (HostInfo(host_name, ssh_private_key),)) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/setup_ci_codebuild.py b/exasol_script_languages_developer_sandbox/cli/commands/setup_ci_codebuild.py deleted file mode 100644 index 92edc690..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/setup_ci_codebuild.py +++ /dev/null @@ -1,20 +0,0 @@ -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.setup_ci_codebuild.ci_codebuild import run_setup_ci_codebuild -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -def setup_ci_codebuild( - aws_profile: str, - log_level: str): - """ - Command to deploy the CI CodeBuild stack - """ - set_log_level(log_level) - run_setup_ci_codebuild(AwsAccess(aws_profile)) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/setup_ec2.py b/exasol_script_languages_developer_sandbox/cli/commands/setup_ec2.py deleted file mode 100644 index 0e320d7e..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/setup_ec2.py +++ /dev/null @@ -1,31 +0,0 @@ -from typing import Optional - -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.ec2_options import ec2_key_options -from exasol_script_languages_developer_sandbox.cli.options.id_options import id_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_setup_ec2 import run_setup_ec2 - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -@add_options(ec2_key_options) -@add_options(id_options) -def setup_ec2( - aws_profile: str, - ec2_key_file: Optional[str], - ec2_key_name: Optional[str], - asset_id: str, - log_level: str): - """ - Debug command to test setup of an EC-2 instance. - """ - set_log_level(log_level) - run_setup_ec2(AwsAccess(aws_profile), ec2_key_file, ec2_key_name, AssetId(asset_id), default_config_object) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/setup_ec2_and_install_dependencies.py b/exasol_script_languages_developer_sandbox/cli/commands/setup_ec2_and_install_dependencies.py deleted file mode 100644 index 6b9a1204..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/setup_ec2_and_install_dependencies.py +++ /dev/null @@ -1,34 +0,0 @@ -from typing import Optional - -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.ec2_options import ec2_key_options -from exasol_script_languages_developer_sandbox.cli.options.id_options import id_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_setup_ec2_and_install_dependencies import \ - run_setup_ec2_and_install_dependencies - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -@add_options(ec2_key_options) -@add_options(id_options) -def setup_ec2_and_install_dependencies( - aws_profile: str, - ec2_key_file: Optional[str], - ec2_key_name: Optional[str], - asset_id: str, - log_level: str): - """ - Debug command to check setup and installation of an EC-2 instance. - """ - set_log_level(log_level) - run_setup_ec2_and_install_dependencies(AwsAccess(aws_profile), ec2_key_file, ec2_key_name, - AssetId(asset_id), AnsibleAccess(), default_config_object) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/setup_release_codebuild.py b/exasol_script_languages_developer_sandbox/cli/commands/setup_release_codebuild.py deleted file mode 100644 index 3261e229..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/setup_release_codebuild.py +++ /dev/null @@ -1,21 +0,0 @@ -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.setup_release_codebuild.release_codebuild import \ - run_setup_release_codebuild - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -def setup_release_codebuild( - aws_profile: str, - log_level: str): - """ - Command to deploy the Release CodeBuild stack - """ - set_log_level(log_level) - run_setup_release_codebuild(AwsAccess(aws_profile)) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket.py b/exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket.py deleted file mode 100644 index ace569a3..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket.py +++ /dev/null @@ -1,21 +0,0 @@ -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import run_setup_vm_bucket - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -def setup_vm_bucket( - aws_profile: str, - log_level: str): - """ - Command to deploy the VM S3-Bucket - """ - set_log_level(log_level) - run_setup_vm_bucket(AwsAccess(aws_profile), default_config_object) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket_waf.py b/exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket_waf.py deleted file mode 100644 index a280c8d2..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket_waf.py +++ /dev/null @@ -1,26 +0,0 @@ -import click - -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket_waf import run_setup_vm_bucket_waf - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -@click.option('--allowed-ip', type=str, - help="The allowed IP address for which CAPTCHA will not be applied.") -def setup_vm_bucket_waf( - aws_profile: str, - allowed_ip: str, - log_level: str): - """ - Command to deploy the VM S3-Bucket Web Application Firewall. Needs to run before deploying the VM Bucket itself. - """ - set_log_level(log_level) - run_setup_vm_bucket_waf(AwsAccess(aws_profile), allowed_ip, default_config_object) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/update_release.py b/exasol_script_languages_developer_sandbox/cli/commands/update_release.py deleted file mode 100644 index a0ad98e1..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/update_release.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -from typing import Optional - -import click - -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.id_options import id_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.github_release_access import GithubReleaseAccess -from exasol_script_languages_developer_sandbox.lib.update_release.run_update_release import run_update_release - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -@click.option('--release-id', type=int, required=True, - help="""The Github release id which will be updated.""") -@add_options(id_options) -def update_release( - aws_profile: Optional[str], - release_id: int, - asset_id: str, - log_level: str): - """ - This command attaches the links of the release assets (AMI, VM images) to the Github release, - indicated by parameter 'release-id'. - """ - set_log_level(log_level) - run_update_release(AwsAccess(aws_profile), GithubReleaseAccess(os.getenv("GITHUB_TOKEN")), - release_id, AssetId(asset_id)) diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/ci_codebuild.py b/exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/ci_codebuild.py deleted file mode 100644 index 09d797d4..00000000 --- a/exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/ci_codebuild.py +++ /dev/null @@ -1,15 +0,0 @@ -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.render_template import render_template -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket - -STACK_NAME = "DEVELOPER-SANDBOX-CI-TEST-CODEBUILD" - -LOG = get_status_logger(LogType.SETUP_CI_CODEBUILD) - - -def run_setup_ci_codebuild(aws_access: AwsAccess) -> None: - yml = render_template("ci_code_build.jinja.yaml", vm_bucket=find_vm_bucket(aws_access)) - aws_access.upload_cloudformation_stack(yml, STACK_NAME) - LOG.info(f"Deployed cloudformation stack {STACK_NAME}") - diff --git a/exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/release_codebuild.py b/exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/release_codebuild.py deleted file mode 100644 index 0cbd5a40..00000000 --- a/exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/release_codebuild.py +++ /dev/null @@ -1,15 +0,0 @@ -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.render_template import render_template -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket - -RELEASE_CODE_BUILD_STACK_NAME = "DEVELOPER-SANDBOX-RELEASE-CODEBUILD" - -LOG = get_status_logger(LogType.SETUP_RELEASE_BUILD) - - -def run_setup_release_codebuild(aws_access: AwsAccess) -> None: - yml = render_template("release_code_build.jinja.yaml", vm_bucket=find_vm_bucket(aws_access)) - aws_access.upload_cloudformation_stack(yml, RELEASE_CODE_BUILD_STACK_NAME) - LOG.info(f"Deployed cloudformation stack {RELEASE_CODE_BUILD_STACK_NAME}") - diff --git a/exasol_script_languages_developer_sandbox/main.py b/exasol_script_languages_developer_sandbox/main.py deleted file mode 100755 index 2edfaca4..00000000 --- a/exasol_script_languages_developer_sandbox/main.py +++ /dev/null @@ -1,7 +0,0 @@ -#! /usr/bin/env python3 -# -import exasol_script_languages_developer_sandbox.cli.commands -from exasol_script_languages_developer_sandbox.cli.cli import cli - -if __name__ == '__main__': - cli() diff --git a/pyproject.toml b/pyproject.toml index 39d41859..625e1cd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,29 @@ [tool.poetry] -name = "exasol-script-languages-developer-sandbox" -version = "0.2.0" -description = "Manages script-languages developer virtual machines." - +name = "exasol-data-science-sandbox" +version = "0.1.0" +description = "Manages virtual machines for the data science sandbox." +packages = [ {include = "exasol"}, ] license = "MIT" - authors = [ - "Thomas Uebensee " + "Torsten Kilias ", + "Mikhail Beck ", + "Christoph Kuhnke " +] +classifiers = [ + "Programming Language :: Python :: 3", + "Development Status :: 3 - Alpha", + "Framework :: Jupyter :: JupyterLab :: 3", + "License :: OSI Approved :: MIT License", ] +readme = "README.md" + + +[tool.poetry.urls] +"Homepage" = "https://github.com/exasol/data-science-sandbox" +"Documentation" = "https://github.io/exasol/data-science-sandbox" +"Source" = "https://github.io/exasol/data-science-sandbox" +"Issues" = "https://github.io/exasol/data-science-sandbox/issues" + [tool.poetry.dependencies] python = ">=3.8.0,<4.0" diff --git a/test/aws/__init__.py b/test/aws/__init__.py new file mode 100644 index 00000000..a1d3903c --- /dev/null +++ b/test/aws/__init__.py @@ -0,0 +1,5 @@ +""" +This package contains tests involving AWS resources. In order to execute +these tests you need an AWS account, a user with permissions in this account and +an access key. +""" diff --git a/test/cloudformation_validation.py b/test/aws/cloudformation_validation.py similarity index 90% rename from test/cloudformation_validation.py rename to test/aws/cloudformation_validation.py index 110f0b55..7a30153a 100644 --- a/test/cloudformation_validation.py +++ b/test/aws/cloudformation_validation.py @@ -6,7 +6,7 @@ def validate_using_cfn_lint(tmp_path, cloudformation_yml): This test uses cfn-lint to validate the Cloudformation template. (See https://github.com/aws-cloudformation/cfn-lint) """ - out_file = tmp_path / "slc_source_credential.yaml" + out_file = tmp_path / "cloudformation.yaml" with open(out_file, "w") as f: f.write(cloudformation_yml) diff --git a/test/test_deploy_codebuild.py b/test/aws/test_deploy_codebuild.py similarity index 74% rename from test/test_deploy_codebuild.py rename to test/aws/test_deploy_codebuild.py index 41c0715f..5a763e56 100644 --- a/test/test_deploy_codebuild.py +++ b/test/aws/test_deploy_codebuild.py @@ -1,8 +1,8 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.render_template import render_template -from test.cloudformation_validation import validate_using_cfn_lint +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.render_template import render_template +from test.aws.cloudformation_validation import validate_using_cfn_lint codebuild_cloudformation_templates = [ diff --git a/test/test_deploy_ec2.py b/test/aws/test_deploy_ec2.py similarity index 84% rename from test/test_deploy_ec2.py rename to test/aws/test_deploy_ec2.py index 6bbcbe4c..51ccaa13 100644 --- a/test/test_deploy_ec2.py +++ b/test/aws/test_deploy_ec2.py @@ -1,10 +1,10 @@ from unittest.mock import MagicMock -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.setup_ec2.cf_stack import CloudformationStack, \ +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.setup_ec2.cf_stack import CloudformationStack, \ CloudformationStackContextManager -from exasol_script_languages_developer_sandbox.lib.tags import create_default_asset_tag -from test.cloudformation_validation import validate_using_cfn_lint +from exasol.ds.sandbox.lib.tags import create_default_asset_tag +from test.aws.cloudformation_validation import validate_using_cfn_lint def test_deploy_ec2_upload_invoked(ec2_cloudformation_yml, default_asset_id, test_dummy_ami_id): diff --git a/test/test_deploy_vm_bucket.py b/test/aws/test_deploy_vm_bucket.py similarity index 86% rename from test/test_deploy_vm_bucket.py rename to test/aws/test_deploy_vm_bucket.py index 43f75f6d..c19723a1 100644 --- a/test/test_deploy_vm_bucket.py +++ b/test/aws/test_deploy_vm_bucket.py @@ -3,14 +3,14 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import run_setup_vm_bucket, find_vm_bucket, \ +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import run_setup_vm_bucket, find_vm_bucket, \ create_vm_bucket_cf_template from test.aws_mock_data import TEST_BUCKET_ID, get_waf_cloudformation_mock_data, TEST_ACL_ARN, \ get_s3_cloudformation_mock_data -from test.cloudformation_validation import validate_using_cfn_lint +from test.aws.cloudformation_validation import validate_using_cfn_lint from test.mock_cast import mock_cast -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import STACK_NAME as VM_STACK_NAME +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import STACK_NAME as VM_STACK_NAME @pytest.fixture diff --git a/test/test_deploy_vm_bucket_waf.py b/test/aws/test_deploy_vm_bucket_waf.py similarity index 83% rename from test/test_deploy_vm_bucket_waf.py rename to test/aws/test_deploy_vm_bucket_waf.py index 62fb163c..d2e03533 100644 --- a/test/test_deploy_vm_bucket_waf.py +++ b/test/aws/test_deploy_vm_bucket_waf.py @@ -3,11 +3,11 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket_waf import run_setup_vm_bucket_waf, \ +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket_waf import run_setup_vm_bucket_waf, \ find_acl_arn, get_cloudformation_template from test.aws_mock_data import get_waf_cloudformation_mock_data, TEST_ACL_ARN -from test.cloudformation_validation import validate_using_cfn_lint +from test.aws.cloudformation_validation import validate_using_cfn_lint from test.mock_cast import mock_cast TEST_IP = "1.1.1.1" diff --git a/test/test_export_vm.py b/test/aws/test_export_vm.py similarity index 90% rename from test/test_export_vm.py rename to test/aws/test_export_vm.py index d6b71efd..01e00fc1 100644 --- a/test/test_export_vm.py +++ b/test/aws/test_export_vm.py @@ -4,12 +4,12 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.aws_access.ami import Ami -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.export_vm.rename_s3_objects import build_image_source, \ +from exasol.ds.sandbox.lib.aws_access.ami import Ami +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.export_vm.rename_s3_objects import build_image_source, \ build_image_destination -from exasol_script_languages_developer_sandbox.lib.export_vm.run_export_vm import export_vm -from exasol_script_languages_developer_sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat +from exasol.ds.sandbox.lib.export_vm.run_export_vm import export_vm +from exasol.ds.sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat from test.aws_mock_data import get_ami_image_mock_data, TEST_AMI_ID, TEST_ROLE_ID, TEST_BUCKET_ID, INSTANCE_ID, \ get_export_image_task_mock_data, get_s3_cloudformation_mock_data, get_waf_cloudformation_mock_data from test.mock_cast import mock_cast diff --git a/test/aws_local_stack_access.py b/test/aws_local_stack_access.py index 8bc639a8..0a0bf2c8 100644 --- a/test/aws_local_stack_access.py +++ b/test/aws_local_stack_access.py @@ -2,7 +2,7 @@ import boto3 -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess class AwsLocalStackAccess(AwsAccess): diff --git a/test/aws_mock_data.py b/test/aws_mock_data.py index 7621edaf..e9409a3a 100644 --- a/test/aws_mock_data.py +++ b/test/aws_mock_data.py @@ -3,17 +3,17 @@ from dateutil.tz import tzutc -from exasol_script_languages_developer_sandbox.lib.aws_access.ami import Ami -from exasol_script_languages_developer_sandbox.lib.aws_access.cloudformation_stack import CloudformationStack -from exasol_script_languages_developer_sandbox.lib.aws_access.export_image_task import ExportImageTask -from exasol_script_languages_developer_sandbox.lib.aws_access.key_pair import KeyPair -from exasol_script_languages_developer_sandbox.lib.aws_access.s3_object import S3Object -from exasol_script_languages_developer_sandbox.lib.aws_access.snapshot import Snapshot -from exasol_script_languages_developer_sandbox.lib.aws_access.stack_resource import StackResource +from exasol.ds.sandbox.lib.aws_access.ami import Ami +from exasol.ds.sandbox.lib.aws_access.cloudformation_stack import CloudformationStack +from exasol.ds.sandbox.lib.aws_access.export_image_task import ExportImageTask +from exasol.ds.sandbox.lib.aws_access.key_pair import KeyPair +from exasol.ds.sandbox.lib.aws_access.s3_object import S3Object +from exasol.ds.sandbox.lib.aws_access.snapshot import Snapshot +from exasol.ds.sandbox.lib.aws_access.stack_resource import StackResource from test.conftest import DEFAULT_ASSET_ID -TEST_ROLE_ID = 'VM-SLC-Bucket-VMImportRole-TEST' -TEST_BUCKET_ID = 'vm-slc-bucket-vmslcbucket-TEST' +TEST_ROLE_ID = 'VM-DSS-Bucket-VMImportRole-TEST' +TEST_BUCKET_ID = 'vm-dss-bucket-vmdssbucket-TEST' TEST_AMI_ID = "AMI-IMAGE-12345" TEST_CLOUDFRONT_ID = "test-cloudfrontet-TEST" TEST_CLOUDFRONT_DOMAIN_NAME = "test-s3.cloudfront.net" @@ -43,7 +43,7 @@ def get_ami_image_mock_data(state: str) -> Ami: 'Name': DEFAULT_ASSET_ID.ami_name, 'RootDeviceName': '/dev/sda1', 'RootDeviceType': 'ebs', 'SriovNetSupport': 'simple', - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'VirtualizationType': 'hvm'}) @@ -53,7 +53,7 @@ def get_snapshot_mock_data(): 'OwnerId': '123', 'Progress': '100%', 'SnapshotId': 'snap-123', 'StartTime': datetime.datetime(2022, 8, 16, 15, 3, 40, 662000, tzinfo=tzutc()), 'State': 'completed', 'VolumeId': 'vol-123', 'VolumeSize': 100, - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'StorageTier': 'standard' }) @@ -71,7 +71,7 @@ def get_export_image_task_mock_data(in_progress: bool): 'Progress': "50%", "StatusMessage": "creating the image", 'Status': 'active', - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}] + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}] }) else: return ExportImageTask({ @@ -83,7 +83,7 @@ def get_export_image_task_mock_data(in_progress: bool): 'S3Prefix': DEFAULT_ASSET_ID.bucket_prefix }, 'Status': 'completed', - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}] + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}] }) @@ -107,7 +107,7 @@ def get_ec2_cloudformation_mock_data(): 'RollbackConfiguration': {}, 'StackStatus': 'CREATE_COMPLETE', 'DisableRollback': False, 'NotificationARNs': [], 'Capabilities': ['CAPABILITY_IAM'], - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'DriftInformation': {'StackDriftStatus': 'NOT_CHECKED'} }) @@ -137,7 +137,7 @@ def get_ec2_key_pair_mock_data(): 'KeyFingerprint': '12:34:56:78:90:12:34:56:78:90:12:34:56:78:90:12:34:56:78:90', 'KeyName': 'ec2-key-test-key', 'KeyType': 'rsa', - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'CreateTime': datetime.datetime(2022, 8, 16, 15, 30, 41, tzinfo=tzutc()) }) @@ -146,14 +146,14 @@ def get_ec2_key_pair_mock_data(): def get_s3_cloudformation_mock_data() -> List[CloudformationStack]: return [CloudformationStack({ 'StackId': 'test-s3-stack-id', - 'StackName': "DEVELOPER-SANDBOX-VM-SLC-Bucket", + 'StackName': "DATA-SCIENCE-SANDBOX-VM-Bucket", 'ChangeSetId': 'test-stack-changeset-id-2', 'CreationTime': datetime.datetime(2022, 8, 16, 14, 30, 45, 559000, tzinfo=tzutc()), 'LastUpdatedTime': datetime.datetime(2022, 8, 16, 14, 30, 51, 667000, tzinfo=tzutc()), 'RollbackConfiguration': {}, 'StackStatus': 'CREATE_COMPLETE', 'DisableRollback': False, 'NotificationARNs': [], 'Capabilities': ['CAPABILITY_IAM'], - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'DriftInformation': {'StackDriftStatus': 'NOT_CHECKED'}, 'Outputs': [{'OutputKey': 'VMBucketId', 'OutputValue': TEST_BUCKET_ID, 'Description': ''}, @@ -171,14 +171,14 @@ def get_s3_cloudformation_mock_data() -> List[CloudformationStack]: def get_waf_cloudformation_mock_data() -> List[CloudformationStack]: return [CloudformationStack({ 'StackId': 'test-waf-stack-id', - 'StackName': "DEVELOPER-SANDBOX-VM-SLC-Bucket-WAF", + 'StackName': "DATA-SCIENCE-SANDBOX-VM-Bucket-WAF", 'ChangeSetId': 'test-stack-changeset-id-3', 'CreationTime': datetime.datetime(2022, 8, 16, 14, 30, 45, 559000, tzinfo=tzutc()), 'LastUpdatedTime': datetime.datetime(2022, 8, 16, 14, 30, 51, 667000, tzinfo=tzutc()), 'RollbackConfiguration': {}, 'StackStatus': 'CREATE_COMPLETE', 'DisableRollback': False, 'NotificationARNs': [], 'Capabilities': [], - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'DriftInformation': {'StackDriftStatus': 'NOT_CHECKED'}, 'Outputs': [{'OutputKey': 'VMDownloadACLArn', 'OutputValue': TEST_ACL_ARN, 'Description': ''} diff --git a/test/test_ci.py b/test/ci/test_ci.py similarity index 87% rename from test/test_ci.py rename to test/ci/test_ci.py index e08d955c..340ad6fc 100644 --- a/test/test_ci.py +++ b/test/ci/test_ci.py @@ -11,18 +11,18 @@ from invoke import Responder -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import default_config_object, SLC_VERSION -from exasol_script_languages_developer_sandbox.lib.run_create_vm import run_create_vm -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import default_config_object, SLC_VERSION +from exasol.ds.sandbox.lib.run_create_vm import run_create_vm +from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ EC2StackLifecycleContextManager import string import random -from exasol_script_languages_developer_sandbox.lib.tags import DEFAULT_TAG_KEY +from exasol.ds.sandbox.lib.tags import DEFAULT_TAG_KEY def generate_random_password(length) -> str: @@ -99,8 +99,8 @@ def new_ec2_from_ami(): aws_access.remove_snapshot(snapshots[0].id) -@pytest.mark.skipif(os.environ.get('RUN_DEVELOPER_SANDBOX_CI_TEST') != 'true', - reason="CI test need to be activated by env variable RUN_DEVELOPER_SANDBOX_CI_TEST") +@pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', + reason="CI test need to be activated by env variable DSS_RUN_CI_TEST") def test_exaslct_with_ec2_based_on_new_ami(new_ec2_from_ami): """ This test validates that exaslct is correctly working on the EC-2 instance, which was launched from the @@ -116,8 +116,8 @@ def test_exaslct_with_ec2_based_on_new_ami(new_ec2_from_ami): assert result.return_code == 0 -@pytest.mark.skipif(os.environ.get('RUN_DEVELOPER_SANDBOX_CI_TEST') != 'true', - reason="CI test need to be activated by env variable RUN_DEVELOPER_SANDBOX_CI_TEST") +@pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', + reason="CI test need to be activated by env variable DSS_RUN_CI_TEST") def test_jupyter_with_ec2_based_on_new_ami(new_ec2_from_ami): """ This test validates that Jupyterlab is correctly working on the EC-2 instance, which was launched from the @@ -128,8 +128,8 @@ def test_jupyter_with_ec2_based_on_new_ami(new_ec2_from_ami): assert http_conn.status_code == 200 -@pytest.mark.skipif(os.environ.get('RUN_DEVELOPER_SANDBOX_CI_TEST') != 'true', - reason="CI test need to be activated by env variable RUN_DEVELOPER_SANDBOX_CI_TEST") +@pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', + reason="CI test need to be activated by env variable DSS_RUN_CI_TEST") def test_password_changed_on_new_ami(new_ec2_from_ami): """ This test validates that the password has been changed by trying to login via ssh using the old password @@ -146,8 +146,8 @@ def test_password_changed_on_new_ami(new_ec2_from_ami): con.run("uname") -@pytest.mark.skipif(os.environ.get('RUN_DEVELOPER_SANDBOX_CI_TEST') != 'true', - reason="CI test need to be activated by env variable RUN_DEVELOPER_SANDBOX_CI_TEST") +@pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', + reason="CI test need to be activated by env variable DSS_RUN_CI_TEST") def test_jupyter_password_message_shown(new_ec2_from_ami): """ This test validates that the motd password message for Jupyterlab is working as expected. diff --git a/test/test_install_dependencies.py b/test/ci/test_install_dependencies.py similarity index 69% rename from test/test_install_dependencies.py rename to test/ci/test_install_dependencies.py index f094407b..76ad1c28 100644 --- a/test/test_install_dependencies.py +++ b/test/ci/test_install_dependencies.py @@ -1,32 +1,54 @@ import tempfile import requests +import shutil import time from pathlib import Path +from inspect import cleandoc import docker import pytest from docker.types import Mount -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_repository import AnsibleResourceRepository, \ +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.ansible.ansible_repository import AnsibleResourceRepository, \ default_repositories -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import AnsibleRunContext -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext +from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies import test.ansible TEST_CONTAINER_NAME = "ansible-test" -TEST_CONTAINER_IMAGE_TAG = "script_languages_developer_sandbox_test_container:latest" +TEST_CONTAINER_IMAGE_TAG = "data_science_sandbox_test_container:latest" + +# AWS AMI Image is based on Ubuntu20.04 server. +# So we try to simulate same environment here. +DOCKERFILE_CONTENT = cleandoc( + """ + FROM ubuntu:20.04 + ENV DEBIAN_FRONTEND noninteractive + RUN apt-get update && apt-get install -y ubuntu-server + """ +) + +@pytest.fixture(scope="session") +def dockerfile(tmp_path_factory): + dockerfile = tmp_path_factory.mktemp("test_container") / "Dockerfile" + with dockerfile.open("w") as f: + print(DOCKERFILE_CONTENT, file = f) + yield dockerfile + shutil.rmtree(str(dockerfile.parent)) @pytest.fixture(scope="session") -def docker_test_container(test_config): - with tempfile.TemporaryDirectory() as tmp_dir: +def docker_test_container(test_config, dockerfile): + with tempfile.TemporaryDirectory() as tmp_path: docker_env = docker.from_env() - p = Path(__file__).parent / "test_container" - docker_env.images.build(path=str(p), tag=TEST_CONTAINER_IMAGE_TAG) + docker_env.images.build( + path=str(dockerfile.parent), + tag=TEST_CONTAINER_IMAGE_TAG + ) socket_mount = Mount("/var/run/docker.sock", "/var/run/docker.sock", type="bind") - tmp_mount = Mount(tmp_dir, tmp_dir, type="bind") + tmp_mount = Mount(tmp_path, tmp_path, type="bind") mapped_ports = {'8888/tcp': 8888} test_container = docker_env.containers.create(image=TEST_CONTAINER_IMAGE_TAG, name=TEST_CONTAINER_NAME, mounts=[socket_mount, tmp_mount], @@ -36,18 +58,18 @@ def docker_test_container(test_config): ansible_run_context = \ AnsibleRunContext(playbook="slc_setup_test.yml", extra_vars={"test_docker_container": test_container.name, - "slc_dest_folder": f"{tmp_dir}/script-languages-release"}) + "slc_dest_folder": f"{tmp_path}/script-languages-release"}) try: run_install_dependencies(AnsibleAccess(), configuration=test_config, host_infos=tuple(), ansible_run_context=ansible_run_context, ansible_repositories=repos) - yield test_container, tmp_dir + yield test_container, tmp_path # Note: script-languages-release will be cloned by ansible within the docker container. # Because the docker container runs as root, the repository will be owned by root. # For simplicity, we delete the folder from within the Docker container (as root). - # Otherwise, we get a permission problem when tmp_dir tries to clean-up itself. + # Otherwise, we get a permission problem when tmp_path tries to clean-up itself. finally: - test_container.exec_run(f"rm -rf {tmp_dir}/script-languages-release") + test_container.exec_run(f"rm -rf {tmp_path}/script-languages-release") test_container.stop() test_container.remove() diff --git a/test/conftest.py b/test/conftest.py index 8b28bd92..8f500001 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -5,13 +5,13 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.config import default_config_object, ConfigObject -from exasol_script_languages_developer_sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.config import default_config_object, ConfigObject +from exasol.ds.sandbox.lib.render_template import render_template from importlib.metadata import version -from exasol_script_languages_developer_sandbox.lib.tags import DEFAULT_TAG_KEY +from exasol.ds.sandbox.lib.tags import DEFAULT_TAG_KEY -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.asset_id import AssetId DEFAULT_ASSET_ID = AssetId("test", stack_prefix="test-stack", ami_prefix="test-ami") diff --git a/test/integration/__init__.py b/test/integration/__init__.py new file mode 100644 index 00000000..c4cd8e27 --- /dev/null +++ b/test/integration/__init__.py @@ -0,0 +1,4 @@ +""" +This package contains integration tests with longer runtime and maybe +requiring additional resources. +""" diff --git a/test/localstack_test.py b/test/integration/localstack_test.py similarity index 92% rename from test/localstack_test.py rename to test/integration/localstack_test.py index 1eee3f80..6c450a39 100644 --- a/test/localstack_test.py +++ b/test/integration/localstack_test.py @@ -1,10 +1,10 @@ import botocore import pytest -from exasol_script_languages_developer_sandbox.lib.setup_ec2.cf_stack import CloudformationStack, \ +from exasol.ds.sandbox.lib.setup_ec2.cf_stack import CloudformationStack, \ CloudformationStackContextManager -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2 -from exasol_script_languages_developer_sandbox.lib.tags import create_default_asset_tag +from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2 +from exasol.ds.sandbox.lib.tags import create_default_asset_tag from test.aws_local_stack_access import AwsLocalStackAccess @@ -71,7 +71,7 @@ def test_validate_cloudformation_template_fails_with_local_stack(local_stack): ToPort: 22 Tags: - Key: "exasol:project" - Value: "ScriptLanguages" + Value: "DataScienceSandbox" - Key: "exasol:owner" Value: "test user" @@ -90,7 +90,7 @@ def test_validate_cloudformation_template_fails_with_local_stack(local_stack): VolumeSize: 100 Tags: - Key: "exasol:project" - Value: "ScriptLanguages" + Value: "DataScienceSandbox" - Key: "exasol:owner" Value: "test" diff --git a/test/test_aws_codebuild_waiter.py b/test/integration/test_aws_codebuild_waiter.py similarity index 95% rename from test/test_aws_codebuild_waiter.py rename to test/integration/test_aws_codebuild_waiter.py index f92f173c..7e8446cf 100644 --- a/test/test_aws_codebuild_waiter.py +++ b/test/integration/test_aws_codebuild_waiter.py @@ -4,7 +4,7 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.aws_access.waiter.codebuild_waiter import CodeBuildWaiter +from exasol.ds.sandbox.lib.aws_access.waiter.codebuild_waiter import CodeBuildWaiter from test.mock_cast import mock_cast BUILD_ID = 123 diff --git a/test/test_motd_jupyter_template.py b/test/integration/test_motd_jupyter_template.py similarity index 91% rename from test/test_motd_jupyter_template.py rename to test/integration/test_motd_jupyter_template.py index a55f48e1..f54fa48c 100644 --- a/test/test_motd_jupyter_template.py +++ b/test/integration/test_motd_jupyter_template.py @@ -14,8 +14,10 @@ def motd_file(tmp_path): jupyter_server_config_file = tmp_path / "jupyter_server_config.json" python_file = tmp_path / "999_jupyter.py" - src_path = Path(__file__).parent.parent / "exasol_script_languages_developer_sandbox" / "runtime" / \ - "ansible" / "roles" / "jupyter" / "templates" / "etc" /"update-motd.d" / "999-jupyter" + src_path = Path(__file__).parent.parent.parent \ + / "exasol" / "ds" / "sandbox" / "runtime" \ + / "ansible" / "roles" / "jupyter" / "templates" \ + / "etc" /"update-motd.d" / "999-jupyter" with open(src_path, "r") as f: python_code_template = f.read() python_template = Template(python_code_template) diff --git a/test/test_serialization.py b/test/integration/test_serialization.py similarity index 93% rename from test/test_serialization.py rename to test/integration/test_serialization.py index 37c05480..905e65c4 100644 --- a/test/test_serialization.py +++ b/test/integration/test_serialization.py @@ -4,9 +4,9 @@ import traceback from pathlib import Path -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.setup_ec2.cf_stack import CloudformationStack -from exasol_script_languages_developer_sandbox.lib.setup_ec2.key_file_manager import KeyFileManager +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.setup_ec2.cf_stack import CloudformationStack +from exasol.ds.sandbox.lib.setup_ec2.key_file_manager import KeyFileManager from test.aws_local_stack_access import AwsLocalStackAccess import multiprocessing as mp diff --git a/test/test_container/Dockerfile b/test/test_container/Dockerfile deleted file mode 100644 index ca2f4181..00000000 --- a/test/test_container/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM ubuntu:20.04 -ENV DEBIAN_FRONTEND noninteractive -#AWS AMI Image is based on Ubuntu20.04 server. So we try to simulate same environment here -RUN apt-get update && apt-get install -y ubuntu-server \ No newline at end of file diff --git a/test/unit/__init__.py b/test/unit/__init__.py new file mode 100644 index 00000000..a6db5aa6 --- /dev/null +++ b/test/unit/__init__.py @@ -0,0 +1,4 @@ +""" +This package contains unit tests requiring no additional setup or external +resources. +""" diff --git a/test/ansible_conflict/__init__.py b/test/unit/ansible_conflict/__init__.py similarity index 100% rename from test/ansible_conflict/__init__.py rename to test/unit/ansible_conflict/__init__.py diff --git a/test/ansible_conflict/slc_setup.yml b/test/unit/ansible_conflict/slc_setup.yml similarity index 100% rename from test/ansible_conflict/slc_setup.yml rename to test/unit/ansible_conflict/slc_setup.yml diff --git a/test/test_ansible.py b/test/unit/test_ansible.py similarity index 92% rename from test/test_ansible.py rename to test/unit/test_ansible.py index 237a0bde..251cc87e 100644 --- a/test/test_ansible.py +++ b/test/unit/test_ansible.py @@ -4,15 +4,15 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_repository import default_repositories, \ +from exasol.ds.sandbox.lib.ansible.ansible_repository import default_repositories, \ AnsibleResourceRepository -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ default_ansible_run_context -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies import test.ansible -import test.ansible_conflict +import test.unit.ansible_conflict class AnsibleTestAccess: @@ -129,7 +129,7 @@ def test_run_ansible_check_multiple_repositories_with_same_content_causes_except """ Test that multiple repositories containing same files raises an runtime exception. """ - test_repositories = default_repositories + (AnsibleResourceRepository(test.ansible_conflict),) + test_repositories = default_repositories + (AnsibleResourceRepository(test.unit.ansible_conflict),) with pytest.raises(RuntimeError): run_install_dependencies(AnsibleTestAccess(), test_config, host_infos=tuple(), ansible_run_context=default_ansible_run_context, diff --git a/test/test_key_file_manager.py b/test/unit/test_key_file_manager.py similarity index 93% rename from test/test_key_file_manager.py rename to test/unit/test_key_file_manager.py index e3428b26..191fdf59 100644 --- a/test/test_key_file_manager.py +++ b/test/unit/test_key_file_manager.py @@ -1,7 +1,7 @@ import os from unittest.mock import MagicMock -from exasol_script_languages_developer_sandbox.lib.setup_ec2.key_file_manager import KeyFileManager, KeyFileManagerContextManager +from exasol.ds.sandbox.lib.setup_ec2.key_file_manager import KeyFileManager, KeyFileManagerContextManager def test_external_keys(tmp_path, default_asset_id): diff --git a/test/test_make_ami_public.py b/test/unit/test_make_ami_public.py similarity index 87% rename from test/test_make_ami_public.py rename to test/unit/test_make_ami_public.py index 77e1d784..e5e1c359 100644 --- a/test/test_make_ami_public.py +++ b/test/unit/test_make_ami_public.py @@ -4,9 +4,9 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.aws_access.ami import Ami -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.export_vm.run_make_ami_public import run_make_ami_public +from exasol.ds.sandbox.lib.aws_access.ami import Ami +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.export_vm.run_make_ami_public import run_make_ami_public from test.aws_mock_data import get_ami_image_mock_data from test.mock_cast import mock_cast diff --git a/test/test_printing.py b/test/unit/test_printing.py similarity index 96% rename from test/test_printing.py rename to test/unit/test_printing.py index 0c133ce8..c0958eca 100644 --- a/test/test_printing.py +++ b/test/unit/test_printing.py @@ -3,9 +3,9 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.asset_printing.print_assets import print_with_printer, AssetTypes -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.asset_printing.print_assets import print_with_printer, AssetTypes +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from test.aws_mock_data import get_ami_image_mock_data, TEST_AMI_ID, get_snapshot_mock_data, \ get_export_image_task_mock_data, get_s3_object_mock_data, TEST_BUCKET_ID, \ get_ec2_cloudformation_mock_data, get_ec2_cloudformation_stack_resources_mock_data, get_ec2_key_pair_mock_data, \ diff --git a/test/test_printing_factories.py b/test/unit/test_printing_factories.py similarity index 81% rename from test/test_printing_factories.py rename to test/unit/test_printing_factories.py index 28809ab6..538bb2b7 100644 --- a/test/test_printing_factories.py +++ b/test/unit/test_printing_factories.py @@ -2,9 +2,9 @@ from contextlib import redirect_stdout from io import StringIO -import exasol_script_languages_developer_sandbox.lib.asset_printing.rich_console_printer -from exasol_script_languages_developer_sandbox.lib.asset_printing.mark_down_printer import MarkdownPrintingFactory -from exasol_script_languages_developer_sandbox.lib.asset_printing.printing_factory import TextObject +import exasol.ds.sandbox.lib.asset_printing.rich_console_printer +from exasol.ds.sandbox.lib.asset_printing.mark_down_printer import MarkdownPrintingFactory +from exasol.ds.sandbox.lib.asset_printing.printing_factory import TextObject from inspect import cleandoc @@ -35,7 +35,7 @@ def test_printing_rich(): with StringIO() as buf, redirect_stdout(buf): # Need to reload the module here as it already might have been loaded by pytest for another test, # in which case sys.stdout was already loaded too - m = importlib.reload(exasol_script_languages_developer_sandbox.lib.asset_printing.rich_console_printer) + m = importlib.reload(exasol.ds.sandbox.lib.asset_printing.rich_console_printer) factory = m.RichConsolePrintingFactory() table_printer = factory.create_table_printer("test-table") @@ -57,4 +57,4 @@ def test_printing_rich(): └───────┴───────┘ this text will be printed""") - assert output == expected_output \ No newline at end of file + assert output == expected_output diff --git a/test/test_release_build.py b/test/unit/test_release_build.py similarity index 84% rename from test/test_release_build.py rename to test/unit/test_release_build.py index a2d0a842..98f6b1bd 100644 --- a/test/test_release_build.py +++ b/test/unit/test_release_build.py @@ -4,15 +4,15 @@ from dateutil.tz import tzutc -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.aws_access.stack_resource import StackResource -from exasol_script_languages_developer_sandbox.lib.aws_access.waiter.codebuild_waiter import CodeBuildWaiter -from exasol_script_languages_developer_sandbox.lib.github_release_access import GithubReleaseAccess -from exasol_script_languages_developer_sandbox.lib.release_build.run_release_build import run_start_release_build, \ +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.aws_access.stack_resource import StackResource +from exasol.ds.sandbox.lib.aws_access.waiter.codebuild_waiter import CodeBuildWaiter +from exasol.ds.sandbox.lib.github_release_access import GithubReleaseAccess +from exasol.ds.sandbox.lib.release_build.run_release_build import run_start_release_build, \ run_start_test_release_build from test.mock_cast import mock_cast -UPLOAD_URL = "https://uploads.github.com/repos/exasol/script-languages-developer-sandbox/releases/123/assets{?name,label}" +UPLOAD_URL = "https://uploads.github.com/repos/exasol/data-science-sandbox/releases/123/assets{?name,label}" BRANCH = "main" GITHUB_TOKEN = "gh_secret" @@ -29,7 +29,7 @@ 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 39, 1, 806000, tzinfo=tzutc()), 'ResourceStatus': 'CREATE_COMPLETE', 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} }), - StackResource({'LogicalResourceId': 'developerSandboxReleaseCodeBuild', + StackResource({'LogicalResourceId': 'dataScienceSandboxReleaseCodeBuild', 'PhysicalResourceId': 'codebuild-id-123', 'ResourceType': 'AWS::CodeBuild::Project', 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 39, 7, 850000, tzinfo=tzutc()), diff --git a/test/test_run_lifecycle_for_ec2.py b/test/unit/test_run_lifecycle_for_ec2.py similarity index 94% rename from test/test_run_lifecycle_for_ec2.py rename to test/unit/test_run_lifecycle_for_ec2.py index b1ad97a2..46d79bef 100644 --- a/test/test_run_lifecycle_for_ec2.py +++ b/test/unit/test_run_lifecycle_for_ec2.py @@ -2,9 +2,9 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.aws_access.ec2_instance import EC2Instance -from exasol_script_languages_developer_sandbox.lib.aws_access.stack_resource import StackResource -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ +from exasol.ds.sandbox.lib.aws_access.ec2_instance import EC2Instance +from exasol.ds.sandbox.lib.aws_access.stack_resource import StackResource +from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ EC2StackLifecycleContextManager diff --git a/test/test_source_ami.py b/test/unit/test_source_ami.py similarity index 99% rename from test/test_source_ami.py rename to test/unit/test_source_ami.py index cfd1fb51..06bad529 100644 --- a/test/test_source_ami.py +++ b/test/unit/test_source_ami.py @@ -1,7 +1,7 @@ from unittest.mock import MagicMock -from exasol_script_languages_developer_sandbox.lib.aws_access.ami import Ami -from exasol_script_languages_developer_sandbox.lib.setup_ec2.source_ami import find_source_ami +from exasol.ds.sandbox.lib.aws_access.ami import Ami +from exasol.ds.sandbox.lib.setup_ec2.source_ami import find_source_ami # The following is just a dump of data returned from aws cli: # 'aws ec2 --profile exa_individual_mfa describe-images --filters Name=name,Values=ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* --owners 099720109477'