From 2660aab15ad6fab6cdaaf75058fbd0a2680b39ec Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 17 Oct 2023 10:14:43 +0200 Subject: [PATCH 01/21] Applied first round of renamings. Currently changed only names related to AWS resources and build artifacts. --- doc/changes/changelog.md | 1 - doc/changes/changes_0.1.0.md | 46 +++++-------------- doc/changes/changes_0.2.0.md | 32 ------------- .../lib/asset_id.py | 4 +- .../lib/export_vm/rename_s3_objects.py | 4 +- .../lib/github_release_access.py | 2 +- .../lib/setup_ci_codebuild/ci_codebuild.py | 2 +- .../release_codebuild.py | 2 +- .../lib/vm_bucket/vm_slc_bucket.py | 2 +- .../lib/vm_bucket/vm_slc_bucket_waf.py | 2 +- test/aws_mock_data.py | 4 +- test/test_ci.py | 16 +++---- test/test_install_dependencies.py | 2 +- test/test_release_build.py | 4 +- 14 files changed, 33 insertions(+), 90 deletions(-) delete mode 100644 doc/changes/changes_0.2.0.md diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 39f13d8b..9e62d079 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,4 +1,3 @@ # Changes -* [0.2.0](changes_0.2.0.md) * [0.1.0](changes_0.1.0.md) diff --git a/doc/changes/changes_0.1.0.md b/doc/changes/changes_0.1.0.md index 754ca23a..dff00a2c 100644 --- a/doc/changes/changes_0.1.0.md +++ b/doc/changes/changes_0.1.0.md @@ -1,47 +1,23 @@ -# script-languages-developer-sandbox 0.1.0, released 2022-10-06 +# script-languages-developer-sandbox 0.1.0, released t.b.d. Code name: Initial release ## Summary -Initial release of the script-languages-developer-sandbox. It provides the creation of a developer sandbox AMI and virtual machine images for a specific version of the script-languages-release project. +Initial release of the data-science-sandbox. It provides the creation of an Amazon Machone Image (AMI) and virtual machine images for a specific version of the data-science-sanbox-release project. -## Script-Languages-Release +## Data-Science-Sandbox-Release -Version: 5.0.0 +Version: 0.1.0 + +## Features + + - #11: Created a notebook to show training with scikit-learn in the notebook ## Bug Fixes - - - #18: Fixed network connection - - #51: Fixed network connection - - #57: Fixed release build - - #60: Fixed release build command - - #64: Fixed Netplan file name - -## Features / Enhancements - - - #2: Implemented launch of an EC2 instance - - #3: Installed SLC dependencies via Ansible - - #4: Implemented deployment and access of S3 Bucket for VM's - - #5: Implemented export of VM's - - #24: Move CI test to AWS Codebuild - - #25: Implemented motd message about Jupyter password change - - #8: Implemented a release workflow - - #36: Added make-ami-public option - - #43: Added CDN to the S3 VM Bucket - - #45: Protected cloudfront access - - #47: Renamed virtual images - - #38: Included tutorial Jupyterlab notebook - -## Documentation - - - #19: Added user guide and developer guide - - #49: Added tutorial about how to start the VM/AMI + + - #1: Fixed CI build ## Refactoring - - #22: Improved logging - - #26: Implemented search for latest AMI - - #12: Updated the script-languages-release tag with the correct version - - #21: Minor refactoring tasks - - #41: Renamed cloudformation stack \ No newline at end of file + - #5: Renamed all occurrences of "script language developer" by "dat science" diff --git a/doc/changes/changes_0.2.0.md b/doc/changes/changes_0.2.0.md deleted file mode 100644 index 3baa421d..00000000 --- a/doc/changes/changes_0.2.0.md +++ /dev/null @@ -1,32 +0,0 @@ -# script-languages-developer-sandbox 0.2.0, released t.b.d. - -Code name: t.b.d. - -## Summary - -t.b.d. - -## Script-Languages-Release - -Version: 5.0.0 - -## Bug Fixes - - - #78: Fix test_install_dependencies docker cleanup, fix pytest github action - -## Features / Enhancements - -n/a - -## Documentation - - - #67: Mentioned the AMI copy function in tutorial and release notes - -## Refactoring - - - #73: Removed setup.py - -## Security - - - #77: Update Ubuntu packages and added `cloudformation:ListStacks` to the roles for the CI runs - diff --git a/exasol_script_languages_developer_sandbox/lib/asset_id.py b/exasol_script_languages_developer_sandbox/lib/asset_id.py index 4df71477..3bb65ffc 100644 --- a/exasol_script_languages_developer_sandbox/lib/asset_id.py +++ b/exasol_script_languages_developer_sandbox/lib/asset_id.py @@ -1,5 +1,5 @@ class AssetId: - def __init__(self, asset_id: str, stack_prefix="EC2-SLC-DEV-SANDBOX-", ami_prefix="Exasol-SLC-Developer-Sandbox"): + def __init__(self, asset_id: str, stack_prefix="EC2-DATA-SCIENCE-SANDBOX-", ami_prefix="Exasol-Data-Science-Sandbox"): self._asset_id = asset_id self._stack_prefix = stack_prefix self._ami_prefix = ami_prefix @@ -23,4 +23,4 @@ def stack_prefix(self): def __repr__(self): return self._asset_id - BUCKET_PREFIX = "slc_developer_sandbox" + BUCKET_PREFIX = "data_science_sandbox" diff --git a/exasol_script_languages_developer_sandbox/lib/export_vm/rename_s3_objects.py b/exasol_script_languages_developer_sandbox/lib/export_vm/rename_s3_objects.py index db634c6a..a63abdf8 100644 --- a/exasol_script_languages_developer_sandbox/lib/export_vm/rename_s3_objects.py +++ b/exasol_script_languages_developer_sandbox/lib/export_vm/rename_s3_objects.py @@ -14,7 +14,7 @@ def build_image_source(prefix: str, export_image_task_id: str, vm_image_format: def build_image_destination(prefix: str, asset_id: AssetId, vm_image_format: VmDiskImageFormat) -> str: img_format = vm_image_format.value.lower() - return "{bucket_prefix}exasol-script-languages-developer-sandbox-{asset_id}.{img_format}".format( + return "{bucket_prefix}exasol-data-science-developer-sandbox-{asset_id}.{img_format}".format( bucket_prefix=prefix, asset_id=str(asset_id), img_format=img_format) @@ -26,7 +26,7 @@ def rename_image_in_s3(aws_access: AwsAccess, export_image_task: ExportImageTask """ Renames the resulting S3 object of an export-image-task. The source objects always have the format "$export-image-task-id.$format". - The destination objects always have the format "exasol-script-languages-developer-sandbox-{asset_id}.{img_format}" + The destination objects always have the format "exasol-data-science-sandbox-{asset_id}.{img_format}" The bucket and prefix in bucket do not change. :param aws_access: Access proxy to Aws :param export_image_task: The export image task which is expected to be completed successfully. diff --git a/exasol_script_languages_developer_sandbox/lib/github_release_access.py b/exasol_script_languages_developer_sandbox/lib/github_release_access.py index 2708abf7..5b323161 100644 --- a/exasol_script_languages_developer_sandbox/lib/github_release_access.py +++ b/exasol_script_languages_developer_sandbox/lib/github_release_access.py @@ -56,5 +56,5 @@ def upload(self, archive_path: str, label: str, release_id: int, content_type: s @property def _get_repo(self) -> Repository: gh = Github(self._gh_token) - gh_repo = gh.get_repo("exasol/script-languages-developer-sandbox") + gh_repo = gh.get_repo("exasol/data-science-sandbox") return gh_repo diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/ci_codebuild.py b/exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/ci_codebuild.py index 09d797d4..a1d1f0ce 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/ci_codebuild.py +++ b/exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/ci_codebuild.py @@ -3,7 +3,7 @@ from exasol_script_languages_developer_sandbox.lib.render_template import render_template from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket -STACK_NAME = "DEVELOPER-SANDBOX-CI-TEST-CODEBUILD" +STACK_NAME = "DATA-SCIENCE-SANDBOX-CI-TEST-CODEBUILD" LOG = get_status_logger(LogType.SETUP_CI_CODEBUILD) diff --git a/exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/release_codebuild.py b/exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/release_codebuild.py index 0cbd5a40..51e034a3 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/release_codebuild.py +++ b/exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/release_codebuild.py @@ -3,7 +3,7 @@ from exasol_script_languages_developer_sandbox.lib.render_template import render_template from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket -RELEASE_CODE_BUILD_STACK_NAME = "DEVELOPER-SANDBOX-RELEASE-CODEBUILD" +RELEASE_CODE_BUILD_STACK_NAME = "DATA-SCIENCE-SANDBOX-RELEASE-CODEBUILD" LOG = get_status_logger(LogType.SETUP_RELEASE_BUILD) diff --git a/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket.py b/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket.py index 4cb5d4be..24b59855 100644 --- a/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket.py +++ b/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket.py @@ -6,7 +6,7 @@ from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket_waf import find_acl_arn -STACK_NAME = "DEVELOPER-SANDBOX-VM-SLC-Bucket" +STACK_NAME = "DATA-SCIENCE-SANDBOX-VM-SLC-Bucket" class OutputKey(Enum): diff --git a/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket_waf.py b/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket_waf.py index f6b77fa7..59e74f30 100644 --- a/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket_waf.py +++ b/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket_waf.py @@ -3,7 +3,7 @@ from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType from exasol_script_languages_developer_sandbox.lib.render_template import render_template -STACK_NAME = "DEVELOPER-SANDBOX-VM-SLC-Bucket-WAF" +STACK_NAME = "DATA-SCIENCE-SANDBOX-VM-SLC-Bucket-WAF" LOG = get_status_logger(LogType.VM_BUCKET) diff --git a/test/aws_mock_data.py b/test/aws_mock_data.py index 7621edaf..abe8d98a 100644 --- a/test/aws_mock_data.py +++ b/test/aws_mock_data.py @@ -146,7 +146,7 @@ def get_ec2_key_pair_mock_data(): def get_s3_cloudformation_mock_data() -> List[CloudformationStack]: return [CloudformationStack({ 'StackId': 'test-s3-stack-id', - 'StackName': "DEVELOPER-SANDBOX-VM-SLC-Bucket", + 'StackName': "DATA-SCIENCE-SANDBOX-VM-SLC-Bucket", 'ChangeSetId': 'test-stack-changeset-id-2', 'CreationTime': datetime.datetime(2022, 8, 16, 14, 30, 45, 559000, tzinfo=tzutc()), 'LastUpdatedTime': datetime.datetime(2022, 8, 16, 14, 30, 51, 667000, tzinfo=tzutc()), @@ -171,7 +171,7 @@ def get_s3_cloudformation_mock_data() -> List[CloudformationStack]: def get_waf_cloudformation_mock_data() -> List[CloudformationStack]: return [CloudformationStack({ 'StackId': 'test-waf-stack-id', - 'StackName': "DEVELOPER-SANDBOX-VM-SLC-Bucket-WAF", + 'StackName': "DATA-SCIENCE-SANDBOX-VM-SLC-Bucket-WAF", 'ChangeSetId': 'test-stack-changeset-id-3', 'CreationTime': datetime.datetime(2022, 8, 16, 14, 30, 45, 559000, tzinfo=tzutc()), 'LastUpdatedTime': datetime.datetime(2022, 8, 16, 14, 30, 51, 667000, tzinfo=tzutc()), diff --git a/test/test_ci.py b/test/test_ci.py index e08d955c..98ef0c6a 100644 --- a/test/test_ci.py +++ b/test/test_ci.py @@ -99,8 +99,8 @@ def new_ec2_from_ami(): aws_access.remove_snapshot(snapshots[0].id) -@pytest.mark.skipif(os.environ.get('RUN_DEVELOPER_SANDBOX_CI_TEST') != 'true', - reason="CI test need to be activated by env variable RUN_DEVELOPER_SANDBOX_CI_TEST") +@pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', + reason="CI test need to be activated by env variable DSS_RUN_CI_TEST") def test_exaslct_with_ec2_based_on_new_ami(new_ec2_from_ami): """ This test validates that exaslct is correctly working on the EC-2 instance, which was launched from the @@ -116,8 +116,8 @@ def test_exaslct_with_ec2_based_on_new_ami(new_ec2_from_ami): assert result.return_code == 0 -@pytest.mark.skipif(os.environ.get('RUN_DEVELOPER_SANDBOX_CI_TEST') != 'true', - reason="CI test need to be activated by env variable RUN_DEVELOPER_SANDBOX_CI_TEST") +@pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', + reason="CI test need to be activated by env variable DSS_RUN_CI_TEST") def test_jupyter_with_ec2_based_on_new_ami(new_ec2_from_ami): """ This test validates that Jupyterlab is correctly working on the EC-2 instance, which was launched from the @@ -128,8 +128,8 @@ def test_jupyter_with_ec2_based_on_new_ami(new_ec2_from_ami): assert http_conn.status_code == 200 -@pytest.mark.skipif(os.environ.get('RUN_DEVELOPER_SANDBOX_CI_TEST') != 'true', - reason="CI test need to be activated by env variable RUN_DEVELOPER_SANDBOX_CI_TEST") +@pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', + reason="CI test need to be activated by env variable DSS_RUN_CI_TEST") def test_password_changed_on_new_ami(new_ec2_from_ami): """ This test validates that the password has been changed by trying to login via ssh using the old password @@ -146,8 +146,8 @@ def test_password_changed_on_new_ami(new_ec2_from_ami): con.run("uname") -@pytest.mark.skipif(os.environ.get('RUN_DEVELOPER_SANDBOX_CI_TEST') != 'true', - reason="CI test need to be activated by env variable RUN_DEVELOPER_SANDBOX_CI_TEST") +@pytest.mark.skipif(os.environ.get('DSS_RUN_CI_TEST') != 'true', + reason="CI test need to be activated by env variable DSS_RUN_CI_TEST") def test_jupyter_password_message_shown(new_ec2_from_ami): """ This test validates that the motd password message for Jupyterlab is working as expected. diff --git a/test/test_install_dependencies.py b/test/test_install_dependencies.py index f094407b..d680760c 100644 --- a/test/test_install_dependencies.py +++ b/test/test_install_dependencies.py @@ -16,7 +16,7 @@ import test.ansible TEST_CONTAINER_NAME = "ansible-test" -TEST_CONTAINER_IMAGE_TAG = "script_languages_developer_sandbox_test_container:latest" +TEST_CONTAINER_IMAGE_TAG = "data_science_sandbox_test_container:latest" @pytest.fixture(scope="session") diff --git a/test/test_release_build.py b/test/test_release_build.py index a2d0a842..7b5e3d85 100644 --- a/test/test_release_build.py +++ b/test/test_release_build.py @@ -12,7 +12,7 @@ run_start_test_release_build from test.mock_cast import mock_cast -UPLOAD_URL = "https://uploads.github.com/repos/exasol/script-languages-developer-sandbox/releases/123/assets{?name,label}" +UPLOAD_URL = "https://uploads.github.com/repos/exasol/data-science-sandbox/releases/123/assets{?name,label}" BRANCH = "main" GITHUB_TOKEN = "gh_secret" @@ -29,7 +29,7 @@ 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 39, 1, 806000, tzinfo=tzutc()), 'ResourceStatus': 'CREATE_COMPLETE', 'DriftInformation': {'StackResourceDriftStatus': 'NOT_CHECKED'} }), - StackResource({'LogicalResourceId': 'developerSandboxReleaseCodeBuild', + StackResource({'LogicalResourceId': 'dataScienceSandboxReleaseCodeBuild', 'PhysicalResourceId': 'codebuild-id-123', 'ResourceType': 'AWS::CodeBuild::Project', 'LastUpdatedTimestamp': datetime.datetime(2022, 5, 4, 18, 39, 7, 850000, tzinfo=tzutc()), From b18ef84e847b4092265e6f573b5c17abbd6917c0 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 17 Oct 2023 11:16:26 +0200 Subject: [PATCH 02/21] Fixed version number in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 39d41859..ec3f05bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "exasol-script-languages-developer-sandbox" -version = "0.2.0" +version = "0.1.0" description = "Manages script-languages developer virtual machines." license = "MIT" From 411be857abc179877108209dde73b4c23077f839 Mon Sep 17 00:00:00 2001 From: ckunki Date: Tue, 17 Oct 2023 15:16:10 +0200 Subject: [PATCH 03/21] Updated curl from ubuntu2.19 to 2.20 --- .../runtime/ansible/roles/poetry/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/poetry/defaults/main.yml b/exasol_script_languages_developer_sandbox/runtime/ansible/roles/poetry/defaults/main.yml index 4570f7e5..14e06296 100644 --- a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/poetry/defaults/main.yml +++ b/exasol_script_languages_developer_sandbox/runtime/ansible/roles/poetry/defaults/main.yml @@ -1,6 +1,6 @@ --- apt_dependencies: - - curl=7.68.0-1ubuntu2.19 + - curl=7.68.0-1ubuntu2.20 - python3.8-venv=3.8.10-0ubuntu1~20.04.8 - python3-pip=20.0.2-5ubuntu1.9 From 35dea92222058857e519096383db9b96c1397094 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 18 Oct 2023 10:59:22 +0200 Subject: [PATCH 04/21] Renamed python packages from exasol_script_languages_developer_sandbox to exasol.ds.sandbox --- ...ase_droid_upload_github_release_assets.yml | 2 +- .github/workflows/test_release.yml | 4 +- error_code_config.yml | 4 +- .../ds/sandbox}/__init__.py | 0 .../ds/sandbox}/cli/__init__.py | 0 .../ds/sandbox}/cli/cli.py | 0 .../ds/sandbox}/cli/commands/__init__.py | 0 .../ds/sandbox}/cli/commands/create_vm.py | 26 ++++++------- exasol/ds/sandbox/cli/commands/export_vm.py | 39 +++++++++++++++++++ .../cli/commands/install_dependencies.py | 23 +++++++++++ .../sandbox/cli/commands/make_ami_public.py | 24 ++++++++++++ .../ds/sandbox/cli/commands/reset_password.py | 27 +++++++++++++ .../cli/commands/setup_ci_codebuild.py | 20 ++++++++++ exasol/ds/sandbox/cli/commands/setup_ec2.py | 31 +++++++++++++++ .../setup_ec2_and_install_dependencies.py | 34 ++++++++++++++++ .../cli/commands/setup_release_codebuild.py | 21 ++++++++++ .../sandbox/cli/commands/setup_vm_bucket.py | 21 ++++++++++ .../cli/commands/setup_vm_bucket_waf.py | 26 +++++++++++++ .../sandbox}/cli/commands/show_aws_assets.py | 16 ++++---- .../cli/commands/start_release_build.py | 16 ++++---- .../cli/commands/start_test_release_build.py | 16 ++++---- .../ds/sandbox/cli/commands/update_release.py | 35 +++++++++++++++++ .../ds/sandbox}/cli/common.py | 0 .../ds/sandbox}/cli/options/__init__.py | 0 .../ds/sandbox}/cli/options/aws_options.py | 0 .../ds/sandbox}/cli/options/ec2_options.py | 0 .../ds/sandbox}/cli/options/id_options.py | 2 +- .../ds/sandbox}/cli/options/logging.py | 2 +- .../ds/sandbox}/cli/options/vm_options.py | 2 +- .../ds/sandbox}/lib/__init__.py | 0 .../ds/sandbox}/lib/ansible/__init__.py | 0 .../ds/sandbox}/lib/ansible/ansible_access.py | 2 +- .../lib/ansible/ansible_context_manager.py | 6 +-- .../lib/ansible/ansible_repository.py | 6 +-- .../lib/ansible/ansible_run_context.py | 0 .../ds/sandbox}/lib/ansible/ansible_runner.py | 10 ++--- .../ds/sandbox}/lib/asset_id.py | 0 .../sandbox}/lib/asset_printing/__init__.py | 0 .../lib/asset_printing/mark_down_printer.py | 2 +- .../lib/asset_printing/print_assets.py | 16 ++++---- .../lib/asset_printing/printing_factory.py | 0 .../asset_printing/rich_console_printer.py | 2 +- .../ds/sandbox}/lib/aws_access/__init__.py | 0 .../ds/sandbox}/lib/aws_access/ami.py | 0 .../ds/sandbox}/lib/aws_access/aws_access.py | 28 ++++++------- .../lib/aws_access/cloudformation_stack.py | 2 +- .../ds/sandbox}/lib/aws_access/common.py | 0 .../ds/sandbox}/lib/aws_access/deployer.py | 0 .../sandbox}/lib/aws_access/ec2_instance.py | 0 .../lib/aws_access/ec2_instance_status.py | 0 .../lib/aws_access/export_image_task.py | 2 +- .../ds/sandbox}/lib/aws_access/key_pair.py | 0 .../ds/sandbox}/lib/aws_access/s3_object.py | 0 .../ds/sandbox}/lib/aws_access/snapshot.py | 2 +- .../sandbox}/lib/aws_access/stack_resource.py | 0 .../lib/aws_access/waiter/__init__.py | 0 .../lib/aws_access/waiter/codebuild_waiter.py | 2 +- .../ds/sandbox}/lib/config.py | 0 .../ds/sandbox}/lib/export_vm/__init__.py | 0 .../lib/export_vm/rename_s3_objects.py | 8 ++-- .../sandbox}/lib/export_vm/run_export_vm.py | 20 +++++----- .../lib/export_vm/run_make_ami_public.py | 8 ++-- .../lib/export_vm/vm_disk_image_format.py | 0 .../ds/sandbox}/lib/github_release_access.py | 2 +- .../ds/sandbox}/lib/logging.py | 0 .../ds/sandbox}/lib/release_build/__init__.py | 0 .../lib/release_build/run_release_build.py | 12 +++--- .../ds/sandbox}/lib/render_template.py | 2 +- .../ds/sandbox}/lib/run_create_vm.py | 28 ++++++------- .../lib/setup_ci_codebuild/__init__.py | 0 .../lib/setup_ci_codebuild/ci_codebuild.py | 8 ++-- .../ds/sandbox}/lib/setup_ec2/__init__.py | 0 .../ds/sandbox}/lib/setup_ec2/cf_stack.py | 12 +++--- .../ds/sandbox}/lib/setup_ec2/host_info.py | 0 .../lib/setup_ec2/key_file_manager.py | 6 +-- .../lib/setup_ec2/random_string_generator.py | 0 .../lib/setup_ec2/run_install_dependencies.py | 12 +++--- .../lib/setup_ec2/run_reset_password.py | 10 ++--- .../sandbox}/lib/setup_ec2/run_setup_ec2.py | 16 ++++---- .../run_setup_ec2_and_install_dependencies.py | 22 +++++------ .../ds/sandbox}/lib/setup_ec2/source_ami.py | 4 +- .../lib/setup_release_codebuild/__init__.py | 0 .../release_codebuild.py | 8 ++-- .../ds/sandbox}/lib/tags.py | 0 .../sandbox}/lib/update_release/__init__.py | 0 .../lib/update_release/run_update_release.py | 10 ++--- .../ds/sandbox}/lib/vm_bucket/__init__.py | 0 .../sandbox}/lib/vm_bucket/vm_slc_bucket.py | 10 ++--- .../lib/vm_bucket/vm_slc_bucket_waf.py | 8 ++-- exasol/ds/sandbox/main.py | 7 ++++ .../ds/sandbox}/runtime/__init__.py | 0 .../runtime/ansible/reset_password.yml | 0 .../runtime/ansible/reset_password_tasks.yml | 0 .../ansible/roles/docker/defaults/main.yml | 0 .../ansible/roles/docker/tasks/main.yml | 0 .../ansible/roles/jupyter/defaults/main.yml | 0 .../files/heading_jupyter_update_password.txt | 0 .../jupyter/files/notebook/bash_runner.py | 0 .../files/notebook/script-languages.ipynb | 0 .../files/notebook/slc_main_build_steps.svg | 0 .../files/requirements_dependencies.txt | 0 .../jupyter/files/requirements_jupyter.txt | 0 .../jupyter/tasks/install-pip-packages.yml | 0 .../roles/jupyter/tasks/jupyterlab.yml | 0 .../ansible/roles/jupyter/tasks/main.yml | 0 .../ansible/roles/jupyter/tasks/motd.yml | 0 .../ansible/roles/jupyter/tasks/systemd.yml | 0 .../ansible/roles/jupyter/tasks/tutorial.yml | 0 .../etc/systemd/system/jupyter.service | 0 .../templates/etc/update-motd.d/999-jupyter | 0 .../ansible/roles/netplan/defaults/main.yml | 0 .../files/etc/netplan/01-network-manager.yaml | 0 .../ansible/roles/netplan/tasks/main.yml | 0 .../ansible/roles/poetry/defaults/main.yml | 0 .../ansible/roles/poetry/tasks/main.yml | 0 .../roles/script_languages/defaults/main.yml | 0 .../roles/script_languages/tasks/main.yml | 0 .../ds/sandbox}/runtime/ansible/slc_setup.yml | 0 .../runtime/ansible/slc_setup_tasks.yml | 0 .../templates/additional_release_notes.jinja | 0 .../templates/ci_code_build.jinja.yaml | 0 .../templates/ec2_cloudformation.jinja.yaml | 0 .../ds/sandbox}/templates/inventory.jinja | 0 .../templates/release_code_build.jinja.yaml | 0 .../vm_bucket_cloudformation.jinja.yaml | 0 .../templates/waf_cloudformation.jinja.yaml | 0 .../cli/commands/export_vm.py | 39 ------------------- .../cli/commands/install_dependencies.py | 23 ----------- .../cli/commands/make_ami_public.py | 24 ------------ .../cli/commands/reset_password.py | 27 ------------- .../cli/commands/setup_ci_codebuild.py | 20 ---------- .../cli/commands/setup_ec2.py | 31 --------------- .../setup_ec2_and_install_dependencies.py | 34 ---------------- .../cli/commands/setup_release_codebuild.py | 21 ---------- .../cli/commands/setup_vm_bucket.py | 21 ---------- .../cli/commands/setup_vm_bucket_waf.py | 26 ------------- .../cli/commands/update_release.py | 35 ----------------- .../main.py | 7 ---- pyproject.toml | 9 +++-- test/aws_local_stack_access.py | 2 +- test/aws_mock_data.py | 14 +++---- test/conftest.py | 8 ++-- test/localstack_test.py | 6 +-- test/test_ansible.py | 8 ++-- test/test_aws_codebuild_waiter.py | 2 +- test/test_ci.py | 14 +++---- test/test_deploy_codebuild.py | 4 +- test/test_deploy_ec2.py | 6 +-- test/test_deploy_vm_bucket.py | 6 +-- test/test_deploy_vm_bucket_waf.py | 4 +- test/test_export_vm.py | 10 ++--- test/test_install_dependencies.py | 8 ++-- test/test_key_file_manager.py | 2 +- test/test_make_ami_public.py | 6 +-- test/test_motd_jupyter_template.py | 2 +- test/test_printing.py | 6 +-- test/test_printing_factories.py | 10 ++--- test/test_release_build.py | 10 ++--- test/test_run_lifecycle_for_ec2.py | 6 +-- test/test_serialization.py | 6 +-- test/test_source_ami.py | 4 +- 161 files changed, 573 insertions(+), 572 deletions(-) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/cli.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/commands/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/commands/create_vm.py (52%) create mode 100644 exasol/ds/sandbox/cli/commands/export_vm.py create mode 100644 exasol/ds/sandbox/cli/commands/install_dependencies.py create mode 100644 exasol/ds/sandbox/cli/commands/make_ami_public.py create mode 100644 exasol/ds/sandbox/cli/commands/reset_password.py create mode 100644 exasol/ds/sandbox/cli/commands/setup_ci_codebuild.py create mode 100644 exasol/ds/sandbox/cli/commands/setup_ec2.py create mode 100644 exasol/ds/sandbox/cli/commands/setup_ec2_and_install_dependencies.py create mode 100644 exasol/ds/sandbox/cli/commands/setup_release_codebuild.py create mode 100644 exasol/ds/sandbox/cli/commands/setup_vm_bucket.py create mode 100644 exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/commands/show_aws_assets.py (64%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/commands/start_release_build.py (54%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/commands/start_test_release_build.py (55%) create mode 100644 exasol/ds/sandbox/cli/commands/update_release.py rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/common.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/options/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/options/aws_options.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/options/ec2_options.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/options/id_options.py (78%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/options/logging.py (82%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/cli/options/vm_options.py (85%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/ansible/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/ansible/ansible_access.py (86%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/ansible/ansible_context_manager.py (76%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/ansible/ansible_repository.py (91%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/ansible/ansible_run_context.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/ansible/ansible_runner.py (63%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/asset_id.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/asset_printing/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/asset_printing/mark_down_printer.py (89%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/asset_printing/print_assets.py (93%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/asset_printing/printing_factory.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/asset_printing/rich_console_printer.py (90%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/ami.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/aws_access.py (93%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/cloudformation_stack.py (94%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/common.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/deployer.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/ec2_instance.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/ec2_instance_status.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/export_image_task.py (93%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/key_pair.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/s3_object.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/snapshot.py (92%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/stack_resource.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/waiter/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/aws_access/waiter/codebuild_waiter.py (96%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/config.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/export_vm/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/export_vm/rename_s3_objects.py (84%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/export_vm/run_export_vm.py (87%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/export_vm/run_make_ami_public.py (72%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/export_vm/vm_disk_image_format.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/github_release_access.py (96%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/logging.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/release_build/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/release_build/run_release_build.py (88%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/render_template.py (90%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/run_create_vm.py (66%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ci_codebuild/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ci_codebuild/ci_codebuild.py (50%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/cf_stack.py (85%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/host_info.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/key_file_manager.py (89%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/random_string_generator.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/run_install_dependencies.py (71%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/run_reset_password.py (77%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/run_setup_ec2.py (89%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/run_setup_ec2_and_install_dependencies.py (73%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_ec2/source_ami.py (64%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_release_codebuild/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/setup_release_codebuild/release_codebuild.py (54%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/tags.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/update_release/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/update_release/run_update_release.py (65%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/vm_bucket/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/vm_bucket/vm_slc_bucket.py (82%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/lib/vm_bucket/vm_slc_bucket_waf.py (83%) create mode 100755 exasol/ds/sandbox/main.py rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/__init__.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/reset_password.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/reset_password_tasks.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/docker/defaults/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/docker/tasks/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/defaults/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/files/heading_jupyter_update_password.txt (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/files/notebook/bash_runner.py (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/files/notebook/script-languages.ipynb (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/files/notebook/slc_main_build_steps.svg (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/tasks/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/tasks/motd.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/tasks/systemd.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/tasks/tutorial.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/templates/etc/systemd/system/jupyter.service (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/netplan/defaults/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/netplan/files/etc/netplan/01-network-manager.yaml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/netplan/tasks/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/poetry/defaults/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/poetry/tasks/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/script_languages/defaults/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/roles/script_languages/tasks/main.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/slc_setup.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/runtime/ansible/slc_setup_tasks.yml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/templates/additional_release_notes.jinja (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/templates/ci_code_build.jinja.yaml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/templates/ec2_cloudformation.jinja.yaml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/templates/inventory.jinja (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/templates/release_code_build.jinja.yaml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/templates/vm_bucket_cloudformation.jinja.yaml (100%) rename {exasol_script_languages_developer_sandbox => exasol/ds/sandbox}/templates/waf_cloudformation.jinja.yaml (100%) delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/export_vm.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/install_dependencies.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/make_ami_public.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/reset_password.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/setup_ci_codebuild.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/setup_ec2.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/setup_ec2_and_install_dependencies.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/setup_release_codebuild.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket_waf.py delete mode 100644 exasol_script_languages_developer_sandbox/cli/commands/update_release.py delete mode 100755 exasol_script_languages_developer_sandbox/main.py diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml index 1a098118..4b548454 100644 --- a/.github/workflows/release_droid_upload_github_release_assets.yml +++ b/.github/workflows/release_droid_upload_github_release_assets.yml @@ -29,7 +29,7 @@ jobs: - name: Setup Python & Poetry Environment uses: ./.github/actions/prepare_poetry_env - name: Build Release - run: poetry run python3 -m exasol_script_languages_developer_sandbox.main start-release-build --upload-url "${{ github.event.inputs.upload_url }}" --branch "$GITHUB_REF" + run: poetry run python3 -m exasol.ds.sandbox.main start-release-build --upload-url "${{ github.event.inputs.upload_url }}" --branch "$GITHUB_REF" env: # Set the secret as an env variable AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }} diff --git a/.github/workflows/test_release.yml b/.github/workflows/test_release.yml index dcfda8cb..dec14637 100644 --- a/.github/workflows/test_release.yml +++ b/.github/workflows/test_release.yml @@ -21,9 +21,9 @@ jobs: uses: ./.github/actions/prepare_poetry_env - name: Start test release - run: poetry run python3 -m exasol_script_languages_developer_sandbox.main start-test-release-build --release-title "${{ github.event.inputs.release_title }}" --branch "$GITHUB_REF" + run: poetry run python3 -m exasol.ds.sandbox.main start-test-release-build --release-title "${{ github.event.inputs.release_title }}" --branch "$GITHUB_REF" env: # Set the secret as an env variable AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }} AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} - GITHUB_TOKEN: ${{ github.token }} \ No newline at end of file + GITHUB_TOKEN: ${{ github.token }} diff --git a/error_code_config.yml b/error_code_config.yml index 00da1e08..f2db5e1b 100644 --- a/error_code_config.yml +++ b/error_code_config.yml @@ -1,3 +1,3 @@ error-tags: - SLDS: - highest-index: 0 \ No newline at end of file + DSS: + highest-index: 0 diff --git a/exasol_script_languages_developer_sandbox/__init__.py b/exasol/ds/sandbox/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/__init__.py rename to exasol/ds/sandbox/__init__.py diff --git a/exasol_script_languages_developer_sandbox/cli/__init__.py b/exasol/ds/sandbox/cli/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/cli/__init__.py rename to exasol/ds/sandbox/cli/__init__.py diff --git a/exasol_script_languages_developer_sandbox/cli/cli.py b/exasol/ds/sandbox/cli/cli.py similarity index 100% rename from exasol_script_languages_developer_sandbox/cli/cli.py rename to exasol/ds/sandbox/cli/cli.py diff --git a/exasol_script_languages_developer_sandbox/cli/commands/__init__.py b/exasol/ds/sandbox/cli/commands/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/cli/commands/__init__.py rename to exasol/ds/sandbox/cli/commands/__init__.py diff --git a/exasol_script_languages_developer_sandbox/cli/commands/create_vm.py b/exasol/ds/sandbox/cli/commands/create_vm.py similarity index 52% rename from exasol_script_languages_developer_sandbox/cli/commands/create_vm.py rename to exasol/ds/sandbox/cli/commands/create_vm.py index 0e8b0b5e..b00c572d 100644 --- a/exasol_script_languages_developer_sandbox/cli/commands/create_vm.py +++ b/exasol/ds/sandbox/cli/commands/create_vm.py @@ -3,19 +3,19 @@ import click -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.ec2_options import ec2_key_options -from exasol_script_languages_developer_sandbox.cli.options.id_options import id_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.cli.options.vm_options import vm_options -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.run_create_vm import run_create_vm +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.ec2_options import ec2_key_options +from exasol.ds.sandbox.cli.options.id_options import id_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.cli.options.vm_options import vm_options +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.run_create_vm import run_create_vm @cli.command() diff --git a/exasol/ds/sandbox/cli/commands/export_vm.py b/exasol/ds/sandbox/cli/commands/export_vm.py new file mode 100644 index 00000000..424d4657 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/export_vm.py @@ -0,0 +1,39 @@ +from typing import Tuple + +import click + +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.id_options import id_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.cli.options.vm_options import vm_options +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.export_vm.run_export_vm import run_export_vm +from exasol.ds.sandbox.lib.logging import set_log_level + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +@click.option('--stack-name', required=True, + type=str, + help="Existing cloudformation stack containing the EC2 instance.") +@add_options(vm_options) +@add_options(id_options) +def export_vm( + aws_profile: str, + stack_name: str, + vm_image_format: Tuple[str, ...], + no_vm: bool, + asset_id: str, + log_level: str): + """ + Debug command which creates a new VM image from a running EC2-Instance. + """ + current_vm_image_formats = tuple() if no_vm else vm_image_format + set_log_level(log_level) + run_export_vm(AwsAccess(aws_profile), stack_name, current_vm_image_formats, + AssetId(asset_id), default_config_object) diff --git a/exasol/ds/sandbox/cli/commands/install_dependencies.py b/exasol/ds/sandbox/cli/commands/install_dependencies.py new file mode 100644 index 00000000..82115f76 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/install_dependencies.py @@ -0,0 +1,23 @@ +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.ec2_options import ec2_host_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies + + +@cli.command() +@add_options(logging_options) +@add_options(ec2_host_options) +def install_dependencies( + host_name: str, + ssh_private_key: str, + log_level: str): + """ + Debug command to ansible-installation onto an EC-2 instance. + """ + set_log_level(log_level) + run_install_dependencies(AnsibleAccess(), default_config_object, (HostInfo(host_name, ssh_private_key),)) diff --git a/exasol/ds/sandbox/cli/commands/make_ami_public.py b/exasol/ds/sandbox/cli/commands/make_ami_public.py new file mode 100644 index 00000000..bdb8c608 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/make_ami_public.py @@ -0,0 +1,24 @@ +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.id_options import id_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.export_vm.run_make_ami_public import run_make_ami_public +from exasol.ds.sandbox.lib.logging import set_log_level + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +@add_options(id_options) +def make_ami_public( + aws_profile: str, + asset_id: str, + log_level: str): + """ + Debug command which makes an existing AMI public. + """ + set_log_level(log_level) + run_make_ami_public(AwsAccess(aws_profile), AssetId(asset_id)) diff --git a/exasol/ds/sandbox/cli/commands/reset_password.py b/exasol/ds/sandbox/cli/commands/reset_password.py new file mode 100644 index 00000000..05411d6c --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/reset_password.py @@ -0,0 +1,27 @@ +import click + +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.ec2_options import ec2_host_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.setup_ec2.run_reset_password import run_reset_password + + +@cli.command() +@add_options(logging_options) +@add_options(ec2_host_options) +@click.option('--default-password', required=True, type=str, + help="The new (temporary) default password.") +def reset_password( + host_name: str, + ssh_private_key: str, + default_password: str, + log_level: str): + """ + Debug command to reset password on a remote EC-2-instance via ansible. + """ + set_log_level(log_level) + run_reset_password(AnsibleAccess(), default_password, (HostInfo(host_name, ssh_private_key),)) diff --git a/exasol/ds/sandbox/cli/commands/setup_ci_codebuild.py b/exasol/ds/sandbox/cli/commands/setup_ci_codebuild.py new file mode 100644 index 00000000..ce1c91d4 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/setup_ci_codebuild.py @@ -0,0 +1,20 @@ +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.setup_ci_codebuild.ci_codebuild import run_setup_ci_codebuild +from exasol.ds.sandbox.lib.logging import set_log_level + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +def setup_ci_codebuild( + aws_profile: str, + log_level: str): + """ + Command to deploy the CI CodeBuild stack + """ + set_log_level(log_level) + run_setup_ci_codebuild(AwsAccess(aws_profile)) diff --git a/exasol/ds/sandbox/cli/commands/setup_ec2.py b/exasol/ds/sandbox/cli/commands/setup_ec2.py new file mode 100644 index 00000000..48adfbe5 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/setup_ec2.py @@ -0,0 +1,31 @@ +from typing import Optional + +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.ec2_options import ec2_key_options +from exasol.ds.sandbox.cli.options.id_options import id_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_setup_ec2 + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +@add_options(ec2_key_options) +@add_options(id_options) +def setup_ec2( + aws_profile: str, + ec2_key_file: Optional[str], + ec2_key_name: Optional[str], + asset_id: str, + log_level: str): + """ + Debug command to test setup of an EC-2 instance. + """ + set_log_level(log_level) + run_setup_ec2(AwsAccess(aws_profile), ec2_key_file, ec2_key_name, AssetId(asset_id), default_config_object) diff --git a/exasol/ds/sandbox/cli/commands/setup_ec2_and_install_dependencies.py b/exasol/ds/sandbox/cli/commands/setup_ec2_and_install_dependencies.py new file mode 100644 index 00000000..5f2467f6 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/setup_ec2_and_install_dependencies.py @@ -0,0 +1,34 @@ +from typing import Optional + +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.ec2_options import ec2_key_options +from exasol.ds.sandbox.cli.options.id_options import id_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2_and_install_dependencies import \ + run_setup_ec2_and_install_dependencies + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +@add_options(ec2_key_options) +@add_options(id_options) +def setup_ec2_and_install_dependencies( + aws_profile: str, + ec2_key_file: Optional[str], + ec2_key_name: Optional[str], + asset_id: str, + log_level: str): + """ + Debug command to check setup and installation of an EC-2 instance. + """ + set_log_level(log_level) + run_setup_ec2_and_install_dependencies(AwsAccess(aws_profile), ec2_key_file, ec2_key_name, + AssetId(asset_id), AnsibleAccess(), default_config_object) diff --git a/exasol/ds/sandbox/cli/commands/setup_release_codebuild.py b/exasol/ds/sandbox/cli/commands/setup_release_codebuild.py new file mode 100644 index 00000000..40986442 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/setup_release_codebuild.py @@ -0,0 +1,21 @@ +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.setup_release_codebuild.release_codebuild import \ + run_setup_release_codebuild + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +def setup_release_codebuild( + aws_profile: str, + log_level: str): + """ + Command to deploy the Release CodeBuild stack + """ + set_log_level(log_level) + run_setup_release_codebuild(AwsAccess(aws_profile)) diff --git a/exasol/ds/sandbox/cli/commands/setup_vm_bucket.py b/exasol/ds/sandbox/cli/commands/setup_vm_bucket.py new file mode 100644 index 00000000..10d376ac --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/setup_vm_bucket.py @@ -0,0 +1,21 @@ +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import run_setup_vm_bucket + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +def setup_vm_bucket( + aws_profile: str, + log_level: str): + """ + Command to deploy the VM S3-Bucket + """ + set_log_level(log_level) + run_setup_vm_bucket(AwsAccess(aws_profile), default_config_object) diff --git a/exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py b/exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py new file mode 100644 index 00000000..69a4c236 --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py @@ -0,0 +1,26 @@ +import click + +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket_waf import run_setup_vm_bucket_waf + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +@click.option('--allowed-ip', type=str, + help="The allowed IP address for which CAPTCHA will not be applied.") +def setup_vm_bucket_waf( + aws_profile: str, + allowed_ip: str, + log_level: str): + """ + Command to deploy the VM S3-Bucket Web Application Firewall. Needs to run before deploying the VM Bucket itself. + """ + set_log_level(log_level) + run_setup_vm_bucket_waf(AwsAccess(aws_profile), allowed_ip, default_config_object) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/show_aws_assets.py b/exasol/ds/sandbox/cli/commands/show_aws_assets.py similarity index 64% rename from exasol_script_languages_developer_sandbox/cli/commands/show_aws_assets.py rename to exasol/ds/sandbox/cli/commands/show_aws_assets.py index 542ce8f9..63604237 100644 --- a/exasol_script_languages_developer_sandbox/cli/commands/show_aws_assets.py +++ b/exasol/ds/sandbox/cli/commands/show_aws_assets.py @@ -2,14 +2,14 @@ import click -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.asset_printing.print_assets import all_asset_types, print_assets -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.asset_printing.print_assets import all_asset_types, print_assets +from exasol.ds.sandbox.lib.logging import set_log_level @cli.command() diff --git a/exasol_script_languages_developer_sandbox/cli/commands/start_release_build.py b/exasol/ds/sandbox/cli/commands/start_release_build.py similarity index 54% rename from exasol_script_languages_developer_sandbox/cli/commands/start_release_build.py rename to exasol/ds/sandbox/cli/commands/start_release_build.py index 3457b2f9..eedc72de 100644 --- a/exasol_script_languages_developer_sandbox/cli/commands/start_release_build.py +++ b/exasol/ds/sandbox/cli/commands/start_release_build.py @@ -3,14 +3,14 @@ import click -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.release_build.run_release_build import run_start_release_build +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import default_config_object +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.release_build.run_release_build import run_start_release_build @cli.command() diff --git a/exasol_script_languages_developer_sandbox/cli/commands/start_test_release_build.py b/exasol/ds/sandbox/cli/commands/start_test_release_build.py similarity index 55% rename from exasol_script_languages_developer_sandbox/cli/commands/start_test_release_build.py rename to exasol/ds/sandbox/cli/commands/start_test_release_build.py index d12c2102..b4991c95 100644 --- a/exasol_script_languages_developer_sandbox/cli/commands/start_test_release_build.py +++ b/exasol/ds/sandbox/cli/commands/start_test_release_build.py @@ -3,14 +3,14 @@ import click -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.github_release_access import GithubReleaseAccess -from exasol_script_languages_developer_sandbox.lib.release_build.run_release_build import run_start_test_release_build +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.github_release_access import GithubReleaseAccess +from exasol.ds.sandbox.lib.release_build.run_release_build import run_start_test_release_build @cli.command() diff --git a/exasol/ds/sandbox/cli/commands/update_release.py b/exasol/ds/sandbox/cli/commands/update_release.py new file mode 100644 index 00000000..f8346eec --- /dev/null +++ b/exasol/ds/sandbox/cli/commands/update_release.py @@ -0,0 +1,35 @@ +import os +from typing import Optional + +import click + +from exasol.ds.sandbox.cli.cli import cli +from exasol.ds.sandbox.cli.common import add_options +from exasol.ds.sandbox.cli.options.aws_options import aws_options +from exasol.ds.sandbox.cli.options.id_options import id_options +from exasol.ds.sandbox.cli.options.logging import logging_options +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import set_log_level +from exasol.ds.sandbox.lib.github_release_access import GithubReleaseAccess +from exasol.ds.sandbox.lib.update_release.run_update_release import run_update_release + + +@cli.command() +@add_options(aws_options) +@add_options(logging_options) +@click.option('--release-id', type=int, required=True, + help="""The Github release id which will be updated.""") +@add_options(id_options) +def update_release( + aws_profile: Optional[str], + release_id: int, + asset_id: str, + log_level: str): + """ + This command attaches the links of the release assets (AMI, VM images) to the Github release, + indicated by parameter 'release-id'. + """ + set_log_level(log_level) + run_update_release(AwsAccess(aws_profile), GithubReleaseAccess(os.getenv("GITHUB_TOKEN")), + release_id, AssetId(asset_id)) diff --git a/exasol_script_languages_developer_sandbox/cli/common.py b/exasol/ds/sandbox/cli/common.py similarity index 100% rename from exasol_script_languages_developer_sandbox/cli/common.py rename to exasol/ds/sandbox/cli/common.py diff --git a/exasol_script_languages_developer_sandbox/cli/options/__init__.py b/exasol/ds/sandbox/cli/options/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/cli/options/__init__.py rename to exasol/ds/sandbox/cli/options/__init__.py diff --git a/exasol_script_languages_developer_sandbox/cli/options/aws_options.py b/exasol/ds/sandbox/cli/options/aws_options.py similarity index 100% rename from exasol_script_languages_developer_sandbox/cli/options/aws_options.py rename to exasol/ds/sandbox/cli/options/aws_options.py diff --git a/exasol_script_languages_developer_sandbox/cli/options/ec2_options.py b/exasol/ds/sandbox/cli/options/ec2_options.py similarity index 100% rename from exasol_script_languages_developer_sandbox/cli/options/ec2_options.py rename to exasol/ds/sandbox/cli/options/ec2_options.py diff --git a/exasol_script_languages_developer_sandbox/cli/options/id_options.py b/exasol/ds/sandbox/cli/options/id_options.py similarity index 78% rename from exasol_script_languages_developer_sandbox/cli/options/id_options.py rename to exasol/ds/sandbox/cli/options/id_options.py index 7b216d1d..06ad4f54 100644 --- a/exasol_script_languages_developer_sandbox/cli/options/id_options.py +++ b/exasol/ds/sandbox/cli/options/id_options.py @@ -1,6 +1,6 @@ import click -from exasol_script_languages_developer_sandbox.lib.config import SLC_VERSION +from exasol.ds.sandbox.lib.config import SLC_VERSION id_options = [ click.option('--asset-id', type=str, default=SLC_VERSION, diff --git a/exasol_script_languages_developer_sandbox/cli/options/logging.py b/exasol/ds/sandbox/cli/options/logging.py similarity index 82% rename from exasol_script_languages_developer_sandbox/cli/options/logging.py rename to exasol/ds/sandbox/cli/options/logging.py index f703521a..9cd31077 100644 --- a/exasol_script_languages_developer_sandbox/cli/options/logging.py +++ b/exasol/ds/sandbox/cli/options/logging.py @@ -1,6 +1,6 @@ import click -from exasol_script_languages_developer_sandbox.lib.logging import SUPPORTED_LOG_LEVELS +from exasol.ds.sandbox.lib.logging import SUPPORTED_LOG_LEVELS logging_options = [ click.option('--log-level', type=click.Choice(list(SUPPORTED_LOG_LEVELS.keys())), default="normal", diff --git a/exasol_script_languages_developer_sandbox/cli/options/vm_options.py b/exasol/ds/sandbox/cli/options/vm_options.py similarity index 85% rename from exasol_script_languages_developer_sandbox/cli/options/vm_options.py rename to exasol/ds/sandbox/cli/options/vm_options.py index d1b6a5b1..59fa2a6d 100644 --- a/exasol_script_languages_developer_sandbox/cli/options/vm_options.py +++ b/exasol/ds/sandbox/cli/options/vm_options.py @@ -1,6 +1,6 @@ import click -from exasol_script_languages_developer_sandbox.lib.export_vm.vm_disk_image_format \ +from exasol.ds.sandbox.lib.export_vm.vm_disk_image_format \ import VmDiskImageFormat vm_options = [ diff --git a/exasol_script_languages_developer_sandbox/lib/__init__.py b/exasol/ds/sandbox/lib/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/__init__.py rename to exasol/ds/sandbox/lib/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/ansible/__init__.py b/exasol/ds/sandbox/lib/ansible/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/ansible/__init__.py rename to exasol/ds/sandbox/lib/ansible/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_access.py b/exasol/ds/sandbox/lib/ansible/ansible_access.py similarity index 86% rename from exasol_script_languages_developer_sandbox/lib/ansible/ansible_access.py rename to exasol/ds/sandbox/lib/ansible/ansible_access.py index 3d2f7160..9a3770e0 100644 --- a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_access.py +++ b/exasol/ds/sandbox/lib/ansible/ansible_access.py @@ -2,7 +2,7 @@ import ansible_runner -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import AnsibleRunContext +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext class AnsibleException(RuntimeError): diff --git a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_context_manager.py b/exasol/ds/sandbox/lib/ansible/ansible_context_manager.py similarity index 76% rename from exasol_script_languages_developer_sandbox/lib/ansible/ansible_context_manager.py rename to exasol/ds/sandbox/lib/ansible/ansible_context_manager.py index a2b5ef0a..ffc876a1 100644 --- a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_context_manager.py +++ b/exasol/ds/sandbox/lib/ansible/ansible_context_manager.py @@ -2,9 +2,9 @@ from pathlib import Path from typing import Tuple -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_repository import AnsibleRepository -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_runner import AnsibleRunner +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.ansible.ansible_repository import AnsibleRepository +from exasol.ds.sandbox.lib.ansible.ansible_runner import AnsibleRunner class AnsibleContextManager: diff --git a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_repository.py b/exasol/ds/sandbox/lib/ansible/ansible_repository.py similarity index 91% rename from exasol_script_languages_developer_sandbox/lib/ansible/ansible_repository.py rename to exasol/ds/sandbox/lib/ansible/ansible_repository.py index 8e9fd658..c53329a3 100644 --- a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_repository.py +++ b/exasol/ds/sandbox/lib/ansible/ansible_repository.py @@ -1,7 +1,7 @@ from pathlib import Path import importlib_resources as ir -import exasol_script_languages_developer_sandbox.runtime.ansible -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType +import exasol.ds.sandbox.runtime.ansible +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType LOG = get_status_logger(LogType.ANSIBLE) @@ -71,4 +71,4 @@ def copy_to(self, target: Path) -> None: self.copy_importlib_resources_dir_tree(source_path, target) -default_repositories = (AnsibleResourceRepository(exasol_script_languages_developer_sandbox.runtime.ansible),) +default_repositories = (AnsibleResourceRepository(exasol.ds.sandbox.runtime.ansible),) diff --git a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_run_context.py b/exasol/ds/sandbox/lib/ansible/ansible_run_context.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/ansible/ansible_run_context.py rename to exasol/ds/sandbox/lib/ansible/ansible_run_context.py diff --git a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_runner.py b/exasol/ds/sandbox/lib/ansible/ansible_runner.py similarity index 63% rename from exasol_script_languages_developer_sandbox/lib/ansible/ansible_runner.py rename to exasol/ds/sandbox/lib/ansible/ansible_runner.py index e2a16b29..dd9c53dc 100644 --- a/exasol_script_languages_developer_sandbox/lib/ansible/ansible_runner.py +++ b/exasol/ds/sandbox/lib/ansible/ansible_runner.py @@ -1,11 +1,11 @@ from pathlib import Path from typing import Tuple -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import AnsibleRunContext -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo -from exasol_script_languages_developer_sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.render_template import render_template LOG = get_status_logger(LogType.ANSIBLE) diff --git a/exasol_script_languages_developer_sandbox/lib/asset_id.py b/exasol/ds/sandbox/lib/asset_id.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/asset_id.py rename to exasol/ds/sandbox/lib/asset_id.py diff --git a/exasol_script_languages_developer_sandbox/lib/asset_printing/__init__.py b/exasol/ds/sandbox/lib/asset_printing/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/asset_printing/__init__.py rename to exasol/ds/sandbox/lib/asset_printing/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/asset_printing/mark_down_printer.py b/exasol/ds/sandbox/lib/asset_printing/mark_down_printer.py similarity index 89% rename from exasol_script_languages_developer_sandbox/lib/asset_printing/mark_down_printer.py rename to exasol/ds/sandbox/lib/asset_printing/mark_down_printer.py index 6ba324cb..b5dcd4bd 100644 --- a/exasol_script_languages_developer_sandbox/lib/asset_printing/mark_down_printer.py +++ b/exasol/ds/sandbox/lib/asset_printing/mark_down_printer.py @@ -4,7 +4,7 @@ import pandas as pd -from exasol_script_languages_developer_sandbox.lib.asset_printing.printing_factory import PrintingFactory, TablePrinter, \ +from exasol.ds.sandbox.lib.asset_printing.printing_factory import PrintingFactory, TablePrinter, \ TextPrinter, TextObject diff --git a/exasol_script_languages_developer_sandbox/lib/asset_printing/print_assets.py b/exasol/ds/sandbox/lib/asset_printing/print_assets.py similarity index 93% rename from exasol_script_languages_developer_sandbox/lib/asset_printing/print_assets.py rename to exasol/ds/sandbox/lib/asset_printing/print_assets.py index 4c722ae0..c1f6b8f1 100644 --- a/exasol_script_languages_developer_sandbox/lib/asset_printing/print_assets.py +++ b/exasol/ds/sandbox/lib/asset_printing/print_assets.py @@ -4,17 +4,17 @@ import humanfriendly -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.asset_printing.mark_down_printer import MarkdownPrintingFactory -from exasol_script_languages_developer_sandbox.lib.asset_printing.printing_factory import PrintingFactory, TextObject, \ +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.asset_printing.mark_down_printer import MarkdownPrintingFactory +from exasol.ds.sandbox.lib.asset_printing.printing_factory import PrintingFactory, TextObject, \ HighlightedTextObject -from exasol_script_languages_developer_sandbox.lib.asset_printing.rich_console_printer import RichConsolePrintingFactory -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.asset_printing.rich_console_printer import RichConsolePrintingFactory +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from enum import Enum -from exasol_script_languages_developer_sandbox.lib.aws_access.cloudformation_stack import CloudformationStack -from exasol_script_languages_developer_sandbox.lib.tags import DEFAULT_TAG_KEY -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket, find_url_for_bucket +from exasol.ds.sandbox.lib.aws_access.cloudformation_stack import CloudformationStack +from exasol.ds.sandbox.lib.tags import DEFAULT_TAG_KEY +from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket, find_url_for_bucket class AssetTypes(Enum): diff --git a/exasol_script_languages_developer_sandbox/lib/asset_printing/printing_factory.py b/exasol/ds/sandbox/lib/asset_printing/printing_factory.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/asset_printing/printing_factory.py rename to exasol/ds/sandbox/lib/asset_printing/printing_factory.py diff --git a/exasol_script_languages_developer_sandbox/lib/asset_printing/rich_console_printer.py b/exasol/ds/sandbox/lib/asset_printing/rich_console_printer.py similarity index 90% rename from exasol_script_languages_developer_sandbox/lib/asset_printing/rich_console_printer.py rename to exasol/ds/sandbox/lib/asset_printing/rich_console_printer.py index 71370366..58a5e06f 100644 --- a/exasol_script_languages_developer_sandbox/lib/asset_printing/rich_console_printer.py +++ b/exasol/ds/sandbox/lib/asset_printing/rich_console_printer.py @@ -5,7 +5,7 @@ from rich.console import Console from rich.text import Text -from exasol_script_languages_developer_sandbox.lib.asset_printing.printing_factory import PrintingFactory, TablePrinter, \ +from exasol.ds.sandbox.lib.asset_printing.printing_factory import PrintingFactory, TablePrinter, \ TextPrinter, TextObject diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/__init__.py b/exasol/ds/sandbox/lib/aws_access/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/__init__.py rename to exasol/ds/sandbox/lib/aws_access/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/ami.py b/exasol/ds/sandbox/lib/aws_access/ami.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/ami.py rename to exasol/ds/sandbox/lib/aws_access/ami.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/aws_access.py b/exasol/ds/sandbox/lib/aws_access/aws_access.py similarity index 93% rename from exasol_script_languages_developer_sandbox/lib/aws_access/aws_access.py rename to exasol/ds/sandbox/lib/aws_access/aws_access.py index 90280786..847f5b5f 100644 --- a/exasol_script_languages_developer_sandbox/lib/aws_access/aws_access.py +++ b/exasol/ds/sandbox/lib/aws_access/aws_access.py @@ -4,20 +4,20 @@ import boto3 import botocore -from exasol_script_languages_developer_sandbox.lib.aws_access.ami import Ami -from exasol_script_languages_developer_sandbox.lib.aws_access.cloudformation_stack import CloudformationStack -from exasol_script_languages_developer_sandbox.lib.aws_access.deployer import Deployer -from exasol_script_languages_developer_sandbox.lib.aws_access.ec2_instance import EC2Instance -from exasol_script_languages_developer_sandbox.lib.aws_access.ec2_instance_status import EC2InstanceStatus -from exasol_script_languages_developer_sandbox.lib.aws_access.export_image_task import ExportImageTask -from exasol_script_languages_developer_sandbox.lib.aws_access.key_pair import KeyPair -from exasol_script_languages_developer_sandbox.lib.aws_access.s3_object import S3Object -from exasol_script_languages_developer_sandbox.lib.aws_access.snapshot import Snapshot -from exasol_script_languages_developer_sandbox.lib.aws_access.stack_resource import StackResource -from exasol_script_languages_developer_sandbox.lib.aws_access.waiter.codebuild_waiter import CodeBuildWaiter -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.tags import create_default_asset_tag -from exasol_script_languages_developer_sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat +from exasol.ds.sandbox.lib.aws_access.ami import Ami +from exasol.ds.sandbox.lib.aws_access.cloudformation_stack import CloudformationStack +from exasol.ds.sandbox.lib.aws_access.deployer import Deployer +from exasol.ds.sandbox.lib.aws_access.ec2_instance import EC2Instance +from exasol.ds.sandbox.lib.aws_access.ec2_instance_status import EC2InstanceStatus +from exasol.ds.sandbox.lib.aws_access.export_image_task import ExportImageTask +from exasol.ds.sandbox.lib.aws_access.key_pair import KeyPair +from exasol.ds.sandbox.lib.aws_access.s3_object import S3Object +from exasol.ds.sandbox.lib.aws_access.snapshot import Snapshot +from exasol.ds.sandbox.lib.aws_access.stack_resource import StackResource +from exasol.ds.sandbox.lib.aws_access.waiter.codebuild_waiter import CodeBuildWaiter +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.tags import create_default_asset_tag +from exasol.ds.sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat LOG = get_status_logger(LogType.AWS_ACCESS) diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/cloudformation_stack.py b/exasol/ds/sandbox/lib/aws_access/cloudformation_stack.py similarity index 94% rename from exasol_script_languages_developer_sandbox/lib/aws_access/cloudformation_stack.py rename to exasol/ds/sandbox/lib/aws_access/cloudformation_stack.py index 12e6caf4..a53bcf26 100644 --- a/exasol_script_languages_developer_sandbox/lib/aws_access/cloudformation_stack.py +++ b/exasol/ds/sandbox/lib/aws_access/cloudformation_stack.py @@ -2,7 +2,7 @@ from datetime import datetime from typing import List, Dict, Optional -from exasol_script_languages_developer_sandbox.lib.aws_access.common import get_value_safe +from exasol.ds.sandbox.lib.aws_access.common import get_value_safe class CloudformationStack: diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/common.py b/exasol/ds/sandbox/lib/aws_access/common.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/common.py rename to exasol/ds/sandbox/lib/aws_access/common.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/deployer.py b/exasol/ds/sandbox/lib/aws_access/deployer.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/deployer.py rename to exasol/ds/sandbox/lib/aws_access/deployer.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/ec2_instance.py b/exasol/ds/sandbox/lib/aws_access/ec2_instance.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/ec2_instance.py rename to exasol/ds/sandbox/lib/aws_access/ec2_instance.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/ec2_instance_status.py b/exasol/ds/sandbox/lib/aws_access/ec2_instance_status.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/ec2_instance_status.py rename to exasol/ds/sandbox/lib/aws_access/ec2_instance_status.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/export_image_task.py b/exasol/ds/sandbox/lib/aws_access/export_image_task.py similarity index 93% rename from exasol_script_languages_developer_sandbox/lib/aws_access/export_image_task.py rename to exasol/ds/sandbox/lib/aws_access/export_image_task.py index 36d9b08d..bb6cd4e7 100644 --- a/exasol_script_languages_developer_sandbox/lib/aws_access/export_image_task.py +++ b/exasol/ds/sandbox/lib/aws_access/export_image_task.py @@ -1,6 +1,6 @@ from typing import Optional, List, Dict -from exasol_script_languages_developer_sandbox.lib.aws_access.common import get_value_safe +from exasol.ds.sandbox.lib.aws_access.common import get_value_safe class ExportImageTask: diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/key_pair.py b/exasol/ds/sandbox/lib/aws_access/key_pair.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/key_pair.py rename to exasol/ds/sandbox/lib/aws_access/key_pair.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/s3_object.py b/exasol/ds/sandbox/lib/aws_access/s3_object.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/s3_object.py rename to exasol/ds/sandbox/lib/aws_access/s3_object.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/snapshot.py b/exasol/ds/sandbox/lib/aws_access/snapshot.py similarity index 92% rename from exasol_script_languages_developer_sandbox/lib/aws_access/snapshot.py rename to exasol/ds/sandbox/lib/aws_access/snapshot.py index a0ddd4b8..e725c02c 100644 --- a/exasol_script_languages_developer_sandbox/lib/aws_access/snapshot.py +++ b/exasol/ds/sandbox/lib/aws_access/snapshot.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Optional, List, Dict -from exasol_script_languages_developer_sandbox.lib.aws_access.common import get_value_safe +from exasol.ds.sandbox.lib.aws_access.common import get_value_safe class Snapshot: diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/stack_resource.py b/exasol/ds/sandbox/lib/aws_access/stack_resource.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/stack_resource.py rename to exasol/ds/sandbox/lib/aws_access/stack_resource.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/waiter/__init__.py b/exasol/ds/sandbox/lib/aws_access/waiter/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/aws_access/waiter/__init__.py rename to exasol/ds/sandbox/lib/aws_access/waiter/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/aws_access/waiter/codebuild_waiter.py b/exasol/ds/sandbox/lib/aws_access/waiter/codebuild_waiter.py similarity index 96% rename from exasol_script_languages_developer_sandbox/lib/aws_access/waiter/codebuild_waiter.py rename to exasol/ds/sandbox/lib/aws_access/waiter/codebuild_waiter.py index 6ba9472b..52e774a8 100644 --- a/exasol_script_languages_developer_sandbox/lib/aws_access/waiter/codebuild_waiter.py +++ b/exasol/ds/sandbox/lib/aws_access/waiter/codebuild_waiter.py @@ -1,7 +1,7 @@ import time from typing import Iterable, Any -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType LOG = get_status_logger(LogType.AWS_ACCESS) diff --git a/exasol_script_languages_developer_sandbox/lib/config.py b/exasol/ds/sandbox/lib/config.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/config.py rename to exasol/ds/sandbox/lib/config.py diff --git a/exasol_script_languages_developer_sandbox/lib/export_vm/__init__.py b/exasol/ds/sandbox/lib/export_vm/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/export_vm/__init__.py rename to exasol/ds/sandbox/lib/export_vm/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/export_vm/rename_s3_objects.py b/exasol/ds/sandbox/lib/export_vm/rename_s3_objects.py similarity index 84% rename from exasol_script_languages_developer_sandbox/lib/export_vm/rename_s3_objects.py rename to exasol/ds/sandbox/lib/export_vm/rename_s3_objects.py index a63abdf8..d0e35d8d 100644 --- a/exasol_script_languages_developer_sandbox/lib/export_vm/rename_s3_objects.py +++ b/exasol/ds/sandbox/lib/export_vm/rename_s3_objects.py @@ -1,7 +1,7 @@ -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.aws_access.export_image_task import ExportImageTask -from exasol_script_languages_developer_sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.aws_access.export_image_task import ExportImageTask +from exasol.ds.sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat def build_image_source(prefix: str, export_image_task_id: str, vm_image_format: VmDiskImageFormat) -> str: diff --git a/exasol_script_languages_developer_sandbox/lib/export_vm/run_export_vm.py b/exasol/ds/sandbox/lib/export_vm/run_export_vm.py similarity index 87% rename from exasol_script_languages_developer_sandbox/lib/export_vm/run_export_vm.py rename to exasol/ds/sandbox/lib/export_vm/run_export_vm.py index f5875928..e55634c4 100644 --- a/exasol_script_languages_developer_sandbox/lib/export_vm/run_export_vm.py +++ b/exasol/ds/sandbox/lib/export_vm/run_export_vm.py @@ -2,16 +2,16 @@ from dataclasses import dataclass from typing import Tuple -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.aws_access.export_image_task import ExportImageTask -from exasol_script_languages_developer_sandbox.lib.export_vm.rename_s3_objects import rename_image_in_s3 -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.setup_ec2.cf_stack import find_ec2_instance_in_cf_stack -from exasol_script_languages_developer_sandbox.lib.asset_printing.print_assets import print_assets -from exasol_script_languages_developer_sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket, find_vm_import_role +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.aws_access.export_image_task import ExportImageTask +from exasol.ds.sandbox.lib.export_vm.rename_s3_objects import rename_image_in_s3 +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.setup_ec2.cf_stack import find_ec2_instance_in_cf_stack +from exasol.ds.sandbox.lib.asset_printing.print_assets import print_assets +from exasol.ds.sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat +from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket, find_vm_import_role LOG = get_status_logger(LogType.EXPORT) diff --git a/exasol_script_languages_developer_sandbox/lib/export_vm/run_make_ami_public.py b/exasol/ds/sandbox/lib/export_vm/run_make_ami_public.py similarity index 72% rename from exasol_script_languages_developer_sandbox/lib/export_vm/run_make_ami_public.py rename to exasol/ds/sandbox/lib/export_vm/run_make_ami_public.py index eec0beb2..e70f18fa 100644 --- a/exasol_script_languages_developer_sandbox/lib/export_vm/run_make_ami_public.py +++ b/exasol/ds/sandbox/lib/export_vm/run_make_ami_public.py @@ -1,7 +1,7 @@ -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.tags import DEFAULT_TAG_KEY +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.tags import DEFAULT_TAG_KEY LOG = get_status_logger(LogType.EXPORT) diff --git a/exasol_script_languages_developer_sandbox/lib/export_vm/vm_disk_image_format.py b/exasol/ds/sandbox/lib/export_vm/vm_disk_image_format.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/export_vm/vm_disk_image_format.py rename to exasol/ds/sandbox/lib/export_vm/vm_disk_image_format.py diff --git a/exasol_script_languages_developer_sandbox/lib/github_release_access.py b/exasol/ds/sandbox/lib/github_release_access.py similarity index 96% rename from exasol_script_languages_developer_sandbox/lib/github_release_access.py rename to exasol/ds/sandbox/lib/github_release_access.py index 5b323161..842d9ccf 100644 --- a/exasol_script_languages_developer_sandbox/lib/github_release_access.py +++ b/exasol/ds/sandbox/lib/github_release_access.py @@ -3,7 +3,7 @@ from github import Github, GithubException from github.Repository import Repository -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType LOG = get_status_logger(LogType.RELEASE_ACCESS) diff --git a/exasol_script_languages_developer_sandbox/lib/logging.py b/exasol/ds/sandbox/lib/logging.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/logging.py rename to exasol/ds/sandbox/lib/logging.py diff --git a/exasol_script_languages_developer_sandbox/lib/release_build/__init__.py b/exasol/ds/sandbox/lib/release_build/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/release_build/__init__.py rename to exasol/ds/sandbox/lib/release_build/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/release_build/run_release_build.py b/exasol/ds/sandbox/lib/release_build/run_release_build.py similarity index 88% rename from exasol_script_languages_developer_sandbox/lib/release_build/run_release_build.py rename to exasol/ds/sandbox/lib/release_build/run_release_build.py index 9281dfa6..94627525 100644 --- a/exasol_script_languages_developer_sandbox/lib/release_build/run_release_build.py +++ b/exasol/ds/sandbox/lib/release_build/run_release_build.py @@ -2,12 +2,12 @@ import re from typing import Tuple, Dict, List -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.aws_access.stack_resource import StackResource -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.github_release_access import GithubReleaseAccess -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.setup_release_codebuild.release_codebuild import \ +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.aws_access.stack_resource import StackResource +from exasol.ds.sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.github_release_access import GithubReleaseAccess +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.setup_release_codebuild.release_codebuild import \ RELEASE_CODE_BUILD_STACK_NAME LOG = get_status_logger(LogType.RELEASE_BUILD) diff --git a/exasol_script_languages_developer_sandbox/lib/render_template.py b/exasol/ds/sandbox/lib/render_template.py similarity index 90% rename from exasol_script_languages_developer_sandbox/lib/render_template.py rename to exasol/ds/sandbox/lib/render_template.py index 6504321e..b3b2fabb 100644 --- a/exasol_script_languages_developer_sandbox/lib/render_template.py +++ b/exasol/ds/sandbox/lib/render_template.py @@ -3,7 +3,7 @@ def render_template(template: str, **kwargs): - env = jinja2.Environment(loader=jinja2.PackageLoader("exasol_script_languages_developer_sandbox"), + env = jinja2.Environment(loader=jinja2.PackageLoader("exasol.ds.sandbox"), autoescape=jinja2.select_autoescape(), keep_trailing_newline=True, undefined=StrictUndefined) t = env.get_template(template) diff --git a/exasol_script_languages_developer_sandbox/lib/run_create_vm.py b/exasol/ds/sandbox/lib/run_create_vm.py similarity index 66% rename from exasol_script_languages_developer_sandbox/lib/run_create_vm.py rename to exasol/ds/sandbox/lib/run_create_vm.py index 7f53bdb4..64810f9f 100644 --- a/exasol_script_languages_developer_sandbox/lib/run_create_vm.py +++ b/exasol/ds/sandbox/lib/run_create_vm.py @@ -1,24 +1,24 @@ import time from typing import Tuple, Optional -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ default_repositories -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import \ +from exasol.ds.sandbox.lib.ansible.ansible_run_context import \ reset_password_ansible_run_context, default_ansible_run_context -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.export_vm.run_make_ami_public import run_make_ami_public -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.export_vm.run_make_ami_public import run_make_ami_public +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo -from exasol_script_languages_developer_sandbox.lib.export_vm.run_export_vm import export_vm -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_reset_password import run_reset_password -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.export_vm.run_export_vm import export_vm +from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies +from exasol.ds.sandbox.lib.setup_ec2.run_reset_password import run_reset_password +from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ EC2StackLifecycleContextManager -from exasol_script_languages_developer_sandbox.lib.setup_ec2.source_ami import find_source_ami +from exasol.ds.sandbox.lib.setup_ec2.source_ami import find_source_ami LOG = get_status_logger(LogType.CREATE_VM) diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/__init__.py b/exasol/ds/sandbox/lib/setup_ci_codebuild/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/__init__.py rename to exasol/ds/sandbox/lib/setup_ci_codebuild/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/ci_codebuild.py b/exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py similarity index 50% rename from exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/ci_codebuild.py rename to exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py index a1d1f0ce..21e871fe 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ci_codebuild/ci_codebuild.py +++ b/exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py @@ -1,7 +1,7 @@ -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.render_template import render_template -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket STACK_NAME = "DATA-SCIENCE-SANDBOX-CI-TEST-CODEBUILD" diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/__init__.py b/exasol/ds/sandbox/lib/setup_ec2/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/__init__.py rename to exasol/ds/sandbox/lib/setup_ec2/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/cf_stack.py b/exasol/ds/sandbox/lib/setup_ec2/cf_stack.py similarity index 85% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/cf_stack.py rename to exasol/ds/sandbox/lib/setup_ec2/cf_stack.py index 4beec59b..ec275463 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ec2/cf_stack.py +++ b/exasol/ds/sandbox/lib/setup_ec2/cf_stack.py @@ -1,12 +1,12 @@ from __future__ import annotations from typing import Optional -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.setup_ec2.random_string_generator import get_random_str_of_length_n -from exasol_script_languages_developer_sandbox.lib.render_template import render_template -from exasol_script_languages_developer_sandbox.lib.tags import DEFAULT_TAG_KEY, create_default_asset_tag +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.setup_ec2.random_string_generator import get_random_str_of_length_n +from exasol.ds.sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.tags import DEFAULT_TAG_KEY, create_default_asset_tag _MAX_ATTEMPTS_TO_FIND_STACK_NAME = 3 diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/host_info.py b/exasol/ds/sandbox/lib/setup_ec2/host_info.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/host_info.py rename to exasol/ds/sandbox/lib/setup_ec2/host_info.py diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/key_file_manager.py b/exasol/ds/sandbox/lib/setup_ec2/key_file_manager.py similarity index 89% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/key_file_manager.py rename to exasol/ds/sandbox/lib/setup_ec2/key_file_manager.py index 680880e5..7b9e3c5c 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ec2/key_file_manager.py +++ b/exasol/ds/sandbox/lib/setup_ec2/key_file_manager.py @@ -2,9 +2,9 @@ from tempfile import mkstemp from typing import Optional -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.setup_ec2.random_string_generator import get_random_str +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.setup_ec2.random_string_generator import get_random_str LOG = get_status_logger(LogType.SETUP) diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/random_string_generator.py b/exasol/ds/sandbox/lib/setup_ec2/random_string_generator.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/random_string_generator.py rename to exasol/ds/sandbox/lib/setup_ec2/random_string_generator.py diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_install_dependencies.py b/exasol/ds/sandbox/lib/setup_ec2/run_install_dependencies.py similarity index 71% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/run_install_dependencies.py rename to exasol/ds/sandbox/lib/setup_ec2/run_install_dependencies.py index cefc5e36..c3c092ec 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_install_dependencies.py +++ b/exasol/ds/sandbox/lib/setup_ec2/run_install_dependencies.py @@ -1,15 +1,15 @@ from importlib.metadata import version from typing import Tuple -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_context_manager import AnsibleContextManager -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.ansible.ansible_context_manager import AnsibleContextManager +from exasol.ds.sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ default_repositories -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ default_ansible_run_context -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo def run_install_dependencies(ansible_access: AnsibleAccess, diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_reset_password.py b/exasol/ds/sandbox/lib/setup_ec2/run_reset_password.py similarity index 77% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/run_reset_password.py rename to exasol/ds/sandbox/lib/setup_ec2/run_reset_password.py index 17787087..021446db 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_reset_password.py +++ b/exasol/ds/sandbox/lib/setup_ec2/run_reset_password.py @@ -1,13 +1,13 @@ from typing import Tuple -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_context_manager import AnsibleContextManager -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.ansible.ansible_context_manager import AnsibleContextManager +from exasol.ds.sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ default_repositories -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ reset_password_ansible_run_context -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo def run_reset_password(ansible_access: AnsibleAccess, default_password: str, diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_setup_ec2.py b/exasol/ds/sandbox/lib/setup_ec2/run_setup_ec2.py similarity index 89% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/run_setup_ec2.py rename to exasol/ds/sandbox/lib/setup_ec2/run_setup_ec2.py index c1436087..7f5b15aa 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_setup_ec2.py +++ b/exasol/ds/sandbox/lib/setup_ec2/run_setup_ec2.py @@ -2,16 +2,16 @@ import time from typing import Optional, Tuple, Iterator -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.aws_access.ec2_instance import EC2Instance -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.setup_ec2.cf_stack import CloudformationStack, \ +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.aws_access.ec2_instance import EC2Instance +from exasol.ds.sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.setup_ec2.cf_stack import CloudformationStack, \ CloudformationStackContextManager -from exasol_script_languages_developer_sandbox.lib.setup_ec2.key_file_manager import KeyFileManager, \ +from exasol.ds.sandbox.lib.setup_ec2.key_file_manager import KeyFileManager, \ KeyFileManagerContextManager -from exasol_script_languages_developer_sandbox.lib.setup_ec2.source_ami import find_source_ami +from exasol.ds.sandbox.lib.setup_ec2.source_ami import find_source_ami LOG = get_status_logger(LogType.SETUP) diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_setup_ec2_and_install_dependencies.py b/exasol/ds/sandbox/lib/setup_ec2/run_setup_ec2_and_install_dependencies.py similarity index 73% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/run_setup_ec2_and_install_dependencies.py rename to exasol/ds/sandbox/lib/setup_ec2/run_setup_ec2_and_install_dependencies.py index 3676f9a7..e38a0275 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ec2/run_setup_ec2_and_install_dependencies.py +++ b/exasol/ds/sandbox/lib/setup_ec2/run_setup_ec2_and_install_dependencies.py @@ -1,22 +1,22 @@ import signal import time from typing import Tuple, Optional -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.ansible.ansible_repository import AnsibleRepository, \ default_repositories -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ default_ansible_run_context -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ +from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies +from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ EC2StackLifecycleContextManager -from exasol_script_languages_developer_sandbox.lib.setup_ec2.source_ami import find_source_ami +from exasol.ds.sandbox.lib.setup_ec2.source_ami import find_source_ami LOG = get_status_logger(LogType.SETUP) diff --git a/exasol_script_languages_developer_sandbox/lib/setup_ec2/source_ami.py b/exasol/ds/sandbox/lib/setup_ec2/source_ami.py similarity index 64% rename from exasol_script_languages_developer_sandbox/lib/setup_ec2/source_ami.py rename to exasol/ds/sandbox/lib/setup_ec2/source_ami.py index d17d629c..0c21de64 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_ec2/source_ami.py +++ b/exasol/ds/sandbox/lib/setup_ec2/source_ami.py @@ -1,7 +1,7 @@ from typing import Dict -from exasol_script_languages_developer_sandbox.lib.aws_access.ami import Ami -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.aws_access.ami import Ami +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess def find_source_ami(aws_access: AwsAccess, filters: Dict[str, str]) -> Ami: diff --git a/exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/__init__.py b/exasol/ds/sandbox/lib/setup_release_codebuild/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/__init__.py rename to exasol/ds/sandbox/lib/setup_release_codebuild/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/release_codebuild.py b/exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py similarity index 54% rename from exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/release_codebuild.py rename to exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py index 51e034a3..28d50f2e 100644 --- a/exasol_script_languages_developer_sandbox/lib/setup_release_codebuild/release_codebuild.py +++ b/exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py @@ -1,7 +1,7 @@ -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.render_template import render_template -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket RELEASE_CODE_BUILD_STACK_NAME = "DATA-SCIENCE-SANDBOX-RELEASE-CODEBUILD" diff --git a/exasol_script_languages_developer_sandbox/lib/tags.py b/exasol/ds/sandbox/lib/tags.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/tags.py rename to exasol/ds/sandbox/lib/tags.py diff --git a/exasol_script_languages_developer_sandbox/lib/update_release/__init__.py b/exasol/ds/sandbox/lib/update_release/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/update_release/__init__.py rename to exasol/ds/sandbox/lib/update_release/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/update_release/run_update_release.py b/exasol/ds/sandbox/lib/update_release/run_update_release.py similarity index 65% rename from exasol_script_languages_developer_sandbox/lib/update_release/run_update_release.py rename to exasol/ds/sandbox/lib/update_release/run_update_release.py index 51a7329c..5f507415 100644 --- a/exasol_script_languages_developer_sandbox/lib/update_release/run_update_release.py +++ b/exasol/ds/sandbox/lib/update_release/run_update_release.py @@ -1,11 +1,11 @@ import logging import tempfile -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.asset_printing.print_assets import print_assets, AssetTypes -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.github_release_access import GithubReleaseAccess -from exasol_script_languages_developer_sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.asset_printing.print_assets import print_assets, AssetTypes +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.github_release_access import GithubReleaseAccess +from exasol.ds.sandbox.lib.render_template import render_template def run_update_release(aws_access: AwsAccess, gh_access: GithubReleaseAccess, diff --git a/exasol_script_languages_developer_sandbox/lib/vm_bucket/__init__.py b/exasol/ds/sandbox/lib/vm_bucket/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/lib/vm_bucket/__init__.py rename to exasol/ds/sandbox/lib/vm_bucket/__init__.py diff --git a/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket.py b/exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket.py similarity index 82% rename from exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket.py rename to exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket.py index 24b59855..71782e71 100644 --- a/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket.py +++ b/exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket.py @@ -1,10 +1,10 @@ -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.render_template import render_template from enum import Enum -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket_waf import find_acl_arn +from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket_waf import find_acl_arn STACK_NAME = "DATA-SCIENCE-SANDBOX-VM-SLC-Bucket" diff --git a/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket_waf.py b/exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket_waf.py similarity index 83% rename from exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket_waf.py rename to exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket_waf.py index 59e74f30..be84ffe1 100644 --- a/exasol_script_languages_developer_sandbox/lib/vm_bucket/vm_slc_bucket_waf.py +++ b/exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket_waf.py @@ -1,7 +1,7 @@ -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import ConfigObject -from exasol_script_languages_developer_sandbox.lib.logging import get_status_logger, LogType -from exasol_script_languages_developer_sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import ConfigObject +from exasol.ds.sandbox.lib.logging import get_status_logger, LogType +from exasol.ds.sandbox.lib.render_template import render_template STACK_NAME = "DATA-SCIENCE-SANDBOX-VM-SLC-Bucket-WAF" diff --git a/exasol/ds/sandbox/main.py b/exasol/ds/sandbox/main.py new file mode 100755 index 00000000..926cba03 --- /dev/null +++ b/exasol/ds/sandbox/main.py @@ -0,0 +1,7 @@ +#! /usr/bin/env python3 +# +import exasol.ds.sandbox.cli.commands +from exasol.ds.sandbox.cli.cli import cli + +if __name__ == '__main__': + cli() diff --git a/exasol_script_languages_developer_sandbox/runtime/__init__.py b/exasol/ds/sandbox/runtime/__init__.py similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/__init__.py rename to exasol/ds/sandbox/runtime/__init__.py diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/reset_password.yml b/exasol/ds/sandbox/runtime/ansible/reset_password.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/reset_password.yml rename to exasol/ds/sandbox/runtime/ansible/reset_password.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/reset_password_tasks.yml b/exasol/ds/sandbox/runtime/ansible/reset_password_tasks.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/reset_password_tasks.yml rename to exasol/ds/sandbox/runtime/ansible/reset_password_tasks.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/docker/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/docker/defaults/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/docker/defaults/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/docker/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/docker/tasks/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/docker/tasks/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/docker/tasks/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/defaults/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/defaults/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/heading_jupyter_update_password.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/heading_jupyter_update_password.txt similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/heading_jupyter_update_password.txt rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/heading_jupyter_update_password.txt diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/notebook/bash_runner.py b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/bash_runner.py similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/notebook/bash_runner.py rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/bash_runner.py diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/notebook/script-languages.ipynb b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script-languages.ipynb similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/notebook/script-languages.ipynb rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script-languages.ipynb diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/notebook/slc_main_build_steps.svg b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/slc_main_build_steps.svg similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/notebook/slc_main_build_steps.svg rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/slc_main_build_steps.svg diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_dependencies.txt diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/requirements_jupyter.txt diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/install-pip-packages.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/jupyterlab.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/motd.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/motd.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/motd.yml rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/motd.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/systemd.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/systemd.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/systemd.yml rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/systemd.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/tasks/tutorial.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/templates/etc/systemd/system/jupyter.service b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/systemd/system/jupyter.service similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/templates/etc/systemd/system/jupyter.service rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/systemd/system/jupyter.service diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter rename to exasol/ds/sandbox/runtime/ansible/roles/jupyter/templates/etc/update-motd.d/999-jupyter diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/netplan/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/netplan/defaults/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/netplan/defaults/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/netplan/defaults/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/netplan/files/etc/netplan/01-network-manager.yaml b/exasol/ds/sandbox/runtime/ansible/roles/netplan/files/etc/netplan/01-network-manager.yaml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/netplan/files/etc/netplan/01-network-manager.yaml rename to exasol/ds/sandbox/runtime/ansible/roles/netplan/files/etc/netplan/01-network-manager.yaml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/netplan/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/netplan/tasks/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/netplan/tasks/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/netplan/tasks/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/poetry/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/poetry/defaults/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/poetry/defaults/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/poetry/defaults/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/poetry/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/poetry/tasks/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/poetry/tasks/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/poetry/tasks/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/script_languages/defaults/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/script_languages/defaults/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/script_languages/defaults/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/script_languages/defaults/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/roles/script_languages/tasks/main.yml b/exasol/ds/sandbox/runtime/ansible/roles/script_languages/tasks/main.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/roles/script_languages/tasks/main.yml rename to exasol/ds/sandbox/runtime/ansible/roles/script_languages/tasks/main.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/slc_setup.yml b/exasol/ds/sandbox/runtime/ansible/slc_setup.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/slc_setup.yml rename to exasol/ds/sandbox/runtime/ansible/slc_setup.yml diff --git a/exasol_script_languages_developer_sandbox/runtime/ansible/slc_setup_tasks.yml b/exasol/ds/sandbox/runtime/ansible/slc_setup_tasks.yml similarity index 100% rename from exasol_script_languages_developer_sandbox/runtime/ansible/slc_setup_tasks.yml rename to exasol/ds/sandbox/runtime/ansible/slc_setup_tasks.yml diff --git a/exasol_script_languages_developer_sandbox/templates/additional_release_notes.jinja b/exasol/ds/sandbox/templates/additional_release_notes.jinja similarity index 100% rename from exasol_script_languages_developer_sandbox/templates/additional_release_notes.jinja rename to exasol/ds/sandbox/templates/additional_release_notes.jinja diff --git a/exasol_script_languages_developer_sandbox/templates/ci_code_build.jinja.yaml b/exasol/ds/sandbox/templates/ci_code_build.jinja.yaml similarity index 100% rename from exasol_script_languages_developer_sandbox/templates/ci_code_build.jinja.yaml rename to exasol/ds/sandbox/templates/ci_code_build.jinja.yaml diff --git a/exasol_script_languages_developer_sandbox/templates/ec2_cloudformation.jinja.yaml b/exasol/ds/sandbox/templates/ec2_cloudformation.jinja.yaml similarity index 100% rename from exasol_script_languages_developer_sandbox/templates/ec2_cloudformation.jinja.yaml rename to exasol/ds/sandbox/templates/ec2_cloudformation.jinja.yaml diff --git a/exasol_script_languages_developer_sandbox/templates/inventory.jinja b/exasol/ds/sandbox/templates/inventory.jinja similarity index 100% rename from exasol_script_languages_developer_sandbox/templates/inventory.jinja rename to exasol/ds/sandbox/templates/inventory.jinja diff --git a/exasol_script_languages_developer_sandbox/templates/release_code_build.jinja.yaml b/exasol/ds/sandbox/templates/release_code_build.jinja.yaml similarity index 100% rename from exasol_script_languages_developer_sandbox/templates/release_code_build.jinja.yaml rename to exasol/ds/sandbox/templates/release_code_build.jinja.yaml diff --git a/exasol_script_languages_developer_sandbox/templates/vm_bucket_cloudformation.jinja.yaml b/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml similarity index 100% rename from exasol_script_languages_developer_sandbox/templates/vm_bucket_cloudformation.jinja.yaml rename to exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml diff --git a/exasol_script_languages_developer_sandbox/templates/waf_cloudformation.jinja.yaml b/exasol/ds/sandbox/templates/waf_cloudformation.jinja.yaml similarity index 100% rename from exasol_script_languages_developer_sandbox/templates/waf_cloudformation.jinja.yaml rename to exasol/ds/sandbox/templates/waf_cloudformation.jinja.yaml diff --git a/exasol_script_languages_developer_sandbox/cli/commands/export_vm.py b/exasol_script_languages_developer_sandbox/cli/commands/export_vm.py deleted file mode 100644 index 0bf15218..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/export_vm.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Tuple - -import click - -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.id_options import id_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.cli.options.vm_options import vm_options -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.export_vm.run_export_vm import run_export_vm -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -@click.option('--stack-name', required=True, - type=str, - help="Existing cloudformation stack containing the EC2 instance.") -@add_options(vm_options) -@add_options(id_options) -def export_vm( - aws_profile: str, - stack_name: str, - vm_image_format: Tuple[str, ...], - no_vm: bool, - asset_id: str, - log_level: str): - """ - Debug command which creates a new VM image from a running EC2-Instance. - """ - current_vm_image_formats = tuple() if no_vm else vm_image_format - set_log_level(log_level) - run_export_vm(AwsAccess(aws_profile), stack_name, current_vm_image_formats, - AssetId(asset_id), default_config_object) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/install_dependencies.py b/exasol_script_languages_developer_sandbox/cli/commands/install_dependencies.py deleted file mode 100644 index 3b975ca3..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/install_dependencies.py +++ /dev/null @@ -1,23 +0,0 @@ -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.ec2_options import ec2_host_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies - - -@cli.command() -@add_options(logging_options) -@add_options(ec2_host_options) -def install_dependencies( - host_name: str, - ssh_private_key: str, - log_level: str): - """ - Debug command to ansible-installation onto an EC-2 instance. - """ - set_log_level(log_level) - run_install_dependencies(AnsibleAccess(), default_config_object, (HostInfo(host_name, ssh_private_key),)) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/make_ami_public.py b/exasol_script_languages_developer_sandbox/cli/commands/make_ami_public.py deleted file mode 100644 index 331d47cf..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/make_ami_public.py +++ /dev/null @@ -1,24 +0,0 @@ -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.id_options import id_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.export_vm.run_make_ami_public import run_make_ami_public -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -@add_options(id_options) -def make_ami_public( - aws_profile: str, - asset_id: str, - log_level: str): - """ - Debug command which makes an existing AMI public. - """ - set_log_level(log_level) - run_make_ami_public(AwsAccess(aws_profile), AssetId(asset_id)) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/reset_password.py b/exasol_script_languages_developer_sandbox/cli/commands/reset_password.py deleted file mode 100644 index 3ebb2d37..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/reset_password.py +++ /dev/null @@ -1,27 +0,0 @@ -import click - -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.ec2_options import ec2_host_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_reset_password import run_reset_password - - -@cli.command() -@add_options(logging_options) -@add_options(ec2_host_options) -@click.option('--default-password', required=True, type=str, - help="The new (temporary) default password.") -def reset_password( - host_name: str, - ssh_private_key: str, - default_password: str, - log_level: str): - """ - Debug command to reset password on a remote EC-2-instance via ansible. - """ - set_log_level(log_level) - run_reset_password(AnsibleAccess(), default_password, (HostInfo(host_name, ssh_private_key),)) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/setup_ci_codebuild.py b/exasol_script_languages_developer_sandbox/cli/commands/setup_ci_codebuild.py deleted file mode 100644 index 92edc690..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/setup_ci_codebuild.py +++ /dev/null @@ -1,20 +0,0 @@ -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.setup_ci_codebuild.ci_codebuild import run_setup_ci_codebuild -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -def setup_ci_codebuild( - aws_profile: str, - log_level: str): - """ - Command to deploy the CI CodeBuild stack - """ - set_log_level(log_level) - run_setup_ci_codebuild(AwsAccess(aws_profile)) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/setup_ec2.py b/exasol_script_languages_developer_sandbox/cli/commands/setup_ec2.py deleted file mode 100644 index 0e320d7e..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/setup_ec2.py +++ /dev/null @@ -1,31 +0,0 @@ -from typing import Optional - -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.ec2_options import ec2_key_options -from exasol_script_languages_developer_sandbox.cli.options.id_options import id_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_setup_ec2 import run_setup_ec2 - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -@add_options(ec2_key_options) -@add_options(id_options) -def setup_ec2( - aws_profile: str, - ec2_key_file: Optional[str], - ec2_key_name: Optional[str], - asset_id: str, - log_level: str): - """ - Debug command to test setup of an EC-2 instance. - """ - set_log_level(log_level) - run_setup_ec2(AwsAccess(aws_profile), ec2_key_file, ec2_key_name, AssetId(asset_id), default_config_object) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/setup_ec2_and_install_dependencies.py b/exasol_script_languages_developer_sandbox/cli/commands/setup_ec2_and_install_dependencies.py deleted file mode 100644 index 6b9a1204..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/setup_ec2_and_install_dependencies.py +++ /dev/null @@ -1,34 +0,0 @@ -from typing import Optional - -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.ec2_options import ec2_key_options -from exasol_script_languages_developer_sandbox.cli.options.id_options import id_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_setup_ec2_and_install_dependencies import \ - run_setup_ec2_and_install_dependencies - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -@add_options(ec2_key_options) -@add_options(id_options) -def setup_ec2_and_install_dependencies( - aws_profile: str, - ec2_key_file: Optional[str], - ec2_key_name: Optional[str], - asset_id: str, - log_level: str): - """ - Debug command to check setup and installation of an EC-2 instance. - """ - set_log_level(log_level) - run_setup_ec2_and_install_dependencies(AwsAccess(aws_profile), ec2_key_file, ec2_key_name, - AssetId(asset_id), AnsibleAccess(), default_config_object) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/setup_release_codebuild.py b/exasol_script_languages_developer_sandbox/cli/commands/setup_release_codebuild.py deleted file mode 100644 index 3261e229..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/setup_release_codebuild.py +++ /dev/null @@ -1,21 +0,0 @@ -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.setup_release_codebuild.release_codebuild import \ - run_setup_release_codebuild - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -def setup_release_codebuild( - aws_profile: str, - log_level: str): - """ - Command to deploy the Release CodeBuild stack - """ - set_log_level(log_level) - run_setup_release_codebuild(AwsAccess(aws_profile)) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket.py b/exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket.py deleted file mode 100644 index ace569a3..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket.py +++ /dev/null @@ -1,21 +0,0 @@ -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import run_setup_vm_bucket - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -def setup_vm_bucket( - aws_profile: str, - log_level: str): - """ - Command to deploy the VM S3-Bucket - """ - set_log_level(log_level) - run_setup_vm_bucket(AwsAccess(aws_profile), default_config_object) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket_waf.py b/exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket_waf.py deleted file mode 100644 index a280c8d2..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/setup_vm_bucket_waf.py +++ /dev/null @@ -1,26 +0,0 @@ -import click - -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import default_config_object -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket_waf import run_setup_vm_bucket_waf - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -@click.option('--allowed-ip', type=str, - help="The allowed IP address for which CAPTCHA will not be applied.") -def setup_vm_bucket_waf( - aws_profile: str, - allowed_ip: str, - log_level: str): - """ - Command to deploy the VM S3-Bucket Web Application Firewall. Needs to run before deploying the VM Bucket itself. - """ - set_log_level(log_level) - run_setup_vm_bucket_waf(AwsAccess(aws_profile), allowed_ip, default_config_object) diff --git a/exasol_script_languages_developer_sandbox/cli/commands/update_release.py b/exasol_script_languages_developer_sandbox/cli/commands/update_release.py deleted file mode 100644 index a0ad98e1..00000000 --- a/exasol_script_languages_developer_sandbox/cli/commands/update_release.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -from typing import Optional - -import click - -from exasol_script_languages_developer_sandbox.cli.cli import cli -from exasol_script_languages_developer_sandbox.cli.common import add_options -from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options -from exasol_script_languages_developer_sandbox.cli.options.id_options import id_options -from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.logging import set_log_level -from exasol_script_languages_developer_sandbox.lib.github_release_access import GithubReleaseAccess -from exasol_script_languages_developer_sandbox.lib.update_release.run_update_release import run_update_release - - -@cli.command() -@add_options(aws_options) -@add_options(logging_options) -@click.option('--release-id', type=int, required=True, - help="""The Github release id which will be updated.""") -@add_options(id_options) -def update_release( - aws_profile: Optional[str], - release_id: int, - asset_id: str, - log_level: str): - """ - This command attaches the links of the release assets (AMI, VM images) to the Github release, - indicated by parameter 'release-id'. - """ - set_log_level(log_level) - run_update_release(AwsAccess(aws_profile), GithubReleaseAccess(os.getenv("GITHUB_TOKEN")), - release_id, AssetId(asset_id)) diff --git a/exasol_script_languages_developer_sandbox/main.py b/exasol_script_languages_developer_sandbox/main.py deleted file mode 100755 index 2edfaca4..00000000 --- a/exasol_script_languages_developer_sandbox/main.py +++ /dev/null @@ -1,7 +0,0 @@ -#! /usr/bin/env python3 -# -import exasol_script_languages_developer_sandbox.cli.commands -from exasol_script_languages_developer_sandbox.cli.cli import cli - -if __name__ == '__main__': - cli() diff --git a/pyproject.toml b/pyproject.toml index ec3f05bb..0d5f33f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,13 @@ [tool.poetry] -name = "exasol-script-languages-developer-sandbox" +name = "exasol-data-science-sandbox" version = "0.1.0" -description = "Manages script-languages developer virtual machines." +description = "Manages virtual machines for data science sandbox." license = "MIT" - authors = [ - "Thomas Uebensee " + "Torsten Kilias ", + "Mikhail Beck ", + "Christoph Kuhnke " ] [tool.poetry.dependencies] diff --git a/test/aws_local_stack_access.py b/test/aws_local_stack_access.py index 8bc639a8..0a0bf2c8 100644 --- a/test/aws_local_stack_access.py +++ b/test/aws_local_stack_access.py @@ -2,7 +2,7 @@ import boto3 -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess class AwsLocalStackAccess(AwsAccess): diff --git a/test/aws_mock_data.py b/test/aws_mock_data.py index abe8d98a..8f8a99a5 100644 --- a/test/aws_mock_data.py +++ b/test/aws_mock_data.py @@ -3,13 +3,13 @@ from dateutil.tz import tzutc -from exasol_script_languages_developer_sandbox.lib.aws_access.ami import Ami -from exasol_script_languages_developer_sandbox.lib.aws_access.cloudformation_stack import CloudformationStack -from exasol_script_languages_developer_sandbox.lib.aws_access.export_image_task import ExportImageTask -from exasol_script_languages_developer_sandbox.lib.aws_access.key_pair import KeyPair -from exasol_script_languages_developer_sandbox.lib.aws_access.s3_object import S3Object -from exasol_script_languages_developer_sandbox.lib.aws_access.snapshot import Snapshot -from exasol_script_languages_developer_sandbox.lib.aws_access.stack_resource import StackResource +from exasol.ds.sandbox.lib.aws_access.ami import Ami +from exasol.ds.sandbox.lib.aws_access.cloudformation_stack import CloudformationStack +from exasol.ds.sandbox.lib.aws_access.export_image_task import ExportImageTask +from exasol.ds.sandbox.lib.aws_access.key_pair import KeyPair +from exasol.ds.sandbox.lib.aws_access.s3_object import S3Object +from exasol.ds.sandbox.lib.aws_access.snapshot import Snapshot +from exasol.ds.sandbox.lib.aws_access.stack_resource import StackResource from test.conftest import DEFAULT_ASSET_ID TEST_ROLE_ID = 'VM-SLC-Bucket-VMImportRole-TEST' diff --git a/test/conftest.py b/test/conftest.py index 8b28bd92..8f500001 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -5,13 +5,13 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.config import default_config_object, ConfigObject -from exasol_script_languages_developer_sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.config import default_config_object, ConfigObject +from exasol.ds.sandbox.lib.render_template import render_template from importlib.metadata import version -from exasol_script_languages_developer_sandbox.lib.tags import DEFAULT_TAG_KEY +from exasol.ds.sandbox.lib.tags import DEFAULT_TAG_KEY -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.asset_id import AssetId DEFAULT_ASSET_ID = AssetId("test", stack_prefix="test-stack", ami_prefix="test-ami") diff --git a/test/localstack_test.py b/test/localstack_test.py index 1eee3f80..05cd2aca 100644 --- a/test/localstack_test.py +++ b/test/localstack_test.py @@ -1,10 +1,10 @@ import botocore import pytest -from exasol_script_languages_developer_sandbox.lib.setup_ec2.cf_stack import CloudformationStack, \ +from exasol.ds.sandbox.lib.setup_ec2.cf_stack import CloudformationStack, \ CloudformationStackContextManager -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2 -from exasol_script_languages_developer_sandbox.lib.tags import create_default_asset_tag +from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2 +from exasol.ds.sandbox.lib.tags import create_default_asset_tag from test.aws_local_stack_access import AwsLocalStackAccess diff --git a/test/test_ansible.py b/test/test_ansible.py index 237a0bde..ccfe0c7c 100644 --- a/test/test_ansible.py +++ b/test/test_ansible.py @@ -4,12 +4,12 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_repository import default_repositories, \ +from exasol.ds.sandbox.lib.ansible.ansible_repository import default_repositories, \ AnsibleResourceRepository -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext, \ default_ansible_run_context -from exasol_script_languages_developer_sandbox.lib.setup_ec2.host_info import HostInfo -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies +from exasol.ds.sandbox.lib.setup_ec2.host_info import HostInfo +from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies import test.ansible import test.ansible_conflict diff --git a/test/test_aws_codebuild_waiter.py b/test/test_aws_codebuild_waiter.py index f92f173c..7e8446cf 100644 --- a/test/test_aws_codebuild_waiter.py +++ b/test/test_aws_codebuild_waiter.py @@ -4,7 +4,7 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.aws_access.waiter.codebuild_waiter import CodeBuildWaiter +from exasol.ds.sandbox.lib.aws_access.waiter.codebuild_waiter import CodeBuildWaiter from test.mock_cast import mock_cast BUILD_ID = 123 diff --git a/test/test_ci.py b/test/test_ci.py index 98ef0c6a..340ad6fc 100644 --- a/test/test_ci.py +++ b/test/test_ci.py @@ -11,18 +11,18 @@ from invoke import Responder -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.config import default_config_object, SLC_VERSION -from exasol_script_languages_developer_sandbox.lib.run_create_vm import run_create_vm -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.config import default_config_object, SLC_VERSION +from exasol.ds.sandbox.lib.run_create_vm import run_create_vm +from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ EC2StackLifecycleContextManager import string import random -from exasol_script_languages_developer_sandbox.lib.tags import DEFAULT_TAG_KEY +from exasol.ds.sandbox.lib.tags import DEFAULT_TAG_KEY def generate_random_password(length) -> str: diff --git a/test/test_deploy_codebuild.py b/test/test_deploy_codebuild.py index 41c0715f..c7b22e04 100644 --- a/test/test_deploy_codebuild.py +++ b/test/test_deploy_codebuild.py @@ -1,7 +1,7 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.render_template import render_template +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.render_template import render_template from test.cloudformation_validation import validate_using_cfn_lint diff --git a/test/test_deploy_ec2.py b/test/test_deploy_ec2.py index 6bbcbe4c..feda4b7e 100644 --- a/test/test_deploy_ec2.py +++ b/test/test_deploy_ec2.py @@ -1,9 +1,9 @@ from unittest.mock import MagicMock -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.setup_ec2.cf_stack import CloudformationStack, \ +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.setup_ec2.cf_stack import CloudformationStack, \ CloudformationStackContextManager -from exasol_script_languages_developer_sandbox.lib.tags import create_default_asset_tag +from exasol.ds.sandbox.lib.tags import create_default_asset_tag from test.cloudformation_validation import validate_using_cfn_lint diff --git a/test/test_deploy_vm_bucket.py b/test/test_deploy_vm_bucket.py index 43f75f6d..0dccfcd4 100644 --- a/test/test_deploy_vm_bucket.py +++ b/test/test_deploy_vm_bucket.py @@ -3,14 +3,14 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import run_setup_vm_bucket, find_vm_bucket, \ +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import run_setup_vm_bucket, find_vm_bucket, \ create_vm_bucket_cf_template from test.aws_mock_data import TEST_BUCKET_ID, get_waf_cloudformation_mock_data, TEST_ACL_ARN, \ get_s3_cloudformation_mock_data from test.cloudformation_validation import validate_using_cfn_lint from test.mock_cast import mock_cast -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket import STACK_NAME as VM_STACK_NAME +from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import STACK_NAME as VM_STACK_NAME @pytest.fixture diff --git a/test/test_deploy_vm_bucket_waf.py b/test/test_deploy_vm_bucket_waf.py index 62fb163c..03804886 100644 --- a/test/test_deploy_vm_bucket_waf.py +++ b/test/test_deploy_vm_bucket_waf.py @@ -3,8 +3,8 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.vm_bucket.vm_slc_bucket_waf import run_setup_vm_bucket_waf, \ +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket_waf import run_setup_vm_bucket_waf, \ find_acl_arn, get_cloudformation_template from test.aws_mock_data import get_waf_cloudformation_mock_data, TEST_ACL_ARN from test.cloudformation_validation import validate_using_cfn_lint diff --git a/test/test_export_vm.py b/test/test_export_vm.py index d6b71efd..01e00fc1 100644 --- a/test/test_export_vm.py +++ b/test/test_export_vm.py @@ -4,12 +4,12 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.aws_access.ami import Ami -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.export_vm.rename_s3_objects import build_image_source, \ +from exasol.ds.sandbox.lib.aws_access.ami import Ami +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.export_vm.rename_s3_objects import build_image_source, \ build_image_destination -from exasol_script_languages_developer_sandbox.lib.export_vm.run_export_vm import export_vm -from exasol_script_languages_developer_sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat +from exasol.ds.sandbox.lib.export_vm.run_export_vm import export_vm +from exasol.ds.sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat from test.aws_mock_data import get_ami_image_mock_data, TEST_AMI_ID, TEST_ROLE_ID, TEST_BUCKET_ID, INSTANCE_ID, \ get_export_image_task_mock_data, get_s3_cloudformation_mock_data, get_waf_cloudformation_mock_data from test.mock_cast import mock_cast diff --git a/test/test_install_dependencies.py b/test/test_install_dependencies.py index d680760c..dbd7fff6 100644 --- a/test/test_install_dependencies.py +++ b/test/test_install_dependencies.py @@ -8,11 +8,11 @@ import pytest from docker.types import Mount -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_access import AnsibleAccess -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_repository import AnsibleResourceRepository, \ +from exasol.ds.sandbox.lib.ansible.ansible_access import AnsibleAccess +from exasol.ds.sandbox.lib.ansible.ansible_repository import AnsibleResourceRepository, \ default_repositories -from exasol_script_languages_developer_sandbox.lib.ansible.ansible_run_context import AnsibleRunContext -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies +from exasol.ds.sandbox.lib.ansible.ansible_run_context import AnsibleRunContext +from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies import test.ansible TEST_CONTAINER_NAME = "ansible-test" diff --git a/test/test_key_file_manager.py b/test/test_key_file_manager.py index e3428b26..191fdf59 100644 --- a/test/test_key_file_manager.py +++ b/test/test_key_file_manager.py @@ -1,7 +1,7 @@ import os from unittest.mock import MagicMock -from exasol_script_languages_developer_sandbox.lib.setup_ec2.key_file_manager import KeyFileManager, KeyFileManagerContextManager +from exasol.ds.sandbox.lib.setup_ec2.key_file_manager import KeyFileManager, KeyFileManagerContextManager def test_external_keys(tmp_path, default_asset_id): diff --git a/test/test_make_ami_public.py b/test/test_make_ami_public.py index 77e1d784..e5e1c359 100644 --- a/test/test_make_ami_public.py +++ b/test/test_make_ami_public.py @@ -4,9 +4,9 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.aws_access.ami import Ami -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.export_vm.run_make_ami_public import run_make_ami_public +from exasol.ds.sandbox.lib.aws_access.ami import Ami +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.export_vm.run_make_ami_public import run_make_ami_public from test.aws_mock_data import get_ami_image_mock_data from test.mock_cast import mock_cast diff --git a/test/test_motd_jupyter_template.py b/test/test_motd_jupyter_template.py index a55f48e1..586495c8 100644 --- a/test/test_motd_jupyter_template.py +++ b/test/test_motd_jupyter_template.py @@ -14,7 +14,7 @@ def motd_file(tmp_path): jupyter_server_config_file = tmp_path / "jupyter_server_config.json" python_file = tmp_path / "999_jupyter.py" - src_path = Path(__file__).parent.parent / "exasol_script_languages_developer_sandbox" / "runtime" / \ + src_path = Path(__file__).parent.parent / "exasol.ds.sandbox" / "runtime" / \ "ansible" / "roles" / "jupyter" / "templates" / "etc" /"update-motd.d" / "999-jupyter" with open(src_path, "r") as f: python_code_template = f.read() diff --git a/test/test_printing.py b/test/test_printing.py index 0c133ce8..c0958eca 100644 --- a/test/test_printing.py +++ b/test/test_printing.py @@ -3,9 +3,9 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.asset_printing.print_assets import print_with_printer, AssetTypes -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.asset_printing.print_assets import print_with_printer, AssetTypes +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from test.aws_mock_data import get_ami_image_mock_data, TEST_AMI_ID, get_snapshot_mock_data, \ get_export_image_task_mock_data, get_s3_object_mock_data, TEST_BUCKET_ID, \ get_ec2_cloudformation_mock_data, get_ec2_cloudformation_stack_resources_mock_data, get_ec2_key_pair_mock_data, \ diff --git a/test/test_printing_factories.py b/test/test_printing_factories.py index 28809ab6..538bb2b7 100644 --- a/test/test_printing_factories.py +++ b/test/test_printing_factories.py @@ -2,9 +2,9 @@ from contextlib import redirect_stdout from io import StringIO -import exasol_script_languages_developer_sandbox.lib.asset_printing.rich_console_printer -from exasol_script_languages_developer_sandbox.lib.asset_printing.mark_down_printer import MarkdownPrintingFactory -from exasol_script_languages_developer_sandbox.lib.asset_printing.printing_factory import TextObject +import exasol.ds.sandbox.lib.asset_printing.rich_console_printer +from exasol.ds.sandbox.lib.asset_printing.mark_down_printer import MarkdownPrintingFactory +from exasol.ds.sandbox.lib.asset_printing.printing_factory import TextObject from inspect import cleandoc @@ -35,7 +35,7 @@ def test_printing_rich(): with StringIO() as buf, redirect_stdout(buf): # Need to reload the module here as it already might have been loaded by pytest for another test, # in which case sys.stdout was already loaded too - m = importlib.reload(exasol_script_languages_developer_sandbox.lib.asset_printing.rich_console_printer) + m = importlib.reload(exasol.ds.sandbox.lib.asset_printing.rich_console_printer) factory = m.RichConsolePrintingFactory() table_printer = factory.create_table_printer("test-table") @@ -57,4 +57,4 @@ def test_printing_rich(): └───────┴───────┘ this text will be printed""") - assert output == expected_output \ No newline at end of file + assert output == expected_output diff --git a/test/test_release_build.py b/test/test_release_build.py index 7b5e3d85..98f6b1bd 100644 --- a/test/test_release_build.py +++ b/test/test_release_build.py @@ -4,11 +4,11 @@ from dateutil.tz import tzutc -from exasol_script_languages_developer_sandbox.lib.aws_access.aws_access import AwsAccess -from exasol_script_languages_developer_sandbox.lib.aws_access.stack_resource import StackResource -from exasol_script_languages_developer_sandbox.lib.aws_access.waiter.codebuild_waiter import CodeBuildWaiter -from exasol_script_languages_developer_sandbox.lib.github_release_access import GithubReleaseAccess -from exasol_script_languages_developer_sandbox.lib.release_build.run_release_build import run_start_release_build, \ +from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess +from exasol.ds.sandbox.lib.aws_access.stack_resource import StackResource +from exasol.ds.sandbox.lib.aws_access.waiter.codebuild_waiter import CodeBuildWaiter +from exasol.ds.sandbox.lib.github_release_access import GithubReleaseAccess +from exasol.ds.sandbox.lib.release_build.run_release_build import run_start_release_build, \ run_start_test_release_build from test.mock_cast import mock_cast diff --git a/test/test_run_lifecycle_for_ec2.py b/test/test_run_lifecycle_for_ec2.py index b1ad97a2..46d79bef 100644 --- a/test/test_run_lifecycle_for_ec2.py +++ b/test/test_run_lifecycle_for_ec2.py @@ -2,9 +2,9 @@ import pytest -from exasol_script_languages_developer_sandbox.lib.aws_access.ec2_instance import EC2Instance -from exasol_script_languages_developer_sandbox.lib.aws_access.stack_resource import StackResource -from exasol_script_languages_developer_sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ +from exasol.ds.sandbox.lib.aws_access.ec2_instance import EC2Instance +from exasol.ds.sandbox.lib.aws_access.stack_resource import StackResource +from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2, \ EC2StackLifecycleContextManager diff --git a/test/test_serialization.py b/test/test_serialization.py index 37c05480..905e65c4 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -4,9 +4,9 @@ import traceback from pathlib import Path -from exasol_script_languages_developer_sandbox.lib.asset_id import AssetId -from exasol_script_languages_developer_sandbox.lib.setup_ec2.cf_stack import CloudformationStack -from exasol_script_languages_developer_sandbox.lib.setup_ec2.key_file_manager import KeyFileManager +from exasol.ds.sandbox.lib.asset_id import AssetId +from exasol.ds.sandbox.lib.setup_ec2.cf_stack import CloudformationStack +from exasol.ds.sandbox.lib.setup_ec2.key_file_manager import KeyFileManager from test.aws_local_stack_access import AwsLocalStackAccess import multiprocessing as mp diff --git a/test/test_source_ami.py b/test/test_source_ami.py index cfd1fb51..06bad529 100644 --- a/test/test_source_ami.py +++ b/test/test_source_ami.py @@ -1,7 +1,7 @@ from unittest.mock import MagicMock -from exasol_script_languages_developer_sandbox.lib.aws_access.ami import Ami -from exasol_script_languages_developer_sandbox.lib.setup_ec2.source_ami import find_source_ami +from exasol.ds.sandbox.lib.aws_access.ami import Ami +from exasol.ds.sandbox.lib.setup_ec2.source_ami import find_source_ami # The following is just a dump of data returned from aws cli: # 'aws ec2 --profile exa_individual_mfa describe-images --filters Name=name,Values=ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* --owners 099720109477' From bd808a9ad3239208c8afe684bf09cd455e323a63 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 18 Oct 2023 11:29:40 +0200 Subject: [PATCH 05/21] Fixed typo in changes file --- doc/changes/changes_0.1.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changes/changes_0.1.0.md b/doc/changes/changes_0.1.0.md index dff00a2c..8e35ee23 100644 --- a/doc/changes/changes_0.1.0.md +++ b/doc/changes/changes_0.1.0.md @@ -20,4 +20,4 @@ Version: 0.1.0 ## Refactoring - - #5: Renamed all occurrences of "script language developer" by "dat science" + - #5: Renamed all occurrences of "script language developer" by "data science" From 929ea17adef169e7e9162085b90947c6d23eba79 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 18 Oct 2023 12:21:32 +0200 Subject: [PATCH 06/21] Added package instruction to pyproject.toml and updated README.md --- README.md | 7 ++++--- pyproject.toml | 20 ++++++++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 71800e0e..e308515a 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ +# Exasol Data Science Sandbox -# Overview +## Overview This project provides an automated mechanism to build and export virtual machines images (AWS AMI, VMDK,...) -which can be used to develop new script-languages container for the Exasol DB. +enabling users to try out data science algorithms in Jupyter notebooks connected to the the Exasol database. ## Where to find the VM images -The release process will automatically store the links to the images in the [release notes](https://github.com/exasol/script-languages-developer-sandbox/releases/latest), as there will be a specific AMI per release. +The release process will automatically store the links to the images in the [release notes](https://github.com/exasol/data-science-sandbox/releases/latest), as there will be a specific AMI per release. Please check the user guide about details of the image. ## Links diff --git a/pyproject.toml b/pyproject.toml index 0d5f33f8..63e44267 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,30 @@ [tool.poetry] name = "exasol-data-science-sandbox" version = "0.1.0" -description = "Manages virtual machines for data science sandbox." - +description = "Manages virtual machines for the data science sandbox." +packages = [ {include = "exasol"}, ] license = "MIT" authors = [ "Torsten Kilias ", "Mikhail Beck ", "Christoph Kuhnke " ] +classifiers = [ + "Programming Language :: Python :: 3", + "Development Status :: 3 - Alpha", + "Framework :: Jupyter :: JupyterLab :: 3", + "License :: OSI Approved :: MIT License", +] +readme = "README.md" +license = "MIT" + + +[tool.poetry.urls] +"Homepage" = "https://github.com/exasol/data-science-sandbox" +"Documentation" = "https://github.io/exasol/data-science-sandbox" +"Source" = "https://github.io/exasol/data-science-sandbox" +"Issues" = "https://github.io/exasol/data-science-sandbox/issues" + [tool.poetry.dependencies] python = ">=3.8.0,<4.0" From 0014e94c39dd56bed4ec3e7003fdee58bab58288 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 18 Oct 2023 12:26:07 +0200 Subject: [PATCH 07/21] fixed pyproject.toml --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 63e44267..625e1cd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,6 @@ classifiers = [ "License :: OSI Approved :: MIT License", ] readme = "README.md" -license = "MIT" [tool.poetry.urls] From 233d3fd252c552a1d0a04e5b105acd14a6344649 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 18 Oct 2023 15:55:23 +0200 Subject: [PATCH 08/21] Applied additional renamings in additional files that have been overseen initially. --- aws-code-build/ci/buildspec.yaml | 4 ++-- aws-code-build/ci/buildspec_release.yaml | 6 +++--- exasol/ds/sandbox/lib/asset_printing/print_assets.py | 6 +++--- exasol/ds/sandbox/lib/export_vm/rename_s3_objects.py | 2 +- exasol/ds/sandbox/templates/ci_code_build.jinja.yaml | 4 ++-- exasol/ds/sandbox/templates/release_code_build.jinja.yaml | 6 +++--- .../sandbox/templates/vm_bucket_cloudformation.jinja.yaml | 4 ++-- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/aws-code-build/ci/buildspec.yaml b/aws-code-build/ci/buildspec.yaml index 6669259a..c2836fee 100644 --- a/aws-code-build/ci/buildspec.yaml +++ b/aws-code-build/ci/buildspec.yaml @@ -3,7 +3,7 @@ version: 0.2 env: shell: bash variables: - RUN_DEVELOPER_SANDBOX_CI_TEST: "true" + DSS_RUN_CI_TEST: "true" AWS_USER_NAME: "ci_user" phases: @@ -20,7 +20,7 @@ phases: pre_build: commands: - - echo RUN_DEVELOPER_SANDBOX_CI_TEST is "$RUN_DEVELOPER_SANDBOX_CI_TEST" #supposed to be true + - echo DSS_RUN_CI_TEST is "$DSS_RUN_CI_TEST" #supposed to be true build: commands: - poetry run python3 -m pytest -s test/test_ci*.py diff --git a/aws-code-build/ci/buildspec_release.yaml b/aws-code-build/ci/buildspec_release.yaml index 58ede603..20d07264 100644 --- a/aws-code-build/ci/buildspec_release.yaml +++ b/aws-code-build/ci/buildspec_release.yaml @@ -3,7 +3,7 @@ version: 0.2 env: shell: bash variables: - DEFAULT_PASSWORD: "scriptlanguages" + DEFAULT_PASSWORD: "dss" ASSET_ID: "" AWS_USER_NAME: "release_user" MAKE_AMI_PUBLIC_OPTION: "--no-make-ami-public" @@ -27,5 +27,5 @@ phases: - echo MAKE_AMI_PUBLIC_OPTION is "$MAKE_AMI_PUBLIC_OPTION" build: commands: - - poetry run python3 -m exasol_script_languages_developer_sandbox.main create-vm --default-password "$DEFAULT_PASSWORD" --asset-id "$ASSET_ID" $MAKE_AMI_PUBLIC_OPTION - - poetry run python3 -m exasol_script_languages_developer_sandbox.main update-release --release-id "$RELEASE_ID" --asset-id "$ASSET_ID" + - poetry run python3 -m exasol.ds.sandbox.main create-vm --default-password "$DEFAULT_PASSWORD" --asset-id "$ASSET_ID" $MAKE_AMI_PUBLIC_OPTION + - poetry run python3 -m exasol.ds.sandbox.main update-release --release-id "$RELEASE_ID" --asset-id "$ASSET_ID" diff --git a/exasol/ds/sandbox/lib/asset_printing/print_assets.py b/exasol/ds/sandbox/lib/asset_printing/print_assets.py index c1f6b8f1..e3fd72e0 100644 --- a/exasol/ds/sandbox/lib/asset_printing/print_assets.py +++ b/exasol/ds/sandbox/lib/asset_printing/print_assets.py @@ -144,15 +144,15 @@ def print_s3_objects(aws_access: AwsAccess, asset_id: Optional[AssetId], printin # How the filtering works: # 1. The VM are stored under following location in the S3 Bucket: $BUCKET_PREFIX/$AssetId/name.$VM_FORMAT - # For example "slc_developer_sandbox/5.0.0/export-ami-01be860e6a6a98bf8.vhd" + # For example "data_science_sandbox/5.0.0/export-ami-01be860e6a6a98bf8.vhd" # 2. Because S3 list_s3_object does not support wildcards, # we need to implement our own wildcard implementation here. - # We call list_s3_object with the standard prefix (e.g. "slc_developer_sandbox"), + # We call list_s3_object with the standard prefix (e.g. "data_science_sandbox"), # which returns ALL stored vm objects. # 3. If no filter is given (asset_id == None), "prefix" will be empty, and we return all s3 objects # 4. If the variable "prefix" is not empty, we need to ensure that it ends with a wildcard, so that the matching # works correctly. - # => Assume that a filter is given "5.0.0". Variable prefix would be "slc_developer_sandbox/5.0.0". + # => Assume that a filter is given "5.0.0". Variable prefix would be "data_science_sandbox/5.0.0". s3_objects = aws_access.list_s3_objects(bucket=vm_bucket, prefix=AssetId.BUCKET_PREFIX) diff --git a/exasol/ds/sandbox/lib/export_vm/rename_s3_objects.py b/exasol/ds/sandbox/lib/export_vm/rename_s3_objects.py index d0e35d8d..9acb2c95 100644 --- a/exasol/ds/sandbox/lib/export_vm/rename_s3_objects.py +++ b/exasol/ds/sandbox/lib/export_vm/rename_s3_objects.py @@ -14,7 +14,7 @@ def build_image_source(prefix: str, export_image_task_id: str, vm_image_format: def build_image_destination(prefix: str, asset_id: AssetId, vm_image_format: VmDiskImageFormat) -> str: img_format = vm_image_format.value.lower() - return "{bucket_prefix}exasol-data-science-developer-sandbox-{asset_id}.{img_format}".format( + return "{bucket_prefix}exasol-data-science-sandbox-{asset_id}.{img_format}".format( bucket_prefix=prefix, asset_id=str(asset_id), img_format=img_format) diff --git a/exasol/ds/sandbox/templates/ci_code_build.jinja.yaml b/exasol/ds/sandbox/templates/ci_code_build.jinja.yaml index 308cdc50..52e9971e 100644 --- a/exasol/ds/sandbox/templates/ci_code_build.jinja.yaml +++ b/exasol/ds/sandbox/templates/ci_code_build.jinja.yaml @@ -69,7 +69,7 @@ Resources: DeveloperSandboxCICodeBuild: Type: AWS::CodeBuild::Project Properties: - Description: CI Build which runs CI test if script-languages-developer-sandbox + Description: CI Build which runs CI test of data-science-sandbox Triggers: Webhook: True FilterGroups: @@ -89,7 +89,7 @@ Resources: Image: aws/codebuild/standard:5.0 Source: Type: GITHUB - Location: "https://github.com/exasol/script-languages-developer-sandbox" + Location: "https://github.com/exasol/data-science-sandbox" BuildSpec: "aws-code-build/ci/buildspec.yaml" TimeoutInMinutes: 240 diff --git a/exasol/ds/sandbox/templates/release_code_build.jinja.yaml b/exasol/ds/sandbox/templates/release_code_build.jinja.yaml index 59e58180..db84a30c 100644 --- a/exasol/ds/sandbox/templates/release_code_build.jinja.yaml +++ b/exasol/ds/sandbox/templates/release_code_build.jinja.yaml @@ -70,12 +70,12 @@ Resources: - s3:PutObject - s3:GetObject - s3:DeleteObject - Resource: arn:aws:s3:::{{vm_bucket}}/slc_developer_sandbox/* + Resource: arn:aws:s3:::{{vm_bucket}}/data_science_sandbox/* Effect: Allow DeveloperSandboxReleaseCodeBuild: Type: AWS::CodeBuild::Project Properties: - Description: Release Build which creates a new AMI and VM iso images on script-languages-developer-sandbox + Description: Release Build which creates a new AMI and VM iso images on data-science-sandbox ServiceRole: !GetAtt CodeBuildRole.Arn Artifacts: Type: NO_ARTIFACTS @@ -85,7 +85,7 @@ Resources: Image: aws/codebuild/standard:5.0 Source: Type: GITHUB - Location: "https://github.com/exasol/script-languages-developer-sandbox" + Location: "https://github.com/exasol/data-science-sandbox" BuildSpec: "aws-code-build/ci/buildspec_release.yaml" TimeoutInMinutes: 240 diff --git a/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml b/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml index 1775a6ef..0fca111d 100644 --- a/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml +++ b/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml @@ -57,7 +57,7 @@ Resources: Principal: Service: "cloudfront.amazonaws.com" Resource: - - !Sub "arn:aws:s3:::${ VMBucket }/slc_developer_sandbox/*" + - !Sub "arn:aws:s3:::${ VMBucket }/data_science_sandbox/*" Condition: StringEquals: 'AWS:SourceArn': !Sub "arn:aws:cloudfront::${AWS::AccountId}:distribution/${CfDistribution}" @@ -128,4 +128,4 @@ Outputs: Value: !Ref CfDistribution {{CfDistributionDomainName}}: Description: 'Domain name for our cloudfront distribution' - Value: !GetAtt CfDistribution.DomainName \ No newline at end of file + Value: !GetAtt CfDistribution.DomainName From 4290ce0d707dc65d37c8b58e70c7ad066c44d626 Mon Sep 17 00:00:00 2001 From: ckunki Date: Wed, 18 Oct 2023 16:38:19 +0200 Subject: [PATCH 09/21] Applied renamings in documentation I hope that should have been the last remainders. --- doc/changes/changes_0.1.0.md | 2 +- doc/developer_guide/developer_guide.md | 6 ++--- doc/user_guide/user_guide.md | 36 +++++++++++++------------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/doc/changes/changes_0.1.0.md b/doc/changes/changes_0.1.0.md index 8e35ee23..9ffe8876 100644 --- a/doc/changes/changes_0.1.0.md +++ b/doc/changes/changes_0.1.0.md @@ -1,4 +1,4 @@ -# script-languages-developer-sandbox 0.1.0, released t.b.d. +# data-science-sandbox 0.1.0, released t.b.d. Code name: Initial release diff --git a/doc/developer_guide/developer_guide.md b/doc/developer_guide/developer_guide.md index c6caa75b..5e6afbff 100644 --- a/doc/developer_guide/developer_guide.md +++ b/doc/developer_guide/developer_guide.md @@ -1,4 +1,4 @@ -# Script-Languages-Developer-Sandbox Developer Guide +# Data Science Sandbox Developer Guide ## Overview @@ -25,7 +25,7 @@ bash install.sh ## Design Goals -script-languages-developer-sandbox uses AWS as backend, because it provides the possibility to run the whole workflow during a ci-test. +The Data Science Sandbox (DSS) uses AWS as backend, because it provides the possibility to run the whole workflow during a ci-test. This project uses - `boto3` to interact with AWS @@ -119,7 +119,7 @@ The bucket has private access. In order to control access, the Bucket cloudforma The following diagram shows the involved cloudformation stacks: ![image info](./img/cloudformation-stacks.drawio.png) -"DEVELOPER-SANDBOX-VM-SLC-BUCKET", "DEVELOPER-SANDBOX-CI-TEST-CODEBUILD" & "DEVELOPER-SANDBOX-RELEASE-CODEBUILD" are permanent and need to be deployed using the "deploy" commands (see [commands](#deployment-commands)). +"DATA-SCIENCE-SANDBOX-VM-SLC-Bucket", "DATA-SCIENCE-SANDBOX-CI-TEST-CODEBUILD" & "DATA-SCIENCE-SANDBOX-RELEASE-CODEBUILD" are permanent and need to be deployed using the "deploy" commands (see [commands](#deployment-commands)). The EC2-stack lives only during the creation of a new developer sandbox image. ## Tagging diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index 28c10814..af9a3386 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -1,4 +1,4 @@ -# Script-Languages-Developer-Sandbox User Guide +# Data Science Sandbox User Guide ## Overview @@ -18,8 +18,8 @@ The disk of the VM is per default large enough to build and test script-language ### AMI -The AMI id is linked in the [release notes](https://github.com/exasol/script-languages-developer-sandbox/releases/latest) and can be used to start an EC2-instance in your AWS account. -Following name format is used for the AMI: "_Exasol-SLC-Developer-Sandbox-**version**_", e.g. "_Exasol-SLC-Developer-Sandbox-5.0.0_" +The AMI id is linked in the [release notes](https://github.com/exasol/data-science-sandbox/releases/latest) and can be used to start an EC2-instance in your AWS account. +Following name format is used for the AMI: "_Exasol-Data-Science-Sandbox-**version**_", e.g. "_Exasol-Data-Science-Sandbox-5.0.0_" ### VM image @@ -30,7 +30,7 @@ Currently two VM formats are supported: | VMDK | VMware Virtual Machine Disk | | VHD | Virtual Hard Disk by Microsoft | -The links to the images are stored in the release notes. +The links to the images are stored in the release notes. ## Usage @@ -61,15 +61,15 @@ __Important__: The AMI is currently only available in the AWS region `eu-central - Click button "Create Security Group" - Choose name and VPC - For outbound rules keep the default - - Create inbound rules: + - Create inbound rules: - One of type `ssh` - - If you plan to connect to the Jupyter lab, add another rule of type "Custom TCP" with port 8888. + - If you plan to connect to the Jupyter lab, add another rule of type "Custom TCP" with port 8888. - **Important**: With this rule you expose the Jupyter lab to the internet; anybody who has access to the password will be able to execute commands. For a minimum of security you should change the default Jupyter password. Details about how to do that will be shown in the login screen. However, you should consider to use only `ssh` with port forwarding. 4. Go back to the EC2 console 5. Launch the EC2 instance: - In the navigation bar on the left select "Instances" - Click button "Launch instances" - - At field "Application and OS Images" select the AMI id of the developer sandbox (found in the [release notes](https://github.com/exasol/script-languages-developer-sandbox/releases/latest)) + - At field "Application and OS Images" select the AMI id of the developer sandbox (found in the [release notes](https://github.com/exasol/data-science-sandbox/releases/latest)) - Select an appropriate instance type (at least "t2.small" or similar) - Choose your key pair - Choose the security group which your created in step 3. @@ -90,7 +90,7 @@ __Important__: The AMI is currently only available in the AWS region `eu-central ##### Step-by-step -1. Download the `VMDK` file from the [release notes](https://github.com/exasol/script-languages-developer-sandbox/releases/latest). +1. Download the `VMDK` file from the [release notes](https://github.com/exasol/data-science-sandbox/releases/latest). 2. Open Boxes 3. Create a new VM: Click the + Button 4. Choose: "Create virtual machine from file" @@ -106,13 +106,13 @@ __Important__: The AMI is currently only available in the AWS region `eu-central 10. If you want to connect to the Jupyterlab, you need to find out the IP address of the VM: - Click the 3 dots in Boxes, then preferences![image info](./img/tutorial-screenshot-open-preferences.png) - Then the IP address will be shown: ![image info](./img/tutorial-screenshot-show-ip.png) -- Use _IP-Address_:8888 to connect to the [Jupterlab](#Jupyter) +- Use _IP-Address_:8888 to connect to the [Jupterlab](#Jupyter) ### Login Username: **ubuntu** -At the first login to the sandbox (image or AMI) you will be prompted to change your password. +At the first login to the sandbox (image or AMI) you will be prompted to change your password. The default password is: **scriptlanguages** However, we suggest to use ssh-keys for the connection. When you use the AWS AMI, this will work automatically. When you use the VM images, you need to deploy your ssh-keys. After you enabled ssh-keys, we recommend to disable ssh password authentication: @@ -127,10 +127,10 @@ Also, we strongly recommend to change the Jupyter password as soon as possible. ### script-languages-release -**Location**: `/home/ubuntu/script-languages-release` -**Source**: [Github repo](https://github.com/exasol/script-languages-release) -The images are tightly coupled to the releases of script-languages-release; for each release of the script-languages-release there will be a release of the developer sandbox. -By default, the images contain a cloned repository of script-languages-container, including all dependencies to run it (Python, Poetry, Docker, etc.). The checked out version will be the tag of the respective release, e.g. a Developer Sandbox image for release 5.0.0 will have the tag 5.0.0 checked out for script-languages-release. +**Location**: `/home/ubuntu/script-languages-release` +**Source**: [Github repo](https://github.com/exasol/script-languages-release) +The images are tightly coupled to the releases of script-languages-release; for each release of the script-languages-release there will be a release of the data-science sandbox. +By default, the images contain a cloned repository of script-languages-container, including all dependencies to run it (Python, Poetry, Docker, etc.). The checked out version will be the tag of the respective release, e.g. a Sandbox image for release 5.0.0 will have the tag 5.0.0 checked out for script-languages-release. If you aim to customize existing containers, this should be fine. However, if you want to rebuild a container, you might encounter problems as dependant packages might have changed. In that case, we suggest to check out the master branch ```shell git checkout --recurse-submodules master @@ -142,10 +142,10 @@ For information about how to build script-languages-container please check: ### Jupyter -**Location virtual environment**: `/home/ubuntu/jupyterenv` -**Location notebooks**: `/home/ubuntu/notebooks` -**Password**: `script-languages` -**Http Port**: 8888 +**Location virtual environment**: `/home/ubuntu/jupyterenv` +**Location notebooks**: `/home/ubuntu/notebooks` +**Password**: `script-languages` +**Http Port**: 8888 There is a pre-loaded tutorial notebook available which guides you through the usage of `exaslct`: __script-languages.ipynb__. From fc58905780dadcd97e1f8b8bea2e076cc68600a7 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 19 Oct 2023 09:23:41 +0200 Subject: [PATCH 10/21] Fixed review findings of @ahsimb --- README.md | 2 +- doc/developer_guide/developer_guide.md | 2 +- doc/user_guide/user_guide.md | 2 +- exasol/ds/sandbox/cli/commands/start_release_build.py | 2 +- exasol/ds/sandbox/cli/commands/start_test_release_build.py | 2 +- exasol/ds/sandbox/templates/ci_code_build.jinja.yaml | 6 +++--- exasol/ds/sandbox/templates/release_code_build.jinja.yaml | 6 +++--- .../sandbox/templates/vm_bucket_cloudformation.jinja.yaml | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e308515a..94edc287 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ## Overview This project provides an automated mechanism to build and export virtual machines images (AWS AMI, VMDK,...) -enabling users to try out data science algorithms in Jupyter notebooks connected to the the Exasol database. +enabling users to try out data science algorithms in Jupyter notebooks connected to the Exasol database. ## Where to find the VM images diff --git a/doc/developer_guide/developer_guide.md b/doc/developer_guide/developer_guide.md index 5e6afbff..03c626cb 100644 --- a/doc/developer_guide/developer_guide.md +++ b/doc/developer_guide/developer_guide.md @@ -120,7 +120,7 @@ The following diagram shows the involved cloudformation stacks: ![image info](./img/cloudformation-stacks.drawio.png) "DATA-SCIENCE-SANDBOX-VM-SLC-Bucket", "DATA-SCIENCE-SANDBOX-CI-TEST-CODEBUILD" & "DATA-SCIENCE-SANDBOX-RELEASE-CODEBUILD" are permanent and need to be deployed using the "deploy" commands (see [commands](#deployment-commands)). -The EC2-stack lives only during the creation of a new developer sandbox image. +The EC2-stack lives only during the creation of a new sandbox image. ## Tagging diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index af9a3386..e988f6f9 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -69,7 +69,7 @@ __Important__: The AMI is currently only available in the AWS region `eu-central 5. Launch the EC2 instance: - In the navigation bar on the left select "Instances" - Click button "Launch instances" - - At field "Application and OS Images" select the AMI id of the developer sandbox (found in the [release notes](https://github.com/exasol/data-science-sandbox/releases/latest)) + - At field "Application and OS Images" select the AMI id of the sandbox (found in the [release notes](https://github.com/exasol/data-science-sandbox/releases/latest)) - Select an appropriate instance type (at least "t2.small" or similar) - Choose your key pair - Choose the security group which your created in step 3. diff --git a/exasol/ds/sandbox/cli/commands/start_release_build.py b/exasol/ds/sandbox/cli/commands/start_release_build.py index eedc72de..9c1d80ee 100644 --- a/exasol/ds/sandbox/cli/commands/start_release_build.py +++ b/exasol/ds/sandbox/cli/commands/start_release_build.py @@ -26,7 +26,7 @@ def start_release_build( upload_url: str, branch: str): """ - This command triggers the AWS release Codebuild to generate a new developer sandbox version. + This command triggers the AWS release Codebuild to generate a new sandbox version. """ set_log_level(log_level) run_start_release_build(AwsAccess(aws_profile), default_config_object, diff --git a/exasol/ds/sandbox/cli/commands/start_test_release_build.py b/exasol/ds/sandbox/cli/commands/start_test_release_build.py index b4991c95..850dc8c3 100644 --- a/exasol/ds/sandbox/cli/commands/start_test_release_build.py +++ b/exasol/ds/sandbox/cli/commands/start_test_release_build.py @@ -27,7 +27,7 @@ def start_test_release_build( release_title: str ): """ - This command triggers the AWS release Codebuild to generate a new developer sandbox test version. + This command triggers the AWS release Codebuild to generate a new sandbox test version. """ set_log_level(log_level) gh_token = os.getenv("GITHUB_TOKEN") diff --git a/exasol/ds/sandbox/templates/ci_code_build.jinja.yaml b/exasol/ds/sandbox/templates/ci_code_build.jinja.yaml index 52e9971e..41299154 100644 --- a/exasol/ds/sandbox/templates/ci_code_build.jinja.yaml +++ b/exasol/ds/sandbox/templates/ci_code_build.jinja.yaml @@ -66,7 +66,7 @@ Resources: - s3:ListBucket Resource: arn:aws:s3:::{{vm_bucket}} Effect: Allow - DeveloperSandboxCICodeBuild: + DataScienceSandboxCICodeBuild: Type: AWS::CodeBuild::Project Properties: Description: CI Build which runs CI test of data-science-sandbox @@ -97,10 +97,10 @@ Resources: CodeBuildLogGroup: Type: AWS::Logs::LogGroup Properties: - LogGroupName: !Sub "/aws/codebuild/${ DeveloperSandboxCICodeBuild }" + LogGroupName: !Sub "/aws/codebuild/${ DataScienceSandboxCICodeBuild }" RetentionInDays: 14 Outputs: ProjectName: - Value: !Ref DeveloperSandboxCICodeBuild + Value: !Ref DataScienceSandboxCICodeBuild Description: Developer Sandbox CI Build project name diff --git a/exasol/ds/sandbox/templates/release_code_build.jinja.yaml b/exasol/ds/sandbox/templates/release_code_build.jinja.yaml index db84a30c..47f504bd 100644 --- a/exasol/ds/sandbox/templates/release_code_build.jinja.yaml +++ b/exasol/ds/sandbox/templates/release_code_build.jinja.yaml @@ -72,7 +72,7 @@ Resources: - s3:DeleteObject Resource: arn:aws:s3:::{{vm_bucket}}/data_science_sandbox/* Effect: Allow - DeveloperSandboxReleaseCodeBuild: + DataScienceSandboxReleaseCodeBuild: Type: AWS::CodeBuild::Project Properties: Description: Release Build which creates a new AMI and VM iso images on data-science-sandbox @@ -93,10 +93,10 @@ Resources: CodeBuildLogGroup: Type: AWS::Logs::LogGroup Properties: - LogGroupName: !Sub "/aws/codebuild/${ DeveloperSandboxReleaseCodeBuild }" + LogGroupName: !Sub "/aws/codebuild/${ DataScienceSandboxReleaseCodeBuild }" RetentionInDays: 14 Outputs: ProjectName: - Value: !Ref DeveloperSandboxReleaseCodeBuild + Value: !Ref DataScienceSandboxReleaseCodeBuild Description: Developer Sandbox Release Build project name diff --git a/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml b/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml index 0fca111d..41257e07 100644 --- a/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml +++ b/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml @@ -4,7 +4,7 @@ Resources: Properties: AccessControl: 'Private' Metadata: - Comment: 'Bucket to store script-language developer sandbox VM images' + Comment: 'Bucket to store data-science sandbox VM images' DeletionPolicy: Retain UpdateReplacePolicy : Retain VMExportRole: From 3640cfa95ea747f59abac62affdddeccbd621bf5 Mon Sep 17 00:00:00 2001 From: ckunki Date: Thu, 19 Oct 2023 15:24:26 +0200 Subject: [PATCH 11/21] Additional renamings as aligned with TK. --- doc/developer_guide/developer_guide.md | 2 +- .../sandbox/cli/commands/setup_vm_bucket.py | 2 +- .../cli/commands/setup_vm_bucket_waf.py | 2 +- .../lib/asset_printing/print_assets.py | 2 +- exasol/ds/sandbox/lib/aws_access/deployer.py | 6 +++-- .../ds/sandbox/lib/export_vm/run_export_vm.py | 2 +- exasol/ds/sandbox/lib/run_create_vm.py | 3 ++- .../lib/setup_ci_codebuild/ci_codebuild.py | 2 +- .../release_codebuild.py | 2 +- exasol/ds/sandbox/lib/tags.py | 2 +- .../{vm_slc_bucket.py => vm_dss_bucket.py} | 4 ++-- ...slc_bucket_waf.py => vm_dss_bucket_waf.py} | 0 test/aws_mock_data.py | 24 +++++++++---------- test/cloudformation_validation.py | 2 +- test/test_deploy_vm_bucket.py | 4 ++-- test/test_deploy_vm_bucket_waf.py | 2 +- 16 files changed, 32 insertions(+), 29 deletions(-) rename exasol/ds/sandbox/lib/vm_bucket/{vm_slc_bucket.py => vm_dss_bucket.py} (95%) rename exasol/ds/sandbox/lib/vm_bucket/{vm_slc_bucket_waf.py => vm_dss_bucket_waf.py} (100%) diff --git a/doc/developer_guide/developer_guide.md b/doc/developer_guide/developer_guide.md index 03c626cb..495f81e9 100644 --- a/doc/developer_guide/developer_guide.md +++ b/doc/developer_guide/developer_guide.md @@ -119,7 +119,7 @@ The bucket has private access. In order to control access, the Bucket cloudforma The following diagram shows the involved cloudformation stacks: ![image info](./img/cloudformation-stacks.drawio.png) -"DATA-SCIENCE-SANDBOX-VM-SLC-Bucket", "DATA-SCIENCE-SANDBOX-CI-TEST-CODEBUILD" & "DATA-SCIENCE-SANDBOX-RELEASE-CODEBUILD" are permanent and need to be deployed using the "deploy" commands (see [commands](#deployment-commands)). +"DATA-SCIENCE-SANDBOX-VM-Bucket", "DATA-SCIENCE-SANDBOX-CI-TEST-CODEBUILD" & "DATA-SCIENCE-SANDBOX-RELEASE-CODEBUILD" are permanent and need to be deployed using the "deploy" commands (see [commands](#deployment-commands)). The EC2-stack lives only during the creation of a new sandbox image. ## Tagging diff --git a/exasol/ds/sandbox/cli/commands/setup_vm_bucket.py b/exasol/ds/sandbox/cli/commands/setup_vm_bucket.py index 10d376ac..11ccfe4e 100644 --- a/exasol/ds/sandbox/cli/commands/setup_vm_bucket.py +++ b/exasol/ds/sandbox/cli/commands/setup_vm_bucket.py @@ -5,7 +5,7 @@ from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from exasol.ds.sandbox.lib.config import default_config_object from exasol.ds.sandbox.lib.logging import set_log_level -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import run_setup_vm_bucket +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import run_setup_vm_bucket @cli.command() diff --git a/exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py b/exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py index 69a4c236..7426297c 100644 --- a/exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py +++ b/exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py @@ -7,7 +7,7 @@ from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from exasol.ds.sandbox.lib.config import default_config_object from exasol.ds.sandbox.lib.logging import set_log_level -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket_waf import run_setup_vm_bucket_waf +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket_waf import run_setup_vm_bucket_waf @cli.command() diff --git a/exasol/ds/sandbox/lib/asset_printing/print_assets.py b/exasol/ds/sandbox/lib/asset_printing/print_assets.py index e3fd72e0..bb0362a6 100644 --- a/exasol/ds/sandbox/lib/asset_printing/print_assets.py +++ b/exasol/ds/sandbox/lib/asset_printing/print_assets.py @@ -14,7 +14,7 @@ from exasol.ds.sandbox.lib.aws_access.cloudformation_stack import CloudformationStack from exasol.ds.sandbox.lib.tags import DEFAULT_TAG_KEY -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket, find_url_for_bucket +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import find_vm_bucket, find_url_for_bucket class AssetTypes(Enum): diff --git a/exasol/ds/sandbox/lib/aws_access/deployer.py b/exasol/ds/sandbox/lib/aws_access/deployer.py index ef643943..6df0903f 100644 --- a/exasol/ds/sandbox/lib/aws_access/deployer.py +++ b/exasol/ds/sandbox/lib/aws_access/deployer.py @@ -28,10 +28,12 @@ "ChangeSetResult", ["changeset_id", "changeset_type"]) +DEFAULT_CHANGE_SET_PREFIX="dss-ci-setup-deploy-" + + class Deployer(object): - def __init__(self, cloudformation_client, - changeset_prefix="slc-ci-setup-deploy-"): + def __init__(self, cloudformation_client, changeset_prefix=DEFAULT_CHANGE_SET_PREFIX): self._client = cloudformation_client self.changeset_prefix = changeset_prefix diff --git a/exasol/ds/sandbox/lib/export_vm/run_export_vm.py b/exasol/ds/sandbox/lib/export_vm/run_export_vm.py index e55634c4..cdd49181 100644 --- a/exasol/ds/sandbox/lib/export_vm/run_export_vm.py +++ b/exasol/ds/sandbox/lib/export_vm/run_export_vm.py @@ -11,7 +11,7 @@ from exasol.ds.sandbox.lib.setup_ec2.cf_stack import find_ec2_instance_in_cf_stack from exasol.ds.sandbox.lib.asset_printing.print_assets import print_assets from exasol.ds.sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket, find_vm_import_role +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import find_vm_bucket, find_vm_import_role LOG = get_status_logger(LogType.EXPORT) diff --git a/exasol/ds/sandbox/lib/run_create_vm.py b/exasol/ds/sandbox/lib/run_create_vm.py index 64810f9f..6ee5e4e5 100644 --- a/exasol/ds/sandbox/lib/run_create_vm.py +++ b/exasol/ds/sandbox/lib/run_create_vm.py @@ -36,7 +36,8 @@ def run_create_vm(aws_access: AwsAccess, ec2_key_file: Optional[str], ec2_key_na ansible_repositories: Tuple[AnsibleRepository, ...] = default_repositories) -> None: """ Runs setup of an EC2 instance and then installs all dependencies via Ansible, - and finally exports the VM to the S3 Bucket (which must be already created by the stack ("VM-SLC-Bucket"). + and finally exports the VM to the S3 Bucket (which must be already created + by the stack "DATA-SCIENCE-SANDBOX-VM-Bucket"). If anything goes wrong the cloudformation stack of the EC-2 instance will be removed. For debuging you can use the available debug commands. """ diff --git a/exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py b/exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py index 21e871fe..d8112607 100644 --- a/exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py +++ b/exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py @@ -1,7 +1,7 @@ from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from exasol.ds.sandbox.lib.logging import get_status_logger, LogType from exasol.ds.sandbox.lib.render_template import render_template -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import find_vm_bucket STACK_NAME = "DATA-SCIENCE-SANDBOX-CI-TEST-CODEBUILD" diff --git a/exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py b/exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py index 28d50f2e..5fb3beba 100644 --- a/exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py +++ b/exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py @@ -1,7 +1,7 @@ from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from exasol.ds.sandbox.lib.logging import get_status_logger, LogType from exasol.ds.sandbox.lib.render_template import render_template -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import find_vm_bucket RELEASE_CODE_BUILD_STACK_NAME = "DATA-SCIENCE-SANDBOX-RELEASE-CODEBUILD" diff --git a/exasol/ds/sandbox/lib/tags.py b/exasol/ds/sandbox/lib/tags.py index 129c6d89..0b4100d0 100644 --- a/exasol/ds/sandbox/lib/tags.py +++ b/exasol/ds/sandbox/lib/tags.py @@ -1,4 +1,4 @@ -DEFAULT_TAG_KEY = "exa_slc_id" +DEFAULT_TAG_KEY = "exa_dss_id" def create_default_asset_tag(value: str) -> list: diff --git a/exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket.py b/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket.py similarity index 95% rename from exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket.py rename to exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket.py index 71782e71..8bd5864f 100644 --- a/exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket.py +++ b/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket.py @@ -4,9 +4,9 @@ from exasol.ds.sandbox.lib.render_template import render_template from enum import Enum -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket_waf import find_acl_arn +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket_waf import find_acl_arn -STACK_NAME = "DATA-SCIENCE-SANDBOX-VM-SLC-Bucket" +STACK_NAME = "DATA-SCIENCE-SANDBOX-VM-Bucket" class OutputKey(Enum): diff --git a/exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket_waf.py b/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py similarity index 100% rename from exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket_waf.py rename to exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py diff --git a/test/aws_mock_data.py b/test/aws_mock_data.py index 8f8a99a5..e9409a3a 100644 --- a/test/aws_mock_data.py +++ b/test/aws_mock_data.py @@ -12,8 +12,8 @@ from exasol.ds.sandbox.lib.aws_access.stack_resource import StackResource from test.conftest import DEFAULT_ASSET_ID -TEST_ROLE_ID = 'VM-SLC-Bucket-VMImportRole-TEST' -TEST_BUCKET_ID = 'vm-slc-bucket-vmslcbucket-TEST' +TEST_ROLE_ID = 'VM-DSS-Bucket-VMImportRole-TEST' +TEST_BUCKET_ID = 'vm-dss-bucket-vmdssbucket-TEST' TEST_AMI_ID = "AMI-IMAGE-12345" TEST_CLOUDFRONT_ID = "test-cloudfrontet-TEST" TEST_CLOUDFRONT_DOMAIN_NAME = "test-s3.cloudfront.net" @@ -43,7 +43,7 @@ def get_ami_image_mock_data(state: str) -> Ami: 'Name': DEFAULT_ASSET_ID.ami_name, 'RootDeviceName': '/dev/sda1', 'RootDeviceType': 'ebs', 'SriovNetSupport': 'simple', - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'VirtualizationType': 'hvm'}) @@ -53,7 +53,7 @@ def get_snapshot_mock_data(): 'OwnerId': '123', 'Progress': '100%', 'SnapshotId': 'snap-123', 'StartTime': datetime.datetime(2022, 8, 16, 15, 3, 40, 662000, tzinfo=tzutc()), 'State': 'completed', 'VolumeId': 'vol-123', 'VolumeSize': 100, - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'StorageTier': 'standard' }) @@ -71,7 +71,7 @@ def get_export_image_task_mock_data(in_progress: bool): 'Progress': "50%", "StatusMessage": "creating the image", 'Status': 'active', - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}] + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}] }) else: return ExportImageTask({ @@ -83,7 +83,7 @@ def get_export_image_task_mock_data(in_progress: bool): 'S3Prefix': DEFAULT_ASSET_ID.bucket_prefix }, 'Status': 'completed', - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}] + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}] }) @@ -107,7 +107,7 @@ def get_ec2_cloudformation_mock_data(): 'RollbackConfiguration': {}, 'StackStatus': 'CREATE_COMPLETE', 'DisableRollback': False, 'NotificationARNs': [], 'Capabilities': ['CAPABILITY_IAM'], - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'DriftInformation': {'StackDriftStatus': 'NOT_CHECKED'} }) @@ -137,7 +137,7 @@ def get_ec2_key_pair_mock_data(): 'KeyFingerprint': '12:34:56:78:90:12:34:56:78:90:12:34:56:78:90:12:34:56:78:90', 'KeyName': 'ec2-key-test-key', 'KeyType': 'rsa', - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'CreateTime': datetime.datetime(2022, 8, 16, 15, 30, 41, tzinfo=tzutc()) }) @@ -146,14 +146,14 @@ def get_ec2_key_pair_mock_data(): def get_s3_cloudformation_mock_data() -> List[CloudformationStack]: return [CloudformationStack({ 'StackId': 'test-s3-stack-id', - 'StackName': "DATA-SCIENCE-SANDBOX-VM-SLC-Bucket", + 'StackName': "DATA-SCIENCE-SANDBOX-VM-Bucket", 'ChangeSetId': 'test-stack-changeset-id-2', 'CreationTime': datetime.datetime(2022, 8, 16, 14, 30, 45, 559000, tzinfo=tzutc()), 'LastUpdatedTime': datetime.datetime(2022, 8, 16, 14, 30, 51, 667000, tzinfo=tzutc()), 'RollbackConfiguration': {}, 'StackStatus': 'CREATE_COMPLETE', 'DisableRollback': False, 'NotificationARNs': [], 'Capabilities': ['CAPABILITY_IAM'], - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'DriftInformation': {'StackDriftStatus': 'NOT_CHECKED'}, 'Outputs': [{'OutputKey': 'VMBucketId', 'OutputValue': TEST_BUCKET_ID, 'Description': ''}, @@ -171,14 +171,14 @@ def get_s3_cloudformation_mock_data() -> List[CloudformationStack]: def get_waf_cloudformation_mock_data() -> List[CloudformationStack]: return [CloudformationStack({ 'StackId': 'test-waf-stack-id', - 'StackName': "DATA-SCIENCE-SANDBOX-VM-SLC-Bucket-WAF", + 'StackName': "DATA-SCIENCE-SANDBOX-VM-Bucket-WAF", 'ChangeSetId': 'test-stack-changeset-id-3', 'CreationTime': datetime.datetime(2022, 8, 16, 14, 30, 45, 559000, tzinfo=tzutc()), 'LastUpdatedTime': datetime.datetime(2022, 8, 16, 14, 30, 51, 667000, tzinfo=tzutc()), 'RollbackConfiguration': {}, 'StackStatus': 'CREATE_COMPLETE', 'DisableRollback': False, 'NotificationARNs': [], 'Capabilities': [], - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'DriftInformation': {'StackDriftStatus': 'NOT_CHECKED'}, 'Outputs': [{'OutputKey': 'VMDownloadACLArn', 'OutputValue': TEST_ACL_ARN, 'Description': ''} diff --git a/test/cloudformation_validation.py b/test/cloudformation_validation.py index 110f0b55..7a30153a 100644 --- a/test/cloudformation_validation.py +++ b/test/cloudformation_validation.py @@ -6,7 +6,7 @@ def validate_using_cfn_lint(tmp_path, cloudformation_yml): This test uses cfn-lint to validate the Cloudformation template. (See https://github.com/aws-cloudformation/cfn-lint) """ - out_file = tmp_path / "slc_source_credential.yaml" + out_file = tmp_path / "cloudformation.yaml" with open(out_file, "w") as f: f.write(cloudformation_yml) diff --git a/test/test_deploy_vm_bucket.py b/test/test_deploy_vm_bucket.py index 0dccfcd4..7d40f799 100644 --- a/test/test_deploy_vm_bucket.py +++ b/test/test_deploy_vm_bucket.py @@ -4,13 +4,13 @@ import pytest from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import run_setup_vm_bucket, find_vm_bucket, \ +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import run_setup_vm_bucket, find_vm_bucket, \ create_vm_bucket_cf_template from test.aws_mock_data import TEST_BUCKET_ID, get_waf_cloudformation_mock_data, TEST_ACL_ARN, \ get_s3_cloudformation_mock_data from test.cloudformation_validation import validate_using_cfn_lint from test.mock_cast import mock_cast -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import STACK_NAME as VM_STACK_NAME +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import STACK_NAME as VM_STACK_NAME @pytest.fixture diff --git a/test/test_deploy_vm_bucket_waf.py b/test/test_deploy_vm_bucket_waf.py index 03804886..e9e37abf 100644 --- a/test/test_deploy_vm_bucket_waf.py +++ b/test/test_deploy_vm_bucket_waf.py @@ -4,7 +4,7 @@ import pytest from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket_waf import run_setup_vm_bucket_waf, \ +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket_waf import run_setup_vm_bucket_waf, \ find_acl_arn, get_cloudformation_template from test.aws_mock_data import get_waf_cloudformation_mock_data, TEST_ACL_ARN from test.cloudformation_validation import validate_using_cfn_lint From b5921e03d5baddf25060045ed0bf806b62557f88 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 20 Oct 2023 13:41:59 +0200 Subject: [PATCH 12/21] Fixed tests and organized into folders. --- .github/workflows/check_ci.yaml | 4 +- doc/developer_guide/developer_guide.md | 99 +++++++++++++++---- .../lib/vm_bucket/vm_dss_bucket_waf.py | 2 +- .../templates/ec2_cloudformation.jinja.yaml | 4 +- test/aws/__init__.py | 5 + test/{ => aws}/aws_mock_data.py | 0 test/{ => aws}/cloudformation_validation.py | 0 test/{ => aws}/test_deploy_codebuild.py | 2 +- test/{ => aws}/test_deploy_ec2.py | 2 +- test/{ => aws}/test_deploy_vm_bucket.py | 4 +- test/{ => aws}/test_deploy_vm_bucket_waf.py | 4 +- test/{ => aws}/test_export_vm.py | 2 +- test/integration/__init__.py | 4 + .../aws_local_stack_access.py | 0 test/{ => integration}/localstack_test.py | 6 +- test/{ => integration}/test_ci.py | 0 .../test_install_dependencies.py | 0 test/unit/__init__.py | 4 + test/{ => unit}/ansible_conflict/__init__.py | 0 .../{ => unit}/ansible_conflict/slc_setup.yml | 0 test/{ => unit}/test_ansible.py | 4 +- test/{ => unit}/test_aws_codebuild_waiter.py | 0 test/{ => unit}/test_key_file_manager.py | 0 test/{ => unit}/test_make_ami_public.py | 0 test/{ => unit}/test_motd_jupyter_template.py | 2 +- test/{ => unit}/test_printing.py | 0 test/{ => unit}/test_printing_factories.py | 0 test/{ => unit}/test_release_build.py | 0 test/{ => unit}/test_run_lifecycle_for_ec2.py | 0 test/{ => unit}/test_serialization.py | 0 test/{ => unit}/test_source_ami.py | 0 31 files changed, 109 insertions(+), 39 deletions(-) create mode 100644 test/aws/__init__.py rename test/{ => aws}/aws_mock_data.py (100%) rename test/{ => aws}/cloudformation_validation.py (100%) rename test/{ => aws}/test_deploy_codebuild.py (92%) rename test/{ => aws}/test_deploy_ec2.py (96%) rename test/{ => aws}/test_deploy_vm_bucket.py (94%) rename test/{ => aws}/test_deploy_vm_bucket_waf.py (90%) rename test/{ => aws}/test_export_vm.py (97%) create mode 100644 test/integration/__init__.py rename test/{ => integration}/aws_local_stack_access.py (100%) rename test/{ => integration}/localstack_test.py (96%) rename test/{ => integration}/test_ci.py (100%) rename test/{ => integration}/test_install_dependencies.py (100%) create mode 100644 test/unit/__init__.py rename test/{ => unit}/ansible_conflict/__init__.py (100%) rename test/{ => unit}/ansible_conflict/slc_setup.yml (100%) rename test/{ => unit}/test_ansible.py (98%) rename test/{ => unit}/test_aws_codebuild_waiter.py (100%) rename test/{ => unit}/test_key_file_manager.py (100%) rename test/{ => unit}/test_make_ami_public.py (100%) rename test/{ => unit}/test_motd_jupyter_template.py (96%) rename test/{ => unit}/test_printing.py (100%) rename test/{ => unit}/test_printing_factories.py (100%) rename test/{ => unit}/test_release_build.py (100%) rename test/{ => unit}/test_run_lifecycle_for_ec2.py (100%) rename test/{ => unit}/test_serialization.py (100%) rename test/{ => unit}/test_source_ami.py (100%) diff --git a/.github/workflows/check_ci.yaml b/.github/workflows/check_ci.yaml index dbe9ecb5..616b8be9 100644 --- a/.github/workflows/check_ci.yaml +++ b/.github/workflows/check_ci.yaml @@ -1,4 +1,4 @@ -name: Run Unit Tests +name: Run Tests for CI Build on: push: @@ -18,7 +18,7 @@ jobs: uses: ./.github/actions/prepare_poetry_env - name: Run pytest - run: poetry run pytest test/test_install_dependencies.py + run: poetry run pytest test/integration/test_install_dependencies.py env: # Set the secret as an env variable AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }} diff --git a/doc/developer_guide/developer_guide.md b/doc/developer_guide/developer_guide.md index 495f81e9..a5691e1e 100644 --- a/doc/developer_guide/developer_guide.md +++ b/doc/developer_guide/developer_guide.md @@ -27,11 +27,11 @@ bash install.sh The Data Science Sandbox (DSS) uses AWS as backend, because it provides the possibility to run the whole workflow during a ci-test. -This project uses +This project uses - `boto3` to interact with AWS - `pygithub` to interact with the Github releases -- `ansible-runner` to interact with Ansible. -Proxy classes to those projects are injected at the CLI layer. This allows to inject mock classes in the unit tests. +- `ansible-runner` to interact with Ansible. +Proxy classes to those projects are injected at the CLI layer. This allows to inject mock classes in the unit tests. A CLI command has normally a respective function in the `lib` submodule. Hence, the CLI layer should not contain any logic, but invoke the respective library function only. Also, the proxy classes which abstract the dependant packages shall not contain too much logic. Ideally they should invoke only one function to the respective package. @@ -39,18 +39,18 @@ A CLI command has normally a respective function in the `lib` submodule. Hence, There are generally three types of commands: -| Type | Explanation | +| Type | Explanation | | ----- | --------- | -| Release Commands | used during the release | -| Deployment Commands | used to deploy infrastructure onto AWS cloud | -| Development Commands | used to identify problems or for testing | +| Release Commands | used during the release | +| Deployment Commands | used to deploy infrastructure onto AWS cloud | +| Development Commands | used to identify problems or for testing | ### Release commands The following commands are used during the release AWS Codebuild job: - `create-vm` - creates a new AMI and VM images - `update-release` - updates release notes of an existing Github release -- `start-release-build` - starts the release on AWS codebuild +- `start-release-build` - starts the release on AWS codebuild ### Developer commands @@ -62,7 +62,7 @@ All other commands provide a subset of the features of the release commands, and - `setup-ec2-and-install-dependencies` - starts a new EC2 instance and install dependencies via Ansible - `show-aws-assets` - shows AWS entities associated with a specific keyword (called __asset-id__) - `start-test-release` - starts a Test Release flow -- `make-ami-public` - Changes permissions of an existing AMI such that it becomes public +- `make-ami-public` - Changes permissions of an existing AMI such that it becomes public ### Deployment commands @@ -70,11 +70,11 @@ The following commands can be used to deploy the infrastructure onto a given AWS - `setup-ci-codebuild` - deploys the AWS Codebuild cloudformation stack which will run the ci-test - `setup-vm-bucket` - deploys the AWS Bucket cloudformation stack which will be used to deploy the VM images - `setup-release-codebuild` - deploys the AWS Codebuild cloudformation stack which will be used for the release-build -- `setup-vm-bucket-waf` - deploys the AWS Codebuild cloudformation stack which contains the WAF Acl configuration for the Cloudfront distribution of the VM Bucket +- `setup-vm-bucket-waf` - deploys the AWS Codebuild cloudformation stack which contains the WAF Acl configuration for the Cloudfront distribution of the VM Bucket ## Flow -The following diagram shows the high-level steps to generate the images: +The following diagram shows the high-level steps to generate the images: ![image info](./img/create-vm-overview.drawio.png) ### Setup EC2 @@ -94,6 +94,55 @@ Installs all dependencies via Ansible: Finally, the default password will be set, and also the password will be marked as expired, such that the user will be forced to enter a new password during initial login. Also, the ssh password authentication will be enabled, and for security reasons the folder "~/.ssh" will be removed. +### Tests + +DSS comes with a number of tests in directory `test`. +There are subdirectories clustering tests with common scope and prerequisites e.g. external resources. + +| Directory | Content | +|---------------------|---------| +| `test/unit` | Simple unit tests requiring no additional setup or external resources. | +| `test/integration` | Integration tests with longer runtime and some requiring additional resources. | +| `test/aws` | Tests involving AWS resources. In order to execute these tests you need an AWS account, a user with permissions in this account, and an access key. | + +To run the tests in file `test/integration/test_ci.py` please use +```shell +export DSS_RUN_CI_TEST=true +poetry run test/integration/test_ci.py +``` + +#### Executing tests involving AWS resources + +In AWS web interface, IAM create an access key for CLI usage and save or download the *access key id* and the *secret access key*. + +In file `~/.aws/config` add lines +``` +[profile dss_aws_tests] +region = eu-central-1 +``` + +In file `~/.aws/credentials` add +``` +[dss_aws_tests] +aws_access_key_id=... +aws_secret_access_key=... +``` + +From [product-integration-tool-chest](https://github.com/exasol/product-integration-tool-chest) call +```shell +aws-store-session-token dss_aws_tests +``` + +This will ask you for your MFA code and add or update profile `[dss_aws_tests_mfa]` in file `~/.aws/credentials`. + +Now you can set an environment variable and execute the tests involing AWS resources: + +```shell +export AWS_PROFILE=dss_aws_tests_mfa +poetry run pytest test/test_deploy_codebuild.py +``` + + ### Export The export creates an AMI based on the running EC2 instance and exports the AMI as VM image in the default formats to a S3 bucket. @@ -107,10 +156,10 @@ The release is executed in a AWS Codebuild job, the following diagram shows the The bucket has private access. In order to control access, the Bucket cloudformation stack also contains a Cloudfront distribution. Public Https access is only possibly through Cloudfront. Another stack contains a Web application firewall (WAF), which will be used by the Cloudfront distribution. Due to restrictions in AWS, the WAF stack needs to be deployed in region "us-east-1". The WAF stack provides two rules which aim to minimize a possible bot attack: -| Name | Explanation | Priority | +| Name | Explanation | Priority | |----------------------|-----------------------------------------------------------------------------------------|----------| -| VMBucketRateLimit | Declares the minimum possible rate limit for access: 100 requests in a 5 min interval. | 0 | -| CAPTCHA | Forces a captcha action for any IP which does not matcha predefined set of IP-addresses | 1 | +| VMBucketRateLimit | Declares the minimum possible rate limit for access: 100 requests in a 5 min interval. | 0 | +| CAPTCHA | Forces a captcha action for any IP which does not matcha predefined set of IP-addresses | 1 | @@ -119,18 +168,26 @@ The bucket has private access. In order to control access, the Bucket cloudforma The following diagram shows the involved cloudformation stacks: ![image info](./img/cloudformation-stacks.drawio.png) -"DATA-SCIENCE-SANDBOX-VM-Bucket", "DATA-SCIENCE-SANDBOX-CI-TEST-CODEBUILD" & "DATA-SCIENCE-SANDBOX-RELEASE-CODEBUILD" are permanent and need to be deployed using the "deploy" commands (see [commands](#deployment-commands)). +The following resources are permanent and need to be deployed using the "deploy" [commands](#deployment-commands): +* `DATA-SCIENCE-SANDBOX-VM-Bucket` +* `DATA-SCIENCE-SANDBOX-CI-TEST-CODEBUILD` +* `DATA-SCIENCE-SANDBOX-RELEASE-CODEBUILD` + The EC2-stack lives only during the creation of a new sandbox image. ## Tagging Each of the involved resources might cause costs: cloudformation stacks, AMI, EC2 key-pairs. -To enable you to keep track of all these resources, the implementation tags them after creation with a specific keyword (called __asset-id__). -The S3 objects are identified by the prefix in the S3 bucket. The command tags only the dynamically created entities with the asset-id but not the permanent cloudformation stacks. -You can use the command `show-aws-assets` to get a list of all assets which were created during the execution. -This is very useful if an error occured. -If the creation of a sandbox finished normally it is expected to have only the AMI, images (S3 objects) and the export tasks (one for each image) listed. +To enable keeping track of all these resources, the implementation tags them after creation with a specific keyword (called __asset-id__). + +The S3 objects are identified by the prefix in the S3 bucket. + +The command tags only the dynamically created entities with the *asset-id* but not the permanent cloudformation stacks. + +The command `show-aws-assets` lists all assets which were created during the execution. +* This is very useful if an error occured. +* If the creation of a sandbox finished normally the list should contain only the AMI, images (S3 objects) and the export tasks (one for each image). ## How to contribute @@ -139,4 +196,4 @@ The project has two types of CI tests: - A system test which runs on a AWS Codebuild Both ci tests need to pass before the approval of a Github PR. -The Github workflow will run on each push to a branch in the Github repository. However, the AWS Codebuild will only run after you push a commit containing the string "[CodeBuild]" in the commit message. \ No newline at end of file +The Github workflow will run on each push to a branch in the Github repository. However, the AWS Codebuild will only run after you push a commit containing the string "[CodeBuild]" in the commit message. \ No newline at end of file diff --git a/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py b/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py index be84ffe1..fb478075 100644 --- a/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py +++ b/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py @@ -3,7 +3,7 @@ from exasol.ds.sandbox.lib.logging import get_status_logger, LogType from exasol.ds.sandbox.lib.render_template import render_template -STACK_NAME = "DATA-SCIENCE-SANDBOX-VM-SLC-Bucket-WAF" +STACK_NAME = "DATA-SCIENCE-SANDBOX-VM-Bucket-WAF" LOG = get_status_logger(LogType.VM_BUCKET) diff --git a/exasol/ds/sandbox/templates/ec2_cloudformation.jinja.yaml b/exasol/ds/sandbox/templates/ec2_cloudformation.jinja.yaml index 88979f7d..cce42d74 100644 --- a/exasol/ds/sandbox/templates/ec2_cloudformation.jinja.yaml +++ b/exasol/ds/sandbox/templates/ec2_cloudformation.jinja.yaml @@ -14,7 +14,7 @@ Resources: ToPort: 8888 Tags: - Key: "exasol:project" - Value: "ScriptLanguages" + Value: "DataScienceSandbox" - Key: "exasol:owner" Value: {{user_name}} - Key: {{trace_tag}} @@ -35,7 +35,7 @@ Resources: VolumeSize: 100 Tags: - Key: "exasol:project" - Value: "ScriptLanguages" + Value: "DataScienceSandbox" - Key: "exasol:owner" Value: {{user_name}} - Key: {{trace_tag}} diff --git a/test/aws/__init__.py b/test/aws/__init__.py new file mode 100644 index 00000000..a1d3903c --- /dev/null +++ b/test/aws/__init__.py @@ -0,0 +1,5 @@ +""" +This package contains tests involving AWS resources. In order to execute +these tests you need an AWS account, a user with permissions in this account and +an access key. +""" diff --git a/test/aws_mock_data.py b/test/aws/aws_mock_data.py similarity index 100% rename from test/aws_mock_data.py rename to test/aws/aws_mock_data.py diff --git a/test/cloudformation_validation.py b/test/aws/cloudformation_validation.py similarity index 100% rename from test/cloudformation_validation.py rename to test/aws/cloudformation_validation.py diff --git a/test/test_deploy_codebuild.py b/test/aws/test_deploy_codebuild.py similarity index 92% rename from test/test_deploy_codebuild.py rename to test/aws/test_deploy_codebuild.py index c7b22e04..5a763e56 100644 --- a/test/test_deploy_codebuild.py +++ b/test/aws/test_deploy_codebuild.py @@ -2,7 +2,7 @@ from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from exasol.ds.sandbox.lib.render_template import render_template -from test.cloudformation_validation import validate_using_cfn_lint +from test.aws.cloudformation_validation import validate_using_cfn_lint codebuild_cloudformation_templates = [ diff --git a/test/test_deploy_ec2.py b/test/aws/test_deploy_ec2.py similarity index 96% rename from test/test_deploy_ec2.py rename to test/aws/test_deploy_ec2.py index feda4b7e..51ccaa13 100644 --- a/test/test_deploy_ec2.py +++ b/test/aws/test_deploy_ec2.py @@ -4,7 +4,7 @@ from exasol.ds.sandbox.lib.setup_ec2.cf_stack import CloudformationStack, \ CloudformationStackContextManager from exasol.ds.sandbox.lib.tags import create_default_asset_tag -from test.cloudformation_validation import validate_using_cfn_lint +from test.aws.cloudformation_validation import validate_using_cfn_lint def test_deploy_ec2_upload_invoked(ec2_cloudformation_yml, default_asset_id, test_dummy_ami_id): diff --git a/test/test_deploy_vm_bucket.py b/test/aws/test_deploy_vm_bucket.py similarity index 94% rename from test/test_deploy_vm_bucket.py rename to test/aws/test_deploy_vm_bucket.py index 7d40f799..3d257d70 100644 --- a/test/test_deploy_vm_bucket.py +++ b/test/aws/test_deploy_vm_bucket.py @@ -6,9 +6,9 @@ from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import run_setup_vm_bucket, find_vm_bucket, \ create_vm_bucket_cf_template -from test.aws_mock_data import TEST_BUCKET_ID, get_waf_cloudformation_mock_data, TEST_ACL_ARN, \ +from test.aws.aws_mock_data import TEST_BUCKET_ID, get_waf_cloudformation_mock_data, TEST_ACL_ARN, \ get_s3_cloudformation_mock_data -from test.cloudformation_validation import validate_using_cfn_lint +from test.aws.cloudformation_validation import validate_using_cfn_lint from test.mock_cast import mock_cast from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import STACK_NAME as VM_STACK_NAME diff --git a/test/test_deploy_vm_bucket_waf.py b/test/aws/test_deploy_vm_bucket_waf.py similarity index 90% rename from test/test_deploy_vm_bucket_waf.py rename to test/aws/test_deploy_vm_bucket_waf.py index e9e37abf..f3d30bd7 100644 --- a/test/test_deploy_vm_bucket_waf.py +++ b/test/aws/test_deploy_vm_bucket_waf.py @@ -6,8 +6,8 @@ from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket_waf import run_setup_vm_bucket_waf, \ find_acl_arn, get_cloudformation_template -from test.aws_mock_data import get_waf_cloudformation_mock_data, TEST_ACL_ARN -from test.cloudformation_validation import validate_using_cfn_lint +from test.aws.aws_mock_data import get_waf_cloudformation_mock_data, TEST_ACL_ARN +from test.aws.cloudformation_validation import validate_using_cfn_lint from test.mock_cast import mock_cast TEST_IP = "1.1.1.1" diff --git a/test/test_export_vm.py b/test/aws/test_export_vm.py similarity index 97% rename from test/test_export_vm.py rename to test/aws/test_export_vm.py index 01e00fc1..b854b996 100644 --- a/test/test_export_vm.py +++ b/test/aws/test_export_vm.py @@ -10,7 +10,7 @@ build_image_destination from exasol.ds.sandbox.lib.export_vm.run_export_vm import export_vm from exasol.ds.sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat -from test.aws_mock_data import get_ami_image_mock_data, TEST_AMI_ID, TEST_ROLE_ID, TEST_BUCKET_ID, INSTANCE_ID, \ +from test.aws.aws_mock_data import get_ami_image_mock_data, TEST_AMI_ID, TEST_ROLE_ID, TEST_BUCKET_ID, INSTANCE_ID, \ get_export_image_task_mock_data, get_s3_cloudformation_mock_data, get_waf_cloudformation_mock_data from test.mock_cast import mock_cast diff --git a/test/integration/__init__.py b/test/integration/__init__.py new file mode 100644 index 00000000..c4cd8e27 --- /dev/null +++ b/test/integration/__init__.py @@ -0,0 +1,4 @@ +""" +This package contains integration tests with longer runtime and maybe +requiring additional resources. +""" diff --git a/test/aws_local_stack_access.py b/test/integration/aws_local_stack_access.py similarity index 100% rename from test/aws_local_stack_access.py rename to test/integration/aws_local_stack_access.py diff --git a/test/localstack_test.py b/test/integration/localstack_test.py similarity index 96% rename from test/localstack_test.py rename to test/integration/localstack_test.py index 05cd2aca..95669087 100644 --- a/test/localstack_test.py +++ b/test/integration/localstack_test.py @@ -5,7 +5,7 @@ CloudformationStackContextManager from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2 from exasol.ds.sandbox.lib.tags import create_default_asset_tag -from test.aws_local_stack_access import AwsLocalStackAccess +from test.integration.aws_local_stack_access import AwsLocalStackAccess def test_ec2_lifecycle_with_local_stack(local_stack, default_asset_id, test_dummy_ami_id): @@ -71,7 +71,7 @@ def test_validate_cloudformation_template_fails_with_local_stack(local_stack): ToPort: 22 Tags: - Key: "exasol:project" - Value: "ScriptLanguages" + Value: "DataScienceSandbox" - Key: "exasol:owner" Value: "test user" @@ -90,7 +90,7 @@ def test_validate_cloudformation_template_fails_with_local_stack(local_stack): VolumeSize: 100 Tags: - Key: "exasol:project" - Value: "ScriptLanguages" + Value: "DataScienceSandbox" - Key: "exasol:owner" Value: "test" diff --git a/test/test_ci.py b/test/integration/test_ci.py similarity index 100% rename from test/test_ci.py rename to test/integration/test_ci.py diff --git a/test/test_install_dependencies.py b/test/integration/test_install_dependencies.py similarity index 100% rename from test/test_install_dependencies.py rename to test/integration/test_install_dependencies.py diff --git a/test/unit/__init__.py b/test/unit/__init__.py new file mode 100644 index 00000000..a6db5aa6 --- /dev/null +++ b/test/unit/__init__.py @@ -0,0 +1,4 @@ +""" +This package contains unit tests requiring no additional setup or external +resources. +""" diff --git a/test/ansible_conflict/__init__.py b/test/unit/ansible_conflict/__init__.py similarity index 100% rename from test/ansible_conflict/__init__.py rename to test/unit/ansible_conflict/__init__.py diff --git a/test/ansible_conflict/slc_setup.yml b/test/unit/ansible_conflict/slc_setup.yml similarity index 100% rename from test/ansible_conflict/slc_setup.yml rename to test/unit/ansible_conflict/slc_setup.yml diff --git a/test/test_ansible.py b/test/unit/test_ansible.py similarity index 98% rename from test/test_ansible.py rename to test/unit/test_ansible.py index ccfe0c7c..251cc87e 100644 --- a/test/test_ansible.py +++ b/test/unit/test_ansible.py @@ -12,7 +12,7 @@ from exasol.ds.sandbox.lib.setup_ec2.run_install_dependencies import run_install_dependencies import test.ansible -import test.ansible_conflict +import test.unit.ansible_conflict class AnsibleTestAccess: @@ -129,7 +129,7 @@ def test_run_ansible_check_multiple_repositories_with_same_content_causes_except """ Test that multiple repositories containing same files raises an runtime exception. """ - test_repositories = default_repositories + (AnsibleResourceRepository(test.ansible_conflict),) + test_repositories = default_repositories + (AnsibleResourceRepository(test.unit.ansible_conflict),) with pytest.raises(RuntimeError): run_install_dependencies(AnsibleTestAccess(), test_config, host_infos=tuple(), ansible_run_context=default_ansible_run_context, diff --git a/test/test_aws_codebuild_waiter.py b/test/unit/test_aws_codebuild_waiter.py similarity index 100% rename from test/test_aws_codebuild_waiter.py rename to test/unit/test_aws_codebuild_waiter.py diff --git a/test/test_key_file_manager.py b/test/unit/test_key_file_manager.py similarity index 100% rename from test/test_key_file_manager.py rename to test/unit/test_key_file_manager.py diff --git a/test/test_make_ami_public.py b/test/unit/test_make_ami_public.py similarity index 100% rename from test/test_make_ami_public.py rename to test/unit/test_make_ami_public.py diff --git a/test/test_motd_jupyter_template.py b/test/unit/test_motd_jupyter_template.py similarity index 96% rename from test/test_motd_jupyter_template.py rename to test/unit/test_motd_jupyter_template.py index 586495c8..821fc6ae 100644 --- a/test/test_motd_jupyter_template.py +++ b/test/unit/test_motd_jupyter_template.py @@ -14,7 +14,7 @@ def motd_file(tmp_path): jupyter_server_config_file = tmp_path / "jupyter_server_config.json" python_file = tmp_path / "999_jupyter.py" - src_path = Path(__file__).parent.parent / "exasol.ds.sandbox" / "runtime" / \ + src_path = Path(__file__).parent.parent / "exasol" / "ds" / "sandbox" / "runtime" / \ "ansible" / "roles" / "jupyter" / "templates" / "etc" /"update-motd.d" / "999-jupyter" with open(src_path, "r") as f: python_code_template = f.read() diff --git a/test/test_printing.py b/test/unit/test_printing.py similarity index 100% rename from test/test_printing.py rename to test/unit/test_printing.py diff --git a/test/test_printing_factories.py b/test/unit/test_printing_factories.py similarity index 100% rename from test/test_printing_factories.py rename to test/unit/test_printing_factories.py diff --git a/test/test_release_build.py b/test/unit/test_release_build.py similarity index 100% rename from test/test_release_build.py rename to test/unit/test_release_build.py diff --git a/test/test_run_lifecycle_for_ec2.py b/test/unit/test_run_lifecycle_for_ec2.py similarity index 100% rename from test/test_run_lifecycle_for_ec2.py rename to test/unit/test_run_lifecycle_for_ec2.py diff --git a/test/test_serialization.py b/test/unit/test_serialization.py similarity index 100% rename from test/test_serialization.py rename to test/unit/test_serialization.py diff --git a/test/test_source_ami.py b/test/unit/test_source_ami.py similarity index 100% rename from test/test_source_ami.py rename to test/unit/test_source_ami.py From 737cbbf382bb0f4dcb3d9516f573090d777f6cea Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 20 Oct 2023 13:46:53 +0200 Subject: [PATCH 13/21] moved folder test_container to integration --- test/{ => integration}/test_container/Dockerfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{ => integration}/test_container/Dockerfile (100%) diff --git a/test/test_container/Dockerfile b/test/integration/test_container/Dockerfile similarity index 100% rename from test/test_container/Dockerfile rename to test/integration/test_container/Dockerfile From df49615c7a6e3eed4bec91bb8db94bdb61e0b6ff Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 20 Oct 2023 14:07:58 +0200 Subject: [PATCH 14/21] simplified test_install_dependencies --- .github/workflows/check_ci.yaml | 2 +- test/integration/test_install_dependencies.py | 81 ++++++++++++------- 2 files changed, 52 insertions(+), 31 deletions(-) diff --git a/.github/workflows/check_ci.yaml b/.github/workflows/check_ci.yaml index 616b8be9..a1819a3d 100644 --- a/.github/workflows/check_ci.yaml +++ b/.github/workflows/check_ci.yaml @@ -6,7 +6,7 @@ on: - "main" jobs: - run_unit_tests: + run_ci_tests: environment: AWS_CI_TESTS runs-on: ubuntu-latest diff --git a/test/integration/test_install_dependencies.py b/test/integration/test_install_dependencies.py index dbd7fff6..f8ea5da7 100644 --- a/test/integration/test_install_dependencies.py +++ b/test/integration/test_install_dependencies.py @@ -3,6 +3,7 @@ import requests import time from pathlib import Path +from inspect import cleandoc import docker import pytest @@ -18,38 +19,58 @@ TEST_CONTAINER_NAME = "ansible-test" TEST_CONTAINER_IMAGE_TAG = "data_science_sandbox_test_container:latest" +DOCKERFILE_CONTENT = cleandoc( + """ + FROM ubuntu:20.04 + ENV DEBIAN_FRONTEND noninteractive + #AWS AMI Image is based on Ubuntu20.04 server. So we try to simulate same environment here + RUN apt-get update && apt-get install -y ubuntu-server + """ +) + +@pytest.fixture +def test_container(tmp_path): + dockerfile = tmp_path / "test_container" / "Dockerfile" + dockerfile.parent.mkdir() + with dockerfile.open("w") as f: + print(DOCKERFILE_CONTENT, file = f) + yield dockerfile + @pytest.fixture(scope="session") -def docker_test_container(test_config): - with tempfile.TemporaryDirectory() as tmp_dir: - docker_env = docker.from_env() - p = Path(__file__).parent / "test_container" - docker_env.images.build(path=str(p), tag=TEST_CONTAINER_IMAGE_TAG) - socket_mount = Mount("/var/run/docker.sock", "/var/run/docker.sock", type="bind") - tmp_mount = Mount(tmp_dir, tmp_dir, type="bind") - mapped_ports = {'8888/tcp': 8888} - test_container = docker_env.containers.create(image=TEST_CONTAINER_IMAGE_TAG, - name=TEST_CONTAINER_NAME, mounts=[socket_mount, tmp_mount], - command="sleep infinity", detach=True, ports=mapped_ports) - test_container.start() - repos = default_repositories + (AnsibleResourceRepository(test.ansible),) - ansible_run_context = \ - AnsibleRunContext(playbook="slc_setup_test.yml", - extra_vars={"test_docker_container": test_container.name, - "slc_dest_folder": f"{tmp_dir}/script-languages-release"}) - try: - run_install_dependencies(AnsibleAccess(), configuration=test_config, - host_infos=tuple(), ansible_run_context=ansible_run_context, - ansible_repositories=repos) - yield test_container, tmp_dir - # Note: script-languages-release will be cloned by ansible within the docker container. - # Because the docker container runs as root, the repository will be owned by root. - # For simplicity, we delete the folder from within the Docker container (as root). - # Otherwise, we get a permission problem when tmp_dir tries to clean-up itself. - finally: - test_container.exec_run(f"rm -rf {tmp_dir}/script-languages-release") - test_container.stop() - test_container.remove() +def docker_test_container(test_config, dockerfile, tmp_path): + # with tempfile.TemporaryDirectory() as tmp_path: + docker_env = docker.from_env() + # p = Path(__file__).parent / "test_container" + docker_env.images.build( + path=str(dockerfile.parent), + tag=TEST_CONTAINER_IMAGE_TAG + ) + socket_mount = Mount("/var/run/docker.sock", "/var/run/docker.sock", type="bind") + tmp_mount = Mount(tmp_path, tmp_path, type="bind") + mapped_ports = {'8888/tcp': 8888} + test_container = docker_env.containers.create(image=TEST_CONTAINER_IMAGE_TAG, + name=TEST_CONTAINER_NAME, mounts=[socket_mount, tmp_mount], + command="sleep infinity", detach=True, ports=mapped_ports) + test_container.start() + repos = default_repositories + (AnsibleResourceRepository(test.ansible),) + ansible_run_context = \ + AnsibleRunContext(playbook="slc_setup_test.yml", + extra_vars={"test_docker_container": test_container.name, + "slc_dest_folder": f"{tmp_path}/script-languages-release"}) + try: + run_install_dependencies(AnsibleAccess(), configuration=test_config, + host_infos=tuple(), ansible_run_context=ansible_run_context, + ansible_repositories=repos) + yield test_container, tmp_path + # Note: script-languages-release will be cloned by ansible within the docker container. + # Because the docker container runs as root, the repository will be owned by root. + # For simplicity, we delete the folder from within the Docker container (as root). + # Otherwise, we get a permission problem when tmp_path tries to clean-up itself. + finally: + test_container.exec_run(f"rm -rf {tmp_path}/script-languages-release") + test_container.stop() + test_container.remove() def test_install_dependencies_jupyterlab(docker_test_container): From 9090707b217364630a58390a31d2eaf310de9967 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 20 Oct 2023 14:11:27 +0200 Subject: [PATCH 15/21] Fixed test --- test/integration/test_install_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/test_install_dependencies.py b/test/integration/test_install_dependencies.py index f8ea5da7..8d248c8a 100644 --- a/test/integration/test_install_dependencies.py +++ b/test/integration/test_install_dependencies.py @@ -29,7 +29,7 @@ ) @pytest.fixture -def test_container(tmp_path): +def dockerfile(tmp_path): dockerfile = tmp_path / "test_container" / "Dockerfile" dockerfile.parent.mkdir() with dockerfile.open("w") as f: From 07a67e0fe67c7ad2e2280a805cc2cd473ed29ad0 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 20 Oct 2023 14:13:35 +0200 Subject: [PATCH 16/21] Fixed test --- test/integration/test_install_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/test_install_dependencies.py b/test/integration/test_install_dependencies.py index 8d248c8a..4a113925 100644 --- a/test/integration/test_install_dependencies.py +++ b/test/integration/test_install_dependencies.py @@ -28,7 +28,7 @@ """ ) -@pytest.fixture +@pytest.fixture(scope="session") def dockerfile(tmp_path): dockerfile = tmp_path / "test_container" / "Dockerfile" dockerfile.parent.mkdir() From 23e3ca0f780bc16f4c164ea64fc721997392684c Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 20 Oct 2023 14:25:15 +0200 Subject: [PATCH 17/21] Fixed test --- test/integration/test_install_dependencies.py | 71 +++++++++---------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/test/integration/test_install_dependencies.py b/test/integration/test_install_dependencies.py index 4a113925..37a658b7 100644 --- a/test/integration/test_install_dependencies.py +++ b/test/integration/test_install_dependencies.py @@ -29,48 +29,47 @@ ) @pytest.fixture(scope="session") -def dockerfile(tmp_path): - dockerfile = tmp_path / "test_container" / "Dockerfile" - dockerfile.parent.mkdir() +def dockerfile(tmpdir_factory): + dockerfile = tmpdir_factory.mktemp("test_container") / "Dockerfile" with dockerfile.open("w") as f: print(DOCKERFILE_CONTENT, file = f) yield dockerfile @pytest.fixture(scope="session") -def docker_test_container(test_config, dockerfile, tmp_path): - # with tempfile.TemporaryDirectory() as tmp_path: - docker_env = docker.from_env() - # p = Path(__file__).parent / "test_container" - docker_env.images.build( - path=str(dockerfile.parent), - tag=TEST_CONTAINER_IMAGE_TAG - ) - socket_mount = Mount("/var/run/docker.sock", "/var/run/docker.sock", type="bind") - tmp_mount = Mount(tmp_path, tmp_path, type="bind") - mapped_ports = {'8888/tcp': 8888} - test_container = docker_env.containers.create(image=TEST_CONTAINER_IMAGE_TAG, - name=TEST_CONTAINER_NAME, mounts=[socket_mount, tmp_mount], - command="sleep infinity", detach=True, ports=mapped_ports) - test_container.start() - repos = default_repositories + (AnsibleResourceRepository(test.ansible),) - ansible_run_context = \ - AnsibleRunContext(playbook="slc_setup_test.yml", - extra_vars={"test_docker_container": test_container.name, - "slc_dest_folder": f"{tmp_path}/script-languages-release"}) - try: - run_install_dependencies(AnsibleAccess(), configuration=test_config, - host_infos=tuple(), ansible_run_context=ansible_run_context, - ansible_repositories=repos) - yield test_container, tmp_path - # Note: script-languages-release will be cloned by ansible within the docker container. - # Because the docker container runs as root, the repository will be owned by root. - # For simplicity, we delete the folder from within the Docker container (as root). - # Otherwise, we get a permission problem when tmp_path tries to clean-up itself. - finally: - test_container.exec_run(f"rm -rf {tmp_path}/script-languages-release") - test_container.stop() - test_container.remove() +def docker_test_container(test_config, dockerfile): + with tempfile.TemporaryDirectory() as tmp_path: + docker_env = docker.from_env() + # p = Path(__file__).parent / "test_container" + docker_env.images.build( + path=str(dockerfile.parent), + tag=TEST_CONTAINER_IMAGE_TAG + ) + socket_mount = Mount("/var/run/docker.sock", "/var/run/docker.sock", type="bind") + tmp_mount = Mount(tmp_path, tmp_path, type="bind") + mapped_ports = {'8888/tcp': 8888} + test_container = docker_env.containers.create(image=TEST_CONTAINER_IMAGE_TAG, + name=TEST_CONTAINER_NAME, mounts=[socket_mount, tmp_mount], + command="sleep infinity", detach=True, ports=mapped_ports) + test_container.start() + repos = default_repositories + (AnsibleResourceRepository(test.ansible),) + ansible_run_context = \ + AnsibleRunContext(playbook="slc_setup_test.yml", + extra_vars={"test_docker_container": test_container.name, + "slc_dest_folder": f"{tmp_path}/script-languages-release"}) + try: + run_install_dependencies(AnsibleAccess(), configuration=test_config, + host_infos=tuple(), ansible_run_context=ansible_run_context, + ansible_repositories=repos) + yield test_container, tmp_path + # Note: script-languages-release will be cloned by ansible within the docker container. + # Because the docker container runs as root, the repository will be owned by root. + # For simplicity, we delete the folder from within the Docker container (as root). + # Otherwise, we get a permission problem when tmp_path tries to clean-up itself. + finally: + test_container.exec_run(f"rm -rf {tmp_path}/script-languages-release") + test_container.stop() + test_container.remove() def test_install_dependencies_jupyterlab(docker_test_container): From 3f9fd473e2148601c08cc6156d7855ce307ee19e Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 20 Oct 2023 14:31:47 +0200 Subject: [PATCH 18/21] Fixed test --- test/integration/test_install_dependencies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/test_install_dependencies.py b/test/integration/test_install_dependencies.py index 37a658b7..9e782b4f 100644 --- a/test/integration/test_install_dependencies.py +++ b/test/integration/test_install_dependencies.py @@ -29,8 +29,8 @@ ) @pytest.fixture(scope="session") -def dockerfile(tmpdir_factory): - dockerfile = tmpdir_factory.mktemp("test_container") / "Dockerfile" +def dockerfile(tmp_path_factory): + dockerfile = tmp_path_factory.mktemp("test_container") / "Dockerfile" with dockerfile.open("w") as f: print(DOCKERFILE_CONTENT, file = f) yield dockerfile From ae35521cac9874593664a664fbd912487605a098 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 20 Oct 2023 14:49:53 +0200 Subject: [PATCH 19/21] replace test resource file by fixture --- test/integration/test_container/Dockerfile | 4 ---- test/integration/test_install_dependencies.py | 6 ++++-- 2 files changed, 4 insertions(+), 6 deletions(-) delete mode 100644 test/integration/test_container/Dockerfile diff --git a/test/integration/test_container/Dockerfile b/test/integration/test_container/Dockerfile deleted file mode 100644 index ca2f4181..00000000 --- a/test/integration/test_container/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM ubuntu:20.04 -ENV DEBIAN_FRONTEND noninteractive -#AWS AMI Image is based on Ubuntu20.04 server. So we try to simulate same environment here -RUN apt-get update && apt-get install -y ubuntu-server \ No newline at end of file diff --git a/test/integration/test_install_dependencies.py b/test/integration/test_install_dependencies.py index 9e782b4f..76ad1c28 100644 --- a/test/integration/test_install_dependencies.py +++ b/test/integration/test_install_dependencies.py @@ -1,6 +1,7 @@ import tempfile import requests +import shutil import time from pathlib import Path from inspect import cleandoc @@ -19,11 +20,12 @@ TEST_CONTAINER_NAME = "ansible-test" TEST_CONTAINER_IMAGE_TAG = "data_science_sandbox_test_container:latest" +# AWS AMI Image is based on Ubuntu20.04 server. +# So we try to simulate same environment here. DOCKERFILE_CONTENT = cleandoc( """ FROM ubuntu:20.04 ENV DEBIAN_FRONTEND noninteractive - #AWS AMI Image is based on Ubuntu20.04 server. So we try to simulate same environment here RUN apt-get update && apt-get install -y ubuntu-server """ ) @@ -34,13 +36,13 @@ def dockerfile(tmp_path_factory): with dockerfile.open("w") as f: print(DOCKERFILE_CONTENT, file = f) yield dockerfile + shutil.rmtree(str(dockerfile.parent)) @pytest.fixture(scope="session") def docker_test_container(test_config, dockerfile): with tempfile.TemporaryDirectory() as tmp_path: docker_env = docker.from_env() - # p = Path(__file__).parent / "test_container" docker_env.images.build( path=str(dockerfile.parent), tag=TEST_CONTAINER_IMAGE_TAG From c077e7c8de34e0e1e8c7c221e29073450f6c6b88 Mon Sep 17 00:00:00 2001 From: ckunki Date: Fri, 20 Oct 2023 15:31:24 +0200 Subject: [PATCH 20/21] Moved test files --- .github/workflows/check_ci.yaml | 2 +- aws-code-build/ci/buildspec.yaml | 2 +- test/aws/test_deploy_vm_bucket.py | 2 +- test/aws/test_deploy_vm_bucket_waf.py | 2 +- test/aws/test_export_vm.py | 2 +- test/{integration => }/aws_local_stack_access.py | 0 test/{aws => }/aws_mock_data.py | 0 test/{integration => ci}/test_ci.py | 0 test/{integration => ci}/test_install_dependencies.py | 0 test/integration/localstack_test.py | 2 +- test/{unit => integration}/test_aws_codebuild_waiter.py | 0 test/{unit => integration}/test_motd_jupyter_template.py | 6 ++++-- test/{unit => integration}/test_serialization.py | 0 13 files changed, 10 insertions(+), 8 deletions(-) rename test/{integration => }/aws_local_stack_access.py (100%) rename test/{aws => }/aws_mock_data.py (100%) rename test/{integration => ci}/test_ci.py (100%) rename test/{integration => ci}/test_install_dependencies.py (100%) rename test/{unit => integration}/test_aws_codebuild_waiter.py (100%) rename test/{unit => integration}/test_motd_jupyter_template.py (91%) rename test/{unit => integration}/test_serialization.py (100%) diff --git a/.github/workflows/check_ci.yaml b/.github/workflows/check_ci.yaml index a1819a3d..d74bea25 100644 --- a/.github/workflows/check_ci.yaml +++ b/.github/workflows/check_ci.yaml @@ -18,7 +18,7 @@ jobs: uses: ./.github/actions/prepare_poetry_env - name: Run pytest - run: poetry run pytest test/integration/test_install_dependencies.py + run: poetry run pytest test/ci/test_install_dependencies.py env: # Set the secret as an env variable AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }} diff --git a/aws-code-build/ci/buildspec.yaml b/aws-code-build/ci/buildspec.yaml index c2836fee..1b7793a1 100644 --- a/aws-code-build/ci/buildspec.yaml +++ b/aws-code-build/ci/buildspec.yaml @@ -23,4 +23,4 @@ phases: - echo DSS_RUN_CI_TEST is "$DSS_RUN_CI_TEST" #supposed to be true build: commands: - - poetry run python3 -m pytest -s test/test_ci*.py + - poetry run python3 -m pytest -s test/ci/test_ci*.py diff --git a/test/aws/test_deploy_vm_bucket.py b/test/aws/test_deploy_vm_bucket.py index 3d257d70..c19723a1 100644 --- a/test/aws/test_deploy_vm_bucket.py +++ b/test/aws/test_deploy_vm_bucket.py @@ -6,7 +6,7 @@ from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import run_setup_vm_bucket, find_vm_bucket, \ create_vm_bucket_cf_template -from test.aws.aws_mock_data import TEST_BUCKET_ID, get_waf_cloudformation_mock_data, TEST_ACL_ARN, \ +from test.aws_mock_data import TEST_BUCKET_ID, get_waf_cloudformation_mock_data, TEST_ACL_ARN, \ get_s3_cloudformation_mock_data from test.aws.cloudformation_validation import validate_using_cfn_lint from test.mock_cast import mock_cast diff --git a/test/aws/test_deploy_vm_bucket_waf.py b/test/aws/test_deploy_vm_bucket_waf.py index f3d30bd7..d2e03533 100644 --- a/test/aws/test_deploy_vm_bucket_waf.py +++ b/test/aws/test_deploy_vm_bucket_waf.py @@ -6,7 +6,7 @@ from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket_waf import run_setup_vm_bucket_waf, \ find_acl_arn, get_cloudformation_template -from test.aws.aws_mock_data import get_waf_cloudformation_mock_data, TEST_ACL_ARN +from test.aws_mock_data import get_waf_cloudformation_mock_data, TEST_ACL_ARN from test.aws.cloudformation_validation import validate_using_cfn_lint from test.mock_cast import mock_cast diff --git a/test/aws/test_export_vm.py b/test/aws/test_export_vm.py index b854b996..01e00fc1 100644 --- a/test/aws/test_export_vm.py +++ b/test/aws/test_export_vm.py @@ -10,7 +10,7 @@ build_image_destination from exasol.ds.sandbox.lib.export_vm.run_export_vm import export_vm from exasol.ds.sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat -from test.aws.aws_mock_data import get_ami_image_mock_data, TEST_AMI_ID, TEST_ROLE_ID, TEST_BUCKET_ID, INSTANCE_ID, \ +from test.aws_mock_data import get_ami_image_mock_data, TEST_AMI_ID, TEST_ROLE_ID, TEST_BUCKET_ID, INSTANCE_ID, \ get_export_image_task_mock_data, get_s3_cloudformation_mock_data, get_waf_cloudformation_mock_data from test.mock_cast import mock_cast diff --git a/test/integration/aws_local_stack_access.py b/test/aws_local_stack_access.py similarity index 100% rename from test/integration/aws_local_stack_access.py rename to test/aws_local_stack_access.py diff --git a/test/aws/aws_mock_data.py b/test/aws_mock_data.py similarity index 100% rename from test/aws/aws_mock_data.py rename to test/aws_mock_data.py diff --git a/test/integration/test_ci.py b/test/ci/test_ci.py similarity index 100% rename from test/integration/test_ci.py rename to test/ci/test_ci.py diff --git a/test/integration/test_install_dependencies.py b/test/ci/test_install_dependencies.py similarity index 100% rename from test/integration/test_install_dependencies.py rename to test/ci/test_install_dependencies.py diff --git a/test/integration/localstack_test.py b/test/integration/localstack_test.py index 95669087..6c450a39 100644 --- a/test/integration/localstack_test.py +++ b/test/integration/localstack_test.py @@ -5,7 +5,7 @@ CloudformationStackContextManager from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2 from exasol.ds.sandbox.lib.tags import create_default_asset_tag -from test.integration.aws_local_stack_access import AwsLocalStackAccess +from test.aws_local_stack_access import AwsLocalStackAccess def test_ec2_lifecycle_with_local_stack(local_stack, default_asset_id, test_dummy_ami_id): diff --git a/test/unit/test_aws_codebuild_waiter.py b/test/integration/test_aws_codebuild_waiter.py similarity index 100% rename from test/unit/test_aws_codebuild_waiter.py rename to test/integration/test_aws_codebuild_waiter.py diff --git a/test/unit/test_motd_jupyter_template.py b/test/integration/test_motd_jupyter_template.py similarity index 91% rename from test/unit/test_motd_jupyter_template.py rename to test/integration/test_motd_jupyter_template.py index 821fc6ae..f54fa48c 100644 --- a/test/unit/test_motd_jupyter_template.py +++ b/test/integration/test_motd_jupyter_template.py @@ -14,8 +14,10 @@ def motd_file(tmp_path): jupyter_server_config_file = tmp_path / "jupyter_server_config.json" python_file = tmp_path / "999_jupyter.py" - src_path = Path(__file__).parent.parent / "exasol" / "ds" / "sandbox" / "runtime" / \ - "ansible" / "roles" / "jupyter" / "templates" / "etc" /"update-motd.d" / "999-jupyter" + src_path = Path(__file__).parent.parent.parent \ + / "exasol" / "ds" / "sandbox" / "runtime" \ + / "ansible" / "roles" / "jupyter" / "templates" \ + / "etc" /"update-motd.d" / "999-jupyter" with open(src_path, "r") as f: python_code_template = f.read() python_template = Template(python_code_template) diff --git a/test/unit/test_serialization.py b/test/integration/test_serialization.py similarity index 100% rename from test/unit/test_serialization.py rename to test/integration/test_serialization.py From 197ac8be540c1915b669534eaa1b6d81c9757730 Mon Sep 17 00:00:00 2001 From: ckunki Date: Mon, 23 Oct 2023 12:52:33 +0200 Subject: [PATCH 21/21] More renamings after alignment with TK. --- doc/developer_guide/developer_guide.md | 10 ++-------- doc/user_guide/user_guide.md | 2 +- exasol/ds/sandbox/lib/logging.py | 2 +- .../sandbox/lib/vm_bucket/vm_dss_bucket_waf.py | 2 +- .../templates/release_code_build.jinja.yaml | 2 +- .../vm_bucket_cloudformation.jinja.yaml | 2 +- .../templates/waf_cloudformation.jinja.yaml | 16 ++++++++-------- 7 files changed, 15 insertions(+), 21 deletions(-) diff --git a/doc/developer_guide/developer_guide.md b/doc/developer_guide/developer_guide.md index a5691e1e..012c5e3b 100644 --- a/doc/developer_guide/developer_guide.md +++ b/doc/developer_guide/developer_guide.md @@ -128,21 +128,15 @@ aws_access_key_id=... aws_secret_access_key=... ``` -From [product-integration-tool-chest](https://github.com/exasol/product-integration-tool-chest) call -```shell -aws-store-session-token dss_aws_tests -``` +In case your are using MFA authentication please allocate a temporary token. -This will ask you for your MFA code and add or update profile `[dss_aws_tests_mfa]` in file `~/.aws/credentials`. - -Now you can set an environment variable and execute the tests involing AWS resources: +After that you can set an environment variable and execute the tests involving AWS resources: ```shell export AWS_PROFILE=dss_aws_tests_mfa poetry run pytest test/test_deploy_codebuild.py ``` - ### Export The export creates an AMI based on the running EC2 instance and exports the AMI as VM image in the default formats to a S3 bucket. diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index e988f6f9..fccd564d 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -144,7 +144,7 @@ For information about how to build script-languages-container please check: **Location virtual environment**: `/home/ubuntu/jupyterenv` **Location notebooks**: `/home/ubuntu/notebooks` -**Password**: `script-languages` +**Password**: `dss` **Http Port**: 8888 There is a pre-loaded tutorial notebook available which guides you through the usage of `exaslct`: __script-languages.ipynb__. diff --git a/exasol/ds/sandbox/lib/logging.py b/exasol/ds/sandbox/lib/logging.py index e6b0132d..5d54d2a5 100644 --- a/exasol/ds/sandbox/lib/logging.py +++ b/exasol/ds/sandbox/lib/logging.py @@ -20,7 +20,7 @@ class LogType(Enum): def get_status_logger(log_type: LogType) -> logging.Logger: - return logging.getLogger(f"eslds-{log_type.value}") + return logging.getLogger(f"edss-{log_type.value}") def set_log_level(level: str): diff --git a/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py b/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py index fb478075..59f7e703 100644 --- a/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py +++ b/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py @@ -18,7 +18,7 @@ def get_cloudformation_template(allowed_ip: str) -> str: def run_setup_vm_bucket_waf(aws_access: AwsAccess, allowed_ip: str, config: ConfigObject) -> None: """ Deploys the WAF Cloudformation stack. - It automatically deploys to AWS region indicated by config paramaer "waf_region". + It automatically deploys to AWS region indicated by configuration parameter "waf_region". """ yml = get_cloudformation_template(allowed_ip=allowed_ip) diff --git a/exasol/ds/sandbox/templates/release_code_build.jinja.yaml b/exasol/ds/sandbox/templates/release_code_build.jinja.yaml index 47f504bd..43e17b5c 100644 --- a/exasol/ds/sandbox/templates/release_code_build.jinja.yaml +++ b/exasol/ds/sandbox/templates/release_code_build.jinja.yaml @@ -33,7 +33,7 @@ Resources: - cloudformation:DescribeChangeSet - cloudformation:ExecuteChangeSet - cloudformation:DeleteStack - Resource: !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/EC2-SLC-DEV-SANDBOX-*" + Resource: !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/EC2-DATA-SCIENCE-SANDBOX-*" Effect: Allow - Action: - ec2:CreateSecurityGroup diff --git a/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml b/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml index 41257e07..d55a9d8b 100644 --- a/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml +++ b/exasol/ds/sandbox/templates/vm_bucket_cloudformation.jinja.yaml @@ -111,7 +111,7 @@ Resources: Properties: OriginAccessControlConfig: Description: Origin Access Control config for the VM S3 Bucket - Name: VM-Bucket-OAC + Name: DssVMBucketOAC OriginAccessControlOriginType: s3 SigningBehavior: always SigningProtocol: sigv4 diff --git a/exasol/ds/sandbox/templates/waf_cloudformation.jinja.yaml b/exasol/ds/sandbox/templates/waf_cloudformation.jinja.yaml index 43148b3c..5b03725b 100644 --- a/exasol/ds/sandbox/templates/waf_cloudformation.jinja.yaml +++ b/exasol/ds/sandbox/templates/waf_cloudformation.jinja.yaml @@ -2,7 +2,7 @@ Resources: VMDownloadACL: Type: AWS::WAFv2::WebACL Properties: - Name: VMDownloadLimitationWebACL + Name: DssVMDownloadLimitationWebACL Scope: CLOUDFRONT Description: This Web ACL contains rule to control access to the VM Bucket cloudfront distribution. DefaultAction: @@ -10,29 +10,29 @@ Resources: VisibilityConfig: SampledRequestsEnabled: true CloudWatchMetricsEnabled: true - MetricName: MetricForVMDownloads + MetricName: DssMetricForVMDownloads Rules: - - Name: Captcha + - Name: DssVmDownloadCaptcha Priority: 1 Action: Captcha: {} VisibilityConfig: SampledRequestsEnabled: true CloudWatchMetricsEnabled: true - MetricName: CaptchaMetricForVMDownloads + MetricName: DssCaptchaMetricForVMDownloads Statement: NotStatement: Statement: IPSetReferenceStatement: Arn: !GetAtt AllowedIpSet.Arn - - Name: VMBucketRateLimit + - Name: DssVMBucketRateLimit Priority: 0 Action: Block: {} VisibilityConfig: SampledRequestsEnabled: true CloudWatchMetricsEnabled: true - MetricName: RateLimitMetricForVMDownloads + MetricName: DssRateLimitMetricForVMDownloads Statement: RateBasedStatement: AggregateKeyType: IP @@ -40,8 +40,8 @@ Resources: AllowedIpSet: Type: 'AWS::WAFv2::IPSet' Properties: - Description: AllowedIp - Name: AllowedIp + Description: DssAllowedIp + Name: DssAllowedIp Scope: CLOUDFRONT IPAddressVersion: IPV4 Addresses: