diff --git a/doc/developer_guide/developer_guide.md b/doc/developer_guide/developer_guide.md index 03c626cb..495f81e9 100644 --- a/doc/developer_guide/developer_guide.md +++ b/doc/developer_guide/developer_guide.md @@ -119,7 +119,7 @@ The bucket has private access. In order to control access, the Bucket cloudforma The following diagram shows the involved cloudformation stacks: ![image info](./img/cloudformation-stacks.drawio.png) -"DATA-SCIENCE-SANDBOX-VM-SLC-Bucket", "DATA-SCIENCE-SANDBOX-CI-TEST-CODEBUILD" & "DATA-SCIENCE-SANDBOX-RELEASE-CODEBUILD" are permanent and need to be deployed using the "deploy" commands (see [commands](#deployment-commands)). +"DATA-SCIENCE-SANDBOX-VM-Bucket", "DATA-SCIENCE-SANDBOX-CI-TEST-CODEBUILD" & "DATA-SCIENCE-SANDBOX-RELEASE-CODEBUILD" are permanent and need to be deployed using the "deploy" commands (see [commands](#deployment-commands)). The EC2-stack lives only during the creation of a new sandbox image. ## Tagging diff --git a/exasol/ds/sandbox/cli/commands/setup_vm_bucket.py b/exasol/ds/sandbox/cli/commands/setup_vm_bucket.py index 10d376ac..11ccfe4e 100644 --- a/exasol/ds/sandbox/cli/commands/setup_vm_bucket.py +++ b/exasol/ds/sandbox/cli/commands/setup_vm_bucket.py @@ -5,7 +5,7 @@ from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from exasol.ds.sandbox.lib.config import default_config_object from exasol.ds.sandbox.lib.logging import set_log_level -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import run_setup_vm_bucket +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import run_setup_vm_bucket @cli.command() diff --git a/exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py b/exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py index 69a4c236..7426297c 100644 --- a/exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py +++ b/exasol/ds/sandbox/cli/commands/setup_vm_bucket_waf.py @@ -7,7 +7,7 @@ from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from exasol.ds.sandbox.lib.config import default_config_object from exasol.ds.sandbox.lib.logging import set_log_level -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket_waf import run_setup_vm_bucket_waf +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket_waf import run_setup_vm_bucket_waf @cli.command() diff --git a/exasol/ds/sandbox/lib/asset_printing/print_assets.py b/exasol/ds/sandbox/lib/asset_printing/print_assets.py index e3fd72e0..bb0362a6 100644 --- a/exasol/ds/sandbox/lib/asset_printing/print_assets.py +++ b/exasol/ds/sandbox/lib/asset_printing/print_assets.py @@ -14,7 +14,7 @@ from exasol.ds.sandbox.lib.aws_access.cloudformation_stack import CloudformationStack from exasol.ds.sandbox.lib.tags import DEFAULT_TAG_KEY -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket, find_url_for_bucket +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import find_vm_bucket, find_url_for_bucket class AssetTypes(Enum): diff --git a/exasol/ds/sandbox/lib/aws_access/deployer.py b/exasol/ds/sandbox/lib/aws_access/deployer.py index ef643943..6df0903f 100644 --- a/exasol/ds/sandbox/lib/aws_access/deployer.py +++ b/exasol/ds/sandbox/lib/aws_access/deployer.py @@ -28,10 +28,12 @@ "ChangeSetResult", ["changeset_id", "changeset_type"]) +DEFAULT_CHANGE_SET_PREFIX="dss-ci-setup-deploy-" + + class Deployer(object): - def __init__(self, cloudformation_client, - changeset_prefix="slc-ci-setup-deploy-"): + def __init__(self, cloudformation_client, changeset_prefix=DEFAULT_CHANGE_SET_PREFIX): self._client = cloudformation_client self.changeset_prefix = changeset_prefix diff --git a/exasol/ds/sandbox/lib/export_vm/run_export_vm.py b/exasol/ds/sandbox/lib/export_vm/run_export_vm.py index e55634c4..cdd49181 100644 --- a/exasol/ds/sandbox/lib/export_vm/run_export_vm.py +++ b/exasol/ds/sandbox/lib/export_vm/run_export_vm.py @@ -11,7 +11,7 @@ from exasol.ds.sandbox.lib.setup_ec2.cf_stack import find_ec2_instance_in_cf_stack from exasol.ds.sandbox.lib.asset_printing.print_assets import print_assets from exasol.ds.sandbox.lib.export_vm.vm_disk_image_format import VmDiskImageFormat -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket, find_vm_import_role +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import find_vm_bucket, find_vm_import_role LOG = get_status_logger(LogType.EXPORT) diff --git a/exasol/ds/sandbox/lib/run_create_vm.py b/exasol/ds/sandbox/lib/run_create_vm.py index 64810f9f..6ee5e4e5 100644 --- a/exasol/ds/sandbox/lib/run_create_vm.py +++ b/exasol/ds/sandbox/lib/run_create_vm.py @@ -36,7 +36,8 @@ def run_create_vm(aws_access: AwsAccess, ec2_key_file: Optional[str], ec2_key_na ansible_repositories: Tuple[AnsibleRepository, ...] = default_repositories) -> None: """ Runs setup of an EC2 instance and then installs all dependencies via Ansible, - and finally exports the VM to the S3 Bucket (which must be already created by the stack ("VM-SLC-Bucket"). + and finally exports the VM to the S3 Bucket (which must be already created + by the stack "DATA-SCIENCE-SANDBOX-VM-Bucket"). If anything goes wrong the cloudformation stack of the EC-2 instance will be removed. For debuging you can use the available debug commands. """ diff --git a/exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py b/exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py index 21e871fe..d8112607 100644 --- a/exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py +++ b/exasol/ds/sandbox/lib/setup_ci_codebuild/ci_codebuild.py @@ -1,7 +1,7 @@ from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from exasol.ds.sandbox.lib.logging import get_status_logger, LogType from exasol.ds.sandbox.lib.render_template import render_template -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import find_vm_bucket STACK_NAME = "DATA-SCIENCE-SANDBOX-CI-TEST-CODEBUILD" diff --git a/exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py b/exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py index 28d50f2e..5fb3beba 100644 --- a/exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py +++ b/exasol/ds/sandbox/lib/setup_release_codebuild/release_codebuild.py @@ -1,7 +1,7 @@ from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess from exasol.ds.sandbox.lib.logging import get_status_logger, LogType from exasol.ds.sandbox.lib.render_template import render_template -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import find_vm_bucket +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import find_vm_bucket RELEASE_CODE_BUILD_STACK_NAME = "DATA-SCIENCE-SANDBOX-RELEASE-CODEBUILD" diff --git a/exasol/ds/sandbox/lib/tags.py b/exasol/ds/sandbox/lib/tags.py index 129c6d89..0b4100d0 100644 --- a/exasol/ds/sandbox/lib/tags.py +++ b/exasol/ds/sandbox/lib/tags.py @@ -1,4 +1,4 @@ -DEFAULT_TAG_KEY = "exa_slc_id" +DEFAULT_TAG_KEY = "exa_dss_id" def create_default_asset_tag(value: str) -> list: diff --git a/exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket.py b/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket.py similarity index 95% rename from exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket.py rename to exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket.py index 71782e71..8bd5864f 100644 --- a/exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket.py +++ b/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket.py @@ -4,9 +4,9 @@ from exasol.ds.sandbox.lib.render_template import render_template from enum import Enum -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket_waf import find_acl_arn +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket_waf import find_acl_arn -STACK_NAME = "DATA-SCIENCE-SANDBOX-VM-SLC-Bucket" +STACK_NAME = "DATA-SCIENCE-SANDBOX-VM-Bucket" class OutputKey(Enum): diff --git a/exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket_waf.py b/exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py similarity index 100% rename from exasol/ds/sandbox/lib/vm_bucket/vm_slc_bucket_waf.py rename to exasol/ds/sandbox/lib/vm_bucket/vm_dss_bucket_waf.py diff --git a/test/aws_mock_data.py b/test/aws_mock_data.py index 8f8a99a5..e9409a3a 100644 --- a/test/aws_mock_data.py +++ b/test/aws_mock_data.py @@ -12,8 +12,8 @@ from exasol.ds.sandbox.lib.aws_access.stack_resource import StackResource from test.conftest import DEFAULT_ASSET_ID -TEST_ROLE_ID = 'VM-SLC-Bucket-VMImportRole-TEST' -TEST_BUCKET_ID = 'vm-slc-bucket-vmslcbucket-TEST' +TEST_ROLE_ID = 'VM-DSS-Bucket-VMImportRole-TEST' +TEST_BUCKET_ID = 'vm-dss-bucket-vmdssbucket-TEST' TEST_AMI_ID = "AMI-IMAGE-12345" TEST_CLOUDFRONT_ID = "test-cloudfrontet-TEST" TEST_CLOUDFRONT_DOMAIN_NAME = "test-s3.cloudfront.net" @@ -43,7 +43,7 @@ def get_ami_image_mock_data(state: str) -> Ami: 'Name': DEFAULT_ASSET_ID.ami_name, 'RootDeviceName': '/dev/sda1', 'RootDeviceType': 'ebs', 'SriovNetSupport': 'simple', - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'VirtualizationType': 'hvm'}) @@ -53,7 +53,7 @@ def get_snapshot_mock_data(): 'OwnerId': '123', 'Progress': '100%', 'SnapshotId': 'snap-123', 'StartTime': datetime.datetime(2022, 8, 16, 15, 3, 40, 662000, tzinfo=tzutc()), 'State': 'completed', 'VolumeId': 'vol-123', 'VolumeSize': 100, - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'StorageTier': 'standard' }) @@ -71,7 +71,7 @@ def get_export_image_task_mock_data(in_progress: bool): 'Progress': "50%", "StatusMessage": "creating the image", 'Status': 'active', - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}] + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}] }) else: return ExportImageTask({ @@ -83,7 +83,7 @@ def get_export_image_task_mock_data(in_progress: bool): 'S3Prefix': DEFAULT_ASSET_ID.bucket_prefix }, 'Status': 'completed', - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}] + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}] }) @@ -107,7 +107,7 @@ def get_ec2_cloudformation_mock_data(): 'RollbackConfiguration': {}, 'StackStatus': 'CREATE_COMPLETE', 'DisableRollback': False, 'NotificationARNs': [], 'Capabilities': ['CAPABILITY_IAM'], - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'DriftInformation': {'StackDriftStatus': 'NOT_CHECKED'} }) @@ -137,7 +137,7 @@ def get_ec2_key_pair_mock_data(): 'KeyFingerprint': '12:34:56:78:90:12:34:56:78:90:12:34:56:78:90:12:34:56:78:90', 'KeyName': 'ec2-key-test-key', 'KeyType': 'rsa', - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'CreateTime': datetime.datetime(2022, 8, 16, 15, 30, 41, tzinfo=tzutc()) }) @@ -146,14 +146,14 @@ def get_ec2_key_pair_mock_data(): def get_s3_cloudformation_mock_data() -> List[CloudformationStack]: return [CloudformationStack({ 'StackId': 'test-s3-stack-id', - 'StackName': "DATA-SCIENCE-SANDBOX-VM-SLC-Bucket", + 'StackName': "DATA-SCIENCE-SANDBOX-VM-Bucket", 'ChangeSetId': 'test-stack-changeset-id-2', 'CreationTime': datetime.datetime(2022, 8, 16, 14, 30, 45, 559000, tzinfo=tzutc()), 'LastUpdatedTime': datetime.datetime(2022, 8, 16, 14, 30, 51, 667000, tzinfo=tzutc()), 'RollbackConfiguration': {}, 'StackStatus': 'CREATE_COMPLETE', 'DisableRollback': False, 'NotificationARNs': [], 'Capabilities': ['CAPABILITY_IAM'], - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'DriftInformation': {'StackDriftStatus': 'NOT_CHECKED'}, 'Outputs': [{'OutputKey': 'VMBucketId', 'OutputValue': TEST_BUCKET_ID, 'Description': ''}, @@ -171,14 +171,14 @@ def get_s3_cloudformation_mock_data() -> List[CloudformationStack]: def get_waf_cloudformation_mock_data() -> List[CloudformationStack]: return [CloudformationStack({ 'StackId': 'test-waf-stack-id', - 'StackName': "DATA-SCIENCE-SANDBOX-VM-SLC-Bucket-WAF", + 'StackName': "DATA-SCIENCE-SANDBOX-VM-Bucket-WAF", 'ChangeSetId': 'test-stack-changeset-id-3', 'CreationTime': datetime.datetime(2022, 8, 16, 14, 30, 45, 559000, tzinfo=tzutc()), 'LastUpdatedTime': datetime.datetime(2022, 8, 16, 14, 30, 51, 667000, tzinfo=tzutc()), 'RollbackConfiguration': {}, 'StackStatus': 'CREATE_COMPLETE', 'DisableRollback': False, 'NotificationARNs': [], 'Capabilities': [], - 'Tags': [{'Key': 'exa_slc_id', 'Value': DEFAULT_ASSET_ID.tag_value}], + 'Tags': [{'Key': 'exa_dss_id', 'Value': DEFAULT_ASSET_ID.tag_value}], 'DriftInformation': {'StackDriftStatus': 'NOT_CHECKED'}, 'Outputs': [{'OutputKey': 'VMDownloadACLArn', 'OutputValue': TEST_ACL_ARN, 'Description': ''} diff --git a/test/cloudformation_validation.py b/test/cloudformation_validation.py index 110f0b55..7a30153a 100644 --- a/test/cloudformation_validation.py +++ b/test/cloudformation_validation.py @@ -6,7 +6,7 @@ def validate_using_cfn_lint(tmp_path, cloudformation_yml): This test uses cfn-lint to validate the Cloudformation template. (See https://github.com/aws-cloudformation/cfn-lint) """ - out_file = tmp_path / "slc_source_credential.yaml" + out_file = tmp_path / "cloudformation.yaml" with open(out_file, "w") as f: f.write(cloudformation_yml) diff --git a/test/test_deploy_vm_bucket.py b/test/test_deploy_vm_bucket.py index 0dccfcd4..7d40f799 100644 --- a/test/test_deploy_vm_bucket.py +++ b/test/test_deploy_vm_bucket.py @@ -4,13 +4,13 @@ import pytest from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import run_setup_vm_bucket, find_vm_bucket, \ +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import run_setup_vm_bucket, find_vm_bucket, \ create_vm_bucket_cf_template from test.aws_mock_data import TEST_BUCKET_ID, get_waf_cloudformation_mock_data, TEST_ACL_ARN, \ get_s3_cloudformation_mock_data from test.cloudformation_validation import validate_using_cfn_lint from test.mock_cast import mock_cast -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket import STACK_NAME as VM_STACK_NAME +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import STACK_NAME as VM_STACK_NAME @pytest.fixture diff --git a/test/test_deploy_vm_bucket_waf.py b/test/test_deploy_vm_bucket_waf.py index 03804886..e9e37abf 100644 --- a/test/test_deploy_vm_bucket_waf.py +++ b/test/test_deploy_vm_bucket_waf.py @@ -4,7 +4,7 @@ import pytest from exasol.ds.sandbox.lib.aws_access.aws_access import AwsAccess -from exasol.ds.sandbox.lib.vm_bucket.vm_slc_bucket_waf import run_setup_vm_bucket_waf, \ +from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket_waf import run_setup_vm_bucket_waf, \ find_acl_arn, get_cloudformation_template from test.aws_mock_data import get_waf_cloudformation_mock_data, TEST_ACL_ARN from test.cloudformation_validation import validate_using_cfn_lint