split AWS tests into unit and integration tests

doc/changes/changes_1.1.0.md

@@ -35,3 +35,4 @@ n/a
 * #220: Changed default ports in the external database configuration.
 * #221: Changed wording in the main configuration notebook, as suggested by PM.
 * #66: Used a non-root user to run Jupyter in the Docker Image ai-lab
+* #149: Split AWS tests

test/aws/fixtures.py

@@ -0,0 +1,78 @@
+import pytest
+
+from exasol.ds.sandbox.lib.tags import DEFAULT_TAG_KEY
+from exasol.ds.sandbox.lib.asset_id import AssetId
+from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket import create_vm_bucket_cf_template
+from exasol.ds.sandbox.lib.vm_bucket.vm_dss_bucket_waf import get_cloudformation_template
+from exasol.ds.sandbox.lib.render_template import render_template
+from test.aws.local_stack_access import AwsLocalStackAccess
+
+TEST_DUMMY_AMI_ID = "ami-123"
+DEFAULT_ASSET_ID = AssetId("test", stack_prefix="test-stack", ami_prefix="test-ami")
+TEST_ACL_ARN = "TEST-DOWNLOAD-ACL"
+TEST_IP = "1.1.1.1"
+
+
+@pytest.fixture
+def default_asset_id():
+    return DEFAULT_ASSET_ID
+
+
+@pytest.fixture()
+def test_dummy_ami_id():
+    return TEST_DUMMY_AMI_ID
+
+
+def vm_bucket_template():
+    return create_vm_bucket_cf_template(TEST_ACL_ARN)
+
+
+@pytest.fixture
+def vm_bucket_cloudformation_yml():
+    return vm_bucket_template()
+
+
+def ec2_template():
+    return render_template(
+        "ec2_cloudformation.jinja.yaml",
+        key_name="test_key",
+        user_name="test_user",
+        trace_tag=DEFAULT_TAG_KEY,
+        trace_tag_value=DEFAULT_ASSET_ID.tag_value,
+        ami_id=TEST_DUMMY_AMI_ID,
+    )
+
+
+@pytest.fixture
+def ec2_cloudformation_yml():
+    return ec2_template()
+
+
+def waf_template():
+    return get_cloudformation_template(TEST_IP)
+
+
+@pytest.fixture
+def waf_cloudformation_yml():
+    return waf_template()
+
+
+def ci_codebuild_template():
+    return render_template(
+        "ci_code_build.jinja.yaml",
+        vm_bucket="test-bucket-123",
+    )
+
+
+def release_codebuild_template():
+    return render_template(
+        "release_code_build.jinja.yaml",
+        vm_bucket="test-bucket-123",
+        path_in_bucket=AssetId.BUCKET_PREFIX,
+        dockerhub_secret_arn="secret_arn",
+    )
+
+
+@pytest.fixture(scope="session")
+def local_stack_aws_access(local_stack):
+    return AwsLocalStackAccess().with_user("default_user")

test/aws/localstack.py

@@ -0,0 +1,39 @@
+import os
+import pytest
+import subprocess
+import shlex
+
+from importlib.metadata import version
+
+@pytest.fixture(scope="session")
+def local_stack():
+    """
+    This fixture starts/stops localstack as a context manager.
+    """
+    command = "localstack start -d"
+
+    image_version = version('localstack')
+    # See https://github.com/localstack/localstack/issues/8254
+    # and https://github.com/localstack/localstack/issues/9939
+    #
+    # Until an official release of localstack Docker image with a concrete
+    # version is available incl. a fix for issue 9939 we only can use version
+    # "latest".
+    #
+    # See https://github.com/exasol/ai-lab/issues/200 for replacing this with
+    # a concrete version in order to make CI tests more robust.
+    image_version = "3.2.0"
+    image_name = {"IMAGE_NAME": f"localstack/localstack:{image_version}"}
+    env_variables = {**os.environ, **image_name}
+
+    process = subprocess.run(shlex.split(command), env=env_variables)
+    assert process.returncode == 0
+
+    command = "localstack wait -t 30"
+
+    process = subprocess.run(shlex.split(command), env=env_variables)
+    assert process.returncode == 0
+    yield None
+
+    command = "localstack stop"
+    subprocess.run(shlex.split(command), env=env_variables)

test/aws_mock_data.py → test/aws/mock_data.py

@@ -11,14 +11,15 @@
 from exasol.ds.sandbox.lib.aws_access.snapshot import Snapshot
 from exasol.ds.sandbox.lib.aws_access.stack_resource import StackResource
 from exasol.ds.sandbox.lib.tags import create_default_asset_tag
-from test.conftest import DEFAULT_ASSET_ID
+from test.aws.fixtures import DEFAULT_ASSET_ID
+from test.aws.fixtures import TEST_ACL_ARN
 
 TEST_ROLE_ID = 'VM-DSS-Bucket-VMImportRole-TEST'
 TEST_BUCKET_ID = 'vm-dss-bucket-vmdssbucket-TEST'
 TEST_AMI_ID = "AMI-IMAGE-12345"
 TEST_CLOUDFRONT_ID = "test-cloudfrontet-TEST"
 TEST_CLOUDFRONT_DOMAIN_NAME = "test-s3.cloudfront.net"
-TEST_ACL_ARN = "TEST-DOWNLOAD-ACL"
+# TEST_ACL_ARN = "TEST-DOWNLOAD-ACL"
 INSTANCE_ID = "test-instance"
 
 

test/conftest.py

@@ -10,80 +10,41 @@
 from importlib.metadata import version
 from exasol.ds.sandbox.lib.tags import DEFAULT_TAG_KEY
 from exasol.ds.sandbox.lib.asset_id import AssetId
-from test.aws_local_stack_access import AwsLocalStackAccess
+from test.aws.local_stack_access import AwsLocalStackAccess
 
-DEFAULT_ASSET_ID = AssetId("test", stack_prefix="test-stack", ami_prefix="test-ami")
-
-TEST_DUMMY_AMI_ID = "ami-123"
+# DEFAULT_ASSET_ID = AssetId("test", stack_prefix="test-stack", ami_prefix="test-ami")
+# 
+# TEST_DUMMY_AMI_ID = "ami-123"
 
 pytest_plugins = (
     "test.docker.dss_docker_image",
 )
 
 
-@pytest.fixture
-def default_asset_id():
-    return DEFAULT_ASSET_ID
+# @pytest.fixture
+# def default_asset_id():
+#     return DEFAULT_ASSET_ID
 
 
 @pytest.fixture
 def jupyter_port():
     return 49494
 
 
-@pytest.fixture
-def ec2_cloudformation_yml():
-    return render_template("ec2_cloudformation.jinja.yaml", key_name="test_key", user_name="test_user",
-                           trace_tag=DEFAULT_TAG_KEY, trace_tag_value=DEFAULT_ASSET_ID.tag_value,
-                           ami_id=TEST_DUMMY_AMI_ID)
-
-
-@pytest.fixture(scope="session")
-def local_stack():
-    """
-    This fixture starts/stops localstack as a context manager.
-    """
-    command = "localstack start -d"
-
-    image_version = version('localstack')
-    # See https://github.com/localstack/localstack/issues/8254
-    # and https://github.com/localstack/localstack/issues/9939
-    #
-    # Until an official release of localstack Docker image with a concrete
-    # version is available incl. a fix for issue 9939 we only can use version
-    # "latest".
-    #
-    # See ai-lab issue for replacing this with a concrete version in order to
-    # make CI tests more robust.
-    image_version = "latest"
-    image_name = {"IMAGE_NAME": f"localstack/localstack:{image_version}"}
-    env_variables = {**os.environ, **image_name}
-
-    process = subprocess.run(shlex.split(command), env=env_variables)
-    assert process.returncode == 0
-
-    command = "localstack wait -t 30"
-
-    process = subprocess.run(shlex.split(command), env=env_variables)
-    assert process.returncode == 0
-    yield None
-
-    command = "localstack stop"
-    subprocess.run(shlex.split(command), env=env_variables)
-
-
-@pytest.fixture(scope="session")
-def local_stack_aws_access(local_stack):
-    return AwsLocalStackAccess().with_user("default_user")
+# @pytest.fixture
+# def ec2_cloudformation_yml():
+#     return render_template(
+#         "ec2_cloudformation.jinja.yaml",
+#         key_name="test_key",
+#         user_name="test_user",
+#         trace_tag=DEFAULT_TAG_KEY,
+#         trace_tag_value=DEFAULT_ASSET_ID.tag_value,
+#         ami_id=TEST_DUMMY_AMI_ID,
+#     )
 
 
 @pytest.fixture(scope="session")
 def test_config():
     test_config = copy(default_config_object)
     test_config.time_to_wait_for_polling = 0.1
     return test_config
-
-
-@pytest.fixture()
-def test_dummy_ami_id():
-    return TEST_DUMMY_AMI_ID

test/integration/localstack_test.py → test/integration/aws/localstack_test.py

@@ -1,13 +1,27 @@
 import botocore
 import pytest
 
-from exasol.ds.sandbox.lib.setup_ec2.cf_stack import CloudformationStack, \
-    CloudformationStackContextManager
+from exasol.ds.sandbox.lib.setup_ec2.cf_stack import (
+    CloudformationStack,
+    CloudformationStackContextManager,
+)
 from exasol.ds.sandbox.lib.setup_ec2.run_setup_ec2 import run_lifecycle_for_ec2
 from exasol.ds.sandbox.lib.tags import create_default_asset_tag
-
-
-def test_ec2_lifecycle_with_local_stack(local_stack_aws_access, default_asset_id, test_dummy_ami_id):
+from test.aws.fixtures import (
+    default_asset_id,
+    test_dummy_ami_id,
+    ec2_cloudformation_yml,
+    local_stack_aws_access,
+)
+from test.aws.localstack import (
+    local_stack,
+)
+
+def test_ec2_lifecycle_with_local_stack(
+        local_stack_aws_access,
+        default_asset_id,
+        test_dummy_ami_id,
+):
     """
     This test uses localstack to simulate lifecycle of an EC-2 instance
     """
@@ -36,7 +50,11 @@ def test_ec2_manage_keypair_with_local_stack(local_stack_aws_access, default_ass
     aws.delete_ec2_key_pair("test")
 
 
-def test_cloudformation_with_localstack(default_asset_id, local_stack_aws_access, ec2_cloudformation_yml):
+def test_cloudformation_with_localstack(
+        default_asset_id,
+        local_stack_aws_access,
+        ec2_cloudformation_yml,
+):
     aws = local_stack_aws_access
     aws.upload_cloudformation_stack(
         ec2_cloudformation_yml, stack_name="test_stack",