Merge branch 'main' into ctr32-armv8

aws · Jul 29, 2024 · 55c0ac8 · 55c0ac8
2 parents 9ee3377 + 916b3d1
commit 55c0ac8
Show file tree

Hide file tree

Showing 14 changed files with 565 additions and 258 deletions.
diff --git a/tests/ci/cdk/cdk/aws_lc_analytics_stack.py b/tests/ci/cdk/cdk/aws_lc_analytics_stack.py
@@ -55,4 +55,4 @@ def __init__(self,
             build_spec=BuildSpecLoader.load(spec_file_path))
         analytics.enable_batch_builds()
 
-        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=analytics)
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=analytics, ec2_permissions=False)
diff --git a/tests/ci/cdk/cdk/aws_lc_android_ci_stack.py b/tests/ci/cdk/cdk/aws_lc_android_ci_stack.py
@@ -65,4 +65,4 @@ def __init__(self,
             build_spec=BuildSpecLoader.load(spec_file_path))
         project.enable_batch_builds()
 
-        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=False)
diff --git a/tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py b/tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py
@@ -108,7 +108,7 @@ def __init__(self,
             })
         project.enable_batch_builds()
 
-        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=True)
 
         # Define logs for SSM.
         log_group_name = "{}-cw-logs".format(id)

diff --git a/tests/ci/cdk/cdk/aws_lc_github_ci_stack.py b/tests/ci/cdk/cdk/aws_lc_github_ci_stack.py
@@ -84,4 +84,4 @@ def __init__(self,
         cfn_project.add_property_override("ResourceAccessRole", resource_access_role.role_arn)
         project.enable_batch_builds()
 
-        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=False)
diff --git a/tests/ci/cdk/cdk/aws_lc_github_fuzz_ci_stack.py b/tests/ci/cdk/cdk/aws_lc_github_fuzz_ci_stack.py
@@ -128,4 +128,4 @@ def __init__(self,
           "Type": "EFS"
         }])
 
-        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=fuzz_codebuild)
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=fuzz_codebuild, ec2_permissions=False)
diff --git a/tests/ci/cdk/cdk/bm_framework_stack.py b/tests/ci/cdk/cdk/bm_framework_stack.py
@@ -71,7 +71,7 @@ def __init__(self,
             build_spec=BuildSpecLoader.load(spec_file_path))
         project.enable_batch_builds()
 
-        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=False)
 
         # use boto3 to determine if a cloudwatch logs group with the name we want exists, and if it doesn't, create it
         logs_client = boto3.client('logs', region_name=AWS_REGION)

diff --git a/tests/ci/cdk/cdk/codebuild/ec2_test_framework_omnibus.yaml b/tests/ci/cdk/cdk/codebuild/ec2_test_framework_omnibus.yaml
@@ -7,7 +7,7 @@ version: 0.2
 batch:
   build-list:
     # Actual tests are ran on an Graviton2 ec2 instance via SSM Commands.
-    - identifier: graviton2_tests
+    - identifier: graviton2_tests_asan
       buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml
       env:
         type: LINUX_CONTAINER
@@ -18,3 +18,44 @@ batch:
           EC2_AMI: "ami-0c29a2c5cf69b5a9c"
           EC2_INSTANCE_TYPE: "c6g.2xlarge"
           ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer"
+          TARGET_TEST_SCRIPT: "./tests/ci/run_posix_sanitizers.sh"
+
+    - identifier: graviton2_tests_fips
+      buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml
+      env:
+        type: LINUX_CONTAINER
+        privileged-mode: false
+        compute-type: BUILD_GENERAL1_SMALL
+        image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-20.04_clang-7x-bm-framework_latest
+        variables:
+          EC2_AMI: "ami-0c29a2c5cf69b5a9c"
+          EC2_INSTANCE_TYPE: "c6g.2xlarge"
+          ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer"
+          TARGET_TEST_SCRIPT: "./tests/ci/run_fips_tests.sh"
+
+    # Actual tests are ran on an Graviton4 ec2 instance via SSM Commands.
+    - identifier: graviton4_tests_asan
+      buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml
+      env:
+        type: LINUX_CONTAINER
+        privileged-mode: false
+        compute-type: BUILD_GENERAL1_SMALL
+        image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-20.04_clang-7x-bm-framework_latest
+        variables:
+          EC2_AMI: "ami-0c29a2c5cf69b5a9c"
+          EC2_INSTANCE_TYPE: "r8g.2xlarge"
+          ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer"
+          TARGET_TEST_SCRIPT: "./tests/ci/run_posix_sanitizers.sh"
+
+    - identifier: graviton4_tests_fips
+      buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml
+      env:
+        type: LINUX_CONTAINER
+        privileged-mode: false
+        compute-type: BUILD_GENERAL1_SMALL
+        image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-20.04_clang-7x-bm-framework_latest
+        variables:
+          EC2_AMI: "ami-0c29a2c5cf69b5a9c"
+          EC2_INSTANCE_TYPE: "r8g.2xlarge"
+          ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer"
+          TARGET_TEST_SCRIPT: "./tests/ci/run_fips_tests.sh"
diff --git a/tests/ci/cdk/cdk/components.py b/tests/ci/cdk/cdk/components.py
@@ -4,11 +4,11 @@
     aws_events as events, aws_events_targets as events_targets, aws_iam as iam, Duration
 
 from constructs import Construct
-from util.metadata import GITHUB_REPO_OWNER, GITHUB_TOKEN_SECRET_NAME
+from util.metadata import AWS_REGION, AWS_ACCOUNT, GITHUB_REPO_OWNER, GITHUB_TOKEN_SECRET_NAME
 
 
 class PruneStaleGitHubBuilds(Construct):
-    def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject) -> None:
+    def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject, ec2_permissions: bool) -> None:
         super().__init__(scope, id)
 
         github_token_secret = sm.Secret.from_secret_name_v2(scope=self,
@@ -36,12 +36,33 @@ def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject) ->
                                 actions=[
                                     "codebuild:BatchGetBuildBatches",
                                     "codebuild:ListBuildBatchesForProject",
-                                    "codebuild:StopBuildBatch",
+                                    "codebuild:StopBuildBatch"
                                 ],
                                 resources=[project.project_arn]))
 
+        if ec2_permissions:
+            lambda_function.add_to_role_policy(
+                iam.PolicyStatement(effect=iam.Effect.ALLOW,
+                                actions=[
+                                    "ec2:TerminateInstances",
+                                ],
+                                resources=["arn:aws:ec2:{}:{}:instance/*".format(AWS_REGION, AWS_ACCOUNT)],
+                                conditions={
+                                    "StringEquals": {
+                                        "ec2:ResourceTag/ec2-framework-host": "ec2-framework-host"
+                                    }
+                                }))
+            # ec2:Describe* API actions do not support resource-level permissions.
+            lambda_function.add_to_role_policy(
+                iam.PolicyStatement(effect=iam.Effect.ALLOW,
+                                    actions=[
+                                        "ec2:DescribeInstances",
+                                    ],
+                                    resources=["*"]))
+
+
         events.Rule(scope=self, id="PurgeEventRule",
-                    description="Purge stale GitHub codebuild jobs (once per minute)",
+                    description="Purge stale GitHub codebuild jobs and ec2 instances (once per minute)",
                     enabled=True,
                     schedule=events.Schedule.rate(Duration.minutes(1)),
                     targets=[events_targets.LambdaFunction(handler=lambda_function)])
diff --git a/tests/ci/cdk/cdk/ssm/general_test_run_ssm_document.yaml b/tests/ci/cdk/cdk/ssm/general_test_run_ssm_document.yaml
@@ -59,13 +59,8 @@ mainSteps:
         # Check if container was spun up succesfully. Then run test scripts and check the output.
         - >
           if [ -n "$exec_docker" ]; then
-            chmod +x ./tests/ci/run_posix_sanitizers.sh
-            $exec_docker ./tests/ci/run_posix_sanitizers.sh
-            if [ $? != 0 ]; then
-              exit 1
-            fi
-            chmod +x ./tests/ci/run_fips_tests.sh
-            $exec_docker ./tests/ci/run_fips_tests.sh
+            chmod +x {TARGET_TEST_SCRIPT}
+            $exec_docker {TARGET_TEST_SCRIPT}
             if [ $? != 0 ]; then
               exit 1
             fi

diff --git a/tests/ci/codebuild/common/run_ec2_target.yml b/tests/ci/codebuild/common/run_ec2_target.yml
@@ -10,4 +10,4 @@ env:
 phases:
   build:
     commands:
-      - ./tests/ci/run_ec2_test_framework.sh "${EC2_AMI}" "${EC2_INSTANCE_TYPE}" "${ECR_DOCKER_TAG}"
+      - ./tests/ci/run_ec2_test_framework.sh "${EC2_AMI}" "${EC2_INSTANCE_TYPE}" "${ECR_DOCKER_TAG}" "${TARGET_TEST_SCRIPT}"