From 916b3d1d25ce6939f30ab0526fa02433e76fa40b Mon Sep 17 00:00:00 2001 From: Samuel Chiang Date: Fri, 26 Jul 2024 16:24:42 -0700 Subject: [PATCH] ec2-test-framework enhancements and graviton 4 testing (#1715) 1. Some of our ec2 instances from the ec2-test-framework were failing to be properly stopped. This adds logic to the job pruner to stop hanging instances. 2. Also parallelizing the test runs. GV2 is significantly slower which causes the run to be much longer. This also makes it much easier for us to extend testing against new test scripts. 3. Also added graviton 4 testing to the ec2-test-framework. By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license and the ISC license. --- tests/ci/cdk/cdk/aws_lc_analytics_stack.py | 2 +- tests/ci/cdk/cdk/aws_lc_android_ci_stack.py | 2 +- .../cdk/aws_lc_ec2_test_framework_ci_stack.py | 2 +- tests/ci/cdk/cdk/aws_lc_github_ci_stack.py | 2 +- .../ci/cdk/cdk/aws_lc_github_fuzz_ci_stack.py | 2 +- tests/ci/cdk/cdk/bm_framework_stack.py | 2 +- .../codebuild/ec2_test_framework_omnibus.yaml | 43 +- tests/ci/cdk/cdk/components.py | 29 +- .../ssm/general_test_run_ssm_document.yaml | 9 +- tests/ci/codebuild/common/run_ec2_target.yml | 2 +- tests/ci/lambda/Cargo.lock | 622 ++++++++++++------ tests/ci/lambda/Cargo.toml | 7 +- tests/ci/lambda/src/bin/purge-stale-builds.rs | 82 ++- tests/ci/run_ec2_test_framework.sh | 17 +- 14 files changed, 565 insertions(+), 258 deletions(-) diff --git a/tests/ci/cdk/cdk/aws_lc_analytics_stack.py b/tests/ci/cdk/cdk/aws_lc_analytics_stack.py index d8f847fbf6..216c33e4db 100644 --- a/tests/ci/cdk/cdk/aws_lc_analytics_stack.py +++ b/tests/ci/cdk/cdk/aws_lc_analytics_stack.py @@ -55,4 +55,4 @@ def __init__(self, build_spec=BuildSpecLoader.load(spec_file_path)) analytics.enable_batch_builds() - PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=analytics) + PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=analytics, ec2_permissions=False) diff --git a/tests/ci/cdk/cdk/aws_lc_android_ci_stack.py b/tests/ci/cdk/cdk/aws_lc_android_ci_stack.py index f72a325c9b..9a3fc00a61 100644 --- a/tests/ci/cdk/cdk/aws_lc_android_ci_stack.py +++ b/tests/ci/cdk/cdk/aws_lc_android_ci_stack.py @@ -65,4 +65,4 @@ def __init__(self, build_spec=BuildSpecLoader.load(spec_file_path)) project.enable_batch_builds() - PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project) + PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=False) diff --git a/tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py b/tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py index 6bdc980d57..0dccf5b02a 100644 --- a/tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py +++ b/tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py @@ -108,7 +108,7 @@ def __init__(self, }) project.enable_batch_builds() - PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project) + PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=True) # Define logs for SSM. log_group_name = "{}-cw-logs".format(id) diff --git a/tests/ci/cdk/cdk/aws_lc_github_ci_stack.py b/tests/ci/cdk/cdk/aws_lc_github_ci_stack.py index ac55714e79..f3a262af3d 100644 --- a/tests/ci/cdk/cdk/aws_lc_github_ci_stack.py +++ b/tests/ci/cdk/cdk/aws_lc_github_ci_stack.py @@ -84,4 +84,4 @@ def __init__(self, cfn_project.add_property_override("ResourceAccessRole", resource_access_role.role_arn) project.enable_batch_builds() - PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project) + PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=False) diff --git a/tests/ci/cdk/cdk/aws_lc_github_fuzz_ci_stack.py b/tests/ci/cdk/cdk/aws_lc_github_fuzz_ci_stack.py index d99cc4532d..6f6e2d4d06 100644 --- a/tests/ci/cdk/cdk/aws_lc_github_fuzz_ci_stack.py +++ b/tests/ci/cdk/cdk/aws_lc_github_fuzz_ci_stack.py @@ -128,4 +128,4 @@ def __init__(self, "Type": "EFS" }]) - PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=fuzz_codebuild) + PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=fuzz_codebuild, ec2_permissions=False) diff --git a/tests/ci/cdk/cdk/bm_framework_stack.py b/tests/ci/cdk/cdk/bm_framework_stack.py index d465ea2a27..c9a880ab45 100644 --- a/tests/ci/cdk/cdk/bm_framework_stack.py +++ b/tests/ci/cdk/cdk/bm_framework_stack.py @@ -71,7 +71,7 @@ def __init__(self, build_spec=BuildSpecLoader.load(spec_file_path)) project.enable_batch_builds() - PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project) + PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=False) # use boto3 to determine if a cloudwatch logs group with the name we want exists, and if it doesn't, create it logs_client = boto3.client('logs', region_name=AWS_REGION) diff --git a/tests/ci/cdk/cdk/codebuild/ec2_test_framework_omnibus.yaml b/tests/ci/cdk/cdk/codebuild/ec2_test_framework_omnibus.yaml index 88d617ed26..315273f6dc 100644 --- a/tests/ci/cdk/cdk/codebuild/ec2_test_framework_omnibus.yaml +++ b/tests/ci/cdk/cdk/codebuild/ec2_test_framework_omnibus.yaml @@ -7,7 +7,7 @@ version: 0.2 batch: build-list: # Actual tests are ran on an Graviton2 ec2 instance via SSM Commands. - - identifier: graviton2_tests + - identifier: graviton2_tests_asan buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml env: type: LINUX_CONTAINER @@ -18,3 +18,44 @@ batch: EC2_AMI: "ami-0c29a2c5cf69b5a9c" EC2_INSTANCE_TYPE: "c6g.2xlarge" ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer" + TARGET_TEST_SCRIPT: "./tests/ci/run_posix_sanitizers.sh" + + - identifier: graviton2_tests_fips + buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_SMALL + image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-20.04_clang-7x-bm-framework_latest + variables: + EC2_AMI: "ami-0c29a2c5cf69b5a9c" + EC2_INSTANCE_TYPE: "c6g.2xlarge" + ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer" + TARGET_TEST_SCRIPT: "./tests/ci/run_fips_tests.sh" + + # Actual tests are ran on an Graviton4 ec2 instance via SSM Commands. + - identifier: graviton4_tests_asan + buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_SMALL + image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-20.04_clang-7x-bm-framework_latest + variables: + EC2_AMI: "ami-0c29a2c5cf69b5a9c" + EC2_INSTANCE_TYPE: "r8g.2xlarge" + ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer" + TARGET_TEST_SCRIPT: "./tests/ci/run_posix_sanitizers.sh" + + - identifier: graviton4_tests_fips + buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml + env: + type: LINUX_CONTAINER + privileged-mode: false + compute-type: BUILD_GENERAL1_SMALL + image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-20.04_clang-7x-bm-framework_latest + variables: + EC2_AMI: "ami-0c29a2c5cf69b5a9c" + EC2_INSTANCE_TYPE: "r8g.2xlarge" + ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer" + TARGET_TEST_SCRIPT: "./tests/ci/run_fips_tests.sh" diff --git a/tests/ci/cdk/cdk/components.py b/tests/ci/cdk/cdk/components.py index 58881bd495..8b1b9a4b57 100644 --- a/tests/ci/cdk/cdk/components.py +++ b/tests/ci/cdk/cdk/components.py @@ -4,11 +4,11 @@ aws_events as events, aws_events_targets as events_targets, aws_iam as iam, Duration from constructs import Construct -from util.metadata import GITHUB_REPO_OWNER, GITHUB_TOKEN_SECRET_NAME +from util.metadata import AWS_REGION, AWS_ACCOUNT, GITHUB_REPO_OWNER, GITHUB_TOKEN_SECRET_NAME class PruneStaleGitHubBuilds(Construct): - def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject) -> None: + def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject, ec2_permissions: bool) -> None: super().__init__(scope, id) github_token_secret = sm.Secret.from_secret_name_v2(scope=self, @@ -36,12 +36,33 @@ def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject) -> actions=[ "codebuild:BatchGetBuildBatches", "codebuild:ListBuildBatchesForProject", - "codebuild:StopBuildBatch", + "codebuild:StopBuildBatch" ], resources=[project.project_arn])) + if ec2_permissions: + lambda_function.add_to_role_policy( + iam.PolicyStatement(effect=iam.Effect.ALLOW, + actions=[ + "ec2:TerminateInstances", + ], + resources=["arn:aws:ec2:{}:{}:instance/*".format(AWS_REGION, AWS_ACCOUNT)], + conditions={ + "StringEquals": { + "ec2:ResourceTag/ec2-framework-host": "ec2-framework-host" + } + })) + # ec2:Describe* API actions do not support resource-level permissions. + lambda_function.add_to_role_policy( + iam.PolicyStatement(effect=iam.Effect.ALLOW, + actions=[ + "ec2:DescribeInstances", + ], + resources=["*"])) + + events.Rule(scope=self, id="PurgeEventRule", - description="Purge stale GitHub codebuild jobs (once per minute)", + description="Purge stale GitHub codebuild jobs and ec2 instances (once per minute)", enabled=True, schedule=events.Schedule.rate(Duration.minutes(1)), targets=[events_targets.LambdaFunction(handler=lambda_function)]) diff --git a/tests/ci/cdk/cdk/ssm/general_test_run_ssm_document.yaml b/tests/ci/cdk/cdk/ssm/general_test_run_ssm_document.yaml index 0d4b5faca2..69503f5847 100644 --- a/tests/ci/cdk/cdk/ssm/general_test_run_ssm_document.yaml +++ b/tests/ci/cdk/cdk/ssm/general_test_run_ssm_document.yaml @@ -59,13 +59,8 @@ mainSteps: # Check if container was spun up succesfully. Then run test scripts and check the output. - > if [ -n "$exec_docker" ]; then - chmod +x ./tests/ci/run_posix_sanitizers.sh - $exec_docker ./tests/ci/run_posix_sanitizers.sh - if [ $? != 0 ]; then - exit 1 - fi - chmod +x ./tests/ci/run_fips_tests.sh - $exec_docker ./tests/ci/run_fips_tests.sh + chmod +x {TARGET_TEST_SCRIPT} + $exec_docker {TARGET_TEST_SCRIPT} if [ $? != 0 ]; then exit 1 fi diff --git a/tests/ci/codebuild/common/run_ec2_target.yml b/tests/ci/codebuild/common/run_ec2_target.yml index bf2b6b9525..b702ed572e 100755 --- a/tests/ci/codebuild/common/run_ec2_target.yml +++ b/tests/ci/codebuild/common/run_ec2_target.yml @@ -10,4 +10,4 @@ env: phases: build: commands: - - ./tests/ci/run_ec2_test_framework.sh "${EC2_AMI}" "${EC2_INSTANCE_TYPE}" "${ECR_DOCKER_TAG}" + - ./tests/ci/run_ec2_test_framework.sh "${EC2_AMI}" "${EC2_INSTANCE_TYPE}" "${ECR_DOCKER_TAG}" "${TARGET_TEST_SCRIPT}" diff --git a/tests/ci/lambda/Cargo.lock b/tests/ci/lambda/Cargo.lock index 3fbec9a434..7080254873 100644 --- a/tests/ci/lambda/Cargo.lock +++ b/tests/ci/lambda/Cargo.lock @@ -82,227 +82,241 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "0.55.1" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd62464d1c4ad70f8b6cd693e7f30229f36bebdcdf3fce8c11803e1bdc0bc052" +checksum = "caf6cfe2881cb1fcbba9ae946fb9a6480d3b7a714ca84c74925014a89ef3387a" dependencies = [ "aws-credential-types", - "aws-http", + "aws-runtime", "aws-sdk-sso", + "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.1.0", "hex", - "http", + "http 0.2.9", "hyper", - "ring", + "ring 0.17.8", "time", "tokio", - "tower", "tracing", + "url", "zeroize", ] [[package]] name = "aws-credential-types" -version = "0.55.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4232d3729eefc287adc0d5a8adc97b7d94eefffe6bbe94312cc86c7ab6b06ce" +checksum = "e16838e6c9e12125face1c1eff1343c75e3ff540de98ff7ebd61874a89bcfeb9" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api", "aws-smithy-types", - "fastrand", - "tokio", - "tracing", "zeroize", ] [[package]] -name = "aws-endpoint" -version = "0.55.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f04ab03b3f1cca91f7cccaa213056d732accb14e2e65debfacc1d28627d162" +name = "aws-lc-ci-lambda" +version = "0.1.0" dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "aws-types", - "http", - "regex", - "tracing", + "aws-config", + "aws-sdk-codebuild", + "aws-sdk-ec2", + "aws-sdk-secretsmanager", + "env_logger", + "lambda_runtime", + "log", + "octocrab", + "serde_json", + "tokio", + "tokio-stream", ] [[package]] -name = "aws-http" -version = "0.55.1" +name = "aws-runtime" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ad8c53f7560baaf635b6aa811f3213d39b50555d100f83e43801652d4e318e" +checksum = "87c5f920ffd1e0526ec9e70e50bf444db50b204395a0fa7016bbf9e31ea1698f" dependencies = [ "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "http", - "http-body", - "lazy_static", + "fastrand 2.1.0", + "http 0.2.9", + "http-body 0.4.5", "percent-encoding", "pin-project-lite", "tracing", + "uuid", ] [[package]] -name = "aws-lc-ci-lambda" -version = "0.1.0" +name = "aws-sdk-codebuild" +version = "1.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a16e29dd17ab98c0464ae2f9641f07cb150eab659455c248274a89fec2a53ea8" dependencies = [ - "aws-config", - "aws-sdk-codebuild", - "aws-sdk-secretsmanager", - "env_logger", - "lambda_runtime", - "log", - "octocrab", - "serde_json", - "tokio", - "tokio-stream", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.9", + "once_cell", + "regex-lite", + "tracing", ] [[package]] -name = "aws-sdk-codebuild" -version = "0.26.0" +name = "aws-sdk-ec2" +version = "1.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be70472243a65c735a6705b87704cada9f9da2b528a0d6a1f7620183d81582a" +checksum = "91d145fb9e8ef6c783263d86d41d56c87b5e59b8ce28d9986de794a1ddfc42dc" dependencies = [ "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", + "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", + "aws-smithy-xml", "aws-types", - "bytes", - "http", - "regex", - "tokio-stream", - "tower", + "fastrand 2.1.0", + "http 0.2.9", + "once_cell", + "regex-lite", "tracing", ] [[package]] name = "aws-sdk-secretsmanager" -version = "0.26.0" +version = "1.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5b95c594d44acf1e105642e71e59b9eed06b1a11f4e4bb0a52e37b8c4c27f25" +checksum = "c9d617a00db081815ef63137359e26819ad755464514ff2c67219189eaf6288c" dependencies = [ "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", + "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "fastrand", - "http", - "regex", - "tokio-stream", - "tower", + "fastrand 2.1.0", + "http 0.2.9", + "once_cell", + "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sso" -version = "0.26.0" +version = "1.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143953d46f77a0b18480e7d8bb1a651080b9484e0bb94c27b8645eaeb3c3e231" +checksum = "fc3ef4ee9cdd19ec6e8b10d963b79637844bbf41c31177b77a188eaa941e69f7" dependencies = [ "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", + "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "http", - "regex", - "tokio-stream", - "tower", + "http 0.2.9", + "once_cell", + "regex-lite", "tracing", ] [[package]] -name = "aws-sdk-sts" -version = "0.26.0" +name = "aws-sdk-ssooidc" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7255c0d8053b89e8b5cdabb52e1dbf596e9968b1f45dce7a56b2cd57038fcfc9" +checksum = "527f3da450ea1f09f95155dba6153bd0d83fe0923344a12e1944dfa5d0b32064" dependencies = [ "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", + "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", - "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", - "aws-smithy-xml", "aws-types", "bytes", - "http", - "regex", - "tower", + "http 0.2.9", + "once_cell", + "regex-lite", "tracing", ] [[package]] -name = "aws-sig-auth" -version = "0.55.1" +name = "aws-sdk-sts" +version = "1.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d77d879ab210e958ba65a6d3842969a596738c024989cd3e490cf9f9b560ec" +checksum = "94316606a4aa2cb7a302388411b8776b3fbd254e8506e2dc43918286d8212e9b" dependencies = [ "aws-credential-types", - "aws-sigv4", + "aws-runtime", + "aws-smithy-async", "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", "aws-types", - "http", + "http 0.2.9", + "once_cell", + "regex-lite", "tracing", ] [[package]] name = "aws-sigv4" -version = "0.55.1" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ab4eebc8ec484fb9eab04b15a5d1e71f3dc13bee8fdd2d9ed78bcd6ecbd7192" +checksum = "5df1b0fa6be58efe9d4ccc257df0a53b89cd8909e86591a13ca54817c87517be" dependencies = [ + "aws-credential-types", "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", "form_urlencoded", "hex", "hmac", - "http", + "http 0.2.9", + "http 1.1.0", "once_cell", "percent-encoding", - "regex", "sha2", "time", "tracing", @@ -310,131 +324,143 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "0.55.1" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88573bcfbe1dcfd54d4912846df028b42d6255cbf9ce07be216b1bbfd11fc4b9" +checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" dependencies = [ "futures-util", "pin-project-lite", "tokio", - "tokio-stream", -] - -[[package]] -name = "aws-smithy-client" -version = "0.55.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2f52352bae50d3337d5d6151b695d31a8c10ebea113eca5bead531f8301b067" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-types", - "bytes", - "fastrand", - "http", - "http-body", - "hyper", - "hyper-rustls", - "lazy_static", - "pin-project-lite", - "rustls", - "tokio", - "tower", - "tracing", ] [[package]] name = "aws-smithy-http" -version = "0.55.1" +version = "0.60.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bcc02d7ed9649d855c8ce4a735e9848d7b8f7568aad0504c158e3baa955df8" +checksum = "d9cd0ae3d97daa0a2bf377a4d8e8e1362cae590c4a1aad0d40058ebca18eb91e" dependencies = [ + "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "bytes-utils", "futures-core", - "http", - "http-body", - "hyper", + "http 0.2.9", + "http-body 0.4.5", "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", - "tokio", - "tokio-util", "tracing", ] [[package]] -name = "aws-smithy-http-tower" -version = "0.55.1" +name = "aws-smithy-json" +version = "0.60.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da88b3a860f65505996c29192d800f1aeb9480440f56d63aad33a3c12045017a" +checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" dependencies = [ - "aws-smithy-http", "aws-smithy-types", - "bytes", - "http", - "http-body", - "pin-project-lite", - "tower", - "tracing", ] [[package]] -name = "aws-smithy-json" -version = "0.55.1" +name = "aws-smithy-query" +version = "0.60.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b0c1e87d75cac889dca2a7f5ba280da2cde8122448e7fec1d614194dfa00c70" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" dependencies = [ "aws-smithy-types", + "urlencoding", ] [[package]] -name = "aws-smithy-query" -version = "0.55.1" +name = "aws-smithy-runtime" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b50d15f446c19e088009ecb00e2fb2d13133d6fe1db702e9aa67ad135bf6a6" +checksum = "ce87155eba55e11768b8c1afa607f3e864ae82f03caf63258b37455b0ad02537" dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", - "urlencoding", + "bytes", + "fastrand 2.1.0", + "h2", + "http 0.2.9", + "http-body 0.4.5", + "http-body 1.0.1", + "httparse", + "hyper", + "hyper-rustls", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30819352ed0a04ecf6a2f3477e344d2d1ba33d43e0f09ad9047c12e0d923616f" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.9", + "http 1.1.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", ] [[package]] name = "aws-smithy-types" -version = "0.55.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0afc731fd1417d791f9145a1e0c30e23ae0beaab9b4814017708ead2fc20f1" +checksum = "cfe321a6b21f5d8eabd0ade9c55d3d0335f3c3157fc2b3e87f05f34b539e4df5" dependencies = [ "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.9", + "http 1.1.0", + "http-body 0.4.5", + "http-body 1.0.1", + "http-body-util", "itoa", "num-integer", + "pin-project-lite", + "pin-utils", "ryu", + "serde", "time", + "tokio", + "tokio-util", ] [[package]] name = "aws-smithy-xml" -version = "0.55.1" +version = "0.60.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5398c1c25dfc6f8c282b1552a66aa807c9d6e15e1b3a84b94aa44e7859bec3" +checksum = "d123fbc2a4adc3c301652ba8e149bf4bc1d1725affb9784eb20c953ace06bf55" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.55.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9b082e329d9a304d39e193ad5c7ab363a0d6507aca6965e0673a746686fb0cc" +checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" dependencies = [ "aws-credential-types", "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", - "http", "rustc_version", "tracing", ] @@ -515,9 +541,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.79" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "324c74f2155653c90b04f25b2a47a8a631360cb908f92a772695f430c7e31052" [[package]] name = "cfg-if" @@ -702,6 +728,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + [[package]] name = "fnv" version = "1.0.7" @@ -749,9 +781,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -759,9 +791,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" @@ -776,15 +808,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -793,21 +825,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -831,6 +863,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "gimli" version = "0.27.2" @@ -848,7 +891,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.9", "indexmap", "slab", "tokio", @@ -909,6 +952,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.5" @@ -916,7 +970,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", - "http", + "http 0.2.9", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -949,8 +1026,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", + "http 0.2.9", + "http-body 0.4.5", "httparse", "httpdate", "itoa", @@ -964,11 +1041,12 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.23.2" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ - "http", + "futures-util", + "http 0.2.9", "hyper", "log", "rustls", @@ -1095,7 +1173,7 @@ checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ "base64 0.21.0", "pem", - "ring", + "ring 0.16.20", "serde", "serde_json", "simple_asn1", @@ -1110,7 +1188,7 @@ dependencies = [ "async-stream", "bytes", "futures", - "http", + "http 0.2.9", "hyper", "lambda_runtime_api_client", "serde", @@ -1127,7 +1205,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "690c5ae01f3acac8c9c3348b556fc443054e9b7f1deaf53e9ebab716282bf0ed" dependencies = [ - "http", + "http 0.2.9", "hyper", "tokio", "tower-service", @@ -1141,9 +1219,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.142" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "link-cplusplus" @@ -1306,9 +1384,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl" @@ -1438,9 +1516,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -1483,6 +1561,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.7.1" @@ -1501,8 +1585,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", + "http 0.2.9", + "http-body 0.4.5", "hyper", "hyper-tls", "ipnet", @@ -1535,12 +1619,27 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", - "untrusted", + "spin 0.5.2", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -1572,14 +1671,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.8" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", - "ring", + "ring 0.17.8", + "rustls-webpki", "sct", - "webpki", ] [[package]] @@ -1603,6 +1702,16 @@ dependencies = [ "base64 0.21.0", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring 0.17.8", + "untrusted 0.9.0", +] + [[package]] name = "ryu" version = "1.0.13" @@ -1636,8 +1745,8 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -1816,6 +1925,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "subtle" version = "2.4.1" @@ -1851,7 +1966,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if", - "fastrand", + "fastrand 1.9.0", "redox_syscall 0.3.5", "rustix", "windows-sys 0.45.0", @@ -1970,13 +2085,12 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.23.4" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls", "tokio", - "webpki", ] [[package]] @@ -2014,7 +2128,6 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio", "tower-layer", "tower-service", "tracing", @@ -2109,6 +2222,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.3.1" @@ -2127,6 +2246,12 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" + [[package]] name = "vcpkg" version = "0.2.15" @@ -2237,16 +2362,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "winapi" version = "0.3.9" @@ -2320,6 +2435,15 @@ dependencies = [ "windows-targets 0.48.0", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -2350,6 +2474,22 @@ dependencies = [ "windows_x86_64_msvc 0.48.0", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -2362,6 +2502,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -2374,6 +2520,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -2386,6 +2538,18 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -2398,6 +2562,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -2410,6 +2580,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -2422,6 +2598,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -2434,6 +2616,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "winreg" version = "0.10.1" diff --git a/tests/ci/lambda/Cargo.toml b/tests/ci/lambda/Cargo.toml index 62792f4842..8cc31631d7 100644 --- a/tests/ci/lambda/Cargo.toml +++ b/tests/ci/lambda/Cargo.toml @@ -7,9 +7,10 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -aws-config = "0.55.1" -aws-sdk-codebuild = "0.26.0" -aws-sdk-secretsmanager = "0.26.0" +aws-config = "1.1.7" +aws-sdk-codebuild = "1.44.0" +aws-sdk-ec2 = "1.59.0" +aws-sdk-secretsmanager = "1.39.0" lambda_runtime = "0.8.0" log = "0.4.17" env_logger = "0.10.0" diff --git a/tests/ci/lambda/src/bin/purge-stale-builds.rs b/tests/ci/lambda/src/bin/purge-stale-builds.rs index f69306c291..076299c630 100644 --- a/tests/ci/lambda/src/bin/purge-stale-builds.rs +++ b/tests/ci/lambda/src/bin/purge-stale-builds.rs @@ -1,10 +1,11 @@ +use aws_config::BehaviorVersion; use std::collections::HashMap; use std::time::{SystemTime, UNIX_EPOCH}; -use aws_sdk_codebuild::{types::BuildBatchFilter, Client}; +use aws_sdk_codebuild::types::BuildBatchFilter; +use aws_sdk_ec2::types::Filter; use lambda_runtime::{service_fn, Error, LambdaEvent}; use serde_json::{json, Value}; -use tokio_stream::StreamExt; #[tokio::main] async fn main() -> Result<(), Error> { @@ -18,7 +19,10 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { let region_provider = aws_config::meta::region::RegionProviderChain::default_provider().or_else("us-west-2"); - let config = aws_config::from_env().region(region_provider).load().await; + let config = aws_config::defaults(BehaviorVersion::latest()) + .region(region_provider) + .load() + .await; let sm_client = aws_sdk_secretsmanager::Client::new(&config); @@ -32,7 +36,7 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { let github = octocrab::initialise(octocrab::Octocrab::builder().personal_token(github_token)) .map_err(|e| format!("failed to build github client: {e}"))?; - let codebuild_client = Client::new(&config); + let codebuild_client = aws_sdk_codebuild::Client::new(&config); let project = std::env::var("CODEBUILD_PROJECT_NAME").unwrap(); @@ -59,6 +63,31 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { .or_insert(builds); } + // Filters for aws-lc-ec2-test-framework specific hosts. + let ec2_client = aws_sdk_ec2::Client::new(&config); + let ec2_framework_filters = vec![ + Filter::builder() + .name("instance-state-name") + .values("running") + .build(), + Filter::builder() + .name("instance.group-name") + .values("codebuild_ec2_sg") + .build(), + Filter::builder() + .name("tag-key") + .values("ec-framework-commit-tag") + .build(), + ]; + + let ec2_describe_response = ec2_client + .describe_instances() + .set_filters(Some(ec2_framework_filters)) + .send() + .await + .map_err(|e| format!("No IAM Permissions to DescribeInstances: {}", e)); + + let mut ec2_terminated_instances: Vec = vec![]; let mut stopped_builds: u64 = 0; for (k, v) in pull_requests.iter() { @@ -69,7 +98,7 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { .pulls(&github_repo_owner, "aws-lc") .get(*k) .await - .map_err(|e| format!("failed to retrieve GitHub pull requests: {}", e))?; + .map_err(|e| format!("failed to retrieve GitHub pull requests: {:?}", e))?; let commit: String = pull.head.sha; for cb in v.iter() { let build_id = cb.build(); @@ -78,6 +107,8 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { continue; } let old_commit = cb.commit(); + + // Prune unneeded codebuild batches. log::info!("{build_id} pr/{k} at old head({old_commit}) will be canceled"); codebuild_client .stop_build_batch() @@ -86,9 +117,35 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { .await .map_err(|e| format!("failed to stop_build_batch: {}", e))?; stopped_builds += 1; + + // Prune unneeded ec2 instances. + for reservation in ec2_describe_response.as_ref().unwrap().reservations() { + log::info!("Checking Instance {:?}", reservation.instances()); + for instance in reservation.instances() { + for tag in instance.tags() { + log::info!("Tag: {:?}", tag); + if tag.key().unwrap().to_string() == "ec-framework-commit-tag" + && tag.value().unwrap().to_string() == old_commit + { + ec2_terminated_instances + .push(instance.instance_id().unwrap().to_string()); + } + } + } + } } } + log::info!("Terminating instances {:?}", ec2_terminated_instances); + if !ec2_terminated_instances.is_empty() { + ec2_client + .terminate_instances() + .set_instance_ids(Some(ec2_terminated_instances.clone())) + .send() + .await + .map_err(|e| format!("failed to terminate hanging instances: {}", e))?; + } + let timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() @@ -114,7 +171,8 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { "Timestamp": timestamp }, "Project": &project, - "PrunedGitHubBuilds": stopped_builds + "PrunedGitHubBuilds": stopped_builds, + "Terminated EC2 Instances": ec2_terminated_instances }) .to_string() ); @@ -158,7 +216,7 @@ impl CommitBuild { } async fn gather_pull_request_builds( - client: &Client, + client: &aws_sdk_codebuild::Client, builds: Vec, ) -> Result>, String> { let mut pull_requests: HashMap> = HashMap::new(); @@ -180,14 +238,12 @@ async fn gather_pull_request_builds( )); } - for bb in batch.unwrap().build_batches().unwrap_or_default() { + for bb in batch.unwrap().build_batches() { if bb.source().is_none() { continue; } let source = bb.source().unwrap(); - if source.r#type().is_none() - || source.r#type().unwrap().as_str() - != aws_sdk_codebuild::types::SourceType::Github.as_str() + if source.r#type().as_str() != aws_sdk_codebuild::types::SourceType::Github.as_str() || bb.source_version().is_none() || bb.resolved_source_version().is_none() { @@ -223,7 +279,7 @@ async fn gather_pull_request_builds( } async fn get_project_build_batches( - client: &Client, + client: &aws_sdk_codebuild::Client, project: String, ) -> Result, String> { let mut builds: Vec = vec![]; @@ -247,7 +303,7 @@ async fn get_project_build_batches( )); } - let mut ids = Vec::from(result.unwrap().ids().unwrap_or(&[])); + let mut ids = Vec::from(result.unwrap().ids()); builds.append(&mut ids); } diff --git a/tests/ci/run_ec2_test_framework.sh b/tests/ci/run_ec2_test_framework.sh index 45e39f298d..73def20f31 100755 --- a/tests/ci/run_ec2_test_framework.sh +++ b/tests/ci/run_ec2_test_framework.sh @@ -24,8 +24,9 @@ generate_ssm_document_file() { -e "s,{SOURCE},${CODEBUILD_SOURCE_REPO_URL},g" \ -e "s,{S3_BUCKET},${s3_bucket_name},g" \ -e "s,{ECR_DOCKER_TAG},${ecr_docker_tag},g" \ + -e "s,{TARGET_TEST_SCRIPT},${target_test_script},g" \ tests/ci/cdk/cdk/ssm/general_test_run_ssm_document.yaml \ - > "tests/ci/cdk/cdk/ssm/${ec2_ami_id}_ssm_document.yaml" + > "tests/ci/cdk/cdk/ssm/${1}_ssm_document.yaml" } #$1 for ami, $2 for instance-type, echos the instance id so we can capture the output @@ -34,7 +35,9 @@ create_ec2_instances() { instance_id="$(aws ec2 run-instances --image-id "$1" --count 1 \ --instance-type "$2" --security-group-ids "${EC2_SECURITY_GROUP_ID}" --subnet-id "${EC2_SUBNET_ID}" \ --block-device-mappings 'DeviceName="/dev/sda1",Ebs={DeleteOnTermination=True,VolumeSize=200}' \ - --tag-specifications 'ResourceType="instance",Tags=[{Key="Name",Value="ec2-test-'"$CODEBUILD_WEBHOOK_TRIGGER"'"}]' \ + --tag-specifications 'ResourceType="instance",Tags=[{Key="Name",Value="ec2-test-'"$CODEBUILD_WEBHOOK_TRIGGER"'"}, + {Key="ec2-framework-host",Value="ec2-framework-host"}, + {Key="ec-framework-commit-tag",Value="'"$CODEBUILD_SOURCE_VERSION"'"}]' \ --iam-instance-profile Name=aws-lc-ci-ec2-test-framework-ec2-profile \ --placement 'AvailabilityZone=us-west-2a' \ --instance-initiated-shutdown-behavior terminate \ @@ -51,10 +54,12 @@ echo Source: "${CODEBUILD_SOURCE_REPO_URL}" export ec2_ami_id="$1" export ec2_instance_type="$2" export ecr_docker_tag="$3" +export target_test_script="$4" export s3_bucket_name="aws-lc-codebuild" # create the ssm documents that will be used for the various ssm commands -generate_ssm_document_file +ssm_prefix="$(echo "$ec2_instance_type" | awk -F'.' '{print $1}')_$(basename "$target_test_script" .sh)" +generate_ssm_document_file "${ssm_prefix}" # create ec2 instances instance_id=$(create_ec2_instances "${ec2_ami_id}" "${ec2_instance_type}") @@ -77,13 +82,13 @@ for i in {1..30}; do sleep 60 done +# Create, and run ssm command. +ssm_doc_name=$(create_ssm_document "${ssm_prefix}") + # Wait 5 minutes for instance to "warm up"? echo "Instances need to initialize a few minutes before SSM commands can be properly run" sleep 300 -# Create, and run ssm command. -ssm_doc_name=$(create_ssm_document "${ec2_ami_id}") - cloudwatch_group_name="aws-lc-ci-ec2-test-framework-cw-logs" ec2_test_ssm_command_id=$(run_ssm_command "${ssm_doc_name}" "${instance_id}" ${cloudwatch_group_name})