diff --git a/.projen/deps.json b/.projen/deps.json index 6ef161cbf..82a70ce8f 100644 --- a/.projen/deps.json +++ b/.projen/deps.json @@ -39,6 +39,10 @@ "name": "@smithy/types", "type": "build" }, + { + "name": "@smithy/util-retry", + "type": "build" + }, { "name": "@smithy/util-stream", "type": "build" diff --git a/.projen/tasks.json b/.projen/tasks.json index 66abcb2f8..908d122fd 100644 --- a/.projen/tasks.json +++ b/.projen/tasks.json @@ -1201,13 +1201,13 @@ }, "steps": [ { - "exec": "npx npm-check-updates@16 --upgrade --target=minor --peer --dep=dev --filter=@aws-sdk/client-codeartifact,@aws-sdk/client-s3,@aws-sdk/client-sfn,@jsii/spec,@octokit/rest,@smithy/types,@smithy/util-stream,@types/aws-lambda,@types/fs-extra,@types/jest,@types/markdown-it,@types/markdown-it-emoji,@types/semver,@types/streamx,@types/tar-stream,@types/tough-cookie,@types/uuid,async-sema,aws-embedded-metrics,aws-sdk,aws-sdk-client-mock,aws-sdk-client-mock-jest,aws-sdk-mock,aws-xray-sdk-core,case,cdk-dia,cdk-watchful,cdklabs-projen-project-types,chalk,construct-hub-webapp,dotenv,esbuild,eslint-config-prettier,eslint-import-resolver-typescript,eslint-plugin-import,eslint-plugin-prettier,feed,fs-extra,glob,got,jest,jsii-diff,jsii-pacmak,JSONStream,markdown-it,markdown-it-emoji,nock,normalize-registry-metadata,prettier,projen,semver,spdx-license-list,streamcount,streamx,tar-stream,ts-node,uuid,yaml" + "exec": "npx npm-check-updates@16 --upgrade --target=minor --peer --dep=dev --filter=@aws-sdk/client-codeartifact,@aws-sdk/client-s3,@aws-sdk/client-sfn,@jsii/spec,@octokit/rest,@smithy/types,@smithy/util-retry,@smithy/util-stream,@types/aws-lambda,@types/fs-extra,@types/jest,@types/markdown-it,@types/markdown-it-emoji,@types/semver,@types/streamx,@types/tar-stream,@types/tough-cookie,@types/uuid,async-sema,aws-embedded-metrics,aws-sdk,aws-sdk-client-mock,aws-sdk-client-mock-jest,aws-sdk-mock,aws-xray-sdk-core,case,cdk-dia,cdk-watchful,cdklabs-projen-project-types,chalk,construct-hub-webapp,dotenv,esbuild,eslint-config-prettier,eslint-import-resolver-typescript,eslint-plugin-import,eslint-plugin-prettier,feed,fs-extra,glob,got,jest,jsii-diff,jsii-pacmak,JSONStream,markdown-it,markdown-it-emoji,nock,normalize-registry-metadata,prettier,projen,semver,spdx-license-list,streamcount,streamx,tar-stream,ts-node,uuid,yaml" }, { "exec": "yarn install --check-files" }, { - "exec": "yarn upgrade @aws-cdk/aws-servicecatalogappregistry-alpha @aws-cdk/integ-runner @aws-cdk/integ-tests-alpha @aws-sdk/client-codeartifact @aws-sdk/client-s3 @aws-sdk/client-sfn @jsii/spec @octokit/rest @smithy/types @smithy/util-stream @types/aws-lambda @types/fs-extra @types/jest @types/markdown-it @types/markdown-it-emoji @types/node @types/semver @types/streamx @types/tar-stream @types/tough-cookie @types/uuid @typescript-eslint/eslint-plugin @typescript-eslint/parser async-sema aws-cdk-lib aws-cdk aws-embedded-metrics aws-sdk aws-sdk-client-mock aws-sdk-client-mock-jest aws-sdk-mock aws-xray-sdk-core case cdk-dia cdk-watchful cdklabs-projen-project-types chalk commit-and-tag-version construct-hub-webapp dotenv esbuild eslint-config-prettier eslint-import-resolver-typescript eslint-plugin-import eslint-plugin-prettier eslint feed fs-extra glob got jest jest-junit jsii-diff jsii-docgen jsii-pacmak jsii-rosetta jsii JSONStream markdown-it markdown-it-emoji nock normalize-registry-metadata prettier projen semver spdx-license-list streamcount streamx tar-stream ts-node typescript uuid yaml" + "exec": "yarn upgrade @aws-cdk/aws-servicecatalogappregistry-alpha @aws-cdk/integ-runner @aws-cdk/integ-tests-alpha @aws-sdk/client-codeartifact @aws-sdk/client-s3 @aws-sdk/client-sfn @jsii/spec @octokit/rest @smithy/types @smithy/util-retry @smithy/util-stream @types/aws-lambda @types/fs-extra @types/jest @types/markdown-it @types/markdown-it-emoji @types/node @types/semver @types/streamx @types/tar-stream @types/tough-cookie @types/uuid @typescript-eslint/eslint-plugin @typescript-eslint/parser async-sema aws-cdk-lib aws-cdk aws-embedded-metrics aws-sdk aws-sdk-client-mock aws-sdk-client-mock-jest aws-sdk-mock aws-xray-sdk-core case cdk-dia cdk-watchful cdklabs-projen-project-types chalk commit-and-tag-version construct-hub-webapp dotenv esbuild eslint-config-prettier eslint-import-resolver-typescript eslint-plugin-import eslint-plugin-prettier eslint feed fs-extra glob got jest jest-junit jsii-diff jsii-docgen jsii-pacmak jsii-rosetta jsii JSONStream markdown-it markdown-it-emoji nock normalize-registry-metadata prettier projen semver spdx-license-list streamcount streamx tar-stream ts-node typescript uuid yaml" }, { "exec": "npx projen" diff --git a/.projenrc.ts b/.projenrc.ts index 1f0c90ab4..af9123401 100644 --- a/.projenrc.ts +++ b/.projenrc.ts @@ -50,6 +50,7 @@ const project = new CdklabsConstructLibrary({ '@aws-sdk/client-codeartifact', '@aws-sdk/client-s3', '@smithy/types', + '@smithy/util-retry', '@smithy/util-stream', 'async-sema', 'aws-embedded-metrics', diff --git a/package.json b/package.json index 4b3253218..75c4f0e47 100644 --- a/package.json +++ b/package.json @@ -121,6 +121,7 @@ "@jsii/spec": "^1.103.1", "@octokit/rest": "^19.0.13", "@smithy/types": "^3.3.0", + "@smithy/util-retry": "^3.0.4", "@smithy/util-stream": "^3.1.4", "@types/aws-lambda": "^8.10.145", "@types/fs-extra": "^11.0.4", diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index 460ffd003..2ea4c431f 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -2210,7 +2210,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "92c30ddb1baaf0771dac0f79d4c3b1a6984baf8eac73e12cb0ab04f3f22fd096.zip", + "S3Key": "405dc475090f08c495a802305808b248fba97800ffd9705fc071d3940e869dea.zip", }, "Description": "[ConstructHub/Ingestion] Ingests new package versions into the Construct Hub", "Environment": { @@ -6802,10 +6802,18 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", "Name": "AWS_EMF_ENVIRONMENT", "Value": "Local", }, + { + "Name": "MAX_CONCURRENT_S3_REQUESTS", + "Value": "16", + }, + { + "Name": "MAX_RETRIES_S3_REQUESTS", + "Value": "12", + }, ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -15030,7 +15038,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "92c30ddb1baaf0771dac0f79d4c3b1a6984baf8eac73e12cb0ab04f3f22fd096.zip", + "S3Key": "405dc475090f08c495a802305808b248fba97800ffd9705fc071d3940e869dea.zip", }, "Description": "[ConstructHub/Ingestion] Ingests new package versions into the Construct Hub", "Environment": { @@ -19720,6 +19728,14 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", "Name": "AWS_EMF_ENVIRONMENT", "Value": "Local", }, + { + "Name": "MAX_CONCURRENT_S3_REQUESTS", + "Value": "16", + }, + { + "Name": "MAX_RETRIES_S3_REQUESTS", + "Value": "12", + }, { "Name": "CODE_ARTIFACT_DOMAIN_NAME", "Value": { @@ -19750,7 +19766,7 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -27764,7 +27780,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "92c30ddb1baaf0771dac0f79d4c3b1a6984baf8eac73e12cb0ab04f3f22fd096.zip", + "S3Key": "405dc475090f08c495a802305808b248fba97800ffd9705fc071d3940e869dea.zip", }, "Description": "[ConstructHub/Ingestion] Ingests new package versions into the Construct Hub", "Environment": { @@ -32356,10 +32372,18 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", "Name": "AWS_EMF_ENVIRONMENT", "Value": "Local", }, + { + "Name": "MAX_CONCURRENT_S3_REQUESTS", + "Value": "16", + }, + { + "Name": "MAX_RETRIES_S3_REQUESTS", + "Value": "12", + }, ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -40454,7 +40478,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "92c30ddb1baaf0771dac0f79d4c3b1a6984baf8eac73e12cb0ab04f3f22fd096.zip", + "S3Key": "405dc475090f08c495a802305808b248fba97800ffd9705fc071d3940e869dea.zip", }, "Description": "[ConstructHub/Ingestion] Ingests new package versions into the Construct Hub", "Environment": { @@ -45068,10 +45092,18 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", "Name": "AWS_EMF_ENVIRONMENT", "Value": "Local", }, + { + "Name": "MAX_CONCURRENT_S3_REQUESTS", + "Value": "16", + }, + { + "Name": "MAX_RETRIES_S3_REQUESTS", + "Value": "12", + }, ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -53353,7 +53385,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "92c30ddb1baaf0771dac0f79d4c3b1a6984baf8eac73e12cb0ab04f3f22fd096.zip", + "S3Key": "405dc475090f08c495a802305808b248fba97800ffd9705fc071d3940e869dea.zip", }, "Description": "[ConstructHub/Ingestion] Ingests new package versions into the Construct Hub", "Environment": { @@ -57699,10 +57731,18 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", "Name": "AWS_EMF_ENVIRONMENT", "Value": "Local", }, + { + "Name": "MAX_CONCURRENT_S3_REQUESTS", + "Value": "16", + }, + { + "Name": "MAX_RETRIES_S3_REQUESTS", + "Value": "12", + }, ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b", }, "LogConfiguration": { "LogDriver": "awslogs", diff --git a/src/__tests__/backend/transliterator/__snapshots__/index.test.ts.snap b/src/__tests__/backend/transliterator/__snapshots__/index.test.ts.snap index 482f246de..5ede42fda 100644 --- a/src/__tests__/backend/transliterator/__snapshots__/index.test.ts.snap +++ b/src/__tests__/backend/transliterator/__snapshots__/index.test.ts.snap @@ -88,6 +88,14 @@ exports[`CodeArtifact repository 1`] = ` "Name": "AWS_EMF_ENVIRONMENT", "Value": "Local", }, + { + "Name": "MAX_CONCURRENT_S3_REQUESTS", + "Value": "16", + }, + { + "Name": "MAX_RETRIES_S3_REQUESTS", + "Value": "12", + }, { "Name": "CODE_ARTIFACT_DOMAIN_NAME", "Value": { @@ -118,7 +126,7 @@ exports[`CodeArtifact repository 1`] = ` ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -2399,6 +2407,14 @@ exports[`VPC Endpoints 1`] = ` "Name": "AWS_EMF_ENVIRONMENT", "Value": "Local", }, + { + "Name": "MAX_CONCURRENT_S3_REQUESTS", + "Value": "16", + }, + { + "Name": "MAX_RETRIES_S3_REQUESTS", + "Value": "12", + }, { "Name": "CODE_ARTIFACT_API_ENDPOINT", "Value": { @@ -2426,7 +2442,7 @@ exports[`VPC Endpoints 1`] = ` ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -4731,6 +4747,14 @@ exports[`VPC Endpoints and CodeArtifact repository 1`] = ` "Name": "AWS_EMF_ENVIRONMENT", "Value": "Local", }, + { + "Name": "MAX_CONCURRENT_S3_REQUESTS", + "Value": "16", + }, + { + "Name": "MAX_RETRIES_S3_REQUESTS", + "Value": "12", + }, { "Name": "CODE_ARTIFACT_API_ENDPOINT", "Value": { @@ -4785,7 +4809,7 @@ exports[`VPC Endpoints and CodeArtifact repository 1`] = ` ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -7086,10 +7110,18 @@ exports[`basic use 1`] = ` "Name": "AWS_EMF_ENVIRONMENT", "Value": "Local", }, + { + "Name": "MAX_CONCURRENT_S3_REQUESTS", + "Value": "16", + }, + { + "Name": "MAX_RETRIES_S3_REQUESTS", + "Value": "12", + }, ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b", }, "LogConfiguration": { "LogDriver": "awslogs", diff --git a/src/__tests__/backend/transliterator/index.test.ts b/src/__tests__/backend/transliterator/index.test.ts index be766544c..d1417caaa 100644 --- a/src/__tests__/backend/transliterator/index.test.ts +++ b/src/__tests__/backend/transliterator/index.test.ts @@ -58,6 +58,14 @@ test('CodeArtifact repository', () => { Environment: stack.resolve([ { Name: 'HEADER_SPAN', Value: 'true' }, { Name: 'AWS_EMF_ENVIRONMENT', Value: 'Local' }, + { + Name: 'MAX_CONCURRENT_S3_REQUESTS', + Value: '16', + }, + { + Name: 'MAX_RETRIES_S3_REQUESTS', + Value: '12', + }, { Name: 'CODE_ARTIFACT_DOMAIN_NAME', Value: codeArtifact.repositoryDomainName, @@ -138,6 +146,14 @@ test('VPC Endpoints', () => { Environment: stack.resolve([ { Name: 'HEADER_SPAN', Value: 'true' }, { Name: 'AWS_EMF_ENVIRONMENT', Value: 'Local' }, + { + Name: 'MAX_CONCURRENT_S3_REQUESTS', + Value: '16', + }, + { + Name: 'MAX_RETRIES_S3_REQUESTS', + Value: '12', + }, { Name: 'CODE_ARTIFACT_API_ENDPOINT', Value: Fn.select( @@ -214,6 +230,14 @@ test('VPC Endpoints and CodeArtifact repository', () => { Environment: stack.resolve([ { Name: 'HEADER_SPAN', Value: 'true' }, { Name: 'AWS_EMF_ENVIRONMENT', Value: 'Local' }, + { + Name: 'MAX_CONCURRENT_S3_REQUESTS', + Value: '16', + }, + { + Name: 'MAX_RETRIES_S3_REQUESTS', + Value: '12', + }, { Name: 'CODE_ARTIFACT_API_ENDPOINT', Value: Fn.select( diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 7877f8b97..05065ec9f 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -2665,7 +2665,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "92c30ddb1baaf0771dac0f79d4c3b1a6984baf8eac73e12cb0ab04f3f22fd096.zip", + "S3Key": "405dc475090f08c495a802305808b248fba97800ffd9705fc071d3940e869dea.zip", }, "Description": "[ConstructHub/Ingestion] Ingests new package versions into the Construct Hub", "Environment": { @@ -7017,6 +7017,14 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", "Name": "AWS_EMF_ENVIRONMENT", "Value": "Local", }, + { + "Name": "MAX_CONCURRENT_S3_REQUESTS", + "Value": "16", + }, + { + "Name": "MAX_RETRIES_S3_REQUESTS", + "Value": "12", + }, { "Name": "CODE_ARTIFACT_API_ENDPOINT", "Value": { @@ -7071,7 +7079,7 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b", }, "LogConfiguration": { "LogDriver": "awslogs", diff --git a/src/backend/transliterator/index.ts b/src/backend/transliterator/index.ts index a1822c505..aff779ed4 100644 --- a/src/backend/transliterator/index.ts +++ b/src/backend/transliterator/index.ts @@ -127,11 +127,15 @@ export class Transliterator extends Construct { : props.bucket; const environment: Record = { - // temporaty hack to generate construct-hub compliant markdown. + // temporary hack to generate construct-hub compliant markdown. // see https://github.com/cdklabs/jsii-docgen/blob/master/src/docgen/render/markdown.ts#L172 HEADER_SPAN: 'true', // Set embedded metrics format environment to "Local", to have a consistent experience. AWS_EMF_ENVIRONMENT: 'Local', + // Limit the number of concurrent S3 requests + MAX_CONCURRENT_S3_REQUESTS: '16', + // The maximal number of retries for throttled S3 requests + MAX_RETRIES_S3_REQUESTS: '12', }; if (props.vpcEndpoints?.codeArtifactApi) { // Those are returned as an array of HOSTED_ZONE_ID:DNS_NAME... We care diff --git a/src/backend/transliterator/transliterator.ecstask.ts b/src/backend/transliterator/transliterator.ecstask.ts index 574449f68..047dba4d2 100644 --- a/src/backend/transliterator/transliterator.ecstask.ts +++ b/src/backend/transliterator/transliterator.ecstask.ts @@ -13,6 +13,7 @@ import { } from '@aws-sdk/client-s3'; import { Assembly } from '@jsii/spec'; import { StreamingBlobPayloadInputTypes } from '@smithy/types'; +import { AdaptiveRetryStrategy } from '@smithy/util-retry'; import { Sema } from 'async-sema'; import { metricScope, Unit } from 'aws-embedded-metrics'; import * as docgen from 'jsii-docgen'; @@ -35,10 +36,16 @@ const S3_SEMAPHORE = new Sema( parseInt(process.env.MAX_CONCURRENT_S3_REQUESTS ?? '16', 10) ); +const MAX_RETRIES_S3_REQUESTS = parseInt( + process.env.MAX_RETRIES_S3_REQUESTS ?? '12', + 10 +); const S3_CLIENT = new S3Client({ // https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html#standardvsadaptive // This works because S3 throttles requests based on prefix - retryMode: 'ADAPTIVE', + retryStrategy: new AdaptiveRetryStrategy( + async () => MAX_RETRIES_S3_REQUESTS /* maxAttempts */ + ), }); /** diff --git a/test/integ.transliterator.ecstask.ts.snapshot/TransliteratorEcsTaskInteg.assets.json b/test/integ.transliterator.ecstask.ts.snapshot/TransliteratorEcsTaskInteg.assets.json index 2b5b7e228..2353ac33b 100644 --- a/test/integ.transliterator.ecstask.ts.snapshot/TransliteratorEcsTaskInteg.assets.json +++ b/test/integ.transliterator.ecstask.ts.snapshot/TransliteratorEcsTaskInteg.assets.json @@ -40,7 +40,7 @@ } } }, - "5734d7ab060d267f335136d0ba9378f40a5698385392dee3bdcd54caf075dadc": { + "d629183b3b5a048fa164c7601cf0c01fe6a87cc1b7c2535a08a8413e2708ab95": { "source": { "path": "TransliteratorEcsTaskInteg.template.json", "packaging": "file" @@ -48,22 +48,22 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "5734d7ab060d267f335136d0ba9378f40a5698385392dee3bdcd54caf075dadc.json", + "objectKey": "d629183b3b5a048fa164c7601cf0c01fe6a87cc1b7c2535a08a8413e2708ab95.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } } }, "dockerImages": { - "163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564": { + "94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b": { "source": { - "directory": "asset.163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564.bundle", + "directory": "asset.94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b.bundle", "platform": "linux/arm64" }, "destinations": { "current_account-current_region": { "repositoryName": "cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}", - "imageTag": "163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564", + "imageTag": "94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-image-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/test/integ.transliterator.ecstask.ts.snapshot/TransliteratorEcsTaskInteg.template.json b/test/integ.transliterator.ecstask.ts.snapshot/TransliteratorEcsTaskInteg.template.json index 4192459ba..5eafbf72f 100644 --- a/test/integ.transliterator.ecstask.ts.snapshot/TransliteratorEcsTaskInteg.template.json +++ b/test/integ.transliterator.ecstask.ts.snapshot/TransliteratorEcsTaskInteg.template.json @@ -1481,11 +1481,19 @@ { "Name": "AWS_EMF_ENVIRONMENT", "Value": "Local" + }, + { + "Name": "MAX_CONCURRENT_S3_REQUESTS", + "Value": "16" + }, + { + "Name": "MAX_RETRIES_S3_REQUESTS", + "Value": "12" } ], "Essential": true, "Image": { - "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}:163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564" + "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}:94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b" }, "LogConfiguration": { "LogDriver": "awslogs", diff --git a/test/integ.transliterator.ecstask.ts.snapshot/manifest.json b/test/integ.transliterator.ecstask.ts.snapshot/manifest.json index 83cdd520c..7d6abc450 100644 --- a/test/integ.transliterator.ecstask.ts.snapshot/manifest.json +++ b/test/integ.transliterator.ecstask.ts.snapshot/manifest.json @@ -18,7 +18,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/5734d7ab060d267f335136d0ba9378f40a5698385392dee3bdcd54caf075dadc.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/d629183b3b5a048fa164c7601cf0c01fe6a87cc1b7c2535a08a8413e2708ab95.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -217,7 +217,10 @@ "/TransliteratorEcsTaskInteg/Transliterator/TaskDefinition/Resource": [ { "type": "aws:cdk:logicalId", - "data": "TransliteratorTaskDefinition16158D86" + "data": "TransliteratorTaskDefinition16158D86", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_REPLACE" + ] } ], "/TransliteratorEcsTaskInteg/Transliterator/TaskDefinition/ExecutionRole/Resource": [ @@ -330,7 +333,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/c58e15f764cc8151a7d2c4b0acf232e94fca93b829e20ef090a20cf5517bf58c.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/d1c573df1179c0a28edfc45dc8da18afec8fceb6ffcfd0bcaea00a1fde15e24a.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ diff --git a/test/integ.transliterator.ecstask.ts.snapshot/transliteratorintegDefaultTestDeployAssert109DC8B0.assets.json b/test/integ.transliterator.ecstask.ts.snapshot/transliteratorintegDefaultTestDeployAssert109DC8B0.assets.json index 88d2c401a..9e43d2baa 100644 --- a/test/integ.transliterator.ecstask.ts.snapshot/transliteratorintegDefaultTestDeployAssert109DC8B0.assets.json +++ b/test/integ.transliterator.ecstask.ts.snapshot/transliteratorintegDefaultTestDeployAssert109DC8B0.assets.json @@ -14,7 +14,7 @@ } } }, - "c58e15f764cc8151a7d2c4b0acf232e94fca93b829e20ef090a20cf5517bf58c": { + "d1c573df1179c0a28edfc45dc8da18afec8fceb6ffcfd0bcaea00a1fde15e24a": { "source": { "path": "transliteratorintegDefaultTestDeployAssert109DC8B0.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "c58e15f764cc8151a7d2c4b0acf232e94fca93b829e20ef090a20cf5517bf58c.json", + "objectKey": "d1c573df1179c0a28edfc45dc8da18afec8fceb6ffcfd0bcaea00a1fde15e24a.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/test/integ.transliterator.ecstask.ts.snapshot/transliteratorintegDefaultTestDeployAssert109DC8B0.template.json b/test/integ.transliterator.ecstask.ts.snapshot/transliteratorintegDefaultTestDeployAssert109DC8B0.template.json index f8bf1fd0f..f4b5bd6db 100644 --- a/test/integ.transliterator.ecstask.ts.snapshot/transliteratorintegDefaultTestDeployAssert109DC8B0.template.json +++ b/test/integ.transliterator.ecstask.ts.snapshot/transliteratorintegDefaultTestDeployAssert109DC8B0.template.json @@ -41,7 +41,7 @@ "outputPaths": [ "executionArn" ], - "salt": "1725967583740" + "salt": "1725976481239" }, "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" @@ -158,7 +158,7 @@ } }, "flattenResponse": "false", - "salt": "1725967583741" + "salt": "1725976481241" }, "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" diff --git a/test/integ.transliterator.ecstask.ts.snapshot/tree.json b/test/integ.transliterator.ecstask.ts.snapshot/tree.json index 0e7124fec..c26f3c042 100644 --- a/test/integ.transliterator.ecstask.ts.snapshot/tree.json +++ b/test/integ.transliterator.ecstask.ts.snapshot/tree.json @@ -1875,7 +1875,7 @@ { "essential": true, "image": { - "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}:163c30075527137d291e3aef3865f2d7fabc82baf8e77bfcba58e32d2ce57564" + "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}:94f2518f19be80aed25d2146b42522cce8c3b71210c12d3f8b9eeef608433e8b" }, "name": "Resource", "logConfiguration": { @@ -1898,6 +1898,14 @@ { "name": "AWS_EMF_ENVIRONMENT", "value": "Local" + }, + { + "name": "MAX_CONCURRENT_S3_REQUESTS", + "value": "16" + }, + { + "name": "MAX_RETRIES_S3_REQUESTS", + "value": "12" } ] } diff --git a/yarn.lock b/yarn.lock index 8842ca785..6044e64c8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2841,6 +2841,13 @@ dependencies: "@smithy/types" "^3.3.0" +"@smithy/service-error-classification@^3.0.4": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@smithy/service-error-classification/-/service-error-classification-3.0.4.tgz#60e07b596b38d316aca453e06bfe33464c622fb5" + integrity sha512-KciDHHKFVTb9A1KlJHBt2F26PBaDtoE23uTZy5qRvPzHPqrooXFi6fmx98lJb3Jl38PuUTqIuCUmmY3pacuMBQ== + dependencies: + "@smithy/types" "^3.4.0" + "@smithy/shared-ini-file-loader@^3.1.4": version "3.1.4" resolved "https://registry.yarnpkg.com/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.4.tgz#7dceaf5a5307a2ee347ace8aba17312a1a3ede15" @@ -3008,6 +3015,15 @@ "@smithy/types" "^3.3.0" tslib "^2.6.2" +"@smithy/util-retry@^3.0.4": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@smithy/util-retry/-/util-retry-3.0.4.tgz#281de3f89458b5e3b86ca92937eb1212bcecf67f" + integrity sha512-JJr6g0tO1qO2tCQyK+n3J18r34ZpvatlFN5ULcLranFIBZPxqoivb77EPyNTVwTGMEvvq2qMnyjm4jMIxjdLFg== + dependencies: + "@smithy/service-error-classification" "^3.0.4" + "@smithy/types" "^3.4.0" + tslib "^2.6.2" + "@smithy/util-stream@^3.1.3": version "3.1.3" resolved "https://registry.yarnpkg.com/@smithy/util-stream/-/util-stream-3.1.3.tgz#699ee2397cc1d474e46d2034039d5263812dca64"