fix(transliterator): s3 requests are capped after 3 retries (#1445)

The previous retry strategy used with the SDK v2 did retry for 12 times
before the back-off tipped over a threshold.
Because we are seeing tasks fail due to S3 requests throttled, this PR
brings the retries for SDK v3 in line with the previous amount. It also
adds environment variables to make the s3 concurrency and retry limits
configurable via environment variables to allow for better testing.


----

*By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license*

.projenrc.ts

@@ -50,6 +50,7 @@ const project = new CdklabsConstructLibrary({
     '@aws-sdk/client-codeartifact',
     '@aws-sdk/client-s3',
     '@smithy/types',
+    '@smithy/util-retry',
     '@smithy/util-stream',
     'async-sema',
     'aws-embedded-metrics',

src/__tests__/backend/transliterator/index.test.ts

@@ -58,6 +58,14 @@ test('CodeArtifact repository', () => {
         Environment: stack.resolve([
           { Name: 'HEADER_SPAN', Value: 'true' },
           { Name: 'AWS_EMF_ENVIRONMENT', Value: 'Local' },
+          {
+            Name: 'MAX_CONCURRENT_S3_REQUESTS',
+            Value: '16',
+          },
+          {
+            Name: 'MAX_RETRIES_S3_REQUESTS',
+            Value: '12',
+          },
           {
             Name: 'CODE_ARTIFACT_DOMAIN_NAME',
             Value: codeArtifact.repositoryDomainName,
@@ -138,6 +146,14 @@ test('VPC Endpoints', () => {
         Environment: stack.resolve([
           { Name: 'HEADER_SPAN', Value: 'true' },
           { Name: 'AWS_EMF_ENVIRONMENT', Value: 'Local' },
+          {
+            Name: 'MAX_CONCURRENT_S3_REQUESTS',
+            Value: '16',
+          },
+          {
+            Name: 'MAX_RETRIES_S3_REQUESTS',
+            Value: '12',
+          },
           {
             Name: 'CODE_ARTIFACT_API_ENDPOINT',
             Value: Fn.select(
@@ -214,6 +230,14 @@ test('VPC Endpoints and CodeArtifact repository', () => {
         Environment: stack.resolve([
           { Name: 'HEADER_SPAN', Value: 'true' },
           { Name: 'AWS_EMF_ENVIRONMENT', Value: 'Local' },
+          {
+            Name: 'MAX_CONCURRENT_S3_REQUESTS',
+            Value: '16',
+          },
+          {
+            Name: 'MAX_RETRIES_S3_REQUESTS',
+            Value: '12',
+          },
           {
             Name: 'CODE_ARTIFACT_API_ENDPOINT',
             Value: Fn.select(