diff --git a/clients/client-s3/src/S3Client.ts b/clients/client-s3/src/S3Client.ts index 57d8b7200ed1..6f72b42688f8 100644 --- a/clients/client-s3/src/S3Client.ts +++ b/clients/client-s3/src/S3Client.ts @@ -9,6 +9,7 @@ import { import { getLoggerPlugin } from "@aws-sdk/middleware-logger"; import { getRecursionDetectionPlugin } from "@aws-sdk/middleware-recursion-detection"; import { + getRegionRedirectMiddlewarePlugin, getValidateBucketNamePlugin, resolveS3Config, S3InputConfig, @@ -778,6 +779,7 @@ export class S3Client extends __Client< this.middlewareStack.use(getAwsAuthPlugin(this.config)); this.middlewareStack.use(getValidateBucketNamePlugin(this.config)); this.middlewareStack.use(getAddExpectContinuePlugin(this.config)); + this.middlewareStack.use(getRegionRedirectMiddlewarePlugin(this.config)); this.middlewareStack.use(getUserAgentPlugin(this.config)); } diff --git a/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java b/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java index 4d86227d61c6..39fa5aea7299 100644 --- a/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java +++ b/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java @@ -226,6 +226,11 @@ && isS3(s)) && isS3(s) && !isEndpointsV2Service(s) && containsInputMembers(m, o, BUCKET_ENDPOINT_INPUT_KEYS)) + .build(), + RuntimeClientPlugin.builder() + .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "RegionRedirectMiddleware", + HAS_MIDDLEWARE) + .servicePredicate((m, s) -> isS3(s)) .build() ); } diff --git a/packages/middleware-sdk-s3/jest.config.e2e.js b/packages/middleware-sdk-s3/jest.config.e2e.js new file mode 100644 index 000000000000..b4d9bee23f48 --- /dev/null +++ b/packages/middleware-sdk-s3/jest.config.e2e.js @@ -0,0 +1,4 @@ +module.exports = { + preset: "ts-jest", + testMatch: ["**/*.e2e.spec.ts"], +}; diff --git a/packages/middleware-sdk-s3/package.json b/packages/middleware-sdk-s3/package.json index 4df6f1f61fab..60b8e15bc00c 100644 --- a/packages/middleware-sdk-s3/package.json +++ b/packages/middleware-sdk-s3/package.json @@ -11,6 +11,7 @@ "clean": "rimraf ./dist-* && rimraf *.tsbuildinfo", "test": "jest", "test:integration": "jest -c jest.config.integ.js", + "test:e2e": "jest -c jest.config.e2e.js", "extract:docs": "api-extractor run --local" }, "main": "./dist-cjs/index.js", diff --git a/packages/middleware-sdk-s3/src/index.ts b/packages/middleware-sdk-s3/src/index.ts index 92c9f7291a67..a23a0aadceba 100644 --- a/packages/middleware-sdk-s3/src/index.ts +++ b/packages/middleware-sdk-s3/src/index.ts @@ -1,4 +1,6 @@ export * from "./check-content-length-header"; +export * from "./region-redirect-endpoint-middleware"; +export * from "./region-redirect-middleware"; export * from "./s3Configuration"; export * from "./throw-200-exceptions"; export * from "./validate-bucket-name"; diff --git a/packages/middleware-sdk-s3/src/region-redirect-endpoint-middleware.ts b/packages/middleware-sdk-s3/src/region-redirect-endpoint-middleware.ts new file mode 100644 index 000000000000..ece8a964ace1 --- /dev/null +++ b/packages/middleware-sdk-s3/src/region-redirect-endpoint-middleware.ts @@ -0,0 +1,50 @@ +import { + HandlerExecutionContext, + MetadataBearer, + RelativeMiddlewareOptions, + SerializeHandler, + SerializeHandlerArguments, + SerializeHandlerOutput, + SerializeMiddleware, +} from "@smithy/types"; + +import { PreviouslyResolved } from "./region-redirect-middleware"; + +/** + * @internal + */ +export const regionRedirectEndpointMiddleware = (config: PreviouslyResolved): SerializeMiddleware => { + return ( + next: SerializeHandler, + context: HandlerExecutionContext + ): SerializeHandler => + async (args: SerializeHandlerArguments): Promise> => { + const originalRegion = await config.region(); + const regionProviderRef = config.region; + if (context.__s3RegionRedirect) { + config.region = async () => { + config.region = regionProviderRef; + return context.__s3RegionRedirect; + }; + } + const result = await next(args); + if (context.__s3RegionRedirect) { + const region = await config.region(); + if (originalRegion !== region) { + throw new Error("Region was not restored following S3 region redirect."); + } + } + return result; + }; +}; + +/** + * @internal + */ +export const regionRedirectEndpointMiddlewareOptions: RelativeMiddlewareOptions = { + tags: ["REGION_REDIRECT", "S3"], + name: "regionRedirectEndpointMiddleware", + override: true, + relation: "before", + toMiddleware: "endpointV2Middleware", +}; diff --git a/packages/middleware-sdk-s3/src/region-redirect-middleware.e2e.spec.ts b/packages/middleware-sdk-s3/src/region-redirect-middleware.e2e.spec.ts new file mode 100644 index 000000000000..7d68644570d0 --- /dev/null +++ b/packages/middleware-sdk-s3/src/region-redirect-middleware.e2e.spec.ts @@ -0,0 +1,106 @@ +import { S3 } from "@aws-sdk/client-s3"; +import { GetCallerIdentityCommandOutput, STS } from "@aws-sdk/client-sts"; + +const testValue = "Hello S3 global client!"; + +describe("S3 Global Client Test", () => { + const regionConfigs = [ + { region: "us-east-1", followRegionRedirects: true }, + { region: "eu-west-1", followRegionRedirects: true }, + { region: "us-west-2", followRegionRedirects: true }, + ]; + const s3Clients = regionConfigs.map((config) => new S3(config)); + const stsClient = new STS({}); + + let callerID = null as unknown as GetCallerIdentityCommandOutput; + let bucketNames = [] as string[]; + + beforeAll(async () => { + jest.setTimeout(500000); + callerID = await stsClient.getCallerIdentity({}); + bucketNames = regionConfigs.map((config) => `${callerID.Account}-redirect-${config.region}`); + await Promise.all(bucketNames.map((bucketName, index) => deleteBucket(s3Clients[index], bucketName))); + await Promise.all(bucketNames.map((bucketName, index) => s3Clients[index].createBucket({ Bucket: bucketName }))); + }); + + afterAll(async () => { + await Promise.all(bucketNames.map((bucketName, index) => deleteBucket(s3Clients[index], bucketName))); + }); + + it("Should be able to put objects following region redirect", async () => { + // Upload objects to each bucket + for (const bucketName of bucketNames) { + for (const s3Client of s3Clients) { + const objKey = `object-from-${await s3Client.config.region()}-client`; + await s3Client.putObject({ Bucket: bucketName, Key: objKey, Body: testValue }); + } + } + }, 50000); + + it("Should be able to get objects following region redirect", async () => { + // Fetch and assert objects + for (const bucketName of bucketNames) { + for (const s3Client of s3Clients) { + const objKey = `object-from-${await s3Client.config.region()}-client`; + const { Body } = await s3Client.getObject({ Bucket: bucketName, Key: objKey }); + const data = await Body?.transformToString(); + expect(data).toEqual(testValue); + } + } + }, 50000); + + it("Should delete objects following region redirect", async () => { + for (const bucketName of bucketNames) { + for (const s3Client of s3Clients) { + const objKey = `object-from-${await s3Client.config.region()}-client`; + await s3Client.deleteObject({ Bucket: bucketName, Key: objKey }); + } + } + }, 50000); +}); + +async function deleteBucket(s3: S3, bucketName: string) { + const Bucket = bucketName; + + try { + await s3.headBucket({ + Bucket, + }); + } catch (e) { + return; + } + + const list = await s3 + .listObjects({ + Bucket, + }) + .catch((e) => { + if (!String(e).includes("NoSuchBucket")) { + throw e; + } + return { + Contents: [], + }; + }); + + const promises = [] as any[]; + for (const key of list.Contents ?? []) { + promises.push( + s3.deleteObject({ + Bucket, + Key: key.Key, + }) + ); + } + await Promise.all(promises); + + try { + return await s3.deleteBucket({ + Bucket, + }); + } catch (e) { + if (!String(e).includes("NoSuchBucket")) { + throw e; + } + } +} diff --git a/packages/middleware-sdk-s3/src/region-redirect-middleware.spec.ts b/packages/middleware-sdk-s3/src/region-redirect-middleware.spec.ts new file mode 100644 index 000000000000..b159333623ca --- /dev/null +++ b/packages/middleware-sdk-s3/src/region-redirect-middleware.spec.ts @@ -0,0 +1,50 @@ +import { HandlerExecutionContext } from "@smithy/types"; + +import { regionRedirectMiddleware } from "./region-redirect-middleware"; + +describe(regionRedirectMiddleware.name, () => { + const region = async () => "us-east-1"; + const redirectRegion = "us-west-2"; + let call = 0; + const next = (arg: any) => { + if (call === 0) { + call++; + throw Object.assign(new Error(), { + name: "PermanentRedirect", + $metadata: { httpStatusCode: 301 }, + $response: { headers: { "x-amz-bucket-region": redirectRegion } }, + }); + } + return null as any; + }; + + beforeEach(() => { + call = 0; + }); + + it("set S3 region redirect on context if receiving a PermanentRedirect error code with status 301", async () => { + const middleware = regionRedirectMiddleware({ region, followRegionRedirects: true }); + const context = {} as HandlerExecutionContext; + const handler = middleware(next, context); + await handler({ input: null }); + expect(context.__s3RegionRedirect).toEqual(redirectRegion); + }); + + it("does not follow the redirect when followRegionRedirects is false", async () => { + const middleware = regionRedirectMiddleware({ region, followRegionRedirects: false }); + const context = {} as HandlerExecutionContext; + const handler = middleware(next, context); + // Simulating a PermanentRedirect error with status 301 + await expect(async () => { + await handler({ input: null }); + }).rejects.toThrowError( + Object.assign(new Error(), { + Code: "PermanentRedirect", + $metadata: { httpStatusCode: 301 }, + $response: { headers: { "x-amz-bucket-region": redirectRegion } }, + }) + ); + // Ensure that context.__s3RegionRedirect is not set + expect(context.__s3RegionRedirect).toBeUndefined(); + }); +}); diff --git a/packages/middleware-sdk-s3/src/region-redirect-middleware.ts b/packages/middleware-sdk-s3/src/region-redirect-middleware.ts new file mode 100644 index 000000000000..f16bfd7465ff --- /dev/null +++ b/packages/middleware-sdk-s3/src/region-redirect-middleware.ts @@ -0,0 +1,77 @@ +import { + HandlerExecutionContext, + InitializeHandler, + InitializeHandlerArguments, + InitializeHandlerOptions, + InitializeHandlerOutput, + InitializeMiddleware, + MetadataBearer, + Pluggable, + Provider, +} from "@smithy/types"; + +import { + regionRedirectEndpointMiddleware, + regionRedirectEndpointMiddlewareOptions, +} from "./region-redirect-endpoint-middleware"; + +/** + * @internal + */ +export interface PreviouslyResolved { + region: Provider; + followRegionRedirects: boolean; +} + +/** + * @internal + */ +export function regionRedirectMiddleware(clientConfig: PreviouslyResolved): InitializeMiddleware { + return ( + next: InitializeHandler, + context: HandlerExecutionContext + ): InitializeHandler => + async (args: InitializeHandlerArguments): Promise> => { + try { + return await next(args); + } catch (err) { + // console.log("Region Redirect", clientConfig.followRegionRedirects, err.name, err.$metadata.httpStatusCode); + if ( + clientConfig.followRegionRedirects && + err.name === "PermanentRedirect" && + err.$metadata.httpStatusCode === 301 + ) { + try { + const actualRegion = err.$response.headers["x-amz-bucket-region"]; + context.logger?.debug(`Redirecting from ${await clientConfig.region()} to ${actualRegion}`); + context.__s3RegionRedirect = actualRegion; + } catch (e) { + throw new Error("Region redirect failed: " + e); + } + return next(args); + } else { + throw err; + } + } + }; +} + +/** + * @internal + */ +export const regionRedirectMiddlewareOptions: InitializeHandlerOptions = { + step: "initialize", + tags: ["REGION_REDIRECT", "S3"], + name: "regionRedirectMiddleware", + override: true, +}; + +/** + * @internal + */ +export const getRegionRedirectMiddlewarePlugin = (clientConfig: PreviouslyResolved): Pluggable => ({ + applyToStack: (clientStack) => { + clientStack.add(regionRedirectMiddleware(clientConfig), regionRedirectMiddlewareOptions); + clientStack.addRelativeTo(regionRedirectEndpointMiddleware(clientConfig), regionRedirectEndpointMiddlewareOptions); + }, +}); diff --git a/packages/middleware-sdk-s3/src/s3Configuration.ts b/packages/middleware-sdk-s3/src/s3Configuration.ts index ba91f3f16293..33993c23aafa 100644 --- a/packages/middleware-sdk-s3/src/s3Configuration.ts +++ b/packages/middleware-sdk-s3/src/s3Configuration.ts @@ -1,6 +1,6 @@ /** * @public - * + * * All endpoint parameters with built-in bindings of AWS::S3::* */ export interface S3InputConfig { @@ -17,12 +17,20 @@ export interface S3InputConfig { * Whether multi-region access points (MRAP) should be disabled. */ disableMultiregionAccessPoints?: boolean; + /** + * This feature was previously called the S3 Global Client. + * This can result in additional latency as failed requests are retried + * with a corrected region when receiving a permanent redirect error with status 301. + * This feature should only be used as a last resort if you do not know the region of your bucket(s) ahead of time. + */ + followRegionRedirects?: boolean; } export interface S3ResolvedConfig { forcePathStyle: boolean; useAccelerateEndpoint: boolean; disableMultiregionAccessPoints: boolean; + followRegionRedirects: boolean; } export const resolveS3Config = (input: T & S3InputConfig): T & S3ResolvedConfig => ({ @@ -30,4 +38,5 @@ export const resolveS3Config = (input: T & S3InputConfig): T & S3ResolvedConf forcePathStyle: input.forcePathStyle ?? false, useAccelerateEndpoint: input.useAccelerateEndpoint ?? false, disableMultiregionAccessPoints: input.disableMultiregionAccessPoints ?? false, + followRegionRedirects: input.followRegionRedirects ?? false, });