-
Notifications
You must be signed in to change notification settings - Fork 575
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(middleware-sdk-s3): add middleware for following region redirects (
#5185) * feat(middleware-sdk-s3): add middleware for following region redirects * test(middleware-sdk-s3): add initial unit tests for region redirect middleware * test(middleware-sdk-s3): unit test addition and initial E2E test * test(middleware-sdk-s3): increase timeout for E2E test * chore(middleware-sdk-s3): adding an await and nit fix * chore(middleware-sdk-s3): split region redirect middlewares in different files * fix(middleware-sdk-s3): bug fix for middleware and test refactor * chore(middleware-sdk-s3): doc update
- Loading branch information
Showing
10 changed files
with
307 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
module.exports = { | ||
preset: "ts-jest", | ||
testMatch: ["**/*.e2e.spec.ts"], | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,6 @@ | ||
export * from "./check-content-length-header"; | ||
export * from "./region-redirect-endpoint-middleware"; | ||
export * from "./region-redirect-middleware"; | ||
export * from "./s3Configuration"; | ||
export * from "./throw-200-exceptions"; | ||
export * from "./validate-bucket-name"; |
50 changes: 50 additions & 0 deletions
50
packages/middleware-sdk-s3/src/region-redirect-endpoint-middleware.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
import { | ||
HandlerExecutionContext, | ||
MetadataBearer, | ||
RelativeMiddlewareOptions, | ||
SerializeHandler, | ||
SerializeHandlerArguments, | ||
SerializeHandlerOutput, | ||
SerializeMiddleware, | ||
} from "@smithy/types"; | ||
|
||
import { PreviouslyResolved } from "./region-redirect-middleware"; | ||
|
||
/** | ||
* @internal | ||
*/ | ||
export const regionRedirectEndpointMiddleware = (config: PreviouslyResolved): SerializeMiddleware<any, any> => { | ||
return <Output extends MetadataBearer>( | ||
next: SerializeHandler<any, Output>, | ||
context: HandlerExecutionContext | ||
): SerializeHandler<any, Output> => | ||
async (args: SerializeHandlerArguments<any>): Promise<SerializeHandlerOutput<Output>> => { | ||
const originalRegion = await config.region(); | ||
const regionProviderRef = config.region; | ||
if (context.__s3RegionRedirect) { | ||
config.region = async () => { | ||
config.region = regionProviderRef; | ||
return context.__s3RegionRedirect; | ||
}; | ||
} | ||
const result = await next(args); | ||
if (context.__s3RegionRedirect) { | ||
const region = await config.region(); | ||
if (originalRegion !== region) { | ||
throw new Error("Region was not restored following S3 region redirect."); | ||
} | ||
} | ||
return result; | ||
}; | ||
}; | ||
|
||
/** | ||
* @internal | ||
*/ | ||
export const regionRedirectEndpointMiddlewareOptions: RelativeMiddlewareOptions = { | ||
tags: ["REGION_REDIRECT", "S3"], | ||
name: "regionRedirectEndpointMiddleware", | ||
override: true, | ||
relation: "before", | ||
toMiddleware: "endpointV2Middleware", | ||
}; |
106 changes: 106 additions & 0 deletions
106
packages/middleware-sdk-s3/src/region-redirect-middleware.e2e.spec.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,106 @@ | ||
import { S3 } from "@aws-sdk/client-s3"; | ||
import { GetCallerIdentityCommandOutput, STS } from "@aws-sdk/client-sts"; | ||
|
||
const testValue = "Hello S3 global client!"; | ||
|
||
describe("S3 Global Client Test", () => { | ||
const regionConfigs = [ | ||
{ region: "us-east-1", followRegionRedirects: true }, | ||
{ region: "eu-west-1", followRegionRedirects: true }, | ||
{ region: "us-west-2", followRegionRedirects: true }, | ||
]; | ||
const s3Clients = regionConfigs.map((config) => new S3(config)); | ||
const stsClient = new STS({}); | ||
|
||
let callerID = null as unknown as GetCallerIdentityCommandOutput; | ||
let bucketNames = [] as string[]; | ||
|
||
beforeAll(async () => { | ||
jest.setTimeout(500000); | ||
callerID = await stsClient.getCallerIdentity({}); | ||
bucketNames = regionConfigs.map((config) => `${callerID.Account}-redirect-${config.region}`); | ||
await Promise.all(bucketNames.map((bucketName, index) => deleteBucket(s3Clients[index], bucketName))); | ||
await Promise.all(bucketNames.map((bucketName, index) => s3Clients[index].createBucket({ Bucket: bucketName }))); | ||
}); | ||
|
||
afterAll(async () => { | ||
await Promise.all(bucketNames.map((bucketName, index) => deleteBucket(s3Clients[index], bucketName))); | ||
}); | ||
|
||
it("Should be able to put objects following region redirect", async () => { | ||
// Upload objects to each bucket | ||
for (const bucketName of bucketNames) { | ||
for (const s3Client of s3Clients) { | ||
const objKey = `object-from-${await s3Client.config.region()}-client`; | ||
await s3Client.putObject({ Bucket: bucketName, Key: objKey, Body: testValue }); | ||
} | ||
} | ||
}, 50000); | ||
|
||
it("Should be able to get objects following region redirect", async () => { | ||
// Fetch and assert objects | ||
for (const bucketName of bucketNames) { | ||
for (const s3Client of s3Clients) { | ||
const objKey = `object-from-${await s3Client.config.region()}-client`; | ||
const { Body } = await s3Client.getObject({ Bucket: bucketName, Key: objKey }); | ||
const data = await Body?.transformToString(); | ||
expect(data).toEqual(testValue); | ||
} | ||
} | ||
}, 50000); | ||
|
||
it("Should delete objects following region redirect", async () => { | ||
for (const bucketName of bucketNames) { | ||
for (const s3Client of s3Clients) { | ||
const objKey = `object-from-${await s3Client.config.region()}-client`; | ||
await s3Client.deleteObject({ Bucket: bucketName, Key: objKey }); | ||
} | ||
} | ||
}, 50000); | ||
}); | ||
|
||
async function deleteBucket(s3: S3, bucketName: string) { | ||
const Bucket = bucketName; | ||
|
||
try { | ||
await s3.headBucket({ | ||
Bucket, | ||
}); | ||
} catch (e) { | ||
return; | ||
} | ||
|
||
const list = await s3 | ||
.listObjects({ | ||
Bucket, | ||
}) | ||
.catch((e) => { | ||
if (!String(e).includes("NoSuchBucket")) { | ||
throw e; | ||
} | ||
return { | ||
Contents: [], | ||
}; | ||
}); | ||
|
||
const promises = [] as any[]; | ||
for (const key of list.Contents ?? []) { | ||
promises.push( | ||
s3.deleteObject({ | ||
Bucket, | ||
Key: key.Key, | ||
}) | ||
); | ||
} | ||
await Promise.all(promises); | ||
|
||
try { | ||
return await s3.deleteBucket({ | ||
Bucket, | ||
}); | ||
} catch (e) { | ||
if (!String(e).includes("NoSuchBucket")) { | ||
throw e; | ||
} | ||
} | ||
} |
50 changes: 50 additions & 0 deletions
50
packages/middleware-sdk-s3/src/region-redirect-middleware.spec.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
import { HandlerExecutionContext } from "@smithy/types"; | ||
|
||
import { regionRedirectMiddleware } from "./region-redirect-middleware"; | ||
|
||
describe(regionRedirectMiddleware.name, () => { | ||
const region = async () => "us-east-1"; | ||
const redirectRegion = "us-west-2"; | ||
let call = 0; | ||
const next = (arg: any) => { | ||
if (call === 0) { | ||
call++; | ||
throw Object.assign(new Error(), { | ||
name: "PermanentRedirect", | ||
$metadata: { httpStatusCode: 301 }, | ||
$response: { headers: { "x-amz-bucket-region": redirectRegion } }, | ||
}); | ||
} | ||
return null as any; | ||
}; | ||
|
||
beforeEach(() => { | ||
call = 0; | ||
}); | ||
|
||
it("set S3 region redirect on context if receiving a PermanentRedirect error code with status 301", async () => { | ||
const middleware = regionRedirectMiddleware({ region, followRegionRedirects: true }); | ||
const context = {} as HandlerExecutionContext; | ||
const handler = middleware(next, context); | ||
await handler({ input: null }); | ||
expect(context.__s3RegionRedirect).toEqual(redirectRegion); | ||
}); | ||
|
||
it("does not follow the redirect when followRegionRedirects is false", async () => { | ||
const middleware = regionRedirectMiddleware({ region, followRegionRedirects: false }); | ||
const context = {} as HandlerExecutionContext; | ||
const handler = middleware(next, context); | ||
// Simulating a PermanentRedirect error with status 301 | ||
await expect(async () => { | ||
await handler({ input: null }); | ||
}).rejects.toThrowError( | ||
Object.assign(new Error(), { | ||
Code: "PermanentRedirect", | ||
$metadata: { httpStatusCode: 301 }, | ||
$response: { headers: { "x-amz-bucket-region": redirectRegion } }, | ||
}) | ||
); | ||
// Ensure that context.__s3RegionRedirect is not set | ||
expect(context.__s3RegionRedirect).toBeUndefined(); | ||
}); | ||
}); |
77 changes: 77 additions & 0 deletions
77
packages/middleware-sdk-s3/src/region-redirect-middleware.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,77 @@ | ||
import { | ||
HandlerExecutionContext, | ||
InitializeHandler, | ||
InitializeHandlerArguments, | ||
InitializeHandlerOptions, | ||
InitializeHandlerOutput, | ||
InitializeMiddleware, | ||
MetadataBearer, | ||
Pluggable, | ||
Provider, | ||
} from "@smithy/types"; | ||
|
||
import { | ||
regionRedirectEndpointMiddleware, | ||
regionRedirectEndpointMiddlewareOptions, | ||
} from "./region-redirect-endpoint-middleware"; | ||
|
||
/** | ||
* @internal | ||
*/ | ||
export interface PreviouslyResolved { | ||
region: Provider<string>; | ||
followRegionRedirects: boolean; | ||
} | ||
|
||
/** | ||
* @internal | ||
*/ | ||
export function regionRedirectMiddleware(clientConfig: PreviouslyResolved): InitializeMiddleware<any, any> { | ||
return <Output extends MetadataBearer>( | ||
next: InitializeHandler<any, Output>, | ||
context: HandlerExecutionContext | ||
): InitializeHandler<any, Output> => | ||
async (args: InitializeHandlerArguments<any>): Promise<InitializeHandlerOutput<Output>> => { | ||
try { | ||
return await next(args); | ||
} catch (err) { | ||
// console.log("Region Redirect", clientConfig.followRegionRedirects, err.name, err.$metadata.httpStatusCode); | ||
if ( | ||
clientConfig.followRegionRedirects && | ||
err.name === "PermanentRedirect" && | ||
err.$metadata.httpStatusCode === 301 | ||
) { | ||
try { | ||
const actualRegion = err.$response.headers["x-amz-bucket-region"]; | ||
context.logger?.debug(`Redirecting from ${await clientConfig.region()} to ${actualRegion}`); | ||
context.__s3RegionRedirect = actualRegion; | ||
} catch (e) { | ||
throw new Error("Region redirect failed: " + e); | ||
} | ||
return next(args); | ||
} else { | ||
throw err; | ||
} | ||
} | ||
}; | ||
} | ||
|
||
/** | ||
* @internal | ||
*/ | ||
export const regionRedirectMiddlewareOptions: InitializeHandlerOptions = { | ||
step: "initialize", | ||
tags: ["REGION_REDIRECT", "S3"], | ||
name: "regionRedirectMiddleware", | ||
override: true, | ||
}; | ||
|
||
/** | ||
* @internal | ||
*/ | ||
export const getRegionRedirectMiddlewarePlugin = (clientConfig: PreviouslyResolved): Pluggable<any, any> => ({ | ||
applyToStack: (clientStack) => { | ||
clientStack.add(regionRedirectMiddleware(clientConfig), regionRedirectMiddlewareOptions); | ||
clientStack.addRelativeTo(regionRedirectEndpointMiddleware(clientConfig), regionRedirectEndpointMiddlewareOptions); | ||
}, | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters