Skip to content

Commit

Permalink
feat(endpoint): endpoints 2.0 existing package changes (#3947)
Browse files Browse the repository at this point in the history
* feat(endpoint): endpoints 2.0 existing package changes

* feat(endpoint): enable middleware stack identification

* feat(endpoint): update existing packages for endpoints v2

* feat(endpoint): address PR style comments

* feat(endpoint): fixes to types

* feat(endpoint): additional fixes from testing

* feat(endpoint): s3 presigned post fixes for endpoints 2.0

* feat(endpoint): unit test fixes

* feat(endpoint): type fixes

* feat(endpoint): types and docs updates

* feat(endpoint): s3 bucket name forcePathStyle

* feat(endpoint): type fix

* feat(endpoint): java code format

* feat(endpoint): type fix

Co-authored-by: AllanZhengYP <zheallan@amazon.com>
  • Loading branch information
kuhe and AllanZhengYP committed Sep 23, 2022
1 parent f91b252 commit df99fc3
Show file tree
Hide file tree
Showing 36 changed files with 566 additions and 184 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,8 @@ public final class AddS3Config implements TypeScriptIntegration {
);

private static final String CRT_NOTIFICATION = "<p>Note: To supply the Multi-region Access Point (MRAP) to Bucket,"
+ " you need to install the \"@aws-sdk/signature-v4-crt\" package to your project dependencies. \n"
+ "For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>";
+ " you need to install the \"@aws-sdk/signature-v4-crt\" package to your project dependencies. \n"
+ "For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>";

@Override
public Model preprocessModel(Model model, TypeScriptSettings settings) {
Expand Down Expand Up @@ -118,17 +118,19 @@ public void addConfigInterfaceFields(
return;
}
writer.writeDocs("Whether to escape request path when signing the request.")
.write("signingEscapePath?: boolean;\n");
.write("signingEscapePath?: boolean;\n");
writer.writeDocs(
"Whether to override the request region with the region inferred from requested resource's ARN."
+ " Defaults to false.")
.addImport("Provider", "Provider", TypeScriptDependency.AWS_SDK_TYPES.packageName)
.write("useArnRegion?: boolean | Provider<boolean>;");
+ " Defaults to false.")
.addImport("Provider", "Provider", TypeScriptDependency.AWS_SDK_TYPES.packageName)
.write("useArnRegion?: boolean | Provider<boolean>;");
}

@Override
public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(TypeScriptSettings settings, Model model,
SymbolProvider symbolProvider, LanguageTarget target) {
public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(
TypeScriptSettings settings, Model model,
SymbolProvider symbolProvider, LanguageTarget target
) {
if (!isS3(settings.getService(model))) {
return Collections.emptyMap();
}
Expand All @@ -140,18 +142,18 @@ public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(TypeScrip
writer.write("false");
}, "signerConstructor", writer -> {
writer.addDependency(AwsDependency.SIGNATURE_V4_MULTIREGION)
.addImport("SignatureV4MultiRegion", "SignatureV4MultiRegion",
AwsDependency.SIGNATURE_V4_MULTIREGION.packageName)
.write("SignatureV4MultiRegion");
.addImport("SignatureV4MultiRegion", "SignatureV4MultiRegion",
AwsDependency.SIGNATURE_V4_MULTIREGION.packageName)
.write("SignatureV4MultiRegion");
});
case NODE:
return MapUtils.of("useArnRegion", writer -> {
writer.addDependency(AwsDependency.NODE_CONFIG_PROVIDER)
.addImport("loadConfig", "loadNodeConfig", AwsDependency.NODE_CONFIG_PROVIDER.packageName)
.addDependency(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE)
.addImport("NODE_USE_ARN_REGION_CONFIG_OPTIONS", "NODE_USE_ARN_REGION_CONFIG_OPTIONS",
AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.packageName)
.write("loadNodeConfig(NODE_USE_ARN_REGION_CONFIG_OPTIONS)");
.addImport("loadConfig", "loadNodeConfig", AwsDependency.NODE_CONFIG_PROVIDER.packageName)
.addDependency(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE)
.addImport("NODE_USE_ARN_REGION_CONFIG_OPTIONS", "NODE_USE_ARN_REGION_CONFIG_OPTIONS",
AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.packageName)
.write("loadNodeConfig(NODE_USE_ARN_REGION_CONFIG_OPTIONS)");
});
default:
return Collections.emptyMap();
Expand All @@ -161,67 +163,67 @@ public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(TypeScrip
@Override
public List<RuntimeClientPlugin> getClientPlugins() {
return ListUtils.of(
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "ValidateBucketName",
HAS_MIDDLEWARE)
.servicePredicate((m, s) -> isS3(s))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "CheckContentLengthHeader",
HAS_MIDDLEWARE)
.operationPredicate((m, s, o) -> isS3(s) && o.getId().getName(s).equals("PutObject"))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "throw200Exceptions",
HAS_MIDDLEWARE)
.operationPredicate(
(m, s, o) -> EXCEPTIONS_OF_200_OPERATIONS.contains(o.getId().getName(s))
&& isS3(s))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency,
"WriteGetObjectResponseEndpoint", HAS_MIDDLEWARE)
.operationPredicate((m, s, o) -> isS3(s)
&& o.getId().getName(s).equals("WriteGetObjectResponse"))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.ADD_EXPECT_CONTINUE.dependency, "AddExpectContinue",
HAS_MIDDLEWARE)
.servicePredicate((m, s) -> isS3(s))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.SSEC_MIDDLEWARE.dependency, "Ssec", HAS_MIDDLEWARE)
.operationPredicate((m, s, o) -> containsInputMembers(m, o, SSEC_INPUT_KEYS)
&& isS3(s))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.LOCATION_CONSTRAINT.dependency, "LocationConstraint",
HAS_MIDDLEWARE)
.operationPredicate((m, s, o) -> o.getId().getName(s).equals("CreateBucket")
&& isS3(s))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "S3",
HAS_CONFIG)
.servicePredicate((m, s) -> isS3(s) && isEndpointsV2Service(s))
.build(),
/*
* BUCKET_ENDPOINT_MIDDLEWARE needs two separate plugins. The first resolves the config in the client.
* The second applies the middleware to bucket endpoint operations.
*/
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint",
HAS_CONFIG)
.servicePredicate((m, s) -> isS3(s) && !isEndpointsV2Service(s))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint",
HAS_MIDDLEWARE)
.operationPredicate((m, s, o) -> !NON_BUCKET_ENDPOINT_OPERATIONS.contains(o.getId().getName(s))
&& isS3(s)
&& !isEndpointsV2Service(s)
&& containsInputMembers(m, o, BUCKET_ENDPOINT_INPUT_KEYS))
.build()
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "ValidateBucketName",
HAS_MIDDLEWARE)
.servicePredicate((m, s) -> isS3(s))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "CheckContentLengthHeader",
HAS_MIDDLEWARE)
.operationPredicate((m, s, o) -> isS3(s) && o.getId().getName(s).equals("PutObject"))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "throw200Exceptions",
HAS_MIDDLEWARE)
.operationPredicate(
(m, s, o) -> EXCEPTIONS_OF_200_OPERATIONS.contains(o.getId().getName(s))
&& isS3(s))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency,
"WriteGetObjectResponseEndpoint", HAS_MIDDLEWARE)
.operationPredicate((m, s, o) -> isS3(s)
&& o.getId().getName(s).equals("WriteGetObjectResponse"))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.ADD_EXPECT_CONTINUE.dependency, "AddExpectContinue",
HAS_MIDDLEWARE)
.servicePredicate((m, s) -> isS3(s))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.SSEC_MIDDLEWARE.dependency, "Ssec", HAS_MIDDLEWARE)
.operationPredicate((m, s, o) -> containsInputMembers(m, o, SSEC_INPUT_KEYS)
&& isS3(s))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.LOCATION_CONSTRAINT.dependency, "LocationConstraint",
HAS_MIDDLEWARE)
.operationPredicate((m, s, o) -> o.getId().getName(s).equals("CreateBucket")
&& isS3(s))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "S3",
HAS_CONFIG)
.servicePredicate((m, s) -> isS3(s) && isEndpointsV2Service(s))
.build(),
/*
* BUCKET_ENDPOINT_MIDDLEWARE needs two separate plugins. The first resolves the config in the client.
* The second applies the middleware to bucket endpoint operations.
*/
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint",
HAS_CONFIG)
.servicePredicate((m, s) -> isS3(s) && !isEndpointsV2Service(s))
.build(),
RuntimeClientPlugin.builder()
.withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint",
HAS_MIDDLEWARE)
.operationPredicate((m, s, o) -> !NON_BUCKET_ENDPOINT_OPERATIONS.contains(o.getId().getName(s))
&& isS3(s)
&& !isEndpointsV2Service(s)
&& containsInputMembers(m, o, BUCKET_ENDPOINT_INPUT_KEYS))
.build()
);
}

Expand All @@ -232,8 +234,8 @@ private static boolean containsInputMembers(
) {
OperationIndex operationIndex = OperationIndex.of(model);
return operationIndex.getInput(operationShape)
.filter(input -> input.getMemberNames().stream().anyMatch(expectedMemberNames::contains))
.isPresent();
.filter(input -> input.getMemberNames().stream().anyMatch(expectedMemberNames::contains))
.isPresent();
}

private static boolean isS3(Shape serviceShape) {
Expand Down
2 changes: 2 additions & 0 deletions lib/lib-storage/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
},
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/middleware-endpoint": "*",
"@aws-sdk/smithy-client": "*",
"buffer": "5.6.0",
"events": "3.3.0",
Expand All @@ -37,6 +38,7 @@
"devDependencies": {
"@aws-sdk/abort-controller": "*",
"@aws-sdk/client-s3": "*",
"@aws-sdk/types": "*",
"@tsconfig/recommended": "1.0.1",
"@types/node": "^14.11.2",
"concurrently": "7.0.0",
Expand Down
27 changes: 20 additions & 7 deletions lib/lib-storage/src/Upload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,14 @@ import {
Tag,
UploadPartCommand,
} from "@aws-sdk/client-s3";
import {
EndpointParameterInstructionsSupplier,
getEndpointFromInstructions,
toEndpointV1,
} from "@aws-sdk/middleware-endpoint";
import { HttpRequest } from "@aws-sdk/protocol-http";
import { extendedEncodeURIComponent } from "@aws-sdk/smithy-client";
import { Endpoint } from "@aws-sdk/types";
import { EventEmitter } from "events";

import { byteLength } from "./bytelength";
Expand Down Expand Up @@ -101,7 +107,8 @@ export class Upload extends EventEmitter {
this.isMultiPart = false;
const params = { ...this.params, Body: dataPart.data };

const requestHandler = this.client.config.requestHandler;
const clientConfig = this.client.config;
const requestHandler = clientConfig.requestHandler;
const eventEmitter: EventEmitter | null = requestHandler instanceof EventEmitter ? requestHandler : null;
const uploadEventListener = (event: ProgressEvent) => {
this.bytesUploadedSoFar = event.loaded;
Expand All @@ -120,14 +127,20 @@ export class Upload extends EventEmitter {
eventEmitter.on("xhr.upload.progress", uploadEventListener);
}

const [putResult, endpoint] = await Promise.all([
this.client.send(new PutObjectCommand(params)),
this.client.config?.endpoint?.(),
]);
const resolved = await Promise.all([this.client.send(new PutObjectCommand(params)), clientConfig?.endpoint?.()]);
const putResult = resolved[0];
let endpoint: Endpoint = resolved[1];

if (!endpoint) {
endpoint = toEndpointV1(
await getEndpointFromInstructions(params, PutObjectCommand as EndpointParameterInstructionsSupplier, {
...clientConfig,
})
);
}

if (!endpoint) {
// TODO(endpointsv2): handle endpoint v2
throw new Error('Could not resolve endpoint from S3 "client.config.endpoint()".');
throw new Error('Could not resolve endpoint from S3 "client.config.endpoint()" nor EndpointsV2.');
}

if (eventEmitter !== null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ import { getEndpointFromRegion } from "./utils/getEndpointFromRegion";

export interface EndpointsInputConfig {
/**
* The fully qualified endpoint of the webservice. This is only required when using a custom endpoint (for example, when using a local version of S3).
* The fully qualified endpoint of the webservice. This is only required when using
* a custom endpoint (for example, when using a local version of S3).
*/
endpoint?: string | Endpoint | Provider<Endpoint>;

Expand Down Expand Up @@ -56,7 +57,7 @@ export const resolveEndpointsConfig = <T>(
endpoint: endpoint
? normalizeProvider(typeof endpoint === "string" ? urlParser(endpoint) : endpoint)
: () => getEndpointFromRegion({ ...input, useDualstackEndpoint, useFipsEndpoint }),
isCustomEndpoint: endpoint ? true : false,
isCustomEndpoint: !!endpoint,
useDualstackEndpoint,
};
};
2 changes: 2 additions & 0 deletions packages/middleware-endpoint/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"build": "concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types'",
"build:cjs": "tsc -p tsconfig.cjs.json",
"build:es": "tsc -p tsconfig.es.json",
"build:include:deps": "lerna run --scope $npm_package_name --include-dependencies build",
"build:types": "tsc -p tsconfig.types.json",
"build:types:downlevel": "downlevel-dts dist-types dist-types/ts3.4",
"clean": "rimraf ./dist-* && rimraf *.tsbuildinfo",
Expand All @@ -19,6 +20,7 @@
},
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/protocol-http": "*",
"@aws-sdk/signature-v4": "*",
"@aws-sdk/types": "*",
"@aws-sdk/util-config-provider": "*",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import { EndpointParameters, EndpointV2, HandlerExecutionContext } from "@aws-sdk/types";

import { EndpointResolvedConfig } from "../resolveEndpointConfig";
import { EndpointParameterInstructions } from "../types";

export type EndpointParameterInstructionsSupplier = Partial<{
getEndpointParameterInstructions(): EndpointParameterInstructions;
}>;

/**
* This step in the endpoint resolution process is exposed as a function
* to allow packages such as signers, lib-upload, etc. to get
* the V2 Endpoint associated to an instance of some api operation command
* without needing to send it or resolve its middleware stack.
*
* @private
* @param commandInput - the input of the Command in question.
* @param instructionsSupplier - this is typically a Command constructor. A static function supplying the
* endpoint parameter instructions will exist for commands in services
* having an endpoints ruleset trait.
* @param clientConfig - config of the service client.
* @param context - optional context.
*/
export const getEndpointFromInstructions = async <
T extends EndpointParameters,
CommandInput extends Record<string, unknown>,
Config extends Record<string, unknown>
>(
commandInput: CommandInput,
instructionsSupplier: EndpointParameterInstructionsSupplier,
clientConfig: Partial<EndpointResolvedConfig<T>> & Config,
context?: HandlerExecutionContext
): Promise<EndpointV2> => {
const endpointParams: EndpointParameters = {};
const instructions: EndpointParameterInstructions =
(instructionsSupplier.getEndpointParameterInstructions || (() => null))() || {};

if (typeof clientConfig.endpointProvider !== "function") {
throw new Error("config.endpointProvider is not set.");
}

for (const [name, instruction] of Object.entries(instructions)) {
switch (instruction.type) {
case "staticContextParams":
endpointParams[name] = instruction.value;
break;
case "contextParams":
endpointParams[name] = commandInput[instruction.name] as string | boolean;
break;
case "clientContextParams":
case "builtInParams":
endpointParams[name] = await createConfigProvider<Config>(instruction.name, clientConfig)();
break;
default:
throw new Error("Unrecognized endpoint parameter instruction: " + JSON.stringify(instruction));
}
}

const endpoint: EndpointV2 = clientConfig.endpointProvider!(endpointParams as T, context);

return endpoint;
};

/**
* Normalize some key of the client config to an async provider.
* @private
*/
const createConfigProvider = <Config extends Record<string, unknown>>(configKey: string, config: Config) => {
const configProvider = async () => {
const configValue: unknown = config[configKey];
if (typeof configValue === "function") {
return configValue();
}
return configValue;
};
return configProvider;
};
2 changes: 2 additions & 0 deletions packages/middleware-endpoint/src/adaptors/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
export * from "./getEndpointFromInstructions";
export * from "./toEndpointV1";
Loading

0 comments on commit df99fc3

Please sign in to comment.