diff --git a/packages/types/src/blob/blob-types.ts b/packages/types/src/blob/blob-types.ts index 3c3b49dd3d837..f8fe75a9014e8 100644 --- a/packages/types/src/blob/blob-types.ts +++ b/packages/types/src/blob/blob-types.ts @@ -3,6 +3,21 @@ import { RuntimeBlobTypes } from "./runtime-blob-types.node"; /** * @public * - * A set of types that can be used as inputs for the blob type. + * A union of types that can be used as inputs for the service model + * "blob" type when it represents the request's entire payload or body. + * + * For example, in Lambda::invoke, the payload is modeled as a blob type + * and this union applies to it. + * In contrast, in Lambda::createFunction the Zip file option is a blob type, + * but is not the (entire) payload and this union does not apply. + * + * Note: not all types are signable by the standard SignatureV4 signer when + * used as the request body. For example, in Node.js a Readable stream + * is not signable by the default signer. + * They are included in the union because it may work in some cases, + * but the expected types are primarily string and Uint8Array. + * + * Additional details may be found in the internal + * function "getPayloadHash" in the SignatureV4 module. */ export type BlobTypes = string | ArrayBuffer | ArrayBufferView | Uint8Array | RuntimeBlobTypes; diff --git a/packages/util-stream/jest.config.e2e.js b/packages/util-stream/jest.config.e2e.js new file mode 100644 index 0000000000000..b4d9bee23f482 --- /dev/null +++ b/packages/util-stream/jest.config.e2e.js @@ -0,0 +1,4 @@ +module.exports = { + preset: "ts-jest", + testMatch: ["**/*.e2e.spec.ts"], +}; diff --git a/packages/util-stream/jest.config.integ.js b/packages/util-stream/jest.config.integ.js new file mode 100644 index 0000000000000..d09aba7398c72 --- /dev/null +++ b/packages/util-stream/jest.config.integ.js @@ -0,0 +1,4 @@ +module.exports = { + preset: "ts-jest", + testMatch: ["**/*.integ.spec.ts"], +}; diff --git a/packages/util-stream/jest.config.js b/packages/util-stream/jest.config.js index 95d8863b22a18..a8d1c2e499123 100644 --- a/packages/util-stream/jest.config.js +++ b/packages/util-stream/jest.config.js @@ -2,5 +2,4 @@ const base = require("../../jest.config.base.js"); module.exports = { ...base, - testMatch: ["**/*.spec.ts"], }; diff --git a/packages/util-stream/package.json b/packages/util-stream/package.json index 1592714783f5e..3e3640b686861 100644 --- a/packages/util-stream/package.json +++ b/packages/util-stream/package.json @@ -9,7 +9,9 @@ "build:types": "tsc -p tsconfig.types.json", "build:types:downlevel": "downlevel-dts dist-types dist-types/ts3.4", "clean": "rimraf ./dist-* && rimraf *.tsbuildinfo", - "test": "jest" + "test": "jest", + "test:integration": "jest -c jest.config.integ.js", + "test:e2e": "jest -c jest.config.e2e.js" }, "main": "./dist-cjs/index.js", "module": "./dist-es/index.js", @@ -52,13 +54,11 @@ ], "browser": { "./dist-es/getAwsChunkedEncodingStream": "./dist-es/getAwsChunkedEncodingStream.browser", - "./dist-es/sdk-stream-mixin": "./dist-es/sdk-stream-mixin.browser", - "./dist-es/blob/decode": "./dist-es/blob/decode.browser" + "./dist-es/sdk-stream-mixin": "./dist-es/sdk-stream-mixin.browser" }, "react-native": { "./dist-es/getAwsChunkedEncodingStream": "./dist-es/getAwsChunkedEncodingStream.browser", - "./dist-es/sdk-stream-mixin": "./dist-es/sdk-stream-mixin.browser", - "./dist-es/blob/decode": "./dist-es/blob/decode.browser" + "./dist-es/sdk-stream-mixin": "./dist-es/sdk-stream-mixin.browser" }, "homepage": "https://github.com/aws/aws-sdk-js-v3/tree/main/packages/util-stream", "repository": { diff --git a/packages/util-stream/src/blob/Uint8ArrayBlobAdapter.spec.ts b/packages/util-stream/src/blob/Uint8ArrayBlobAdapter.spec.ts index 2e7a4a05b8f43..c9c27d248a1aa 100644 --- a/packages/util-stream/src/blob/Uint8ArrayBlobAdapter.spec.ts +++ b/packages/util-stream/src/blob/Uint8ArrayBlobAdapter.spec.ts @@ -1,8 +1,8 @@ -import { BlobAdapter } from "./BlobAdapter"; +import { Uint8ArrayBlobAdapter } from "./Uint8ArrayBlobAdapter"; -describe(BlobAdapter.name, () => { +describe(Uint8ArrayBlobAdapter.name, () => { it("extends Uint8Array", () => { - const blobby = new BlobAdapter(5); + const blobby = new Uint8ArrayBlobAdapter(5); blobby[-1] = 8; blobby[0] = 8; @@ -26,6 +26,7 @@ describe(BlobAdapter.name, () => { }); it("should transform to string synchronously", () => { - throw new Error("NYI"); + const blob = Uint8ArrayBlobAdapter.fromString("test-123"); + expect(blob.transformToString()).toEqual("test-123"); }); }); diff --git a/packages/util-stream/src/blob/Uint8ArrayBlobAdapter.ts b/packages/util-stream/src/blob/Uint8ArrayBlobAdapter.ts index 0a9921995c99d..447be19b21d8e 100644 --- a/packages/util-stream/src/blob/Uint8ArrayBlobAdapter.ts +++ b/packages/util-stream/src/blob/Uint8ArrayBlobAdapter.ts @@ -1,4 +1,4 @@ -import { transformFromObject, transformFromString, transformToString } from "./transforms"; +import { transformFromString, transformToString } from "./transforms"; /** * Adapter for conversions of the native Uint8Array type. @@ -9,12 +9,10 @@ export class Uint8ArrayBlobAdapter extends Uint8Array { * @param source - such as a string or Stream. * @returns a new Uint8ArrayBlobAdapter extending Uint8Array. */ - public static from(source: unknown): Uint8ArrayBlobAdapter { + public static fromString(source: string, encoding = "utf-8"): Uint8ArrayBlobAdapter { switch (typeof source) { case "string": - return transformFromString(source as string); - case "object": - return transformFromObject(source as object); + return transformFromString(source, encoding); default: throw new Error(`Unsupported conversion from ${typeof source} to Uint8ArrayBlobAdapter.`); } diff --git a/packages/util-stream/src/blob/transforms.browser.ts b/packages/util-stream/src/blob/transforms.browser.ts deleted file mode 100644 index 24fd34d0ca6e7..0000000000000 --- a/packages/util-stream/src/blob/transforms.browser.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Uint8ArrayBlobAdapter } from "./Uint8ArrayBlobAdapter"; - -/** - * @internal - */ -export function transformToString(payload: Uint8Array, encoding = "utf-8"): string { - const textDecoder = new TextDecoder(encoding); - return textDecoder.decode(payload); -} - -/** - * @internal - */ -export function transformFromString(str: string): Uint8ArrayBlobAdapter { - const textEncoder = new TextEncoder(); - return new Uint8ArrayBlobAdapter(textEncoder.encode(str)); -} - -/** - * @internal - */ -export function transformFromObject(obj: object): Uint8ArrayBlobAdapter { - return new Uint8ArrayBlobAdapter(obj as any); -} diff --git a/packages/util-stream/src/blob/transforms.ts b/packages/util-stream/src/blob/transforms.ts index 4afd45e2b1e62..bd05bd5c0e754 100644 --- a/packages/util-stream/src/blob/transforms.ts +++ b/packages/util-stream/src/blob/transforms.ts @@ -1,4 +1,5 @@ -import { Readable } from "stream"; +import { fromBase64, toBase64 } from "@aws-sdk/util-base64"; +import { fromUtf8, toUtf8 } from "@aws-sdk/util-utf8"; import { Uint8ArrayBlobAdapter } from "./Uint8ArrayBlobAdapter"; @@ -6,19 +7,18 @@ import { Uint8ArrayBlobAdapter } from "./Uint8ArrayBlobAdapter"; * @internal */ export function transformToString(payload: Uint8Array, encoding = "utf-8"): string { - return Buffer.from(payload).toString(encoding as BufferEncoding); + if (encoding === "base64") { + return toBase64(payload); + } + return toUtf8(payload); } /** * @internal */ -export function transformFromString(str: string): Uint8ArrayBlobAdapter { - return new Uint8ArrayBlobAdapter(Buffer.from(str)); -} - -/** - * @internal - */ -export function transformFromObject(obj: object): Uint8ArrayBlobAdapter { - return new Uint8ArrayBlobAdapter(obj as any); +export function transformFromString(str: string, encoding?: string): Uint8ArrayBlobAdapter { + if (encoding === "base64") { + return Uint8ArrayBlobAdapter.mutate(fromBase64(str)); + } + return Uint8ArrayBlobAdapter.mutate(fromUtf8(str)); } diff --git a/packages/util-stream/src/util-stream.integ.spec.ts b/packages/util-stream/src/util-stream.integ.spec.ts new file mode 100644 index 0000000000000..dc19ee134813f --- /dev/null +++ b/packages/util-stream/src/util-stream.integ.spec.ts @@ -0,0 +1,44 @@ +import { Lambda } from "@aws-sdk/client-lambda"; + +import { requireRequestsFrom } from "../../../private/aws-util-test/src"; + +describe("util-stream", () => { + describe(Lambda.name, () => { + it("should be uniform between string and Uint8Array payloads", async () => { + const client = new Lambda({ region: "us-west-2" }); + requireRequestsFrom(client).toMatch({ + method: "POST", + hostname: "lambda.us-west-2.amazonaws.com", + query: {}, + headers: { + "content-type": "application/octet-stream", + "content-length": "17", + host: "lambda.us-west-2.amazonaws.com", + }, + body(raw) { + expect(raw.toString("utf-8")).toEqual('{"hello":"world"}'); + }, + protocol: "https:", + path: "/2015-03-31/functions/echo/invocations", + }); + + // string + await client.invoke({ + FunctionName: "echo", + Payload: JSON.stringify({ + hello: "world", + }), + }); + + // Uint8Array + await client.invoke({ + FunctionName: "echo", + Payload: Buffer.from( + JSON.stringify({ + hello: "world", + }) + ), + }); + }); + }); +}); diff --git a/packages/util-stream/test/function.zip b/packages/util-stream/test/function.zip new file mode 100644 index 0000000000000..bc37c3b332259 Binary files /dev/null and b/packages/util-stream/test/function.zip differ diff --git a/packages/util-stream/test/index.mjs b/packages/util-stream/test/index.mjs new file mode 100644 index 0000000000000..e1b98f02447f9 --- /dev/null +++ b/packages/util-stream/test/index.mjs @@ -0,0 +1,6 @@ +/** + * function.zip contains this file. + */ +export const handler = async (event) => { + return event; +}; diff --git a/packages/util-stream/test/setup.ts b/packages/util-stream/test/setup.ts new file mode 100644 index 0000000000000..5e4816dfb4b4a --- /dev/null +++ b/packages/util-stream/test/setup.ts @@ -0,0 +1,149 @@ +import { AttachedPolicy, CreateRoleResponse, GetRoleResponse, IAM } from "@aws-sdk/client-iam"; +import { + GetFunctionConfigurationCommandOutput, + Lambda, + Runtime, + waitUntilFunctionActiveV2, + waitUntilFunctionUpdated, +} from "@aws-sdk/client-lambda"; +import fs from "fs"; +export const FunctionName = "aws-sdk-js-v3-e2e-echo"; +export const Handler = "index.handler"; +const LAMBDA_ROLE_NAME = "aws-sdk-js-v3-e2e-LambdaRole"; + +export async function setup() { + const lambda = new Lambda({ + region: "us-west-2", + }); + const getFn: null | GetFunctionConfigurationCommandOutput = await lambda + .getFunctionConfiguration({ + FunctionName, + }) + .catch(() => null); + + if (getFn) { + return; + } + + const iam = new IAM({ + region: "us-west-2", + }); + + const roleName = LAMBDA_ROLE_NAME; + const role: null | GetRoleResponse | CreateRoleResponse = await iam + .getRole({ + RoleName: roleName, + }) + .catch(() => null); + + if (!role) { + console.info("Creating role", roleName); + await iam.createRole({ + RoleName: roleName, + Path: "/", + Description: "aws sdk js v3 lambda test role", + AssumeRolePolicyDocument: JSON.stringify({ + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Action: ["sts:AssumeRole"], + Principal: { + Service: ["lambda.amazonaws.com"], + }, + }, + ], + }), + }); + } else { + console.info("Role exists", roleName); + } + + const listAttachedRolePolicies = await iam.listAttachedRolePolicies({ + RoleName: roleName, + }); + const policies = listAttachedRolePolicies.AttachedPolicies || []; + + const existingPolicies = policies.reduce((acc: Record, cur: AttachedPolicy) => { + if (cur.PolicyName) { + acc[cur.PolicyName] = true; + } + return acc; + }, {} as Record); + + const required = ["AWSLambda_FullAccess"]; + + for (const requiredPolicy of required) { + if (!existingPolicies[requiredPolicy]) { + console.info("Attaching policy to role", requiredPolicy, roleName); + await iam.attachRolePolicy({ + RoleName: roleName, + PolicyArn: `arn:aws:iam::aws:policy/${requiredPolicy}`, + }); + } else { + console.info("Policy exists on role", requiredPolicy, roleName); + } + } + + const getRole: null | GetRoleResponse = await iam + .getRole({ + RoleName: roleName, + }) + .catch(() => null); + if (!getRole) { + throw new Error("Role not found."); + } else { + console.info("Role found", roleName); + } + + const roleArn = getRole.Role!.Arn!; + + if (getFn) { + console.info("Function exists:", FunctionName); + + if ((getFn.Timeout ?? 0) < 5 * 60 || getFn?.Handler !== Handler) { + await lambda.updateFunctionConfiguration({ + FunctionName, + Handler, + Timeout: 5 * 60, + }); + await waitUntilFunctionUpdated( + { + client: lambda, + maxWaitTime: 40, + }, + { + FunctionName, + } + ); + } + // await lambda.updateFunctionCode({ + // FunctionName, + // ZipFile: fs.readFileSync(require.resolve("./function.zip")), + // }); + // console.info("Function code/configuration updated:", FunctionName); + } else { + await lambda.createFunction({ + FunctionName, + Role: roleArn, + Code: { + ZipFile: fs.readFileSync(require.resolve("./function.zip")), + }, + Runtime: Runtime.nodejs16x, + Description: `aws sdk js v3 e2e test echo`, + Timeout: 300, + Handler, + }); + console.info("Function created:", FunctionName); + } + + await waitUntilFunctionActiveV2( + { + maxWaitTime: 40, + client: lambda, + }, + { + FunctionName, + } + ); +} diff --git a/packages/util-stream/test/util-stream-blob.e2e.spec.ts b/packages/util-stream/test/util-stream-blob.e2e.spec.ts new file mode 100644 index 0000000000000..044959410ebae --- /dev/null +++ b/packages/util-stream/test/util-stream-blob.e2e.spec.ts @@ -0,0 +1,56 @@ +import { Lambda } from "@aws-sdk/client-lambda"; +import { Uint8ArrayBlobAdapter } from "@aws-sdk/util-stream"; +import { Readable } from "stream"; + +import { FunctionName, setup } from "./setup"; + +describe("blob e2e", () => { + jest.setTimeout(100000); + + const lambda = new Lambda({ + region: "us-west-2", + }); + + beforeAll(async () => { + await setup(); + }); + + it("should allow string as payload blob and allow conversion of output payload blob to string", async () => { + const payload = JSON.stringify({ hello: "world" }); + const invoke = await lambda.invoke({ FunctionName, Payload: payload }); + expect(JSON.parse(invoke?.Payload?.transformToString() ?? "{}")).toEqual({ hello: "world" }); + }); + + it("should allow Uint8Array as payload blob", async () => { + const payload = Uint8ArrayBlobAdapter.fromString(JSON.stringify({ hello: "world" })); + const invoke = await lambda.invoke({ FunctionName, Payload: payload }); + expect(JSON.parse(invoke?.Payload?.transformToString() ?? "{}")).toEqual({ hello: "world" }); + }); + + it("should allow buffer as payload blob", async () => { + // note: Buffer extends Uint8Array + const payload = Buffer.from(Uint8ArrayBlobAdapter.fromString(JSON.stringify({ hello: "world" }))); + const invoke = await lambda.invoke({ FunctionName, Payload: payload }); + expect(JSON.parse(invoke?.Payload?.transformToString() ?? "{}")).toEqual({ hello: "world" }); + }); + + it("should allow stream as payload blob but not be able to sign it", async () => { + const payload = Readable.from(Buffer.from(Uint8ArrayBlobAdapter.fromString(JSON.stringify({ hello: "world" }))), { + encoding: "utf-8", + }); + expect(JSON.parse(await streamToString(payload))).toEqual({ hello: "world" }); + await lambda.invoke({ FunctionName, Payload: payload }).catch((e) => { + expect(e.toString()).toContain("InvalidSignatureException"); + }); + expect.hasAssertions(); + }); +}); + +function streamToString(stream: Readable): Promise { + const chunks: any[] = []; + return new Promise((resolve, reject) => { + stream.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + stream.on("error", (err) => reject(err)); + stream.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + }); +}