feat(lib-storage): rewrite lib-storage upload

lib/storage/CHANGELOG.md

@@ -7,151 +7,80 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline
 
 **Note:** Version bump only for package @aws-sdk/lib-storage
 
-
-
-
-
 ## [3.4.1](https://github.com/aws/aws-sdk-js-v3/compare/v3.4.0...v3.4.1) (2021-01-29)
 
-
 ### Bug Fixes
 
-* **clients:** use TS 3.4 compatible types for TS 3.9 ([#1978](https://github.com/aws/aws-sdk-js-v3/issues/1978)) ([8bced5c](https://github.com/aws/aws-sdk-js-v3/commit/8bced5c32b9dbc68f1065054d796cb0b8b87bcc4))
-
-
-
-
+- **clients:** use TS 3.4 compatible types for TS 3.9 ([#1978](https://github.com/aws/aws-sdk-js-v3/issues/1978)) ([8bced5c](https://github.com/aws/aws-sdk-js-v3/commit/8bced5c32b9dbc68f1065054d796cb0b8b87bcc4))
 
 # [3.4.0](https://github.com/aws/aws-sdk-js-v3/compare/v3.3.0...v3.4.0) (2021-01-28)
 
-
 ### Features
 
-* use downlevel-dts to generate TS 3.4 compatible types ([#1943](https://github.com/aws/aws-sdk-js-v3/issues/1943)) ([63ad215](https://github.com/aws/aws-sdk-js-v3/commit/63ad2151c8bb7be32ea8838a9b0974806ed3906b))
-
-
-
-
+- use downlevel-dts to generate TS 3.4 compatible types ([#1943](https://github.com/aws/aws-sdk-js-v3/issues/1943)) ([63ad215](https://github.com/aws/aws-sdk-js-v3/commit/63ad2151c8bb7be32ea8838a9b0974806ed3906b))
 
 # [3.3.0](https://github.com/aws/aws-sdk-js-v3/compare/v3.2.0...v3.3.0) (2021-01-14)
 
 **Note:** Version bump only for package @aws-sdk/lib-storage
 
-
-
-
-
 # [3.2.0](https://github.com/aws/aws-sdk-js-v3/compare/v3.1.0...v3.2.0) (2021-01-09)
 
-
 ### Bug Fixes
 
-* **lib-storage:** chunk from readable only when defined ([#1886](https://github.com/aws/aws-sdk-js-v3/issues/1886)) ([4cdc08a](https://github.com/aws/aws-sdk-js-v3/commit/4cdc08aef3dc57d1d200f4d6d9e9683426354374))
-
-
-
-
+- **lib-storage:** chunk from readable only when defined ([#1886](https://github.com/aws/aws-sdk-js-v3/issues/1886)) ([4cdc08a](https://github.com/aws/aws-sdk-js-v3/commit/4cdc08aef3dc57d1d200f4d6d9e9683426354374))
 
 # [3.1.0](https://github.com/aws/aws-sdk-js-v3/compare/v3.0.0...v3.1.0) (2020-12-23)
 
-
 ### Bug Fixes
 
-* **lib-storage:** cleanup stream listeners to prevent memory leak ([3d36682](https://github.com/aws/aws-sdk-js-v3/commit/3d3668256467c5fa7a00e19c6c42667b5b3a77d4))
-
-
-
-
+- **lib-storage:** cleanup stream listeners to prevent memory leak ([3d36682](https://github.com/aws/aws-sdk-js-v3/commit/3d3668256467c5fa7a00e19c6c42667b5b3a77d4))
 
 # [3.0.0](https://github.com/aws/aws-sdk-js-v3/compare/v1.0.0-rc.10...v3.0.0) (2020-12-15)
 
-
 ### Features
 
-* bump version to 3.0.0 ([#1793](https://github.com/aws/aws-sdk-js-v3/issues/1793)) ([d8475f8](https://github.com/aws/aws-sdk-js-v3/commit/d8475f8d972d28fbc15cd7e23abfe18f9eab0644))
-
-
-
-
+- bump version to 3.0.0 ([#1793](https://github.com/aws/aws-sdk-js-v3/issues/1793)) ([d8475f8](https://github.com/aws/aws-sdk-js-v3/commit/d8475f8d972d28fbc15cd7e23abfe18f9eab0644))
 
 # [1.0.0-rc.10](https://github.com/aws/aws-sdk-js-v3/compare/v1.0.0-rc.9...v1.0.0-rc.10) (2020-12-15)
 
 **Note:** Version bump only for package @aws-sdk/lib-storage
 
-
-
-
-
 # [1.0.0-rc.9](https://github.com/aws/aws-sdk-js-v3/compare/v1.0.0-rc.8...v1.0.0-rc.9) (2020-12-11)
 
 **Note:** Version bump only for package @aws-sdk/lib-storage
 
-
-
-
-
 # [1.0.0-rc.8](https://github.com/aws/aws-sdk-js-v3/compare/v1.0.0-rc.7...v1.0.0-rc.8) (2020-12-05)
 
 **Note:** Version bump only for package @aws-sdk/lib-storage
 
-
-
-
-
 # [1.0.0-rc.7](https://github.com/aws/aws-sdk-js-v3/compare/v1.0.0-rc.6...v1.0.0-rc.7) (2020-11-20)
 
 **Note:** Version bump only for package @aws-sdk/lib-storage
 
-
-
-
-
 # [1.0.0-rc.6](https://github.com/aws/aws-sdk-js-v3/compare/v1.0.0-rc.5...v1.0.0-rc.6) (2020-11-13)
 
 **Note:** Version bump only for package @aws-sdk/lib-storage
 
-
-
-
-
 # [1.0.0-rc.5](https://github.com/aws/aws-sdk-js-v3/compare/v1.0.0-rc.4...v1.0.0-rc.5) (2020-11-09)
 
 **Note:** Version bump only for package @aws-sdk/lib-storage
 
-
-
-
-
 # [1.0.0-rc.4](https://github.com/aws/aws-sdk-js-v3/compare/v1.0.0-rc.3...v1.0.0-rc.4) (2020-10-31)
 
 **Note:** Version bump only for package @aws-sdk/lib-storage
 
-
-
-
-
 # [1.0.0-rc.3](https://github.com/aws/aws-sdk-js-v3/compare/v1.0.0-rc.2...v1.0.0-rc.3) (2020-10-27)
 
 **Note:** Version bump only for package @aws-sdk/lib-storage
 
-
-
-
-
 # [1.0.0-rc.2](https://github.com/aws/aws-sdk-js-v3/compare/v1.0.0-rc.1...v1.0.0-rc.2) (2020-10-22)
 
-
 ### Bug Fixes
 
-* **storage:** add version and downloads badges ([#1599](https://github.com/aws/aws-sdk-js-v3/issues/1599)) ([230d030](https://github.com/aws/aws-sdk-js-v3/commit/230d0305801c298f93d3c534884833bf5cfa3f3f))
-
-
-
-
+- **storage:** add version and downloads badges ([#1599](https://github.com/aws/aws-sdk-js-v3/issues/1599)) ([230d030](https://github.com/aws/aws-sdk-js-v3/commit/230d0305801c298f93d3c534884833bf5cfa3f3f))
 
 # [1.0.0-rc.1](https://github.com/aws/aws-sdk-js-v3/compare/v1.0.0-gamma.11...v1.0.0-rc.1) (2020-10-19)
 
-
 ### Features
 
-* **storage:** adding lib/storage upload ([#1547](https://github.com/aws/aws-sdk-js-v3/issues/1547)) ([b07e719](https://github.com/aws/aws-sdk-js-v3/commit/b07e71986817d7781e12d9b687532c58d5ea6ba0))
+- **storage:** adding lib/storage upload ([#1547](https://github.com/aws/aws-sdk-js-v3/issues/1547)) ([b07e719](https://github.com/aws/aws-sdk-js-v3/commit/b07e71986817d7781e12d9b687532c58d5ea6ba0))

lib/storage/README.md

@@ -1,18 +1,24 @@
 # @aws-sdk/lib-storage
 
-[![NPM version](https://img.shields.io/npm/v/@aws-sdk/lib-storage/latest.svg)](https://www.npmjs.com/package/@aws-sdk/abort-controller)
-[![NPM downloads](https://img.shields.io/npm/dm/@aws-sdk/lib-storage.svg)](https://www.npmjs.com/package/@aws-sdk/abort-controller)
+[![NPM version](https://img.shields.io/npm/v/@aws-sdk/lib-storage/latest.svg)](https://www.npmjs.com/package/@aws-sdk/lib-storage)
+[![NPM downloads](https://img.shields.io/npm/dm/@aws-sdk/lib-storage.svg)](https://www.npmjs.com/package/@aws-sdk/lib-storage)
 
 ### Upload
 
 Upload allows for easy and efficient uploading of buffers, blobs, or streams, using a configurable amount of concurrency to perform multipart uploads where possible. This abstraction enables uploading large files or streams of unknown size due to the use of multipart uploads under the hood.
 
 ```
   import { Upload } from "@aws-sdk/lib-storage";
+  import { S3Client, S3 } from "@aws-sdk/client-s3";
+
   const target = { Bucket, Key, Body };
   try {
     const paralellUploads3 = new Upload({
-      client: new S3({}),
+      client: new S3({}) || new S3Client({}),
+      tags: [...], // optional tags
+      queueSize: 4, // optional concurrency configuration
+      partSize: 5MB, // optional size of each part
+      leavePartsOnError: false, // optional manually handle dropped parts
       params: target,
     });
 

lib/storage/example-code/config.ts

@@ -0,0 +1,4 @@
+export const configuration = {
+  Bucket: "test-bucket-1234567890",
+  Key: `unique-key-${Date.now}`,
+};

lib/storage/example-code/file-upload.ts

@@ -1,29 +1,24 @@
-import { Upload } from "../src/index";
+import { Upload } from "@aws-sdk/lib-storage";
 import { S3 } from "@aws-sdk/client-s3";
+import { configuration } from "./config";
 
 const fs = require("fs");
-const fileStream = fs.createReadStream("./big.file");
+const fileStream = fs.createReadStream(__dirname + "/big.file");
 
 (async () => {
-  const target = {
-    Bucket: "aws-sdk-js-mock-files",
-    Key: "data_key",
-    Body: fileStream,
-  };
+  const upload = new Upload({
+    params: {
+      Bucket: configuration.Bucket,
+      Key: configuration.Key,
+      Body: fileStream,
+    },
+    client: new S3({}),
+    queueSize: 3,
+  });
 
-  try {
-    const upload = new Upload({
-      params: target,
-      client: new S3({}),
-      queueSize: 3,
-    });
+  upload.on("httpUploadProgress", (progress) => {
+    console.log(progress);
+  });
 
-    upload.on("httpUploadProgress", (progress) => {
-      console.log(progress);
-    });
-
-    await upload.done();
-  } catch (e) {
-    console.log(e);
-  }
+  await upload.done();
 })();

lib/storage/example-code/upload-abort.ts

@@ -0,0 +1,48 @@
+import { S3Client } from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+import { configuration } from "./config";
+import { Readable } from "stream";
+
+const Bucket = configuration.Bucket;
+const region = "us-west-2";
+
+const sleep = async (seconds: number) => {
+  return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+};
+
+async function* generateContents() {
+  for (let index = 0; index < 8; index++) {
+    const time = Math.random() * 5;
+    await sleep(time);
+    console.log(`Delaying part ${index} for ${time}`);
+    yield `[Part ${index}] ${"#".repeat(2000000)}`;
+  }
+}
+
+const uploadIndeterminateLengthStreamNode = async () => {
+  const streamOfUnknownlength = Readable.from(generateContents());
+
+  const Key = configuration.Key;
+  let upload = new Upload({
+    client: new S3Client({ region }),
+    params: {
+      Key,
+      Bucket,
+      Body: streamOfUnknownlength,
+    },
+  });
+
+  upload.on("httpUploadProgress", (progress: ProgressEvent) => {
+    console.log(progress);
+  });
+
+  setTimeout(() => {
+    console.log(" Aborting ....");
+    let res = upload.abort();
+  }, 10 * 1000);
+
+  const uploadResult = await upload.done();
+  console.log("done uploading", uploadResult);
+};
+
+uploadIndeterminateLengthStreamNode();

lib/storage/example-code/upload-string.ts

@@ -1,40 +1,22 @@
-import { Upload } from "../src/index";
-import { S3, S3Client } from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+import { S3 } from "@aws-sdk/client-s3";
+import { configuration } from "./config";
 
-const Bucket = "aws-sdk-js-mock-files";
-const Key = "data_key";
+const Bucket = configuration.Bucket;
+const Key = configuration.Key;
 const Body =
   "Duo Reges: constructio interrete. Qui autem esse poteris, nisi te amor ipse ceperit? Hoc est non modo cor non habere, sed ne palatum quidem. Quantam rem agas, ut Circeis qui habitet totum hunc mundum suum municipium esse existimet? Huius, Lyco, oratione locuples, rebus ipsis ielunior. Sed quid attinet de rebus tam apertis plura requirere? Non quam nostram quidem, inquit Pomponius iocans; Alterum significari idem, ut si diceretur, officia media omnia aut pleraque servantem vivere. Stoici scilicet";
 
 (async () => {
   const target = { Bucket, Key, Body };
-  try {
-    const paralellUploads3 = new Upload({
-      client: new S3({}),
-      params: target,
-    });
+  const paralellUploads3 = new Upload({
+    client: new S3({}),
+    params: target,
+  });
 
-    paralellUploads3.on("httpUploadProgress", (progress) => {
-      console.log(progress);
-    });
+  paralellUploads3.on("httpUploadProgress", (progress) => {
+    console.log(progress);
+  });
 
-    await paralellUploads3.done();
-  } catch (e) {
-    console.log(e);
-  }
-
-  try {
-    const paralellUploads3Client = new Upload({
-      client: new S3Client({}),
-      params: target,
-    });
-
-    paralellUploads3Client.on("httpUploadProgress", (progress) => {
-      console.log(progress);
-    });
-
-    await paralellUploads3Client.done();
-  } catch (e) {
-    console.log(e);
-  }
+  await paralellUploads3.done();
 })();

lib/storage/example-code/upload-unknown-length-browser.ts

@@ -0,0 +1,57 @@
+import { fromCognitoIdentityPool } from "@aws-sdk/credential-provider-cognito-identity";
+import { CognitoIdentityClient } from "@aws-sdk/client-cognito-identity";
+
+import { S3Client } from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+
+import { configuration } from "./config";
+
+const idPool = "us-west-2:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";
+const Bucket = configuration.Bucket;
+const region = "us-west-2";
+
+const sleep = async (seconds: number) => {
+  return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+};
+
+const fakeStreamOfUnknownlength = new ReadableStream({
+  start: async (controller) => {
+    for (let index = 0; index < 8; index++) {
+      const time = Math.random() * 5;
+      await sleep(time);
+      console.log(`Delaying part ${index} for ${time}`);
+      controller.enqueue(`[part ${index}] with ${"#".repeat(2000000)}`);
+    }
+    controller.close();
+  },
+});
+
+const uploadIndeterminateLengthStreamBrowser = async () => {
+  const client = new S3Client({
+    region,
+    credentials: fromCognitoIdentityPool({
+      client: new CognitoIdentityClient({ region }),
+      identityPoolId: idPool,
+    }),
+  });
+
+  const Key = configuration.Key;
+
+  let upload = new Upload({
+    client,
+    params: {
+      Key,
+      Bucket,
+      Body: fakeStreamOfUnknownlength,
+    },
+  });
+
+  upload.on("httpUploadProgress", (progress: ProgressEvent) => {
+    console.log(progress);
+  });
+
+  const uploadResult = await upload.done();
+  console.log("done uploading", uploadResult);
+};
+
+uploadIndeterminateLengthStreamBrowser();