fix: revert to single file upload/data item upload

quote> quote> Some concerns arose about proper control flow when uploading multiple files/data items. For now we will stick with single file uploads and make clients responsible for handling multiple uploads.
ardriveapp · Sep 5, 2023 · 1ccbbfa · 1ccbbfa
1 parent f361dab
commit 1ccbbfa
Show file tree

Hide file tree

Showing 10 changed files with 152 additions and 303 deletions.
diff --git a/examples/node/index.cjs b/examples/node/index.cjs
@@ -64,13 +64,10 @@
   /**
    * Post local files to the Turbo service.
    */
-  console.log('Posting raw files to Turbo service...');
-  const filePaths = [path.join(__dirname, './files/0_kb.txt')];
-  const fileStreamGenerators = filePaths.map(
-    (filePath) => () => fs.createReadStream(filePath),
-  );
-  const uploadResult = await turboAuthClient.uploadFiles({
-    fileStreamGenerators: fileStreamGenerators,
+  console.log('Posting raw file to Turbo service...');
+  const filePath = path.join(__dirname, './files/0_kb.txt');
+  const uploadResult = await turboAuthClient.uploadFile({
+    fileStreamFactory: () => fs.createReadStream(filePath),
   });
   console.log(JSON.stringify(uploadResult, null, 2));
 })();
diff --git a/examples/node/index.mjs b/examples/node/index.mjs
@@ -59,12 +59,9 @@ import {
    * Post local files to the Turbo service.
    */
   console.log('Posting raw files to Turbo service...');
-  const filePaths = [new URL('files/0_kb.txt', import.meta.url).pathname];
-  const fileStreamGenerators = filePaths.map(
-    (filePath) => () => fs.createReadStream(filePath),
-  );
-  const uploadResult = await turboAuthClient.uploadFiles({
-    fileStreamGenerators: fileStreamGenerators,
+  const filePath = new URL('files/0_kb.txt', import.meta.url).pathname;
+  const uploadResult = await turboAuthClient.uploadFile({
+    fileStreamFactory: () => fs.createReadStream(filePath),
   });
   console.log(JSON.stringify(uploadResult, null, 2));
 })();
diff --git a/examples/web/index.html b/examples/web/index.html
@@ -92,24 +92,24 @@ <h1>Upload File</h1>
           const selectedFiles = fileInput.files;
 
           if (selectedFiles.length) {
-            const blobFileGenerators = Object.values(selectedFiles).map(
-              (file) => () => file.stream(),
-            );
-            const response = turbo
-              .uploadFiles({
-                fileStreamGenerators: blobFileGenerators,
-              })
-              .then((res) => {
-                console.log('Successfully uploaded files!', res);
-                document.getElementById(
-                  'uploadStatus',
-                ).innerText = `Successfully uploaded files! ${JSON.stringify(
-                  res,
-                  null,
-                  2,
-                )}`;
-              })
-              .catch((err) => console.log('Error uploading file!', err));
+            // TODO: make this concurrent
+            for (const file of selectedFiles) {
+              const response = turbo
+                .uploadFile({
+                  fileStreamFactory: () => file.stream(),
+                })
+                .then((res) => {
+                  console.log('Successfully uploaded files!', res);
+                  document.getElementById(
+                    'uploadStatus',
+                  ).innerText = `Successfully uploaded files! ${JSON.stringify(
+                    res,
+                    null,
+                    2,
+                  )}`;
+                })
+                .catch((err) => console.log('Error uploading file!', err));
+            }
           } else {
             console.log('No file selected');
           }

diff --git a/src/common/payment.ts b/src/common/payment.ts
@@ -166,6 +166,7 @@ export class TurboAuthenticatedPaymentService
   protected readonly privateKey: JWKInterface;
   protected readonly publicPaymentService: TurboUnauthenticatedPaymentServiceInterface;
 
+  // TODO: replace private key with an internal signer interface
   constructor({
     url = 'https://payment.ardrive.dev',
     retryConfig,

diff --git a/src/common/turbo.ts b/src/common/turbo.ts
@@ -32,7 +32,7 @@ import {
   TurboSignedDataItemFactory,
   TurboUnauthenticatedPaymentServiceInterface,
   TurboUnauthenticatedUploadServiceInterface,
-  TurboUploadDataItemsResponse,
+  TurboUploadDataItemResponse,
 } from '../types/index.js';
 import {
   TurboAuthenticatedPaymentService,
@@ -115,13 +115,13 @@ export class TurboUnauthenticatedClient implements TurboPublicClient {
   }
 
   /**
-   * Verifies signature of signed data items and uploads to the upload service.
+   * Uploads a signed data item to the Turbo Upload Service.
    */
-  async uploadSignedDataItems({
-    dataItemGenerators,
-  }: TurboSignedDataItemFactory): Promise<TurboUploadDataItemsResponse> {
-    return this.uploadService.uploadSignedDataItems({
-      dataItemGenerators,
+  async uploadSignedDataItem({
+    dataItemStreamFactory,
+  }: TurboSignedDataItemFactory): Promise<TurboUploadDataItemResponse> {
+    return this.uploadService.uploadSignedDataItem({
+      dataItemStreamFactory,
     });
   }
 }
@@ -210,22 +210,24 @@ export class TurboAuthenticatedClient implements TurboPrivateClient {
   }
 
   /**
-   * Signs and uploads data to the upload service.
+   * Signs and uploads raw data to the Turbo Upload Service.
+   *
+   * Note: 'privateKey' must be provided to use.
    */
-  async uploadFiles({
-    fileStreamGenerators,
-  }: TurboFileFactory): Promise<TurboUploadDataItemsResponse> {
-    return this.uploadService.uploadFiles({ fileStreamGenerators });
+  async uploadFile({
+    fileStreamFactory,
+  }: TurboFileFactory): Promise<TurboUploadDataItemResponse> {
+    return this.uploadService.uploadFile({ fileStreamFactory });
   }
 
   /**
-   * Verifies signature of signed data items and uploads to the upload service.
+   * Uploads a signed data item to the Turbo Upload Service.
    */
-  async uploadSignedDataItems({
-    dataItemGenerators,
-  }: TurboSignedDataItemFactory): Promise<TurboUploadDataItemsResponse> {
-    return this.uploadService.uploadSignedDataItems({
-      dataItemGenerators,
+  async uploadSignedDataItem({
+    dataItemStreamFactory,
+  }: TurboSignedDataItemFactory): Promise<TurboUploadDataItemResponse> {
+    return this.uploadService.uploadSignedDataItem({
+      dataItemStreamFactory,
     });
   }
 }
diff --git a/src/common/upload.ts b/src/common/upload.ts
@@ -14,12 +14,11 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
-import { AxiosInstance, AxiosResponse } from 'axios';
+import { AxiosInstance } from 'axios';
 
 import { TurboNodeDataItemSigner } from '../node/signer.js';
 import { JWKInterface } from '../types/arweave.js';
 import {
-  TransactionId,
   TurboAuthenticatedUploadServiceConfiguration,
   TurboAuthenticatedUploadServiceInterface,
   TurboDataItemSigner,
@@ -28,9 +27,9 @@ import {
   TurboUnauthenticatedUploadServiceInterface,
   TurboUnauthenticatedUploadServiceInterfaceConfiguration,
   TurboUploadDataItemResponse,
-  TurboUploadDataItemsResponse,
 } from '../types/turbo.js';
 import { createAxiosInstance } from '../utils/axiosClient.js';
+import { FailedRequestError } from '../utils/errors.js';
 
 export class TurboUnauthenticatedUploadService
   implements TurboUnauthenticatedUploadServiceInterface
@@ -49,58 +48,25 @@ export class TurboUnauthenticatedUploadService
     });
   }
 
-  async uploadSignedDataItems({
-    dataItemGenerators,
-  }: TurboSignedDataItemFactory): Promise<TurboUploadDataItemsResponse> {
-    const signedDataItems = dataItemGenerators.map((dataItem) => dataItem());
-
-    // TODO: add p-limit constraint
-    const uploadPromises = signedDataItems.map((signedDataItem) => {
-      return this.axios.post<TurboUploadDataItemResponse>(
+  async uploadSignedDataItem({
+    dataItemStreamFactory,
+  }: TurboSignedDataItemFactory): Promise<TurboUploadDataItemResponse> {
+    // TODO: add p-limit constraint or replace with separate upload class
+    const { status, data, statusText } =
+      await this.axios.post<TurboUploadDataItemResponse>(
         `/tx`,
-        signedDataItem,
+        dataItemStreamFactory(),
         {
           headers: {
             'content-type': 'application/octet-stream',
           },
         },
       );
-    });
 
-    // NOTE: our axios config (validateStatus) swallows errors, so failed data items will be ignored
-    const dataItemResponses = await Promise.all(uploadPromises);
-    const errors: { id: string; status: number; message: string }[] = [];
-    const postedDataItems = dataItemResponses.reduce(
-      (
-        postedDataItemsMap: Record<
-          TransactionId,
-          Omit<TurboUploadDataItemResponse, 'id'>
-        >,
-        dataItemResponse: AxiosResponse<TurboUploadDataItemResponse, 'id'>,
-      ) => {
-        // handle the fulfilled response
-        const { status, data, statusText } = dataItemResponse;
-        if (![200, 202].includes(status)) {
-          // TODO: add to failed data items array
-          errors.push({
-            id: data.id ?? 'unknown',
-
-            status,
-            message: statusText,
-          });
-          return postedDataItemsMap;
-        }
-        const { id, ...dataItemCache } = data;
-        postedDataItemsMap[id] = dataItemCache;
-        return postedDataItemsMap;
-      },
-      {},
-    );
-
-    return {
-      dataItems: postedDataItems,
-      errors,
-    };
+    if (![202, 200].includes(status)) {
+      throw new FailedRequestError(status, statusText);
+    }
+    return data;
   }
 }
 
@@ -127,74 +93,36 @@ export class TurboAuthenticatedUploadService
     this.dataItemSigner = dataItemSigner;
   }
 
-  async uploadSignedDataItems({
-    dataItemGenerators,
-  }: TurboSignedDataItemFactory): Promise<TurboUploadDataItemsResponse> {
-    const signedDataItems = dataItemGenerators.map((dataItem) => dataItem());
-
-    console.log('upload signed data items here');
-
-    // TODO: add p-limit constraint
-    const uploadPromises = signedDataItems.map((signedDataItem) => {
-      return this.axios.post<TurboUploadDataItemResponse>(
+  async uploadSignedDataItem({
+    dataItemStreamFactory,
+  }: TurboSignedDataItemFactory): Promise<TurboUploadDataItemResponse> {
+    // TODO: add p-limit constraint or replace with separate upload class
+    const { status, data, statusText } =
+      await this.axios.post<TurboUploadDataItemResponse>(
         `/tx`,
-        signedDataItem,
+        dataItemStreamFactory(),
         {
           headers: {
             'content-type': 'application/octet-stream',
           },
         },
       );
-    });
-
-    // NOTE: our axios config (validateStatus) swallows errors, so failed data items will be ignored
-    const dataItemResponses = await Promise.all(uploadPromises);
-    const errors: { id: string; status: number; message: string }[] = [];
-    const postedDataItems = dataItemResponses.reduce(
-      (
-        postedDataItemsMap: Record<
-          TransactionId,
-          Omit<TurboUploadDataItemResponse, 'id'>
-        >,
-        dataItemResponse: AxiosResponse<TurboUploadDataItemResponse, 'id'>,
-      ) => {
-        // handle the fulfilled response
-        const { status, data, statusText } = dataItemResponse;
-        if (![200, 202].includes(status)) {
-          errors.push({
-            id: data.id ?? 'unknown',
 
-            status,
-            message: statusText,
-          });
-          return postedDataItemsMap;
-        }
-        const { id, ...dataItemCache } = data;
-        postedDataItemsMap[id] = dataItemCache;
-        return postedDataItemsMap;
-      },
-      {},
-    );
-
-    return {
-      dataItems: postedDataItems,
-      errors,
-    };
+    if (![202, 200].includes(status)) {
+      throw new FailedRequestError(status, statusText);
+    }
+    return data;
   }
 
-  async uploadFiles({
-    fileStreamGenerators,
-  }: TurboFileFactory): Promise<TurboUploadDataItemsResponse> {
-    const signedDataItemPromises = this.dataItemSigner.signDataItems({
-      fileStreamGenerators,
+  async uploadFile({
+    fileStreamFactory,
+  }: TurboFileFactory): Promise<TurboUploadDataItemResponse> {
+    const signedDataItem = await this.dataItemSigner.signDataItem({
+      fileStreamFactory,
     });
-
-    // TODO: we probably don't want to Promise.all, do .allSettled and only return successful signed data items
-    const signedDataItems = await Promise.all(signedDataItemPromises);
-
-    // TODO: add p-limit constraint
-    const uploadPromises = signedDataItems.map((signedDataItem) => {
-      return this.axios.post<TurboUploadDataItemResponse>(
+    // TODO: add p-limit constraint or replace with separate upload class
+    const { status, data, statusText } =
+      await this.axios.post<TurboUploadDataItemResponse>(
         `/tx`,
         signedDataItem,
         {
@@ -203,40 +131,10 @@ export class TurboAuthenticatedUploadService
           },
         },
       );
-    });
-
-    // NOTE: our axios config (validateStatus) swallows errors, so failed data items will be ignored
-    const dataItemResponses = await Promise.all(uploadPromises);
-    const errors: { id: string; status: number; message: string }[] = [];
-    const postedDataItems = dataItemResponses.reduce(
-      (
-        postedDataItemsMap: Record<
-          TransactionId,
-          Omit<TurboUploadDataItemResponse, 'id'>
-        >,
-        dataItemResponse: AxiosResponse<TurboUploadDataItemResponse, 'id'>,
-      ) => {
-        // handle the fulfilled response
-        const { status, data, statusText } = dataItemResponse;
-        if (![200, 202].includes(status)) {
-          // TODO: add to failed data items array
-          errors.push({
-            id: data.id ?? 'unknown',
-            status,
-            message: statusText,
-          });
-          return postedDataItemsMap;
-        }
-        const { id, ...dataItemCache } = data;
-        postedDataItemsMap[id] = dataItemCache;
-        return postedDataItemsMap;
-      },
-      {},
-    );
 
-    return {
-      dataItems: postedDataItems,
-      errors,
-    };
+    if (![202, 200].includes(status)) {
+      throw new FailedRequestError(status, statusText);
+    }
+    return data;
   }
 }