feat: introduce uploadSignedDataItem interface, implement for node

package.json

@@ -51,6 +51,8 @@
     "arbundles": "^0.9.9",
     "arweave": "^1.14.4",
     "axios": "^1.4.0",
+    "elliptic": "^6.5.4",
+    "jwk-to-pem": "^2.0.5",
     "retry-axios": "^3.0.0",
     "winston": "^3.10.0"
   },

src/common/turbo.ts

@@ -16,7 +16,6 @@
  */
 import { Readable } from 'stream';
 
-import { TurboNodeDataItemSigner } from '../node/signer.js';
 import {
   Currency,
   TurboBalanceResponse,
@@ -33,6 +32,7 @@ import {
   TurboPublicPaymentService,
   TurboPublicUploadService,
   TurboRatesResponse,
+  TurboSignedDataItemFactory,
   TurboUploadDataItemsResponse,
 } from '../types/index.js';
 import {
@@ -114,6 +114,21 @@ export class TurboUnauthenticatedClient implements TurboPublicClient {
   }): Promise<Omit<TurboPriceResponse, 'adjustments'>> {
     return this.paymentService.getWincForFiat({ amount, currency });
   }
+
+  /**
+   * Verifies signature of signed data items and uploads to the upload service.
+   */
+  async uploadSignedDataItems({
+    dataItemGenerator,
+    signature,
+    publicKey,
+  }): Promise<TurboUploadDataItemsResponse> {
+    return this.uploadService.uploadSignedDataItems({
+      dataItemGenerator,
+      signature,
+      publicKey,
+    });
+  }
 }
 
 export class TurboAuthenticatedClient implements TurboPrivateClient {
@@ -125,7 +140,6 @@ export class TurboAuthenticatedClient implements TurboPrivateClient {
     paymentService = new TurboAuthenticatedPaymentService({ privateKey }),
     uploadService = new TurboAuthenticatedUploadService({
       privateKey,
-      dataItemSigner: new TurboNodeDataItemSigner({ privateKey }),
     }),
   }: TurboPrivateClientConfiguration) {
     this.paymentService = paymentService;
@@ -212,4 +226,19 @@ export class TurboAuthenticatedClient implements TurboPrivateClient {
   }): Promise<TurboUploadDataItemsResponse> {
     return this.uploadService.uploadFiles({ fileStreamGenerator, bundle });
   }
+
+  /**
+   * Verifies signature of signed data items and uploads to the upload service.
+   */
+  async uploadSignedDataItems({
+    dataItemGenerator,
+    signature,
+    publicKey,
+  }: TurboSignedDataItemFactory): Promise<TurboUploadDataItemsResponse> {
+    return this.uploadService.uploadSignedDataItems({
+      dataItemGenerator,
+      signature,
+      publicKey,
+    });
+  }
 }

src/common/upload.ts

@@ -17,7 +17,10 @@
 import { AxiosInstance, AxiosResponse } from 'axios';
 import { Readable } from 'stream';
 
-import { TurboNodeDataItemSigner } from '../node/signer.js';
+import {
+  TurboNodeDataItemSigner,
+  TurboNodeDataItemVerifier,
+} from '../node/signer.js';
 import { JWKInterface } from '../types/arweave.js';
 import {
   TransactionId,
@@ -26,6 +29,7 @@ import {
   TurboPrivateUploadServiceConfiguration,
   TurboPublicUploadService,
   TurboPublicUploadServiceConfiguration,
+  TurboSignedDataItemFactory,
   TurboUploadDataItemResponse,
   TurboUploadDataItemsResponse,
 } from '../types/turbo.js';
@@ -37,8 +41,11 @@ export class TurboUnauthenticatedUploadService
   implements TurboPublicUploadService
 {
   protected axios: AxiosInstance;
+  protected dataItemVerifier: TurboNodeDataItemVerifier;
+
   constructor({
     url = 'https://upload.ardrive.dev',
+    dataItemVerifier = new TurboNodeDataItemVerifier(),
     retryConfig,
   }: TurboPublicUploadServiceConfiguration) {
     this.axios = createAxiosInstance({
@@ -47,9 +54,27 @@ export class TurboUnauthenticatedUploadService
       },
       retryConfig,
     });
+    this.dataItemVerifier = dataItemVerifier;
   }
 
   // TODO: any public upload service APIS
+  async uploadSignedDataItems({
+    dataItemGenerator,
+    signature,
+    publicKey,
+  }: TurboSignedDataItemFactory): Promise<TurboUploadDataItemsResponse> {
+    const verified = await this.dataItemVerifier.verifySignedDataItems({
+      dataItemGenerator,
+      signature,
+      publicKey,
+    });
+    if (!verified) {
+      throw new Error('One or more data items failed signature validation');
+    }
+
+    // TODO: upload the files
+    return {} as TurboUploadDataItemsResponse;
+  }
 }
 
 export class TurboAuthenticatedUploadService
@@ -58,11 +83,13 @@ export class TurboAuthenticatedUploadService
   protected axios: AxiosInstance;
   protected privateKey: JWKInterface | undefined;
   protected dataItemSigner: TurboDataItemSigner;
+  protected dataItemVerifier: TurboNodeDataItemVerifier;
 
   constructor({
     url = 'https://upload.ardrive.dev',
     privateKey,
     dataItemSigner = new TurboNodeDataItemSigner({ privateKey }),
+    dataItemVerifier = new TurboNodeDataItemVerifier(),
     retryConfig,
   }: TurboPrivateUploadServiceConfiguration) {
     this.axios = createAxiosInstance({
@@ -73,6 +100,67 @@ export class TurboAuthenticatedUploadService
     });
     this.privateKey = privateKey;
     this.dataItemSigner = dataItemSigner;
+    this.dataItemVerifier = dataItemVerifier;
+  }
+
+  async uploadSignedDataItems({
+    dataItemGenerator,
+    signature,
+    publicKey,
+  }: TurboSignedDataItemFactory): Promise<TurboUploadDataItemsResponse> {
+    const verified = await this.dataItemVerifier.verifySignedDataItems({
+      dataItemGenerator,
+      signature,
+      publicKey,
+    });
+    if (!verified) {
+      throw new Error('One or more data items failed signature validation');
+    }
+
+    const signedDataItems: Readable[] = dataItemGenerator.map((dataItem) =>
+      dataItem(),
+    );
+
+    // TODO: add p-limit constraint
+    const uploadPromises = signedDataItems.map((signedDataItem) => {
+      return this.axios.post<TurboUploadDataItemResponse>(
+        `/tx`,
+        signedDataItem,
+        {
+          headers: {
+            'content-type': 'application/octet-stream',
+          },
+        },
+      );
+    });
+
+    // NOTE: our axios config (validateStatus) swallows errors, so failed data items will be ignored
+    const dataItemResponses = await Promise.all(uploadPromises);
+    const postedDataItems = dataItemResponses.reduce(
+      (
+        postedDataItemsMap: Record<
+          TransactionId,
+          Omit<TurboUploadDataItemResponse, 'id'>
+        >,
+        dataItemResponse: AxiosResponse<TurboUploadDataItemResponse, 'id'>,
+      ) => {
+        // handle the fulfilled response
+        const { status, data } = dataItemResponse;
+        if (![200, 202].includes(status)) {
+          // TODO: add to failed data items array
+          return postedDataItemsMap;
+        }
+        const { id, ...dataItemCache } = data;
+        postedDataItemsMap[id] = dataItemCache;
+        return postedDataItemsMap;
+      },
+      {},
+    );
+
+    return {
+      ownerAddress: publicKey,
+      dataItems: postedDataItems,
+    };
   }
 
   async uploadFiles({
@@ -107,25 +195,18 @@ export class TurboAuthenticatedUploadService
       );
     });
 
-    const dataItemResponses = await Promise.allSettled(uploadPromises);
+    // NOTE: our axios config (validateStatus) swallows errors, so failed data items will be ignored
+    const dataItemResponses = await Promise.all(uploadPromises);
     const postedDataItems = dataItemResponses.reduce(
       (
         postedDataItemsMap: Record<
           TransactionId,
           Omit<TurboUploadDataItemResponse, 'id'>
         >,
-        dataItemResponse:
-          | PromiseFulfilledResult<
-              AxiosResponse<TurboUploadDataItemResponse, 'id'>
-            >
-          | PromiseRejectedResult,
+        dataItemResponse: AxiosResponse<TurboUploadDataItemResponse, 'id'>,
       ) => {
-        // NOTE: with validateStatus set to true on the axios config we could use Promise.all and remove this check
-        if (dataItemResponse.status === 'rejected') {
-          return postedDataItemsMap;
-        }
         // handle the fulfilled response
-        const { status, data } = dataItemResponse.value;
+        const { status, data } = dataItemResponse;
         if (![200, 202].includes(status)) {
           // TODO: add to failed data items array
           return postedDataItemsMap;

src/node/signer.ts

@@ -16,16 +16,64 @@
  */
 import { ArweaveSigner, streamSigner } from 'arbundles';
 import { AxiosInstance } from 'axios';
-import { PassThrough, Readable } from 'stream';
+import crypto from 'crypto';
+import jwkToPem from 'jwk-to-pem';
+import { Readable } from 'stream';
 
 import { JWKInterface } from '../types/arweave.js';
-import { TurboDataItemSigner } from '../types/turbo.js';
+import {
+  TurboDataItemSigner,
+  TurboDataItemVerifier,
+  TurboSignedDataItemFactory,
+} from '../types/turbo.js';
+import { UnauthenticatedRequestError } from '../utils/errors.js';
+
+export class TurboNodeDataItemVerifier implements TurboDataItemVerifier {
+  async verifySignedDataItems({
+    dataItemGenerator,
+    signature,
+    publicKey,
+  }: TurboSignedDataItemFactory): Promise<boolean> {
+    const fullKey = {
+      kty: 'RSA',
+      e: 'AQAB',
+      n: publicKey,
+    };
+
+    const pem = jwkToPem(fullKey);
+    const verifiedDataItems: boolean[] = [];
+
+    // TODO: do this in parallel
+    for (const generateDataItem of dataItemGenerator) {
+      const verify = crypto.createVerify('sha256');
+      const signedDataItem = generateDataItem();
+      signedDataItem.on('data', (chunk) => {
+        verify.update(chunk);
+      });
+      signedDataItem.on('end', () => {
+        const dataItemVerified = verify.verify(
+          {
+            key: pem,
+            padding: crypto.constants.RSA_PKCS1_PSS_PADDING,
+          },
+          signature,
+        );
+        verifiedDataItems.push(dataItemVerified);
+      });
+      signedDataItem.on('error', () => {
+        verifiedDataItems.push(false);
+      });
+    }
+
+    return verifiedDataItems.every((dataItem) => dataItem);
+  }
+}
 
 export class TurboNodeDataItemSigner implements TurboDataItemSigner {
   protected axios: AxiosInstance;
-  protected privateKey: JWKInterface;
+  protected privateKey: JWKInterface | undefined; // TODO: break into separate classes
 
-  constructor({ privateKey }: { privateKey: JWKInterface }) {
+  constructor({ privateKey }: { privateKey?: JWKInterface } = {}) {
     this.privateKey = privateKey;
   }
 
@@ -35,13 +83,19 @@ export class TurboNodeDataItemSigner implements TurboDataItemSigner {
   }: {
     fileStreamGenerator: (() => Readable)[];
     bundle?: boolean;
-  }): Promise<PassThrough>[] {
+  }): Promise<Readable>[] {
+    // TODO: break this into separate classes
+    if (!this.privateKey) {
+      throw new UnauthenticatedRequestError();
+    }
+
     if (bundle) {
       throw new Error('Not implemented!');
     }
 
     const signer = new ArweaveSigner(this.privateKey);
 
+    // these are technically PassThrough's which are subclasses of streams
     const signedDataItemPromises = fileStreamGenerator.map(
       (fileStreamGenerator) => {
         const [stream1, stream2] = [