From 1ccbbfa1cdec9b560c6045cbf66b52a1cce3f7e5 Mon Sep 17 00:00:00 2001 From: dtfiedler Date: Tue, 5 Sep 2023 09:55:31 -0500 Subject: [PATCH] fix: revert to single file upload/data item upload quote> quote> Some concerns arose about proper control flow when uploading multiple files/data items. For now we will stick with single file uploads and make clients responsible for handling multiple uploads. --- examples/node/index.cjs | 11 +-- examples/node/index.mjs | 9 +-- examples/web/index.html | 36 ++++----- src/common/payment.ts | 1 + src/common/turbo.ts | 38 ++++----- src/common/upload.ts | 174 +++++++++------------------------------- src/node/signer.ts | 38 +++------ src/types/turbo.ts | 34 ++++---- src/web/signer.ts | 34 ++++---- tests/turbo.test.ts | 80 +++++++----------- 10 files changed, 152 insertions(+), 303 deletions(-) diff --git a/examples/node/index.cjs b/examples/node/index.cjs index b2b62a38..c9c7ff66 100644 --- a/examples/node/index.cjs +++ b/examples/node/index.cjs @@ -64,13 +64,10 @@ /** * Post local files to the Turbo service. */ - console.log('Posting raw files to Turbo service...'); - const filePaths = [path.join(__dirname, './files/0_kb.txt')]; - const fileStreamGenerators = filePaths.map( - (filePath) => () => fs.createReadStream(filePath), - ); - const uploadResult = await turboAuthClient.uploadFiles({ - fileStreamGenerators: fileStreamGenerators, + console.log('Posting raw file to Turbo service...'); + const filePath = path.join(__dirname, './files/0_kb.txt'); + const uploadResult = await turboAuthClient.uploadFile({ + fileStreamFactory: () => fs.createReadStream(filePath), }); console.log(JSON.stringify(uploadResult, null, 2)); })(); diff --git a/examples/node/index.mjs b/examples/node/index.mjs index 16ffeef2..470deb28 100644 --- a/examples/node/index.mjs +++ b/examples/node/index.mjs @@ -59,12 +59,9 @@ import { * Post local files to the Turbo service. */ console.log('Posting raw files to Turbo service...'); - const filePaths = [new URL('files/0_kb.txt', import.meta.url).pathname]; - const fileStreamGenerators = filePaths.map( - (filePath) => () => fs.createReadStream(filePath), - ); - const uploadResult = await turboAuthClient.uploadFiles({ - fileStreamGenerators: fileStreamGenerators, + const filePath = new URL('files/0_kb.txt', import.meta.url).pathname; + const uploadResult = await turboAuthClient.uploadFile({ + fileStreamFactory: () => fs.createReadStream(filePath), }); console.log(JSON.stringify(uploadResult, null, 2)); })(); diff --git a/examples/web/index.html b/examples/web/index.html index 5d18c08d..35bbf948 100644 --- a/examples/web/index.html +++ b/examples/web/index.html @@ -92,24 +92,24 @@

Upload File

const selectedFiles = fileInput.files; if (selectedFiles.length) { - const blobFileGenerators = Object.values(selectedFiles).map( - (file) => () => file.stream(), - ); - const response = turbo - .uploadFiles({ - fileStreamGenerators: blobFileGenerators, - }) - .then((res) => { - console.log('Successfully uploaded files!', res); - document.getElementById( - 'uploadStatus', - ).innerText = `Successfully uploaded files! ${JSON.stringify( - res, - null, - 2, - )}`; - }) - .catch((err) => console.log('Error uploading file!', err)); + // TODO: make this concurrent + for (const file of selectedFiles) { + const response = turbo + .uploadFile({ + fileStreamFactory: () => file.stream(), + }) + .then((res) => { + console.log('Successfully uploaded files!', res); + document.getElementById( + 'uploadStatus', + ).innerText = `Successfully uploaded files! ${JSON.stringify( + res, + null, + 2, + )}`; + }) + .catch((err) => console.log('Error uploading file!', err)); + } } else { console.log('No file selected'); } diff --git a/src/common/payment.ts b/src/common/payment.ts index 077a11a8..9ff4403d 100644 --- a/src/common/payment.ts +++ b/src/common/payment.ts @@ -166,6 +166,7 @@ export class TurboAuthenticatedPaymentService protected readonly privateKey: JWKInterface; protected readonly publicPaymentService: TurboUnauthenticatedPaymentServiceInterface; + // TODO: replace private key with an internal signer interface constructor({ url = 'https://payment.ardrive.dev', retryConfig, diff --git a/src/common/turbo.ts b/src/common/turbo.ts index 95b24588..89c2b236 100644 --- a/src/common/turbo.ts +++ b/src/common/turbo.ts @@ -32,7 +32,7 @@ import { TurboSignedDataItemFactory, TurboUnauthenticatedPaymentServiceInterface, TurboUnauthenticatedUploadServiceInterface, - TurboUploadDataItemsResponse, + TurboUploadDataItemResponse, } from '../types/index.js'; import { TurboAuthenticatedPaymentService, @@ -115,13 +115,13 @@ export class TurboUnauthenticatedClient implements TurboPublicClient { } /** - * Verifies signature of signed data items and uploads to the upload service. + * Uploads a signed data item to the Turbo Upload Service. */ - async uploadSignedDataItems({ - dataItemGenerators, - }: TurboSignedDataItemFactory): Promise { - return this.uploadService.uploadSignedDataItems({ - dataItemGenerators, + async uploadSignedDataItem({ + dataItemStreamFactory, + }: TurboSignedDataItemFactory): Promise { + return this.uploadService.uploadSignedDataItem({ + dataItemStreamFactory, }); } } @@ -210,22 +210,24 @@ export class TurboAuthenticatedClient implements TurboPrivateClient { } /** - * Signs and uploads data to the upload service. + * Signs and uploads raw data to the Turbo Upload Service. + * + * Note: 'privateKey' must be provided to use. */ - async uploadFiles({ - fileStreamGenerators, - }: TurboFileFactory): Promise { - return this.uploadService.uploadFiles({ fileStreamGenerators }); + async uploadFile({ + fileStreamFactory, + }: TurboFileFactory): Promise { + return this.uploadService.uploadFile({ fileStreamFactory }); } /** - * Verifies signature of signed data items and uploads to the upload service. + * Uploads a signed data item to the Turbo Upload Service. */ - async uploadSignedDataItems({ - dataItemGenerators, - }: TurboSignedDataItemFactory): Promise { - return this.uploadService.uploadSignedDataItems({ - dataItemGenerators, + async uploadSignedDataItem({ + dataItemStreamFactory, + }: TurboSignedDataItemFactory): Promise { + return this.uploadService.uploadSignedDataItem({ + dataItemStreamFactory, }); } } diff --git a/src/common/upload.ts b/src/common/upload.ts index 3b167578..c80354a8 100644 --- a/src/common/upload.ts +++ b/src/common/upload.ts @@ -14,12 +14,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -import { AxiosInstance, AxiosResponse } from 'axios'; +import { AxiosInstance } from 'axios'; import { TurboNodeDataItemSigner } from '../node/signer.js'; import { JWKInterface } from '../types/arweave.js'; import { - TransactionId, TurboAuthenticatedUploadServiceConfiguration, TurboAuthenticatedUploadServiceInterface, TurboDataItemSigner, @@ -28,9 +27,9 @@ import { TurboUnauthenticatedUploadServiceInterface, TurboUnauthenticatedUploadServiceInterfaceConfiguration, TurboUploadDataItemResponse, - TurboUploadDataItemsResponse, } from '../types/turbo.js'; import { createAxiosInstance } from '../utils/axiosClient.js'; +import { FailedRequestError } from '../utils/errors.js'; export class TurboUnauthenticatedUploadService implements TurboUnauthenticatedUploadServiceInterface @@ -49,58 +48,25 @@ export class TurboUnauthenticatedUploadService }); } - async uploadSignedDataItems({ - dataItemGenerators, - }: TurboSignedDataItemFactory): Promise { - const signedDataItems = dataItemGenerators.map((dataItem) => dataItem()); - - // TODO: add p-limit constraint - const uploadPromises = signedDataItems.map((signedDataItem) => { - return this.axios.post( + async uploadSignedDataItem({ + dataItemStreamFactory, + }: TurboSignedDataItemFactory): Promise { + // TODO: add p-limit constraint or replace with separate upload class + const { status, data, statusText } = + await this.axios.post( `/tx`, - signedDataItem, + dataItemStreamFactory(), { headers: { 'content-type': 'application/octet-stream', }, }, ); - }); - // NOTE: our axios config (validateStatus) swallows errors, so failed data items will be ignored - const dataItemResponses = await Promise.all(uploadPromises); - const errors: { id: string; status: number; message: string }[] = []; - const postedDataItems = dataItemResponses.reduce( - ( - postedDataItemsMap: Record< - TransactionId, - Omit - >, - dataItemResponse: AxiosResponse, - ) => { - // handle the fulfilled response - const { status, data, statusText } = dataItemResponse; - if (![200, 202].includes(status)) { - // TODO: add to failed data items array - errors.push({ - id: data.id ?? 'unknown', - - status, - message: statusText, - }); - return postedDataItemsMap; - } - const { id, ...dataItemCache } = data; - postedDataItemsMap[id] = dataItemCache; - return postedDataItemsMap; - }, - {}, - ); - - return { - dataItems: postedDataItems, - errors, - }; + if (![202, 200].includes(status)) { + throw new FailedRequestError(status, statusText); + } + return data; } } @@ -127,74 +93,36 @@ export class TurboAuthenticatedUploadService this.dataItemSigner = dataItemSigner; } - async uploadSignedDataItems({ - dataItemGenerators, - }: TurboSignedDataItemFactory): Promise { - const signedDataItems = dataItemGenerators.map((dataItem) => dataItem()); - - console.log('upload signed data items here'); - - // TODO: add p-limit constraint - const uploadPromises = signedDataItems.map((signedDataItem) => { - return this.axios.post( + async uploadSignedDataItem({ + dataItemStreamFactory, + }: TurboSignedDataItemFactory): Promise { + // TODO: add p-limit constraint or replace with separate upload class + const { status, data, statusText } = + await this.axios.post( `/tx`, - signedDataItem, + dataItemStreamFactory(), { headers: { 'content-type': 'application/octet-stream', }, }, ); - }); - - // NOTE: our axios config (validateStatus) swallows errors, so failed data items will be ignored - const dataItemResponses = await Promise.all(uploadPromises); - const errors: { id: string; status: number; message: string }[] = []; - const postedDataItems = dataItemResponses.reduce( - ( - postedDataItemsMap: Record< - TransactionId, - Omit - >, - dataItemResponse: AxiosResponse, - ) => { - // handle the fulfilled response - const { status, data, statusText } = dataItemResponse; - if (![200, 202].includes(status)) { - errors.push({ - id: data.id ?? 'unknown', - status, - message: statusText, - }); - return postedDataItemsMap; - } - const { id, ...dataItemCache } = data; - postedDataItemsMap[id] = dataItemCache; - return postedDataItemsMap; - }, - {}, - ); - - return { - dataItems: postedDataItems, - errors, - }; + if (![202, 200].includes(status)) { + throw new FailedRequestError(status, statusText); + } + return data; } - async uploadFiles({ - fileStreamGenerators, - }: TurboFileFactory): Promise { - const signedDataItemPromises = this.dataItemSigner.signDataItems({ - fileStreamGenerators, + async uploadFile({ + fileStreamFactory, + }: TurboFileFactory): Promise { + const signedDataItem = await this.dataItemSigner.signDataItem({ + fileStreamFactory, }); - - // TODO: we probably don't want to Promise.all, do .allSettled and only return successful signed data items - const signedDataItems = await Promise.all(signedDataItemPromises); - - // TODO: add p-limit constraint - const uploadPromises = signedDataItems.map((signedDataItem) => { - return this.axios.post( + // TODO: add p-limit constraint or replace with separate upload class + const { status, data, statusText } = + await this.axios.post( `/tx`, signedDataItem, { @@ -203,40 +131,10 @@ export class TurboAuthenticatedUploadService }, }, ); - }); - - // NOTE: our axios config (validateStatus) swallows errors, so failed data items will be ignored - const dataItemResponses = await Promise.all(uploadPromises); - const errors: { id: string; status: number; message: string }[] = []; - const postedDataItems = dataItemResponses.reduce( - ( - postedDataItemsMap: Record< - TransactionId, - Omit - >, - dataItemResponse: AxiosResponse, - ) => { - // handle the fulfilled response - const { status, data, statusText } = dataItemResponse; - if (![200, 202].includes(status)) { - // TODO: add to failed data items array - errors.push({ - id: data.id ?? 'unknown', - status, - message: statusText, - }); - return postedDataItemsMap; - } - const { id, ...dataItemCache } = data; - postedDataItemsMap[id] = dataItemCache; - return postedDataItemsMap; - }, - {}, - ); - return { - dataItems: postedDataItems, - errors, - }; + if (![202, 200].includes(status)) { + throw new FailedRequestError(status, statusText); + } + return data; } } diff --git a/src/node/signer.ts b/src/node/signer.ts index 9e301baf..f357a753 100644 --- a/src/node/signer.ts +++ b/src/node/signer.ts @@ -19,40 +19,24 @@ import { AxiosInstance } from 'axios'; import { Readable } from 'node:stream'; import { JWKInterface } from '../types/arweave.js'; -import { TurboDataItemSigner, TurboFileFactory } from '../types/turbo.js'; -import { UnauthenticatedRequestError } from '../utils/errors.js'; +import { TurboDataItemSigner } from '../types/turbo.js'; export class TurboNodeDataItemSigner implements TurboDataItemSigner { protected axios: AxiosInstance; - protected privateKey: JWKInterface | undefined; // TODO: break into separate classes + protected privateKey: JWKInterface; - constructor({ privateKey }: { privateKey?: JWKInterface } = {}) { + // TODO: replace with internal signer class + constructor({ privateKey }: { privateKey: JWKInterface }) { this.privateKey = privateKey; } - signDataItems({ - fileStreamGenerators, - }: Omit & { - fileStreamGenerators: (() => Readable)[]; - }): Promise[] { - // TODO: break this into separate classes - if (!this.privateKey) { - throw new UnauthenticatedRequestError(); - } - + signDataItem({ + fileStreamFactory, + }: { + fileStreamFactory: () => Readable; + }): Promise { const signer = new ArweaveSigner(this.privateKey); - - // these are technically PassThrough's which are subclasses of streams - const signedDataItemPromises = fileStreamGenerators.map( - (fileStreamGenerators) => { - const [stream1, stream2] = [ - fileStreamGenerators(), - fileStreamGenerators(), - ]; - // TODO: this will not work with BDIs as is, we may need to add an additional stream signer - return streamSigner(stream1, stream2, signer); - }, - ); - return signedDataItemPromises; + const [stream1, stream2] = [fileStreamFactory(), fileStreamFactory()]; + return streamSigner(stream1, stream2, signer); } } diff --git a/src/types/turbo.ts b/src/types/turbo.ts index 71116ed2..de2a80ad 100644 --- a/src/types/turbo.ts +++ b/src/types/turbo.ts @@ -60,11 +60,6 @@ export type TurboUploadDataItemResponse = { id: TransactionId; }; -export type TurboUploadDataItemsResponse = { - dataItems: Record>; - errors: { id: string; status: number; message: string }[]; -}; - export type TurboSignedRequestHeaders = { 'x-public-key': string; 'x-nonce': string; @@ -111,21 +106,24 @@ export type TurboPrivateClientConfiguration = { uploadService: TurboAuthenticatedUploadServiceInterface; } & TurboAuthConfiguration; +export type FileStreamFactory = + | (() => Readable) + | (() => ReadableStream) + | (() => Buffer); +export type SignedDataStreamFactory = FileStreamFactory; export type TurboFileFactory = { - fileStreamGenerators: (() => Readable)[] | (() => ReadableStream)[]; + fileStreamFactory: FileStreamFactory; // TODO: allow multiple files // bundle?: boolean; // TODO: add bundling into BDIs - // TODO: add payload size }; -// TODO: add web one for ReadableStream or Buffer depending on how best to implement export type TurboSignedDataItemFactory = { - dataItemGenerators: (() => Readable | Buffer | ReadableStream)[]; + dataItemStreamFactory: SignedDataStreamFactory; // TODO: allow multiple data items }; export interface TurboDataItemSigner { - signDataItems({ - fileStreamGenerators, - }: TurboFileFactory): Promise[] | Promise[]; + signDataItem({ + fileStreamFactory, + }: TurboFileFactory): Promise | Promise; } export interface TurboUnauthenticatedPaymentServiceInterface { @@ -153,16 +151,16 @@ export interface TurboAuthenticatedPaymentServiceInterface } export interface TurboUnauthenticatedUploadServiceInterface { - uploadSignedDataItems({ - dataItemGenerators, - }: TurboSignedDataItemFactory): Promise; + uploadSignedDataItem({ + dataItemStreamFactory, + }: TurboSignedDataItemFactory): Promise; } export interface TurboAuthenticatedUploadServiceInterface extends TurboUnauthenticatedUploadServiceInterface { - uploadFiles({ - fileStreamGenerators, - }: TurboFileFactory): Promise; + uploadFile({ + fileStreamFactory, + }: TurboFileFactory): Promise; } export interface TurboPublicClient diff --git a/src/web/signer.ts b/src/web/signer.ts index 81d4f958..d186391e 100644 --- a/src/web/signer.ts +++ b/src/web/signer.ts @@ -19,7 +19,7 @@ import { AxiosInstance } from 'axios'; import { ReadableStream } from 'node:stream/web'; import { JWKInterface } from '../types/arweave.js'; -import { TurboDataItemSigner, TurboFileFactory } from '../types/turbo.js'; +import { TurboDataItemSigner } from '../types/turbo.js'; import { readableStreamToBuffer } from '../utils/readableStream.js'; export class TurboWebDataItemSigner implements TurboDataItemSigner { @@ -30,25 +30,19 @@ export class TurboWebDataItemSigner implements TurboDataItemSigner { this.privateKey = privateKey; } - signDataItems({ - fileStreamGenerators, - }: Omit & { - fileStreamGenerators: (() => ReadableStream)[]; - }): Promise[] { + async signDataItem({ + fileStreamFactory, + }: { + fileStreamFactory: () => ReadableStream; + }): Promise { + // TODO: replace this with an internal signer class const signer = new ArweaveSigner(this.privateKey); - - const signedDataItemPromises = fileStreamGenerators.map( - async (streamGenerator: () => ReadableStream) => { - // Convert the readable stream to a buffer - const buffer = await readableStreamToBuffer({ - stream: streamGenerator(), - }); - const dataItem = createData(buffer, signer); - await dataItem.sign(signer); - return dataItem.getRaw(); - }, - ); - - return signedDataItemPromises; + // Convert the readable stream to a buffer + const buffer = await readableStreamToBuffer({ + stream: fileStreamFactory(), + }); + const dataItem = createData(buffer, signer); + await dataItem.sign(signer); + return dataItem.getRaw(); } } diff --git a/tests/turbo.test.ts b/tests/turbo.test.ts index 53553256..0ce92aa5 100644 --- a/tests/turbo.test.ts +++ b/tests/turbo.test.ts @@ -130,26 +130,21 @@ describe('Node environment', () => { expect(+winc).to.be.greaterThan(0); }); - describe('uploadSignedDataItems()', async () => { + describe('uploadSignedDataItem()', async () => { it('supports sending a signed Buffer to turbo', async () => { const jwk = await Arweave.crypto.generateJWK(); const signer = new ArweaveSigner(jwk); const signedDataItem = createData('signed data item', signer, {}); await signedDataItem.sign(signer); - const response = await turbo.uploadSignedDataItems({ - dataItemGenerators: [() => signedDataItem.getRaw()], + const response = await turbo.uploadSignedDataItem({ + dataItemStreamFactory: () => signedDataItem.getRaw(), }); expect(response).to.not.be.undefined; - expect(response).to.have.property('dataItems'); - expect(response).to.have.property('errors'); - expect(response['errors']).to.have.length(0); - for (const dataItem of Object.values(response['dataItems'])) { - expect(dataItem).to.have.property('fastFinalityIndexes'); - expect(dataItem).to.have.property('dataCaches'); - expect(dataItem).to.have.property('owner'); - expect(dataItem['owner']).to.equal(jwkToPublicArweaveAddress(jwk)); - } + expect(response).to.have.property('fastFinalityIndexes'); + expect(response).to.have.property('dataCaches'); + expect(response).to.have.property('owner'); + expect(response['owner']).to.equal(jwkToPublicArweaveAddress(jwk)); }); it('supports sending a signed Readable to turbo', async () => { @@ -158,19 +153,14 @@ describe('Node environment', () => { const signedDataItem = createData('signed data item', signer, {}); await signedDataItem.sign(signer); - const response = await turbo.uploadSignedDataItems({ - dataItemGenerators: [() => Readable.from(signedDataItem.getRaw())], + const response = await turbo.uploadSignedDataItem({ + dataItemStreamFactory: () => Readable.from(signedDataItem.getRaw()), }); expect(response).to.not.be.undefined; - expect(response).to.have.property('dataItems'); - expect(response).to.have.property('errors'); - expect(response['errors']).to.have.length(0); - for (const dataItem of Object.values(response['dataItems'])) { - expect(dataItem).to.have.property('fastFinalityIndexes'); - expect(dataItem).to.have.property('dataCaches'); - expect(dataItem).to.have.property('owner'); - expect(dataItem['owner']).to.equal(jwkToPublicArweaveAddress(jwk)); - } + expect(response).to.have.property('fastFinalityIndexes'); + expect(response).to.have.property('dataCaches'); + expect(response).to.have.property('owner'); + expect(response['owner']).to.equal(jwkToPublicArweaveAddress(jwk)); }); }); }); @@ -193,25 +183,18 @@ describe('Node environment', () => { expect(+balance.winc).to.equal(0); }); - it('uploadFiles()', async () => { + it('uploadFile()', async () => { const file = new URL('files/0_kb.txt', import.meta.url).pathname; - const streamGenerator = [() => fs.createReadStream(file)]; - const response = await turbo.uploadFiles({ - fileStreamGenerators: streamGenerator, + const streamGenerator = () => fs.createReadStream(file); + const response = await turbo.uploadFile({ + fileStreamFactory: streamGenerator, }); expect(response).to.not.be.undefined; - expect(response).to.have.property('dataItems'); - expect(Object.keys(response['dataItems']).length).to.equal( - streamGenerator.length, - ); - expect(response).to.have.property('errors'); - expect(response['errors']).to.have.length(0); - for (const dataItem of Object.values(response['dataItems'])) { - expect(dataItem).to.have.property('fastFinalityIndexes'); - expect(dataItem).to.have.property('dataCaches'); - expect(dataItem).to.have.property('owner'); - expect(dataItem['owner']).to.equal(jwkToPublicArweaveAddress(jwk)); - } + expect(response).to.not.be.undefined; + expect(response).to.have.property('fastFinalityIndexes'); + expect(response).to.have.property('dataCaches'); + expect(response).to.have.property('owner'); + expect(response['owner']).to.equal(jwkToPublicArweaveAddress(jwk)); }); }); }); @@ -310,26 +293,21 @@ describe('Browser environment', () => { expect(+winc).to.be.greaterThan(0); }); - describe('uploadSignedDataItems()', async () => { + describe('uploadSignedDataItem()', async () => { it('supports sending a signed Buffer to turbo', async () => { const jwk = await Arweave.crypto.generateJWK(); const signer = new ArweaveSigner(jwk); const signedDataItem = createData('signed data item', signer); await signedDataItem.sign(signer); - const response = await turbo.uploadSignedDataItems({ - dataItemGenerators: [() => signedDataItem.getRaw()], + const response = await turbo.uploadSignedDataItem({ + dataItemStreamFactory: () => signedDataItem.getRaw(), }); expect(response).to.not.be.undefined; - expect(response).to.have.property('dataItems'); - expect(response).to.have.property('errors'); - expect(response['errors']).to.have.length(0); - for (const dataItem of Object.values(response['dataItems'])) { - expect(dataItem).to.have.property('fastFinalityIndexes'); - expect(dataItem).to.have.property('dataCaches'); - expect(dataItem).to.have.property('owner'); - expect(dataItem['owner']).to.equal(jwkToPublicArweaveAddress(jwk)); - } + expect(response).to.have.property('fastFinalityIndexes'); + expect(response).to.have.property('dataCaches'); + expect(response).to.have.property('owner'); + expect(response['owner']).to.equal(jwkToPublicArweaveAddress(jwk)); }); // TODO: add test that polyfills posting a signed ReadableStream to turbo