From 63c1ab2f2eaa4fe4c2d2866ef13d64e706208c29 Mon Sep 17 00:00:00 2001 From: pubkey <8926560+pubkey@users.noreply.github.com> Date: Sat, 8 Apr 2023 15:23:05 +0200 Subject: [PATCH] IMPROVE performance of categorizeBulkWriteRows --- src/rx-storage-helper.ts | 80 ++++++++++++++++------------- test/unit.test.ts | 1 + test/unit/rx-storage-helper.test.ts | 72 ++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 35 deletions(-) create mode 100644 test/unit/rx-storage-helper.test.ts diff --git a/src/rx-storage-helper.ts b/src/rx-storage-helper.ts index 524052a94e9..3d472f75244 100644 --- a/src/rx-storage-helper.ts +++ b/src/rx-storage-helper.ts @@ -181,8 +181,9 @@ export function categorizeBulkWriteRows( const bulkUpdateDocs: BulkWriteRowProcessed[] = []; const errors: ById> = {}; const changedDocumentIds: RxDocType[StringKeys][] = []; + const eventBulkId = randomCouchString(10); const eventBulk: EventBulk>, any> = { - id: randomCouchString(10), + id: eventBulkId, events: [], checkpoint: null, context @@ -207,13 +208,18 @@ export function categorizeBulkWriteRows( const startTime = now(); const docsByIdIsMap = typeof docsInDb.get === 'function'; + const hasDocsInDb = docsByIdIsMap ? (docsInDb as Map).size > 0 : Object.keys(docsInDb).length > 0; + let newestRow: BulkWriteRowProcessed | undefined; const rowAmount = bulkWriteRows.length; - for (let i = 0; i < rowAmount; i++) { - const writeRow = bulkWriteRows[i]; - const id = writeRow.document[primaryPath]; - const documentInDb = docsByIdIsMap ? (docsInDb as any).get(id) : (docsInDb as any)[id]; + for (let rowId = 0; rowId < rowAmount; rowId++) { + const writeRow = bulkWriteRows[rowId]; + const docId = writeRow.document[primaryPath] as string; + let documentInDb: RxDocumentData | false = false; + if (hasDocsInDb) { + documentInDb = docsByIdIsMap ? (docsInDb as any).get(docId) : (docsInDb as any)[docId]; + } let attachmentError: RxStorageWriteErrorAttachment | undefined; if (!documentInDb) { @@ -228,16 +234,16 @@ export function categorizeBulkWriteRows( !(attachmentData as RxAttachmentWriteData).data ) { attachmentError = { - documentId: id as any, + documentId: docId, isError: true, status: 510, writeRow, attachmentId }; - errors[id as any] = attachmentError; + errors[docId] = attachmentError; } else { attachmentsAdd.push({ - documentId: id as any, + documentId: docId, attachmentId, attachmentData: attachmentData as any }); @@ -259,10 +265,15 @@ export function categorizeBulkWriteRows( } if (!insertedIsDeleted) { - changedDocumentIds.push(id); + changedDocumentIds.push(docId as any); eventBulk.events.push({ - eventId: getUniqueDeterministicEventKey(storageInstance, primaryPath as any, writeRow), - documentId: id as any, + eventId: getUniqueDeterministicEventKey( + eventBulkId, + rowId, + docId, + writeRow + ), + documentId: docId, operation: 'INSERT', documentData: hasAttachments ? stripAttachmentsDataFromDocument(writeRow.document) : writeRow.document as any, previousDocumentData: hasAttachments && writeRow.previous ? stripAttachmentsDataFromDocument(writeRow.previous) : writeRow.previous as any, @@ -290,11 +301,11 @@ export function categorizeBulkWriteRows( const err: RxStorageWriteError = { isError: true, status: 409, - documentId: id as any, + documentId: docId, writeRow: writeRow, documentInDb }; - errors[id as any] = err; + errors[docId] = err; continue; } @@ -311,7 +322,7 @@ export function categorizeBulkWriteRows( .keys(writeRow.previous._attachments) .forEach(attachmentId => { attachmentsRemove.push({ - documentId: id as any, + documentId: docId, attachmentId }); }); @@ -327,8 +338,8 @@ export function categorizeBulkWriteRows( !(attachmentData as RxAttachmentWriteData).data ) { attachmentError = { - documentId: id as any, - documentInDb, + documentId: docId, + documentInDb: documentInDb as any, isError: true, status: 510, writeRow, @@ -344,7 +355,7 @@ export function categorizeBulkWriteRows( const previousAttachmentData = writeRow.previous ? writeRow.previous._attachments[attachmentId] : undefined; if (!previousAttachmentData) { attachmentsAdd.push({ - documentId: id as any, + documentId: docId, attachmentId, attachmentData: attachmentData as any }); @@ -359,7 +370,7 @@ export function categorizeBulkWriteRows( previousAttachmentData.digest !== newDigest ) { attachmentsUpdate.push({ - documentId: id as any, + documentId: docId, attachmentId, attachmentData: attachmentData as RxAttachmentWriteData }); @@ -371,7 +382,7 @@ export function categorizeBulkWriteRows( } if (attachmentError) { - errors[id as any] = attachmentError; + errors[docId] = attachmentError; } else { bulkUpdateDocs.push(updatedRow); if ( @@ -403,11 +414,16 @@ export function categorizeBulkWriteRows( throw newRxError('SNH', { args: { writeRow } }); } - changedDocumentIds.push(id); + changedDocumentIds.push(docId as any); eventBulk.events.push({ - eventId: getUniqueDeterministicEventKey(storageInstance, primaryPath as any, writeRow), - documentId: id as any, - documentData: ensureNotFalsy(eventDocumentData), + eventId: getUniqueDeterministicEventKey( + eventBulkId, + rowId, + docId, + writeRow + ), + documentId: docId, + documentData: eventDocumentData as RxDocumentData, previousDocumentData: previousEventDocumentData, operation: operation, startTime, @@ -485,22 +501,16 @@ export function flatCloneDocWithMeta( /** * Each event is labeled with the id - * to make it easy to filter out duplicates. + * to make it easy to filter out duplicates + * even on flattened eventBulks */ export function getUniqueDeterministicEventKey( - storageInstance: RxStorageInstance, - primaryPath: string, + eventBulkId: string, + rowId: number, + docId: string, writeRow: BulkWriteRow ): string { - const docId = writeRow.document[primaryPath]; - const binaryValues: boolean[] = [ - !!writeRow.previous, - (writeRow.previous && writeRow.previous._deleted), - !!writeRow.document._deleted - ]; - const binary = binaryValues.map(v => v ? '1' : '0').join(''); - const eventKey = storageInstance.databaseName + '|' + storageInstance.collectionName + '|' + docId + '|' + '|' + binary + '|' + writeRow.document._rev; - return eventKey; + return eventBulkId + '|' + rowId + '|' + docId + '|' + writeRow.document._rev; } diff --git a/test/unit.test.ts b/test/unit.test.ts index 162a7edc2c5..74d45493dfd 100644 --- a/test/unit.test.ts +++ b/test/unit.test.ts @@ -19,6 +19,7 @@ import './unit/query-planner.test'; */ import './unit/rx-storage-implementations.test'; import './unit/rx-storage-query-correctness.test'; +import './unit/rx-storage-helper.test'; import './unit/rx-storage-lokijs.test'; import './unit/rx-storage-dexie.test'; diff --git a/test/unit/rx-storage-helper.test.ts b/test/unit/rx-storage-helper.test.ts new file mode 100644 index 00000000000..e4a1dfa1705 --- /dev/null +++ b/test/unit/rx-storage-helper.test.ts @@ -0,0 +1,72 @@ + +import config from './config'; +import * as schemaObjects from '../helper/schema-objects'; +import { + randomCouchString, + now, + fillWithDefaultSettings, + categorizeBulkWriteRows, + getPrimaryFieldOfPrimaryKey, + BulkWriteRow +} from '../../'; +import * as schemas from '../helper/schemas'; +import { + EXAMPLE_REVISION_1 +} from '../helper/revisions'; +import assert from 'assert'; + + +const testContext = 'rx-storage-helper.test.ts'; + +config.parallel('rx-storage-helper.test.ts', () => { + describe('.categorizeBulkWriteRows()', () => { + it('performance', async () => { + + const instance = await config.storage.getStorage().createStorageInstance({ + databaseInstanceToken: randomCouchString(10), + databaseName: randomCouchString(10), + collectionName: randomCouchString(10), + schema: fillWithDefaultSettings(schemas.human), + options: {}, + multiInstance: false, + devMode: true + }); + const primaryPath = getPrimaryFieldOfPrimaryKey(schemas.human.primaryKey); + const amount = config.isFastMode() ? 100 : 10000; + const writeRows: BulkWriteRow[] = new Array(amount).fill(0).map(() => { + const document = Object.assign( + schemaObjects.human(), + { + _deleted: false, + _rev: EXAMPLE_REVISION_1, + _attachments: {}, + _meta: { + lwt: now() + } + } + ); + return { document }; + }); + + const startTime = performance.now(); + + categorizeBulkWriteRows( + instance, + primaryPath, + new Map(), + writeRows, + testContext + ); + + const endTime = performance.now(); + const time = endTime - startTime; + + // console.log('time ' + time); + // process.exit(); + + + assert.ok(time); + instance.close(); + }); + }); +});