From 08e34d8ebb3ce3e33d76e0cee11c766cbdf029c9 Mon Sep 17 00:00:00 2001 From: pubkey <8926560+pubkey@users.noreply.github.com> Date: Fri, 7 Apr 2023 10:58:39 +0200 Subject: [PATCH] IMPROVE performance of index string creation --- orga/performance-trackings.md | 5 ++ src/custom-index.ts | 100 +++++++++++++++++---------------- test/unit/custom-index.test.ts | 27 ++++++++- 3 files changed, 83 insertions(+), 49 deletions(-) diff --git a/orga/performance-trackings.md b/orga/performance-trackings.md index a26a62c9416..4b26b453fc9 100644 --- a/orga/performance-trackings.md +++ b/orga/performance-trackings.md @@ -1359,3 +1359,8 @@ AFTER(fix iteration) 70.70821499824524 70.67861998081207 71.3054929971695 + +AFTER(use inner monad) +64.17834001779556 +72.26790100336075 +69.85145297646523 diff --git a/src/custom-index.ts b/src/custom-index.ts index 2db756d7cce..47d547de727 100644 --- a/src/custom-index.ts +++ b/src/custom-index.ts @@ -9,7 +9,9 @@ * Run performance tests before and after you touch anything here! */ -import { getSchemaByObjectPath } from './rx-schema-helper'; +import { + getSchemaByObjectPath +} from './rx-schema-helper'; import type { JsonSchema, RxDocumentData, @@ -20,7 +22,10 @@ import { objectPathMonad, ObjectPathMonadFunction } from './plugins/utils'; -import { INDEX_MAX, INDEX_MIN } from './query-planner'; +import { + INDEX_MAX, + INDEX_MIN +} from './query-planner'; /** @@ -31,17 +36,14 @@ import { INDEX_MAX, INDEX_MIN } from './query-planner'; * function is called many times. */ type IndexMetaField = { - // getValue() function - v: ObjectPathMonadFunction; - // type - t: | 0 // string - | 1 // boolean - | 2 // number - ; - // maxLength - ml: number; - // parsed lengths (only on number fields) - pl: ParsedLengths | undefined; + fieldName: string; + schemaPart: JsonSchema; + /* + * Only in number fields. + */ + parsedLengths?: ParsedLengths; + getValue: ObjectPathMonadFunction; + getIndexStringPart: (docData: RxDocumentData) => string; }; export function getIndexMeta( @@ -64,19 +66,39 @@ export function getIndexMeta( ); } - let typeId: IndexMetaField['t'] = 2; + const getValue = objectPathMonad(fieldName); + const maxLength = schemaPart.maxLength ? schemaPart.maxLength : 0; + + let getIndexStringPart: (docData: RxDocumentData) => string; if (type === 'string') { - typeId = 0; - } - if (type === 'boolean') { - typeId = 1; + getIndexStringPart = docData => { + let fieldValue = getValue(docData); + if (!fieldValue) { + fieldValue = ''; + } + return fieldValue.padEnd(maxLength, ' '); + }; + } else if (type === 'boolean') { + getIndexStringPart = docData => { + const fieldValue = getValue(docData); + return fieldValue ? '1' : '0'; + }; + } else { // number + getIndexStringPart = docData => { + const fieldValue = getValue(docData); + return getNumberIndexString( + parsedLengths as any, + fieldValue + ); + }; } const ret: IndexMetaField = { - v: objectPathMonad(fieldName), - t: typeId, - ml: schemaPart.maxLength ? schemaPart.maxLength : 0, - pl: parsedLengths + fieldName, + schemaPart, + parsedLengths, + getValue, + getIndexStringPart }; return ret; }); @@ -101,6 +123,7 @@ export function getIndexableStringMonad( ): (docData: RxDocumentData) => string { const fieldNameProperties = getIndexMeta(schema, index); const fieldNamePropertiesAmount = fieldNameProperties.length; + const indexPartsFunctions = fieldNameProperties.map(r => r.getIndexStringPart); /** @@ -109,27 +132,7 @@ export function getIndexableStringMonad( const ret = function (docData: RxDocumentData): string { let str = ''; for (let i = 0; i < fieldNamePropertiesAmount; ++i) { - const props = fieldNameProperties[i]; - const typeId = props.t; - let fieldValue = props.v(docData); - if (typeId === 0) { - // is string - if (!fieldValue) { - fieldValue = ''; - } - str += fieldValue.padEnd(props.ml, ' '); - } else if (typeId === 1) { - // is boolean - const boolToStr = fieldValue ? '1' : '0'; - str += boolToStr; - } else { - // is number - const parsedLengths = props.pl as ParsedLengths; - str += getNumberIndexString( - parsedLengths, - fieldValue - ); - } + str += indexPartsFunctions[i](docData); } return str; }; @@ -175,14 +178,15 @@ export function getIndexStringLength( const fieldNameProperties = getIndexMeta(schema, index); let length = 0; fieldNameProperties.forEach(props => { - const typeId = props.t; + const schemaPart = props.schemaPart; + const type = schemaPart.type; - if (typeId === 0) { - length += props.ml; - } else if (typeId === 1) { + if (type === 'string') { + length += schemaPart.maxLength as number; + } else if (type === 'boolean') { length += 1; } else { - const parsedLengths = props.pl as ParsedLengths; + const parsedLengths = props.parsedLengths as ParsedLengths; length = length + parsedLengths.nonDecimals + parsedLengths.decimals; } diff --git a/test/unit/custom-index.test.ts b/test/unit/custom-index.test.ts index 36fd79ee5f5..f495b00a0fb 100644 --- a/test/unit/custom-index.test.ts +++ b/test/unit/custom-index.test.ts @@ -3,7 +3,8 @@ import { clone, randomBoolean, randomNumber, - randomString + randomString, + wait } from 'async-test-util'; import { getIndexableStringMonad, @@ -19,6 +20,8 @@ import { ensureNotFalsy } from '../../'; import { EXAMPLE_REVISION_1 } from '../helper/revisions'; +import * as schemas from '../helper/schemas'; +import * as schemaObjects from '../helper/schema-objects'; import config from './config'; config.parallel('custom-index.test.ts', () => { @@ -255,6 +258,28 @@ config.parallel('custom-index.test.ts', () => { strA.split('').forEach(char => assert.strictEqual(char, ' ')); }); }); + describe('Performance', () => { + it('Run performance test', async () => { + const averageSchema = fillWithDefaultSettings(schemas.averageSchema()); + const docsAmount = 20000; + + const documents = new Array(docsAmount).fill(0).map(() => schemaObjects.averageSchema()); + const fns = ensureNotFalsy(averageSchema.indexes).map(index => getIndexableStringMonad(averageSchema, index as any)); + await wait(100); + const startTime = performance.now(); + for (const fn of fns) { + for (let i = 0; i < docsAmount; ++i) { + const doc = documents[i]; + fn(doc as any); + } + } + const endTime = performance.now(); + const time = endTime - startTime; + assert.ok(time); + console.log('time: ' + time); + // process.exit(); + }); + }); }); describe('.getIndexStringLength()', () => { it('get the correct length', () => {