feat: Improve PouchDB query performances with partialIndex

The react-native PouchDB adapter does not handle partialIndexes: it means that any predicate in partialIndex will not be evaluated at indexing time, but at querying time. When dealing with huge database, this can be quite impactful because it will result in full-scan of huge indexes. Therefore, we now force the indexing of any field in partialIndex, to guarantee a doc will be indexed if and only if it has all the necessary attributes. We measured performances for this query: https://github.com/cozy/mespapiers/blob/ e456faaa00dcefef13c6eee88459caffcac4f2d4/src/queries/index.js#L64-L105 **Before:** Query time: 51 499 ms **After:** Query time: 1 854 ms
cozy · Oct 3, 2024 · c6315e6 · c6315e6
1 parent 5b0b12d
commit c6315e6
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 12 deletions.
diff --git a/docs/api/cozy-pouch-link/classes/PouchLink.md b/docs/api/cozy-pouch-link/classes/PouchLink.md
@@ -132,7 +132,7 @@ CozyLink.constructor
 
 *Defined in*
 
-[CozyPouchLink.js:689](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L689)
+[CozyPouchLink.js:702](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L702)
 
 ***
 
@@ -152,7 +152,7 @@ CozyLink.constructor
 
 *Defined in*
 
-[CozyPouchLink.js:650](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L650)
+[CozyPouchLink.js:663](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L663)
 
 ***
 
@@ -199,7 +199,7 @@ Create the PouchDB index if not existing
 
 *Defined in*
 
-[CozyPouchLink.js:693](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L693)
+[CozyPouchLink.js:706](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L706)
 
 ***
 
@@ -219,7 +219,7 @@ Create the PouchDB index if not existing
 
 *Defined in*
 
-[CozyPouchLink.js:678](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L678)
+[CozyPouchLink.js:691](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L691)
 
 ***
 
@@ -241,7 +241,7 @@ Create the PouchDB index if not existing
 
 *Defined in*
 
-[CozyPouchLink.js:620](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L620)
+[CozyPouchLink.js:633](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L633)
 
 ***
 
@@ -261,7 +261,7 @@ Create the PouchDB index if not existing
 
 *Defined in*
 
-[CozyPouchLink.js:558](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L558)
+[CozyPouchLink.js:571](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L571)
 
 ***
 
@@ -701,7 +701,7 @@ Emits pouchlink:sync:stop event
 
 *Defined in*
 
-[CozyPouchLink.js:715](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L715)
+[CozyPouchLink.js:728](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L728)
 
 ***
 
@@ -721,7 +721,7 @@ Emits pouchlink:sync:stop event
 
 *Defined in*
 
-[CozyPouchLink.js:655](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L655)
+[CozyPouchLink.js:668](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L668)
 
 ***
 
@@ -741,7 +741,7 @@ Emits pouchlink:sync:stop event
 
 *Defined in*
 
-[CozyPouchLink.js:660](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L660)
+[CozyPouchLink.js:673](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/CozyPouchLink.js#L673)
 
 ***
 

diff --git a/packages/cozy-pouch-link/src/CozyPouchLink.js b/packages/cozy-pouch-link/src/CozyPouchLink.js
@@ -539,6 +539,19 @@ class PouchLink extends CozyLink {
 
     if (!indexedFields) {
       indexedFields = getIndexFields(options)
+    } else if (partialFilter) {
+      // Some pouch adapters does not support partialIndex, e.g. with websql in react-native
+      // Therefore, we need to force the indexing the partialIndex fields to ensure they will be
+      // included in the actual index. Thanks to this, docs with missing fields will be excluded
+      // from the index.
+      // Note the $exists: false case should be handled in-memory.
+      indexedFields = Array.from(
+        new Set([...indexedFields, ...Object.keys(partialFilter)])
+      )
+      // FIXME: should properly handle n-level attributes
+      indexedFields = indexedFields.filter(
+        field => field !== '$and' && field !== '$or'
+      )
     }
 
     const indexName = getIndexNameFromFields(indexedFields, partialFilter)

diff --git a/packages/cozy-pouch-link/src/CozyPouchLink.spec.js b/packages/cozy-pouch-link/src/CozyPouchLink.spec.js
@@ -608,9 +608,10 @@ describe('CozyPouchLink', () => {
       expect(spy).toHaveBeenCalled()
       expect(spy).toHaveBeenCalledWith({
         index: {
-          ddoc: 'by_myIndex_filter_(SOME_FIELD_$exists_true)',
-          fields: ['myIndex'],
-          indexName: 'by_myIndex_filter_(SOME_FIELD_$exists_true)',
+          ddoc: 'by_myIndex_and_SOME_FIELD_filter_(SOME_FIELD_$exists_true)',
+          fields: ['myIndex', 'SOME_FIELD'],
+          indexName:
+            'by_myIndex_and_SOME_FIELD_filter_(SOME_FIELD_$exists_true)',
           partial_filter_selector: {
             SOME_FIELD: {
               $exists: true
@@ -620,6 +621,38 @@ describe('CozyPouchLink', () => {
       })
     })
 
+    it('should exclude $and and $or operators from fields with partialIndex', async () => {
+      spy = jest.spyOn(PouchDB.prototype, 'createIndex').mockReturnValue({})
+      await setup()
+      await link.ensureIndex(TODO_DOCTYPE, {
+        indexedFields: ['myIndex'],
+        partialFilter: {
+          $and: [
+            { SOME_FIELD: { $exists: true } },
+            { SOME_FIELD: { $gt: null } }
+          ],
+          $or: [{ SOME_FIELD: { $eq: '1' } }, { SOME_FIELD: { $eq: '2' } }]
+        }
+      })
+      expect(spy).toHaveBeenCalled()
+      expect(spy).toHaveBeenCalledWith({
+        index: {
+          ddoc:
+            'by_myIndex_filter_((SOME_FIELD_$exists_true)_$and_(SOME_FIELD_$gt_null))_and_((SOME_FIELD_$eq_1)_$or_(SOME_FIELD_$eq_2))',
+          fields: ['myIndex'],
+          indexName:
+            'by_myIndex_filter_((SOME_FIELD_$exists_true)_$and_(SOME_FIELD_$gt_null))_and_((SOME_FIELD_$eq_1)_$or_(SOME_FIELD_$eq_2))',
+          partial_filter_selector: {
+            $and: [
+              { SOME_FIELD: { $exists: true } },
+              { SOME_FIELD: { $gt: null } }
+            ],
+            $or: [{ SOME_FIELD: { $eq: '1' } }, { SOME_FIELD: { $eq: '2' } }]
+          }
+        }
+      })
+    })
+
     it('uses the specified index', async () => {
       let spyIndex = jest
         .spyOn(CozyPouchLink.prototype, 'ensureIndex')