Merge pull request #308 from colonial-heritage/search-ignore-0-return…

…-10000-terms fix: ignore terms with 0 results; return up to 10000 terms
colonial-heritage · Nov 7, 2023 · 19329e4 · 19329e4 · vercel · Nov 7, 2023
2 parents 13593c5 + bddbf03
commit 19329e4
Show file tree

Hide file tree

Showing 5 changed files with 32 additions and 217 deletions.
diff --git a/apps/researcher/jest.config.js b/apps/researcher/jest.config.js
@@ -4,6 +4,7 @@ const createJestConfig = nextJest();
 
 /** @type {import('jest').Config} */
 const customJestConfig = {
+  testTimeout: 60000,
   testMatch: ['**/*.test.ts(x)?'],
   collectCoverage: true,
   setupFiles: ['<rootDir>/jest.setup.js'],

diff --git a/apps/researcher/src/lib/api/objects/searcher-request.ts b/apps/researcher/src/lib/api/objects/searcher-request.ts
@@ -1,9 +1,8 @@
 export function buildAggregation(id: string) {
   const aggregation = {
     terms: {
-      size: 1000, // TBD: what's a good size?
+      size: 10000, // TBD: revisit this - return fewer terms instead
       field: id,
-      order: {_key: 'asc'},
     },
   };
 

diff --git a/apps/researcher/src/lib/api/objects/searcher-result.ts b/apps/researcher/src/lib/api/objects/searcher-result.ts
@@ -1,14 +1,6 @@
 import type {SearchResultFilter} from '../definitions';
 import type {RawBucket} from './searcher';
 
-function toUnmatchedFilter(bucket: RawBucket): SearchResultFilter {
-  const totalCount = 0; // Initial count; will be overridden by the matching filter, if any
-  const id = bucket.key;
-  const name = bucket.key; // Replace with labelFetcher as soon as we have IRIs
-
-  return {totalCount, id, name};
-}
-
 function toMatchedFilter(bucket: RawBucket): SearchResultFilter {
   const totalCount = bucket.doc_count; // Actual count if a filter matched the query
   const id = bucket.key;
@@ -17,36 +9,12 @@ function toMatchedFilter(bucket: RawBucket): SearchResultFilter {
   return {totalCount, id, name};
 }
 
-function combineUnmatchedWithMatchedFilters(
-  unmatchedFilters: SearchResultFilter[],
-  matchedFilters: SearchResultFilter[]
-) {
-  const combinedFilters = unmatchedFilters.map(filter => {
-    const matchedFilter = matchedFilters.find(
-      matchedFilter => matchedFilter.id === filter.id
-    );
-    return matchedFilter ?? filter;
-  });
-
-  return combinedFilters;
-}
-
-export function buildFilters(
-  rawUnmatchedFilters: RawBucket[],
-  rawMatchedFilters: RawBucket[]
-) {
-  const unmatchedFilters = rawUnmatchedFilters.map(rawUnmatchedFilter => {
-    return toUnmatchedFilter(rawUnmatchedFilter);
-  });
+export function buildFilters(rawMatchedFilters: RawBucket[]) {
   const matchedFilters = rawMatchedFilters.map(rawMatchedFilter => {
     return toMatchedFilter(rawMatchedFilter);
   });
-  const combinedFilters = combineUnmatchedWithMatchedFilters(
-    unmatchedFilters,
-    matchedFilters
-  );
 
   // TBD: sort filters by totalCount, descending + subsort by totalCount, ascending?
 
-  return combinedFilters;
+  return matchedFilters;
 }
diff --git a/apps/researcher/src/lib/api/objects/searcher.integration.test.ts b/apps/researcher/src/lib/api/objects/searcher.integration.test.ts
@@ -271,16 +271,16 @@ describe('search', () => {
       ],
       filters: {
         owners: [
-          {
-            totalCount: 1,
-            id: 'Library',
-            name: 'Library',
-          },
           {
             totalCount: 2,
             id: 'Museum',
             name: 'Museum',
           },
+          {
+            totalCount: 1,
+            id: 'Library',
+            name: 'Library',
+          },
           {
             totalCount: 1,
             id: 'Research Organisation',
@@ -310,16 +310,16 @@ describe('search', () => {
           },
         ],
         subjects: [
-          {
-            totalCount: 1,
-            id: 'Castle',
-            name: 'Castle',
-          },
           {
             totalCount: 2,
             id: 'Celebrations',
             name: 'Celebrations',
           },
+          {
+            totalCount: 1,
+            id: 'Castle',
+            name: 'Castle',
+          },
           {
             totalCount: 1,
             id: 'Cottage',
@@ -354,11 +354,6 @@ describe('search', () => {
             id: 'Canvas',
             name: 'Canvas',
           },
-          {
-            totalCount: 1,
-            id: 'Ink',
-            name: 'Ink',
-          },
           {
             totalCount: 2,
             id: 'Oilpaint',
@@ -369,6 +364,11 @@ describe('search', () => {
             id: 'Paper',
             name: 'Paper',
           },
+          {
+            totalCount: 1,
+            id: 'Ink',
+            name: 'Ink',
+          },
         ],
         creators: [
           {
@@ -389,20 +389,15 @@ describe('search', () => {
         ],
         publishers: [
           {
-            totalCount: 0,
-            id: 'Archive',
-            name: 'Archive',
+            totalCount: 3,
+            id: 'Museum',
+            name: 'Museum',
           },
           {
             totalCount: 1,
             id: 'Library',
             name: 'Library',
           },
-          {
-            totalCount: 3,
-            id: 'Museum',
-            name: 'Museum',
-          },
           {
             totalCount: 1,
             id: 'Onderzoeksinstelling',
@@ -472,15 +467,7 @@ describe('search', () => {
     expect(result).toMatchObject({
       totalCount: 1,
       filters: {
-        owners: [
-          {totalCount: 1, id: 'Library', name: 'Library'},
-          {totalCount: 0, id: 'Museum', name: 'Museum'},
-          {
-            totalCount: 0,
-            id: 'Research Organisation',
-            name: 'Research Organisation',
-          },
-        ],
+        owners: [{totalCount: 1, id: 'Library', name: 'Library'}],
       },
     });
   });
@@ -495,12 +482,7 @@ describe('search', () => {
     expect(result).toMatchObject({
       totalCount: 1,
       filters: {
-        types: [
-          {totalCount: 0, id: 'Canvas Painting', name: 'Canvas Painting'},
-          {totalCount: 0, id: 'Drawing', name: 'Drawing'},
-          {totalCount: 1, id: 'Painting', name: 'Painting'},
-          {totalCount: 0, id: 'Photo', name: 'Photo'},
-        ],
+        types: [{totalCount: 1, id: 'Painting', name: 'Painting'}],
       },
     });
   });
@@ -517,9 +499,7 @@ describe('search', () => {
       filters: {
         subjects: [
           {totalCount: 1, id: 'Castle', name: 'Castle'},
-          {totalCount: 0, id: 'Celebrations', name: 'Celebrations'},
           {totalCount: 1, id: 'Cottage', name: 'Cottage'},
-          {totalCount: 0, id: 'Palace', name: 'Palace'},
         ],
       },
     });
@@ -535,11 +515,7 @@ describe('search', () => {
     expect(result).toMatchObject({
       totalCount: 1,
       filters: {
-        locations: [
-          {totalCount: 0, id: 'Indonesia', name: 'Indonesia'},
-          {totalCount: 1, id: 'Malaysia', name: 'Malaysia'},
-          {totalCount: 0, id: 'Suriname', name: 'Suriname'},
-        ],
+        locations: [{totalCount: 1, id: 'Malaysia', name: 'Malaysia'}],
       },
     });
   });
@@ -560,21 +536,11 @@ describe('search', () => {
             id: 'Canvas',
             name: 'Canvas',
           },
-          {
-            totalCount: 0,
-            id: 'Ink',
-            name: 'Ink',
-          },
           {
             totalCount: 2,
             id: 'Oilpaint',
             name: 'Oilpaint',
           },
-          {
-            totalCount: 0,
-            id: 'Paper',
-            name: 'Paper',
-          },
         ],
       },
     });
@@ -596,16 +562,6 @@ describe('search', () => {
             id: 'Adriaan Boer',
             name: 'Adriaan Boer',
           },
-          {
-            totalCount: 0,
-            id: 'Geeske van Châtellerault',
-            name: 'Geeske van Châtellerault',
-          },
-          {
-            totalCount: 0,
-            id: 'Vincent van Gogh',
-            name: 'Vincent van Gogh',
-          },
         ],
       },
     });
@@ -622,31 +578,11 @@ describe('search', () => {
       totalCount: 1,
       filters: {
         publishers: [
-          {
-            totalCount: 0,
-            id: 'Archive',
-            name: 'Archive',
-          },
           {
             totalCount: 1,
             id: 'Library',
             name: 'Library',
           },
-          {
-            totalCount: 0,
-            id: 'Museum',
-            name: 'Museum',
-          },
-          {
-            totalCount: 0,
-            id: 'Onderzoeksinstelling',
-            name: 'Onderzoeksinstelling',
-          },
-          {
-            totalCount: 0,
-            id: 'Research Organisation',
-            name: 'Research Organisation',
-          },
         ],
       },
     });
@@ -663,21 +599,6 @@ describe('search', () => {
       totalCount: 1,
       filters: {
         dateCreatedStart: [
-          {
-            totalCount: 0,
-            id: 1725,
-            name: 1725,
-          },
-          {
-            totalCount: 0,
-            id: 1889,
-            name: 1889,
-          },
-          {
-            totalCount: 0,
-            id: 1895,
-            name: 1895,
-          },
           {
             totalCount: 1,
             id: 1901,
@@ -704,21 +625,6 @@ describe('search', () => {
             id: 1736,
             name: 1736,
           },
-          {
-            totalCount: 0,
-            id: 1890,
-            name: 1890,
-          },
-          {
-            totalCount: 0,
-            id: 1895,
-            name: 1895,
-          },
-          {
-            totalCount: 0,
-            id: 1902,
-            name: 1902,
-          },
         ],
       },
     });
@@ -751,11 +657,6 @@ describe('search', () => {
             id: 1895,
             name: 1895,
           },
-          {
-            totalCount: 0,
-            id: 1901,
-            name: 1901,
-          },
         ],
         dateCreatedEnd: [
           {
@@ -773,11 +674,6 @@ describe('search', () => {
             id: 1895,
             name: 1895,
           },
-          {
-            totalCount: 0,
-            id: 1902,
-            name: 1902,
-          },
         ],
       },
     });