[Data Table] Expensive queries are causing unnecessary load and delay…

…s on Elasticsearch Part of elastic#93770
alexwizp · Apr 30, 2021 · f6795f5 · f6795f5
1 parent 036821d
commit f6795f5
Show file tree

Hide file tree

Showing 2 changed files with 48 additions and 12 deletions.
diff --git a/src/plugins/vis_type_table/server/usage_collector/get_stats.test.ts b/src/plugins/vis_type_table/server/usage_collector/get_stats.test.ts
@@ -42,14 +42,18 @@ const mockVisualizations = {
 
 describe('vis_type_table getStats', () => {
   const mockSoClient = ({
-    find: jest.fn().mockResolvedValue(mockVisualizations),
+    createPointInTimeFinder: jest.fn().mockResolvedValue({
+      find: function* asyncGenerator() {
+        yield mockVisualizations;
+      },
+    }),
   } as unknown) as SavedObjectsClientContract;
 
   test('Returns stats from saved objects for table vis only', async () => {
     const result = await getStats(mockSoClient);
-    expect(mockSoClient.find).toHaveBeenCalledWith({
+    expect(mockSoClient.createPointInTimeFinder).toHaveBeenCalledWith({
       type: 'visualization',
-      perPage: 10000,
+      perPage: 1000,
     });
     expect(result).toEqual({
       total: 4,

diff --git a/src/plugins/vis_type_table/server/usage_collector/get_stats.ts b/src/plugins/vis_type_table/server/usage_collector/get_stats.ts
@@ -6,13 +6,17 @@
  * Side Public License, v 1.
  */
 
-import { ISavedObjectsRepository, SavedObjectsClientContract } from 'kibana/server';
 import {
-  SavedVisState,
-  VisualizationSavedObjectAttributes,
-} from 'src/plugins/visualizations/common';
+  ISavedObjectsRepository,
+  SavedObjectsClientContract,
+  SavedObjectsFindResult,
+} from 'kibana/server';
+import { SavedVisState } from 'src/plugins/visualizations/common';
 import { TableVisParams, VIS_TYPE_TABLE } from '../../common';
 
+// elasticsearch index.max_result_window default value
+const ES_MAX_RESULT_WINDOW_DEFAULT_VALUE = 1000;
+
 export interface VisTypeTableUsage {
   /**
    * Total number of table type visualizations
@@ -38,20 +42,48 @@ export interface VisTypeTableUsage {
   };
 }
 
+/** @internal **/
+type SavedTableVisState = SavedVisState<TableVisParams>;
+
 /*
  * Parse the response data into telemetry payload
  */
 export async function getStats(
   soClient: SavedObjectsClientContract | ISavedObjectsRepository
 ): Promise<VisTypeTableUsage | undefined> {
-  const visualizations = await soClient.find<VisualizationSavedObjectAttributes>({
+  const finder = await soClient.createPointInTimeFinder({
     type: 'visualization',
-    perPage: 10000,
+    perPage: ES_MAX_RESULT_WINDOW_DEFAULT_VALUE,
   });
 
-  const tableVisualizations = visualizations.saved_objects
-    .map<SavedVisState<TableVisParams>>(({ attributes }) => JSON.parse(attributes.visState))
-    .filter(({ type }) => type === VIS_TYPE_TABLE);
+  let tableVisualizations: SavedTableVisState[] = [];
+
+  for await (const response of finder.find()) {
+    tableVisualizations = [
+      ...tableVisualizations,
+      ...(response.saved_objects || []).reduce(
+        (acc: SavedTableVisState[], { attributes }: SavedObjectsFindResult<any>) => {
+          if (attributes?.visState) {
+            try {
+              const visState: SavedVisState = JSON.parse(attributes.visState);
+
+              if (visState.type === VIS_TYPE_TABLE) {
+                acc.push(visState as SavedTableVisState);
+              }
+            } catch {
+              // nothing to be here, "so" not valid
+            }
+          }
+          return acc;
+        },
+        []
+      ),
+    ];
+
+    if (!response.saved_objects.length) {
+      await finder.close();
+    }
+  }
 
   const defaultStats = {
     total: tableVisualizations.length,