Usage collection speed improvements (#71317)

* Usage collection speed improvements * Remove commented code
elastic · Jul 13, 2020 · 5ccc7e9 · 5ccc7e9
1 parent 3a17e81
commit 5ccc7e9
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 24 deletions.
diff --git a/src/plugins/kibana_usage_collection/server/collectors/find_all.ts b/src/plugins/kibana_usage_collection/server/collectors/find_all.ts
@@ -28,7 +28,7 @@ export async function findAll<T extends SavedObjectAttributes>(
   savedObjectsClient: ISavedObjectsRepository,
   opts: SavedObjectsFindOptions
 ): Promise<Array<SavedObject<T>>> {
-  const { page = 1, perPage = 100, ...options } = opts;
+  const { page = 1, perPage = 10000, ...options } = opts;
   const { saved_objects: savedObjects, total } = await savedObjectsClient.find<T>({
     ...options,
     page,

diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts
@@ -83,14 +83,16 @@ export class CollectorSet {
       );
     }
 
-    const collectorTypesNotReady: string[] = [];
-    let allReady = true;
-    for (const collector of collectorSet.collectors.values()) {
-      if (!(await collector.isReady())) {
-        allReady = false;
-        collectorTypesNotReady.push(collector.type);
-      }
-    }
+    const collectorTypesNotReady = (
+      await Promise.all(
+        [...collectorSet.collectors.values()].map(async (collector) => {
+          if (!(await collector.isReady())) {
+            return collector.type;
+          }
+        })
+      )
+    ).filter((collectorType): collectorType is string => !!collectorType);
+    const allReady = collectorTypesNotReady.length === 0;
 
     if (!allReady && this.maximumWaitTimeForAllCollectorsInS >= 0) {
       const nowTimestamp = +new Date();
@@ -119,21 +121,24 @@ export class CollectorSet {
     callCluster: LegacyAPICaller,
     collectors: Map<string, Collector<any, any>> = this.collectors
   ) => {
-    const responses = [];
-    for (const collector of collectors.values()) {
-      this.logger.debug(`Fetching data from ${collector.type} collector`);
-      try {
-        responses.push({
-          type: collector.type,
-          result: await collector.fetch(callCluster),
-        });
-      } catch (err) {
-        this.logger.warn(err);
-        this.logger.warn(`Unable to fetch data from ${collector.type} collector`);
-      }
-    }
-
-    return responses;
+    const responses = await Promise.all(
+      [...collectors.values()].map(async (collector) => {
+        this.logger.debug(`Fetching data from ${collector.type} collector`);
+        try {
+          return {
+            type: collector.type,
+            result: await collector.fetch(callCluster),
+          };
+        } catch (err) {
+          this.logger.warn(err);
+          this.logger.warn(`Unable to fetch data from ${collector.type} collector`);
+        }
+      })
+    );
+
+    return responses.filter(
+      (response): response is { type: string; result: unknown } => typeof response !== 'undefined'
+    );
   };
 
   /*