Switch collector to use task data

x-pack/legacy/plugins/lens/server/usage/collectors.ts

@@ -4,10 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { KibanaConfig } from 'src/legacy/server/kbn_server';
+import { get } from 'lodash';
+import { Server, KibanaConfig } from 'src/legacy/server/kbn_server';
 import { CallCluster } from 'src/legacy/core_plugins/elasticsearch';
 import { CoreSetup, SavedObjectsLegacyService } from 'src/core/server';
-import { getVisualizationCounts } from './visualization_counts';
 import { LensUsage } from './types';
 
 export function registerLensUsageCollector(
@@ -21,13 +21,32 @@ export function registerLensUsageCollector(
       };
     };
     config: KibanaConfig;
+    server: Server;
   }
 ) {
+  let isCollectorReady = false;
+  async function determineIfTaskManagerIsReady() {
+    let isReady = false;
+    try {
+      isReady = await isTaskManagerReady(plugins.server);
+    } catch (err) {} // eslint-disable-line
+
+    if (isReady) {
+      isCollectorReady = true;
+    } else {
+      setTimeout(determineIfTaskManagerIsReady, 500);
+    }
+  }
+  determineIfTaskManagerIsReady();
+
   const lensUsageCollector = plugins.usage.collectorSet.makeUsageCollector({
     type: 'lens',
     fetch: async (callCluster: CallCluster): Promise<LensUsage> => {
       try {
-        return getVisualizationCounts(callCluster, plugins.config);
+        const docs = await fetch(plugins.server);
+        // get the accumulated state from the recurring task
+        return get(docs, '[0].state.stats');
+        // return getVisualizationCounts(callCluster, plugins.config);
       } catch (err) {
         return {
           saved_total: 0,
@@ -36,10 +55,45 @@ export function registerLensUsageCollector(
           visualization_types_overall: {},
           visualization_types_last_30_days: {},
           visualization_types_last_90_days: {},
+
+          clicks_last_30_days: {},
+          clicks_last_90_days: {},
+          suggestion_clicks_last_30_days: {},
+          suggestion_clicks_last_90_days: {},
         };
       }
     },
-    isReady: () => true,
+    isReady: () => isCollectorReady,
   });
   plugins.usage.collectorSet.register(lensUsageCollector);
 }
+
+async function isTaskManagerReady(server: Server) {
+  const result = await fetch(server);
+  return result !== null;
+}
+
+async function fetch(server: Server) {
+  const taskManager = server.plugins.task_manager!;
+
+  let docs;
+  try {
+    ({ docs } = await taskManager.fetch({
+      query: { bool: { filter: { term: { _id: `task:Lens-lens_telemetry` } } } },
+    }));
+  } catch (err) {
+    const errMessage = err && err.message ? err.message : err.toString();
+    /*
+      The usage service WILL to try to fetch from this collector before the task manager has been initialized, because the
+      task manager has to wait for all plugins to initialize first. It's fine to ignore it as next time around it will be
+      initialized (or it will throw a different type of error)
+    */
+    if (errMessage.includes('NotInitialized')) {
+      docs = null;
+    } else {
+      throw err;
+    }
+  }
+
+  return docs;
+}

x-pack/legacy/plugins/lens/server/usage/task.ts

@@ -9,6 +9,7 @@ import { CoreSetup } from 'src/core/server';
 import { CallClusterOptions } from 'src/legacy/core_plugins/elasticsearch';
 import { SearchParams, SearchResponse, DeleteDocumentByQueryResponse } from 'elasticsearch';
 import { RunContext } from '../../../task_manager';
+import { getVisualizationCounts } from './visualization_counts';
 
 // This task is responsible for running daily and aggregating all the Lens click event objects
 // into daily rolled-up documents, which will be used in reporting click stats
@@ -105,6 +106,8 @@ async function doWork(server: Server, callCluster: ClusterSearchType & ClusterDe
     size: 0,
   });
 
+  console.log(JSON.stringify(metrics));
+
   const byDateByType: Record<string, Record<string, number>> = {};
 
   metrics.aggregations.daily.buckets.forEach(bucket => {
@@ -140,21 +143,28 @@ function telemetryTaskRunner(server: Server) {
     const callCluster = server.plugins.elasticsearch.getCluster('admin').callWithInternalUser;
 
     let lensTelemetryTask: Promise<unknown>;
+    let lensVisualizationTask: ReturnType<typeof getVisualizationCounts>;
 
     return {
       async run() {
         try {
           lensTelemetryTask = doWork(server, callCluster);
+
+          lensVisualizationTask = getVisualizationCounts(callCluster, server.config());
         } catch (err) {
           server.log(['warning'], `Error loading lens telemetry: ${err}`);
         }
 
-        return lensTelemetryTask
-          .then((lensTelemetry = {}) => {
+        return Promise.all([lensTelemetryTask, lensVisualizationTask])
+          .then(([lensTelemetry, lensVisualizations]) => {
             return {
               state: {
                 runs: state.runs || 1,
-                stats: lensTelemetry || prevState.stats || {},
+                stats: Object.assign(
+                  {},
+                  lensTelemetry || prevState.stats || {},
+                  lensVisualizations
+                ),
               },
               runAt: getNextMidnight(),
             };

x-pack/legacy/plugins/lens/server/usage/types.ts

@@ -4,11 +4,20 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export interface LensUsage {
+export interface LensVisualizationUsage {
   visualization_types_overall: Record<string, number>;
   visualization_types_last_30_days: Record<string, number>;
   visualization_types_last_90_days: Record<string, number>;
   saved_total: number;
   saved_last_30_days: number;
   saved_last_90_days: number;
 }
+
+export interface LensClickUsage {
+  clicks_last_30_days: Record<string, number>;
+  clicks_last_90_days: Record<string, number>;
+  suggestion_clicks_last_30_days: Record<string, number>;
+  suggestion_clicks_last_90_days: Record<string, number>;
+}
+
+export type LensUsage = LensVisualizationUsage & LensClickUsage;

x-pack/legacy/plugins/lens/server/usage/visualization_counts.ts

@@ -7,7 +7,7 @@
 import { KibanaConfig } from 'src/legacy/server/kbn_server';
 import { CallClusterOptions } from 'src/legacy/core_plugins/elasticsearch';
 import { SearchParams, SearchResponse } from 'elasticsearch';
-import { LensUsage } from './types';
+import { LensVisualizationUsage } from './types';
 
 type ClusterSearchType = (
   endpoint: 'search',
@@ -20,7 +20,7 @@ type ClusterSearchType = (
 export async function getVisualizationCounts(
   callCluster: ClusterSearchType,
   config: KibanaConfig
-): Promise<LensUsage> {
+): Promise<LensVisualizationUsage> {
   const scriptedMetric = {
     scripted_metric: {
       // Each cluster collects its own type data in a key-value Map that looks like: