From 8c818e4df24977ab7434753850f83199e93af8b8 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 11 Oct 2019 16:34:24 -0400 Subject: [PATCH] Switch collector to use task data --- .../plugins/lens/server/usage/collectors.ts | 62 +++++++++++++++++-- .../legacy/plugins/lens/server/usage/task.ts | 16 ++++- .../legacy/plugins/lens/server/usage/types.ts | 11 +++- .../lens/server/usage/visualization_counts.ts | 4 +- 4 files changed, 83 insertions(+), 10 deletions(-) diff --git a/x-pack/legacy/plugins/lens/server/usage/collectors.ts b/x-pack/legacy/plugins/lens/server/usage/collectors.ts index 75f422088ed81..68c1cffce6934 100644 --- a/x-pack/legacy/plugins/lens/server/usage/collectors.ts +++ b/x-pack/legacy/plugins/lens/server/usage/collectors.ts @@ -4,10 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { KibanaConfig } from 'src/legacy/server/kbn_server'; +import { get } from 'lodash'; +import { Server, KibanaConfig } from 'src/legacy/server/kbn_server'; import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; import { CoreSetup, SavedObjectsLegacyService } from 'src/core/server'; -import { getVisualizationCounts } from './visualization_counts'; import { LensUsage } from './types'; export function registerLensUsageCollector( @@ -21,13 +21,32 @@ export function registerLensUsageCollector( }; }; config: KibanaConfig; + server: Server; } ) { + let isCollectorReady = false; + async function determineIfTaskManagerIsReady() { + let isReady = false; + try { + isReady = await isTaskManagerReady(plugins.server); + } catch (err) {} // eslint-disable-line + + if (isReady) { + isCollectorReady = true; + } else { + setTimeout(determineIfTaskManagerIsReady, 500); + } + } + determineIfTaskManagerIsReady(); + const lensUsageCollector = plugins.usage.collectorSet.makeUsageCollector({ type: 'lens', fetch: async (callCluster: CallCluster): Promise => { try { - return getVisualizationCounts(callCluster, plugins.config); + const docs = await fetch(plugins.server); + // get the accumulated state from the recurring task + return get(docs, '[0].state.stats'); + // return getVisualizationCounts(callCluster, plugins.config); } catch (err) { return { saved_total: 0, @@ -36,10 +55,45 @@ export function registerLensUsageCollector( visualization_types_overall: {}, visualization_types_last_30_days: {}, visualization_types_last_90_days: {}, + + clicks_last_30_days: {}, + clicks_last_90_days: {}, + suggestion_clicks_last_30_days: {}, + suggestion_clicks_last_90_days: {}, }; } }, - isReady: () => true, + isReady: () => isCollectorReady, }); plugins.usage.collectorSet.register(lensUsageCollector); } + +async function isTaskManagerReady(server: Server) { + const result = await fetch(server); + return result !== null; +} + +async function fetch(server: Server) { + const taskManager = server.plugins.task_manager!; + + let docs; + try { + ({ docs } = await taskManager.fetch({ + query: { bool: { filter: { term: { _id: `task:Lens-lens_telemetry` } } } }, + })); + } catch (err) { + const errMessage = err && err.message ? err.message : err.toString(); + /* + The usage service WILL to try to fetch from this collector before the task manager has been initialized, because the + task manager has to wait for all plugins to initialize first. It's fine to ignore it as next time around it will be + initialized (or it will throw a different type of error) + */ + if (errMessage.includes('NotInitialized')) { + docs = null; + } else { + throw err; + } + } + + return docs; +} diff --git a/x-pack/legacy/plugins/lens/server/usage/task.ts b/x-pack/legacy/plugins/lens/server/usage/task.ts index 189f1f30558a6..df6e49d357990 100644 --- a/x-pack/legacy/plugins/lens/server/usage/task.ts +++ b/x-pack/legacy/plugins/lens/server/usage/task.ts @@ -9,6 +9,7 @@ import { CoreSetup } from 'src/core/server'; import { CallClusterOptions } from 'src/legacy/core_plugins/elasticsearch'; import { SearchParams, SearchResponse, DeleteDocumentByQueryResponse } from 'elasticsearch'; import { RunContext } from '../../../task_manager'; +import { getVisualizationCounts } from './visualization_counts'; // This task is responsible for running daily and aggregating all the Lens click event objects // into daily rolled-up documents, which will be used in reporting click stats @@ -105,6 +106,8 @@ async function doWork(server: Server, callCluster: ClusterSearchType & ClusterDe size: 0, }); + console.log(JSON.stringify(metrics)); + const byDateByType: Record> = {}; metrics.aggregations.daily.buckets.forEach(bucket => { @@ -140,21 +143,28 @@ function telemetryTaskRunner(server: Server) { const callCluster = server.plugins.elasticsearch.getCluster('admin').callWithInternalUser; let lensTelemetryTask: Promise; + let lensVisualizationTask: ReturnType; return { async run() { try { lensTelemetryTask = doWork(server, callCluster); + + lensVisualizationTask = getVisualizationCounts(callCluster, server.config()); } catch (err) { server.log(['warning'], `Error loading lens telemetry: ${err}`); } - return lensTelemetryTask - .then((lensTelemetry = {}) => { + return Promise.all([lensTelemetryTask, lensVisualizationTask]) + .then(([lensTelemetry, lensVisualizations]) => { return { state: { runs: state.runs || 1, - stats: lensTelemetry || prevState.stats || {}, + stats: Object.assign( + {}, + lensTelemetry || prevState.stats || {}, + lensVisualizations + ), }, runAt: getNextMidnight(), }; diff --git a/x-pack/legacy/plugins/lens/server/usage/types.ts b/x-pack/legacy/plugins/lens/server/usage/types.ts index 909566b09ac8f..4da3e0cb53812 100644 --- a/x-pack/legacy/plugins/lens/server/usage/types.ts +++ b/x-pack/legacy/plugins/lens/server/usage/types.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -export interface LensUsage { +export interface LensVisualizationUsage { visualization_types_overall: Record; visualization_types_last_30_days: Record; visualization_types_last_90_days: Record; @@ -12,3 +12,12 @@ export interface LensUsage { saved_last_30_days: number; saved_last_90_days: number; } + +export interface LensClickUsage { + clicks_last_30_days: Record; + clicks_last_90_days: Record; + suggestion_clicks_last_30_days: Record; + suggestion_clicks_last_90_days: Record; +} + +export type LensUsage = LensVisualizationUsage & LensClickUsage; \ No newline at end of file diff --git a/x-pack/legacy/plugins/lens/server/usage/visualization_counts.ts b/x-pack/legacy/plugins/lens/server/usage/visualization_counts.ts index 0558963374514..4402431897d4e 100644 --- a/x-pack/legacy/plugins/lens/server/usage/visualization_counts.ts +++ b/x-pack/legacy/plugins/lens/server/usage/visualization_counts.ts @@ -7,7 +7,7 @@ import { KibanaConfig } from 'src/legacy/server/kbn_server'; import { CallClusterOptions } from 'src/legacy/core_plugins/elasticsearch'; import { SearchParams, SearchResponse } from 'elasticsearch'; -import { LensUsage } from './types'; +import { LensVisualizationUsage } from './types'; type ClusterSearchType = ( endpoint: 'search', @@ -20,7 +20,7 @@ type ClusterSearchType = ( export async function getVisualizationCounts( callCluster: ClusterSearchType, config: KibanaConfig -): Promise { +): Promise { const scriptedMetric = { scripted_metric: { // Each cluster collects its own type data in a key-value Map that looks like: