diff --git a/x-pack/legacy/plugins/lens/common/clickdata.ts b/x-pack/legacy/plugins/lens/common/clickdata.ts new file mode 100644 index 0000000000000..8e64f83ce57d8 --- /dev/null +++ b/x-pack/legacy/plugins/lens/common/clickdata.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface LensClickEvent { + name: string; + date: string; +} diff --git a/x-pack/legacy/plugins/lens/common/index.ts b/x-pack/legacy/plugins/lens/common/index.ts index eead93dd33480..f5291a5941dd5 100644 --- a/x-pack/legacy/plugins/lens/common/index.ts +++ b/x-pack/legacy/plugins/lens/common/index.ts @@ -6,3 +6,4 @@ export * from './api'; export * from './constants'; +export * from './clickdata'; diff --git a/x-pack/legacy/plugins/lens/index.ts b/x-pack/legacy/plugins/lens/index.ts index f7cadbc6433e7..78b542a26c619 100644 --- a/x-pack/legacy/plugins/lens/index.ts +++ b/x-pack/legacy/plugins/lens/index.ts @@ -88,6 +88,7 @@ export const lens: LegacyPluginInitializer = kibana => { savedObjects: server.savedObjects, usage: server.usage, config: server.config(), + server, }); server.events.on('stop', () => { diff --git a/x-pack/legacy/plugins/lens/mappings.json b/x-pack/legacy/plugins/lens/mappings.json index 832d152eb77a1..e33fb5456360d 100644 --- a/x-pack/legacy/plugins/lens/mappings.json +++ b/x-pack/legacy/plugins/lens/mappings.json @@ -15,5 +15,18 @@ "type": "keyword" } } + }, + "lens-ui-telemetry": { + "properties": { + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "date": { + "type": "date" + } + } } } diff --git a/x-pack/legacy/plugins/lens/public/app_plugin/app.tsx b/x-pack/legacy/plugins/lens/public/app_plugin/app.tsx index 4746b1cac3ecd..7d29ae0284bae 100644 --- a/x-pack/legacy/plugins/lens/public/app_plugin/app.tsx +++ b/x-pack/legacy/plugins/lens/public/app_plugin/app.tsx @@ -25,6 +25,7 @@ import { KibanaContextProvider } from '../../../../../../src/plugins/kibana_reac import { Document, SavedObjectStore } from '../persistence'; import { EditorFrameInstance } from '../types'; import { NativeRenderer } from '../native_renderer'; +import { useLensTelemetry } from '../lens_ui_telemetry'; interface State { isLoading: boolean; @@ -64,6 +65,7 @@ export function App({ const timeDefaults = core.uiSettings.get('timepicker:timeDefaults'); const language = store.get('kibana.userQueryLanguage') || core.uiSettings.get('search:queryLanguage'); + const { trackClick } = useLensTelemetry(); const [state, setState] = useState({ isLoading: !!docId, @@ -84,6 +86,8 @@ export function App({ const subscription = dataShim.filter.filterManager.getUpdates$().subscribe({ next: () => { setState(s => ({ ...s, filters: dataShim.filter.filterManager.getFilters() })); + + trackClick('app_filters_updated'); }, }); return () => { @@ -198,6 +202,7 @@ export function App({ } }) .catch(() => { + trackClick('save_failed'); core.notifications.toasts.addDanger( i18n.translate('xpack.lens.editorFrame.docSavingError', { defaultMessage: 'Error saving document', @@ -222,6 +227,8 @@ export function App({ }, query: query || s.query, })); + + trackClick('date_or_query_change'); }} appName={'lens'} indexPatterns={state.indexPatternsForTopNav} diff --git a/x-pack/legacy/plugins/lens/public/app_plugin/plugin.tsx b/x-pack/legacy/plugins/lens/public/app_plugin/plugin.tsx index 2668946fec47b..686a842194407 100644 --- a/x-pack/legacy/plugins/lens/public/app_plugin/plugin.tsx +++ b/x-pack/legacy/plugins/lens/public/app_plugin/plugin.tsx @@ -25,6 +25,7 @@ import { } from '../datatable_visualization_plugin'; import { App } from './app'; import { EditorFrameInstance } from '../types'; +import { LensReportManager, LensTelemetryContext } from '../lens_ui_telemetry'; export interface LensPluginStartDependencies { data: DataPublicPluginStart; @@ -33,6 +34,7 @@ export interface LensPluginStartDependencies { export class AppPlugin { private instance: EditorFrameInstance | null = null; private store: SavedObjectIndexStore | null = null; + private reporter: LensReportManager | null = null; constructor() {} @@ -63,24 +65,40 @@ export class AppPlugin { this.instance = editorFrameStartInterface.createInstance({}); + this.reporter = new LensReportManager({ + storage: new Storage(localStorage), + basePath: core.http.basePath.get(), + http: core.http, + }); + const renderEditor = (routeProps: RouteComponentProps<{ id?: string }>) => { + if (this.reporter) { + this.reporter.trackClick('loaded'); + } return ( - { - if (!id) { - routeProps.history.push('/'); - } else { - routeProps.history.push(`/edit/${id}`); - } + this.reporter && this.reporter.trackClick(name), + trackSuggestionClick: name => this.reporter && this.reporter.trackSuggestionClick(name), }} - /> + > + { + if (!id) { + routeProps.history.push('/'); + } else { + routeProps.history.push(`/edit/${id}`); + } + }} + /> + ); }; @@ -106,6 +124,10 @@ export class AppPlugin { this.instance.unmount(); } + if (this.reporter) { + this.reporter.stop(); + } + // TODO this will be handled by the plugin platform itself indexPatternDatasourceStop(); xyVisualizationStop(); diff --git a/x-pack/legacy/plugins/lens/public/editor_frame_plugin/plugin.tsx b/x-pack/legacy/plugins/lens/public/editor_frame_plugin/plugin.tsx index cb81ec3d69985..683a039128325 100644 --- a/x-pack/legacy/plugins/lens/public/editor_frame_plugin/plugin.tsx +++ b/x-pack/legacy/plugins/lens/public/editor_frame_plugin/plugin.tsx @@ -31,6 +31,7 @@ import { EditorFrame } from './editor_frame'; import { mergeTables } from './merge_tables'; import { EmbeddableFactory } from './embeddable/embeddable_factory'; import { getActiveDatasourceIdFromDoc } from './editor_frame/state_management'; +import { LensTelemetryContext, useLensTelemetry } from '../lens_ui_telemetry'; export interface EditorFrameSetupPlugins { data: typeof dataSetup; @@ -83,24 +84,28 @@ export class EditorFramePlugin { render( - + + + , domElement ); diff --git a/x-pack/legacy/plugins/lens/public/lens_ui_telemetry/__mocks__/index.ts b/x-pack/legacy/plugins/lens/public/lens_ui_telemetry/__mocks__/index.ts new file mode 100644 index 0000000000000..7f6a7481b0fef --- /dev/null +++ b/x-pack/legacy/plugins/lens/public/lens_ui_telemetry/__mocks__/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createContext, useContext } from 'react'; + +export const LensTelemetryContext = createContext<{ + trackClick: (name: string) => void; + trackSuggestionClick: (name: string, suggestionData: unknown) => void; +}>({ + trackClick: jest.fn(), + trackSuggestionClick: jest.fn(), +}); + +export const useLensTelemetry = () => useContext(LensTelemetryContext); diff --git a/x-pack/legacy/plugins/lens/public/lens_ui_telemetry/factory.ts b/x-pack/legacy/plugins/lens/public/lens_ui_telemetry/factory.ts new file mode 100644 index 0000000000000..0d0304c78f4c8 --- /dev/null +++ b/x-pack/legacy/plugins/lens/public/lens_ui_telemetry/factory.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { HttpServiceBase } from 'src/core/public'; + +import { Storage } from 'src/legacy/core_plugins/data/public/types'; +import { LensClickEvent, BASE_API_URL } from '../../common'; + +const STORAGE_KEY = 'lens-ui-telemetry'; + +export class LensReportManager { + private clicks: LensClickEvent[]; + private suggestionClicks: LensClickEvent[]; + + private storage: Storage; + private http: HttpServiceBase; + private basePath: string; + private timer: ReturnType; + + constructor({ + storage, + http, + basePath, + }: { + storage: Storage; + http: HttpServiceBase; + basePath: string; + }) { + this.storage = storage; + this.http = http; + this.basePath = basePath; + + const unsent = this.storage.get(STORAGE_KEY); + this.clicks = unsent && unsent.clicks ? unsent.clicks : []; + this.suggestionClicks = unsent && unsent.suggestionClicks ? unsent.suggestionClicks : []; + + this.timer = setInterval(() => { + if (this.clicks.length || this.suggestionClicks.length) { + this.postToServer(); + } + }, 10000); + } + + public trackClick(name: string) { + this.clicks.push({ + name, + date: new Date().toISOString(), + }); + this.write(); + } + + public trackSuggestionClick(name: string) { + this.suggestionClicks.push({ + name, + date: new Date().toISOString(), + }); + this.write(); + } + + private write() { + this.storage.set(STORAGE_KEY, { clicks: this.clicks, suggestionClicks: this.suggestionClicks }); + } + + public stop() { + if (this.timer) { + clearInterval(this.timer); + } + } + + private async postToServer() { + try { + await this.http.post(`${this.basePath}${BASE_API_URL}/telemetry`, { + body: JSON.stringify({ + clicks: this.clicks, + suggestionClicks: this.clicks, + }), + }); + this.clicks = []; + this.suggestionClicks = []; + this.write(); + } catch (e) { + // Maybe show an error + console.log(e); + } + } +} diff --git a/x-pack/legacy/plugins/lens/public/lens_ui_telemetry/index.ts b/x-pack/legacy/plugins/lens/public/lens_ui_telemetry/index.ts new file mode 100644 index 0000000000000..d7a8719137104 --- /dev/null +++ b/x-pack/legacy/plugins/lens/public/lens_ui_telemetry/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './factory'; +export * from './provider'; diff --git a/x-pack/legacy/plugins/lens/public/lens_ui_telemetry/provider.ts b/x-pack/legacy/plugins/lens/public/lens_ui_telemetry/provider.ts new file mode 100644 index 0000000000000..b7852cfff2062 --- /dev/null +++ b/x-pack/legacy/plugins/lens/public/lens_ui_telemetry/provider.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createContext, useContext } from 'react'; + +export const LensTelemetryContext = createContext<{ + trackClick: (name: string) => void; + trackSuggestionClick: (name: string, suggestionData: unknown) => void; +}>({ + trackClick: () => {}, + trackSuggestionClick: () => {}, +}); + +export const useLensTelemetry = () => useContext(LensTelemetryContext); diff --git a/x-pack/legacy/plugins/lens/server/plugin.tsx b/x-pack/legacy/plugins/lens/server/plugin.tsx index fdcfe122f60ce..96b0c3db08c7a 100644 --- a/x-pack/legacy/plugins/lens/server/plugin.tsx +++ b/x-pack/legacy/plugins/lens/server/plugin.tsx @@ -4,10 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { KibanaConfig } from 'src/legacy/server/kbn_server'; +import { Server, KibanaConfig } from 'src/legacy/server/kbn_server'; import { Plugin, CoreSetup, SavedObjectsLegacyService } from 'src/core/server'; import { setupRoutes } from './routes'; -import { registerLensUsageCollector } from './usage'; +import { registerLensUsageCollector, initializeLensTelemetry } from './usage'; export class LensServer implements Plugin<{}, {}, {}, {}> { constructor() {} @@ -23,10 +23,12 @@ export class LensServer implements Plugin<{}, {}, {}, {}> { }; }; config: KibanaConfig; + server: Server; } ) { - setupRoutes(core); + setupRoutes(core, plugins); registerLensUsageCollector(core, plugins); + initializeLensTelemetry(core, plugins); return {}; } diff --git a/x-pack/legacy/plugins/lens/server/routes/index.ts b/x-pack/legacy/plugins/lens/server/routes/index.ts index c5f882c8e5714..6d701ad1c93e3 100644 --- a/x-pack/legacy/plugins/lens/server/routes/index.ts +++ b/x-pack/legacy/plugins/lens/server/routes/index.ts @@ -4,11 +4,20 @@ * you may not use this file except in compliance with the Elastic License. */ -import { CoreSetup } from 'src/core/server'; +import { KibanaConfig } from 'src/legacy/server/kbn_server'; +import { CoreSetup, SavedObjectsLegacyService } from 'src/core/server'; import { initStatsRoute } from './index_stats'; import { initFieldsRoute } from './field_stats'; +import { initLensUsageRoute } from './telemetry'; -export function setupRoutes(setup: CoreSetup) { +export function setupRoutes( + setup: CoreSetup, + plugins: { + savedObjects: SavedObjectsLegacyService; + config: KibanaConfig; + } +) { initStatsRoute(setup); initFieldsRoute(setup); + initLensUsageRoute(setup, plugins); } diff --git a/x-pack/legacy/plugins/lens/server/routes/telemetry.ts b/x-pack/legacy/plugins/lens/server/routes/telemetry.ts new file mode 100644 index 0000000000000..b0a92c7d12b5a --- /dev/null +++ b/x-pack/legacy/plugins/lens/server/routes/telemetry.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Boom from 'boom'; +import { KibanaConfig, Server } from 'src/legacy/server/kbn_server'; +import { CoreSetup, SavedObjectsLegacyService } from 'src/core/server'; +import { schema } from '@kbn/config-schema'; +import { BASE_API_URL } from '../../common'; + +export function getSavedObjectsClient( + savedObjects: SavedObjectsLegacyService, + callAsInternalUser: unknown +) { + const { SavedObjectsClient, getSavedObjectsRepository } = savedObjects; + const internalRepository = getSavedObjectsRepository(callAsInternalUser); + return new SavedObjectsClient(internalRepository); +} + +// This route is responsible for taking a batch of click events from the browser +// and writing them to saved objects +export async function initLensUsageRoute( + setup: CoreSetup, + plugins: { + savedObjects: SavedObjectsLegacyService; + config: KibanaConfig; + // server: Server; + } +) { + const router = setup.http.createRouter(); + router.post( + { + path: `${BASE_API_URL}/telemetry`, + validate: { + params: schema.object({}), + body: schema.object({ + clicks: schema.arrayOf( + schema.object({ + name: schema.string(), + date: schema.string(), + }) + ), + suggestionClicks: schema.arrayOf( + schema.object({ + name: schema.string(), + date: schema.string(), + }) + ), + }), + }, + }, + async (context, req, res) => { + const { dataClient } = context.core.elasticsearch; + + const { clicks, suggestionClicks } = req.body; + + try { + const client = getSavedObjectsClient(plugins.savedObjects, dataClient.callAsCurrentUser); + + const clickEvents = clicks.map(event => ({ + type: 'lens-ui-telemetry', + attributes: { + name: event.name, + type: 'click', + date: event.date, + }, + })); + const suggestionEvents = suggestionClicks.map(event => ({ + type: 'lens-ui-telemetry', + attributes: { + name: event.name, + type: 'suggestion', + date: event.date, + }, + })); + + await client.bulkCreate(clickEvents.concat(suggestionEvents)); + + return res.ok({ body: {} }); + } catch (e) { + if (e.status === 404) { + return res.notFound(); + } + if (e.isBoom) { + if (e.output.statusCode === 404) { + return res.notFound(); + } + return res.internalError(e.output.message); + } else { + return res.internalError({ + body: Boom.internal(e.message || e.name), + }); + } + } + } + ); +} diff --git a/x-pack/legacy/plugins/lens/server/usage/index.ts b/x-pack/legacy/plugins/lens/server/usage/index.ts index 4dd74057e0877..e7c09fdda3c05 100644 --- a/x-pack/legacy/plugins/lens/server/usage/index.ts +++ b/x-pack/legacy/plugins/lens/server/usage/index.ts @@ -5,3 +5,4 @@ */ export * from './collectors'; +export * from './task'; diff --git a/x-pack/legacy/plugins/lens/server/usage/task.ts b/x-pack/legacy/plugins/lens/server/usage/task.ts new file mode 100644 index 0000000000000..189f1f30558a6 --- /dev/null +++ b/x-pack/legacy/plugins/lens/server/usage/task.ts @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Server, SavedObjectsClient as SavedObjectsClientType } from 'src/legacy/server/kbn_server'; +import { CoreSetup } from 'src/core/server'; +import { CallClusterOptions } from 'src/legacy/core_plugins/elasticsearch'; +import { SearchParams, SearchResponse, DeleteDocumentByQueryResponse } from 'elasticsearch'; +import { RunContext } from '../../../task_manager'; + +// This task is responsible for running daily and aggregating all the Lens click event objects +// into daily rolled-up documents, which will be used in reporting click stats + +const TELEMETRY_TASK_TYPE = 'lens_telemetry'; + +export const TASK_ID = `Lens-${TELEMETRY_TASK_TYPE}`; + +type ClusterSearchType = ( + endpoint: 'search', + params: SearchParams & { + rest_total_hits_as_int: boolean; + }, + options?: CallClusterOptions +) => Promise>; +type ClusterDeleteType = ( + endpoint: 'deleteByQuery', + params: SearchParams, + options?: CallClusterOptions +) => Promise; + +export function initializeLensTelemetry(core: CoreSetup, { server }: { server: Server }) { + registerLensTelemetryTask(core, { server }); + scheduleTasks(server); +} + +function registerLensTelemetryTask(core: CoreSetup, { server }: { server: Server }) { + const taskManager = server.plugins.task_manager!; + taskManager.registerTaskDefinitions({ + [TELEMETRY_TASK_TYPE]: { + title: 'Lens telemetry fetch task', + type: TELEMETRY_TASK_TYPE, + timeout: '1m', + createTaskRunner: telemetryTaskRunner(server), + }, + }); +} + +function scheduleTasks(server: Server) { + const taskManager = server.plugins.task_manager; + const { kbnServer } = server.plugins.xpack_main.status.plugin; + + kbnServer.afterPluginsInit(() => { + // The code block below can't await directly within "afterPluginsInit" + // callback due to circular dependency The server isn't "ready" until + // this code block finishes. Migrations wait for server to be ready before + // executing. Saved objects repository waits for migrations to finish before + // finishing the request. To avoid this, we'll await within a separate + // function block. + (async () => { + try { + await taskManager.schedule({ + id: TASK_ID, + taskType: TELEMETRY_TASK_TYPE, + state: { stats: {}, runs: 0 }, + params: {}, + }); + } catch (e) { + server.log(['warning', 'telemetry'], `Error scheduling task, received ${e.message}`); + } + })(); + }); +} + +async function doWork(server: Server, callCluster: ClusterSearchType & ClusterDeleteType) { + const kibanaIndex = server.config().get('kibana.index'); + + const metrics = await callCluster('search', { + index: kibanaIndex, + rest_total_hits_as_int: true, + body: { + query: { + bool: { + filter: [ + { term: { type: 'lens-ui-telemetry' } }, + { range: { updated_at: { gte: 'now-90d/d' } } }, + ], + }, + }, + aggs: { + daily: { + date_histogram: { + field: 'updated_at', + calendar_interval: '1d', + }, + aggs: { + names: { + terms: { field: 'lens-ui-telemetry.name', size: 100 }, + }, + }, + }, + }, + }, + size: 0, + }); + + const byDateByType: Record> = {}; + + metrics.aggregations.daily.buckets.forEach(bucket => { + const byType: Record = {}; + bucket.names.buckets.forEach(({ key, doc_count }) => { + byType[key] = doc_count; + }); + byDateByType[bucket.key] = byType; + }); + + if (metrics.hits.total > 0) { + // After aggregating the lens telemetry, we delete the originals which are not needed + await callCluster('deleteByQuery', { + index: kibanaIndex, + body: { + query: { + bool: { + filter: [{ term: { type: 'lens-ui-telemetry' } }], + }, + }, + }, + }); + } + + return byDateByType; +} + +function telemetryTaskRunner(server: Server) { + return ({ taskInstance }: RunContext) => { + const { state } = taskInstance; + const prevState = state; + + const callCluster = server.plugins.elasticsearch.getCluster('admin').callWithInternalUser; + + let lensTelemetryTask: Promise; + + return { + async run() { + try { + lensTelemetryTask = doWork(server, callCluster); + } catch (err) { + server.log(['warning'], `Error loading lens telemetry: ${err}`); + } + + return lensTelemetryTask + .then((lensTelemetry = {}) => { + return { + state: { + runs: state.runs || 1, + stats: lensTelemetry || prevState.stats || {}, + }, + runAt: getNextMidnight(), + }; + }) + .catch(errMsg => + server.log(['warning'], `Error executing lens telemetry task: ${errMsg}`) + ); + }, + }; + }; +} + +function getNextMidnight() { + const nextMidnight = new Date(); + // nextMidnight.setHours(0, 0, 0, 0); + nextMidnight.setHours(nextMidnight.getHours(), nextMidnight.getMinutes() + 1, 0, 0); + // nextMidnight.setDate(nextMidnight.getDate() + 1); + // return nextMidnight; + return nextMidnight; +}