From 6daf5a04674318512d53452b26e102e88059ea2e Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 23 Jul 2018 09:42:48 -0700 Subject: [PATCH 1/3] canvas stats in telemetry bulk uploader --- .../monitoring/server/kibana_monitoring/bulk_uploader.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js index c95baa056c8d6a..f958315bceb433 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js @@ -149,12 +149,14 @@ export class BulkUploader { let statsResult; const [ statsHeader, statsPayload ] = findItem(KIBANA_STATS_TYPE_MONITORING); const [ reportingHeader, reportingPayload ] = findItem(KIBANA_REPORTING_TYPE); + const [ canvasHeader, canvasPayload ] = findItem('kibana_canvas'); if (statsHeader && statsPayload) { statsHeader.index._type = 'kibana_stats'; // HACK to convert kibana_stats_monitoring to just kibana_stats for bwc const [ usageHeader, usagePayload ] = findItem(KIBANA_USAGE_TYPE); const kibanaUsage = (usageHeader && usagePayload) ? usagePayload : null; const reportingUsage = (reportingHeader && reportingPayload) ? reportingPayload : null; + const canvasUsage = (canvasHeader && canvasPayload) ? canvasPayload : null; statsResult = [ statsHeader, statsPayload ]; if (kibanaUsage) { set(statsResult, '[1].usage', kibanaUsage); @@ -162,6 +164,9 @@ export class BulkUploader { if (reportingUsage) { set(statsResult, '[1].usage.xpack.reporting', reportingUsage); } + if (canvasUsage) { + set(statsResult, '[1].usage.xpack.canvas', canvasUsage); + } } // kibana settings From facf8637a866a5b1c24cefc0b1da060d99db6103 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 23 Jul 2018 14:25:16 -0700 Subject: [PATCH 2/3] integrate canvas stats in telemetry payload --- x-pack/plugins/xpack_main/common/constants.js | 6 ++ .../lib/telemetry/monitoring/get_all_stats.js | 20 +++--- .../telemetry/monitoring/get_canvas_stats.js | 62 +++++++++++++++++++ 3 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_canvas_stats.js diff --git a/x-pack/plugins/xpack_main/common/constants.js b/x-pack/plugins/xpack_main/common/constants.js index d939038a3986b1..71b0023caee445 100644 --- a/x-pack/plugins/xpack_main/common/constants.js +++ b/x-pack/plugins/xpack_main/common/constants.js @@ -41,6 +41,12 @@ export const LOGSTASH_SYSTEM_ID = 'logstash'; */ export const REPORTING_SYSTEM_ID = 'reporting'; +/** + * The name of the Kibana System ID used to look up Reporting stats through the Monitoring system. + * @type {string} + */ +export const CANVAS_SYSTEM_ID = 'canvas'; + /** * The amount of time, in milliseconds, to wait between reports when enabled. * diff --git a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_all_stats.js b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_all_stats.js index 71b9741530029a..1eb730213bc114 100644 --- a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_all_stats.js +++ b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_all_stats.js @@ -8,6 +8,7 @@ import { get, set, merge } from 'lodash'; import { KIBANA_SYSTEM_ID, LOGSTASH_SYSTEM_ID, + CANVAS_SYSTEM_ID, REPORTING_SYSTEM_ID, BEATS_SYSTEM_ID, } from '../../../../common/constants'; @@ -15,6 +16,7 @@ import { getClusterUuids } from './get_cluster_uuids'; import { getElasticsearchStats } from './get_es_stats'; import { getKibanaStats } from './get_kibana_stats'; import { getReportingStats } from './get_reporting_stats'; +import { getCanvasStats } from './get_canvas_stats'; import { getBeatsStats } from './get_beats_stats'; import { getHighLevelStats } from './get_high_level_stats'; @@ -66,13 +68,16 @@ function getAllStatsWithCaller(server, callCluster, start, end) { } return Promise.all([ - getElasticsearchStats(server, callCluster, clusterUuids), // cluster_stats, stack_stats.xpack, cluster_name/uuid, license, version - getKibanaStats(server, callCluster, clusterUuids, start, end), // stack_stats.kibana + getElasticsearchStats(server, callCluster, clusterUuids), // cluster_stats, stack_stats.xpack, cluster_name/uuid, license, version + getKibanaStats(server, callCluster, clusterUuids, start, end), // stack_stats.kibana getHighLevelStats(server, callCluster, clusterUuids, start, end, LOGSTASH_SYSTEM_ID), // stack_stats.logstash - getReportingStats(server, callCluster, clusterUuids, start, end), // stack_stats.xpack.reporting - getBeatsStats(server, callCluster, clusterUuids, start, end), // stack_stats.beats - ]) - .then(([esClusters, kibana, logstash, reporting, beats]) => handleAllStats(esClusters, { kibana, logstash, reporting, beats })); + getReportingStats(server, callCluster, clusterUuids, start, end), // stack_stats.xpack.reporting + getCanvasStats(server, callCluster, clusterUuids, start, end), // stack_stats.xpack.kibana_canvas + getBeatsStats(server, callCluster, clusterUuids, start, end), // stack_stats.beats + ]).then(([esClusters, kibana, logstash, reporting, canvas, beats]) => { + return handleAllStats(esClusters, { kibana, logstash, reporting, canvas, beats }); + } + ); }); } @@ -85,12 +90,13 @@ function getAllStatsWithCaller(server, callCluster, start, end) { * @param {Object} logstash The Logstash nodes keyed by Cluster UUID * @return {Array} The clusters joined with the Kibana and Logstash instances under each cluster's {@code stack_stats}. */ -export function handleAllStats(clusters, { kibana, logstash, reporting, beats }) { +export function handleAllStats(clusters, { kibana, logstash, reporting, canvas, beats }) { return clusters.map(cluster => { // if they are using Kibana or Logstash, then add it to the cluster details under cluster.stack_stats addStackStats(cluster, kibana, KIBANA_SYSTEM_ID); addStackStats(cluster, logstash, LOGSTASH_SYSTEM_ID); addStackStats(cluster, beats, BEATS_SYSTEM_ID); + addXPackStats(cluster, canvas, CANVAS_SYSTEM_ID); addXPackStats(cluster, reporting, REPORTING_SYSTEM_ID); mergeXPackStats(cluster, kibana, 'graph_workspace', 'graph'); // copy graph_workspace info out of kibana, merge it into stack_stats.xpack.graph diff --git a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_canvas_stats.js b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_canvas_stats.js new file mode 100644 index 00000000000000..ae8463b146756f --- /dev/null +++ b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_canvas_stats.js @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get } from 'lodash'; +import { createQuery } from './create_query'; +import { KIBANA_SYSTEM_ID, CANVAS_SYSTEM_ID } from '../../../../common/constants'; + +const canvasStatsPath = `${KIBANA_SYSTEM_ID}_stats.usage.xpack.${CANVAS_SYSTEM_ID}`; + +export function fetchHighLevelCanvasStats(server, callCluster, clusterUuids, start, end) { + const config = server.config(); + const params = { + index: config.get(`xpack.monitoring.${KIBANA_SYSTEM_ID}.index_pattern`), + size: config.get('xpack.monitoring.max_bucket_size'), + ignoreUnavailable: true, + body: { + query: createQuery({ + start, + end, + type: `${KIBANA_SYSTEM_ID}_stats`, // canvas stats are in kibana_stats.xpack.canvas + filters: [ { terms: { cluster_uuid: clusterUuids } } ] + }), + collapse: { + field: 'cluster_uuid' + }, + sort: [ + { 'timestamp': 'desc' } + ], + _source: { + includes: [ + 'cluster_uuid', + canvasStatsPath, + ] + }, + } + }; + + return callCluster('search', params); +} + +export function handleHighLevelCanvasStatsResponse(response) { + const hits = get(response, 'hits.hits', []); + return hits.reduce((accum, curr) => { + const clusterUuid = get(curr, '_source.cluster_uuid'); + const stats = get(curr, `_source.${canvasStatsPath}`); + return { + ...accum, + [clusterUuid]: stats + }; + }, {}); +} + +export async function getCanvasStats(server, callCluster, clusterUuids, start, end) { + const rawStats = await fetchHighLevelCanvasStats(server, callCluster, clusterUuids, start, end, CANVAS_SYSTEM_ID); + const stats = handleHighLevelCanvasStatsResponse(rawStats); + + return stats; +} + From 8298e2fdea787b72afafae2b5375c7ba08a776b5 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 24 Jul 2018 10:40:36 -0700 Subject: [PATCH 3/3] kibana_canvas => canvas --- .../monitoring/server/kibana_monitoring/bulk_uploader.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js index f958315bceb433..d60be00ce684dc 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js @@ -149,7 +149,7 @@ export class BulkUploader { let statsResult; const [ statsHeader, statsPayload ] = findItem(KIBANA_STATS_TYPE_MONITORING); const [ reportingHeader, reportingPayload ] = findItem(KIBANA_REPORTING_TYPE); - const [ canvasHeader, canvasPayload ] = findItem('kibana_canvas'); + const [ canvasHeader, canvasPayload ] = findItem('canvas'); // TODO replace with constant if (statsHeader && statsPayload) { statsHeader.index._type = 'kibana_stats'; // HACK to convert kibana_stats_monitoring to just kibana_stats for bwc