diff --git a/src/plugins/telemetry/common/constants.ts b/src/plugins/telemetry/common/constants.ts index 4493d0e3ba31c..d6111d4124a07 100644 --- a/src/plugins/telemetry/common/constants.ts +++ b/src/plugins/telemetry/common/constants.ts @@ -27,21 +27,32 @@ export const PATH_TO_ADVANCED_SETTINGS = '/app/management/kibana/settings'; */ export const PRIVACY_STATEMENT_URL = `https://www.elastic.co/legal/privacy-statement`; +/** + * The telemetry payload content encryption encoding + */ +export const PAYLOAD_CONTENT_ENCODING = 'aes256gcm'; + /** * The endpoint version when hitting the remote telemetry service */ -export const ENDPOINT_VERSION = 'v2'; +export const ENDPOINT_VERSION = 'v3'; + +/** + * The staging telemetry endpoint for the remote telemetry service. + */ + +export const ENDPOINT_STAGING = 'https://telemetry-staging.elastic.co/'; + +/** + * The production telemetry endpoint for the remote telemetry service. + */ + +export const ENDPOINT_PROD = 'https://telemetry.elastic.co/'; /** - * The telemetry endpoints for the remote telemetry service. + * The telemetry channels for the remote telemetry service. */ -export const TELEMETRY_ENDPOINT = { - MAIN_CHANNEL: { - PROD: `https://telemetry.elastic.co/xpack/${ENDPOINT_VERSION}/send`, - STAGING: `https://telemetry-staging.elastic.co/xpack/${ENDPOINT_VERSION}/send`, - }, - OPT_IN_STATUS_CHANNEL: { - PROD: `https://telemetry.elastic.co/opt_in_status/${ENDPOINT_VERSION}/send`, - STAGING: `https://telemetry-staging.elastic.co/opt_in_status/${ENDPOINT_VERSION}/send`, - }, +export const TELEMETRY_CHANNELS = { + SNAPSHOT_CHANNEL: 'kibana-snapshot', + OPT_IN_STATUS_CHANNEL: 'kibana-opt_in_status', }; diff --git a/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.test.ts b/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.test.ts index 74d45f6a9f7d4..c7f9984269581 100644 --- a/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.test.ts +++ b/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.test.ts @@ -6,14 +6,55 @@ * Side Public License, v 1. */ -import { getTelemetryChannelEndpoint } from './get_telemetry_channel_endpoint'; -import { TELEMETRY_ENDPOINT } from '../constants'; +import { + getTelemetryChannelEndpoint, + getChannel, + getBaseUrl, +} from './get_telemetry_channel_endpoint'; + +describe('getBaseUrl', () => { + it('throws on unknown env', () => { + expect(() => + // @ts-expect-error + getBaseUrl('ANY') + ).toThrowErrorMatchingInlineSnapshot(`"Unknown telemetry endpoint env ANY."`); + }); + + it('returns correct prod base url', () => { + const baseUrl = getBaseUrl('prod'); + expect(baseUrl).toMatchInlineSnapshot(`"https://telemetry.elastic.co/"`); + }); + + it('returns correct staging base url', () => { + const baseUrl = getBaseUrl('staging'); + expect(baseUrl).toMatchInlineSnapshot(`"https://telemetry-staging.elastic.co/"`); + }); +}); + +describe('getChannel', () => { + it('throws on unknown channel', () => { + expect(() => + // @ts-expect-error + getChannel('ANY') + ).toThrowErrorMatchingInlineSnapshot(`"Unknown telemetry channel ANY."`); + }); + + it('returns correct snapshot channel name', () => { + const channelName = getChannel('snapshot'); + expect(channelName).toMatchInlineSnapshot(`"kibana-snapshot"`); + }); + + it('returns correct optInStatus channel name', () => { + const channelName = getChannel('optInStatus'); + expect(channelName).toMatchInlineSnapshot(`"kibana-opt_in_status"`); + }); +}); describe('getTelemetryChannelEndpoint', () => { it('throws on unknown env', () => { expect(() => // @ts-expect-error - getTelemetryChannelEndpoint({ env: 'ANY', channelName: 'main' }) + getTelemetryChannelEndpoint({ env: 'ANY', channelName: 'snapshot' }) ).toThrowErrorMatchingInlineSnapshot(`"Unknown telemetry endpoint env ANY."`); }); @@ -24,25 +65,33 @@ describe('getTelemetryChannelEndpoint', () => { ).toThrowErrorMatchingInlineSnapshot(`"Unknown telemetry channel ANY."`); }); - describe('main channel', () => { + describe('snapshot channel', () => { it('returns correct prod endpoint', () => { - const endpoint = getTelemetryChannelEndpoint({ env: 'prod', channelName: 'main' }); - expect(endpoint).toBe(TELEMETRY_ENDPOINT.MAIN_CHANNEL.PROD); + const endpoint = getTelemetryChannelEndpoint({ env: 'prod', channelName: 'snapshot' }); + expect(endpoint).toMatchInlineSnapshot( + `"https://telemetry.elastic.co/v3/send/kibana-snapshot"` + ); }); it('returns correct staging endpoint', () => { - const endpoint = getTelemetryChannelEndpoint({ env: 'staging', channelName: 'main' }); - expect(endpoint).toBe(TELEMETRY_ENDPOINT.MAIN_CHANNEL.STAGING); + const endpoint = getTelemetryChannelEndpoint({ env: 'staging', channelName: 'snapshot' }); + expect(endpoint).toMatchInlineSnapshot( + `"https://telemetry-staging.elastic.co/v3/send/kibana-snapshot"` + ); }); }); describe('optInStatus channel', () => { it('returns correct prod endpoint', () => { const endpoint = getTelemetryChannelEndpoint({ env: 'prod', channelName: 'optInStatus' }); - expect(endpoint).toBe(TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL.PROD); + expect(endpoint).toMatchInlineSnapshot( + `"https://telemetry.elastic.co/v3/send/kibana-opt_in_status"` + ); }); it('returns correct staging endpoint', () => { const endpoint = getTelemetryChannelEndpoint({ env: 'staging', channelName: 'optInStatus' }); - expect(endpoint).toBe(TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL.STAGING); + expect(endpoint).toMatchInlineSnapshot( + `"https://telemetry-staging.elastic.co/v3/send/kibana-opt_in_status"` + ); }); }); }); diff --git a/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.ts b/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.ts index a0af7878afef6..75d83611b8c8d 100644 --- a/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.ts +++ b/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.ts @@ -6,29 +6,48 @@ * Side Public License, v 1. */ -import { TELEMETRY_ENDPOINT } from '../constants'; +import { + ENDPOINT_VERSION, + ENDPOINT_STAGING, + ENDPOINT_PROD, + TELEMETRY_CHANNELS, +} from '../constants'; +export type ChannelName = 'snapshot' | 'optInStatus'; +export type TelemetryEnv = 'staging' | 'prod'; export interface GetTelemetryChannelEndpointConfig { - channelName: 'main' | 'optInStatus'; - env: 'staging' | 'prod'; + channelName: ChannelName; + env: TelemetryEnv; } -export function getTelemetryChannelEndpoint({ - channelName, - env, -}: GetTelemetryChannelEndpointConfig): string { - if (env !== 'staging' && env !== 'prod') { - throw new Error(`Unknown telemetry endpoint env ${env}.`); - } - - const endpointEnv = env === 'staging' ? 'STAGING' : 'PROD'; - +export function getChannel(channelName: ChannelName): string { switch (channelName) { - case 'main': - return TELEMETRY_ENDPOINT.MAIN_CHANNEL[endpointEnv]; + case 'snapshot': + return TELEMETRY_CHANNELS.SNAPSHOT_CHANNEL; case 'optInStatus': - return TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL[endpointEnv]; + return TELEMETRY_CHANNELS.OPT_IN_STATUS_CHANNEL; default: throw new Error(`Unknown telemetry channel ${channelName}.`); } } + +export function getBaseUrl(env: TelemetryEnv): string { + switch (env) { + case 'prod': + return ENDPOINT_PROD; + case 'staging': + return ENDPOINT_STAGING; + default: + throw new Error(`Unknown telemetry endpoint env ${env}.`); + } +} + +export function getTelemetryChannelEndpoint({ + channelName, + env, +}: GetTelemetryChannelEndpointConfig): string { + const baseUrl = getBaseUrl(env); + const channelPath = getChannel(channelName); + + return `${baseUrl}${ENDPOINT_VERSION}/send/${channelPath}`; +} diff --git a/src/plugins/telemetry/common/telemetry_config/index.ts b/src/plugins/telemetry/common/telemetry_config/index.ts index eb268639cad91..b15475280fe85 100644 --- a/src/plugins/telemetry/common/telemetry_config/index.ts +++ b/src/plugins/telemetry/common/telemetry_config/index.ts @@ -12,4 +12,8 @@ export { getTelemetryAllowChangingOptInStatus } from './get_telemetry_allow_chan export { getTelemetryFailureDetails } from './get_telemetry_failure_details'; export type { TelemetryFailureDetails } from './get_telemetry_failure_details'; export { getTelemetryChannelEndpoint } from './get_telemetry_channel_endpoint'; -export type { GetTelemetryChannelEndpointConfig } from './get_telemetry_channel_endpoint'; +export type { + GetTelemetryChannelEndpointConfig, + ChannelName, + TelemetryEnv, +} from './get_telemetry_channel_endpoint'; diff --git a/src/plugins/telemetry/common/types.ts b/src/plugins/telemetry/common/types.ts new file mode 100644 index 0000000000000..aefbbd2358861 --- /dev/null +++ b/src/plugins/telemetry/common/types.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export type EncryptedTelemetryPayload = Array<{ clusterUuid: string; stats: string }>; +export type UnencryptedTelemetryPayload = Array<{ clusterUuid: string; stats: object }>; diff --git a/src/plugins/telemetry/public/services/telemetry_sender.test.ts b/src/plugins/telemetry/public/services/telemetry_sender.test.ts index 50738b11e508d..10da46fe2761d 100644 --- a/src/plugins/telemetry/public/services/telemetry_sender.test.ts +++ b/src/plugins/telemetry/public/services/telemetry_sender.test.ts @@ -171,8 +171,11 @@ describe('TelemetrySender', () => { }); it('sends report if due', async () => { + const mockClusterUuid = 'mk_uuid'; const mockTelemetryUrl = 'telemetry_cluster_url'; - const mockTelemetryPayload = ['hashed_cluster_usage_data1']; + const mockTelemetryPayload = [ + { clusterUuid: mockClusterUuid, stats: 'hashed_cluster_usage_data1' }, + ]; const telemetryService = mockTelemetryService(); const telemetrySender = new TelemetrySender(telemetryService); @@ -184,14 +187,21 @@ describe('TelemetrySender', () => { expect(telemetryService.fetchTelemetry).toBeCalledTimes(1); expect(mockFetch).toBeCalledTimes(1); - expect(mockFetch).toBeCalledWith(mockTelemetryUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Elastic-Stack-Version': telemetryService.currentKibanaVersion, - }, - body: mockTelemetryPayload[0], - }); + expect(mockFetch.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "telemetry_cluster_url", + Object { + "body": "hashed_cluster_usage_data1", + "headers": Object { + "Content-Type": "application/json", + "X-Elastic-Cluster-ID": "mk_uuid", + "X-Elastic-Content-Encoding": "aes256gcm", + "X-Elastic-Stack-Version": "mockKibanaVersion", + }, + "method": "POST", + }, + ] + `); }); it('sends report separately for every cluster', async () => { diff --git a/src/plugins/telemetry/public/services/telemetry_sender.ts b/src/plugins/telemetry/public/services/telemetry_sender.ts index fa97334495122..87287a420e725 100644 --- a/src/plugins/telemetry/public/services/telemetry_sender.ts +++ b/src/plugins/telemetry/public/services/telemetry_sender.ts @@ -6,9 +6,14 @@ * Side Public License, v 1. */ -import { REPORT_INTERVAL_MS, LOCALSTORAGE_KEY } from '../../common/constants'; +import { + REPORT_INTERVAL_MS, + LOCALSTORAGE_KEY, + PAYLOAD_CONTENT_ENCODING, +} from '../../common/constants'; import { TelemetryService } from './telemetry_service'; import { Storage } from '../../../kibana_utils/public'; +import type { EncryptedTelemetryPayload } from '../../common/types'; export class TelemetrySender { private readonly telemetryService: TelemetryService; @@ -57,18 +62,21 @@ export class TelemetrySender { this.isSending = true; try { const telemetryUrl = this.telemetryService.getTelemetryUrl(); - const telemetryData: string | string[] = await this.telemetryService.fetchTelemetry(); - const clusters: string[] = ([] as string[]).concat(telemetryData); + const telemetryPayload: EncryptedTelemetryPayload = + await this.telemetryService.fetchTelemetry(); + await Promise.all( - clusters.map( - async (cluster) => + telemetryPayload.map( + async ({ clusterUuid, stats }) => await fetch(telemetryUrl, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Elastic-Stack-Version': this.telemetryService.currentKibanaVersion, + 'X-Elastic-Cluster-ID': clusterUuid, + 'X-Elastic-Content-Encoding': PAYLOAD_CONTENT_ENCODING, }, - body: cluster, + body: stats, }) ) ); diff --git a/src/plugins/telemetry/public/services/telemetry_service.test.ts b/src/plugins/telemetry/public/services/telemetry_service.test.ts index b23ba127c1522..ca4af0a903400 100644 --- a/src/plugins/telemetry/public/services/telemetry_service.test.ts +++ b/src/plugins/telemetry/public/services/telemetry_service.test.ts @@ -10,7 +10,7 @@ /* eslint-disable dot-notation */ import { mockTelemetryService } from '../mocks'; -import { TELEMETRY_ENDPOINT } from '../../common/constants'; + describe('TelemetryService', () => { describe('fetchTelemetry', () => { it('calls expected URL with 20 minutes - now', async () => { @@ -142,7 +142,9 @@ describe('TelemetryService', () => { config: { sendUsageTo: 'staging' }, }); - expect(telemetryService.getTelemetryUrl()).toBe(TELEMETRY_ENDPOINT.MAIN_CHANNEL.STAGING); + expect(telemetryService.getTelemetryUrl()).toMatchInlineSnapshot( + `"https://telemetry-staging.elastic.co/v3/send/kibana-snapshot"` + ); }); it('should return prod endpoint when sendUsageTo is set to prod', async () => { @@ -150,7 +152,9 @@ describe('TelemetryService', () => { config: { sendUsageTo: 'prod' }, }); - expect(telemetryService.getTelemetryUrl()).toBe(TELEMETRY_ENDPOINT.MAIN_CHANNEL.PROD); + expect(telemetryService.getTelemetryUrl()).toMatchInlineSnapshot( + `"https://telemetry.elastic.co/v3/send/kibana-snapshot"` + ); }); }); @@ -160,8 +164,8 @@ describe('TelemetryService', () => { config: { sendUsageTo: 'staging' }, }); - expect(telemetryService.getOptInStatusUrl()).toBe( - TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL.STAGING + expect(telemetryService.getOptInStatusUrl()).toMatchInlineSnapshot( + `"https://telemetry-staging.elastic.co/v3/send/kibana-opt_in_status"` ); }); @@ -170,8 +174,8 @@ describe('TelemetryService', () => { config: { sendUsageTo: 'prod' }, }); - expect(telemetryService.getOptInStatusUrl()).toBe( - TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL.PROD + expect(telemetryService.getOptInStatusUrl()).toMatchInlineSnapshot( + `"https://telemetry.elastic.co/v3/send/kibana-opt_in_status"` ); }); }); @@ -247,7 +251,7 @@ describe('TelemetryService', () => { const telemetryService = mockTelemetryService({ config: { userCanChangeSettings: undefined }, }); - const mockPayload = ['mock_hashed_opt_in_status_payload']; + const mockPayload = [{ clusterUuid: 'mk_uuid', stats: 'mock_hashed_opt_in_status_payload' }]; const mockUrl = 'mock_telemetry_optin_status_url'; const mockGetOptInStatusUrl = jest.fn().mockReturnValue(mockUrl); @@ -257,21 +261,28 @@ describe('TelemetryService', () => { expect(mockGetOptInStatusUrl).toBeCalledTimes(1); expect(mockFetch).toBeCalledTimes(1); - expect(mockFetch).toBeCalledWith(mockUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Elastic-Stack-Version': 'mockKibanaVersion', - }, - body: JSON.stringify(mockPayload), - }); + expect(mockFetch.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "mock_telemetry_optin_status_url", + Object { + "body": "mock_hashed_opt_in_status_payload", + "headers": Object { + "Content-Type": "application/json", + "X-Elastic-Cluster-ID": "mk_uuid", + "X-Elastic-Content-Encoding": "aes256gcm", + "X-Elastic-Stack-Version": "mockKibanaVersion", + }, + "method": "POST", + }, + ] + `); }); it('swallows errors if fetch fails', async () => { const telemetryService = mockTelemetryService({ config: { userCanChangeSettings: undefined }, }); - const mockPayload = ['mock_hashed_opt_in_status_payload']; + const mockPayload = [{ clusterUuid: 'mk_uuid', stats: 'mock_hashed_opt_in_status_payload' }]; const mockUrl = 'mock_telemetry_optin_status_url'; const mockGetOptInStatusUrl = jest.fn().mockReturnValue(mockUrl); diff --git a/src/plugins/telemetry/public/services/telemetry_service.ts b/src/plugins/telemetry/public/services/telemetry_service.ts index 4e52ec3a7e6ed..63e9b66a49a92 100644 --- a/src/plugins/telemetry/public/services/telemetry_service.ts +++ b/src/plugins/telemetry/public/services/telemetry_service.ts @@ -10,6 +10,8 @@ import { i18n } from '@kbn/i18n'; import { CoreStart } from 'kibana/public'; import { TelemetryPluginConfig } from '../plugin'; import { getTelemetryChannelEndpoint } from '../../common/telemetry_config'; +import type { UnencryptedTelemetryPayload, EncryptedTelemetryPayload } from '../../common/types'; +import { PAYLOAD_CONTENT_ENCODING } from '../../common/constants'; interface TelemetryServiceConstructor { config: TelemetryPluginConfig; @@ -101,7 +103,7 @@ export class TelemetryService { /** Retrieve the URL to report telemetry **/ public getTelemetryUrl = () => { const { sendUsageTo } = this.config; - return getTelemetryChannelEndpoint({ channelName: 'main', env: sendUsageTo }); + return getTelemetryChannelEndpoint({ channelName: 'snapshot', env: sendUsageTo }); }; /** @@ -137,7 +139,7 @@ export class TelemetryService { }; /** Fetches an unencrypted telemetry payload so we can show it to the user **/ - public fetchExample = async () => { + public fetchExample = async (): Promise => { return await this.fetchTelemetry({ unencrypted: true }); }; @@ -145,11 +147,11 @@ export class TelemetryService { * Fetches telemetry payload * @param unencrypted Default `false`. Whether the returned payload should be encrypted or not. */ - public fetchTelemetry = async ({ unencrypted = false } = {}) => { + public fetchTelemetry = async ({ + unencrypted = false, + } = {}): Promise => { return this.http.post('/api/telemetry/v2/clusters/_stats', { - body: JSON.stringify({ - unencrypted, - }), + body: JSON.stringify({ unencrypted }), }); }; @@ -167,13 +169,16 @@ export class TelemetryService { try { // Report the option to the Kibana server to store the settings. // It returns the encrypted update to send to the telemetry cluster [{cluster_uuid, opt_in_status}] - const optInPayload = await this.http.post('/api/telemetry/v2/optIn', { - body: JSON.stringify({ enabled: optedIn }), - }); + const optInStatusPayload = await this.http.post( + '/api/telemetry/v2/optIn', + { + body: JSON.stringify({ enabled: optedIn }), + } + ); if (this.reportOptInStatusChange) { // Use the response to report about the change to the remote telemetry cluster. // If it's opt-out, this will be the last communication to the remote service. - await this.reportOptInStatus(optInPayload); + await this.reportOptInStatus(optInStatusPayload); } this.isOptedIn = optedIn; } catch (err) { @@ -216,18 +221,26 @@ export class TelemetryService { * Pushes the encrypted payload [{cluster_uuid, opt_in_status}] to the remote telemetry service * @param optInPayload [{cluster_uuid, opt_in_status}] encrypted by the server into an array of strings */ - private reportOptInStatus = async (optInPayload: string[]): Promise => { + private reportOptInStatus = async ( + optInStatusPayload: EncryptedTelemetryPayload + ): Promise => { const telemetryOptInStatusUrl = this.getOptInStatusUrl(); try { - await fetch(telemetryOptInStatusUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Elastic-Stack-Version': this.currentKibanaVersion, - }, - body: JSON.stringify(optInPayload), - }); + await Promise.all( + optInStatusPayload.map(async ({ clusterUuid, stats }) => { + return await fetch(telemetryOptInStatusUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Elastic-Stack-Version': this.currentKibanaVersion, + 'X-Elastic-Cluster-ID': clusterUuid, + 'X-Elastic-Content-Encoding': PAYLOAD_CONTENT_ENCODING, + }, + body: stats, + }); + }) + ); } catch (err) { // Sending the ping is best-effort. Telemetry tries to send the ping once and discards it immediately if sending fails. // swallow any errors diff --git a/src/plugins/telemetry/server/fetcher.test.ts b/src/plugins/telemetry/server/fetcher.test.ts index 15b40d2b3e4e5..8d427808bb5e1 100644 --- a/src/plugins/telemetry/server/fetcher.test.ts +++ b/src/plugins/telemetry/server/fetcher.test.ts @@ -71,7 +71,11 @@ describe('FetcherTask', () => { const initializerContext = coreMock.createPluginInitializerContext({}); const fetcherTask = new FetcherTask(initializerContext); const mockTelemetryUrl = 'mock_telemetry_url'; - const mockClusters = ['cluster_1', 'cluster_2']; + const mockClusters = [ + { clusterUuid: 'mk_uuid_1', stats: 'cluster_1' }, + { clusterUuid: 'mk_uuid_2', stats: 'cluster_2' }, + ]; + const getCurrentConfigs = jest.fn().mockResolvedValue({ telemetryUrl: mockTelemetryUrl, }); @@ -95,9 +99,8 @@ describe('FetcherTask', () => { expect(areAllCollectorsReady).toBeCalledTimes(1); expect(fetchTelemetry).toBeCalledTimes(1); - expect(sendTelemetry).toBeCalledTimes(2); - expect(sendTelemetry).toHaveBeenNthCalledWith(1, mockTelemetryUrl, mockClusters[0]); - expect(sendTelemetry).toHaveBeenNthCalledWith(2, mockTelemetryUrl, mockClusters[1]); + expect(sendTelemetry).toBeCalledTimes(1); + expect(sendTelemetry).toHaveBeenNthCalledWith(1, mockTelemetryUrl, mockClusters); expect(updateReportFailure).toBeCalledTimes(0); }); }); diff --git a/src/plugins/telemetry/server/fetcher.ts b/src/plugins/telemetry/server/fetcher.ts index e15b5be2604ec..02ac428b07667 100644 --- a/src/plugins/telemetry/server/fetcher.ts +++ b/src/plugins/telemetry/server/fetcher.ts @@ -25,7 +25,8 @@ import { getTelemetryFailureDetails, } from '../common/telemetry_config'; import { getTelemetrySavedObject, updateTelemetrySavedObject } from './telemetry_repository'; -import { REPORT_INTERVAL_MS } from '../common/constants'; +import { REPORT_INTERVAL_MS, PAYLOAD_CONTENT_ENCODING } from '../common/constants'; +import type { EncryptedTelemetryPayload } from '../common/types'; import { TelemetryConfigType } from './config'; export interface FetcherTaskDepsStart { @@ -103,7 +104,7 @@ export class FetcherTask { return; } - let clusters: string[] = []; + let clusters: EncryptedTelemetryPayload = []; this.isSending = true; try { @@ -120,9 +121,7 @@ export class FetcherTask { try { const { telemetryUrl } = telemetryConfig; - for (const cluster of clusters) { - await this.sendTelemetry(telemetryUrl, cluster); - } + await this.sendTelemetry(telemetryUrl, clusters); await this.updateLastReported(); } catch (err) { @@ -141,7 +140,7 @@ export class FetcherTask { const allowChangingOptInStatus = config.allowChangingOptInStatus; const configTelemetryOptIn = typeof config.optIn === 'undefined' ? null : config.optIn; const telemetryUrl = getTelemetryChannelEndpoint({ - channelName: 'main', + channelName: 'snapshot', env: config.sendUsageTo, }); const { failureCount, failureVersion } = getTelemetryFailureDetails({ @@ -206,13 +205,16 @@ export class FetcherTask { return false; } - private async fetchTelemetry() { + private async fetchTelemetry(): Promise { return await this.telemetryCollectionManager!.getStats({ unencrypted: false, }); } - private async sendTelemetry(telemetryUrl: string, cluster: string): Promise { + private async sendTelemetry( + telemetryUrl: string, + payload: EncryptedTelemetryPayload + ): Promise { this.logger.debug(`Sending usage stats.`); /** * send OPTIONS before sending usage data. @@ -222,10 +224,18 @@ export class FetcherTask { method: 'options', }); - await fetch(telemetryUrl, { - method: 'post', - body: cluster, - headers: { 'X-Elastic-Stack-Version': this.currentKibanaVersion }, - }); + await Promise.all( + payload.map(async ({ clusterUuid, stats }) => { + await fetch(telemetryUrl, { + method: 'post', + body: stats, + headers: { + 'X-Elastic-Stack-Version': this.currentKibanaVersion, + 'X-Elastic-Cluster-ID': clusterUuid, + 'X-Elastic-Content-Encoding': PAYLOAD_CONTENT_ENCODING, + }, + }); + }) + ); } } diff --git a/src/plugins/telemetry/server/plugin.ts b/src/plugins/telemetry/server/plugin.ts index 21fd85018d6db..aa22410358f72 100644 --- a/src/plugins/telemetry/server/plugin.ts +++ b/src/plugins/telemetry/server/plugin.ts @@ -115,7 +115,10 @@ export class TelemetryPlugin implements Plugin { const { sendUsageTo } = await config$.pipe(take(1)).toPromise(); - const telemetryUrl = getTelemetryChannelEndpoint({ env: sendUsageTo, channelName: 'main' }); + const telemetryUrl = getTelemetryChannelEndpoint({ + env: sendUsageTo, + channelName: 'snapshot', + }); return new URL(telemetryUrl); }, diff --git a/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.test.ts b/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.test.ts index acc9a863af61b..edf9cf5b5e18c 100644 --- a/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.test.ts +++ b/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.test.ts @@ -10,11 +10,14 @@ jest.mock('node-fetch'); import fetch from 'node-fetch'; import { sendTelemetryOptInStatus } from './telemetry_opt_in_stats'; import { StatsGetterConfig } from 'src/plugins/telemetry_collection_manager/server'; -import { TELEMETRY_ENDPOINT } from '../../common/constants'; + describe('sendTelemetryOptInStatus', () => { + const mockClusterUuid = 'mk_uuid'; const mockStatsGetterConfig = { unencrypted: false } as StatsGetterConfig; const mockTelemetryCollectionManager = { - getOptInStats: jest.fn().mockResolvedValue(['mock_opt_in_hashed_value']), + getOptInStats: jest + .fn() + .mockResolvedValue([{ clusterUuid: mockClusterUuid, stats: 'mock_opt_in_hashed_value' }]), }; beforeEach(() => { @@ -35,11 +38,21 @@ describe('sendTelemetryOptInStatus', () => { ); expect(result).toBeUndefined(); expect(fetch).toBeCalledTimes(1); - expect(fetch).toBeCalledWith(TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL.PROD, { - method: 'post', - body: '["mock_opt_in_hashed_value"]', - headers: { 'X-Elastic-Stack-Version': mockConfig.currentKibanaVersion }, - }); + expect((fetch as jest.MockedFunction).mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "https://telemetry.elastic.co/v3/send/kibana-opt_in_status", + Object { + "body": "mock_opt_in_hashed_value", + "headers": Object { + "Content-Type": "application/json", + "X-Elastic-Cluster-ID": "mk_uuid", + "X-Elastic-Content-Encoding": "aes256gcm", + "X-Elastic-Stack-Version": "mock_kibana_version", + }, + "method": "post", + }, + ] + `); }); it('sends to staging endpoint on "sendUsageTo: staging"', async () => { @@ -56,10 +69,20 @@ describe('sendTelemetryOptInStatus', () => { ); expect(fetch).toBeCalledTimes(1); - expect(fetch).toBeCalledWith(TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL.STAGING, { - method: 'post', - body: '["mock_opt_in_hashed_value"]', - headers: { 'X-Elastic-Stack-Version': mockConfig.currentKibanaVersion }, - }); + expect((fetch as jest.MockedFunction).mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "https://telemetry-staging.elastic.co/v3/send/kibana-opt_in_status", + Object { + "body": "mock_opt_in_hashed_value", + "headers": Object { + "Content-Type": "application/json", + "X-Elastic-Cluster-ID": "mk_uuid", + "X-Elastic-Content-Encoding": "aes256gcm", + "X-Elastic-Stack-Version": "mock_kibana_version", + }, + "method": "post", + }, + ] + `); }); }); diff --git a/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.ts b/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.ts index f6b7eddcbe765..2a95665662194 100644 --- a/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.ts +++ b/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.ts @@ -15,6 +15,8 @@ import { StatsGetterConfig, } from 'src/plugins/telemetry_collection_manager/server'; import { getTelemetryChannelEndpoint } from '../../common/telemetry_config'; +import { PAYLOAD_CONTENT_ENCODING } from '../../common/constants'; +import type { UnencryptedTelemetryPayload } from '../../common/types'; interface SendTelemetryOptInStatusConfig { sendUsageTo: 'staging' | 'prod'; @@ -26,23 +28,30 @@ export async function sendTelemetryOptInStatus( telemetryCollectionManager: Pick, config: SendTelemetryOptInStatusConfig, statsGetterConfig: StatsGetterConfig -) { +): Promise { const { sendUsageTo, newOptInStatus, currentKibanaVersion } = config; const optInStatusUrl = getTelemetryChannelEndpoint({ env: sendUsageTo, channelName: 'optInStatus', }); - const optInStatus = await telemetryCollectionManager.getOptInStats( - newOptInStatus, - statsGetterConfig - ); + const optInStatusPayload: UnencryptedTelemetryPayload = + await telemetryCollectionManager.getOptInStats(newOptInStatus, statsGetterConfig); - await fetch(optInStatusUrl, { - method: 'post', - body: JSON.stringify(optInStatus), - headers: { 'X-Elastic-Stack-Version': currentKibanaVersion }, - }); + await Promise.all( + optInStatusPayload.map(async ({ clusterUuid, stats }) => { + return await fetch(optInStatusUrl, { + method: 'post', + body: typeof stats === 'string' ? stats : JSON.stringify(stats), + headers: { + 'Content-Type': 'application/json', + 'X-Elastic-Stack-Version': currentKibanaVersion, + 'X-Elastic-Cluster-ID': clusterUuid, + 'X-Elastic-Content-Encoding': PAYLOAD_CONTENT_ENCODING, + }, + }); + }) + ); } export function registerTelemetryOptInStatsRoutes( diff --git a/src/plugins/telemetry_collection_manager/server/encryption/encrypt.ts b/src/plugins/telemetry_collection_manager/server/encryption/encrypt.ts index 1b80a2c29b362..2ed69c2f8a944 100644 --- a/src/plugins/telemetry_collection_manager/server/encryption/encrypt.ts +++ b/src/plugins/telemetry_collection_manager/server/encryption/encrypt.ts @@ -14,11 +14,11 @@ export function getKID(useProdKey = false): string { } export async function encryptTelemetry( - payload: Payload | Payload[], + payload: Payload, { useProdKey = false } = {} -): Promise { +): Promise { const kid = getKID(useProdKey); const encryptor = await createRequestEncryptor(telemetryJWKS); - const clusters = ([] as Payload[]).concat(payload); - return Promise.all(clusters.map((cluster) => encryptor.encrypt(kid, cluster))); + + return await encryptor.encrypt(kid, payload); } diff --git a/src/plugins/telemetry_collection_manager/server/plugin.test.ts b/src/plugins/telemetry_collection_manager/server/plugin.test.ts index d05799f82c354..6e37ef5ffd4f5 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.test.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.test.ts @@ -91,7 +91,9 @@ describe('Telemetry Collection Manager', () => { cluster_uuid: 'clusterUuid', cluster_name: 'clusterName', timestamp: new Date().toISOString(), - cluster_stats: {}, + cluster_stats: { + cluster_uuid: 'clusterUuid', + }, stack_stats: {}, version: 'version', }; @@ -120,7 +122,12 @@ describe('Telemetry Collection Manager', () => { { clusterUuid: 'clusterUuid' }, ]); collectionStrategy.statsGetter.mockResolvedValue([basicStats]); - await expect(setupApi.getStats(config)).resolves.toStrictEqual([expect.any(String)]); + await expect(setupApi.getStats(config)).resolves.toStrictEqual([ + { + clusterUuid: 'clusterUuid', + stats: expect.any(String), + }, + ]); expect( collectionStrategy.clusterDetailsGetter.mock.calls[0][0].soClient ).toBeInstanceOf(TelemetrySavedObjectsClient); @@ -141,7 +148,10 @@ describe('Telemetry Collection Manager', () => { { clusterUuid: 'clusterUuid' }, ]); await expect(setupApi.getOptInStats(true, config)).resolves.toStrictEqual([ - expect.any(String), + { + clusterUuid: 'clusterUuid', + stats: expect.any(String), + }, ]); expect( collectionStrategy.clusterDetailsGetter.mock.calls[0][0].soClient @@ -153,7 +163,10 @@ describe('Telemetry Collection Manager', () => { { clusterUuid: 'clusterUuid' }, ]); await expect(setupApi.getOptInStats(false, config)).resolves.toStrictEqual([ - expect.any(String), + { + clusterUuid: 'clusterUuid', + stats: expect.any(String), + }, ]); expect( collectionStrategy.clusterDetailsGetter.mock.calls[0][0].soClient @@ -181,7 +194,10 @@ describe('Telemetry Collection Manager', () => { ]); collectionStrategy.statsGetter.mockResolvedValue([basicStats]); await expect(setupApi.getStats(config)).resolves.toStrictEqual([ - { ...basicStats, collectionSource: 'test_collection' }, + { + clusterUuid: 'clusterUuid', + stats: { ...basicStats, collectionSource: 'test_collection' }, + }, ]); expect( collectionStrategy.clusterDetailsGetter.mock.calls[0][0].soClient @@ -203,7 +219,10 @@ describe('Telemetry Collection Manager', () => { { clusterUuid: 'clusterUuid' }, ]); await expect(setupApi.getOptInStats(true, config)).resolves.toStrictEqual([ - { cluster_uuid: 'clusterUuid', opt_in_status: true }, + { + clusterUuid: 'clusterUuid', + stats: { opt_in_status: true, cluster_uuid: 'clusterUuid' }, + }, ]); expect( collectionStrategy.clusterDetailsGetter.mock.calls[0][0].soClient @@ -215,7 +234,10 @@ describe('Telemetry Collection Manager', () => { { clusterUuid: 'clusterUuid' }, ]); await expect(setupApi.getOptInStats(false, config)).resolves.toStrictEqual([ - { cluster_uuid: 'clusterUuid', opt_in_status: false }, + { + clusterUuid: 'clusterUuid', + stats: { opt_in_status: false, cluster_uuid: 'clusterUuid' }, + }, ]); expect( collectionStrategy.clusterDetailsGetter.mock.calls[0][0].soClient diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index 9770395e0ec0c..6dd1de65a8bdc 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -28,6 +28,7 @@ import type { StatsGetterConfig, StatsCollectionConfig, UsageStatsPayload, + OptInStatsPayload, StatsCollectionContext, UnencryptedStatsGetterConfig, EncryptedStatsGetterConfig, @@ -163,6 +164,14 @@ export class TelemetryCollectionManagerPlugin } } + private async getOptInStats( + optInStatus: boolean, + config: UnencryptedStatsGetterConfig + ): Promise>; + private async getOptInStats( + optInStatus: boolean, + config: EncryptedStatsGetterConfig + ): Promise>; private async getOptInStats(optInStatus: boolean, config: StatsGetterConfig) { if (!this.usageCollection) { return []; @@ -179,13 +188,23 @@ export class TelemetryCollectionManagerPlugin optInStatus, statsCollectionConfig ); - if (optInStats && optInStats.length) { - this.logger.debug(`Got Opt In stats using ${collection.title} collection.`); - if (config.unencrypted) { - return optInStats; - } - return encryptTelemetry(optInStats, { useProdKey: this.isDistributable }); - } + + this.logger.debug(`Received Opt In stats using ${collection.title} collection.`); + + return await Promise.all( + optInStats.map(async (clusterStats) => { + const clusterUuid = clusterStats.cluster_uuid; + + return { + clusterUuid, + stats: config.unencrypted + ? clusterStats + : await encryptTelemetry(clusterStats, { + useProdKey: this.isDistributable, + }), + }; + }) + ); } catch (err) { this.logger.debug( `Failed to collect any opt in stats with collection ${collection.title}.` @@ -205,7 +224,7 @@ export class TelemetryCollectionManagerPlugin collection: CollectionStrategy, optInStatus: boolean, statsCollectionConfig: StatsCollectionConfig - ) => { + ): Promise => { const context: StatsCollectionContext = { logger: this.logger.get(collection.title), version: this.version, @@ -218,8 +237,12 @@ export class TelemetryCollectionManagerPlugin })); }; - private async getStats(config: UnencryptedStatsGetterConfig): Promise; - private async getStats(config: EncryptedStatsGetterConfig): Promise; + private async getStats( + config: UnencryptedStatsGetterConfig + ): Promise>; + private async getStats( + config: EncryptedStatsGetterConfig + ): Promise>; private async getStats(config: StatsGetterConfig) { if (!this.usageCollection) { return []; @@ -231,16 +254,25 @@ export class TelemetryCollectionManagerPlugin if (statsCollectionConfig) { try { const usageData = await this.getUsageForCollection(collection, statsCollectionConfig); - if (usageData.length) { - this.logger.debug(`Got Usage using ${collection.title} collection.`); - if (config.unencrypted) { - return usageData; - } - - return await encryptTelemetry(usageData, { - useProdKey: this.isDistributable, - }); - } + this.logger.debug(`Received Usage using ${collection.title} collection.`); + + return await Promise.all( + usageData.map(async (clusterStats) => { + const { cluster_uuid: clusterUuid } = clusterStats.cluster_stats as Record< + string, + string + >; + + return { + clusterUuid, + stats: config.unencrypted + ? clusterStats + : await encryptTelemetry(clusterStats, { + useProdKey: this.isDistributable, + }), + }; + }) + ); } catch (err) { this.logger.debug( `Failed to collect any usage with registered collection ${collection.title}.` diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts index 985eff409c1de..648e457f9a238 100644 --- a/src/plugins/telemetry_collection_manager/server/types.ts +++ b/src/plugins/telemetry_collection_manager/server/types.ts @@ -74,6 +74,11 @@ export interface UsageStatsPayload extends BasicStatsPayload { collectionSource: string; } +export interface OptInStatsPayload { + cluster_uuid: string; + opt_in_status: boolean; +} + export interface StatsCollectionContext { logger: Logger | Console; version: string; diff --git a/x-pack/test/api_integration/apis/telemetry/telemetry.ts b/x-pack/test/api_integration/apis/telemetry/telemetry.ts index c5b8b40368302..527d755123f26 100644 --- a/x-pack/test/api_integration/apis/telemetry/telemetry.ts +++ b/x-pack/test/api_integration/apis/telemetry/telemetry.ts @@ -19,6 +19,7 @@ import monitoringRootTelemetrySchema from '../../../../plugins/telemetry_collect import ossPluginsTelemetrySchema from '../../../../../src/plugins/telemetry/schema/oss_plugins.json'; import xpackPluginsTelemetrySchema from '../../../../plugins/telemetry_collection_xpack/schema/xpack_plugins.json'; import { assertTelemetryPayload } from '../../../../../test/api_integration/apis/telemetry/utils'; +import type { UnencryptedTelemetryPayload } from '../../../../../src/plugins/telemetry/common/types'; /** * Update the .monitoring-* documents loaded via the archiver to the recent `timestamp` @@ -92,15 +93,16 @@ export default function ({ getService }: FtrProviderContext) { await esArchiver.load(archive); await updateMonitoringDates(esSupertest, fromTimestamp, toTimestamp, timestamp); - const { body } = await supertest + const { body }: { body: UnencryptedTelemetryPayload } = await supertest .post('/api/telemetry/v2/clusters/_stats') .set('kbn-xsrf', 'xxx') .send({ unencrypted: true }) .expect(200); expect(body.length).to.be.greaterThan(1); - localXPack = body.shift(); - monitoring = body; + const telemetryStats = body.map(({ stats }) => stats); + localXPack = telemetryStats.shift() as Record; + monitoring = telemetryStats as Array>; }); after(() => esArchiver.unload(archive)); @@ -142,15 +144,17 @@ export default function ({ getService }: FtrProviderContext) { }); after(() => esArchiver.unload(archive)); it('should load non-expiring basic cluster', async () => { - const { body } = await supertest + const { body }: { body: UnencryptedTelemetryPayload } = await supertest .post('/api/telemetry/v2/clusters/_stats') .set('kbn-xsrf', 'xxx') .send({ unencrypted: true }) .expect(200); expect(body).length(2); - const [localXPack, ...monitoring] = body; - expect(localXPack.collectionSource).to.eql('local_xpack'); + const telemetryStats = body.map(({ stats }) => stats); + + const [localXPack, ...monitoring] = telemetryStats; + expect((localXPack as Record).collectionSource).to.eql('local_xpack'); expect(monitoring).to.eql(basicClusterFixture.map((item) => ({ ...item, timestamp }))); }); }); diff --git a/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts b/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts index 508a6584e9246..e34e0fff25888 100644 --- a/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts +++ b/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts @@ -47,7 +47,7 @@ export default function ({ getService }: FtrProviderContext) { .expect(200); expect(body.length).to.be(1); - stats = body[0]; + stats = body[0].stats; }); it('should pass the schema validation', () => { diff --git a/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts b/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts index ed79d7200c4ed..0d8f38c55c7f8 100644 --- a/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts +++ b/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts @@ -107,7 +107,7 @@ export default function (providerContext: FtrProviderContext) { it('should return the correct telemetry values for fleet', async () => { const { - body: [apiResponse], + body: [{ stats: apiResponse }], } = await supertest .post(`/api/telemetry/v2/clusters/_stats`) .set('kbn-xsrf', 'xxxx') diff --git a/x-pack/test/functional/apps/infra/logs_source_configuration.ts b/x-pack/test/functional/apps/infra/logs_source_configuration.ts index dcbe30864640b..34a50530df993 100644 --- a/x-pack/test/functional/apps/infra/logs_source_configuration.ts +++ b/x-pack/test/functional/apps/infra/logs_source_configuration.ts @@ -113,7 +113,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { await logsUi.logStreamPage.getStreamEntries(); - const resp = await supertest + const [{ stats }] = await supertest .post(`/api/telemetry/v2/clusters/_stats`) .set(COMMON_REQUEST_HEADERS) .set('Accept', 'application/json') @@ -123,9 +123,9 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { .expect(200) .then((res: any) => res.body); - expect( - resp[0].stack_stats.kibana.plugins.infraops.last_24_hours.hits.logs - ).to.be.greaterThan(0); + expect(stats.stack_stats.kibana.plugins.infraops.last_24_hours.hits.logs).to.be.greaterThan( + 0 + ); }); it('can change the log columns', async () => { diff --git a/x-pack/test/saved_object_tagging/api_integration/tagging_api/apis/usage_collection.ts b/x-pack/test/saved_object_tagging/api_integration/tagging_api/apis/usage_collection.ts index b6ec4aa8dcfa5..03494edccd648 100644 --- a/x-pack/test/saved_object_tagging/api_integration/tagging_api/apis/usage_collection.ts +++ b/x-pack/test/saved_object_tagging/api_integration/tagging_api/apis/usage_collection.ts @@ -40,11 +40,11 @@ export default function ({ getService }: FtrProviderContext) { * - vis-3: ref to tag-3 */ it('collects the expected data', async () => { - const telemetryStats = (await usageAPI.getTelemetryStats({ + const [{ stats: telemetryStats }] = (await usageAPI.getTelemetryStats({ unencrypted: true, })) as any; - const taggingStats = telemetryStats[0].stack_stats.kibana.plugins.saved_objects_tagging; + const taggingStats = telemetryStats.stack_stats.kibana.plugins.saved_objects_tagging; expect(taggingStats).to.eql({ usedTags: 4, taggedObjects: 5,