From def871d3dfbb18bc56ffc0730d5eda496aebcd12 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Fri, 15 Oct 2021 14:41:12 +0200 Subject: [PATCH 01/27] draft of upgrade usage collector --- .../collectors/package_upgrade_collectors.ts | 36 +++++++++++++++++++ .../fleet/server/collectors/register.ts | 3 ++ .../fleet/server/saved_objects/index.ts | 24 +++++++++++++ .../fleet/server/services/package_policy.ts | 17 +++++++++ .../fleet/server/services/upgrade_usage.ts | 26 ++++++++++++++ 5 files changed, 106 insertions(+) create mode 100644 x-pack/plugins/fleet/server/collectors/package_upgrade_collectors.ts create mode 100644 x-pack/plugins/fleet/server/services/upgrade_usage.ts diff --git a/x-pack/plugins/fleet/server/collectors/package_upgrade_collectors.ts b/x-pack/plugins/fleet/server/collectors/package_upgrade_collectors.ts new file mode 100644 index 0000000000000..3c76d6cd806ea --- /dev/null +++ b/x-pack/plugins/fleet/server/collectors/package_upgrade_collectors.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClient } from 'kibana/server'; + +import { deleteUpgradeUsages } from '../services/upgrade_usage'; + +export interface PackagePolicyUpgradeUsage { + package_name: string; + current_version: string; + new_version: string; + status: 'success' | 'failure'; + error?: Array<{ key?: string; message: string }>; +} + +export const getPackagePolicyUpgradeUsage = async ( + soClient?: SavedObjectsClient +): Promise => { + if (!soClient) { + return []; + } + const telemetryObjects = await soClient.find({ + type: 'package-policy-upgrade-telemetry', + }); + + const usages = telemetryObjects.saved_objects.map((so) => so.attributes); + deleteUpgradeUsages( + soClient, + telemetryObjects.saved_objects.map((so) => so.id) + ); + return usages; +}; diff --git a/x-pack/plugins/fleet/server/collectors/register.ts b/x-pack/plugins/fleet/server/collectors/register.ts index a097d423e7dd2..2ddd9030b5910 100644 --- a/x-pack/plugins/fleet/server/collectors/register.ts +++ b/x-pack/plugins/fleet/server/collectors/register.ts @@ -18,6 +18,7 @@ import { getPackageUsage } from './package_collectors'; import type { PackageUsage } from './package_collectors'; import { getFleetServerUsage } from './fleet_server_collector'; import type { FleetServerUsage } from './fleet_server_collector'; +import { getPackagePolicyUpgradeUsage } from './package_upgrade_collectors'; interface Usage { agents_enabled: boolean; @@ -48,6 +49,7 @@ export function registerFleetUsageCollector( agents: await getAgentUsage(config, soClient, esClient), fleet_server: await getFleetServerUsage(soClient, esClient), packages: await getPackageUsage(soClient), + package_policy_upgrades: await getPackagePolicyUpgradeUsage(soClient), }; }, schema: { @@ -143,6 +145,7 @@ export function registerFleetUsageCollector( enabled: { type: 'boolean' }, }, }, + // TODO add package_policy_upgrades schema }, }); diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index ac5ca401da000..29b9c014b56e8 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -369,6 +369,30 @@ const getSavedObjectTypes = ( }, }, }, + // TODO create constant + ['package-policy-upgrade-telemetry']: { + name: 'package-policy-upgrade-telemetry', + hidden: false, + namespaceType: 'agnostic', + management: { + importableAndExportable: false, + }, + mappings: { + properties: { + package_name: { type: 'keyword' }, + current_version: { type: 'keyword' }, + new_version: { type: 'keyword' }, + status: { type: 'keyword' }, + error: { + type: 'nested', + properties: { + key: { type: 'keyword' }, + message: { type: 'text' }, + }, + }, + }, + }, + }, }); export function registerSavedObjects( diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 546e267b8402b..be3f72f421da0 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -66,6 +66,7 @@ import { compileTemplate } from './epm/agent/agent'; import { normalizeKuery } from './saved_object'; import { appContextService } from '.'; import { removeOldAssets } from './epm/packages/cleanup'; +import { createUpgradeUsage } from './upgrade_usage'; export type InputsOverride = Partial & { vars?: Array; @@ -576,6 +577,12 @@ class PackagePolicyService { name: packagePolicy.name, success: true, }); + createUpgradeUsage(soClient, { + package_name: packageInfo.name, + current_version: packagePolicy.package.version, + new_version: packageInfo.version, + status: 'success', + }); await removeOldAssets({ soClient, pkgName: packageInfo.name, @@ -627,6 +634,16 @@ class PackagePolicyService { const hasErrors = 'errors' in updatedPackagePolicy; + if (hasErrors) { + createUpgradeUsage(soClient, { + package_name: packageInfo.name, + current_version: packagePolicy.package.version, + new_version: packageInfo.version, + status: hasErrors ? 'failure' : 'success', + error: updatedPackagePolicy.errors, + }); + } + return { name: updatedPackagePolicy.name, diff: [packagePolicy, updatedPackagePolicy], diff --git a/x-pack/plugins/fleet/server/services/upgrade_usage.ts b/x-pack/plugins/fleet/server/services/upgrade_usage.ts new file mode 100644 index 0000000000000..46780fc87ee08 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/upgrade_usage.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from 'kibana/server'; + +import type { PackagePolicyUpgradeUsage } from '../collectors/package_upgrade_collectors'; + +export function createUpgradeUsage( + soClient: SavedObjectsClientContract, + upgradeUsage: PackagePolicyUpgradeUsage +) { + soClient.create('package-policy-upgrade-telemetry', upgradeUsage, { + id: `${upgradeUsage.package_name}_${upgradeUsage.current_version}_${upgradeUsage.new_version}_${upgradeUsage.status}`, + overwrite: true, + }); +} + +export function deleteUpgradeUsages(soClient: SavedObjectsClientContract, ids: string[]) { + for (const id of ids) { + soClient.delete('package-policy-upgrade-telemetry', id); + } +} From 49576907a5164198999d3efaa44aa4e9aefa760d Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Mon, 18 Oct 2021 12:00:11 +0200 Subject: [PATCH 02/27] telemetry sender service --- x-pack/plugins/fleet/kibana.json | 2 +- .../collectors/package_upgrade_collectors.ts | 2 + x-pack/plugins/fleet/server/mocks/index.ts | 1 + x-pack/plugins/fleet/server/plugin.ts | 19 ++ .../fleet/server/services/app_context.ts | 7 + .../fleet/server/services/package_policy.ts | 57 +++- .../fleet/server/services/upgrade_usage.ts | 23 ++ .../fleet/server/telemetry/__mocks__/index.ts | 103 +++++++ .../fleet/server/telemetry/receiver.ts | 67 +++++ .../fleet/server/telemetry/sender.test.ts | 121 ++++++++ .../plugins/fleet/server/telemetry/sender.ts | 232 +++++++++++++++ .../server/telemetry/telemetry/helpers.ts | 207 ++++++++++++++ .../telemetry/telemetry/tasks/endpoint.ts | 269 ++++++++++++++++++ .../plugins/fleet/server/telemetry/types.ts | 49 ++++ 14 files changed, 1143 insertions(+), 16 deletions(-) create mode 100644 x-pack/plugins/fleet/server/telemetry/__mocks__/index.ts create mode 100644 x-pack/plugins/fleet/server/telemetry/receiver.ts create mode 100644 x-pack/plugins/fleet/server/telemetry/sender.test.ts create mode 100644 x-pack/plugins/fleet/server/telemetry/sender.ts create mode 100644 x-pack/plugins/fleet/server/telemetry/telemetry/helpers.ts create mode 100644 x-pack/plugins/fleet/server/telemetry/telemetry/tasks/endpoint.ts create mode 100644 x-pack/plugins/fleet/server/telemetry/types.ts diff --git a/x-pack/plugins/fleet/kibana.json b/x-pack/plugins/fleet/kibana.json index 9de538ee91b8c..6c442757677e2 100644 --- a/x-pack/plugins/fleet/kibana.json +++ b/x-pack/plugins/fleet/kibana.json @@ -9,7 +9,7 @@ "ui": true, "configPath": ["xpack", "fleet"], "requiredPlugins": ["licensing", "data", "encryptedSavedObjects", "navigation", "customIntegrations", "share"], - "optionalPlugins": ["security", "features", "cloud", "usageCollection", "home", "globalSearch"], + "optionalPlugins": ["security", "features", "cloud", "usageCollection", "home", "globalSearch", "telemetry"], "extraPublicDirs": ["common"], "requiredBundles": ["kibanaReact", "esUiShared", "home", "infra", "kibanaUtils"] } diff --git a/x-pack/plugins/fleet/server/collectors/package_upgrade_collectors.ts b/x-pack/plugins/fleet/server/collectors/package_upgrade_collectors.ts index 3c76d6cd806ea..7183091fcd865 100644 --- a/x-pack/plugins/fleet/server/collectors/package_upgrade_collectors.ts +++ b/x-pack/plugins/fleet/server/collectors/package_upgrade_collectors.ts @@ -27,6 +27,8 @@ export const getPackagePolicyUpgradeUsage = async ( type: 'package-policy-upgrade-telemetry', }); + // TODO cap if becomes too large + const usages = telemetryObjects.saved_objects.map((so) => so.attributes); deleteUpgradeUsages( soClient, diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 0e7b335da6775..8da503601a4b0 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -49,6 +49,7 @@ export const createAppContextStartContractMock = (): FleetAppContext => { config$, kibanaVersion: '8.0.0', kibanaBranch: 'master', + telemetryEventsSender: undefined, }; }; diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 697ea0fa30d69..436909b7d584d 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -18,6 +18,8 @@ import type { } from 'kibana/server'; import type { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import type { TelemetryPluginSetup, TelemetryPluginStart } from 'src/plugins/telemetry/server'; + import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server'; import type { PluginStart as DataPluginStart } from '../../../../src/plugins/data/server'; import type { LicensingPluginSetup, ILicense } from '../../licensing/server'; @@ -83,6 +85,8 @@ import { getInstallation } from './services/epm/packages'; import { makeRouterEnforcingSuperuser } from './routes/security'; import { startFleetServerSetup } from './services/fleet_server'; import { FleetArtifactsClient } from './services/artifacts'; +import { TelemetryReceiver } from './telemetry/receiver'; +import { TelemetryEventsSender } from './telemetry/sender'; export interface FleetSetupDeps { licensing: LicensingPluginSetup; @@ -91,12 +95,14 @@ export interface FleetSetupDeps { encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; cloud?: CloudSetup; usageCollection?: UsageCollectionSetup; + telemetry?: TelemetryPluginSetup; } export interface FleetStartDeps { data: DataPluginStart; encryptedSavedObjects: EncryptedSavedObjectsPluginStart; security?: SecurityPluginStart; + telemetry?: TelemetryPluginStart; } export interface FleetAppContext { @@ -115,6 +121,7 @@ export interface FleetAppContext { cloud?: CloudSetup; logger?: Logger; httpSetup?: HttpServiceSetup; + telemetryEventsSender: TelemetryEventsSender; } export type FleetSetupContract = void; @@ -176,6 +183,8 @@ export class FleetPlugin private httpSetup?: HttpServiceSetup; private securitySetup?: SecurityPluginSetup; private encryptedSavedObjectsSetup?: EncryptedSavedObjectsPluginSetup; + private readonly telemetryReceiver: TelemetryReceiver; + private readonly telemetryEventsSender: TelemetryEventsSender; constructor(private readonly initializerContext: PluginInitializerContext) { this.config$ = this.initializerContext.config.create(); @@ -184,6 +193,8 @@ export class FleetPlugin this.kibanaBranch = this.initializerContext.env.packageInfo.branch; this.logger = this.initializerContext.logger.get(); this.configInitialValue = this.initializerContext.config.get(); + this.telemetryEventsSender = new TelemetryEventsSender(this.logger); + this.telemetryReceiver = new TelemetryReceiver(this.logger); } public setup(core: CoreSetup, deps: FleetSetupDeps) { @@ -274,6 +285,8 @@ export class FleetPlugin registerEnrollmentApiKeyRoutes(routerSuperuserOnly); } } + + this.telemetryEventsSender.setup(deps.telemetry); } public start(core: CoreStart, plugins: FleetStartDeps): FleetStartContract { @@ -293,11 +306,16 @@ export class FleetPlugin httpSetup: this.httpSetup, cloud: this.cloud, logger: this.logger, + telemetryEventsSender: this.telemetryEventsSender, }); licenseService.start(this.licensing$); const fleetServerSetup = startFleetServerSetup(); + this.telemetryReceiver.start(core); + + this.telemetryEventsSender.start(plugins.telemetry, this.telemetryReceiver); + return { fleetSetupCompleted: () => new Promise((resolve) => { @@ -334,5 +352,6 @@ export class FleetPlugin public async stop() { appContextService.stop(); licenseService.stop(); + this.telemetryEventsSender.stop(); } } diff --git a/x-pack/plugins/fleet/server/services/app_context.ts b/x-pack/plugins/fleet/server/services/app_context.ts index a1e6ef4545aef..7ec1607598b8a 100644 --- a/x-pack/plugins/fleet/server/services/app_context.ts +++ b/x-pack/plugins/fleet/server/services/app_context.ts @@ -33,6 +33,7 @@ import type { } from '../types'; import type { FleetAppContext } from '../plugin'; import type { CloudSetup } from '../../../cloud/server'; +import type { TelemetryEventsSender } from '../telemetry/sender'; class AppContextService { private encryptedSavedObjects: EncryptedSavedObjectsClient | undefined; @@ -51,6 +52,7 @@ class AppContextService { private logger: Logger | undefined; private httpSetup?: HttpServiceSetup; private externalCallbacks: ExternalCallbacksStorage = new Map(); + private telemetryEventsSender: TelemetryEventsSender | undefined; public start(appContext: FleetAppContext) { this.data = appContext.data; @@ -66,6 +68,7 @@ class AppContextService { this.kibanaVersion = appContext.kibanaVersion; this.kibanaBranch = appContext.kibanaBranch; this.httpSetup = appContext.httpSetup; + this.telemetryEventsSender = appContext.telemetryEventsSender; if (appContext.config$) { this.config$ = appContext.config$; @@ -203,6 +206,10 @@ class AppContextService { >; } } + + public getTelemetryEventsSender() { + return this.telemetryEventsSender; + } } export const appContextService = new AppContextService(); diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 276cd94700f2f..cffc71ef99940 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -66,7 +66,7 @@ import { compileTemplate } from './epm/agent/agent'; import { normalizeKuery } from './saved_object'; import { appContextService } from '.'; import { removeOldAssets } from './epm/packages/cleanup'; -import { createUpgradeUsage } from './upgrade_usage'; +import { createUpgradeUsage, sendAlertTelemetryEvents } from './upgrade_usage'; export type InputsOverride = Partial & { vars?: Array; @@ -577,12 +577,25 @@ class PackagePolicyService { name: packagePolicy.name, success: true, }); - createUpgradeUsage(soClient, { - package_name: packageInfo.name, - current_version: packagePolicy.package.version, - new_version: packageInfo.version, - status: 'success', - }); + if (packagePolicy.package.version !== packageInfo.version) { + createUpgradeUsage(soClient, { + package_name: packageInfo.name, + current_version: packagePolicy.package.version, + new_version: packageInfo.version, + status: 'success', + }); + sendAlertTelemetryEvents( + appContextService.getLogger(), + appContextService.getTelemetryEventsSender(), + { + package_name: packageInfo.name, + current_version: packagePolicy.package.version, + new_version: packageInfo.version, + status: 'success', + } + ); + } + await removeOldAssets({ soClient, pkgName: packageInfo.name, @@ -634,14 +647,28 @@ class PackagePolicyService { const hasErrors = 'errors' in updatedPackagePolicy; - if (hasErrors) { - createUpgradeUsage(soClient, { - package_name: packageInfo.name, - current_version: packagePolicy.package.version, - new_version: packageInfo.version, - status: hasErrors ? 'failure' : 'success', - error: updatedPackagePolicy.errors, - }); + if (packagePolicy.package.version !== packageInfo.version) { + if (hasErrors) { + createUpgradeUsage(soClient, { + package_name: packageInfo.name, + current_version: packagePolicy.package.version, + new_version: packageInfo.version, + status: hasErrors ? 'failure' : 'success', + error: updatedPackagePolicy.errors, + }); + } + // TODO move inside hasErrors + sendAlertTelemetryEvents( + appContextService.getLogger(), + appContextService.getTelemetryEventsSender(), + { + package_name: packageInfo.name, + current_version: packagePolicy.package.version, + new_version: packageInfo.version, + status: hasErrors ? 'failure' : 'success', + error: updatedPackagePolicy.errors, + } + ); } return { diff --git a/x-pack/plugins/fleet/server/services/upgrade_usage.ts b/x-pack/plugins/fleet/server/services/upgrade_usage.ts index 46780fc87ee08..5c849e4cb65e6 100644 --- a/x-pack/plugins/fleet/server/services/upgrade_usage.ts +++ b/x-pack/plugins/fleet/server/services/upgrade_usage.ts @@ -6,8 +6,10 @@ */ import type { SavedObjectsClientContract } from 'kibana/server'; +import type { Logger } from 'src/core/server'; import type { PackagePolicyUpgradeUsage } from '../collectors/package_upgrade_collectors'; +import type { TelemetryEventsSender } from '../telemetry/sender'; export function createUpgradeUsage( soClient: SavedObjectsClientContract, @@ -24,3 +26,24 @@ export function deleteUpgradeUsages(soClient: SavedObjectsClientContract, ids: s soClient.delete('package-policy-upgrade-telemetry', id); } } + +export function sendAlertTelemetryEvents( + logger: Logger, + eventsTelemetry: TelemetryEventsSender | undefined, + upgradeUsage: PackagePolicyUpgradeUsage +) { + if (eventsTelemetry === undefined) { + return; + } + + try { + eventsTelemetry.queueTelemetryEvents([ + { + package_policy_upgrade: { ...upgradeUsage }, + id: `${upgradeUsage.package_name}_${upgradeUsage.current_version}_${upgradeUsage.new_version}_${upgradeUsage.status}`, + }, + ]); + } catch (exc) { + logger.error(`queing telemetry events failed ${exc}`); + } +} diff --git a/x-pack/plugins/fleet/server/telemetry/__mocks__/index.ts b/x-pack/plugins/fleet/server/telemetry/__mocks__/index.ts new file mode 100644 index 0000000000000..b4fbeeda4ef4b --- /dev/null +++ b/x-pack/plugins/fleet/server/telemetry/__mocks__/index.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ConcreteTaskInstance } from '../../../../../task_manager/server'; +import { TaskStatus } from '../../../../../task_manager/server'; +import type { TelemetryEventsSender } from '../sender'; +import type { TelemetryReceiver } from '../receiver'; +import type { SecurityTelemetryTaskConfig } from '../task'; +import type { PackagePolicy } from '../../../../../fleet/common/types/models/package_policy'; + +/** + * Creates a mocked Telemetry Events Sender + */ +export const createMockTelemetryEventsSender = ( + enableTelemetry?: boolean +): jest.Mocked => { + return { + setup: jest.fn(), + start: jest.fn(), + stop: jest.fn(), + fetchTelemetryUrl: jest.fn(), + queueTelemetryEvents: jest.fn(), + processEvents: jest.fn(), + isTelemetryOptedIn: jest.fn().mockReturnValue(enableTelemetry ?? jest.fn()), + sendIfDue: jest.fn(), + sendEvents: jest.fn(), + } as unknown as jest.Mocked; +}; + +export const createMockTelemetryReceiver = ( + diagnosticsAlert?: unknown +): jest.Mocked => { + return { + start: jest.fn(), + fetchClusterInfo: jest.fn(), + fetchLicenseInfo: jest.fn(), + copyLicenseFields: jest.fn(), + fetchFleetAgents: jest.fn(), + fetchDiagnosticAlerts: jest.fn().mockReturnValue(diagnosticsAlert ?? jest.fn()), + fetchEndpointMetrics: jest.fn(), + fetchEndpointPolicyResponses: jest.fn(), + fetchTrustedApplications: jest.fn(), + fetchEndpointList: jest.fn(), + fetchDetectionRules: jest.fn().mockReturnValue({ body: null }), + } as unknown as jest.Mocked; +}; + +/** + * Creates a mocked package policy + */ +export const createMockPackagePolicy = (): jest.Mocked => { + return { + id: jest.fn(), + inputs: jest.fn(), + version: jest.fn(), + revision: jest.fn(), + updated_at: jest.fn(), + updated_by: jest.fn(), + created_at: jest.fn(), + created_by: jest.fn(), + } as unknown as jest.Mocked; +}; + +/** + * Creates a mocked Security Telemetry Task Config + */ +export const createMockSecurityTelemetryTask = ( + testType?: string, + testLastTimestamp?: string +): jest.Mocked => { + return { + type: testType, + title: 'test title', + interval: '0m', + timeout: '0m', + version: '0.0.0', + getLastExecutionTime: jest.fn().mockReturnValue(testLastTimestamp ?? jest.fn()), + runTask: jest.fn(), + } as unknown as jest.Mocked; +}; + +/** + * Creates a mocked Task Instance + */ +export const createMockTaskInstance = (testId: string, testType: string): ConcreteTaskInstance => { + return { + id: testId, + runAt: new Date(), + attempts: 0, + ownerId: '', + status: TaskStatus.Running, + startedAt: new Date(), + scheduledAt: new Date(), + retryAt: new Date(), + params: {}, + state: {}, + taskType: testType, + } as ConcreteTaskInstance; +}; diff --git a/x-pack/plugins/fleet/server/telemetry/receiver.ts b/x-pack/plugins/fleet/server/telemetry/receiver.ts new file mode 100644 index 0000000000000..d51f9654808ce --- /dev/null +++ b/x-pack/plugins/fleet/server/telemetry/receiver.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger, CoreStart, ElasticsearchClient } from 'src/core/server'; + +import type { ESLicense, ESClusterInfo } from './types'; + +export class TelemetryReceiver { + private readonly logger: Logger; + private esClient?: ElasticsearchClient; + + constructor(logger: Logger) { + this.logger = logger.get('telemetry_events'); + } + + public async start(core?: CoreStart) { + this.esClient = core?.elasticsearch.client.asInternalUser; + } + + public async fetchClusterInfo(): Promise { + if (this.esClient === undefined || this.esClient === null) { + throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation'); + } + + const { body } = await this.esClient.info(); + return body; + } + + public async fetchLicenseInfo(): Promise { + if (this.esClient === undefined || this.esClient === null) { + throw Error('elasticsearch client is unavailable: cannot retrieve license information'); + } + + try { + const ret = ( + await this.esClient.transport.request({ + method: 'GET', + path: '/_license', + querystring: { + local: true, + // For versions >= 7.6 and < 8.0, this flag is needed otherwise 'platinum' is returned for 'enterprise' license. + accept_enterprise: 'true', + }, + }) + ).body as Promise<{ license: ESLicense }>; + + return (await ret).license; + } catch (err) { + this.logger.debug(`failed retrieving license: ${err}`); + return undefined; + } + } + + public copyLicenseFields(lic: ESLicense) { + return { + uid: lic.uid, + status: lic.status, + type: lic.type, + ...(lic.issued_to ? { issued_to: lic.issued_to } : {}), + ...(lic.issuer ? { issuer: lic.issuer } : {}), + }; + } +} diff --git a/x-pack/plugins/fleet/server/telemetry/sender.test.ts b/x-pack/plugins/fleet/server/telemetry/sender.test.ts new file mode 100644 index 0000000000000..49f98d3401267 --- /dev/null +++ b/x-pack/plugins/fleet/server/telemetry/sender.test.ts @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable dot-notation */ +import { URL } from 'url'; + +import { loggingSystemMock } from 'src/core/server/mocks'; + +import { usageCountersServiceMock } from 'src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock'; + +import { TelemetryEventsSender } from './sender'; + +describe('TelemetryEventsSender', () => { + let logger: ReturnType; + const usageCountersServiceSetup = usageCountersServiceMock.createSetupContract(); + const telemetryUsageCounter = usageCountersServiceSetup.createUsageCounter( + 'testTelemetryUsageCounter' + ); + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + }); + + describe('queueTelemetryEvents', () => { + it('queues two events', () => { + const sender = new TelemetryEventsSender(logger); + sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); + expect(sender['queue'].length).toBe(2); + }); + + it('queues more than maxQueueSize events', () => { + const sender = new TelemetryEventsSender(logger); + sender['maxQueueSize'] = 5; + sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); + sender.queueTelemetryEvents([{ 'event.kind': '3' }, { 'event.kind': '4' }]); + sender.queueTelemetryEvents([{ 'event.kind': '5' }, { 'event.kind': '6' }]); + sender.queueTelemetryEvents([{ 'event.kind': '7' }, { 'event.kind': '8' }]); + expect(sender['queue'].length).toBe(5); + }); + + it('empties the queue when sending', async () => { + const sender = new TelemetryEventsSender(logger); + sender['telemetryStart'] = { + getIsOptedIn: jest.fn(async () => true), + }; + sender['telemetrySetup'] = { + getTelemetryUrl: jest.fn(async () => new URL('https://telemetry.elastic.co')), + }; + sender['telemetryUsageCounter'] = telemetryUsageCounter; + sender['sendEvents'] = jest.fn(async () => { + sender['telemetryUsageCounter']?.incrementCounter({ + counterName: 'test_counter', + counterType: 'invoked', + incrementBy: 1, + }); + }); + + sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); + expect(sender['queue'].length).toBe(2); + await sender['sendIfDue'](); + expect(sender['queue'].length).toBe(0); + expect(sender['sendEvents']).toBeCalledTimes(1); + sender.queueTelemetryEvents([{ 'event.kind': '3' }, { 'event.kind': '4' }]); + sender.queueTelemetryEvents([{ 'event.kind': '5' }, { 'event.kind': '6' }]); + expect(sender['queue'].length).toBe(4); + await sender['sendIfDue'](); + expect(sender['queue'].length).toBe(0); + expect(sender['sendEvents']).toBeCalledTimes(2); + expect(sender['telemetryUsageCounter'].incrementCounter).toBeCalledTimes(2); + }); + + it("shouldn't send when telemetry is disabled", async () => { + const sender = new TelemetryEventsSender(logger); + sender['sendEvents'] = jest.fn(); + const telemetryStart = { + getIsOptedIn: jest.fn(async () => false), + }; + sender['telemetryStart'] = telemetryStart; + + sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); + expect(sender['queue'].length).toBe(2); + await sender['sendIfDue'](); + + expect(sender['queue'].length).toBe(0); + expect(sender['sendEvents']).toBeCalledTimes(0); + }); + }); +}); + +describe('getV3UrlFromV2', () => { + let logger: ReturnType; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + }); + + it('should return prod url', () => { + const sender = new TelemetryEventsSender(logger); + expect( + sender.getV3UrlFromV2('https://telemetry.elastic.co/xpack/v2/send', 'alerts-endpoint') + ).toBe('https://telemetry.elastic.co/v3/send/alerts-endpoint'); + }); + + it('should return staging url', () => { + const sender = new TelemetryEventsSender(logger); + expect( + sender.getV3UrlFromV2('https://telemetry-staging.elastic.co/xpack/v2/send', 'alerts-endpoint') + ).toBe('https://telemetry-staging.elastic.co/v3-dev/send/alerts-endpoint'); + }); + + it('should support ports and auth', () => { + const sender = new TelemetryEventsSender(logger); + expect( + sender.getV3UrlFromV2('http://user:pass@myproxy.local:1337/xpack/v2/send', 'alerts-endpoint') + ).toBe('http://user:pass@myproxy.local:1337/v3/send/alerts-endpoint'); + }); +}); diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts new file mode 100644 index 0000000000000..a6d2e28fbc586 --- /dev/null +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -0,0 +1,232 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { URL } from 'url'; + +import { cloneDeep } from 'lodash'; +import axios from 'axios'; +import type { Logger } from 'src/core/server'; +import type { TelemetryPluginStart, TelemetryPluginSetup } from 'src/plugins/telemetry/server'; + +import type { TelemetryEvent } from './types'; +import type { TelemetryReceiver } from './receiver'; + +export const TELEMETRY_MAX_BUFFER_SIZE = 100; +export const FLEET_CHANNEL_NAME = 'fleet-stats'; + +export class TelemetryEventsSender { + private readonly initialCheckDelayMs = 10 * 1000; + private readonly checkIntervalMs = 10 * 1000; + private readonly logger: Logger; + private maxQueueSize = TELEMETRY_MAX_BUFFER_SIZE; + private telemetryStart?: TelemetryPluginStart; + private telemetrySetup?: TelemetryPluginSetup; + private intervalId?: NodeJS.Timeout; + private isSending = false; + private receiver: TelemetryReceiver | undefined; + private queue: TelemetryEvent[] = []; + private isOptedIn?: boolean = true; // Assume true until the first check + + constructor(logger: Logger) { + this.logger = logger.get('telemetry_events'); + } + + public setup(telemetrySetup?: TelemetryPluginSetup) { + this.telemetrySetup = telemetrySetup; + } + + public start(telemetryStart?: TelemetryPluginStart, receiver?: TelemetryReceiver) { + this.telemetryStart = telemetryStart; + this.receiver = receiver; + + this.logger.debug(`Starting local task`); + setTimeout(() => { + this.sendIfDue(); + this.intervalId = setInterval(() => this.sendIfDue(), this.checkIntervalMs); + }, this.initialCheckDelayMs); + } + + public stop() { + if (this.intervalId) { + clearInterval(this.intervalId); + } + } + + public queueTelemetryEvents(events: TelemetryEvent[]) { + const qlength = this.queue.length; + + if (events.length === 0) { + return; + } + + events = events.filter((event) => !this.queue.find((qItem) => qItem.id === event.id)); + + this.logger.info(`Queue events ` + JSON.stringify(events)); + + if (qlength >= this.maxQueueSize) { + // we're full already + return; + } + + if (events.length > this.maxQueueSize - qlength) { + this.queue.push(...events.slice(0, this.maxQueueSize - qlength)); + } else { + this.queue.push(...events); + } + } + + public async isTelemetryOptedIn() { + this.isOptedIn = await this.telemetryStart?.getIsOptedIn(); + return this.isOptedIn === true; + } + + private async sendIfDue() { + if (this.isSending) { + return; + } + + if (this.queue.length === 0) { + return; + } + + try { + this.isSending = true; + + this.isOptedIn = true; // await this.isTelemetryOptedIn(); + if (!this.isOptedIn) { + this.logger.info(`Telemetry is not opted-in.`); + this.queue = []; + this.isSending = false; + return; + } + + const [telemetryUrl, clusterInfo, licenseInfo] = await Promise.all([ + this.fetchTelemetryUrl(FLEET_CHANNEL_NAME), + this.receiver?.fetchClusterInfo(), + this.receiver?.fetchLicenseInfo(), + ]); + + this.logger.debug(`Telemetry URL: ${telemetryUrl}`); + this.logger.debug( + `cluster_uuid: ${clusterInfo?.cluster_uuid} cluster_name: ${clusterInfo?.cluster_name}` + ); + + const toSend: TelemetryEvent[] = cloneDeep(this.queue).map((event) => ({ + ...event, + ...(licenseInfo ? { license: this.receiver?.copyLicenseFields(licenseInfo) } : {}), + cluster_uuid: clusterInfo?.cluster_uuid, + cluster_name: clusterInfo?.cluster_name, + })); + this.queue = []; + + this.logger.info('sending events to: ' + telemetryUrl); + this.logger.info(JSON.stringify(toSend)); + + await this.sendEvents( + toSend, + telemetryUrl, + clusterInfo?.cluster_uuid, + clusterInfo?.version?.number, + licenseInfo?.uid + ); + } catch (err) { + this.logger.warn(`Error sending telemetry events data: ${err}`); + this.queue = []; + } + this.isSending = false; + } + + /** + * This function sends events to the elastic telemetry channel. Caution is required + * because it does no allowlist filtering at send time. The function call site is + * responsible for ensuring sure no sensitive material is in telemetry events. + * + * @param channel the elastic telemetry channel + * @param toSend telemetry events + */ + // public async sendOnDemand(channel: string, toSend: unknown[]) { + // try { + // const [telemetryUrl, clusterInfo, licenseInfo] = await Promise.all([ + // this.fetchTelemetryUrl(channel), + // this.receiver?.fetchClusterInfo(), + // this.receiver?.fetchLicenseInfo(), + // ]); + + // this.logger.debug(`Telemetry URL: ${telemetryUrl}`); + // this.logger.debug( + // `cluster_uuid: ${clusterInfo?.cluster_uuid} cluster_name: ${clusterInfo?.cluster_name}` + // ); + + // await this.sendEvents( + // toSend, + // telemetryUrl, + // channel, + // clusterInfo?.cluster_uuid, + // clusterInfo?.version?.number, + // licenseInfo?.uid + // ); + // } catch (err) { + // this.logger.warn(`Error sending telemetry events data: ${err}`); + // } + // } + + private async fetchTelemetryUrl(channel: string): Promise { + const telemetryUrl = await this.telemetrySetup?.getTelemetryUrl(); + if (!telemetryUrl) { + throw Error("Couldn't get telemetry URL"); + } + return this.getV3UrlFromV2(telemetryUrl.toString(), channel); + } + + // Forms URLs like: + // https://telemetry.elastic.co/v3/send/my-channel-name or + // https://telemetry-staging.elastic.co/v3-dev/send/my-channel-name + public getV3UrlFromV2(v2url: string, channel: string): string { + const url = new URL(v2url); + if (!url.hostname.includes('staging')) { + url.pathname = `/v3/send/${channel}`; + } else { + url.pathname = `/v3-dev/send/${channel}`; + } + return url.toString(); + } + + private async sendEvents( + events: unknown[], + telemetryUrl: string, + clusterUuid: string | undefined, + clusterVersionNumber: string | undefined, + licenseId: string | undefined + ) { + const ndjson = this.transformDataToNdjson(events); + + try { + const resp = await axios.post(telemetryUrl, ndjson, { + headers: { + 'Content-Type': 'application/x-ndjson', + 'X-Elastic-Cluster-ID': clusterUuid, + 'X-Elastic-Stack-Version': clusterVersionNumber ? clusterVersionNumber : '7.16.0', + ...(licenseId ? { 'X-Elastic-License-ID': licenseId } : {}), + }, + }); + this.logger.debug(`Events sent!. Response: ${resp.status} ${JSON.stringify(resp.data)}`); + } catch (err) { + this.logger.warn( + `Error sending events: ${err.response.status} ${JSON.stringify(err.response.data)}` + ); + } + } + + private transformDataToNdjson = (data: unknown[]): string => { + if (data.length !== 0) { + const dataString = data.map((dataItem) => JSON.stringify(dataItem)).join('\n'); + return `${dataString}\n`; + } else { + return ''; + } + }; +} diff --git a/x-pack/plugins/fleet/server/telemetry/telemetry/helpers.ts b/x-pack/plugins/fleet/server/telemetry/telemetry/helpers.ts new file mode 100644 index 0000000000000..3ea48fc652e9f --- /dev/null +++ b/x-pack/plugins/fleet/server/telemetry/telemetry/helpers.ts @@ -0,0 +1,207 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; + +import type { PackagePolicy } from '../../../common/types/models/package_policy'; + +import type { TrustedApp } from '../../../common/endpoint/types'; + +import { copyAllowlistedFields, exceptionListEventFields } from './filters'; +import type { ExceptionListItem, ListTemplate, TelemetryEvent } from './types'; +import { + LIST_DETECTION_RULE_EXCEPTION, + LIST_ENDPOINT_EXCEPTION, + LIST_ENDPOINT_EVENT_FILTER, + LIST_TRUSTED_APPLICATION, +} from './constants'; + +/** + * Determines the when the last run was in order to execute to. + * + * @param executeTo + * @param lastExecutionTimestamp + * @returns the timestamp to search from + */ +export const getPreviousDiagTaskTimestamp = ( + executeTo: string, + lastExecutionTimestamp?: string +) => { + if (lastExecutionTimestamp === undefined) { + return moment(executeTo).subtract(5, 'minutes').toISOString(); + } + + if (moment(executeTo).diff(lastExecutionTimestamp, 'minutes') >= 10) { + return moment(executeTo).subtract(10, 'minutes').toISOString(); + } + + return lastExecutionTimestamp; +}; + +/** + * Determines the when the last run was in order to execute to. + * + * @param executeTo + * @param lastExecutionTimestamp + * @returns the timestamp to search from + */ +export const getPreviousDailyTaskTimestamp = ( + executeTo: string, + lastExecutionTimestamp?: string +) => { + if (lastExecutionTimestamp === undefined) { + return moment(executeTo).subtract(24, 'hours').toISOString(); + } + + if (moment(executeTo).diff(lastExecutionTimestamp, 'hours') >= 24) { + return moment(executeTo).subtract(24, 'hours').toISOString(); + } + + return lastExecutionTimestamp; +}; + +/** + * Chunks an Array into an Array> + * This is to prevent overloading the telemetry channel + user resources + * + * @param telemetryRecords + * @param batchSize + * @returns the batch of records + */ +export const batchTelemetryRecords = ( + telemetryRecords: unknown[], + batchSize: number +): unknown[][] => + [...Array(Math.ceil(telemetryRecords.length / batchSize))].map((_) => + telemetryRecords.splice(0, batchSize) + ); + +/** + * User defined type guard for PackagePolicy + * + * @param data the union type of package policies + * @returns type confirmation + */ +export function isPackagePolicyList( + data: string[] | PackagePolicy[] | undefined +): data is PackagePolicy[] { + if (data === undefined || data.length < 1) { + return false; + } + + return (data as PackagePolicy[])[0].inputs !== undefined; +} + +/** + * Maps trusted application to shared telemetry object + * + * @param exceptionListItem + * @returns collection of trusted applications + */ +export const trustedApplicationToTelemetryEntry = (trustedApplication: TrustedApp) => { + return { + id: trustedApplication.id, + name: trustedApplication.name, + created_at: trustedApplication.created_at, + updated_at: trustedApplication.updated_at, + entries: trustedApplication.entries, + os_types: [trustedApplication.os], + } as ExceptionListItem; +}; + +/** + * Maps endpoint lists to shared telemetry object + * + * @param exceptionListItem + * @returns collection of endpoint exceptions + */ +export const exceptionListItemToTelemetryEntry = (exceptionListItem: ExceptionListItemSchema) => { + return { + id: exceptionListItem.id, + name: exceptionListItem.name, + created_at: exceptionListItem.created_at, + updated_at: exceptionListItem.updated_at, + entries: exceptionListItem.entries, + os_types: exceptionListItem.os_types, + } as ExceptionListItem; +}; + +/** + * Maps detection rule exception list items to shared telemetry object + * + * @param exceptionListItem + * @param ruleVersion + * @returns collection of detection rule exceptions + */ +export const ruleExceptionListItemToTelemetryEvent = ( + exceptionListItem: ExceptionListItemSchema, + ruleVersion: number +) => { + return { + id: exceptionListItem.item_id, + name: exceptionListItem.description, + rule_version: ruleVersion, + created_at: exceptionListItem.created_at, + updated_at: exceptionListItem.updated_at, + entries: exceptionListItem.entries, + os_types: exceptionListItem.os_types, + } as ExceptionListItem; +}; + +/** + * Consructs the list telemetry schema from a collection of endpoint exceptions + * + * @param listData + * @param listType + * @returns lists telemetry schema + */ +export const templateExceptionList = (listData: ExceptionListItem[], listType: string) => { + return listData.map((item) => { + const template: ListTemplate = { + '@timestamp': new Date().getTime(), + }; + + // cast exception list type to a TelemetryEvent for allowlist filtering + const filteredListItem = copyAllowlistedFields( + exceptionListEventFields, + item as unknown as TelemetryEvent + ); + + if (listType === LIST_DETECTION_RULE_EXCEPTION) { + template.detection_rule = filteredListItem; + return template; + } + + if (listType === LIST_TRUSTED_APPLICATION) { + template.trusted_application = filteredListItem; + return template; + } + + if (listType === LIST_ENDPOINT_EXCEPTION) { + template.endpoint_exception = filteredListItem; + return template; + } + + if (listType === LIST_ENDPOINT_EVENT_FILTER) { + template.endpoint_event_filter = filteredListItem; + return template; + } + + return null; + }); +}; + +/** + * Convert counter label list to kebab case + * + * @param label_list the list of labels to create standardized UsageCounter from + * @returns a string label for usage in the UsageCounter + */ +export function createUsageCounterLabel(labelList: string[]): string { + return labelList.join('-'); +} diff --git a/x-pack/plugins/fleet/server/telemetry/telemetry/tasks/endpoint.ts b/x-pack/plugins/fleet/server/telemetry/telemetry/tasks/endpoint.ts new file mode 100644 index 0000000000000..a60c4940cdaaa --- /dev/null +++ b/x-pack/plugins/fleet/server/telemetry/telemetry/tasks/endpoint.ts @@ -0,0 +1,269 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from 'src/core/server'; + +import type { TelemetryEventsSender } from '../sender'; +import type { + EndpointMetricsAggregation, + EndpointPolicyResponseAggregation, + EndpointPolicyResponseDocument, +} from '../types'; +import type { TelemetryReceiver } from '../receiver'; +import type { TaskExecutionPeriod } from '../task'; +import { + batchTelemetryRecords, + getPreviousDailyTaskTimestamp, + isPackagePolicyList, +} from '../helpers'; +import type { PolicyData } from '../../../../common/endpoint/types'; +import { FLEET_ENDPOINT_PACKAGE } from '../../../../common'; +import { TELEMETRY_CHANNEL_ENDPOINT_META } from '../constants'; + +// Endpoint agent uses this Policy ID while it's installing. +const DefaultEndpointPolicyIdToIgnore = '00000000-0000-0000-0000-000000000000'; + +const EmptyFleetAgentResponse = { + agents: [], + total: 0, + page: 0, + perPage: 0, +}; + +export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { + return { + type: 'security:endpoint-meta-telemetry', + title: 'Security Solution Telemetry Endpoint Metrics and Info task', + interval: '24h', + timeout: '5m', + version: '1.0.0', + getLastExecutionTime: getPreviousDailyTaskTimestamp, + runTask: async ( + taskId: string, + logger: Logger, + receiver: TelemetryReceiver, + sender: TelemetryEventsSender, + taskExecutionPeriod: TaskExecutionPeriod + ) => { + if (!taskExecutionPeriod.last) { + throw new Error('last execution timestamp is required'); + } + + const endpointData = await fetchEndpointData( + receiver, + taskExecutionPeriod.last, + taskExecutionPeriod.current + ); + + /** STAGE 1 - Fetch Endpoint Agent Metrics + * + * Reads Endpoint Agent metrics out of the `.ds-metrics-endpoint.metrics` data stream + * and buckets them by Endpoint Agent id and sorts by the top hit. The EP agent will + * report its metrics once per day OR every time a policy change has occured. If + * a metric document(s) exists for an EP agent we map to fleet agent and policy + */ + if (endpointData.endpointMetrics === undefined) { + logger.debug(`no endpoint metrics to report`); + return 0; + } + + const { body: endpointMetricsResponse } = endpointData.endpointMetrics as unknown as { + body: EndpointMetricsAggregation; + }; + + if (endpointMetricsResponse.aggregations === undefined) { + logger.debug(`no endpoint metrics to report`); + return 0; + } + + const endpointMetrics = endpointMetricsResponse.aggregations.endpoint_agents.buckets.map( + (epMetrics) => { + return { + endpoint_agent: epMetrics.latest_metrics.hits.hits[0]._source.agent.id, + endpoint_version: epMetrics.latest_metrics.hits.hits[0]._source.agent.version, + endpoint_metrics: epMetrics.latest_metrics.hits.hits[0]._source, + }; + } + ); + + /** STAGE 2 - Fetch Fleet Agent Config + * + * As the policy id + policy version does not exist on the Endpoint Metrics document + * we need to fetch information about the Fleet Agent and sync the metrics document + * with the Agent's policy data. + * + */ + const agentsResponse = endpointData.fleetAgentsResponse; + + if (agentsResponse === undefined) { + logger.debug('no fleet agent information available'); + return 0; + } + + const fleetAgents = agentsResponse.agents.reduce((cache, agent) => { + if (agent.id === DefaultEndpointPolicyIdToIgnore) { + return cache; + } + + if (agent.policy_id !== null && agent.policy_id !== undefined) { + cache.set(agent.id, agent.policy_id); + } + + return cache; + }, new Map()); + + const endpointPolicyCache = new Map(); + for (const policyInfo of fleetAgents.values()) { + if ( + policyInfo !== null && + policyInfo !== undefined && + !endpointPolicyCache.has(policyInfo) + ) { + const agentPolicy = await receiver.fetchPolicyConfigs(policyInfo); + const packagePolicies = agentPolicy?.package_policies; + + if (packagePolicies !== undefined && isPackagePolicyList(packagePolicies)) { + packagePolicies + .map((pPolicy) => pPolicy as PolicyData) + .forEach((pPolicy) => { + if (pPolicy.inputs[0].config !== undefined) { + pPolicy.inputs.forEach((input) => { + if ( + input.type === FLEET_ENDPOINT_PACKAGE && + input.config !== undefined && + policyInfo !== undefined + ) { + endpointPolicyCache.set(policyInfo, pPolicy); + } + }); + } + }); + } + } + } + + /** STAGE 3 - Fetch Endpoint Policy Responses + * + * Reads Endpoint Agent policy responses out of the `.ds-metrics-endpoint.policy*` data + * stream and creates a local K/V structure that stores the policy response (V) with + * the Endpoint Agent Id (K). A value will only exist if there has been a endpoint + * enrolled in the last 24 hours OR a policy change has occurred. We only send + * non-successful responses. If the field is null, we assume no responses in + * the last 24h or no failures/warnings in the policy applied. + * + */ + const { body: failedPolicyResponses } = endpointData.epPolicyResponse as unknown as { + body: EndpointPolicyResponseAggregation; + }; + + // If there is no policy responses in the 24h > now then we will continue + const policyResponses = failedPolicyResponses.aggregations + ? failedPolicyResponses.aggregations.policy_responses.buckets.reduce( + (cache, endpointAgentId) => { + const doc = endpointAgentId.latest_response.hits.hits[0]; + cache.set(endpointAgentId.key, doc); + return cache; + }, + new Map() + ) + : new Map(); + + /** STAGE 4 - Create the telemetry log records + * + * Iterates through the endpoint metrics documents at STAGE 1 and joins them together + * to form the telemetry log that is sent back to Elastic Security developers to + * make improvements to the product. + * + */ + try { + const telemetryPayloads = endpointMetrics.map((endpoint) => { + let policyConfig = null; + let failedPolicy = null; + + const fleetAgentId = endpoint.endpoint_metrics.elastic.agent.id; + const endpointAgentId = endpoint.endpoint_agent; + + const policyInformation = fleetAgents.get(fleetAgentId); + if (policyInformation) { + policyConfig = endpointPolicyCache.get(policyInformation) || null; + + if (policyConfig) { + failedPolicy = policyResponses.get(endpointAgentId); + } + } + + const { cpu, memory, uptime } = endpoint.endpoint_metrics.Endpoint.metrics; + + return { + '@timestamp': taskExecutionPeriod.current, + endpoint_id: endpointAgentId, + endpoint_version: endpoint.endpoint_version, + endpoint_package_version: policyConfig?.package?.version || null, + endpoint_metrics: { + cpu: cpu.endpoint, + memory: memory.endpoint.private, + uptime, + }, + endpoint_meta: { + os: endpoint.endpoint_metrics.host.os, + }, + policy_config: policyConfig !== null ? policyConfig?.inputs[0].config.policy : {}, + policy_response: + failedPolicy !== null && failedPolicy !== undefined + ? { + agent_policy_status: failedPolicy._source.event.agent_id_status, + manifest_version: + failedPolicy._source.Endpoint.policy.applied.artifacts.global.version, + status: failedPolicy._source.Endpoint.policy.applied.status, + actions: failedPolicy._source.Endpoint.policy.applied.actions + .map((action) => (action.status !== 'success' ? action : null)) + .filter((action) => action !== null), + } + : {}, + telemetry_meta: { + metrics_timestamp: endpoint.endpoint_metrics['@timestamp'], + }, + }; + }); + + /** + * STAGE 5 - Send the documents + * + * Send the documents in a batches of maxTelemetryBatch + */ + batchTelemetryRecords(telemetryPayloads, maxTelemetryBatch).forEach((telemetryBatch) => + sender.sendOnDemand(TELEMETRY_CHANNEL_ENDPOINT_META, telemetryBatch) + ); + return telemetryPayloads.length; + } catch (err) { + logger.warn('could not complete endpoint alert telemetry task'); + return 0; + } + }, + }; +} + +async function fetchEndpointData( + receiver: TelemetryReceiver, + executeFrom: string, + executeTo: string +) { + const [fleetAgentsResponse, epMetricsResponse, policyResponse] = await Promise.allSettled([ + receiver.fetchFleetAgents(), + receiver.fetchEndpointMetrics(executeFrom, executeTo), + receiver.fetchEndpointPolicyResponses(executeFrom, executeTo), + ]); + + return { + fleetAgentsResponse: + fleetAgentsResponse.status === 'fulfilled' + ? fleetAgentsResponse.value + : EmptyFleetAgentResponse, + endpointMetrics: epMetricsResponse.status === 'fulfilled' ? epMetricsResponse.value : undefined, + epPolicyResponse: policyResponse.status === 'fulfilled' ? policyResponse.value : undefined, + }; +} diff --git a/x-pack/plugins/fleet/server/telemetry/types.ts b/x-pack/plugins/fleet/server/telemetry/types.ts new file mode 100644 index 0000000000000..917b9b9bfc1ba --- /dev/null +++ b/x-pack/plugins/fleet/server/telemetry/types.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +type BaseSearchTypes = string | number | boolean | object; +export type SearchTypes = BaseSearchTypes | BaseSearchTypes[] | undefined; + +// For getting cluster info. Copied from telemetry_collection/get_cluster_info.ts +export interface ESClusterInfo { + cluster_uuid: string; + cluster_name: string; + version?: { + number: string; + build_flavor: string; + build_type: string; + build_hash: string; + build_date: string; + build_snapshot?: boolean; + lucene_version: string; + minimum_wire_compatibility_version: string; + minimum_index_compatibility_version: string; + }; +} + +// From https://www.elastic.co/guide/en/elasticsearch/reference/current/get-license.html +export interface ESLicense { + status: string; + uid: string; + type: string; + issue_date?: string; + issue_date_in_millis?: number; + expiry_date?: string; + expirty_date_in_millis?: number; + max_nodes?: number; + issued_to?: string; + issuer?: string; + start_date_in_millis?: number; +} + +export interface TelemetryEvent { + [key: string]: SearchTypes; + '@timestamp'?: string; + cluster_name?: string; + cluster_uuid?: string; + license?: ESLicense; +} From 1d2deffef9d90f409ac10048f3d78ea9fcfc215d Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Mon, 18 Oct 2021 12:56:42 +0200 Subject: [PATCH 03/27] fixed tests and types --- x-pack/plugins/fleet/server/mocks/index.ts | 3 +- .../fleet/server/telemetry/__mocks__/index.ts | 57 ---- .../fleet/server/telemetry/sender.test.ts | 16 +- .../plugins/fleet/server/telemetry/sender.ts | 4 +- .../server/telemetry/telemetry/helpers.ts | 207 -------------- .../telemetry/telemetry/tasks/endpoint.ts | 269 ------------------ 6 files changed, 6 insertions(+), 550 deletions(-) delete mode 100644 x-pack/plugins/fleet/server/telemetry/telemetry/helpers.ts delete mode 100644 x-pack/plugins/fleet/server/telemetry/telemetry/tasks/endpoint.ts diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 8da503601a4b0..e0f8e680dce1b 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -19,6 +19,7 @@ import { securityMock } from '../../../security/server/mocks'; import type { PackagePolicyServiceInterface } from '../services/package_policy'; import type { AgentPolicyServiceInterface, AgentService } from '../services'; import type { FleetAppContext } from '../plugin'; +import { createMockTelemetryEventsSender } from '../telemetry/__mocks__'; // Export all mocks from artifacts export * from '../services/artifacts/mocks'; @@ -49,7 +50,7 @@ export const createAppContextStartContractMock = (): FleetAppContext => { config$, kibanaVersion: '8.0.0', kibanaBranch: 'master', - telemetryEventsSender: undefined, + telemetryEventsSender: createMockTelemetryEventsSender(), }; }; diff --git a/x-pack/plugins/fleet/server/telemetry/__mocks__/index.ts b/x-pack/plugins/fleet/server/telemetry/__mocks__/index.ts index b4fbeeda4ef4b..0eeac6fd1d7e5 100644 --- a/x-pack/plugins/fleet/server/telemetry/__mocks__/index.ts +++ b/x-pack/plugins/fleet/server/telemetry/__mocks__/index.ts @@ -5,12 +5,8 @@ * 2.0. */ -import type { ConcreteTaskInstance } from '../../../../../task_manager/server'; -import { TaskStatus } from '../../../../../task_manager/server'; import type { TelemetryEventsSender } from '../sender'; import type { TelemetryReceiver } from '../receiver'; -import type { SecurityTelemetryTaskConfig } from '../task'; -import type { PackagePolicy } from '../../../../../fleet/common/types/models/package_policy'; /** * Creates a mocked Telemetry Events Sender @@ -48,56 +44,3 @@ export const createMockTelemetryReceiver = ( fetchDetectionRules: jest.fn().mockReturnValue({ body: null }), } as unknown as jest.Mocked; }; - -/** - * Creates a mocked package policy - */ -export const createMockPackagePolicy = (): jest.Mocked => { - return { - id: jest.fn(), - inputs: jest.fn(), - version: jest.fn(), - revision: jest.fn(), - updated_at: jest.fn(), - updated_by: jest.fn(), - created_at: jest.fn(), - created_by: jest.fn(), - } as unknown as jest.Mocked; -}; - -/** - * Creates a mocked Security Telemetry Task Config - */ -export const createMockSecurityTelemetryTask = ( - testType?: string, - testLastTimestamp?: string -): jest.Mocked => { - return { - type: testType, - title: 'test title', - interval: '0m', - timeout: '0m', - version: '0.0.0', - getLastExecutionTime: jest.fn().mockReturnValue(testLastTimestamp ?? jest.fn()), - runTask: jest.fn(), - } as unknown as jest.Mocked; -}; - -/** - * Creates a mocked Task Instance - */ -export const createMockTaskInstance = (testId: string, testType: string): ConcreteTaskInstance => { - return { - id: testId, - runAt: new Date(), - attempts: 0, - ownerId: '', - status: TaskStatus.Running, - startedAt: new Date(), - scheduledAt: new Date(), - retryAt: new Date(), - params: {}, - state: {}, - taskType: testType, - } as ConcreteTaskInstance; -}; diff --git a/x-pack/plugins/fleet/server/telemetry/sender.test.ts b/x-pack/plugins/fleet/server/telemetry/sender.test.ts index 49f98d3401267..713ccdef8729d 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.test.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.test.ts @@ -10,16 +10,10 @@ import { URL } from 'url'; import { loggingSystemMock } from 'src/core/server/mocks'; -import { usageCountersServiceMock } from 'src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock'; - import { TelemetryEventsSender } from './sender'; describe('TelemetryEventsSender', () => { let logger: ReturnType; - const usageCountersServiceSetup = usageCountersServiceMock.createSetupContract(); - const telemetryUsageCounter = usageCountersServiceSetup.createUsageCounter( - 'testTelemetryUsageCounter' - ); beforeEach(() => { logger = loggingSystemMock.createLogger(); @@ -50,14 +44,7 @@ describe('TelemetryEventsSender', () => { sender['telemetrySetup'] = { getTelemetryUrl: jest.fn(async () => new URL('https://telemetry.elastic.co')), }; - sender['telemetryUsageCounter'] = telemetryUsageCounter; - sender['sendEvents'] = jest.fn(async () => { - sender['telemetryUsageCounter']?.incrementCounter({ - counterName: 'test_counter', - counterType: 'invoked', - incrementBy: 1, - }); - }); + sender['sendEvents'] = jest.fn(); sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); expect(sender['queue'].length).toBe(2); @@ -70,7 +57,6 @@ describe('TelemetryEventsSender', () => { await sender['sendIfDue'](); expect(sender['queue'].length).toBe(0); expect(sender['sendEvents']).toBeCalledTimes(2); - expect(sender['telemetryUsageCounter'].incrementCounter).toBeCalledTimes(2); }); it("shouldn't send when telemetry is disabled", async () => { diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index a6d2e28fbc586..aa7b2dc4803f2 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -63,7 +63,9 @@ export class TelemetryEventsSender { return; } - events = events.filter((event) => !this.queue.find((qItem) => qItem.id === event.id)); + events = events.filter( + (event) => !this.queue.find((qItem) => qItem.id && event.id && qItem.id === event.id) + ); this.logger.info(`Queue events ` + JSON.stringify(events)); diff --git a/x-pack/plugins/fleet/server/telemetry/telemetry/helpers.ts b/x-pack/plugins/fleet/server/telemetry/telemetry/helpers.ts deleted file mode 100644 index 3ea48fc652e9f..0000000000000 --- a/x-pack/plugins/fleet/server/telemetry/telemetry/helpers.ts +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import moment from 'moment'; -import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; - -import type { PackagePolicy } from '../../../common/types/models/package_policy'; - -import type { TrustedApp } from '../../../common/endpoint/types'; - -import { copyAllowlistedFields, exceptionListEventFields } from './filters'; -import type { ExceptionListItem, ListTemplate, TelemetryEvent } from './types'; -import { - LIST_DETECTION_RULE_EXCEPTION, - LIST_ENDPOINT_EXCEPTION, - LIST_ENDPOINT_EVENT_FILTER, - LIST_TRUSTED_APPLICATION, -} from './constants'; - -/** - * Determines the when the last run was in order to execute to. - * - * @param executeTo - * @param lastExecutionTimestamp - * @returns the timestamp to search from - */ -export const getPreviousDiagTaskTimestamp = ( - executeTo: string, - lastExecutionTimestamp?: string -) => { - if (lastExecutionTimestamp === undefined) { - return moment(executeTo).subtract(5, 'minutes').toISOString(); - } - - if (moment(executeTo).diff(lastExecutionTimestamp, 'minutes') >= 10) { - return moment(executeTo).subtract(10, 'minutes').toISOString(); - } - - return lastExecutionTimestamp; -}; - -/** - * Determines the when the last run was in order to execute to. - * - * @param executeTo - * @param lastExecutionTimestamp - * @returns the timestamp to search from - */ -export const getPreviousDailyTaskTimestamp = ( - executeTo: string, - lastExecutionTimestamp?: string -) => { - if (lastExecutionTimestamp === undefined) { - return moment(executeTo).subtract(24, 'hours').toISOString(); - } - - if (moment(executeTo).diff(lastExecutionTimestamp, 'hours') >= 24) { - return moment(executeTo).subtract(24, 'hours').toISOString(); - } - - return lastExecutionTimestamp; -}; - -/** - * Chunks an Array into an Array> - * This is to prevent overloading the telemetry channel + user resources - * - * @param telemetryRecords - * @param batchSize - * @returns the batch of records - */ -export const batchTelemetryRecords = ( - telemetryRecords: unknown[], - batchSize: number -): unknown[][] => - [...Array(Math.ceil(telemetryRecords.length / batchSize))].map((_) => - telemetryRecords.splice(0, batchSize) - ); - -/** - * User defined type guard for PackagePolicy - * - * @param data the union type of package policies - * @returns type confirmation - */ -export function isPackagePolicyList( - data: string[] | PackagePolicy[] | undefined -): data is PackagePolicy[] { - if (data === undefined || data.length < 1) { - return false; - } - - return (data as PackagePolicy[])[0].inputs !== undefined; -} - -/** - * Maps trusted application to shared telemetry object - * - * @param exceptionListItem - * @returns collection of trusted applications - */ -export const trustedApplicationToTelemetryEntry = (trustedApplication: TrustedApp) => { - return { - id: trustedApplication.id, - name: trustedApplication.name, - created_at: trustedApplication.created_at, - updated_at: trustedApplication.updated_at, - entries: trustedApplication.entries, - os_types: [trustedApplication.os], - } as ExceptionListItem; -}; - -/** - * Maps endpoint lists to shared telemetry object - * - * @param exceptionListItem - * @returns collection of endpoint exceptions - */ -export const exceptionListItemToTelemetryEntry = (exceptionListItem: ExceptionListItemSchema) => { - return { - id: exceptionListItem.id, - name: exceptionListItem.name, - created_at: exceptionListItem.created_at, - updated_at: exceptionListItem.updated_at, - entries: exceptionListItem.entries, - os_types: exceptionListItem.os_types, - } as ExceptionListItem; -}; - -/** - * Maps detection rule exception list items to shared telemetry object - * - * @param exceptionListItem - * @param ruleVersion - * @returns collection of detection rule exceptions - */ -export const ruleExceptionListItemToTelemetryEvent = ( - exceptionListItem: ExceptionListItemSchema, - ruleVersion: number -) => { - return { - id: exceptionListItem.item_id, - name: exceptionListItem.description, - rule_version: ruleVersion, - created_at: exceptionListItem.created_at, - updated_at: exceptionListItem.updated_at, - entries: exceptionListItem.entries, - os_types: exceptionListItem.os_types, - } as ExceptionListItem; -}; - -/** - * Consructs the list telemetry schema from a collection of endpoint exceptions - * - * @param listData - * @param listType - * @returns lists telemetry schema - */ -export const templateExceptionList = (listData: ExceptionListItem[], listType: string) => { - return listData.map((item) => { - const template: ListTemplate = { - '@timestamp': new Date().getTime(), - }; - - // cast exception list type to a TelemetryEvent for allowlist filtering - const filteredListItem = copyAllowlistedFields( - exceptionListEventFields, - item as unknown as TelemetryEvent - ); - - if (listType === LIST_DETECTION_RULE_EXCEPTION) { - template.detection_rule = filteredListItem; - return template; - } - - if (listType === LIST_TRUSTED_APPLICATION) { - template.trusted_application = filteredListItem; - return template; - } - - if (listType === LIST_ENDPOINT_EXCEPTION) { - template.endpoint_exception = filteredListItem; - return template; - } - - if (listType === LIST_ENDPOINT_EVENT_FILTER) { - template.endpoint_event_filter = filteredListItem; - return template; - } - - return null; - }); -}; - -/** - * Convert counter label list to kebab case - * - * @param label_list the list of labels to create standardized UsageCounter from - * @returns a string label for usage in the UsageCounter - */ -export function createUsageCounterLabel(labelList: string[]): string { - return labelList.join('-'); -} diff --git a/x-pack/plugins/fleet/server/telemetry/telemetry/tasks/endpoint.ts b/x-pack/plugins/fleet/server/telemetry/telemetry/tasks/endpoint.ts deleted file mode 100644 index a60c4940cdaaa..0000000000000 --- a/x-pack/plugins/fleet/server/telemetry/telemetry/tasks/endpoint.ts +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { Logger } from 'src/core/server'; - -import type { TelemetryEventsSender } from '../sender'; -import type { - EndpointMetricsAggregation, - EndpointPolicyResponseAggregation, - EndpointPolicyResponseDocument, -} from '../types'; -import type { TelemetryReceiver } from '../receiver'; -import type { TaskExecutionPeriod } from '../task'; -import { - batchTelemetryRecords, - getPreviousDailyTaskTimestamp, - isPackagePolicyList, -} from '../helpers'; -import type { PolicyData } from '../../../../common/endpoint/types'; -import { FLEET_ENDPOINT_PACKAGE } from '../../../../common'; -import { TELEMETRY_CHANNEL_ENDPOINT_META } from '../constants'; - -// Endpoint agent uses this Policy ID while it's installing. -const DefaultEndpointPolicyIdToIgnore = '00000000-0000-0000-0000-000000000000'; - -const EmptyFleetAgentResponse = { - agents: [], - total: 0, - page: 0, - perPage: 0, -}; - -export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { - return { - type: 'security:endpoint-meta-telemetry', - title: 'Security Solution Telemetry Endpoint Metrics and Info task', - interval: '24h', - timeout: '5m', - version: '1.0.0', - getLastExecutionTime: getPreviousDailyTaskTimestamp, - runTask: async ( - taskId: string, - logger: Logger, - receiver: TelemetryReceiver, - sender: TelemetryEventsSender, - taskExecutionPeriod: TaskExecutionPeriod - ) => { - if (!taskExecutionPeriod.last) { - throw new Error('last execution timestamp is required'); - } - - const endpointData = await fetchEndpointData( - receiver, - taskExecutionPeriod.last, - taskExecutionPeriod.current - ); - - /** STAGE 1 - Fetch Endpoint Agent Metrics - * - * Reads Endpoint Agent metrics out of the `.ds-metrics-endpoint.metrics` data stream - * and buckets them by Endpoint Agent id and sorts by the top hit. The EP agent will - * report its metrics once per day OR every time a policy change has occured. If - * a metric document(s) exists for an EP agent we map to fleet agent and policy - */ - if (endpointData.endpointMetrics === undefined) { - logger.debug(`no endpoint metrics to report`); - return 0; - } - - const { body: endpointMetricsResponse } = endpointData.endpointMetrics as unknown as { - body: EndpointMetricsAggregation; - }; - - if (endpointMetricsResponse.aggregations === undefined) { - logger.debug(`no endpoint metrics to report`); - return 0; - } - - const endpointMetrics = endpointMetricsResponse.aggregations.endpoint_agents.buckets.map( - (epMetrics) => { - return { - endpoint_agent: epMetrics.latest_metrics.hits.hits[0]._source.agent.id, - endpoint_version: epMetrics.latest_metrics.hits.hits[0]._source.agent.version, - endpoint_metrics: epMetrics.latest_metrics.hits.hits[0]._source, - }; - } - ); - - /** STAGE 2 - Fetch Fleet Agent Config - * - * As the policy id + policy version does not exist on the Endpoint Metrics document - * we need to fetch information about the Fleet Agent and sync the metrics document - * with the Agent's policy data. - * - */ - const agentsResponse = endpointData.fleetAgentsResponse; - - if (agentsResponse === undefined) { - logger.debug('no fleet agent information available'); - return 0; - } - - const fleetAgents = agentsResponse.agents.reduce((cache, agent) => { - if (agent.id === DefaultEndpointPolicyIdToIgnore) { - return cache; - } - - if (agent.policy_id !== null && agent.policy_id !== undefined) { - cache.set(agent.id, agent.policy_id); - } - - return cache; - }, new Map()); - - const endpointPolicyCache = new Map(); - for (const policyInfo of fleetAgents.values()) { - if ( - policyInfo !== null && - policyInfo !== undefined && - !endpointPolicyCache.has(policyInfo) - ) { - const agentPolicy = await receiver.fetchPolicyConfigs(policyInfo); - const packagePolicies = agentPolicy?.package_policies; - - if (packagePolicies !== undefined && isPackagePolicyList(packagePolicies)) { - packagePolicies - .map((pPolicy) => pPolicy as PolicyData) - .forEach((pPolicy) => { - if (pPolicy.inputs[0].config !== undefined) { - pPolicy.inputs.forEach((input) => { - if ( - input.type === FLEET_ENDPOINT_PACKAGE && - input.config !== undefined && - policyInfo !== undefined - ) { - endpointPolicyCache.set(policyInfo, pPolicy); - } - }); - } - }); - } - } - } - - /** STAGE 3 - Fetch Endpoint Policy Responses - * - * Reads Endpoint Agent policy responses out of the `.ds-metrics-endpoint.policy*` data - * stream and creates a local K/V structure that stores the policy response (V) with - * the Endpoint Agent Id (K). A value will only exist if there has been a endpoint - * enrolled in the last 24 hours OR a policy change has occurred. We only send - * non-successful responses. If the field is null, we assume no responses in - * the last 24h or no failures/warnings in the policy applied. - * - */ - const { body: failedPolicyResponses } = endpointData.epPolicyResponse as unknown as { - body: EndpointPolicyResponseAggregation; - }; - - // If there is no policy responses in the 24h > now then we will continue - const policyResponses = failedPolicyResponses.aggregations - ? failedPolicyResponses.aggregations.policy_responses.buckets.reduce( - (cache, endpointAgentId) => { - const doc = endpointAgentId.latest_response.hits.hits[0]; - cache.set(endpointAgentId.key, doc); - return cache; - }, - new Map() - ) - : new Map(); - - /** STAGE 4 - Create the telemetry log records - * - * Iterates through the endpoint metrics documents at STAGE 1 and joins them together - * to form the telemetry log that is sent back to Elastic Security developers to - * make improvements to the product. - * - */ - try { - const telemetryPayloads = endpointMetrics.map((endpoint) => { - let policyConfig = null; - let failedPolicy = null; - - const fleetAgentId = endpoint.endpoint_metrics.elastic.agent.id; - const endpointAgentId = endpoint.endpoint_agent; - - const policyInformation = fleetAgents.get(fleetAgentId); - if (policyInformation) { - policyConfig = endpointPolicyCache.get(policyInformation) || null; - - if (policyConfig) { - failedPolicy = policyResponses.get(endpointAgentId); - } - } - - const { cpu, memory, uptime } = endpoint.endpoint_metrics.Endpoint.metrics; - - return { - '@timestamp': taskExecutionPeriod.current, - endpoint_id: endpointAgentId, - endpoint_version: endpoint.endpoint_version, - endpoint_package_version: policyConfig?.package?.version || null, - endpoint_metrics: { - cpu: cpu.endpoint, - memory: memory.endpoint.private, - uptime, - }, - endpoint_meta: { - os: endpoint.endpoint_metrics.host.os, - }, - policy_config: policyConfig !== null ? policyConfig?.inputs[0].config.policy : {}, - policy_response: - failedPolicy !== null && failedPolicy !== undefined - ? { - agent_policy_status: failedPolicy._source.event.agent_id_status, - manifest_version: - failedPolicy._source.Endpoint.policy.applied.artifacts.global.version, - status: failedPolicy._source.Endpoint.policy.applied.status, - actions: failedPolicy._source.Endpoint.policy.applied.actions - .map((action) => (action.status !== 'success' ? action : null)) - .filter((action) => action !== null), - } - : {}, - telemetry_meta: { - metrics_timestamp: endpoint.endpoint_metrics['@timestamp'], - }, - }; - }); - - /** - * STAGE 5 - Send the documents - * - * Send the documents in a batches of maxTelemetryBatch - */ - batchTelemetryRecords(telemetryPayloads, maxTelemetryBatch).forEach((telemetryBatch) => - sender.sendOnDemand(TELEMETRY_CHANNEL_ENDPOINT_META, telemetryBatch) - ); - return telemetryPayloads.length; - } catch (err) { - logger.warn('could not complete endpoint alert telemetry task'); - return 0; - } - }, - }; -} - -async function fetchEndpointData( - receiver: TelemetryReceiver, - executeFrom: string, - executeTo: string -) { - const [fleetAgentsResponse, epMetricsResponse, policyResponse] = await Promise.allSettled([ - receiver.fetchFleetAgents(), - receiver.fetchEndpointMetrics(executeFrom, executeTo), - receiver.fetchEndpointPolicyResponses(executeFrom, executeTo), - ]); - - return { - fleetAgentsResponse: - fleetAgentsResponse.status === 'fulfilled' - ? fleetAgentsResponse.value - : EmptyFleetAgentResponse, - endpointMetrics: epMetricsResponse.status === 'fulfilled' ? epMetricsResponse.value : undefined, - epPolicyResponse: policyResponse.status === 'fulfilled' ? policyResponse.value : undefined, - }; -} From 719be2d6006e636bd85485a04dc5ca9f25afff4f Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Mon, 18 Oct 2021 13:29:33 +0200 Subject: [PATCH 04/27] cleanup --- .../fleet/server/services/package_policy.ts | 54 +++++++++---------- .../fleet/server/services/upgrade_usage.ts | 12 ++++- .../plugins/fleet/server/telemetry/sender.ts | 11 ++-- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index cffc71ef99940..5b7aea5ddf27a 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -66,7 +66,7 @@ import { compileTemplate } from './epm/agent/agent'; import { normalizeKuery } from './saved_object'; import { appContextService } from '.'; import { removeOldAssets } from './epm/packages/cleanup'; -import { createUpgradeUsage, sendAlertTelemetryEvents } from './upgrade_usage'; +import { createUpgradeUsage, sendTelemetryEvents } from './upgrade_usage'; export type InputsOverride = Partial & { vars?: Array; @@ -578,13 +578,13 @@ class PackagePolicyService { success: true, }); if (packagePolicy.package.version !== packageInfo.version) { - createUpgradeUsage(soClient, { - package_name: packageInfo.name, - current_version: packagePolicy.package.version, - new_version: packageInfo.version, - status: 'success', - }); - sendAlertTelemetryEvents( + // createUpgradeUsage(soClient, { + // package_name: packageInfo.name, + // current_version: packagePolicy.package.version, + // new_version: packageInfo.version, + // status: 'success', + // }); + sendTelemetryEvents( appContextService.getLogger(), appContextService.getTelemetryEventsSender(), { @@ -649,26 +649,26 @@ class PackagePolicyService { if (packagePolicy.package.version !== packageInfo.version) { if (hasErrors) { - createUpgradeUsage(soClient, { - package_name: packageInfo.name, - current_version: packagePolicy.package.version, - new_version: packageInfo.version, - status: hasErrors ? 'failure' : 'success', - error: updatedPackagePolicy.errors, - }); + // createUpgradeUsage(soClient, { + // package_name: packageInfo.name, + // current_version: packagePolicy.package.version, + // new_version: packageInfo.version, + // status: hasErrors ? 'failure' : 'success', + // error: updatedPackagePolicy.errors, + // }); + + sendTelemetryEvents( + appContextService.getLogger(), + appContextService.getTelemetryEventsSender(), + { + package_name: packageInfo.name, + current_version: packagePolicy.package.version, + new_version: packageInfo.version, + status: hasErrors ? 'failure' : 'success', + error: updatedPackagePolicy.errors, + } + ); } - // TODO move inside hasErrors - sendAlertTelemetryEvents( - appContextService.getLogger(), - appContextService.getTelemetryEventsSender(), - { - package_name: packageInfo.name, - current_version: packagePolicy.package.version, - new_version: packageInfo.version, - status: hasErrors ? 'failure' : 'success', - error: updatedPackagePolicy.errors, - } - ); } return { diff --git a/x-pack/plugins/fleet/server/services/upgrade_usage.ts b/x-pack/plugins/fleet/server/services/upgrade_usage.ts index 5c849e4cb65e6..bcb00aa1d0144 100644 --- a/x-pack/plugins/fleet/server/services/upgrade_usage.ts +++ b/x-pack/plugins/fleet/server/services/upgrade_usage.ts @@ -27,7 +27,7 @@ export function deleteUpgradeUsages(soClient: SavedObjectsClientContract, ids: s } } -export function sendAlertTelemetryEvents( +export function sendTelemetryEvents( logger: Logger, eventsTelemetry: TelemetryEventsSender | undefined, upgradeUsage: PackagePolicyUpgradeUsage @@ -39,7 +39,10 @@ export function sendAlertTelemetryEvents( try { eventsTelemetry.queueTelemetryEvents([ { - package_policy_upgrade: { ...upgradeUsage }, + package_policy_upgrade: { + ...upgradeUsage, + error: capErrorSize(upgradeUsage.error), + }, id: `${upgradeUsage.package_name}_${upgradeUsage.current_version}_${upgradeUsage.new_version}_${upgradeUsage.status}`, }, ]); @@ -47,3 +50,8 @@ export function sendAlertTelemetryEvents( logger.error(`queing telemetry events failed ${exc}`); } } + +function capErrorSize(errors?: any[]) { + const MAX_ERROR_SIZE = 100; + return (errors || []).length > MAX_ERROR_SIZE ? errors?.slice(0, MAX_ERROR_SIZE) : errors; +} diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index aa7b2dc4803f2..a3c6d27496229 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -20,7 +20,7 @@ export const FLEET_CHANNEL_NAME = 'fleet-stats'; export class TelemetryEventsSender { private readonly initialCheckDelayMs = 10 * 1000; - private readonly checkIntervalMs = 10 * 1000; + private readonly checkIntervalMs = 30 * 1000; private readonly logger: Logger; private maxQueueSize = TELEMETRY_MAX_BUFFER_SIZE; private telemetryStart?: TelemetryPluginStart; @@ -67,8 +67,6 @@ export class TelemetryEventsSender { (event) => !this.queue.find((qItem) => qItem.id && event.id && qItem.id === event.id) ); - this.logger.info(`Queue events ` + JSON.stringify(events)); - if (qlength >= this.maxQueueSize) { // we're full already return; @@ -98,9 +96,9 @@ export class TelemetryEventsSender { try { this.isSending = true; - this.isOptedIn = true; // await this.isTelemetryOptedIn(); + this.isOptedIn = await this.isTelemetryOptedIn(); if (!this.isOptedIn) { - this.logger.info(`Telemetry is not opted-in.`); + this.logger.debug(`Telemetry is not opted-in.`); this.queue = []; this.isSending = false; return; @@ -125,8 +123,7 @@ export class TelemetryEventsSender { })); this.queue = []; - this.logger.info('sending events to: ' + telemetryUrl); - this.logger.info(JSON.stringify(toSend)); + this.logger.debug(JSON.stringify(toSend)); await this.sendEvents( toSend, From 0434d464db9fde23387b02626e0c94419100959d Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Mon, 18 Oct 2021 14:41:25 +0200 Subject: [PATCH 05/27] type fix --- .../fleet/server/services/package_policy.ts | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 5b7aea5ddf27a..1f70aece4506b 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -578,12 +578,12 @@ class PackagePolicyService { success: true, }); if (packagePolicy.package.version !== packageInfo.version) { - // createUpgradeUsage(soClient, { - // package_name: packageInfo.name, - // current_version: packagePolicy.package.version, - // new_version: packageInfo.version, - // status: 'success', - // }); + createUpgradeUsage(soClient, { + package_name: packageInfo.name, + current_version: packagePolicy.package.version, + new_version: packageInfo.version, + status: 'success', + }); sendTelemetryEvents( appContextService.getLogger(), appContextService.getTelemetryEventsSender(), @@ -649,13 +649,13 @@ class PackagePolicyService { if (packagePolicy.package.version !== packageInfo.version) { if (hasErrors) { - // createUpgradeUsage(soClient, { - // package_name: packageInfo.name, - // current_version: packagePolicy.package.version, - // new_version: packageInfo.version, - // status: hasErrors ? 'failure' : 'success', - // error: updatedPackagePolicy.errors, - // }); + createUpgradeUsage(soClient, { + package_name: packageInfo.name, + current_version: packagePolicy.package.version, + new_version: packageInfo.version, + status: hasErrors ? 'failure' : 'success', + error: updatedPackagePolicy.errors, + }); sendTelemetryEvents( appContextService.getLogger(), From 0e8fa09862224bbbf61faa2af27bf14c4cffc49f Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Tue, 19 Oct 2021 12:55:42 +0200 Subject: [PATCH 06/27] removed collector --- .../collectors/package_upgrade_collectors.ts | 38 ------------------- .../fleet/server/collectors/register.ts | 3 -- .../fleet/server/saved_objects/index.ts | 24 ------------ .../fleet/server/services/package_policy.ts | 16 +------- .../fleet/server/services/upgrade_usage.ts | 22 +++-------- 5 files changed, 7 insertions(+), 96 deletions(-) delete mode 100644 x-pack/plugins/fleet/server/collectors/package_upgrade_collectors.ts diff --git a/x-pack/plugins/fleet/server/collectors/package_upgrade_collectors.ts b/x-pack/plugins/fleet/server/collectors/package_upgrade_collectors.ts deleted file mode 100644 index 7183091fcd865..0000000000000 --- a/x-pack/plugins/fleet/server/collectors/package_upgrade_collectors.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { SavedObjectsClient } from 'kibana/server'; - -import { deleteUpgradeUsages } from '../services/upgrade_usage'; - -export interface PackagePolicyUpgradeUsage { - package_name: string; - current_version: string; - new_version: string; - status: 'success' | 'failure'; - error?: Array<{ key?: string; message: string }>; -} - -export const getPackagePolicyUpgradeUsage = async ( - soClient?: SavedObjectsClient -): Promise => { - if (!soClient) { - return []; - } - const telemetryObjects = await soClient.find({ - type: 'package-policy-upgrade-telemetry', - }); - - // TODO cap if becomes too large - - const usages = telemetryObjects.saved_objects.map((so) => so.attributes); - deleteUpgradeUsages( - soClient, - telemetryObjects.saved_objects.map((so) => so.id) - ); - return usages; -}; diff --git a/x-pack/plugins/fleet/server/collectors/register.ts b/x-pack/plugins/fleet/server/collectors/register.ts index 2ddd9030b5910..a097d423e7dd2 100644 --- a/x-pack/plugins/fleet/server/collectors/register.ts +++ b/x-pack/plugins/fleet/server/collectors/register.ts @@ -18,7 +18,6 @@ import { getPackageUsage } from './package_collectors'; import type { PackageUsage } from './package_collectors'; import { getFleetServerUsage } from './fleet_server_collector'; import type { FleetServerUsage } from './fleet_server_collector'; -import { getPackagePolicyUpgradeUsage } from './package_upgrade_collectors'; interface Usage { agents_enabled: boolean; @@ -49,7 +48,6 @@ export function registerFleetUsageCollector( agents: await getAgentUsage(config, soClient, esClient), fleet_server: await getFleetServerUsage(soClient, esClient), packages: await getPackageUsage(soClient), - package_policy_upgrades: await getPackagePolicyUpgradeUsage(soClient), }; }, schema: { @@ -145,7 +143,6 @@ export function registerFleetUsageCollector( enabled: { type: 'boolean' }, }, }, - // TODO add package_policy_upgrades schema }, }); diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 29b9c014b56e8..ac5ca401da000 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -369,30 +369,6 @@ const getSavedObjectTypes = ( }, }, }, - // TODO create constant - ['package-policy-upgrade-telemetry']: { - name: 'package-policy-upgrade-telemetry', - hidden: false, - namespaceType: 'agnostic', - management: { - importableAndExportable: false, - }, - mappings: { - properties: { - package_name: { type: 'keyword' }, - current_version: { type: 'keyword' }, - new_version: { type: 'keyword' }, - status: { type: 'keyword' }, - error: { - type: 'nested', - properties: { - key: { type: 'keyword' }, - message: { type: 'text' }, - }, - }, - }, - }, - }, }); export function registerSavedObjects( diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 1f70aece4506b..612d8fafeab01 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -66,7 +66,7 @@ import { compileTemplate } from './epm/agent/agent'; import { normalizeKuery } from './saved_object'; import { appContextService } from '.'; import { removeOldAssets } from './epm/packages/cleanup'; -import { createUpgradeUsage, sendTelemetryEvents } from './upgrade_usage'; +import { sendTelemetryEvents } from './upgrade_usage'; export type InputsOverride = Partial & { vars?: Array; @@ -578,12 +578,6 @@ class PackagePolicyService { success: true, }); if (packagePolicy.package.version !== packageInfo.version) { - createUpgradeUsage(soClient, { - package_name: packageInfo.name, - current_version: packagePolicy.package.version, - new_version: packageInfo.version, - status: 'success', - }); sendTelemetryEvents( appContextService.getLogger(), appContextService.getTelemetryEventsSender(), @@ -649,14 +643,6 @@ class PackagePolicyService { if (packagePolicy.package.version !== packageInfo.version) { if (hasErrors) { - createUpgradeUsage(soClient, { - package_name: packageInfo.name, - current_version: packagePolicy.package.version, - new_version: packageInfo.version, - status: hasErrors ? 'failure' : 'success', - error: updatedPackagePolicy.errors, - }); - sendTelemetryEvents( appContextService.getLogger(), appContextService.getTelemetryEventsSender(), diff --git a/x-pack/plugins/fleet/server/services/upgrade_usage.ts b/x-pack/plugins/fleet/server/services/upgrade_usage.ts index bcb00aa1d0144..1c88ffcde5085 100644 --- a/x-pack/plugins/fleet/server/services/upgrade_usage.ts +++ b/x-pack/plugins/fleet/server/services/upgrade_usage.ts @@ -5,26 +5,16 @@ * 2.0. */ -import type { SavedObjectsClientContract } from 'kibana/server'; import type { Logger } from 'src/core/server'; -import type { PackagePolicyUpgradeUsage } from '../collectors/package_upgrade_collectors'; import type { TelemetryEventsSender } from '../telemetry/sender'; -export function createUpgradeUsage( - soClient: SavedObjectsClientContract, - upgradeUsage: PackagePolicyUpgradeUsage -) { - soClient.create('package-policy-upgrade-telemetry', upgradeUsage, { - id: `${upgradeUsage.package_name}_${upgradeUsage.current_version}_${upgradeUsage.new_version}_${upgradeUsage.status}`, - overwrite: true, - }); -} - -export function deleteUpgradeUsages(soClient: SavedObjectsClientContract, ids: string[]) { - for (const id of ids) { - soClient.delete('package-policy-upgrade-telemetry', id); - } +export interface PackagePolicyUpgradeUsage { + package_name: string; + current_version: string; + new_version: string; + status: 'success' | 'failure'; + error?: Array<{ key?: string; message: string }>; } export function sendTelemetryEvents( From 316ff8b78ae6445d04130d35e288e821527a4c3b Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Tue, 19 Oct 2021 14:05:14 +0200 Subject: [PATCH 07/27] made required field message generic, added test --- .../server/services/upgrade_usage.test.ts | 71 +++++++++++++++++++ .../fleet/server/services/upgrade_usage.ts | 31 ++++++-- .../plugins/fleet/server/telemetry/sender.ts | 38 ++-------- 3 files changed, 101 insertions(+), 39 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/upgrade_usage.test.ts diff --git a/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts b/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts new file mode 100644 index 0000000000000..369da5903577f --- /dev/null +++ b/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from 'src/core/server'; +import { loggingSystemMock } from 'src/core/server/mocks'; + +import type { TelemetryEventsSender } from '../telemetry/sender'; +import { createMockTelemetryEventsSender } from '../telemetry/__mocks__'; + +import { sendTelemetryEvents, capErrorSize } from './upgrade_usage'; +import type { PackagePolicyUpgradeUsage } from './upgrade_usage'; + +describe('sendTelemetryEvents', () => { + let eventsTelemetryMock: jest.Mocked; + let loggerMock: jest.Mocked; + + beforeEach(() => { + eventsTelemetryMock = createMockTelemetryEventsSender(); + loggerMock = loggingSystemMock.createLogger(); + }); + + it('should queue telemetry events with generic error', () => { + const upgardeMessage: PackagePolicyUpgradeUsage = { + package_name: 'aws', + current_version: '0.6.1', + new_version: '1.3.0', + status: 'failure', + error: [ + { key: 'fieldX', message: ['Field X is required'] }, + { key: 'fieldX', message: 'Invalid format' }, + ], + }; + + sendTelemetryEvents(loggerMock, eventsTelemetryMock, upgardeMessage); + + expect(eventsTelemetryMock.queueTelemetryEvents).toHaveBeenCalledWith([ + { + id: 'aws_0.6.1_1.3.0_failure', + package_policy_upgrade: { + current_version: '0.6.1', + error: [ + { + key: 'fieldX', + message: ['Field is required'], + }, + { + key: 'fieldX', + message: 'Invalid format', + }, + ], + new_version: '1.3.0', + package_name: 'aws', + status: 'failure', + }, + }, + ]); + }); + + it('should cap error size', () => { + const maxSize = 2; + const errors = [{ message: '1' }, { message: '2' }, { message: '3' }]; + + const result = capErrorSize(errors, maxSize); + + expect(result.length).toEqual(maxSize); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/upgrade_usage.ts b/x-pack/plugins/fleet/server/services/upgrade_usage.ts index 1c88ffcde5085..361af4a7b3f75 100644 --- a/x-pack/plugins/fleet/server/services/upgrade_usage.ts +++ b/x-pack/plugins/fleet/server/services/upgrade_usage.ts @@ -14,9 +14,16 @@ export interface PackagePolicyUpgradeUsage { current_version: string; new_version: string; status: 'success' | 'failure'; - error?: Array<{ key?: string; message: string }>; + error?: UpgradeError[]; } +export interface UpgradeError { + key?: string; + message: string | string[]; +} + +export const MAX_ERROR_SIZE = 100; + export function sendTelemetryEvents( logger: Logger, eventsTelemetry: TelemetryEventsSender | undefined, @@ -31,7 +38,9 @@ export function sendTelemetryEvents( { package_policy_upgrade: { ...upgradeUsage, - error: capErrorSize(upgradeUsage.error), + error: upgradeUsage.error + ? makeErrorGeneric(capErrorSize(upgradeUsage.error, MAX_ERROR_SIZE)) + : undefined, }, id: `${upgradeUsage.package_name}_${upgradeUsage.current_version}_${upgradeUsage.new_version}_${upgradeUsage.status}`, }, @@ -41,7 +50,19 @@ export function sendTelemetryEvents( } } -function capErrorSize(errors?: any[]) { - const MAX_ERROR_SIZE = 100; - return (errors || []).length > MAX_ERROR_SIZE ? errors?.slice(0, MAX_ERROR_SIZE) : errors; +export function capErrorSize(errors: UpgradeError[], maxSize: number): UpgradeError[] { + return errors.length > maxSize ? errors?.slice(0, maxSize) : errors; +} + +function makeErrorGeneric(errors: UpgradeError[]): UpgradeError[] { + return errors.map((error) => { + let message = error.message; + if (error.key && Array.isArray(error.message) && error.message[0].indexOf('is required') > -1) { + message = ['Field is required']; + } + return { + key: error.key, + message, + }; + }); } diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index a3c6d27496229..96c0c1bc40ffc 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -18,6 +18,10 @@ import type { TelemetryReceiver } from './receiver'; export const TELEMETRY_MAX_BUFFER_SIZE = 100; export const FLEET_CHANNEL_NAME = 'fleet-stats'; +/** + * Simplified version of https://github.com/elastic/kibana/blob/master/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts + * Sends batched events to telemetry v3 api + */ export class TelemetryEventsSender { private readonly initialCheckDelayMs = 10 * 1000; private readonly checkIntervalMs = 30 * 1000; @@ -139,40 +143,6 @@ export class TelemetryEventsSender { this.isSending = false; } - /** - * This function sends events to the elastic telemetry channel. Caution is required - * because it does no allowlist filtering at send time. The function call site is - * responsible for ensuring sure no sensitive material is in telemetry events. - * - * @param channel the elastic telemetry channel - * @param toSend telemetry events - */ - // public async sendOnDemand(channel: string, toSend: unknown[]) { - // try { - // const [telemetryUrl, clusterInfo, licenseInfo] = await Promise.all([ - // this.fetchTelemetryUrl(channel), - // this.receiver?.fetchClusterInfo(), - // this.receiver?.fetchLicenseInfo(), - // ]); - - // this.logger.debug(`Telemetry URL: ${telemetryUrl}`); - // this.logger.debug( - // `cluster_uuid: ${clusterInfo?.cluster_uuid} cluster_name: ${clusterInfo?.cluster_name}` - // ); - - // await this.sendEvents( - // toSend, - // telemetryUrl, - // channel, - // clusterInfo?.cluster_uuid, - // clusterInfo?.version?.number, - // licenseInfo?.uid - // ); - // } catch (err) { - // this.logger.warn(`Error sending telemetry events data: ${err}`); - // } - // } - private async fetchTelemetryUrl(channel: string): Promise { const telemetryUrl = await this.telemetrySetup?.getTelemetryUrl(); if (!telemetryUrl) { From 217ef5556480cb5a1a15ba5a7dbf1c8d69db0c95 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Tue, 19 Oct 2021 14:09:15 +0200 Subject: [PATCH 08/27] cleanup --- .../fleet/server/telemetry/__mocks__/index.ts | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/x-pack/plugins/fleet/server/telemetry/__mocks__/index.ts b/x-pack/plugins/fleet/server/telemetry/__mocks__/index.ts index 0eeac6fd1d7e5..2070aeca20861 100644 --- a/x-pack/plugins/fleet/server/telemetry/__mocks__/index.ts +++ b/x-pack/plugins/fleet/server/telemetry/__mocks__/index.ts @@ -6,7 +6,6 @@ */ import type { TelemetryEventsSender } from '../sender'; -import type { TelemetryReceiver } from '../receiver'; /** * Creates a mocked Telemetry Events Sender @@ -20,27 +19,8 @@ export const createMockTelemetryEventsSender = ( stop: jest.fn(), fetchTelemetryUrl: jest.fn(), queueTelemetryEvents: jest.fn(), - processEvents: jest.fn(), isTelemetryOptedIn: jest.fn().mockReturnValue(enableTelemetry ?? jest.fn()), sendIfDue: jest.fn(), sendEvents: jest.fn(), } as unknown as jest.Mocked; }; - -export const createMockTelemetryReceiver = ( - diagnosticsAlert?: unknown -): jest.Mocked => { - return { - start: jest.fn(), - fetchClusterInfo: jest.fn(), - fetchLicenseInfo: jest.fn(), - copyLicenseFields: jest.fn(), - fetchFleetAgents: jest.fn(), - fetchDiagnosticAlerts: jest.fn().mockReturnValue(diagnosticsAlert ?? jest.fn()), - fetchEndpointMetrics: jest.fn(), - fetchEndpointPolicyResponses: jest.fn(), - fetchTrustedApplications: jest.fn(), - fetchEndpointList: jest.fn(), - fetchDetectionRules: jest.fn().mockReturnValue({ body: null }), - } as unknown as jest.Mocked; -}; From d0620df05cccaec91d1138cb6d25d703d9e43755 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Tue, 19 Oct 2021 14:11:03 +0200 Subject: [PATCH 09/27] cleanup --- x-pack/plugins/fleet/server/telemetry/types.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/telemetry/types.ts b/x-pack/plugins/fleet/server/telemetry/types.ts index 917b9b9bfc1ba..9b96fa9a9968a 100644 --- a/x-pack/plugins/fleet/server/telemetry/types.ts +++ b/x-pack/plugins/fleet/server/telemetry/types.ts @@ -42,7 +42,6 @@ export interface ESLicense { export interface TelemetryEvent { [key: string]: SearchTypes; - '@timestamp'?: string; cluster_name?: string; cluster_uuid?: string; license?: ESLicense; From 03d39ad9d90377e5f067784c93e0eb81f1951e70 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Tue, 19 Oct 2021 15:47:01 +0200 Subject: [PATCH 10/27] cleanup --- x-pack/plugins/fleet/server/plugin.ts | 2 +- .../fleet/server/telemetry/receiver.ts | 44 +------------------ .../plugins/fleet/server/telemetry/sender.ts | 11 ++--- .../plugins/fleet/server/telemetry/types.ts | 16 ------- 4 files changed, 6 insertions(+), 67 deletions(-) diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 11e566a75669c..52e1028827f13 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -195,7 +195,7 @@ export class FleetPlugin this.logger = this.initializerContext.logger.get(); this.configInitialValue = this.initializerContext.config.get(); this.telemetryEventsSender = new TelemetryEventsSender(this.logger); - this.telemetryReceiver = new TelemetryReceiver(this.logger); + this.telemetryReceiver = new TelemetryReceiver(); } public setup(core: CoreSetup, deps: FleetSetupDeps) { diff --git a/x-pack/plugins/fleet/server/telemetry/receiver.ts b/x-pack/plugins/fleet/server/telemetry/receiver.ts index d51f9654808ce..487461e5ed1ab 100644 --- a/x-pack/plugins/fleet/server/telemetry/receiver.ts +++ b/x-pack/plugins/fleet/server/telemetry/receiver.ts @@ -5,18 +5,13 @@ * 2.0. */ -import type { Logger, CoreStart, ElasticsearchClient } from 'src/core/server'; +import type { CoreStart, ElasticsearchClient } from 'src/core/server'; -import type { ESLicense, ESClusterInfo } from './types'; +import type { ESClusterInfo } from './types'; export class TelemetryReceiver { - private readonly logger: Logger; private esClient?: ElasticsearchClient; - constructor(logger: Logger) { - this.logger = logger.get('telemetry_events'); - } - public async start(core?: CoreStart) { this.esClient = core?.elasticsearch.client.asInternalUser; } @@ -29,39 +24,4 @@ export class TelemetryReceiver { const { body } = await this.esClient.info(); return body; } - - public async fetchLicenseInfo(): Promise { - if (this.esClient === undefined || this.esClient === null) { - throw Error('elasticsearch client is unavailable: cannot retrieve license information'); - } - - try { - const ret = ( - await this.esClient.transport.request({ - method: 'GET', - path: '/_license', - querystring: { - local: true, - // For versions >= 7.6 and < 8.0, this flag is needed otherwise 'platinum' is returned for 'enterprise' license. - accept_enterprise: 'true', - }, - }) - ).body as Promise<{ license: ESLicense }>; - - return (await ret).license; - } catch (err) { - this.logger.debug(`failed retrieving license: ${err}`); - return undefined; - } - } - - public copyLicenseFields(lic: ESLicense) { - return { - uid: lic.uid, - status: lic.status, - type: lic.type, - ...(lic.issued_to ? { issued_to: lic.issued_to } : {}), - ...(lic.issuer ? { issuer: lic.issuer } : {}), - }; - } } diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index 96c0c1bc40ffc..8c7e3a913f205 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -108,10 +108,9 @@ export class TelemetryEventsSender { return; } - const [telemetryUrl, clusterInfo, licenseInfo] = await Promise.all([ + const [telemetryUrl, clusterInfo] = await Promise.all([ this.fetchTelemetryUrl(FLEET_CHANNEL_NAME), this.receiver?.fetchClusterInfo(), - this.receiver?.fetchLicenseInfo(), ]); this.logger.debug(`Telemetry URL: ${telemetryUrl}`); @@ -121,7 +120,6 @@ export class TelemetryEventsSender { const toSend: TelemetryEvent[] = cloneDeep(this.queue).map((event) => ({ ...event, - ...(licenseInfo ? { license: this.receiver?.copyLicenseFields(licenseInfo) } : {}), cluster_uuid: clusterInfo?.cluster_uuid, cluster_name: clusterInfo?.cluster_name, })); @@ -133,8 +131,7 @@ export class TelemetryEventsSender { toSend, telemetryUrl, clusterInfo?.cluster_uuid, - clusterInfo?.version?.number, - licenseInfo?.uid + clusterInfo?.version?.number ); } catch (err) { this.logger.warn(`Error sending telemetry events data: ${err}`); @@ -168,8 +165,7 @@ export class TelemetryEventsSender { events: unknown[], telemetryUrl: string, clusterUuid: string | undefined, - clusterVersionNumber: string | undefined, - licenseId: string | undefined + clusterVersionNumber: string | undefined ) { const ndjson = this.transformDataToNdjson(events); @@ -179,7 +175,6 @@ export class TelemetryEventsSender { 'Content-Type': 'application/x-ndjson', 'X-Elastic-Cluster-ID': clusterUuid, 'X-Elastic-Stack-Version': clusterVersionNumber ? clusterVersionNumber : '7.16.0', - ...(licenseId ? { 'X-Elastic-License-ID': licenseId } : {}), }, }); this.logger.debug(`Events sent!. Response: ${resp.status} ${JSON.stringify(resp.data)}`); diff --git a/x-pack/plugins/fleet/server/telemetry/types.ts b/x-pack/plugins/fleet/server/telemetry/types.ts index 9b96fa9a9968a..13dff52ca7da9 100644 --- a/x-pack/plugins/fleet/server/telemetry/types.ts +++ b/x-pack/plugins/fleet/server/telemetry/types.ts @@ -25,24 +25,8 @@ export interface ESClusterInfo { }; } -// From https://www.elastic.co/guide/en/elasticsearch/reference/current/get-license.html -export interface ESLicense { - status: string; - uid: string; - type: string; - issue_date?: string; - issue_date_in_millis?: number; - expiry_date?: string; - expirty_date_in_millis?: number; - max_nodes?: number; - issued_to?: string; - issuer?: string; - start_date_in_millis?: number; -} - export interface TelemetryEvent { [key: string]: SearchTypes; cluster_name?: string; cluster_uuid?: string; - license?: ESLicense; } From 1fe58f168fefb84fa05b1e75307e27b8384efd9e Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Tue, 19 Oct 2021 16:24:16 +0200 Subject: [PATCH 11/27] removed v3-dev as outdated --- x-pack/plugins/fleet/server/telemetry/sender.test.ts | 2 +- x-pack/plugins/fleet/server/telemetry/sender.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/fleet/server/telemetry/sender.test.ts b/x-pack/plugins/fleet/server/telemetry/sender.test.ts index 713ccdef8729d..ece7a3cdb7a0b 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.test.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.test.ts @@ -95,7 +95,7 @@ describe('getV3UrlFromV2', () => { const sender = new TelemetryEventsSender(logger); expect( sender.getV3UrlFromV2('https://telemetry-staging.elastic.co/xpack/v2/send', 'alerts-endpoint') - ).toBe('https://telemetry-staging.elastic.co/v3-dev/send/alerts-endpoint'); + ).toBe('https://telemetry-staging.elastic.co/v3/send/alerts-endpoint'); }); it('should support ports and auth', () => { diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index 8c7e3a913f205..ca30a308d2aa9 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -150,13 +150,13 @@ export class TelemetryEventsSender { // Forms URLs like: // https://telemetry.elastic.co/v3/send/my-channel-name or - // https://telemetry-staging.elastic.co/v3-dev/send/my-channel-name + // https://telemetry-staging.elastic.co/v3/send/my-channel-name public getV3UrlFromV2(v2url: string, channel: string): string { const url = new URL(v2url); if (!url.hostname.includes('staging')) { url.pathname = `/v3/send/${channel}`; } else { - url.pathname = `/v3-dev/send/${channel}`; + url.pathname = `/v3/send/${channel}`; } return url.toString(); } From 9f18ad3ce81413d4ae512aa472361d96a9908673 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Tue, 19 Oct 2021 16:53:21 +0200 Subject: [PATCH 12/27] removed conditional from telemetry url creation --- x-pack/plugins/fleet/server/telemetry/sender.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index ca30a308d2aa9..8b5bef71872fb 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -153,11 +153,7 @@ export class TelemetryEventsSender { // https://telemetry-staging.elastic.co/v3/send/my-channel-name public getV3UrlFromV2(v2url: string, channel: string): string { const url = new URL(v2url); - if (!url.hostname.includes('staging')) { - url.pathname = `/v3/send/${channel}`; - } else { - url.pathname = `/v3/send/${channel}`; - } + url.pathname = `/v3/send/${channel}`; return url.toString(); } From a3b101140eb5f91ed51b7aa932dfccf23e9e249b Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Wed, 20 Oct 2021 14:28:33 +0200 Subject: [PATCH 13/27] supporting multiple channels in sender --- .../fleet/server/services/upgrade_usage.ts | 24 ++-- .../fleet/server/telemetry/queue.test.ts | 62 +++++++++ .../plugins/fleet/server/telemetry/queue.ts | 115 +++++++++++++++++ .../fleet/server/telemetry/sender.test.ts | 80 ++++++------ .../plugins/fleet/server/telemetry/sender.ts | 121 ++++-------------- 5 files changed, 258 insertions(+), 144 deletions(-) create mode 100644 x-pack/plugins/fleet/server/telemetry/queue.test.ts create mode 100644 x-pack/plugins/fleet/server/telemetry/queue.ts diff --git a/x-pack/plugins/fleet/server/services/upgrade_usage.ts b/x-pack/plugins/fleet/server/services/upgrade_usage.ts index 361af4a7b3f75..c69ddf0261cd2 100644 --- a/x-pack/plugins/fleet/server/services/upgrade_usage.ts +++ b/x-pack/plugins/fleet/server/services/upgrade_usage.ts @@ -23,6 +23,7 @@ export interface UpgradeError { } export const MAX_ERROR_SIZE = 100; +export const FLEET_UPGRADES_CHANNEL_NAME = 'fleet-upgrades'; export function sendTelemetryEvents( logger: Logger, @@ -34,17 +35,20 @@ export function sendTelemetryEvents( } try { - eventsTelemetry.queueTelemetryEvents([ - { - package_policy_upgrade: { - ...upgradeUsage, - error: upgradeUsage.error - ? makeErrorGeneric(capErrorSize(upgradeUsage.error, MAX_ERROR_SIZE)) - : undefined, + eventsTelemetry.queueTelemetryEvents( + [ + { + package_policy_upgrade: { + ...upgradeUsage, + error: upgradeUsage.error + ? makeErrorGeneric(capErrorSize(upgradeUsage.error, MAX_ERROR_SIZE)) + : undefined, + }, + id: `${upgradeUsage.package_name}_${upgradeUsage.current_version}_${upgradeUsage.new_version}_${upgradeUsage.status}`, }, - id: `${upgradeUsage.package_name}_${upgradeUsage.current_version}_${upgradeUsage.new_version}_${upgradeUsage.status}`, - }, - ]); + ], + FLEET_UPGRADES_CHANNEL_NAME + ); } catch (exc) { logger.error(`queing telemetry events failed ${exc}`); } diff --git a/x-pack/plugins/fleet/server/telemetry/queue.test.ts b/x-pack/plugins/fleet/server/telemetry/queue.test.ts new file mode 100644 index 0000000000000..a79ca1004d0d6 --- /dev/null +++ b/x-pack/plugins/fleet/server/telemetry/queue.test.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from 'src/core/server/mocks'; + +import type { ESClusterInfo } from './types'; +import { TelemetryQueue } from './queue'; + +describe('TelemetryQueue', () => { + let logger: ReturnType; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + }); + + describe('queueTelemetryEvents', () => { + it('queues two events', () => { + const queue = new TelemetryQueue(logger); + queue.addEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); + expect(queue.queue.length).toBe(2); + }); + + it('queues more than maxQueueSize events', () => { + const queue = new TelemetryQueue(logger); + queue.addEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); + queue.maxQueueSize = 5; + queue.addEvents([{ 'event.kind': '3' }, { 'event.kind': '4' }]); + queue.addEvents([{ 'event.kind': '5' }, { 'event.kind': '6' }]); + queue.addEvents([{ 'event.kind': '7' }, { 'event.kind': '8' }]); + expect(queue.queue.length).toBe(5); + }); + + it('empties the queue when sending', async () => { + const queue = new TelemetryQueue(logger); + queue.send = jest.fn(); + queue.addEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); + + expect(queue.queue.length).toBe(2); + await queue.sendEvents('https://telemetry.elastic.co/v3/send/my-channel', { + cluster_uuid: '1', + cluster_name: 'cluster', + version: { + number: '7.16.0', + }, + } as ESClusterInfo); + expect(queue.queue.length).toBe(0); + expect(queue.send).toHaveBeenCalledWith( + [ + { cluster_name: 'cluster', cluster_uuid: '1', 'event.kind': '1' }, + { cluster_name: 'cluster', cluster_uuid: '1', 'event.kind': '2' }, + ], + 'https://telemetry.elastic.co/v3/send/my-channel', + '1', + '7.16.0' + ); + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/telemetry/queue.ts b/x-pack/plugins/fleet/server/telemetry/queue.ts new file mode 100644 index 0000000000000..736f65e8d89b9 --- /dev/null +++ b/x-pack/plugins/fleet/server/telemetry/queue.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { cloneDeep } from 'lodash'; +import type { Logger } from 'src/core/server'; + +import axios from 'axios'; + +import type { ESClusterInfo, TelemetryEvent } from './types'; + +export const TELEMETRY_MAX_BUFFER_SIZE = 100; + +export class TelemetryQueue { + private maxQueueSize = TELEMETRY_MAX_BUFFER_SIZE; + private queue: TelemetryEvent[] = []; + private readonly logger: Logger; + + constructor(logger: Logger) { + this.logger = logger; + } + + public addEvents(events: TelemetryEvent[]) { + const qlength = this.queue.length; + + if (events.length === 0) { + return; + } + + // do not add events with same id + events = events.filter( + (event) => !this.queue.find((qItem) => qItem.id && event.id && qItem.id === event.id) + ); + + if (qlength >= this.maxQueueSize) { + // we're full already + return; + } + + if (events.length > this.maxQueueSize - qlength) { + this.queue.push(...events.slice(0, this.maxQueueSize - qlength)); + } else { + this.queue.push(...events); + } + } + + public clearEvents() { + this.queue = []; + } + + public async sendEvents(telemetryUrl: string, clusterInfo: ESClusterInfo | undefined) { + if (this.queue.length === 0) { + return; + } + + try { + this.logger.debug(`Telemetry URL: ${telemetryUrl}`); + + const toSend: TelemetryEvent[] = cloneDeep(this.queue).map((event) => ({ + ...event, + cluster_uuid: clusterInfo?.cluster_uuid, + cluster_name: clusterInfo?.cluster_name, + })); + this.queue = []; + + this.logger.debug(JSON.stringify(toSend)); + + await this.send( + toSend, + telemetryUrl, + clusterInfo?.cluster_uuid, + clusterInfo?.version?.number + ); + } catch (err) { + this.logger.warn(`Error sending telemetry events data: ${err}`); + this.queue = []; + } + } + + private async send( + events: unknown[], + telemetryUrl: string, + clusterUuid: string | undefined, + clusterVersionNumber: string | undefined + ) { + const ndjson = this.transformDataToNdjson(events); + + try { + const resp = await axios.post(telemetryUrl, ndjson, { + headers: { + 'Content-Type': 'application/x-ndjson', + 'X-Elastic-Cluster-ID': clusterUuid, + 'X-Elastic-Stack-Version': clusterVersionNumber ? clusterVersionNumber : '7.16.0', + }, + }); + this.logger.debug(`Events sent!. Response: ${resp.status} ${JSON.stringify(resp.data)}`); + } catch (err) { + this.logger.warn( + `Error sending events: ${err.response.status} ${JSON.stringify(err.response.data)}` + ); + } + } + + private transformDataToNdjson = (data: unknown[]): string => { + if (data.length !== 0) { + const dataString = data.map((dataItem) => JSON.stringify(dataItem)).join('\n'); + return `${dataString}\n`; + } else { + return ''; + } + }; +} diff --git a/x-pack/plugins/fleet/server/telemetry/sender.test.ts b/x-pack/plugins/fleet/server/telemetry/sender.test.ts index ece7a3cdb7a0b..1ceaa5d68b3c3 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.test.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.test.ts @@ -22,21 +22,11 @@ describe('TelemetryEventsSender', () => { describe('queueTelemetryEvents', () => { it('queues two events', () => { const sender = new TelemetryEventsSender(logger); - sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); - expect(sender['queue'].length).toBe(2); + sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }], 'my-channel'); + expect(sender['queuesPerChannel']['my-channel']).toBeDefined(); }); - it('queues more than maxQueueSize events', () => { - const sender = new TelemetryEventsSender(logger); - sender['maxQueueSize'] = 5; - sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); - sender.queueTelemetryEvents([{ 'event.kind': '3' }, { 'event.kind': '4' }]); - sender.queueTelemetryEvents([{ 'event.kind': '5' }, { 'event.kind': '6' }]); - sender.queueTelemetryEvents([{ 'event.kind': '7' }, { 'event.kind': '8' }]); - expect(sender['queue'].length).toBe(5); - }); - - it('empties the queue when sending', async () => { + it('should send events when due', async () => { const sender = new TelemetryEventsSender(logger); sender['telemetryStart'] = { getIsOptedIn: jest.fn(async () => true), @@ -44,35 +34,53 @@ describe('TelemetryEventsSender', () => { sender['telemetrySetup'] = { getTelemetryUrl: jest.fn(async () => new URL('https://telemetry.elastic.co')), }; - sender['sendEvents'] = jest.fn(); - sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); - expect(sender['queue'].length).toBe(2); - await sender['sendIfDue'](); - expect(sender['queue'].length).toBe(0); - expect(sender['sendEvents']).toBeCalledTimes(1); - sender.queueTelemetryEvents([{ 'event.kind': '3' }, { 'event.kind': '4' }]); - sender.queueTelemetryEvents([{ 'event.kind': '5' }, { 'event.kind': '6' }]); - expect(sender['queue'].length).toBe(4); + sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }], 'my-channel'); + sender['queuesPerChannel']['my-channel']['sendEvents'] = jest.fn(); + await sender['sendIfDue'](); - expect(sender['queue'].length).toBe(0); - expect(sender['sendEvents']).toBeCalledTimes(2); + + expect(sender['queuesPerChannel']['my-channel']['sendEvents']).toBeCalledTimes(1); }); it("shouldn't send when telemetry is disabled", async () => { const sender = new TelemetryEventsSender(logger); - sender['sendEvents'] = jest.fn(); const telemetryStart = { getIsOptedIn: jest.fn(async () => false), }; sender['telemetryStart'] = telemetryStart; - sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); - expect(sender['queue'].length).toBe(2); + sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }], 'my-channel'); + sender['queuesPerChannel']['my-channel']['sendEvents'] = jest.fn(); + + await sender['sendIfDue'](); + + expect(sender['queuesPerChannel']['my-channel']['sendEvents']).toBeCalledTimes(0); + }); + + it('should send events to separate channels', async () => { + const sender = new TelemetryEventsSender(logger); + sender['telemetryStart'] = { + getIsOptedIn: jest.fn(async () => true), + }; + sender['telemetrySetup'] = { + getTelemetryUrl: jest.fn(async () => new URL('https://telemetry.elastic.co')), + }; + + sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }], 'my-channel'); + sender['queuesPerChannel']['my-channel']['sendEvents'] = jest.fn(); + + expect(sender['queuesPerChannel']['my-channel']['queue'].length).toBe(2); + + sender.queueTelemetryEvents([{ 'event.kind': '3' }], 'my-channel2'); + sender['queuesPerChannel']['my-channel2']['sendEvents'] = jest.fn(); + + expect(sender['queuesPerChannel']['my-channel2']['queue'].length).toBe(1); + await sender['sendIfDue'](); - expect(sender['queue'].length).toBe(0); - expect(sender['sendEvents']).toBeCalledTimes(0); + expect(sender['queuesPerChannel']['my-channel']['sendEvents']).toBeCalledTimes(1); + expect(sender['queuesPerChannel']['my-channel2']['sendEvents']).toBeCalledTimes(1); }); }); }); @@ -86,22 +94,22 @@ describe('getV3UrlFromV2', () => { it('should return prod url', () => { const sender = new TelemetryEventsSender(logger); - expect( - sender.getV3UrlFromV2('https://telemetry.elastic.co/xpack/v2/send', 'alerts-endpoint') - ).toBe('https://telemetry.elastic.co/v3/send/alerts-endpoint'); + expect(sender.getV3UrlFromV2('https://telemetry.elastic.co/xpack/v2/send', 'my-channel')).toBe( + 'https://telemetry.elastic.co/v3/send/my-channel' + ); }); it('should return staging url', () => { const sender = new TelemetryEventsSender(logger); expect( - sender.getV3UrlFromV2('https://telemetry-staging.elastic.co/xpack/v2/send', 'alerts-endpoint') - ).toBe('https://telemetry-staging.elastic.co/v3/send/alerts-endpoint'); + sender.getV3UrlFromV2('https://telemetry-staging.elastic.co/xpack/v2/send', 'my-channel') + ).toBe('https://telemetry-staging.elastic.co/v3/send/my-channel'); }); it('should support ports and auth', () => { const sender = new TelemetryEventsSender(logger); expect( - sender.getV3UrlFromV2('http://user:pass@myproxy.local:1337/xpack/v2/send', 'alerts-endpoint') - ).toBe('http://user:pass@myproxy.local:1337/v3/send/alerts-endpoint'); + sender.getV3UrlFromV2('http://user:pass@myproxy.local:1337/xpack/v2/send', 'my-channel') + ).toBe('http://user:pass@myproxy.local:1337/v3/send/my-channel'); }); }); diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index 8b5bef71872fb..c9b3b16f045d9 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -7,16 +7,12 @@ import { URL } from 'url'; -import { cloneDeep } from 'lodash'; -import axios from 'axios'; import type { Logger } from 'src/core/server'; import type { TelemetryPluginStart, TelemetryPluginSetup } from 'src/plugins/telemetry/server'; import type { TelemetryEvent } from './types'; import type { TelemetryReceiver } from './receiver'; - -export const TELEMETRY_MAX_BUFFER_SIZE = 100; -export const FLEET_CHANNEL_NAME = 'fleet-stats'; +import { TelemetryQueue } from './queue'; /** * Simplified version of https://github.com/elastic/kibana/blob/master/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -26,13 +22,13 @@ export class TelemetryEventsSender { private readonly initialCheckDelayMs = 10 * 1000; private readonly checkIntervalMs = 30 * 1000; private readonly logger: Logger; - private maxQueueSize = TELEMETRY_MAX_BUFFER_SIZE; + private telemetryStart?: TelemetryPluginStart; private telemetrySetup?: TelemetryPluginSetup; private intervalId?: NodeJS.Timeout; private isSending = false; private receiver: TelemetryReceiver | undefined; - private queue: TelemetryEvent[] = []; + private queuesPerChannel: { [channel: string]: TelemetryQueue } = {}; private isOptedIn?: boolean = true; // Assume true until the first check constructor(logger: Logger) { @@ -60,27 +56,11 @@ export class TelemetryEventsSender { } } - public queueTelemetryEvents(events: TelemetryEvent[]) { - const qlength = this.queue.length; - - if (events.length === 0) { - return; - } - - events = events.filter( - (event) => !this.queue.find((qItem) => qItem.id && event.id && qItem.id === event.id) - ); - - if (qlength >= this.maxQueueSize) { - // we're full already - return; - } - - if (events.length > this.maxQueueSize - qlength) { - this.queue.push(...events.slice(0, this.maxQueueSize - qlength)); - } else { - this.queue.push(...events); + public queueTelemetryEvents(events: TelemetryEvent[], channel: string) { + if (!this.queuesPerChannel[channel]) { + this.queuesPerChannel[channel] = new TelemetryQueue(this.logger); } + this.queuesPerChannel[channel].addEvents(events); } public async isTelemetryOptedIn() { @@ -93,53 +73,31 @@ export class TelemetryEventsSender { return; } - if (this.queue.length === 0) { - return; - } - - try { - this.isSending = true; + this.isSending = true; - this.isOptedIn = await this.isTelemetryOptedIn(); - if (!this.isOptedIn) { - this.logger.debug(`Telemetry is not opted-in.`); - this.queue = []; - this.isSending = false; - return; + this.isOptedIn = await this.isTelemetryOptedIn(); + if (!this.isOptedIn) { + this.logger.debug(`Telemetry is not opted-in.`); + for (const channel of Object.keys(this.queuesPerChannel)) { + this.queuesPerChannel[channel].clearEvents(); } + this.isSending = false; + return; + } - const [telemetryUrl, clusterInfo] = await Promise.all([ - this.fetchTelemetryUrl(FLEET_CHANNEL_NAME), - this.receiver?.fetchClusterInfo(), - ]); - - this.logger.debug(`Telemetry URL: ${telemetryUrl}`); - this.logger.debug( - `cluster_uuid: ${clusterInfo?.cluster_uuid} cluster_name: ${clusterInfo?.cluster_name}` - ); - - const toSend: TelemetryEvent[] = cloneDeep(this.queue).map((event) => ({ - ...event, - cluster_uuid: clusterInfo?.cluster_uuid, - cluster_name: clusterInfo?.cluster_name, - })); - this.queue = []; - - this.logger.debug(JSON.stringify(toSend)); + const clusterInfo = await this.receiver?.fetchClusterInfo(); - await this.sendEvents( - toSend, - telemetryUrl, - clusterInfo?.cluster_uuid, - clusterInfo?.version?.number + for (const channel of Object.keys(this.queuesPerChannel)) { + await this.queuesPerChannel[channel].sendEvents( + await this.fetchTelemetryUrl(channel), + clusterInfo ); - } catch (err) { - this.logger.warn(`Error sending telemetry events data: ${err}`); - this.queue = []; } + this.isSending = false; } + // TODO update once kibana uses v3 too https://github.com/elastic/kibana/pull/113525 private async fetchTelemetryUrl(channel: string): Promise { const telemetryUrl = await this.telemetrySetup?.getTelemetryUrl(); if (!telemetryUrl) { @@ -156,37 +114,4 @@ export class TelemetryEventsSender { url.pathname = `/v3/send/${channel}`; return url.toString(); } - - private async sendEvents( - events: unknown[], - telemetryUrl: string, - clusterUuid: string | undefined, - clusterVersionNumber: string | undefined - ) { - const ndjson = this.transformDataToNdjson(events); - - try { - const resp = await axios.post(telemetryUrl, ndjson, { - headers: { - 'Content-Type': 'application/x-ndjson', - 'X-Elastic-Cluster-ID': clusterUuid, - 'X-Elastic-Stack-Version': clusterVersionNumber ? clusterVersionNumber : '7.16.0', - }, - }); - this.logger.debug(`Events sent!. Response: ${resp.status} ${JSON.stringify(resp.data)}`); - } catch (err) { - this.logger.warn( - `Error sending events: ${err.response.status} ${JSON.stringify(err.response.data)}` - ); - } - } - - private transformDataToNdjson = (data: unknown[]): string => { - if (data.length !== 0) { - const dataString = data.map((dataItem) => JSON.stringify(dataItem)).join('\n'); - return `${dataString}\n`; - } else { - return ''; - } - }; } From caef79a0ff118dc7f73c6eed237571f113c2e1b4 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Wed, 20 Oct 2021 15:00:21 +0200 Subject: [PATCH 14/27] fix types --- .../plugins/fleet/server/telemetry/queue.test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/fleet/server/telemetry/queue.test.ts b/x-pack/plugins/fleet/server/telemetry/queue.test.ts index a79ca1004d0d6..671a9714d9977 100644 --- a/x-pack/plugins/fleet/server/telemetry/queue.test.ts +++ b/x-pack/plugins/fleet/server/telemetry/queue.test.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +/* eslint-disable dot-notation */ import { loggingSystemMock } from 'src/core/server/mocks'; import type { ESClusterInfo } from './types'; @@ -21,25 +21,25 @@ describe('TelemetryQueue', () => { it('queues two events', () => { const queue = new TelemetryQueue(logger); queue.addEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); - expect(queue.queue.length).toBe(2); + expect(queue['queue'].length).toBe(2); }); it('queues more than maxQueueSize events', () => { const queue = new TelemetryQueue(logger); queue.addEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); - queue.maxQueueSize = 5; + queue['maxQueueSize'] = 5; queue.addEvents([{ 'event.kind': '3' }, { 'event.kind': '4' }]); queue.addEvents([{ 'event.kind': '5' }, { 'event.kind': '6' }]); queue.addEvents([{ 'event.kind': '7' }, { 'event.kind': '8' }]); - expect(queue.queue.length).toBe(5); + expect(queue['queue'].length).toBe(5); }); it('empties the queue when sending', async () => { const queue = new TelemetryQueue(logger); - queue.send = jest.fn(); + queue['send'] = jest.fn(); queue.addEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); - expect(queue.queue.length).toBe(2); + expect(queue['queue'].length).toBe(2); await queue.sendEvents('https://telemetry.elastic.co/v3/send/my-channel', { cluster_uuid: '1', cluster_name: 'cluster', @@ -47,8 +47,8 @@ describe('TelemetryQueue', () => { number: '7.16.0', }, } as ESClusterInfo); - expect(queue.queue.length).toBe(0); - expect(queue.send).toHaveBeenCalledWith( + expect(queue['queue'].length).toBe(0); + expect(queue['send']).toHaveBeenCalledWith( [ { cluster_name: 'cluster', cluster_uuid: '1', 'event.kind': '1' }, { cluster_name: 'cluster', cluster_uuid: '1', 'event.kind': '2' }, From ae3b673f288ba635701c01557fd471649d61c582 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Wed, 20 Oct 2021 16:30:30 +0200 Subject: [PATCH 15/27] refactor --- .../fleet/server/telemetry/queue.test.ts | 39 ++------- .../plugins/fleet/server/telemetry/queue.ts | 74 +---------------- .../fleet/server/telemetry/sender.test.ts | 39 +++++++-- .../plugins/fleet/server/telemetry/sender.ts | 81 ++++++++++++++++++- 4 files changed, 119 insertions(+), 114 deletions(-) diff --git a/x-pack/plugins/fleet/server/telemetry/queue.test.ts b/x-pack/plugins/fleet/server/telemetry/queue.test.ts index 671a9714d9977..510b898387036 100644 --- a/x-pack/plugins/fleet/server/telemetry/queue.test.ts +++ b/x-pack/plugins/fleet/server/telemetry/queue.test.ts @@ -5,27 +5,18 @@ * 2.0. */ /* eslint-disable dot-notation */ -import { loggingSystemMock } from 'src/core/server/mocks'; - -import type { ESClusterInfo } from './types'; import { TelemetryQueue } from './queue'; describe('TelemetryQueue', () => { - let logger: ReturnType; - - beforeEach(() => { - logger = loggingSystemMock.createLogger(); - }); - describe('queueTelemetryEvents', () => { it('queues two events', () => { - const queue = new TelemetryQueue(logger); + const queue = new TelemetryQueue(); queue.addEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); expect(queue['queue'].length).toBe(2); }); it('queues more than maxQueueSize events', () => { - const queue = new TelemetryQueue(logger); + const queue = new TelemetryQueue(); queue.addEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); queue['maxQueueSize'] = 5; queue.addEvents([{ 'event.kind': '3' }, { 'event.kind': '4' }]); @@ -34,29 +25,15 @@ describe('TelemetryQueue', () => { expect(queue['queue'].length).toBe(5); }); - it('empties the queue when sending', async () => { - const queue = new TelemetryQueue(logger); - queue['send'] = jest.fn(); + it('get and clear events', async () => { + const queue = new TelemetryQueue(); queue.addEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); - expect(queue['queue'].length).toBe(2); - await queue.sendEvents('https://telemetry.elastic.co/v3/send/my-channel', { - cluster_uuid: '1', - cluster_name: 'cluster', - version: { - number: '7.16.0', - }, - } as ESClusterInfo); + expect(queue.getEvents().length).toBe(2); + + queue.clearEvents(); + expect(queue['queue'].length).toBe(0); - expect(queue['send']).toHaveBeenCalledWith( - [ - { cluster_name: 'cluster', cluster_uuid: '1', 'event.kind': '1' }, - { cluster_name: 'cluster', cluster_uuid: '1', 'event.kind': '2' }, - ], - 'https://telemetry.elastic.co/v3/send/my-channel', - '1', - '7.16.0' - ); }); }); }); diff --git a/x-pack/plugins/fleet/server/telemetry/queue.ts b/x-pack/plugins/fleet/server/telemetry/queue.ts index 736f65e8d89b9..773dc764996ad 100644 --- a/x-pack/plugins/fleet/server/telemetry/queue.ts +++ b/x-pack/plugins/fleet/server/telemetry/queue.ts @@ -5,23 +5,13 @@ * 2.0. */ -import { cloneDeep } from 'lodash'; -import type { Logger } from 'src/core/server'; - -import axios from 'axios'; - -import type { ESClusterInfo, TelemetryEvent } from './types'; +import type { TelemetryEvent } from './types'; export const TELEMETRY_MAX_BUFFER_SIZE = 100; export class TelemetryQueue { private maxQueueSize = TELEMETRY_MAX_BUFFER_SIZE; private queue: TelemetryEvent[] = []; - private readonly logger: Logger; - - constructor(logger: Logger) { - this.logger = logger; - } public addEvents(events: TelemetryEvent[]) { const qlength = this.queue.length; @@ -51,65 +41,7 @@ export class TelemetryQueue { this.queue = []; } - public async sendEvents(telemetryUrl: string, clusterInfo: ESClusterInfo | undefined) { - if (this.queue.length === 0) { - return; - } - - try { - this.logger.debug(`Telemetry URL: ${telemetryUrl}`); - - const toSend: TelemetryEvent[] = cloneDeep(this.queue).map((event) => ({ - ...event, - cluster_uuid: clusterInfo?.cluster_uuid, - cluster_name: clusterInfo?.cluster_name, - })); - this.queue = []; - - this.logger.debug(JSON.stringify(toSend)); - - await this.send( - toSend, - telemetryUrl, - clusterInfo?.cluster_uuid, - clusterInfo?.version?.number - ); - } catch (err) { - this.logger.warn(`Error sending telemetry events data: ${err}`); - this.queue = []; - } - } - - private async send( - events: unknown[], - telemetryUrl: string, - clusterUuid: string | undefined, - clusterVersionNumber: string | undefined - ) { - const ndjson = this.transformDataToNdjson(events); - - try { - const resp = await axios.post(telemetryUrl, ndjson, { - headers: { - 'Content-Type': 'application/x-ndjson', - 'X-Elastic-Cluster-ID': clusterUuid, - 'X-Elastic-Stack-Version': clusterVersionNumber ? clusterVersionNumber : '7.16.0', - }, - }); - this.logger.debug(`Events sent!. Response: ${resp.status} ${JSON.stringify(resp.data)}`); - } catch (err) { - this.logger.warn( - `Error sending events: ${err.response.status} ${JSON.stringify(err.response.data)}` - ); - } + public getEvents(): TelemetryEvent[] { + return this.queue; } - - private transformDataToNdjson = (data: unknown[]): string => { - if (data.length !== 0) { - const dataString = data.map((dataItem) => JSON.stringify(dataItem)).join('\n'); - return `${dataString}\n`; - } else { - return ''; - } - }; } diff --git a/x-pack/plugins/fleet/server/telemetry/sender.test.ts b/x-pack/plugins/fleet/server/telemetry/sender.test.ts index 1ceaa5d68b3c3..06672b8966f4d 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.test.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.test.ts @@ -10,8 +10,16 @@ import { URL } from 'url'; import { loggingSystemMock } from 'src/core/server/mocks'; +import axios from 'axios'; + import { TelemetryEventsSender } from './sender'; +jest.mock('axios', () => { + return { + post: jest.fn(), + }; +}); + describe('TelemetryEventsSender', () => { let logger: ReturnType; @@ -36,11 +44,11 @@ describe('TelemetryEventsSender', () => { }; sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }], 'my-channel'); - sender['queuesPerChannel']['my-channel']['sendEvents'] = jest.fn(); + sender['sendEvents'] = jest.fn(); await sender['sendIfDue'](); - expect(sender['queuesPerChannel']['my-channel']['sendEvents']).toBeCalledTimes(1); + expect(sender['sendEvents']).toBeCalledTimes(1); }); it("shouldn't send when telemetry is disabled", async () => { @@ -51,11 +59,11 @@ describe('TelemetryEventsSender', () => { sender['telemetryStart'] = telemetryStart; sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }], 'my-channel'); - sender['queuesPerChannel']['my-channel']['sendEvents'] = jest.fn(); + sender['sendEvents'] = jest.fn(); await sender['sendIfDue'](); - expect(sender['queuesPerChannel']['my-channel']['sendEvents']).toBeCalledTimes(0); + expect(sender['sendEvents']).toBeCalledTimes(0); }); it('should send events to separate channels', async () => { @@ -68,19 +76,34 @@ describe('TelemetryEventsSender', () => { }; sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }], 'my-channel'); - sender['queuesPerChannel']['my-channel']['sendEvents'] = jest.fn(); + sender['queuesPerChannel']['my-channel']['getEvents'] = jest.fn(() => [ + { 'event.kind': '1' }, + { 'event.kind': '2' }, + ]); expect(sender['queuesPerChannel']['my-channel']['queue'].length).toBe(2); sender.queueTelemetryEvents([{ 'event.kind': '3' }], 'my-channel2'); - sender['queuesPerChannel']['my-channel2']['sendEvents'] = jest.fn(); + sender['queuesPerChannel']['my-channel2']['getEvents'] = jest.fn(() => [ + { 'event.kind': '3' }, + ]); expect(sender['queuesPerChannel']['my-channel2']['queue'].length).toBe(1); await sender['sendIfDue'](); - expect(sender['queuesPerChannel']['my-channel']['sendEvents']).toBeCalledTimes(1); - expect(sender['queuesPerChannel']['my-channel2']['sendEvents']).toBeCalledTimes(1); + expect(sender['queuesPerChannel']['my-channel']['getEvents']).toBeCalledTimes(1); + expect(sender['queuesPerChannel']['my-channel2']['getEvents']).toBeCalledTimes(1); + expect(axios.post).toHaveBeenCalledWith( + 'https://telemetry.elastic.co/v3/send/my-channel', + expect.anything(), + expect.anything() + ); + expect(axios.post).toHaveBeenCalledWith( + 'https://telemetry.elastic.co/v3/send/my-channel2', + expect.anything(), + expect.anything() + ); }); }); }); diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index c9b3b16f045d9..a7f334349ebb7 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -10,10 +10,15 @@ import { URL } from 'url'; import type { Logger } from 'src/core/server'; import type { TelemetryPluginStart, TelemetryPluginSetup } from 'src/plugins/telemetry/server'; -import type { TelemetryEvent } from './types'; +import { cloneDeep } from 'lodash'; + +import axios from 'axios'; + import type { TelemetryReceiver } from './receiver'; import { TelemetryQueue } from './queue'; +import type { ESClusterInfo, TelemetryEvent } from './types'; + /** * Simplified version of https://github.com/elastic/kibana/blob/master/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts * Sends batched events to telemetry v3 api @@ -58,7 +63,7 @@ export class TelemetryEventsSender { public queueTelemetryEvents(events: TelemetryEvent[], channel: string) { if (!this.queuesPerChannel[channel]) { - this.queuesPerChannel[channel] = new TelemetryQueue(this.logger); + this.queuesPerChannel[channel] = new TelemetryQueue(); } this.queuesPerChannel[channel].addEvents(events); } @@ -88,15 +93,50 @@ export class TelemetryEventsSender { const clusterInfo = await this.receiver?.fetchClusterInfo(); for (const channel of Object.keys(this.queuesPerChannel)) { - await this.queuesPerChannel[channel].sendEvents( + await this.sendEvents( await this.fetchTelemetryUrl(channel), - clusterInfo + clusterInfo, + this.queuesPerChannel[channel] ); } this.isSending = false; } + public async sendEvents( + telemetryUrl: string, + clusterInfo: ESClusterInfo | undefined, + queue: TelemetryQueue + ) { + const events = queue.getEvents(); + if (events.length === 0) { + return; + } + + try { + this.logger.debug(`Telemetry URL: ${telemetryUrl}`); + + const toSend: TelemetryEvent[] = cloneDeep(events).map((event) => ({ + ...event, + cluster_uuid: clusterInfo?.cluster_uuid, + cluster_name: clusterInfo?.cluster_name, + })); + queue.clearEvents(); + + this.logger.debug(JSON.stringify(toSend)); + + await this.send( + toSend, + telemetryUrl, + clusterInfo?.cluster_uuid, + clusterInfo?.version?.number + ); + } catch (err) { + this.logger.warn(`Error sending telemetry events data: ${err}`); + queue.clearEvents(); + } + } + // TODO update once kibana uses v3 too https://github.com/elastic/kibana/pull/113525 private async fetchTelemetryUrl(channel: string): Promise { const telemetryUrl = await this.telemetrySetup?.getTelemetryUrl(); @@ -114,4 +154,37 @@ export class TelemetryEventsSender { url.pathname = `/v3/send/${channel}`; return url.toString(); } + + private async send( + events: unknown[], + telemetryUrl: string, + clusterUuid: string | undefined, + clusterVersionNumber: string | undefined + ) { + const ndjson = this.transformDataToNdjson(events); + + try { + const resp = await axios.post(telemetryUrl, ndjson, { + headers: { + 'Content-Type': 'application/x-ndjson', + 'X-Elastic-Cluster-ID': clusterUuid, + 'X-Elastic-Stack-Version': clusterVersionNumber ? clusterVersionNumber : '7.16.0', + }, + }); + this.logger.debug(`Events sent!. Response: ${resp.status} ${JSON.stringify(resp.data)}`); + } catch (err) { + this.logger.warn( + `Error sending events: ${err.response.status} ${JSON.stringify(err.response.data)}` + ); + } + } + + private transformDataToNdjson = (data: unknown[]): string => { + if (data.length !== 0) { + const dataString = data.map((dataItem) => JSON.stringify(dataItem)).join('\n'); + return `${dataString}\n`; + } else { + return ''; + } + }; } From 3798547de1a918294a91e754447219f8a2def96c Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Wed, 20 Oct 2021 17:00:10 +0200 Subject: [PATCH 16/27] using json content type --- .../fleet/server/telemetry/sender.test.ts | 46 +++++++++++++------ .../plugins/fleet/server/telemetry/sender.ts | 15 +----- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/fleet/server/telemetry/sender.test.ts b/x-pack/plugins/fleet/server/telemetry/sender.test.ts index 06672b8966f4d..91255b7c2d1c2 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.test.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.test.ts @@ -13,6 +13,7 @@ import { loggingSystemMock } from 'src/core/server/mocks'; import axios from 'axios'; import { TelemetryEventsSender } from './sender'; +import type { ESClusterInfo } from './types'; jest.mock('axios', () => { return { @@ -74,19 +75,28 @@ describe('TelemetryEventsSender', () => { sender['telemetrySetup'] = { getTelemetryUrl: jest.fn(async () => new URL('https://telemetry.elastic.co')), }; + sender['receiver'] = { + start: jest.fn(), + fetchClusterInfo: jest.fn(async () => { + return { + cluster_uuid: '1', + cluster_name: 'name', + version: { + number: '8.0.0', + }, + } as ESClusterInfo; + }), + }; - sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }], 'my-channel'); - sender['queuesPerChannel']['my-channel']['getEvents'] = jest.fn(() => [ - { 'event.kind': '1' }, - { 'event.kind': '2' }, - ]); + const myChannelEvents = [{ 'event.kind': '1' }, { 'event.kind': '2' }]; + sender.queueTelemetryEvents(myChannelEvents, 'my-channel'); + sender['queuesPerChannel']['my-channel']['getEvents'] = jest.fn(() => myChannelEvents); expect(sender['queuesPerChannel']['my-channel']['queue'].length).toBe(2); - sender.queueTelemetryEvents([{ 'event.kind': '3' }], 'my-channel2'); - sender['queuesPerChannel']['my-channel2']['getEvents'] = jest.fn(() => [ - { 'event.kind': '3' }, - ]); + const myChannel2Events = [{ 'event.kind': '3' }]; + sender.queueTelemetryEvents(myChannel2Events, 'my-channel2'); + sender['queuesPerChannel']['my-channel2']['getEvents'] = jest.fn(() => myChannel2Events); expect(sender['queuesPerChannel']['my-channel2']['queue'].length).toBe(1); @@ -94,15 +104,25 @@ describe('TelemetryEventsSender', () => { expect(sender['queuesPerChannel']['my-channel']['getEvents']).toBeCalledTimes(1); expect(sender['queuesPerChannel']['my-channel2']['getEvents']).toBeCalledTimes(1); + const headers = { + headers: { + 'Content-Type': 'application/json', + 'X-Elastic-Cluster-ID': '1', + 'X-Elastic-Stack-Version': '8.0.0', + }, + }; expect(axios.post).toHaveBeenCalledWith( 'https://telemetry.elastic.co/v3/send/my-channel', - expect.anything(), - expect.anything() + [ + { cluster_name: 'name', cluster_uuid: '1', 'event.kind': '1' }, + { cluster_name: 'name', cluster_uuid: '1', 'event.kind': '2' }, + ], + headers ); expect(axios.post).toHaveBeenCalledWith( 'https://telemetry.elastic.co/v3/send/my-channel2', - expect.anything(), - expect.anything() + [{ cluster_name: 'name', cluster_uuid: '1', 'event.kind': '3' }], + headers ); }); }); diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index a7f334349ebb7..3da0663ec5052 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -161,12 +161,10 @@ export class TelemetryEventsSender { clusterUuid: string | undefined, clusterVersionNumber: string | undefined ) { - const ndjson = this.transformDataToNdjson(events); - try { - const resp = await axios.post(telemetryUrl, ndjson, { + const resp = await axios.post(telemetryUrl, events, { headers: { - 'Content-Type': 'application/x-ndjson', + 'Content-Type': 'application/json', 'X-Elastic-Cluster-ID': clusterUuid, 'X-Elastic-Stack-Version': clusterVersionNumber ? clusterVersionNumber : '7.16.0', }, @@ -178,13 +176,4 @@ export class TelemetryEventsSender { ); } } - - private transformDataToNdjson = (data: unknown[]): string => { - if (data.length !== 0) { - const dataString = data.map((dataItem) => JSON.stringify(dataItem)).join('\n'); - return `${dataString}\n`; - } else { - return ''; - } - }; } From b210f64010428e3cbea64bea7724da12e67cae88 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Wed, 20 Oct 2021 18:34:15 +0200 Subject: [PATCH 17/27] fix test --- .../server/services/upgrade_usage.test.ts | 43 ++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts b/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts index 369da5903577f..c9c1a290d4090 100644 --- a/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts +++ b/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts @@ -37,27 +37,30 @@ describe('sendTelemetryEvents', () => { sendTelemetryEvents(loggerMock, eventsTelemetryMock, upgardeMessage); - expect(eventsTelemetryMock.queueTelemetryEvents).toHaveBeenCalledWith([ - { - id: 'aws_0.6.1_1.3.0_failure', - package_policy_upgrade: { - current_version: '0.6.1', - error: [ - { - key: 'fieldX', - message: ['Field is required'], - }, - { - key: 'fieldX', - message: 'Invalid format', - }, - ], - new_version: '1.3.0', - package_name: 'aws', - status: 'failure', + expect(eventsTelemetryMock.queueTelemetryEvents).toHaveBeenCalledWith( + [ + { + id: 'aws_0.6.1_1.3.0_failure', + package_policy_upgrade: { + current_version: '0.6.1', + error: [ + { + key: 'fieldX', + message: ['Field is required'], + }, + { + key: 'fieldX', + message: 'Invalid format', + }, + ], + new_version: '1.3.0', + package_name: 'aws', + status: 'failure', + }, }, - }, - ]); + ], + 'fleet-upgrades' + ); }); it('should cap error size', () => { From 786b63f5b7d129d848b4b3f7d942fb3fd11e7016 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Thu, 21 Oct 2021 09:23:12 +0200 Subject: [PATCH 18/27] simplified telemetry url --- .../fleet/server/telemetry/sender.test.ts | 43 +++++-------------- .../plugins/fleet/server/telemetry/sender.ts | 13 ++---- 2 files changed, 15 insertions(+), 41 deletions(-) diff --git a/x-pack/plugins/fleet/server/telemetry/sender.test.ts b/x-pack/plugins/fleet/server/telemetry/sender.test.ts index 91255b7c2d1c2..4a5d1fd4d01e3 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.test.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.test.ts @@ -41,7 +41,9 @@ describe('TelemetryEventsSender', () => { getIsOptedIn: jest.fn(async () => true), }; sender['telemetrySetup'] = { - getTelemetryUrl: jest.fn(async () => new URL('https://telemetry.elastic.co')), + getTelemetryUrl: jest.fn( + async () => new URL('https://telemetry-staging.elastic.co/v3/send/snapshot') + ), }; sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }], 'my-channel'); @@ -49,7 +51,11 @@ describe('TelemetryEventsSender', () => { await sender['sendIfDue'](); - expect(sender['sendEvents']).toBeCalledTimes(1); + expect(sender['sendEvents']).toHaveBeenCalledWith( + 'https://telemetry-staging.elastic.co/v3/send/my-channel', + undefined, + expect.anything() + ); }); it("shouldn't send when telemetry is disabled", async () => { @@ -73,7 +79,9 @@ describe('TelemetryEventsSender', () => { getIsOptedIn: jest.fn(async () => true), }; sender['telemetrySetup'] = { - getTelemetryUrl: jest.fn(async () => new URL('https://telemetry.elastic.co')), + getTelemetryUrl: jest.fn( + async () => new URL('https://telemetry.elastic.co/v3/send/snapshot') + ), }; sender['receiver'] = { start: jest.fn(), @@ -127,32 +135,3 @@ describe('TelemetryEventsSender', () => { }); }); }); - -describe('getV3UrlFromV2', () => { - let logger: ReturnType; - - beforeEach(() => { - logger = loggingSystemMock.createLogger(); - }); - - it('should return prod url', () => { - const sender = new TelemetryEventsSender(logger); - expect(sender.getV3UrlFromV2('https://telemetry.elastic.co/xpack/v2/send', 'my-channel')).toBe( - 'https://telemetry.elastic.co/v3/send/my-channel' - ); - }); - - it('should return staging url', () => { - const sender = new TelemetryEventsSender(logger); - expect( - sender.getV3UrlFromV2('https://telemetry-staging.elastic.co/xpack/v2/send', 'my-channel') - ).toBe('https://telemetry-staging.elastic.co/v3/send/my-channel'); - }); - - it('should support ports and auth', () => { - const sender = new TelemetryEventsSender(logger); - expect( - sender.getV3UrlFromV2('http://user:pass@myproxy.local:1337/xpack/v2/send', 'my-channel') - ).toBe('http://user:pass@myproxy.local:1337/v3/send/my-channel'); - }); -}); diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index 3da0663ec5052..5e8d6cacdd5ce 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -137,20 +137,15 @@ export class TelemetryEventsSender { } } - // TODO update once kibana uses v3 too https://github.com/elastic/kibana/pull/113525 + // Forms URLs like: + // https://telemetry.elastic.co/v3/send/my-channel-name or + // https://telemetry-staging.elastic.co/v3/send/my-channel-name private async fetchTelemetryUrl(channel: string): Promise { const telemetryUrl = await this.telemetrySetup?.getTelemetryUrl(); if (!telemetryUrl) { throw Error("Couldn't get telemetry URL"); } - return this.getV3UrlFromV2(telemetryUrl.toString(), channel); - } - - // Forms URLs like: - // https://telemetry.elastic.co/v3/send/my-channel-name or - // https://telemetry-staging.elastic.co/v3/send/my-channel-name - public getV3UrlFromV2(v2url: string, channel: string): string { - const url = new URL(v2url); + const url = new URL(telemetryUrl); url.pathname = `/v3/send/${channel}`; return url.toString(); } From ec3b97748d3edaee24788c0e56ba18a98ab0ec10 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Thu, 21 Oct 2021 10:47:12 +0200 Subject: [PATCH 19/27] fixed type --- x-pack/plugins/fleet/server/telemetry/sender.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index 5e8d6cacdd5ce..5de65e3aaf575 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -5,8 +5,6 @@ * 2.0. */ -import { URL } from 'url'; - import type { Logger } from 'src/core/server'; import type { TelemetryPluginStart, TelemetryPluginSetup } from 'src/plugins/telemetry/server'; @@ -145,9 +143,8 @@ export class TelemetryEventsSender { if (!telemetryUrl) { throw Error("Couldn't get telemetry URL"); } - const url = new URL(telemetryUrl); - url.pathname = `/v3/send/${channel}`; - return url.toString(); + telemetryUrl.pathname = `/v3/send/${channel}`; + return telemetryUrl.toString(); } private async send( From b0bf43b02d59ef9b46040e11f808ed1ede4496dd Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Sat, 23 Oct 2021 11:33:52 +0200 Subject: [PATCH 20/27] added back ndjson --- .../fleet/server/telemetry/sender.test.ts | 9 +++------ x-pack/plugins/fleet/server/telemetry/sender.ts | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/fleet/server/telemetry/sender.test.ts b/x-pack/plugins/fleet/server/telemetry/sender.test.ts index 4a5d1fd4d01e3..924d2b9f3d148 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.test.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.test.ts @@ -114,22 +114,19 @@ describe('TelemetryEventsSender', () => { expect(sender['queuesPerChannel']['my-channel2']['getEvents']).toBeCalledTimes(1); const headers = { headers: { - 'Content-Type': 'application/json', + 'Content-Type': 'application/x-ndjson', 'X-Elastic-Cluster-ID': '1', 'X-Elastic-Stack-Version': '8.0.0', }, }; expect(axios.post).toHaveBeenCalledWith( 'https://telemetry.elastic.co/v3/send/my-channel', - [ - { cluster_name: 'name', cluster_uuid: '1', 'event.kind': '1' }, - { cluster_name: 'name', cluster_uuid: '1', 'event.kind': '2' }, - ], + '{"event.kind":"1","cluster_uuid":"1","cluster_name":"name"}\n{"event.kind":"2","cluster_uuid":"1","cluster_name":"name"}\n', headers ); expect(axios.post).toHaveBeenCalledWith( 'https://telemetry.elastic.co/v3/send/my-channel2', - [{ cluster_name: 'name', cluster_uuid: '1', 'event.kind': '3' }], + '{"event.kind":"3","cluster_uuid":"1","cluster_name":"name"}\n', headers ); }); diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index 5de65e3aaf575..3ccab058167c7 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -153,10 +153,14 @@ export class TelemetryEventsSender { clusterUuid: string | undefined, clusterVersionNumber: string | undefined ) { + // using ndjson so that each line will be wrapped in json envelope on server side + // see https://github.com/elastic/infra/blob/master/docs/telemetry/telemetry-next-dataflow.md#json-envelope + const ndjson = this.transformDataToNdjson(events); + try { - const resp = await axios.post(telemetryUrl, events, { + const resp = await axios.post(telemetryUrl, ndjson, { headers: { - 'Content-Type': 'application/json', + 'Content-Type': 'application/x-ndjson', 'X-Elastic-Cluster-ID': clusterUuid, 'X-Elastic-Stack-Version': clusterVersionNumber ? clusterVersionNumber : '7.16.0', }, @@ -168,4 +172,13 @@ export class TelemetryEventsSender { ); } } + + private transformDataToNdjson = (data: unknown[]): string => { + if (data.length !== 0) { + const dataString = data.map((dataItem) => JSON.stringify(dataItem)).join('\n'); + return `${dataString}\n`; + } else { + return ''; + } + }; } From c77d7f6af262cd6959492bf1efde3a846a8c2a94 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Mon, 25 Oct 2021 10:52:10 +0200 Subject: [PATCH 21/27] moved telemetry to update, added dryrun --- .../server/routes/package_policy/handlers.ts | 3 +- .../server/services/package_policy.test.ts | 24 +++++-- .../fleet/server/services/package_policy.ts | 64 +++++++++++-------- .../server/services/upgrade_usage.test.ts | 4 +- .../fleet/server/services/upgrade_usage.ts | 3 +- 5 files changed, 63 insertions(+), 35 deletions(-) diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts index 77e7a2c4ede1a..58463bfa5569d 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts @@ -146,7 +146,8 @@ export const updatePackagePolicyHandler: RequestHandler< esClient, request.params.packagePolicyId, { ...newData, package: pkg, inputs }, - { user } + { user }, + packagePolicy.package?.version ); return response.ok({ body: { item: updatedPackagePolicy }, diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts index c25a1db753c73..a141203bfc81a 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts @@ -134,6 +134,12 @@ jest.mock('./epm/packages/cleanup', () => { }; }); +jest.mock('./upgrade_usage', () => { + return { + sendTelemetryEvents: jest.fn(), + }; +}); + const mockedFetchInfo = fetchInfo as jest.Mock>; type CombinedExternalCallback = PutPackagePolicyUpdateCallback | PostPackagePolicyCreateCallback; @@ -578,6 +584,12 @@ describe('Package policy service', () => { }); describe('update', () => { + beforeEach(() => { + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(() => { + appContextService.stop(); + }); it('should fail to update on version conflict', async () => { const savedObjectsClient = savedObjectsClientMock.create(); savedObjectsClient.get.mockResolvedValue({ @@ -601,7 +613,8 @@ describe('Package policy service', () => { savedObjectsClient, elasticsearchClient, 'the-package-policy-id', - createPackagePolicyMock() + createPackagePolicyMock(), + 'current-version' ) ).rejects.toThrow('Saved object [abc/123] conflict'); }); @@ -721,7 +734,8 @@ describe('Package policy service', () => { savedObjectsClient, elasticsearchClient, 'the-package-policy-id', - { ...mockPackagePolicy, inputs: inputsUpdate } + { ...mockPackagePolicy, inputs: inputsUpdate }, + 'current-version' ); const [modifiedInput] = result.inputs; @@ -844,7 +858,8 @@ describe('Package policy service', () => { savedObjectsClient, elasticsearchClient, 'the-package-policy-id', - { ...mockPackagePolicy, inputs: inputsUpdate } + { ...mockPackagePolicy, inputs: inputsUpdate }, + 'current-version' ); const [modifiedInput] = result.inputs; @@ -903,7 +918,8 @@ describe('Package policy service', () => { savedObjectsClient, elasticsearchClient, 'the-package-policy-id', - { ...mockPackagePolicy, inputs: [] } + { ...mockPackagePolicy, inputs: [] }, + 'current-version' ); expect(result.elasticsearch).toMatchObject({ privileges: { cluster: ['monitor'] } }); diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index b0cbfb08090e6..5b2cce5417bc2 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -359,7 +359,8 @@ class PackagePolicyService { esClient: ElasticsearchClient, id: string, packagePolicy: UpdatePackagePolicy, - options?: { user?: AuthenticatedUser } + options?: { user?: AuthenticatedUser }, + currentVersion?: string ): Promise { const oldPackagePolicy = await this.get(soClient, id); const { version, ...restOfPackagePolicy } = packagePolicy; @@ -433,6 +434,20 @@ class PackagePolicyService { pkgName: packagePolicy.package.name, currentVersion: packagePolicy.package.version, }); + + if (packagePolicy.package.version !== currentVersion) { + sendTelemetryEvents( + appContextService.getLogger(), + appContextService.getTelemetryEventsSender(), + { + package_name: packagePolicy.package.name, + current_version: currentVersion || 'unknown', + new_version: packagePolicy.package.version, + status: 'success', + dryRun: false, + } + ); + } } return newPolicy; @@ -601,25 +616,19 @@ class PackagePolicyService { ); updatePackagePolicy.elasticsearch = registryPkgInfo.elasticsearch; - await this.update(soClient, esClient, id, updatePackagePolicy, options); + await this.update( + soClient, + esClient, + id, + updatePackagePolicy, + options, + packagePolicy.package.version + ); result.push({ id, name: packagePolicy.name, success: true, }); - - if (packagePolicy.package.version !== packageInfo.version) { - sendTelemetryEvents( - appContextService.getLogger(), - appContextService.getTelemetryEventsSender(), - { - package_name: packageInfo.name, - current_version: packagePolicy.package.version, - new_version: packageInfo.version, - status: 'success', - } - ); - } } catch (error) { // We only want to specifically handle validation errors for the new package policy. If a more severe or // general error is thrown elsewhere during the upgrade process, we want to surface that directly in @@ -676,19 +685,18 @@ class PackagePolicyService { const hasErrors = 'errors' in updatedPackagePolicy; if (packagePolicy.package.version !== packageInfo.version) { - if (hasErrors) { - sendTelemetryEvents( - appContextService.getLogger(), - appContextService.getTelemetryEventsSender(), - { - package_name: packageInfo.name, - current_version: packagePolicy.package.version, - new_version: packageInfo.version, - status: hasErrors ? 'failure' : 'success', - error: updatedPackagePolicy.errors, - } - ); - } + sendTelemetryEvents( + appContextService.getLogger(), + appContextService.getTelemetryEventsSender(), + { + package_name: packageInfo.name, + current_version: packagePolicy.package.version, + new_version: packageInfo.version, + status: hasErrors ? 'failure' : 'success', + error: hasErrors ? updatedPackagePolicy.errors : undefined, + dryRun: true, + } + ); } return { diff --git a/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts b/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts index c9c1a290d4090..e14aa37e4a5e7 100644 --- a/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts +++ b/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts @@ -33,6 +33,7 @@ describe('sendTelemetryEvents', () => { { key: 'fieldX', message: ['Field X is required'] }, { key: 'fieldX', message: 'Invalid format' }, ], + dryRun: true, }; sendTelemetryEvents(loggerMock, eventsTelemetryMock, upgardeMessage); @@ -40,7 +41,7 @@ describe('sendTelemetryEvents', () => { expect(eventsTelemetryMock.queueTelemetryEvents).toHaveBeenCalledWith( [ { - id: 'aws_0.6.1_1.3.0_failure', + id: 'aws_0.6.1_1.3.0_failure_true', package_policy_upgrade: { current_version: '0.6.1', error: [ @@ -56,6 +57,7 @@ describe('sendTelemetryEvents', () => { new_version: '1.3.0', package_name: 'aws', status: 'failure', + dryRun: true, }, }, ], diff --git a/x-pack/plugins/fleet/server/services/upgrade_usage.ts b/x-pack/plugins/fleet/server/services/upgrade_usage.ts index c69ddf0261cd2..9be25ad3e8cbb 100644 --- a/x-pack/plugins/fleet/server/services/upgrade_usage.ts +++ b/x-pack/plugins/fleet/server/services/upgrade_usage.ts @@ -15,6 +15,7 @@ export interface PackagePolicyUpgradeUsage { new_version: string; status: 'success' | 'failure'; error?: UpgradeError[]; + dryRun?: boolean; } export interface UpgradeError { @@ -44,7 +45,7 @@ export function sendTelemetryEvents( ? makeErrorGeneric(capErrorSize(upgradeUsage.error, MAX_ERROR_SIZE)) : undefined, }, - id: `${upgradeUsage.package_name}_${upgradeUsage.current_version}_${upgradeUsage.new_version}_${upgradeUsage.status}`, + id: `${upgradeUsage.package_name}_${upgradeUsage.current_version}_${upgradeUsage.new_version}_${upgradeUsage.status}_${upgradeUsage.dryRun}`, }, ], FLEET_UPGRADES_CHANNEL_NAME From ee2c91699f886e523457ed8bc4f2c26441462ca4 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Mon, 25 Oct 2021 11:21:53 +0200 Subject: [PATCH 22/27] fix types --- .../fleet/server/services/package_policy.test.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts index a141203bfc81a..73e74eed5c27d 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts @@ -613,8 +613,7 @@ describe('Package policy service', () => { savedObjectsClient, elasticsearchClient, 'the-package-policy-id', - createPackagePolicyMock(), - 'current-version' + createPackagePolicyMock() ) ).rejects.toThrow('Saved object [abc/123] conflict'); }); @@ -734,8 +733,7 @@ describe('Package policy service', () => { savedObjectsClient, elasticsearchClient, 'the-package-policy-id', - { ...mockPackagePolicy, inputs: inputsUpdate }, - 'current-version' + { ...mockPackagePolicy, inputs: inputsUpdate } ); const [modifiedInput] = result.inputs; @@ -858,8 +856,7 @@ describe('Package policy service', () => { savedObjectsClient, elasticsearchClient, 'the-package-policy-id', - { ...mockPackagePolicy, inputs: inputsUpdate }, - 'current-version' + { ...mockPackagePolicy, inputs: inputsUpdate } ); const [modifiedInput] = result.inputs; @@ -918,8 +915,7 @@ describe('Package policy service', () => { savedObjectsClient, elasticsearchClient, 'the-package-policy-id', - { ...mockPackagePolicy, inputs: [] }, - 'current-version' + { ...mockPackagePolicy, inputs: [] } ); expect(result.elasticsearch).toMatchObject({ privileges: { cluster: ['monitor'] } }); From 74b6c1f967f8ae2c0fcdbc538781bd712806e154 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Tue, 26 Oct 2021 09:54:23 +0200 Subject: [PATCH 23/27] fix prettier --- x-pack/plugins/fleet/server/services/package_policy.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 6d4e77bd1aefc..245132dafea59 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -436,11 +436,7 @@ class PackagePolicyService { }); if (packagePolicy.package.version !== currentVersion) { - appContextService - .getLogger() - .info( - `Package policy upgraded successfully` - ); + appContextService.getLogger().info(`Package policy upgraded successfully`); sendTelemetryEvents( appContextService.getLogger(), appContextService.getTelemetryEventsSender(), From cb591ad4562d496847bc85927be0b211654e7c10 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Wed, 27 Oct 2021 18:33:39 +0200 Subject: [PATCH 24/27] updated after review --- x-pack/plugins/fleet/server/plugin.ts | 9 +-- .../fleet/server/services/package_policy.ts | 45 +++++++------- .../server/services/upgrade_usage.test.ts | 42 ++++++------- .../fleet/server/services/upgrade_usage.ts | 22 +++---- .../plugins/fleet/server/telemetry/queue.ts | 15 ++--- .../fleet/server/telemetry/receiver.ts | 27 --------- .../fleet/server/telemetry/sender.test.ts | 60 +++++++++++-------- .../plugins/fleet/server/telemetry/sender.ts | 50 +++++++++------- .../plugins/fleet/server/telemetry/types.ts | 27 ++------- 9 files changed, 125 insertions(+), 172 deletions(-) delete mode 100644 x-pack/plugins/fleet/server/telemetry/receiver.ts diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index c361d1790b855..7cc1b8b1cfcc9 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -85,7 +85,6 @@ import { RouterWrappers } from './routes/security'; import { startFleetServerSetup } from './services/fleet_server'; import { FleetArtifactsClient } from './services/artifacts'; import type { FleetRouter } from './types/request_context'; -import { TelemetryReceiver } from './telemetry/receiver'; import { TelemetryEventsSender } from './telemetry/sender'; export interface FleetSetupDeps { @@ -183,7 +182,6 @@ export class FleetPlugin private httpSetup?: HttpServiceSetup; private securitySetup?: SecurityPluginSetup; private encryptedSavedObjectsSetup?: EncryptedSavedObjectsPluginSetup; - private readonly telemetryReceiver: TelemetryReceiver; private readonly telemetryEventsSender: TelemetryEventsSender; constructor(private readonly initializerContext: PluginInitializerContext) { @@ -193,8 +191,7 @@ export class FleetPlugin this.kibanaBranch = this.initializerContext.env.packageInfo.branch; this.logger = this.initializerContext.logger.get(); this.configInitialValue = this.initializerContext.config.get(); - this.telemetryEventsSender = new TelemetryEventsSender(this.logger); - this.telemetryReceiver = new TelemetryReceiver(); + this.telemetryEventsSender = new TelemetryEventsSender(this.logger.get('telemetry_events')); } public setup(core: CoreSetup, deps: FleetSetupDeps) { @@ -340,9 +337,7 @@ export class FleetPlugin const fleetServerSetup = startFleetServerSetup(); - this.telemetryReceiver.start(core); - - this.telemetryEventsSender.start(plugins.telemetry, this.telemetryReceiver); + this.telemetryEventsSender.start(plugins.telemetry, core); return { fleetSetupCompleted: () => diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 245132dafea59..c81c77eb0fb14 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -68,6 +68,7 @@ import { compileTemplate } from './epm/agent/agent'; import { normalizeKuery } from './saved_object'; import { appContextService } from '.'; import { removeOldAssets } from './epm/packages/cleanup'; +import type { PackagePolicyUpgradeUsage } from './upgrade_usage'; import { sendTelemetryEvents } from './upgrade_usage'; export type InputsOverride = Partial & { @@ -436,18 +437,20 @@ class PackagePolicyService { }); if (packagePolicy.package.version !== currentVersion) { - appContextService.getLogger().info(`Package policy upgraded successfully`); + const upgradeTelemetry: PackagePolicyUpgradeUsage = { + package_name: packagePolicy.package.name, + current_version: currentVersion || 'unknown', + new_version: packagePolicy.package.version, + status: 'success', + dryRun: false, + }; sendTelemetryEvents( appContextService.getLogger(), appContextService.getTelemetryEventsSender(), - { - package_name: packagePolicy.package.name, - current_version: currentVersion || 'unknown', - new_version: packagePolicy.package.version, - status: 'success', - dryRun: false, - } + upgradeTelemetry ); + appContextService.getLogger().info(`Package policy upgraded successfully`); + appContextService.getLogger().debug(JSON.stringify(upgradeTelemetry)); } } @@ -686,6 +689,19 @@ class PackagePolicyService { const hasErrors = 'errors' in updatedPackagePolicy; if (packagePolicy.package.version !== packageInfo.version) { + const upgradeTelemetry: PackagePolicyUpgradeUsage = { + package_name: packageInfo.name, + current_version: packagePolicy.package.version, + new_version: packageInfo.version, + status: hasErrors ? 'failure' : 'success', + error: hasErrors ? updatedPackagePolicy.errors : undefined, + dryRun: true, + }; + sendTelemetryEvents( + appContextService.getLogger(), + appContextService.getTelemetryEventsSender(), + upgradeTelemetry + ); appContextService .getLogger() .info( @@ -693,18 +709,7 @@ class PackagePolicyService { hasErrors ? 'resulted in errors' : 'ran successfully' }` ); - sendTelemetryEvents( - appContextService.getLogger(), - appContextService.getTelemetryEventsSender(), - { - package_name: packageInfo.name, - current_version: packagePolicy.package.version, - new_version: packageInfo.version, - status: hasErrors ? 'failure' : 'success', - error: hasErrors ? updatedPackagePolicy.errors : undefined, - dryRun: true, - } - ); + appContextService.getLogger().debug(JSON.stringify(upgradeTelemetry)); } return { diff --git a/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts b/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts index e14aa37e4a5e7..5cdd345fb65d1 100644 --- a/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts +++ b/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts @@ -38,31 +38,25 @@ describe('sendTelemetryEvents', () => { sendTelemetryEvents(loggerMock, eventsTelemetryMock, upgardeMessage); - expect(eventsTelemetryMock.queueTelemetryEvents).toHaveBeenCalledWith( - [ - { - id: 'aws_0.6.1_1.3.0_failure_true', - package_policy_upgrade: { - current_version: '0.6.1', - error: [ - { - key: 'fieldX', - message: ['Field is required'], - }, - { - key: 'fieldX', - message: 'Invalid format', - }, - ], - new_version: '1.3.0', - package_name: 'aws', - status: 'failure', - dryRun: true, + expect(eventsTelemetryMock.queueTelemetryEvents).toHaveBeenCalledWith('fleet-upgrades', [ + { + current_version: '0.6.1', + error: [ + { + key: 'fieldX', + message: ['Field is required'], }, - }, - ], - 'fleet-upgrades' - ); + { + key: 'fieldX', + message: 'Invalid format', + }, + ], + new_version: '1.3.0', + package_name: 'aws', + status: 'failure', + dryRun: true, + }, + ]); }); it('should cap error size', () => { diff --git a/x-pack/plugins/fleet/server/services/upgrade_usage.ts b/x-pack/plugins/fleet/server/services/upgrade_usage.ts index 9be25ad3e8cbb..dad7228a13621 100644 --- a/x-pack/plugins/fleet/server/services/upgrade_usage.ts +++ b/x-pack/plugins/fleet/server/services/upgrade_usage.ts @@ -36,20 +36,14 @@ export function sendTelemetryEvents( } try { - eventsTelemetry.queueTelemetryEvents( - [ - { - package_policy_upgrade: { - ...upgradeUsage, - error: upgradeUsage.error - ? makeErrorGeneric(capErrorSize(upgradeUsage.error, MAX_ERROR_SIZE)) - : undefined, - }, - id: `${upgradeUsage.package_name}_${upgradeUsage.current_version}_${upgradeUsage.new_version}_${upgradeUsage.status}_${upgradeUsage.dryRun}`, - }, - ], - FLEET_UPGRADES_CHANNEL_NAME - ); + eventsTelemetry.queueTelemetryEvents(FLEET_UPGRADES_CHANNEL_NAME, [ + { + ...upgradeUsage, + error: upgradeUsage.error + ? makeErrorGeneric(capErrorSize(upgradeUsage.error, MAX_ERROR_SIZE)) + : undefined, + }, + ]); } catch (exc) { logger.error(`queing telemetry events failed ${exc}`); } diff --git a/x-pack/plugins/fleet/server/telemetry/queue.ts b/x-pack/plugins/fleet/server/telemetry/queue.ts index 773dc764996ad..9c451563ad5f0 100644 --- a/x-pack/plugins/fleet/server/telemetry/queue.ts +++ b/x-pack/plugins/fleet/server/telemetry/queue.ts @@ -5,26 +5,19 @@ * 2.0. */ -import type { TelemetryEvent } from './types'; - export const TELEMETRY_MAX_BUFFER_SIZE = 100; -export class TelemetryQueue { +export class TelemetryQueue { private maxQueueSize = TELEMETRY_MAX_BUFFER_SIZE; - private queue: TelemetryEvent[] = []; + private queue: T[] = []; - public addEvents(events: TelemetryEvent[]) { + public addEvents(events: T[]) { const qlength = this.queue.length; if (events.length === 0) { return; } - // do not add events with same id - events = events.filter( - (event) => !this.queue.find((qItem) => qItem.id && event.id && qItem.id === event.id) - ); - if (qlength >= this.maxQueueSize) { // we're full already return; @@ -41,7 +34,7 @@ export class TelemetryQueue { this.queue = []; } - public getEvents(): TelemetryEvent[] { + public getEvents(): T[] { return this.queue; } } diff --git a/x-pack/plugins/fleet/server/telemetry/receiver.ts b/x-pack/plugins/fleet/server/telemetry/receiver.ts deleted file mode 100644 index 487461e5ed1ab..0000000000000 --- a/x-pack/plugins/fleet/server/telemetry/receiver.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { CoreStart, ElasticsearchClient } from 'src/core/server'; - -import type { ESClusterInfo } from './types'; - -export class TelemetryReceiver { - private esClient?: ElasticsearchClient; - - public async start(core?: CoreStart) { - this.esClient = core?.elasticsearch.client.asInternalUser; - } - - public async fetchClusterInfo(): Promise { - if (this.esClient === undefined || this.esClient === null) { - throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation'); - } - - const { body } = await this.esClient.info(); - return body; - } -} diff --git a/x-pack/plugins/fleet/server/telemetry/sender.test.ts b/x-pack/plugins/fleet/server/telemetry/sender.test.ts index 924d2b9f3d148..171ace12bdf77 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.test.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.test.ts @@ -12,8 +12,9 @@ import { loggingSystemMock } from 'src/core/server/mocks'; import axios from 'axios'; +import type { InfoResponse } from '@elastic/elasticsearch/api/types'; + import { TelemetryEventsSender } from './sender'; -import type { ESClusterInfo } from './types'; jest.mock('axios', () => { return { @@ -23,20 +24,25 @@ jest.mock('axios', () => { describe('TelemetryEventsSender', () => { let logger: ReturnType; + let sender: TelemetryEventsSender; beforeEach(() => { logger = loggingSystemMock.createLogger(); + sender = new TelemetryEventsSender(logger); + sender.start(undefined, { + elasticsearch: { client: { asInternalUser: { info: jest.fn(async () => ({})) } } }, + } as any); }); describe('queueTelemetryEvents', () => { it('queues two events', () => { - const sender = new TelemetryEventsSender(logger); - sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }], 'my-channel'); - expect(sender['queuesPerChannel']['my-channel']).toBeDefined(); + sender.queueTelemetryEvents('fleet-upgrades', [ + { package_name: 'system', current_version: '0.3', new_version: '1.0', status: 'success' }, + ]); + expect(sender['queuesPerChannel']['fleet-upgrades']).toBeDefined(); }); it('should send events when due', async () => { - const sender = new TelemetryEventsSender(logger); sender['telemetryStart'] = { getIsOptedIn: jest.fn(async () => true), }; @@ -46,26 +52,29 @@ describe('TelemetryEventsSender', () => { ), }; - sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }], 'my-channel'); + sender.queueTelemetryEvents('fleet-upgrades', [ + { package_name: 'apache', current_version: '0.3', new_version: '1.0', status: 'success' }, + ]); sender['sendEvents'] = jest.fn(); await sender['sendIfDue'](); expect(sender['sendEvents']).toHaveBeenCalledWith( - 'https://telemetry-staging.elastic.co/v3/send/my-channel', + 'https://telemetry-staging.elastic.co/v3/send/fleet-upgrades', undefined, expect.anything() ); }); it("shouldn't send when telemetry is disabled", async () => { - const sender = new TelemetryEventsSender(logger); const telemetryStart = { getIsOptedIn: jest.fn(async () => false), }; sender['telemetryStart'] = telemetryStart; - sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }], 'my-channel'); + sender.queueTelemetryEvents('fleet-upgrades', [ + { package_name: 'system', current_version: '0.3', new_version: '1.0', status: 'success' }, + ]); sender['sendEvents'] = jest.fn(); await sender['sendIfDue'](); @@ -74,7 +83,6 @@ describe('TelemetryEventsSender', () => { }); it('should send events to separate channels', async () => { - const sender = new TelemetryEventsSender(logger); sender['telemetryStart'] = { getIsOptedIn: jest.fn(async () => true), }; @@ -83,27 +91,27 @@ describe('TelemetryEventsSender', () => { async () => new URL('https://telemetry.elastic.co/v3/send/snapshot') ), }; - sender['receiver'] = { - start: jest.fn(), - fetchClusterInfo: jest.fn(async () => { - return { - cluster_uuid: '1', - cluster_name: 'name', - version: { - number: '8.0.0', - }, - } as ESClusterInfo; - }), - }; + + sender['fetchClusterInfo'] = jest.fn(async () => { + return { + cluster_uuid: '1', + cluster_name: 'name', + version: { + number: '8.0.0', + }, + } as InfoResponse; + }); const myChannelEvents = [{ 'event.kind': '1' }, { 'event.kind': '2' }]; - sender.queueTelemetryEvents(myChannelEvents, 'my-channel'); + // @ts-ignore + sender.queueTelemetryEvents('my-channel', myChannelEvents); sender['queuesPerChannel']['my-channel']['getEvents'] = jest.fn(() => myChannelEvents); expect(sender['queuesPerChannel']['my-channel']['queue'].length).toBe(2); const myChannel2Events = [{ 'event.kind': '3' }]; - sender.queueTelemetryEvents(myChannel2Events, 'my-channel2'); + // @ts-ignore + sender.queueTelemetryEvents('my-channel2', myChannel2Events); sender['queuesPerChannel']['my-channel2']['getEvents'] = jest.fn(() => myChannel2Events); expect(sender['queuesPerChannel']['my-channel2']['queue'].length).toBe(1); @@ -121,12 +129,12 @@ describe('TelemetryEventsSender', () => { }; expect(axios.post).toHaveBeenCalledWith( 'https://telemetry.elastic.co/v3/send/my-channel', - '{"event.kind":"1","cluster_uuid":"1","cluster_name":"name"}\n{"event.kind":"2","cluster_uuid":"1","cluster_name":"name"}\n', + '{"event.kind":"1"}\n{"event.kind":"2"}\n', headers ); expect(axios.post).toHaveBeenCalledWith( 'https://telemetry.elastic.co/v3/send/my-channel2', - '{"event.kind":"3","cluster_uuid":"1","cluster_name":"name"}\n', + '{"event.kind":"3"}\n', headers ); }); diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index 3ccab058167c7..410a88f60bae8 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -5,17 +5,18 @@ * 2.0. */ -import type { Logger } from 'src/core/server'; +import type { CoreStart, ElasticsearchClient, Logger } from 'src/core/server'; import type { TelemetryPluginStart, TelemetryPluginSetup } from 'src/plugins/telemetry/server'; import { cloneDeep } from 'lodash'; import axios from 'axios'; -import type { TelemetryReceiver } from './receiver'; +import type { InfoResponse } from '@elastic/elasticsearch/api/types'; + import { TelemetryQueue } from './queue'; -import type { ESClusterInfo, TelemetryEvent } from './types'; +import type { FleetTelemetryChannel, FleetTelemetryChannelEvents } from './types'; /** * Simplified version of https://github.com/elastic/kibana/blob/master/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -30,21 +31,21 @@ export class TelemetryEventsSender { private telemetrySetup?: TelemetryPluginSetup; private intervalId?: NodeJS.Timeout; private isSending = false; - private receiver: TelemetryReceiver | undefined; - private queuesPerChannel: { [channel: string]: TelemetryQueue } = {}; + private queuesPerChannel: { [channel: string]: TelemetryQueue } = {}; private isOptedIn?: boolean = true; // Assume true until the first check + private esClient?: ElasticsearchClient; constructor(logger: Logger) { - this.logger = logger.get('telemetry_events'); + this.logger = logger; } public setup(telemetrySetup?: TelemetryPluginSetup) { this.telemetrySetup = telemetrySetup; } - public start(telemetryStart?: TelemetryPluginStart, receiver?: TelemetryReceiver) { + public async start(telemetryStart?: TelemetryPluginStart, core?: CoreStart) { this.telemetryStart = telemetryStart; - this.receiver = receiver; + this.esClient = core?.elasticsearch.client.asInternalUser; this.logger.debug(`Starting local task`); setTimeout(() => { @@ -59,11 +60,14 @@ export class TelemetryEventsSender { } } - public queueTelemetryEvents(events: TelemetryEvent[], channel: string) { + public queueTelemetryEvents( + channel: T, + events: Array + ) { if (!this.queuesPerChannel[channel]) { - this.queuesPerChannel[channel] = new TelemetryQueue(); + this.queuesPerChannel[channel] = new TelemetryQueue(); } - this.queuesPerChannel[channel].addEvents(events); + this.queuesPerChannel[channel].addEvents(cloneDeep(events)); } public async isTelemetryOptedIn() { @@ -88,7 +92,7 @@ export class TelemetryEventsSender { return; } - const clusterInfo = await this.receiver?.fetchClusterInfo(); + const clusterInfo = await this.fetchClusterInfo(); for (const channel of Object.keys(this.queuesPerChannel)) { await this.sendEvents( @@ -101,10 +105,19 @@ export class TelemetryEventsSender { this.isSending = false; } + private async fetchClusterInfo(): Promise { + if (this.esClient === undefined || this.esClient === null) { + throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation'); + } + + const { body } = await this.esClient.info(); + return body; + } + public async sendEvents( telemetryUrl: string, - clusterInfo: ESClusterInfo | undefined, - queue: TelemetryQueue + clusterInfo: InfoResponse | undefined, + queue: TelemetryQueue ) { const events = queue.getEvents(); if (events.length === 0) { @@ -114,17 +127,12 @@ export class TelemetryEventsSender { try { this.logger.debug(`Telemetry URL: ${telemetryUrl}`); - const toSend: TelemetryEvent[] = cloneDeep(events).map((event) => ({ - ...event, - cluster_uuid: clusterInfo?.cluster_uuid, - cluster_name: clusterInfo?.cluster_name, - })); queue.clearEvents(); - this.logger.debug(JSON.stringify(toSend)); + this.logger.debug(JSON.stringify(events)); await this.send( - toSend, + events, telemetryUrl, clusterInfo?.cluster_uuid, clusterInfo?.version?.number diff --git a/x-pack/plugins/fleet/server/telemetry/types.ts b/x-pack/plugins/fleet/server/telemetry/types.ts index 13dff52ca7da9..4351546ecdf02 100644 --- a/x-pack/plugins/fleet/server/telemetry/types.ts +++ b/x-pack/plugins/fleet/server/telemetry/types.ts @@ -5,28 +5,11 @@ * 2.0. */ -type BaseSearchTypes = string | number | boolean | object; -export type SearchTypes = BaseSearchTypes | BaseSearchTypes[] | undefined; +import type { PackagePolicyUpgradeUsage } from '../services/upgrade_usage'; -// For getting cluster info. Copied from telemetry_collection/get_cluster_info.ts -export interface ESClusterInfo { - cluster_uuid: string; - cluster_name: string; - version?: { - number: string; - build_flavor: string; - build_type: string; - build_hash: string; - build_date: string; - build_snapshot?: boolean; - lucene_version: string; - minimum_wire_compatibility_version: string; - minimum_index_compatibility_version: string; - }; +export interface FleetTelemetryChannelEvents { + // channel name => event type + 'fleet-upgrades': PackagePolicyUpgradeUsage; } -export interface TelemetryEvent { - [key: string]: SearchTypes; - cluster_name?: string; - cluster_uuid?: string; -} +export type FleetTelemetryChannel = keyof FleetTelemetryChannelEvents; From b4a89d9bf193a0e6f26bf37f907771f69495ba75 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Thu, 28 Oct 2021 09:31:26 +0200 Subject: [PATCH 25/27] fix imports --- x-pack/plugins/fleet/server/telemetry/sender.test.ts | 2 +- x-pack/plugins/fleet/server/telemetry/sender.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/server/telemetry/sender.test.ts b/x-pack/plugins/fleet/server/telemetry/sender.test.ts index 171ace12bdf77..6ab4f0aeeb811 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.test.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.test.ts @@ -12,7 +12,7 @@ import { loggingSystemMock } from 'src/core/server/mocks'; import axios from 'axios'; -import type { InfoResponse } from '@elastic/elasticsearch/api/types'; +import type { InfoResponse } from '@elastic/elasticsearch/lib/api/types'; import { TelemetryEventsSender } from './sender'; diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index 410a88f60bae8..3bda17fbd1d79 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -12,7 +12,7 @@ import { cloneDeep } from 'lodash'; import axios from 'axios'; -import type { InfoResponse } from '@elastic/elasticsearch/api/types'; +import type { InfoResponse } from '@elastic/elasticsearch/lib/api/types'; import { TelemetryQueue } from './queue'; From f0a8cd22455e90e35b0ab32619d0915bdb5f6d58 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Thu, 28 Oct 2021 11:41:10 +0200 Subject: [PATCH 26/27] added error_message field --- .../server/services/upgrade_usage.test.ts | 10 +++++----- .../fleet/server/services/upgrade_usage.ts | 20 +++++++++---------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts b/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts index 5cdd345fb65d1..5445ad233eddc 100644 --- a/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts +++ b/x-pack/plugins/fleet/server/services/upgrade_usage.test.ts @@ -30,8 +30,8 @@ describe('sendTelemetryEvents', () => { new_version: '1.3.0', status: 'failure', error: [ - { key: 'fieldX', message: ['Field X is required'] }, - { key: 'fieldX', message: 'Invalid format' }, + { key: 'queueUrl', message: ['Queue URL is required'] }, + { message: 'Invalid format' }, ], dryRun: true, }; @@ -43,14 +43,14 @@ describe('sendTelemetryEvents', () => { current_version: '0.6.1', error: [ { - key: 'fieldX', - message: ['Field is required'], + key: 'queueUrl', + message: ['Queue URL is required'], }, { - key: 'fieldX', message: 'Invalid format', }, ], + error_message: ['Field is required', 'Invalid format'], new_version: '1.3.0', package_name: 'aws', status: 'failure', diff --git a/x-pack/plugins/fleet/server/services/upgrade_usage.ts b/x-pack/plugins/fleet/server/services/upgrade_usage.ts index dad7228a13621..68bb126496e01 100644 --- a/x-pack/plugins/fleet/server/services/upgrade_usage.ts +++ b/x-pack/plugins/fleet/server/services/upgrade_usage.ts @@ -16,6 +16,7 @@ export interface PackagePolicyUpgradeUsage { status: 'success' | 'failure'; error?: UpgradeError[]; dryRun?: boolean; + error_message?: string[]; } export interface UpgradeError { @@ -36,12 +37,12 @@ export function sendTelemetryEvents( } try { + const cappedErrors = capErrorSize(upgradeUsage.error || [], MAX_ERROR_SIZE); eventsTelemetry.queueTelemetryEvents(FLEET_UPGRADES_CHANNEL_NAME, [ { ...upgradeUsage, - error: upgradeUsage.error - ? makeErrorGeneric(capErrorSize(upgradeUsage.error, MAX_ERROR_SIZE)) - : undefined, + error: upgradeUsage.error ? cappedErrors : undefined, + error_message: makeErrorGeneric(cappedErrors), }, ]); } catch (exc) { @@ -53,15 +54,12 @@ export function capErrorSize(errors: UpgradeError[], maxSize: number): UpgradeEr return errors.length > maxSize ? errors?.slice(0, maxSize) : errors; } -function makeErrorGeneric(errors: UpgradeError[]): UpgradeError[] { +function makeErrorGeneric(errors: UpgradeError[]): string[] { return errors.map((error) => { - let message = error.message; - if (error.key && Array.isArray(error.message) && error.message[0].indexOf('is required') > -1) { - message = ['Field is required']; + if (Array.isArray(error.message)) { + const firstMessage = error.message[0]; + return firstMessage?.indexOf('is required') > -1 ? 'Field is required' : firstMessage; } - return { - key: error.key, - message, - }; + return error.message as string; }); } From 1d876cd3ac78b0feac556c2c43aaa462c34e2ace Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Mon, 1 Nov 2021 09:09:49 +0100 Subject: [PATCH 27/27] review fixes --- x-pack/plugins/fleet/server/telemetry/queue.ts | 4 ++-- x-pack/plugins/fleet/server/telemetry/sender.test.ts | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/fleet/server/telemetry/queue.ts b/x-pack/plugins/fleet/server/telemetry/queue.ts index 9c451563ad5f0..3496cfb94915d 100644 --- a/x-pack/plugins/fleet/server/telemetry/queue.ts +++ b/x-pack/plugins/fleet/server/telemetry/queue.ts @@ -5,10 +5,10 @@ * 2.0. */ -export const TELEMETRY_MAX_BUFFER_SIZE = 100; +export const TELEMETRY_MAX_QUEUE_SIZE = 100; export class TelemetryQueue { - private maxQueueSize = TELEMETRY_MAX_BUFFER_SIZE; + private maxQueueSize = TELEMETRY_MAX_QUEUE_SIZE; private queue: T[] = []; public addEvents(events: T[]) { diff --git a/x-pack/plugins/fleet/server/telemetry/sender.test.ts b/x-pack/plugins/fleet/server/telemetry/sender.test.ts index 6ab4f0aeeb811..8fe4c6e150ff9 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.test.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.test.ts @@ -6,14 +6,15 @@ */ /* eslint-disable dot-notation */ -import { URL } from 'url'; -import { loggingSystemMock } from 'src/core/server/mocks'; +import { URL } from 'url'; import axios from 'axios'; import type { InfoResponse } from '@elastic/elasticsearch/lib/api/types'; +import { loggingSystemMock } from 'src/core/server/mocks'; + import { TelemetryEventsSender } from './sender'; jest.mock('axios', () => {