From d016f5abcbdec26bc545b521e650a4a2e15b89b3 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Fri, 4 Feb 2022 09:41:20 -0700 Subject: [PATCH 01/10] wip --- .../server/usage/detections/constants.ts | 27 +++++++ .../detections/detection_rule_helpers.ts | 73 +++++-------------- .../server/usage/detections/fetch_with_pit.ts | 61 ++++++++++++++++ .../server/usage/detections/get_alerts.ts | 54 ++++++++++++++ .../server/usage/detections/get_cases.ts | 42 +++++++++++ .../usage/detections/get_detection_rules.ts | 61 ++++++++++++++++ .../server/usage/detections/types.ts | 2 +- 7 files changed, 263 insertions(+), 57 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/usage/detections/constants.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/fetch_with_pit.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/get_alerts.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/get_cases.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/get_detection_rules.ts diff --git a/x-pack/plugins/security_solution/server/usage/detections/constants.ts b/x-pack/plugins/security_solution/server/usage/detections/constants.ts new file mode 100644 index 0000000000000..27b191d7dac9a --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/constants.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * We limit the max results window to prevent in-memory from blowing up when we do correlation. + * This is limiting us to 10,000 cases and 10,000 elastic detection rules to do telemetry and correlation + * and the choice was based on the initial "index.max_result_window" before this turned into a PIT (Point In Time) + * implementation. + * + * This number could be changed, and the implementation details of how we correlate could change as well (maybe) + * to avoid pulling 10,000 worth of cases and elastic rules into memory. + * + * However, for now, we are keeping this maximum as the original and the in-memory implementation + */ +export const MAX_RESULTS_WINDOW = 10_000; + +/** + * We arbitrarily choose our max per page based on 100 as that + * appears to be what others are choosing here in documentation: + * https://www.elastic.co/guide/en/elasticsearch/reference/current/point-in-time-api.html + * and within the saved objects client examples and documentation. + */ +export const MAX_PER_PAGE = 100; diff --git a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts b/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts index 446fc956c0c65..c52188d1287ef 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts @@ -33,6 +33,9 @@ import type { import { legacyRuleActionsSavedObjectType } from '../../lib/detection_engine/rule_actions/legacy_saved_object_mappings'; // eslint-disable-next-line no-restricted-imports import { LegacyIRuleActionsAttributesSavedObjectAttributes } from '../../lib/detection_engine/rule_actions/legacy_types'; +import { getDetectionRules } from './get_detection_rules'; +import { MAX_PER_PAGE, MAX_RESULTS_WINDOW } from './constants'; +import { getAlerts } from './get_alerts'; /** * Initial detection metrics initialized. @@ -338,8 +341,6 @@ export const updateDetectionRuleUsage = ( return updatedUsage; }; -const MAX_RESULTS_WINDOW = 10_000; // elasticsearch index.max_result_window default value - export const getDetectionRuleMetrics = async ( kibanaIndex: string, signalsIndex: string, @@ -347,59 +348,19 @@ export const getDetectionRuleMetrics = async ( savedObjectClient: SavedObjectsClientContract ): Promise => { let rulesUsage: DetectionRulesTypeUsage = initialDetectionRulesUsage; - const ruleSearchOptions: RuleSearchParams = { - body: { - query: { - bool: { - filter: { - terms: { - 'alert.alertTypeId': [ - SIGNALS_ID, - EQL_RULE_TYPE_ID, - ML_RULE_TYPE_ID, - QUERY_RULE_TYPE_ID, - SAVED_QUERY_RULE_TYPE_ID, - INDICATOR_RULE_TYPE_ID, - THRESHOLD_RULE_TYPE_ID, - ], - }, - }, - }, - }, - }, - filter_path: [], - ignore_unavailable: true, - index: kibanaIndex, - size: MAX_RESULTS_WINDOW, - }; - try { - const { body: ruleResults } = await esClient.search(ruleSearchOptions); - const { body: detectionAlertsResp } = (await esClient.search({ - index: `${signalsIndex}*`, - size: MAX_RESULTS_WINDOW, - body: { - aggs: { - detectionAlerts: { - terms: { field: ALERT_RULE_UUID }, - }, - }, - query: { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: 'now-24h', - lte: 'now', - }, - }, - }, - ], - }, - }, - }, - })) as { body: AlertsAggregationResponse }; + const ruleResults = await getDetectionRules({ + esClient, + kibanaIndex, + maxPerPage: MAX_PER_PAGE, + maxSize: MAX_RESULTS_WINDOW, + }); + const detectionAlertsResp = await getAlerts({ + esClient, + signalsIndex: `${signalsIndex}*`, + maxPerPage: MAX_PER_PAGE, + maxSize: MAX_RESULTS_WINDOW, + }); const cases = await savedObjectClient.find({ type: CASE_COMMENT_SAVED_OBJECT, @@ -449,8 +410,8 @@ export const getDetectionRuleMetrics = async ( const alertsCache = new Map(); alertBuckets.map((bucket) => alertsCache.set(bucket.key, bucket.doc_count)); - if (ruleResults.hits?.hits?.length > 0) { - const ruleObjects = ruleResults.hits.hits.map((hit) => { + if (ruleResults.length > 0) { + const ruleObjects = ruleResults.map((hit) => { const ruleId = hit._id.split(':')[1]; const isElastic = isElasticRule(hit._source?.alert.tags); diff --git a/x-pack/plugins/security_solution/server/usage/detections/fetch_with_pit.ts b/x-pack/plugins/security_solution/server/usage/detections/fetch_with_pit.ts new file mode 100644 index 0000000000000..59bdb32cb0feb --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/fetch_with_pit.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + OpenPointInTimeResponse, + SearchHit, + SortResults, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { SearchRequest } from '@elastic/elasticsearch/lib/api/types'; +import { ElasticsearchClient } from 'kibana/server'; + +export interface FetchWithPitOptions { + esClient: ElasticsearchClient; + index: string; + maxSize: number; + maxPerPage: number; + searchRequest: SearchRequest; +} +export const fetchWithPit = async ({ + esClient, + index, + searchRequest, + maxSize, + maxPerPage, +}: FetchWithPitOptions): Promise>> => { + // create and assign an initial point in time + let pitId: OpenPointInTimeResponse['id'] = ( + await esClient.openPointInTime({ + index, + keep_alive: '5m', // default is from looking at Kibana saved objects and online documentation + }) + ).body.id; + + let searchAfter: SortResults | undefined; + let hits: Array> = []; + let fetchMore = true; + while (fetchMore) { + const ruleSearchOptions: SearchRequest = { + ...searchRequest, + search_after: searchAfter, + sort: [{ _shard_doc: 'desc' }] as unknown as string[], // FUNFACT: This is not typed correctly https://github.com/elastic/elasticsearch-js/issues/1589 + pit: { id: pitId }, + size: Math.min(maxPerPage, maxSize - hits.length), + }; + const { body } = await esClient.search(ruleSearchOptions); + hits = [...hits, ...body.hits.hits]; + searchAfter = + body.hits.hits.length !== 0 ? body.hits.hits[body.hits.hits.length - 1].sort : undefined; + + fetchMore = searchAfter != null && hits.length <= maxSize; + if (body.pit_id != null) { + pitId = body.pit_id; + } + } + esClient.closePointInTime({ id: pitId }); + return hits; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_alerts.ts b/x-pack/plugins/security_solution/server/usage/detections/get_alerts.ts new file mode 100644 index 0000000000000..a8dbb18807f0e --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/get_alerts.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { ALERT_RULE_UUID } from '@kbn/rule-data-utils'; +import { ElasticsearchClient } from 'kibana/server'; +import { fetchWithPit } from './fetch_with_pit'; +import { AlertsAggregationResponse } from './types'; + +export interface GetAlertsOptions { + esClient: ElasticsearchClient; + signalsIndex: string; + maxSize: number; + maxPerPage: number; +} + +export const getAlerts = async ({ + esClient, + signalsIndex, + maxSize, + maxPerPage, +}: GetAlertsOptions): Promise>> => { + return fetchWithPit({ + esClient, + index: signalsIndex, + maxSize, + maxPerPage, + searchRequest: { + aggs: { + detectionAlerts: { + terms: { field: ALERT_RULE_UUID }, + }, + }, + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: 'now-24h', + lte: 'now', + }, + }, + }, + ], + }, + }, + }, + }); +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_cases.ts b/x-pack/plugins/security_solution/server/usage/detections/get_cases.ts new file mode 100644 index 0000000000000..aa93a73fdb652 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/get_cases.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SavedObjectsClientContract, + SavedObjectsFindResult, +} from '../../../../../../src/core/server'; +import type { CasesSavedObject } from './types'; +import { CASE_COMMENT_SAVED_OBJECT } from '../../../../cases/common/constants'; + +export interface GetCasesOptions { + savedObjectsClient: SavedObjectsClientContract; + maxSize: number; + maxPerPage: number; +} + +export const getCases = async ({ + savedObjectsClient, + maxSize, + maxPerPage, +}: GetCasesOptions): Promise>> => { + const finder = savedObjectsClient.createPointInTimeFinder({ + type: CASE_COMMENT_SAVED_OBJECT, + perPage: maxPerPage, + namespaces: ['*'], + filter: `${CASE_COMMENT_SAVED_OBJECT}.attributes.type: alert`, + }); + let responses: Array> = []; + for await (const response of finder.find()) { + const extra = responses.length + response.saved_objects.length - maxSize; + if (extra > 0) { + responses = [...responses, ...response.saved_objects.splice(extra)]; + } else { + responses = [...responses, ...response.saved_objects]; + } + } + return responses; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_detection_rules.ts b/x-pack/plugins/security_solution/server/usage/detections/get_detection_rules.ts new file mode 100644 index 0000000000000..5b68ea8c2d25e --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/get_detection_rules.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { + SIGNALS_ID, + EQL_RULE_TYPE_ID, + INDICATOR_RULE_TYPE_ID, + ML_RULE_TYPE_ID, + QUERY_RULE_TYPE_ID, + THRESHOLD_RULE_TYPE_ID, + SAVED_QUERY_RULE_TYPE_ID, +} from '@kbn/securitysolution-rules'; +import { ElasticsearchClient } from 'kibana/server'; +import { fetchWithPit } from './fetch_with_pit'; + +import { RuleSearchResult } from './types'; + +export interface GetDetectionRulesOptions { + esClient: ElasticsearchClient; + kibanaIndex: string; + maxSize: number; + maxPerPage: number; +} + +export const getDetectionRules = async ({ + esClient, + kibanaIndex, + maxSize, + maxPerPage, +}: GetDetectionRulesOptions): Promise>> => { + return fetchWithPit({ + esClient, + index: kibanaIndex, + maxSize, + maxPerPage, + searchRequest: { + query: { + bool: { + filter: { + terms: { + 'alert.alertTypeId': [ + SIGNALS_ID, + EQL_RULE_TYPE_ID, + ML_RULE_TYPE_ID, + QUERY_RULE_TYPE_ID, + SAVED_QUERY_RULE_TYPE_ID, + INDICATOR_RULE_TYPE_ID, + THRESHOLD_RULE_TYPE_ID, + ], + }, + }, + }, + }, + }, + }); +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/types.ts b/x-pack/plugins/security_solution/server/usage/detections/types.ts index a7eb4c387d4ba..8c2459b2eb57d 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/types.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/types.ts @@ -5,7 +5,7 @@ * 2.0. */ -interface RuleSearchBody { +export interface RuleSearchBody { query: { bool: { filter: { From 230d385943c0ef6ce0c340ba4e2e50452f50e38b Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Mon, 7 Feb 2022 17:41:31 -0700 Subject: [PATCH 02/10] Added the Point In Time (PIT) work and cleaned up the types --- .../rules/perform_bulk_action_route.test.ts | 4 +- .../routes/rules/perform_bulk_action_route.ts | 2 +- .../security_solution/server/plugin.ts | 1 + .../server/usage/collector.ts | 525 +----------------- .../usage/{detections => }/constants.ts | 0 .../usage/detections/detection_ml_helpers.ts | 175 ------ .../detections/detection_rule_helpers.ts | 464 ---------------- .../server/usage/detections/get_alerts.ts | 54 -- .../usage/detections/get_initial_usage.ts | 25 + ...detections.test.ts => get_metrics.test.ts} | 122 ++-- .../server/usage/detections/get_metrics.ts | 49 ++ .../server/usage/detections/index.ts | 41 -- .../detections/ml_jobs/get_initial_usage.ts | 22 + .../get_metrics.mocks.ts} | 225 +++----- .../usage/detections/ml_jobs/get_metrics.ts | 102 ++++ .../server/usage/detections/ml_jobs/schema.ts | 202 +++++++ .../transform_utils/get_job_correlations.ts | 71 +++ .../server/usage/detections/ml_jobs/types.ts | 56 ++ .../update_usage.test.ts} | 18 +- .../usage/detections/ml_jobs/update_usage.ts | 47 ++ .../detections/rules/get_initial_usage.ts | 84 +++ .../detections/rules/get_metrics.mocks.ts | 86 +++ .../usage/detections/rules/get_metrics.ts | 119 ++++ .../server/usage/detections/rules/schema.ts | 285 ++++++++++ .../get_alert_id_to_count_map.ts | 14 + .../get_rule_id_to_cases_map.ts | 30 + .../get_rule_id_to_enabled_map.ts | 32 ++ .../get_rule_object_correlations.ts | 60 ++ .../server/usage/detections/rules/types.ts | 47 ++ .../update_usage.test.ts} | 35 +- .../usage/detections/rules/update_usage.ts | 85 +++ .../get_notifications_enabled_disabled.ts | 26 + .../rules/usage_utils/update_query_usage.ts | 48 ++ .../rules/usage_utils/update_total_usage.ts | 51 ++ .../server/usage/detections/types.ts | 163 +----- .../get_internal_saved_objects_client.ts | 25 + .../server/usage/queries/get_alerts.ts | 114 ++++ .../get_case_comments.ts} | 22 +- .../get_detection_rules.ts | 15 +- .../usage/queries/legacy_get_rule_actions.ts | 61 ++ .../utils/fetch_hits_with_pit.ts} | 34 +- .../usage/queries/utils/is_elastic_rule.ts | 11 + .../security_solution/server/usage/schema.ts | 20 + .../security_solution/server/usage/types.ts | 27 +- .../telemetry/usage_collector/all_types.ts | 4 +- .../usage_collector/detection_rules.ts | 8 +- 46 files changed, 2067 insertions(+), 1644 deletions(-) rename x-pack/plugins/security_solution/server/usage/{detections => }/constants.ts (100%) delete mode 100644 x-pack/plugins/security_solution/server/usage/detections/detection_ml_helpers.ts delete mode 100644 x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts delete mode 100644 x-pack/plugins/security_solution/server/usage/detections/get_alerts.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/get_initial_usage.ts rename x-pack/plugins/security_solution/server/usage/detections/{detections.test.ts => get_metrics.test.ts} (75%) create mode 100644 x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts delete mode 100644 x-pack/plugins/security_solution/server/usage/detections/index.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_initial_usage.ts rename x-pack/plugins/security_solution/server/usage/detections/{detections.mocks.ts => ml_jobs/get_metrics.mocks.ts} (87%) create mode 100644 x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/ml_jobs/schema.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/ml_jobs/transform_utils/get_job_correlations.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/ml_jobs/types.ts rename x-pack/plugins/security_solution/server/usage/detections/{detection_ml_helpers.test.ts => ml_jobs/update_usage.test.ts} (70%) create mode 100644 x-pack/plugins/security_solution/server/usage/detections/ml_jobs/update_usage.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/rules/get_initial_usage.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.mocks.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/rules/schema.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_alert_id_to_count_map.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_id_to_cases_map.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_id_to_enabled_map.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_object_correlations.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/rules/types.ts rename x-pack/plugins/security_solution/server/usage/detections/{detection_rule_helpers.test.ts => rules/update_usage.test.ts} (92%) create mode 100644 x-pack/plugins/security_solution/server/usage/detections/rules/update_usage.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/get_notifications_enabled_disabled.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/update_query_usage.ts create mode 100644 x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/update_total_usage.ts create mode 100644 x-pack/plugins/security_solution/server/usage/get_internal_saved_objects_client.ts create mode 100644 x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts rename x-pack/plugins/security_solution/server/usage/{detections/get_cases.ts => queries/get_case_comments.ts} (59%) rename x-pack/plugins/security_solution/server/usage/{detections => queries}/get_detection_rules.ts (79%) create mode 100644 x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts rename x-pack/plugins/security_solution/server/usage/{detections/fetch_with_pit.ts => queries/utils/fetch_hits_with_pit.ts} (57%) create mode 100644 x-pack/plugins/security_solution/server/usage/queries/utils/is_elastic_rule.ts create mode 100644 x-pack/plugins/security_solution/server/usage/schema.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/perform_bulk_action_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/perform_bulk_action_route.test.ts index 0d968eb402717..fce7031a05b92 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/perform_bulk_action_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/perform_bulk_action_route.test.ts @@ -19,11 +19,11 @@ import { requestContextMock, serverMock, requestMock } from '../__mocks__'; import { performBulkActionRoute } from './perform_bulk_action_route'; import { getPerformBulkActionSchemaMock } from '../../../../../common/detection_engine/schemas/request/perform_bulk_action_schema.mock'; import { loggingSystemMock } from 'src/core/server/mocks'; -import { isElasticRule } from '../../../../usage/detections'; +import { isElasticRule } from '../../../../usage/queries/utils/is_elastic_rule'; import { readRules } from '../../rules/read_rules'; jest.mock('../../../machine_learning/authz', () => mockMlAuthzFactory.create()); -jest.mock('../../../../usage/detections', () => ({ isElasticRule: jest.fn() })); +jest.mock('../../../../usage/queries/utils/is_elastic_rule', () => ({ isElasticRule: jest.fn() })); jest.mock('../../rules/read_rules', () => ({ readRules: jest.fn() })); describe.each([ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/perform_bulk_action_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/perform_bulk_action_route.ts index 369bff6f615fc..8d326e67d8889 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/perform_bulk_action_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/perform_bulk_action_route.ts @@ -25,7 +25,7 @@ import type { SecuritySolutionPluginRouter } from '../../../../types'; import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; import { routeLimitedConcurrencyTag } from '../../../../utils/route_limited_concurrency_tag'; import { initPromisePool } from '../../../../utils/promise_pool'; -import { isElasticRule } from '../../../../usage/detections'; +import { isElasticRule } from '../../../../usage/queries/utils/is_elastic_rule'; import { buildMlAuthz } from '../../../machine_learning/authz'; import { throwHttpError } from '../../../machine_learning/validation'; import { deleteRules } from '../../rules/delete_rules'; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 08696b02c5403..0d2eafafb2cc7 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -171,6 +171,7 @@ export class Plugin implements ISecuritySolutionPlugin { signalsIndex: DEFAULT_ALERTS_INDEX, ml: plugins.ml, usageCollection: plugins.usageCollection, + logger, }); this.telemetryUsageCounter = plugins.usageCollection?.createUsageCounter(APP_ID); diff --git a/x-pack/plugins/security_solution/server/usage/collector.ts b/x-pack/plugins/security_solution/server/usage/collector.ts index 4530dac725c7b..7dd6c23ac97f5 100644 --- a/x-pack/plugins/security_solution/server/usage/collector.ts +++ b/x-pack/plugins/security_solution/server/usage/collector.ts @@ -5,29 +5,13 @@ * 2.0. */ -import { CoreSetup, SavedObjectsClientContract } from '../../../../../src/core/server'; -import { CollectorFetchContext } from '../../../../../src/plugins/usage_collection/server'; -import { CollectorDependencies } from './types'; -import { fetchDetectionsMetrics } from './detections'; -import { SAVED_OBJECT_TYPES } from '../../../cases/common/constants'; -// eslint-disable-next-line no-restricted-imports -import { legacyRuleActionsSavedObjectType } from '../lib/detection_engine/rule_actions/legacy_saved_object_mappings'; +import type { CollectorFetchContext } from '../../../../../src/plugins/usage_collection/server'; +import type { CollectorDependencies } from './types'; +import { getDetectionsMetrics } from './detections/get_metrics'; +import { schema, UsageData } from './schema'; +import { getInternalSavedObjectsClient } from './get_internal_saved_objects_client'; export type RegisterCollector = (deps: CollectorDependencies) => void; -export interface UsageData { - detectionMetrics: {}; -} - -export async function getInternalSavedObjectsClient(core: CoreSetup) { - return core.getStartServices().then(async ([coreStart]) => { - // note: we include the "cases" and "alert" hidden types here otherwise we would not be able to query them. If at some point cases and alert is not considered a hidden type this can be removed - return coreStart.savedObjects.createInternalRepository([ - 'alert', - legacyRuleActionsSavedObjectType, - ...SAVED_OBJECT_TYPES, - ]); - }); -} export const registerCollector: RegisterCollector = ({ core, @@ -35,502 +19,29 @@ export const registerCollector: RegisterCollector = ({ signalsIndex, ml, usageCollection, + logger, }) => { if (!usageCollection) { + logger.debug('Usage collection is undefined, therefore returning early without registering it'); return; } const collector = usageCollection.makeUsageCollector({ type: 'security_solution', - schema: { - detectionMetrics: { - detection_rules: { - detection_rule_usage: { - query: { - enabled: { type: 'long', _meta: { description: 'Number of query rules enabled' } }, - disabled: { type: 'long', _meta: { description: 'Number of query rules disabled' } }, - alerts: { - type: 'long', - _meta: { description: 'Number of alerts generated by query rules' }, - }, - cases: { - type: 'long', - _meta: { description: 'Number of cases attached to query detection rule alerts' }, - }, - legacy_notifications_enabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications enabled' }, - }, - legacy_notifications_disabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications disabled' }, - }, - notifications_enabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - notifications_disabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - }, - threshold: { - enabled: { - type: 'long', - _meta: { description: 'Number of threshold rules enabled' }, - }, - disabled: { - type: 'long', - _meta: { description: 'Number of threshold rules disabled' }, - }, - alerts: { - type: 'long', - _meta: { description: 'Number of alerts generated by threshold rules' }, - }, - cases: { - type: 'long', - _meta: { - description: 'Number of cases attached to threshold detection rule alerts', - }, - }, - legacy_notifications_enabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications enabled' }, - }, - legacy_notifications_disabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications disabled' }, - }, - notifications_enabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - notifications_disabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - }, - eql: { - enabled: { type: 'long', _meta: { description: 'Number of eql rules enabled' } }, - disabled: { type: 'long', _meta: { description: 'Number of eql rules disabled' } }, - alerts: { - type: 'long', - _meta: { description: 'Number of alerts generated by eql rules' }, - }, - cases: { - type: 'long', - _meta: { description: 'Number of cases attached to eql detection rule alerts' }, - }, - legacy_notifications_enabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications enabled' }, - }, - legacy_notifications_disabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications disabled' }, - }, - notifications_enabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - notifications_disabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - }, - machine_learning: { - enabled: { - type: 'long', - _meta: { description: 'Number of machine_learning rules enabled' }, - }, - disabled: { - type: 'long', - _meta: { description: 'Number of machine_learning rules disabled' }, - }, - alerts: { - type: 'long', - _meta: { description: 'Number of alerts generated by machine_learning rules' }, - }, - cases: { - type: 'long', - _meta: { - description: 'Number of cases attached to machine_learning detection rule alerts', - }, - }, - legacy_notifications_enabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications enabled' }, - }, - legacy_notifications_disabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications disabled' }, - }, - notifications_enabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - notifications_disabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - }, - threat_match: { - enabled: { - type: 'long', - _meta: { description: 'Number of threat_match rules enabled' }, - }, - disabled: { - type: 'long', - _meta: { description: 'Number of threat_match rules disabled' }, - }, - alerts: { - type: 'long', - _meta: { description: 'Number of alerts generated by threat_match rules' }, - }, - cases: { - type: 'long', - _meta: { - description: 'Number of cases attached to threat_match detection rule alerts', - }, - }, - legacy_notifications_enabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications enabled' }, - }, - legacy_notifications_disabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications disabled' }, - }, - notifications_enabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - notifications_disabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - }, - elastic_total: { - enabled: { type: 'long', _meta: { description: 'Number of elastic rules enabled' } }, - disabled: { - type: 'long', - _meta: { description: 'Number of elastic rules disabled' }, - }, - alerts: { - type: 'long', - _meta: { description: 'Number of alerts generated by elastic rules' }, - }, - cases: { - type: 'long', - _meta: { description: 'Number of cases attached to elastic detection rule alerts' }, - }, - legacy_notifications_enabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications enabled' }, - }, - legacy_notifications_disabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications disabled' }, - }, - notifications_enabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - notifications_disabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - }, - custom_total: { - enabled: { type: 'long', _meta: { description: 'Number of custom rules enabled' } }, - disabled: { type: 'long', _meta: { description: 'Number of custom rules disabled' } }, - alerts: { - type: 'long', - _meta: { description: 'Number of alerts generated by custom rules' }, - }, - cases: { - type: 'long', - _meta: { description: 'Number of cases attached to custom detection rule alerts' }, - }, - legacy_notifications_enabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications enabled' }, - }, - legacy_notifications_disabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications disabled' }, - }, - notifications_enabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - notifications_disabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - }, - }, - detection_rule_detail: { - type: 'array', - items: { - rule_name: { - type: 'keyword', - _meta: { description: 'The name of the detection rule' }, - }, - rule_id: { - type: 'keyword', - _meta: { description: 'The UUID id of the detection rule' }, - }, - rule_type: { - type: 'keyword', - _meta: { description: 'The type of detection rule. ie eql, query...' }, - }, - rule_version: { type: 'long', _meta: { description: 'The version of the rule' } }, - enabled: { - type: 'boolean', - _meta: { description: 'If the detection rule has been enabled by the user' }, - }, - elastic_rule: { - type: 'boolean', - _meta: { description: 'If the detection rule has been authored by Elastic' }, - }, - created_on: { - type: 'keyword', - _meta: { description: 'When the detection rule was created on the cluster' }, - }, - updated_on: { - type: 'keyword', - _meta: { description: 'When the detection rule was updated on the cluster' }, - }, - alert_count_daily: { - type: 'long', - _meta: { description: 'The number of daily alerts generated by a rule' }, - }, - cases_count_total: { - type: 'long', - _meta: { description: 'The number of total cases generated by a rule' }, - }, - has_legacy_notification: { - type: 'boolean', - _meta: { description: 'True if this rule has a legacy notification' }, - }, - has_notification: { - type: 'boolean', - _meta: { description: 'True if this rule has a notification' }, - }, - }, - }, - }, - ml_jobs: { - ml_job_usage: { - custom: { - enabled: { - type: 'long', - _meta: { description: 'The number of custom ML jobs rules enabled' }, - }, - disabled: { - type: 'long', - _meta: { description: 'The number of custom ML jobs rules disabled' }, - }, - }, - elastic: { - enabled: { - type: 'long', - _meta: { description: 'The number of elastic provided ML jobs rules enabled' }, - }, - disabled: { - type: 'long', - _meta: { description: 'The number of elastic provided ML jobs rules disabled' }, - }, - }, - }, - ml_job_metrics: { - type: 'array', - items: { - job_id: { - type: 'keyword', - _meta: { description: 'Identifier for the anomaly detection job' }, - }, - open_time: { - type: 'keyword', - _meta: { - description: - 'For open jobs only, the elapsed time for which the job has been open', - }, - }, - create_time: { - type: 'keyword', - _meta: { description: 'The time the job was created' }, - }, - finished_time: { - type: 'keyword', - _meta: { - description: 'If the job closed or failed, this is the time the job finished', - }, - }, - state: { - type: 'keyword', - _meta: { description: 'The status of the anomaly detection job' }, - }, - data_counts: { - bucket_count: { - type: 'long', - _meta: { description: 'The number of buckets processed' }, - }, - empty_bucket_count: { - type: 'long', - _meta: { description: 'The number of buckets which did not contain any data' }, - }, - input_bytes: { - type: 'long', - _meta: { - description: - 'The number of bytes of input data posted to the anomaly detection job', - }, - }, - input_record_count: { - type: 'long', - _meta: { - description: - 'The number of input documents posted to the anomaly detection job', - }, - }, - last_data_time: { - type: 'long', - _meta: { - description: - 'The timestamp at which data was last analyzed, according to server time', - }, - }, - processed_record_count: { - type: 'long', - _meta: { - description: - 'The number of input documents that have been processed by the anomaly detection job', - }, - }, - }, - model_size_stats: { - bucket_allocation_failures_count: { - type: 'long', - _meta: { - description: - 'The number of buckets for which new entities in incoming data were not processed due to insufficient model memory', - }, - }, - model_bytes: { - type: 'long', - _meta: { description: 'The number of bytes of memory used by the models' }, - }, - model_bytes_exceeded: { - type: 'long', - _meta: { - description: - 'The number of bytes over the high limit for memory usage at the last allocation failure', - }, - }, - model_bytes_memory_limit: { - type: 'long', - _meta: { - description: - 'The upper limit for model memory usage, checked on increasing values', - }, - }, - peak_model_bytes: { - type: 'long', - _meta: { - description: 'The peak number of bytes of memory ever used by the models', - }, - }, - }, - timing_stats: { - bucket_count: { - type: 'long', - _meta: { description: 'The number of buckets processed' }, - }, - exponential_average_bucket_processing_time_ms: { - type: 'long', - _meta: { - description: - 'Exponential moving average of all bucket processing times, in milliseconds', - }, - }, - exponential_average_bucket_processing_time_per_hour_ms: { - type: 'long', - _meta: { - description: - 'Exponentially-weighted moving average of bucket processing times calculated in a 1 hour time window, in milliseconds', - }, - }, - maximum_bucket_processing_time_ms: { - type: 'long', - _meta: { - description: 'Maximum among all bucket processing times, in milliseconds', - }, - }, - minimum_bucket_processing_time_ms: { - type: 'long', - _meta: { - description: 'Minimum among all bucket processing times, in milliseconds', - }, - }, - total_bucket_processing_time_ms: { - type: 'long', - _meta: { description: 'Sum of all bucket processing times, in milliseconds' }, - }, - }, - datafeed: { - datafeed_id: { - type: 'keyword', - _meta: { - description: - 'A numerical character string that uniquely identifies the datafeed', - }, - }, - state: { - type: 'keyword', - _meta: { description: 'The status of the datafeed' }, - }, - timing_stats: { - average_search_time_per_bucket_ms: { - type: 'long', - _meta: { description: 'The average search time per bucket, in milliseconds' }, - }, - bucket_count: { - type: 'long', - _meta: { description: 'The number of buckets processed' }, - }, - exponential_average_search_time_per_hour_ms: { - type: 'long', - _meta: { - description: 'The exponential average search time per hour, in milliseconds', - }, - }, - search_count: { - type: 'long', - _meta: { description: 'The number of searches run by the datafeed' }, - }, - total_search_time_ms: { - type: 'long', - _meta: { - description: 'The total time the datafeed spent searching, in milliseconds', - }, - }, - }, - }, - }, - }, - }, - }, - }, + schema, isReady: () => true, fetch: async ({ esClient }: CollectorFetchContext): Promise => { - const internalSavedObjectsClient = await getInternalSavedObjectsClient(core); - const soClient = internalSavedObjectsClient as unknown as SavedObjectsClientContract; - + const savedObjectsClient = await getInternalSavedObjectsClient(core); + const detectionMetrics = await getDetectionsMetrics({ + kibanaIndex, + signalsIndex, + esClient, + savedObjectsClient, + logger, + mlClient: ml, + }); return { - detectionMetrics: - (await fetchDetectionsMetrics(kibanaIndex, signalsIndex, esClient, soClient, ml)) || {}, + detectionMetrics: detectionMetrics || {}, }; }, }); diff --git a/x-pack/plugins/security_solution/server/usage/detections/constants.ts b/x-pack/plugins/security_solution/server/usage/constants.ts similarity index 100% rename from x-pack/plugins/security_solution/server/usage/detections/constants.ts rename to x-pack/plugins/security_solution/server/usage/constants.ts diff --git a/x-pack/plugins/security_solution/server/usage/detections/detection_ml_helpers.ts b/x-pack/plugins/security_solution/server/usage/detections/detection_ml_helpers.ts deleted file mode 100644 index 1aadcfdc5478a..0000000000000 --- a/x-pack/plugins/security_solution/server/usage/detections/detection_ml_helpers.ts +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { KibanaRequest, SavedObjectsClientContract } from '../../../../../../src/core/server'; -import { MlDatafeedStats, MlJob, MlPluginSetup } from '../../../../ml/server'; -import { isJobStarted } from '../../../common/machine_learning/helpers'; -import { isSecurityJob } from '../../../common/machine_learning/is_security_job'; -import { DetectionsMetric, MlJobMetric, MlJobsUsage, MlJobUsage } from './types'; - -/** - * Default ml job usage count - */ -export const initialMlJobsUsage: MlJobsUsage = { - custom: { - enabled: 0, - disabled: 0, - }, - elastic: { - enabled: 0, - disabled: 0, - }, -}; - -export const updateMlJobsUsage = (jobMetric: DetectionsMetric, usage: MlJobsUsage): MlJobsUsage => { - const { isEnabled, isElastic } = jobMetric; - if (isEnabled && isElastic) { - return { - ...usage, - elastic: { - ...usage.elastic, - enabled: usage.elastic.enabled + 1, - }, - }; - } else if (!isEnabled && isElastic) { - return { - ...usage, - elastic: { - ...usage.elastic, - disabled: usage.elastic.disabled + 1, - }, - }; - } else if (isEnabled && !isElastic) { - return { - ...usage, - custom: { - ...usage.custom, - enabled: usage.custom.enabled + 1, - }, - }; - } else if (!isEnabled && !isElastic) { - return { - ...usage, - custom: { - ...usage.custom, - disabled: usage.custom.disabled + 1, - }, - }; - } else { - return usage; - } -}; - -export const getMlJobMetrics = async ( - ml: MlPluginSetup | undefined, - savedObjectClient: SavedObjectsClientContract -): Promise => { - let jobsUsage: MlJobsUsage = initialMlJobsUsage; - - if (ml) { - try { - const fakeRequest = { headers: {} } as KibanaRequest; - - const modules = await ml.modulesProvider(fakeRequest, savedObjectClient).listModules(); - const moduleJobs = modules.flatMap((module) => module.jobs); - const jobs = await ml.jobServiceProvider(fakeRequest, savedObjectClient).jobsSummary(); - - jobsUsage = jobs.filter(isSecurityJob).reduce((usage, job) => { - const isElastic = moduleJobs.some((moduleJob) => moduleJob.id === job.id); - const isEnabled = isJobStarted(job.jobState, job.datafeedState); - - return updateMlJobsUsage({ isElastic, isEnabled }, usage); - }, initialMlJobsUsage); - - const jobsType = 'security'; - const securityJobStats = await ml - .anomalyDetectorsProvider(fakeRequest, savedObjectClient) - .jobStats(jobsType); - - const jobDetails = await ml - .anomalyDetectorsProvider(fakeRequest, savedObjectClient) - .jobs(jobsType); - - const jobDetailsCache = new Map(); - jobDetails.jobs.forEach((detail) => jobDetailsCache.set(detail.job_id, detail)); - - const datafeedStats = await ml - .anomalyDetectorsProvider(fakeRequest, savedObjectClient) - .datafeedStats(); - - const datafeedStatsCache = new Map(); - datafeedStats.datafeeds.forEach((datafeedStat) => - datafeedStatsCache.set(`${datafeedStat.datafeed_id}`, datafeedStat) - ); - - const jobMetrics: MlJobMetric[] = securityJobStats.jobs.map((stat) => { - const jobId = stat.job_id; - const jobDetail = jobDetailsCache.get(stat.job_id); - const datafeed = datafeedStatsCache.get(`datafeed-${jobId}`); - - return { - job_id: jobId, - open_time: stat.open_time, - create_time: jobDetail?.create_time, - finished_time: jobDetail?.finished_time, - state: stat.state, - data_counts: { - bucket_count: stat.data_counts.bucket_count, - empty_bucket_count: stat.data_counts.empty_bucket_count, - input_bytes: stat.data_counts.input_bytes, - input_record_count: stat.data_counts.input_record_count, - last_data_time: stat.data_counts.last_data_time, - processed_record_count: stat.data_counts.processed_record_count, - }, - model_size_stats: { - bucket_allocation_failures_count: - stat.model_size_stats.bucket_allocation_failures_count, - memory_status: stat.model_size_stats.memory_status, - model_bytes: stat.model_size_stats.model_bytes, - model_bytes_exceeded: stat.model_size_stats.model_bytes_exceeded, - model_bytes_memory_limit: stat.model_size_stats.model_bytes_memory_limit, - peak_model_bytes: stat.model_size_stats.peak_model_bytes, - }, - timing_stats: { - average_bucket_processing_time_ms: stat.timing_stats.average_bucket_processing_time_ms, - bucket_count: stat.timing_stats.bucket_count, - exponential_average_bucket_processing_time_ms: - stat.timing_stats.exponential_average_bucket_processing_time_ms, - exponential_average_bucket_processing_time_per_hour_ms: - stat.timing_stats.exponential_average_bucket_processing_time_per_hour_ms, - maximum_bucket_processing_time_ms: stat.timing_stats.maximum_bucket_processing_time_ms, - minimum_bucket_processing_time_ms: stat.timing_stats.minimum_bucket_processing_time_ms, - total_bucket_processing_time_ms: stat.timing_stats.total_bucket_processing_time_ms, - }, - datafeed: { - datafeed_id: datafeed?.datafeed_id, - state: datafeed?.state, - timing_stats: { - bucket_count: datafeed?.timing_stats.bucket_count, - exponential_average_search_time_per_hour_ms: - datafeed?.timing_stats.exponential_average_search_time_per_hour_ms, - search_count: datafeed?.timing_stats.search_count, - total_search_time_ms: datafeed?.timing_stats.total_search_time_ms, - }, - }, - } as MlJobMetric; - }); - - return { - ml_job_usage: jobsUsage, - ml_job_metrics: jobMetrics, - }; - } catch (e) { - // ignore failure, usage will be zeroed - } - } - - return { - ml_job_usage: initialMlJobsUsage, - ml_job_metrics: [], - }; -}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts b/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts deleted file mode 100644 index c52188d1287ef..0000000000000 --- a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts +++ /dev/null @@ -1,464 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - SIGNALS_ID, - EQL_RULE_TYPE_ID, - INDICATOR_RULE_TYPE_ID, - ML_RULE_TYPE_ID, - QUERY_RULE_TYPE_ID, - THRESHOLD_RULE_TYPE_ID, - SAVED_QUERY_RULE_TYPE_ID, -} from '@kbn/securitysolution-rules'; -import { ALERT_RULE_UUID } from '@kbn/rule-data-utils'; -import { CASE_COMMENT_SAVED_OBJECT } from '../../../../cases/common/constants'; - -import { ElasticsearchClient, SavedObjectsClientContract } from '../../../../../../src/core/server'; -import { isElasticRule } from './index'; -import type { - AlertsAggregationResponse, - CasesSavedObject, - DetectionRulesTypeUsage, - DetectionRuleMetric, - DetectionRuleAdoption, - RuleSearchParams, - RuleSearchResult, - DetectionMetrics, -} from './types'; -// eslint-disable-next-line no-restricted-imports -import { legacyRuleActionsSavedObjectType } from '../../lib/detection_engine/rule_actions/legacy_saved_object_mappings'; -// eslint-disable-next-line no-restricted-imports -import { LegacyIRuleActionsAttributesSavedObjectAttributes } from '../../lib/detection_engine/rule_actions/legacy_types'; -import { getDetectionRules } from './get_detection_rules'; -import { MAX_PER_PAGE, MAX_RESULTS_WINDOW } from './constants'; -import { getAlerts } from './get_alerts'; - -/** - * Initial detection metrics initialized. - */ -export const getInitialDetectionMetrics = (): DetectionMetrics => ({ - ml_jobs: { - ml_job_usage: { - custom: { - enabled: 0, - disabled: 0, - }, - elastic: { - enabled: 0, - disabled: 0, - }, - }, - ml_job_metrics: [], - }, - detection_rules: { - detection_rule_detail: [], - detection_rule_usage: initialDetectionRulesUsage, - }, -}); - -/** - * Default detection rule usage count, split by type + elastic/custom - */ -export const initialDetectionRulesUsage: DetectionRulesTypeUsage = { - query: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - threshold: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - eql: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - machine_learning: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - threat_match: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - elastic_total: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - custom_total: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, -}; - -/* eslint-disable complexity */ -export const updateDetectionRuleUsage = ( - detectionRuleMetric: DetectionRuleMetric, - usage: DetectionRulesTypeUsage -): DetectionRulesTypeUsage => { - let updatedUsage = usage; - - const legacyNotificationEnabled = - detectionRuleMetric.has_legacy_notification && detectionRuleMetric.enabled; - - const legacyNotificationDisabled = - detectionRuleMetric.has_legacy_notification && !detectionRuleMetric.enabled; - - const notificationEnabled = detectionRuleMetric.has_notification && detectionRuleMetric.enabled; - - const notificationDisabled = detectionRuleMetric.has_notification && !detectionRuleMetric.enabled; - - if (detectionRuleMetric.rule_type === 'query') { - updatedUsage = { - ...usage, - query: { - ...usage.query, - enabled: detectionRuleMetric.enabled ? usage.query.enabled + 1 : usage.query.enabled, - disabled: !detectionRuleMetric.enabled ? usage.query.disabled + 1 : usage.query.disabled, - alerts: usage.query.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.query.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.query.legacy_notifications_enabled + 1 - : usage.query.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.query.legacy_notifications_disabled + 1 - : usage.query.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.query.notifications_enabled + 1 - : usage.query.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.query.notifications_disabled + 1 - : usage.query.notifications_disabled, - }, - }; - } else if (detectionRuleMetric.rule_type === 'threshold') { - updatedUsage = { - ...usage, - threshold: { - ...usage.threshold, - enabled: detectionRuleMetric.enabled - ? usage.threshold.enabled + 1 - : usage.threshold.enabled, - disabled: !detectionRuleMetric.enabled - ? usage.threshold.disabled + 1 - : usage.threshold.disabled, - alerts: usage.threshold.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.threshold.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.threshold.legacy_notifications_enabled + 1 - : usage.threshold.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.threshold.legacy_notifications_disabled + 1 - : usage.threshold.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.threshold.notifications_enabled + 1 - : usage.threshold.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.threshold.notifications_disabled + 1 - : usage.threshold.notifications_disabled, - }, - }; - } else if (detectionRuleMetric.rule_type === 'eql') { - updatedUsage = { - ...usage, - eql: { - ...usage.eql, - enabled: detectionRuleMetric.enabled ? usage.eql.enabled + 1 : usage.eql.enabled, - disabled: !detectionRuleMetric.enabled ? usage.eql.disabled + 1 : usage.eql.disabled, - alerts: usage.eql.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.eql.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.eql.legacy_notifications_enabled + 1 - : usage.eql.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.eql.legacy_notifications_disabled + 1 - : usage.eql.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.eql.notifications_enabled + 1 - : usage.eql.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.eql.notifications_disabled + 1 - : usage.eql.notifications_disabled, - }, - }; - } else if (detectionRuleMetric.rule_type === 'machine_learning') { - updatedUsage = { - ...usage, - machine_learning: { - ...usage.machine_learning, - enabled: detectionRuleMetric.enabled - ? usage.machine_learning.enabled + 1 - : usage.machine_learning.enabled, - disabled: !detectionRuleMetric.enabled - ? usage.machine_learning.disabled + 1 - : usage.machine_learning.disabled, - alerts: usage.machine_learning.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.machine_learning.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.machine_learning.legacy_notifications_enabled + 1 - : usage.machine_learning.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.machine_learning.legacy_notifications_disabled + 1 - : usage.machine_learning.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.machine_learning.notifications_enabled + 1 - : usage.machine_learning.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.machine_learning.notifications_disabled + 1 - : usage.machine_learning.notifications_disabled, - }, - }; - } else if (detectionRuleMetric.rule_type === 'threat_match') { - updatedUsage = { - ...usage, - threat_match: { - ...usage.threat_match, - enabled: detectionRuleMetric.enabled - ? usage.threat_match.enabled + 1 - : usage.threat_match.enabled, - disabled: !detectionRuleMetric.enabled - ? usage.threat_match.disabled + 1 - : usage.threat_match.disabled, - alerts: usage.threat_match.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.threat_match.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.threat_match.legacy_notifications_enabled + 1 - : usage.threat_match.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.threat_match.legacy_notifications_disabled + 1 - : usage.threat_match.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.threat_match.notifications_enabled + 1 - : usage.threat_match.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.threat_match.notifications_disabled + 1 - : usage.threat_match.notifications_disabled, - }, - }; - } - - if (detectionRuleMetric.elastic_rule) { - updatedUsage = { - ...updatedUsage, - elastic_total: { - ...updatedUsage.elastic_total, - enabled: detectionRuleMetric.enabled - ? updatedUsage.elastic_total.enabled + 1 - : updatedUsage.elastic_total.enabled, - disabled: !detectionRuleMetric.enabled - ? updatedUsage.elastic_total.disabled + 1 - : updatedUsage.elastic_total.disabled, - alerts: updatedUsage.elastic_total.alerts + detectionRuleMetric.alert_count_daily, - cases: updatedUsage.elastic_total.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? updatedUsage.elastic_total.legacy_notifications_enabled + 1 - : updatedUsage.elastic_total.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? updatedUsage.elastic_total.legacy_notifications_disabled + 1 - : updatedUsage.elastic_total.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? updatedUsage.elastic_total.notifications_enabled + 1 - : updatedUsage.elastic_total.notifications_enabled, - notifications_disabled: notificationDisabled - ? updatedUsage.elastic_total.notifications_disabled + 1 - : updatedUsage.elastic_total.notifications_disabled, - }, - }; - } else { - updatedUsage = { - ...updatedUsage, - custom_total: { - ...updatedUsage.custom_total, - enabled: detectionRuleMetric.enabled - ? updatedUsage.custom_total.enabled + 1 - : updatedUsage.custom_total.enabled, - disabled: !detectionRuleMetric.enabled - ? updatedUsage.custom_total.disabled + 1 - : updatedUsage.custom_total.disabled, - alerts: updatedUsage.custom_total.alerts + detectionRuleMetric.alert_count_daily, - cases: updatedUsage.custom_total.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? updatedUsage.custom_total.legacy_notifications_enabled + 1 - : updatedUsage.custom_total.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? updatedUsage.custom_total.legacy_notifications_disabled + 1 - : updatedUsage.custom_total.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? updatedUsage.custom_total.notifications_enabled + 1 - : updatedUsage.custom_total.notifications_enabled, - notifications_disabled: notificationDisabled - ? updatedUsage.custom_total.notifications_disabled + 1 - : updatedUsage.custom_total.notifications_disabled, - }, - }; - } - - return updatedUsage; -}; - -export const getDetectionRuleMetrics = async ( - kibanaIndex: string, - signalsIndex: string, - esClient: ElasticsearchClient, - savedObjectClient: SavedObjectsClientContract -): Promise => { - let rulesUsage: DetectionRulesTypeUsage = initialDetectionRulesUsage; - try { - const ruleResults = await getDetectionRules({ - esClient, - kibanaIndex, - maxPerPage: MAX_PER_PAGE, - maxSize: MAX_RESULTS_WINDOW, - }); - const detectionAlertsResp = await getAlerts({ - esClient, - signalsIndex: `${signalsIndex}*`, - maxPerPage: MAX_PER_PAGE, - maxSize: MAX_RESULTS_WINDOW, - }); - - const cases = await savedObjectClient.find({ - type: CASE_COMMENT_SAVED_OBJECT, - page: 1, - perPage: MAX_RESULTS_WINDOW, - namespaces: ['*'], - filter: `${CASE_COMMENT_SAVED_OBJECT}.attributes.type: alert`, - }); - - // Once we are confident all rules relying on side-car actions SO's have been migrated to SO references we should remove this function. - const legacyRuleActions = - await savedObjectClient.find({ - type: legacyRuleActionsSavedObjectType, - page: 1, - perPage: MAX_RESULTS_WINDOW, - namespaces: ['*'], - }); - - const legacyNotificationRuleIds = legacyRuleActions.saved_objects.reduce( - (cache, legacyNotificationsObject) => { - const ruleRef = legacyNotificationsObject.references.find( - (reference) => reference.name === 'alert_0' && reference.type === 'alert' - ); - if (ruleRef != null) { - const enabled = legacyNotificationsObject.attributes.ruleThrottle !== 'no_actions'; - cache.set(ruleRef.id, { enabled }); - } - return cache; - }, - new Map() - ); - - const casesCache = cases.saved_objects.reduce((cache, { attributes: casesObject }) => { - const ruleId = casesObject.rule.id; - if (ruleId != null) { - const cacheCount = cache.get(ruleId); - if (cacheCount === undefined) { - cache.set(ruleId, 1); - } else { - cache.set(ruleId, cacheCount + 1); - } - } - return cache; - }, new Map()); - - const alertBuckets = detectionAlertsResp.aggregations?.detectionAlerts?.buckets ?? []; - - const alertsCache = new Map(); - alertBuckets.map((bucket) => alertsCache.set(bucket.key, bucket.doc_count)); - if (ruleResults.length > 0) { - const ruleObjects = ruleResults.map((hit) => { - const ruleId = hit._id.split(':')[1]; - const isElastic = isElasticRule(hit._source?.alert.tags); - - // Even if the legacy notification is set to "no_actions" we still count the rule as having a legacy notification that is not migrated yet. - const hasLegacyNotification = legacyNotificationRuleIds.get(ruleId) != null; - - // We only count a rule as having a notification and being "enabled" if it is _not_ set to "no_actions"/"muteAll" and it has at least one action within its array. - const hasNotification = - !hasLegacyNotification && - hit._source?.alert.actions != null && - hit._source?.alert.actions.length > 0 && - hit._source?.alert.muteAll !== true; - - return { - rule_name: hit._source?.alert.name, - rule_id: hit._source?.alert.params.ruleId, - rule_type: hit._source?.alert.params.type, - rule_version: Number(hit._source?.alert.params.version), - enabled: hit._source?.alert.enabled, - elastic_rule: isElastic, - created_on: hit._source?.alert.createdAt, - updated_on: hit._source?.alert.updatedAt, - alert_count_daily: alertsCache.get(ruleId) || 0, - cases_count_total: casesCache.get(ruleId) || 0, - has_legacy_notification: hasLegacyNotification, - has_notification: hasNotification, - } as DetectionRuleMetric; - }); - - // Only bring back rule detail on elastic prepackaged detection rules - const elasticRuleObjects = ruleObjects.filter((hit) => hit.elastic_rule === true); - - rulesUsage = ruleObjects.reduce((usage, rule) => { - return updateDetectionRuleUsage(rule, usage); - }, rulesUsage); - - return { - detection_rule_detail: elasticRuleObjects, - detection_rule_usage: rulesUsage, - }; - } - } catch (e) { - // ignore failure, usage will be zeroed - } - - return { - detection_rule_detail: [], - detection_rule_usage: rulesUsage, - }; -}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_alerts.ts b/x-pack/plugins/security_solution/server/usage/detections/get_alerts.ts deleted file mode 100644 index a8dbb18807f0e..0000000000000 --- a/x-pack/plugins/security_solution/server/usage/detections/get_alerts.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { ALERT_RULE_UUID } from '@kbn/rule-data-utils'; -import { ElasticsearchClient } from 'kibana/server'; -import { fetchWithPit } from './fetch_with_pit'; -import { AlertsAggregationResponse } from './types'; - -export interface GetAlertsOptions { - esClient: ElasticsearchClient; - signalsIndex: string; - maxSize: number; - maxPerPage: number; -} - -export const getAlerts = async ({ - esClient, - signalsIndex, - maxSize, - maxPerPage, -}: GetAlertsOptions): Promise>> => { - return fetchWithPit({ - esClient, - index: signalsIndex, - maxSize, - maxPerPage, - searchRequest: { - aggs: { - detectionAlerts: { - terms: { field: ALERT_RULE_UUID }, - }, - }, - query: { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: 'now-24h', - lte: 'now', - }, - }, - }, - ], - }, - }, - }, - }); -}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_initial_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/get_initial_usage.ts new file mode 100644 index 0000000000000..0d885aa3b142c --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/get_initial_usage.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DetectionMetrics } from './types'; + +import { getInitialMlJobUsage } from './ml_jobs/get_initial_usage'; +import { getInitialRulesUsage } from './rules/get_initial_usage'; + +/** + * Initial detection metrics initialized. + */ +export const getInitialDetectionMetrics = (): DetectionMetrics => ({ + ml_jobs: { + ml_job_usage: getInitialMlJobUsage(), + ml_job_metrics: [], + }, + detection_rules: { + detection_rule_detail: [], + detection_rule_usage: getInitialRulesUsage(), + }, +}); diff --git a/x-pack/plugins/security_solution/server/usage/detections/detections.test.ts b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.test.ts similarity index 75% rename from x-pack/plugins/security_solution/server/usage/detections/detections.test.ts rename to x-pack/plugins/security_solution/server/usage/detections/get_metrics.test.ts index d08f915e4428f..3b1e9883b80ef 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/detections.test.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.test.ts @@ -5,47 +5,57 @@ * 2.0. */ -import { ElasticsearchClient } from '../../../../../../src/core/server'; +import type { ElasticsearchClient } from '../../../../../../src/core/server'; +import type { DetectionMetrics } from './types'; + import { elasticsearchServiceMock, + loggingSystemMock, savedObjectsClientMock, } from '../../../../../../src/core/server/mocks'; import { mlServicesMock } from '../../lib/machine_learning/mocks'; -import { fetchDetectionsMetrics } from './index'; import { - getMockJobSummaryResponse, + getMockMlJobSummaryResponse, getMockListModulesResponse, getMockMlJobDetailsResponse, getMockMlJobStatsResponse, getMockMlDatafeedStatsResponse, getMockRuleSearchResponse, - getMockRuleAlertsResponse, - getMockAlertCasesResponse, -} from './detections.mocks'; -import { getInitialDetectionMetrics, initialDetectionRulesUsage } from './detection_rule_helpers'; -import { DetectionMetrics } from './types'; +} from './ml_jobs/get_metrics.mocks'; +import { getMockRuleAlertsResponse, getMockAlertCasesResponse } from './rules/get_metrics.mocks'; +import { getInitialDetectionMetrics } from './get_initial_usage'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; +import { getDetectionsMetrics } from './get_metrics'; +import { getInitialRulesUsage } from './rules/get_initial_usage'; describe('Detections Usage and Metrics', () => { - let esClientMock: jest.Mocked; - let mlMock: ReturnType; + let esClient: jest.Mocked; + let mlClient: ReturnType; let savedObjectsClient: ReturnType; - describe('getDetectionRuleMetrics()', () => { + describe('getRuleMetrics()', () => { beforeEach(() => { - esClientMock = elasticsearchServiceMock.createClusterClient().asInternalUser; - mlMock = mlServicesMock.createSetupContract(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + mlClient = mlServicesMock.createSetupContract(); savedObjectsClient = savedObjectsClientMock.create(); }); it('returns zeroed counts if calls are empty', async () => { - const result = await fetchDetectionsMetrics('', '', esClientMock, savedObjectsClient, mlMock); + const logger = loggingSystemMock.createLogger(); + const result = await getDetectionsMetrics({ + kibanaIndex: '', + signalsIndex: '', + esClient, + savedObjectsClient, + logger, + mlClient, + }); expect(result).toEqual(getInitialDetectionMetrics()); }); it('returns information with rule, alerts and cases', async () => { - esClientMock.search + esClient.search .mockResolvedValueOnce( elasticsearchClientMock.createApiResponse({ body: getMockRuleSearchResponse() }) ) @@ -53,7 +63,15 @@ describe('Detections Usage and Metrics', () => { elasticsearchClientMock.createApiResponse({ body: getMockRuleAlertsResponse(3400) }) ); savedObjectsClient.find.mockResolvedValue(getMockAlertCasesResponse()); - const result = await fetchDetectionsMetrics('', '', esClientMock, savedObjectsClient, mlMock); + const logger = loggingSystemMock.createLogger(); + const result = await getDetectionsMetrics({ + kibanaIndex: '', + signalsIndex: '', + esClient, + savedObjectsClient, + logger, + mlClient, + }); expect(result).toEqual({ ...getInitialDetectionMetrics(), @@ -75,7 +93,7 @@ describe('Detections Usage and Metrics', () => { }, ], detection_rule_usage: { - ...initialDetectionRulesUsage, + ...getInitialRulesUsage(), query: { enabled: 0, disabled: 1, @@ -102,7 +120,7 @@ describe('Detections Usage and Metrics', () => { }); it('returns information with on non elastic prebuilt rule', async () => { - esClientMock.search + esClient.search .mockResolvedValueOnce( elasticsearchClientMock.createApiResponse({ body: getMockRuleSearchResponse('not_immutable'), @@ -112,14 +130,22 @@ describe('Detections Usage and Metrics', () => { elasticsearchClientMock.createApiResponse({ body: getMockRuleAlertsResponse(800) }) ); savedObjectsClient.find.mockResolvedValue(getMockAlertCasesResponse()); - const result = await fetchDetectionsMetrics('', '', esClientMock, savedObjectsClient, mlMock); + const logger = loggingSystemMock.createLogger(); + const result = await getDetectionsMetrics({ + kibanaIndex: '', + signalsIndex: '', + esClient, + savedObjectsClient, + logger, + mlClient, + }); expect(result).toEqual({ ...getInitialDetectionMetrics(), detection_rules: { detection_rule_detail: [], // *should not* contain custom detection rule details detection_rule_usage: { - ...initialDetectionRulesUsage, + ...getInitialRulesUsage(), custom_total: { alerts: 800, cases: 1, @@ -146,7 +172,7 @@ describe('Detections Usage and Metrics', () => { }); it('returns information with rule, no alerts and no cases', async () => { - esClientMock.search + esClient.search .mockResolvedValueOnce( elasticsearchClientMock.createApiResponse({ body: getMockRuleSearchResponse() }) ) @@ -154,7 +180,15 @@ describe('Detections Usage and Metrics', () => { elasticsearchClientMock.createApiResponse({ body: getMockRuleAlertsResponse(0) }) ); savedObjectsClient.find.mockResolvedValue(getMockAlertCasesResponse()); - const result = await fetchDetectionsMetrics('', '', esClientMock, savedObjectsClient, mlMock); + const logger = loggingSystemMock.createLogger(); + const result = await getDetectionsMetrics({ + kibanaIndex: '', + signalsIndex: '', + esClient, + savedObjectsClient, + logger, + mlClient, + }); expect(result).toEqual({ ...getInitialDetectionMetrics(), @@ -176,7 +210,7 @@ describe('Detections Usage and Metrics', () => { }, ], detection_rule_usage: { - ...initialDetectionRulesUsage, + ...getInitialRulesUsage(), elastic_total: { alerts: 0, cases: 1, @@ -203,29 +237,38 @@ describe('Detections Usage and Metrics', () => { }); }); - describe('fetchDetectionsMetrics()', () => { + describe('getDetectionsMetrics()', () => { beforeEach(() => { - esClientMock = elasticsearchServiceMock.createClusterClient().asInternalUser; - mlMock = mlServicesMock.createSetupContract(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + mlClient = mlServicesMock.createSetupContract(); savedObjectsClient = savedObjectsClientMock.create(); }); it('returns an empty array if there is no data', async () => { - mlMock.anomalyDetectorsProvider.mockReturnValue({ + mlClient.anomalyDetectorsProvider.mockReturnValue({ jobs: null, jobStats: null, - } as unknown as ReturnType); - const result = await fetchDetectionsMetrics('', '', esClientMock, savedObjectsClient, mlMock); + } as unknown as ReturnType); + const logger = loggingSystemMock.createLogger(); + const result = await getDetectionsMetrics({ + kibanaIndex: '', + signalsIndex: '', + esClient, + savedObjectsClient, + logger, + mlClient, + }); expect(result).toEqual(getInitialDetectionMetrics()); }); it('returns an ml job telemetry object from anomaly detectors provider', async () => { - const mockJobSummary = jest.fn().mockResolvedValue(getMockJobSummaryResponse()); + const logger = loggingSystemMock.createLogger(); + const mockJobSummary = jest.fn().mockResolvedValue(getMockMlJobSummaryResponse()); const mockListModules = jest.fn().mockResolvedValue(getMockListModulesResponse()); - mlMock.modulesProvider.mockReturnValue({ + mlClient.modulesProvider.mockReturnValue({ listModules: mockListModules, - } as unknown as ReturnType); - mlMock.jobServiceProvider.mockReturnValue({ + } as unknown as ReturnType); + mlClient.jobServiceProvider.mockReturnValue({ jobsSummary: mockJobSummary, }); const mockJobsResponse = jest.fn().mockResolvedValue(getMockMlJobDetailsResponse()); @@ -234,13 +277,20 @@ describe('Detections Usage and Metrics', () => { .fn() .mockResolvedValue(getMockMlDatafeedStatsResponse()); - mlMock.anomalyDetectorsProvider.mockReturnValue({ + mlClient.anomalyDetectorsProvider.mockReturnValue({ jobs: mockJobsResponse, jobStats: mockJobStatsResponse, datafeedStats: mockDatafeedStatsResponse, - } as unknown as ReturnType); + } as unknown as ReturnType); - const result = await fetchDetectionsMetrics('', '', esClientMock, savedObjectsClient, mlMock); + const result = await getDetectionsMetrics({ + kibanaIndex: '', + signalsIndex: '', + esClient, + savedObjectsClient, + logger, + mlClient, + }); expect(result).toEqual( expect.objectContaining({ diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts new file mode 100644 index 0000000000000..e2abd99fb1812 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, SavedObjectsClientContract, Logger } from 'kibana/server'; +import type { MlPluginSetup } from '../../../../ml/server'; +import type { DetectionMetrics } from './types'; + +import { getMlJobMetrics } from './ml_jobs/get_metrics'; +import { getRuleMetrics } from './rules/get_metrics'; +import { getInitialRulesUsage } from './rules/get_initial_usage'; +import { getInitialMlJobUsage } from './ml_jobs/get_initial_usage'; + +export interface GetDetectionsMetricsOptions { + kibanaIndex: string; + signalsIndex: string; + esClient: ElasticsearchClient; + savedObjectsClient: SavedObjectsClientContract; + logger: Logger; + mlClient: MlPluginSetup | undefined; +} + +export const getDetectionsMetrics = async ({ + kibanaIndex, + signalsIndex, + esClient, + savedObjectsClient, + logger, + mlClient, +}: GetDetectionsMetricsOptions): Promise => { + const [mlJobMetrics, detectionRuleMetrics] = await Promise.allSettled([ + getMlJobMetrics({ mlClient, savedObjectsClient, logger }), + getRuleMetrics({ kibanaIndex, signalsIndex, esClient, savedObjectsClient, logger }), + ]); + + return { + ml_jobs: + mlJobMetrics.status === 'fulfilled' + ? mlJobMetrics.value + : { ml_job_metrics: [], ml_job_usage: getInitialMlJobUsage() }, + detection_rules: + detectionRuleMetrics.status === 'fulfilled' + ? detectionRuleMetrics.value + : { detection_rule_detail: [], detection_rule_usage: getInitialRulesUsage() }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/index.ts b/x-pack/plugins/security_solution/server/usage/detections/index.ts deleted file mode 100644 index a8d2ead83eec7..0000000000000 --- a/x-pack/plugins/security_solution/server/usage/detections/index.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient, SavedObjectsClientContract } from '../../../../../../src/core/server'; -import { MlPluginSetup } from '../../../../ml/server'; -import { getDetectionRuleMetrics, initialDetectionRulesUsage } from './detection_rule_helpers'; -import { getMlJobMetrics, initialMlJobsUsage } from './detection_ml_helpers'; -import { DetectionMetrics } from './types'; - -import { INTERNAL_IMMUTABLE_KEY } from '../../../common/constants'; - -export const isElasticRule = (tags: string[] = []) => - tags.includes(`${INTERNAL_IMMUTABLE_KEY}:true`); - -export const fetchDetectionsMetrics = async ( - kibanaIndex: string, - signalsIndex: string, - esClient: ElasticsearchClient, - soClient: SavedObjectsClientContract, - mlClient: MlPluginSetup | undefined -): Promise => { - const [mlJobMetrics, detectionRuleMetrics] = await Promise.allSettled([ - getMlJobMetrics(mlClient, soClient), - getDetectionRuleMetrics(kibanaIndex, signalsIndex, esClient, soClient), - ]); - - return { - ml_jobs: - mlJobMetrics.status === 'fulfilled' - ? mlJobMetrics.value - : { ml_job_metrics: [], ml_job_usage: initialMlJobsUsage }, - detection_rules: - detectionRuleMetrics.status === 'fulfilled' - ? detectionRuleMetrics.value - : { detection_rule_detail: [], detection_rule_usage: initialDetectionRulesUsage }, - }; -}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_initial_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_initial_usage.ts new file mode 100644 index 0000000000000..6e3ab3124baf1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_initial_usage.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MlJobUsage } from './types'; + +/** + * Default ml job usage count + */ +export const getInitialMlJobUsage = (): MlJobUsage => ({ + custom: { + enabled: 0, + disabled: 0, + }, + elastic: { + enabled: 0, + disabled: 0, + }, +}); diff --git a/x-pack/plugins/security_solution/server/usage/detections/detections.mocks.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.mocks.ts similarity index 87% rename from x-pack/plugins/security_solution/server/usage/detections/detections.mocks.ts rename to x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.mocks.ts index e7c1384152c5a..e7870f35a35bc 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/detections.mocks.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.mocks.ts @@ -5,81 +5,7 @@ * 2.0. */ -import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; - -export const getMockJobSummaryResponse = () => [ - { - id: 'linux_anomalous_network_activity_ecs', - description: - 'SIEM Auditbeat: Looks for unusual processes using the network which could indicate command-and-control, lateral movement, persistence, or data exfiltration activity (beta)', - groups: ['auditbeat', 'process', 'siem'], - processed_record_count: 141889, - memory_status: 'ok', - jobState: 'opened', - hasDatafeed: true, - datafeedId: 'datafeed-linux_anomalous_network_activity_ecs', - datafeedIndices: ['auditbeat-*'], - datafeedState: 'started', - latestTimestampMs: 1594085401911, - earliestTimestampMs: 1593054845656, - latestResultsTimestampMs: 1594085401911, - isSingleMetricViewerJob: true, - nodeName: 'node', - }, - { - id: 'linux_anomalous_network_port_activity_ecs', - description: - 'SIEM Auditbeat: Looks for unusual destination port activity that could indicate command-and-control, persistence mechanism, or data exfiltration activity (beta)', - groups: ['auditbeat', 'process', 'siem'], - processed_record_count: 0, - memory_status: 'ok', - jobState: 'closed', - hasDatafeed: true, - datafeedId: 'datafeed-linux_anomalous_network_port_activity_ecs', - datafeedIndices: ['auditbeat-*'], - datafeedState: 'stopped', - isSingleMetricViewerJob: true, - }, - { - id: 'other_job', - description: 'a job that is custom', - groups: ['auditbeat', 'process', 'security'], - processed_record_count: 0, - memory_status: 'ok', - jobState: 'closed', - hasDatafeed: true, - datafeedId: 'datafeed-other', - datafeedIndices: ['auditbeat-*'], - datafeedState: 'stopped', - isSingleMetricViewerJob: true, - }, - { - id: 'another_job', - description: 'another job that is custom', - groups: ['auditbeat', 'process', 'security'], - processed_record_count: 0, - memory_status: 'ok', - jobState: 'opened', - hasDatafeed: true, - datafeedId: 'datafeed-another', - datafeedIndices: ['auditbeat-*'], - datafeedState: 'started', - isSingleMetricViewerJob: true, - }, - { - id: 'irrelevant_job', - description: 'a non-security job', - groups: ['auditbeat', 'process'], - processed_record_count: 0, - memory_status: 'ok', - jobState: 'opened', - hasDatafeed: true, - datafeedId: 'datafeed-another', - datafeedIndices: ['auditbeat-*'], - datafeedState: 'started', - isSingleMetricViewerJob: true, - }, -]; +import type { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; export const getMockListModulesResponse = () => [ { @@ -162,6 +88,80 @@ export const getMockListModulesResponse = () => [ }, ]; +export const getMockMlJobSummaryResponse = () => [ + { + id: 'linux_anomalous_network_activity_ecs', + description: + 'SIEM Auditbeat: Looks for unusual processes using the network which could indicate command-and-control, lateral movement, persistence, or data exfiltration activity (beta)', + groups: ['auditbeat', 'process', 'siem'], + processed_record_count: 141889, + memory_status: 'ok', + jobState: 'opened', + hasDatafeed: true, + datafeedId: 'datafeed-linux_anomalous_network_activity_ecs', + datafeedIndices: ['auditbeat-*'], + datafeedState: 'started', + latestTimestampMs: 1594085401911, + earliestTimestampMs: 1593054845656, + latestResultsTimestampMs: 1594085401911, + isSingleMetricViewerJob: true, + nodeName: 'node', + }, + { + id: 'linux_anomalous_network_port_activity_ecs', + description: + 'SIEM Auditbeat: Looks for unusual destination port activity that could indicate command-and-control, persistence mechanism, or data exfiltration activity (beta)', + groups: ['auditbeat', 'process', 'siem'], + processed_record_count: 0, + memory_status: 'ok', + jobState: 'closed', + hasDatafeed: true, + datafeedId: 'datafeed-linux_anomalous_network_port_activity_ecs', + datafeedIndices: ['auditbeat-*'], + datafeedState: 'stopped', + isSingleMetricViewerJob: true, + }, + { + id: 'other_job', + description: 'a job that is custom', + groups: ['auditbeat', 'process', 'security'], + processed_record_count: 0, + memory_status: 'ok', + jobState: 'closed', + hasDatafeed: true, + datafeedId: 'datafeed-other', + datafeedIndices: ['auditbeat-*'], + datafeedState: 'stopped', + isSingleMetricViewerJob: true, + }, + { + id: 'another_job', + description: 'another job that is custom', + groups: ['auditbeat', 'process', 'security'], + processed_record_count: 0, + memory_status: 'ok', + jobState: 'opened', + hasDatafeed: true, + datafeedId: 'datafeed-another', + datafeedIndices: ['auditbeat-*'], + datafeedState: 'started', + isSingleMetricViewerJob: true, + }, + { + id: 'irrelevant_job', + description: 'a non-security job', + groups: ['auditbeat', 'process'], + processed_record_count: 0, + memory_status: 'ok', + jobState: 'opened', + hasDatafeed: true, + datafeedId: 'datafeed-another', + datafeedIndices: ['auditbeat-*'], + datafeedState: 'started', + isSingleMetricViewerJob: true, + }, +]; + export const getMockMlJobDetailsResponse = () => ({ count: 20, jobs: [ @@ -390,78 +390,3 @@ export const getMockRuleSearchResponse = ( ], }, }); - -export const getMockRuleAlertsResponse = (docCount: number): SearchResponse => ({ - took: 7, - timed_out: false, - _shards: { - total: 1, - successful: 1, - skipped: 0, - failed: 0, - }, - hits: { - total: { - value: 7322, - relation: 'eq', - }, - max_score: null, - hits: [], - }, - aggregations: { - detectionAlerts: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [ - { - key: '6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', - doc_count: docCount, - }, - ], - }, - }, -}); - -export const getMockAlertCasesResponse = () => ({ - page: 1, - per_page: 10000, - total: 4, - saved_objects: [ - { - type: 'cases-comments', - id: '3bb5cc10-9249-11eb-85b7-254c8af1a983', - attributes: { - type: 'alert', - alertId: '54802763917f521249c9f68d0d4be0c26cc538404c26dfed1ae7dcfa94ea2226', - index: '.siem-signals-default-000001', - rule: { - id: '6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', - name: 'Azure Diagnostic Settings Deletion', - }, - created_at: '2021-03-31T17:47:59.449Z', - created_by: { - email: '', - full_name: '', - username: '', - }, - pushed_at: null, - pushed_by: null, - updated_at: null, - updated_by: null, - }, - references: [ - { - type: 'cases', - name: 'associated-cases', - id: '3a3a4fa0-9249-11eb-85b7-254c8af1a983', - }, - ], - migrationVersion: {}, - coreMigrationVersion: '8.0.0', - updated_at: '2021-03-31T17:47:59.818Z', - version: 'WzI3MDIyODMsNF0=', - namespaces: ['default'], - score: 0, - }, - ], -}); diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts new file mode 100644 index 0000000000000..61b4e8bc35b21 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + KibanaRequest, + SavedObjectsClientContract, + Logger, +} from '../../../../../../../src/core/server'; +import type { MlDatafeedStats, MlJob, MlPluginSetup } from '../../../../../ml/server'; +import type { MlJobMetric, MlJobUsageMetric } from './types'; + +import { isJobStarted } from '../../../../common/machine_learning/helpers'; +import { isSecurityJob } from '../../../../common/machine_learning/is_security_job'; +import { getInitialMlJobUsage } from './get_initial_usage'; +import { updateMlJobUsage } from './update_usage'; +import { getJobCorrelations } from './transform_utils/get_job_correlations'; + +export interface GetMlJobMetricsOptions { + mlClient: MlPluginSetup | undefined; + savedObjectsClient: SavedObjectsClientContract; + logger: Logger; +} + +export const getMlJobMetrics = async ({ + mlClient, + savedObjectsClient, + logger, +}: GetMlJobMetricsOptions): Promise => { + let jobsUsage = getInitialMlJobUsage(); + + if (mlClient == null) { + logger.debug( + 'Machine learning client is null/undefined, therefore not collecting telemetry from it' + ); + // early return if we don't have ml client + return { + ml_job_usage: getInitialMlJobUsage(), + ml_job_metrics: [], + }; + } + + try { + const fakeRequest = { headers: {} } as KibanaRequest; + + const modules = await mlClient.modulesProvider(fakeRequest, savedObjectsClient).listModules(); + const moduleJobs = modules.flatMap((module) => module.jobs); + const jobs = await mlClient.jobServiceProvider(fakeRequest, savedObjectsClient).jobsSummary(); + + jobsUsage = jobs.filter(isSecurityJob).reduce((usage, job) => { + const isElastic = moduleJobs.some((moduleJob) => moduleJob.id === job.id); + const isEnabled = isJobStarted(job.jobState, job.datafeedState); + + return updateMlJobUsage({ isElastic, isEnabled }, usage); + }, getInitialMlJobUsage()); + + const jobsType = 'security'; + const securityJobStats = await mlClient + .anomalyDetectorsProvider(fakeRequest, savedObjectsClient) + .jobStats(jobsType); + + const jobDetails = await mlClient + .anomalyDetectorsProvider(fakeRequest, savedObjectsClient) + .jobs(jobsType); + + const jobDetailsCache = new Map(); + jobDetails.jobs.forEach((detail) => jobDetailsCache.set(detail.job_id, detail)); + + const datafeedStats = await mlClient + .anomalyDetectorsProvider(fakeRequest, savedObjectsClient) + .datafeedStats(); + + const datafeedStatsCache = new Map(); + datafeedStats.datafeeds.forEach((datafeedStat) => + datafeedStatsCache.set(`${datafeedStat.datafeed_id}`, datafeedStat) + ); + + const jobMetrics = securityJobStats.jobs.map((stat) => { + const jobId = stat.job_id; + const jobDetail = jobDetailsCache.get(stat.job_id); + const datafeed = datafeedStatsCache.get(`datafeed-${jobId}`); + return getJobCorrelations({ stat, jobDetail, datafeed }); + }); + + return { + ml_job_usage: jobsUsage, + ml_job_metrics: jobMetrics, + }; + } catch (e) { + // ignore failure, usage will be zeroed + logger.error( + `Encountered error in telemetry of message: ${e.message}, error: ${e}. Telemetry for "ml_jobs" will be skipped.` + ); + return { + ml_job_usage: getInitialMlJobUsage(), + ml_job_metrics: [], + }; + } +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/schema.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/schema.ts new file mode 100644 index 0000000000000..6df24516f3d53 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/schema.ts @@ -0,0 +1,202 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// eslint-disable-next-line @typescript-eslint/naming-convention +export const ml_jobs = { + ml_job_usage: { + custom: { + enabled: { + type: 'long', + _meta: { description: 'The number of custom ML jobs rules enabled' }, + }, + disabled: { + type: 'long', + _meta: { description: 'The number of custom ML jobs rules disabled' }, + }, + }, + elastic: { + enabled: { + type: 'long', + _meta: { description: 'The number of elastic provided ML jobs rules enabled' }, + }, + disabled: { + type: 'long', + _meta: { description: 'The number of elastic provided ML jobs rules disabled' }, + }, + }, + }, + ml_job_metrics: { + type: 'array', + items: { + job_id: { + type: 'keyword', + _meta: { description: 'Identifier for the anomaly detection job' }, + }, + open_time: { + type: 'keyword', + _meta: { + description: 'For open jobs only, the elapsed time for which the job has been open', + }, + }, + create_time: { + type: 'keyword', + _meta: { description: 'The time the job was created' }, + }, + finished_time: { + type: 'keyword', + _meta: { + description: 'If the job closed or failed, this is the time the job finished', + }, + }, + state: { + type: 'keyword', + _meta: { description: 'The status of the anomaly detection job' }, + }, + data_counts: { + bucket_count: { + type: 'long', + _meta: { description: 'The number of buckets processed' }, + }, + empty_bucket_count: { + type: 'long', + _meta: { description: 'The number of buckets which did not contain any data' }, + }, + input_bytes: { + type: 'long', + _meta: { + description: 'The number of bytes of input data posted to the anomaly detection job', + }, + }, + input_record_count: { + type: 'long', + _meta: { + description: 'The number of input documents posted to the anomaly detection job', + }, + }, + last_data_time: { + type: 'long', + _meta: { + description: 'The timestamp at which data was last analyzed, according to server time', + }, + }, + processed_record_count: { + type: 'long', + _meta: { + description: + 'The number of input documents that have been processed by the anomaly detection job', + }, + }, + }, + model_size_stats: { + bucket_allocation_failures_count: { + type: 'long', + _meta: { + description: + 'The number of buckets for which new entities in incoming data were not processed due to insufficient model memory', + }, + }, + model_bytes: { + type: 'long', + _meta: { description: 'The number of bytes of memory used by the models' }, + }, + model_bytes_exceeded: { + type: 'long', + _meta: { + description: + 'The number of bytes over the high limit for memory usage at the last allocation failure', + }, + }, + model_bytes_memory_limit: { + type: 'long', + _meta: { + description: 'The upper limit for model memory usage, checked on increasing values', + }, + }, + peak_model_bytes: { + type: 'long', + _meta: { + description: 'The peak number of bytes of memory ever used by the models', + }, + }, + }, + timing_stats: { + bucket_count: { + type: 'long', + _meta: { description: 'The number of buckets processed' }, + }, + exponential_average_bucket_processing_time_ms: { + type: 'long', + _meta: { + description: + 'Exponential moving average of all bucket processing times, in milliseconds', + }, + }, + exponential_average_bucket_processing_time_per_hour_ms: { + type: 'long', + _meta: { + description: + 'Exponentially-weighted moving average of bucket processing times calculated in a 1 hour time window, in milliseconds', + }, + }, + maximum_bucket_processing_time_ms: { + type: 'long', + _meta: { + description: 'Maximum among all bucket processing times, in milliseconds', + }, + }, + minimum_bucket_processing_time_ms: { + type: 'long', + _meta: { + description: 'Minimum among all bucket processing times, in milliseconds', + }, + }, + total_bucket_processing_time_ms: { + type: 'long', + _meta: { description: 'Sum of all bucket processing times, in milliseconds' }, + }, + }, + datafeed: { + datafeed_id: { + type: 'keyword', + _meta: { + description: 'A numerical character string that uniquely identifies the datafeed', + }, + }, + state: { + type: 'keyword', + _meta: { description: 'The status of the datafeed' }, + }, + timing_stats: { + average_search_time_per_bucket_ms: { + type: 'long', + _meta: { description: 'The average search time per bucket, in milliseconds' }, + }, + bucket_count: { + type: 'long', + _meta: { description: 'The number of buckets processed' }, + }, + exponential_average_search_time_per_hour_ms: { + type: 'long', + _meta: { + description: 'The exponential average search time per hour, in milliseconds', + }, + }, + search_count: { + type: 'long', + _meta: { description: 'The number of searches run by the datafeed' }, + }, + total_search_time_ms: { + type: 'long', + _meta: { + description: 'The total time the datafeed spent searching, in milliseconds', + }, + }, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/transform_utils/get_job_correlations.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/transform_utils/get_job_correlations.ts new file mode 100644 index 0000000000000..59a23c5dc7bd1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/transform_utils/get_job_correlations.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + MlDatafeedStats, + MlJob, + MlJobStats, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { MlJobMetric } from '../types'; + +export interface GetJobCorrelations { + stat: MlJobStats; + jobDetail: MlJob | undefined; + datafeed: MlDatafeedStats | undefined; +} + +export const getJobCorrelations = ({ + stat, + jobDetail, + datafeed, +}: GetJobCorrelations): MlJobMetric => { + return { + job_id: stat.job_id, + open_time: stat.open_time, + create_time: jobDetail?.create_time, + finished_time: jobDetail?.finished_time, + state: stat.state, + data_counts: { + bucket_count: stat.data_counts.bucket_count, + empty_bucket_count: stat.data_counts.empty_bucket_count, + input_bytes: stat.data_counts.input_bytes, + input_record_count: stat.data_counts.input_record_count, + last_data_time: stat.data_counts.last_data_time, + processed_record_count: stat.data_counts.processed_record_count, + }, + model_size_stats: { + bucket_allocation_failures_count: stat.model_size_stats.bucket_allocation_failures_count, + memory_status: stat.model_size_stats.memory_status, + model_bytes: stat.model_size_stats.model_bytes, + model_bytes_exceeded: stat.model_size_stats.model_bytes_exceeded, + model_bytes_memory_limit: stat.model_size_stats.model_bytes_memory_limit, + peak_model_bytes: stat.model_size_stats.peak_model_bytes, + }, + timing_stats: { + average_bucket_processing_time_ms: stat.timing_stats.average_bucket_processing_time_ms, + bucket_count: stat.timing_stats.bucket_count, + exponential_average_bucket_processing_time_ms: + stat.timing_stats.exponential_average_bucket_processing_time_ms, + exponential_average_bucket_processing_time_per_hour_ms: + stat.timing_stats.exponential_average_bucket_processing_time_per_hour_ms, + maximum_bucket_processing_time_ms: stat.timing_stats.maximum_bucket_processing_time_ms, + minimum_bucket_processing_time_ms: stat.timing_stats.minimum_bucket_processing_time_ms, + total_bucket_processing_time_ms: stat.timing_stats.total_bucket_processing_time_ms, + }, + datafeed: { + datafeed_id: datafeed?.datafeed_id, + state: datafeed?.state, + timing_stats: { + bucket_count: datafeed?.timing_stats.bucket_count, + exponential_average_search_time_per_hour_ms: + datafeed?.timing_stats.exponential_average_search_time_per_hour_ms, + search_count: datafeed?.timing_stats.search_count, + total_search_time_ms: datafeed?.timing_stats.total_search_time_ms, + }, + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/types.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/types.ts new file mode 100644 index 0000000000000..c50fc3166977a --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/types.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + MlDataCounts, + MlDatafeedState, + MlDatafeedStats, + MlDatafeedTimingStats, + MlJob, + MlJobState, + MlJobStats, + MlJobTimingStats, + MlModelSizeStats, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +interface FeatureUsage { + enabled: number; + disabled: number; +} + +export interface MlJobUsage { + custom: FeatureUsage; + elastic: FeatureUsage; +} + +export interface MlJobMetric { + job_id: MlJobStats['job_id']; + create_time?: MlJob['create_time']; + finished_time?: MlJob['finished_time']; + open_time?: MlJobStats['open_time']; + state: MlJobState; + data_counts: Partial; + model_size_stats: Partial; + timing_stats: Partial; + datafeed: MlDataFeed; +} + +export interface MlJobUsageMetric { + ml_job_usage: MlJobUsage; + ml_job_metrics: MlJobMetric[]; +} + +export interface DetectionsMetric { + isElastic: boolean; + isEnabled: boolean; +} + +export interface MlDataFeed { + datafeed_id?: MlDatafeedStats['datafeed_id']; + state?: MlDatafeedState; + timing_stats: Partial; +} diff --git a/x-pack/plugins/security_solution/server/usage/detections/detection_ml_helpers.test.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/update_usage.test.ts similarity index 70% rename from x-pack/plugins/security_solution/server/usage/detections/detection_ml_helpers.test.ts rename to x-pack/plugins/security_solution/server/usage/detections/ml_jobs/update_usage.test.ts index 3ca0faeca7d36..9d0dc7c02e568 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/detection_ml_helpers.test.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/update_usage.test.ts @@ -5,16 +5,17 @@ * 2.0. */ -import { initialMlJobsUsage, updateMlJobsUsage } from './detection_ml_helpers'; +import { getInitialMlJobUsage } from './get_initial_usage'; +import { updateMlJobUsage } from './update_usage'; describe('Security Machine Learning usage metrics', () => { describe('Updates metrics with job information', () => { it('Should update ML total for elastic rules', async () => { - const initialUsage = initialMlJobsUsage; + const initialUsage = getInitialMlJobUsage(); const isElastic = true; const isEnabled = true; - const updatedUsage = updateMlJobsUsage({ isElastic, isEnabled }, initialUsage); + const updatedUsage = updateMlJobUsage({ isElastic, isEnabled }, initialUsage); expect(updatedUsage).toEqual( expect.objectContaining({ @@ -31,11 +32,11 @@ describe('Security Machine Learning usage metrics', () => { }); it('Should update ML total for custom rules', async () => { - const initialUsage = initialMlJobsUsage; + const initialUsage = getInitialMlJobUsage(); const isElastic = false; const isEnabled = true; - const updatedUsage = updateMlJobsUsage({ isElastic, isEnabled }, initialUsage); + const updatedUsage = updateMlJobUsage({ isElastic, isEnabled }, initialUsage); expect(updatedUsage).toEqual( expect.objectContaining({ @@ -52,10 +53,9 @@ describe('Security Machine Learning usage metrics', () => { }); it('Should update ML total for both elastic and custom rules', async () => { - const initialUsage = initialMlJobsUsage; - - let updatedUsage = updateMlJobsUsage({ isElastic: true, isEnabled: true }, initialUsage); - updatedUsage = updateMlJobsUsage({ isElastic: false, isEnabled: true }, updatedUsage); + const initialUsage = getInitialMlJobUsage(); + let updatedUsage = updateMlJobUsage({ isElastic: true, isEnabled: true }, initialUsage); + updatedUsage = updateMlJobUsage({ isElastic: false, isEnabled: true }, updatedUsage); expect(updatedUsage).toEqual( expect.objectContaining({ diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/update_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/update_usage.ts new file mode 100644 index 0000000000000..2306bfa051a3b --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/update_usage.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DetectionsMetric, MlJobUsage } from './types'; + +export const updateMlJobUsage = (jobMetric: DetectionsMetric, usage: MlJobUsage): MlJobUsage => { + const { isEnabled, isElastic } = jobMetric; + if (isEnabled && isElastic) { + return { + ...usage, + elastic: { + ...usage.elastic, + enabled: usage.elastic.enabled + 1, + }, + }; + } else if (!isEnabled && isElastic) { + return { + ...usage, + elastic: { + ...usage.elastic, + disabled: usage.elastic.disabled + 1, + }, + }; + } else if (isEnabled && !isElastic) { + return { + ...usage, + custom: { + ...usage.custom, + enabled: usage.custom.enabled + 1, + }, + }; + } else if (!isEnabled && !isElastic) { + return { + ...usage, + custom: { + ...usage.custom, + disabled: usage.custom.disabled + 1, + }, + }; + } else { + return usage; + } +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/get_initial_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/get_initial_usage.ts new file mode 100644 index 0000000000000..81ea7aec800e3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/get_initial_usage.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesTypeUsage } from './types'; + +/** + * Default detection rule usage count, split by type + elastic/custom + */ +export const getInitialRulesUsage = (): RulesTypeUsage => ({ + query: { + enabled: 0, + disabled: 0, + alerts: 0, + cases: 0, + legacy_notifications_enabled: 0, + legacy_notifications_disabled: 0, + notifications_enabled: 0, + notifications_disabled: 0, + }, + threshold: { + enabled: 0, + disabled: 0, + alerts: 0, + cases: 0, + legacy_notifications_enabled: 0, + legacy_notifications_disabled: 0, + notifications_enabled: 0, + notifications_disabled: 0, + }, + eql: { + enabled: 0, + disabled: 0, + alerts: 0, + cases: 0, + legacy_notifications_enabled: 0, + legacy_notifications_disabled: 0, + notifications_enabled: 0, + notifications_disabled: 0, + }, + machine_learning: { + enabled: 0, + disabled: 0, + alerts: 0, + cases: 0, + legacy_notifications_enabled: 0, + legacy_notifications_disabled: 0, + notifications_enabled: 0, + notifications_disabled: 0, + }, + threat_match: { + enabled: 0, + disabled: 0, + alerts: 0, + cases: 0, + legacy_notifications_enabled: 0, + legacy_notifications_disabled: 0, + notifications_enabled: 0, + notifications_disabled: 0, + }, + elastic_total: { + enabled: 0, + disabled: 0, + alerts: 0, + cases: 0, + legacy_notifications_enabled: 0, + legacy_notifications_disabled: 0, + notifications_enabled: 0, + notifications_disabled: 0, + }, + custom_total: { + enabled: 0, + disabled: 0, + alerts: 0, + cases: 0, + legacy_notifications_enabled: 0, + legacy_notifications_disabled: 0, + notifications_enabled: 0, + notifications_disabled: 0, + }, +}); diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.mocks.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.mocks.ts new file mode 100644 index 0000000000000..cfdc6b0b992e7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.mocks.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; + +export const getMockRuleAlertsResponse = (docCount: number): SearchResponse => ({ + took: 7, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: { + value: 7322, + relation: 'eq', + }, + max_score: null, + hits: [], + }, + aggregations: { + buckets: { + after_key: { + detectionAlerts: '6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', + }, + buckets: [ + { + key: { + detectionAlerts: '6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', + }, + doc_count: docCount, + }, + ], + }, + }, +}); + +export const getMockAlertCasesResponse = () => ({ + page: 1, + per_page: 10000, + total: 4, + saved_objects: [ + { + type: 'cases-comments', + id: '3bb5cc10-9249-11eb-85b7-254c8af1a983', + attributes: { + type: 'alert', + alertId: '54802763917f521249c9f68d0d4be0c26cc538404c26dfed1ae7dcfa94ea2226', + index: '.siem-signals-default-000001', + rule: { + id: '6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', + name: 'Azure Diagnostic Settings Deletion', + }, + created_at: '2021-03-31T17:47:59.449Z', + created_by: { + email: '', + full_name: '', + username: '', + }, + pushed_at: null, + pushed_by: null, + updated_at: null, + updated_by: null, + }, + references: [ + { + type: 'cases', + name: 'associated-cases', + id: '3a3a4fa0-9249-11eb-85b7-254c8af1a983', + }, + ], + migrationVersion: {}, + coreMigrationVersion: '8.0.0', + updated_at: '2021-03-31T17:47:59.818Z', + version: 'WzI3MDIyODMsNF0=', + namespaces: ['default'], + score: 0, + }, + ], +}); diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts new file mode 100644 index 0000000000000..8423a7bdeb7bd --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, SavedObjectsClientContract, Logger } from 'kibana/server'; +import type { RuleAdoption } from './types'; + +import { updateRuleUsage } from './update_usage'; +import { getDetectionRules } from '../../queries/get_detection_rules'; +import { getAlerts } from '../../queries/get_alerts'; +import { MAX_PER_PAGE, MAX_RESULTS_WINDOW } from '../../constants'; +import { getInitialRulesUsage } from './get_initial_usage'; +import { getCaseComments } from '../../queries/get_case_comments'; +import { getRuleIdToCasesMap } from './transform_utils/get_rule_id_to_cases_map'; +import { getAlertIdToCountMap } from './transform_utils/get_alert_id_to_count_map'; +import { getRuleIdToEnabledMap } from './transform_utils/get_rule_id_to_enabled_map'; +import { getRuleObjectCorrelations } from './transform_utils/get_rule_object_correlations'; + +// eslint-disable-next-line no-restricted-imports +import { legacyGetRuleActions } from '../../queries/legacy_get_rule_actions'; + +export interface GetRuleMetricsOptions { + kibanaIndex: string; + signalsIndex: string; + esClient: ElasticsearchClient; + savedObjectsClient: SavedObjectsClientContract; + logger: Logger; +} + +export const getRuleMetrics = async ({ + kibanaIndex, + signalsIndex, + esClient, + savedObjectsClient, + logger, +}: GetRuleMetricsOptions): Promise => { + try { + // gets rule saved objects + const ruleResults = await getDetectionRules({ + esClient, + kibanaIndex, + maxPerPage: MAX_PER_PAGE, + maxSize: MAX_RESULTS_WINDOW, + logger, + }); + + // early return if we don't have any detection rules then there is no need to query anything else + if (ruleResults.length === 0) { + return { + detection_rule_detail: [], + detection_rule_usage: getInitialRulesUsage(), + }; + } + + // gets the alerts data objects + const detectionAlertsResp = await getAlerts({ + esClient, + signalsIndex: `${signalsIndex}*`, + maxPerPage: MAX_PER_PAGE, + maxSize: MAX_RESULTS_WINDOW, + logger, + }); + + // gets cases saved objects + const caseComments = await getCaseComments({ + savedObjectsClient, + maxSize: MAX_PER_PAGE, + maxPerPage: MAX_RESULTS_WINDOW, + logger, + }); + + // gets the legacy rule actions to track legacy notifications. + const legacyRuleActions = await legacyGetRuleActions({ + savedObjectsClient, + maxSize: MAX_PER_PAGE, + maxPerPage: MAX_RESULTS_WINDOW, + logger, + }); + + // create in-memory maps for correlation + const legacyNotificationRuleIds = getRuleIdToEnabledMap(legacyRuleActions); + const casesRuleIds = getRuleIdToCasesMap(caseComments); + const alertsCounts = getAlertIdToCountMap(detectionAlertsResp); + + // correlate the rule objects to the results + const rulesCorrelated = getRuleObjectCorrelations({ + ruleResults, + legacyNotificationRuleIds, + casesRuleIds, + alertsCounts, + }); + + // Only bring back rule detail on elastic prepackaged detection rules + const elasticRuleObjects = rulesCorrelated.filter((hit) => hit.elastic_rule === true); + + // calculate the rule usage + const rulesUsage = rulesCorrelated.reduce( + (usage, rule) => updateRuleUsage(rule, usage), + getInitialRulesUsage() + ); + + return { + detection_rule_detail: elasticRuleObjects, + detection_rule_usage: rulesUsage, + }; + } catch (e) { + // ignore failure, usage will be zeroed + logger.error( + `Encountered error in telemetry of message: ${e.message}, error: ${e}. Telemetry for "detection rules" being skipped.` + ); + return { + detection_rule_detail: [], + detection_rule_usage: getInitialRulesUsage(), + }; + } +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/schema.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/schema.ts new file mode 100644 index 0000000000000..7088c6665e314 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/schema.ts @@ -0,0 +1,285 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// eslint-disable-next-line @typescript-eslint/naming-convention +export const detection_rules = { + detection_rule_usage: { + query: { + enabled: { type: 'long', _meta: { description: 'Number of query rules enabled' } }, + disabled: { type: 'long', _meta: { description: 'Number of query rules disabled' } }, + alerts: { + type: 'long', + _meta: { description: 'Number of alerts generated by query rules' }, + }, + cases: { + type: 'long', + _meta: { description: 'Number of cases attached to query detection rule alerts' }, + }, + legacy_notifications_enabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications enabled' }, + }, + legacy_notifications_disabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications disabled' }, + }, + notifications_enabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + notifications_disabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + }, + threshold: { + enabled: { + type: 'long', + _meta: { description: 'Number of threshold rules enabled' }, + }, + disabled: { + type: 'long', + _meta: { description: 'Number of threshold rules disabled' }, + }, + alerts: { + type: 'long', + _meta: { description: 'Number of alerts generated by threshold rules' }, + }, + cases: { + type: 'long', + _meta: { + description: 'Number of cases attached to threshold detection rule alerts', + }, + }, + legacy_notifications_enabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications enabled' }, + }, + legacy_notifications_disabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications disabled' }, + }, + notifications_enabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + notifications_disabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + }, + eql: { + enabled: { type: 'long', _meta: { description: 'Number of eql rules enabled' } }, + disabled: { type: 'long', _meta: { description: 'Number of eql rules disabled' } }, + alerts: { + type: 'long', + _meta: { description: 'Number of alerts generated by eql rules' }, + }, + cases: { + type: 'long', + _meta: { description: 'Number of cases attached to eql detection rule alerts' }, + }, + legacy_notifications_enabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications enabled' }, + }, + legacy_notifications_disabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications disabled' }, + }, + notifications_enabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + notifications_disabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + }, + machine_learning: { + enabled: { + type: 'long', + _meta: { description: 'Number of machine_learning rules enabled' }, + }, + disabled: { + type: 'long', + _meta: { description: 'Number of machine_learning rules disabled' }, + }, + alerts: { + type: 'long', + _meta: { description: 'Number of alerts generated by machine_learning rules' }, + }, + cases: { + type: 'long', + _meta: { + description: 'Number of cases attached to machine_learning detection rule alerts', + }, + }, + legacy_notifications_enabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications enabled' }, + }, + legacy_notifications_disabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications disabled' }, + }, + notifications_enabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + notifications_disabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + }, + threat_match: { + enabled: { + type: 'long', + _meta: { description: 'Number of threat_match rules enabled' }, + }, + disabled: { + type: 'long', + _meta: { description: 'Number of threat_match rules disabled' }, + }, + alerts: { + type: 'long', + _meta: { description: 'Number of alerts generated by threat_match rules' }, + }, + cases: { + type: 'long', + _meta: { + description: 'Number of cases attached to threat_match detection rule alerts', + }, + }, + legacy_notifications_enabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications enabled' }, + }, + legacy_notifications_disabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications disabled' }, + }, + notifications_enabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + notifications_disabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + }, + elastic_total: { + enabled: { type: 'long', _meta: { description: 'Number of elastic rules enabled' } }, + disabled: { + type: 'long', + _meta: { description: 'Number of elastic rules disabled' }, + }, + alerts: { + type: 'long', + _meta: { description: 'Number of alerts generated by elastic rules' }, + }, + cases: { + type: 'long', + _meta: { description: 'Number of cases attached to elastic detection rule alerts' }, + }, + legacy_notifications_enabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications enabled' }, + }, + legacy_notifications_disabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications disabled' }, + }, + notifications_enabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + notifications_disabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + }, + custom_total: { + enabled: { type: 'long', _meta: { description: 'Number of custom rules enabled' } }, + disabled: { type: 'long', _meta: { description: 'Number of custom rules disabled' } }, + alerts: { + type: 'long', + _meta: { description: 'Number of alerts generated by custom rules' }, + }, + cases: { + type: 'long', + _meta: { description: 'Number of cases attached to custom detection rule alerts' }, + }, + legacy_notifications_enabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications enabled' }, + }, + legacy_notifications_disabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications disabled' }, + }, + notifications_enabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + notifications_disabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + }, + }, + detection_rule_detail: { + type: 'array', + items: { + rule_name: { + type: 'keyword', + _meta: { description: 'The name of the detection rule' }, + }, + rule_id: { + type: 'keyword', + _meta: { description: 'The UUID id of the detection rule' }, + }, + rule_type: { + type: 'keyword', + _meta: { description: 'The type of detection rule. ie eql, query...' }, + }, + rule_version: { type: 'long', _meta: { description: 'The version of the rule' } }, + enabled: { + type: 'boolean', + _meta: { description: 'If the detection rule has been enabled by the user' }, + }, + elastic_rule: { + type: 'boolean', + _meta: { description: 'If the detection rule has been authored by Elastic' }, + }, + created_on: { + type: 'keyword', + _meta: { description: 'When the detection rule was created on the cluster' }, + }, + updated_on: { + type: 'keyword', + _meta: { description: 'When the detection rule was updated on the cluster' }, + }, + alert_count_daily: { + type: 'long', + _meta: { description: 'The number of daily alerts generated by a rule' }, + }, + cases_count_total: { + type: 'long', + _meta: { description: 'The number of total cases generated by a rule' }, + }, + has_legacy_notification: { + type: 'boolean', + _meta: { description: 'True if this rule has a legacy notification' }, + }, + has_notification: { + type: 'boolean', + _meta: { description: 'True if this rule has a notification' }, + }, + }, + }, +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_alert_id_to_count_map.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_alert_id_to_count_map.ts new file mode 100644 index 0000000000000..ce569564273e4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_alert_id_to_count_map.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AlertBucket } from '../../../types'; + +export const getAlertIdToCountMap = (alerts: AlertBucket[]): Map => { + const alertsCache = new Map(); + alerts.map((bucket) => alertsCache.set(bucket.key.detectionAlerts, bucket.doc_count)); + return alertsCache; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_id_to_cases_map.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_id_to_cases_map.ts new file mode 100644 index 0000000000000..d7ce790be0750 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_id_to_cases_map.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsFindResult } from 'kibana/server'; +import type { CommentAttributes } from '../../../../../../cases/common/api/cases/comment'; + +export const getRuleIdToCasesMap = ( + cases: Array> +): Map => { + return cases.reduce((cache, { attributes: casesObject }) => { + if (casesObject.type === 'alert') { + const ruleId = casesObject.rule.id; + if (ruleId != null) { + const cacheCount = cache.get(ruleId); + if (cacheCount === undefined) { + cache.set(ruleId, 1); + } else { + cache.set(ruleId, cacheCount + 1); + } + } + return cache; + } else { + return cache; + } + }, new Map()); +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_id_to_enabled_map.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_id_to_enabled_map.ts new file mode 100644 index 0000000000000..b280d3a4ba17d --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_id_to_enabled_map.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsFindResult } from 'kibana/server'; +// eslint-disable-next-line no-restricted-imports +import type { LegacyIRuleActionsAttributesSavedObjectAttributes } from '../../../../lib/detection_engine/rule_actions/legacy_types'; + +export const getRuleIdToEnabledMap = ( + legacyRuleActions: Array< + SavedObjectsFindResult + > +): Map< + string, + { + enabled: boolean; + } +> => { + return legacyRuleActions.reduce((cache, legacyNotificationsObject) => { + const ruleRef = legacyNotificationsObject.references.find( + (reference) => reference.name === 'alert_0' && reference.type === 'alert' + ); + if (ruleRef != null) { + const enabled = legacyNotificationsObject.attributes.ruleThrottle !== 'no_actions'; + cache.set(ruleRef.id, { enabled }); + } + return cache; + }, new Map()); +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_object_correlations.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_object_correlations.ts new file mode 100644 index 0000000000000..9fb1bca0ed568 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_object_correlations.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SearchHit } from '@elastic/elasticsearch/lib/api/types'; +import type { RuleSearchResult } from '../../../types'; +import type { RuleMetric } from '../types'; +import { isElasticRule } from '../../../queries/utils/is_elastic_rule'; + +export interface RuleObjectCorrelationsOptions { + ruleResults: Array>; + legacyNotificationRuleIds: Map< + string, + { + enabled: boolean; + } + >; + casesRuleIds: Map; + alertsCounts: Map; +} + +export const getRuleObjectCorrelations = ({ + ruleResults, + legacyNotificationRuleIds, + casesRuleIds, + alertsCounts, +}: RuleObjectCorrelationsOptions): RuleMetric[] => { + return ruleResults.map((hit) => { + const ruleId = hit._id.split(':')[1]; + const isElastic = isElasticRule(hit._source?.alert.tags); + + // Even if the legacy notification is set to "no_actions" we still count the rule as having a legacy notification that is not migrated yet. + const hasLegacyNotification = legacyNotificationRuleIds.get(ruleId) != null; + + // We only count a rule as having a notification and being "enabled" if it is _not_ set to "no_actions"/"muteAll" and it has at least one action within its array. + const hasNotification = + !hasLegacyNotification && + hit._source?.alert.actions != null && + hit._source?.alert.actions.length > 0 && + hit._source?.alert.muteAll !== true; + + return { + rule_name: String(hit._source?.alert.name), + rule_id: String(hit._source?.alert.params.ruleId), + rule_type: String(hit._source?.alert.params.type), + rule_version: Number(hit._source?.alert.params.version), + enabled: Boolean(hit._source?.alert.enabled), + elastic_rule: isElastic, + created_on: String(hit._source?.alert.createdAt), + updated_on: String(hit._source?.alert.updatedAt), + alert_count_daily: alertsCounts.get(ruleId) || 0, + cases_count_total: casesRuleIds.get(ruleId) || 0, + has_legacy_notification: hasLegacyNotification, + has_notification: hasNotification, + }; + }); +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/types.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/types.ts new file mode 100644 index 0000000000000..54b3e6d6a0084 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/types.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface FeatureTypeUsage { + enabled: number; + disabled: number; + alerts: number; + cases: number; + legacy_notifications_enabled: number; + legacy_notifications_disabled: number; + notifications_enabled: number; + notifications_disabled: number; +} + +export interface RulesTypeUsage { + query: FeatureTypeUsage; + threshold: FeatureTypeUsage; + eql: FeatureTypeUsage; + machine_learning: FeatureTypeUsage; + threat_match: FeatureTypeUsage; + elastic_total: FeatureTypeUsage; + custom_total: FeatureTypeUsage; +} + +export interface RuleAdoption { + detection_rule_detail: RuleMetric[]; + detection_rule_usage: RulesTypeUsage; +} + +export interface RuleMetric { + rule_name: string; + rule_id: string; + rule_type: string; + rule_version: number; + enabled: boolean; + elastic_rule: boolean; + created_on: string; + updated_on: string; + alert_count_daily: number; + cases_count_total: number; + has_legacy_notification: boolean; + has_notification: boolean; +} diff --git a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.test.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/update_usage.test.ts similarity index 92% rename from x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.test.ts rename to x-pack/plugins/security_solution/server/usage/detections/rules/update_usage.test.ts index c19e7b18f9e72..d878d0a5145ab 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.test.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/update_usage.test.ts @@ -5,8 +5,9 @@ * 2.0. */ -import { initialDetectionRulesUsage, updateDetectionRuleUsage } from './detection_rule_helpers'; -import { DetectionRuleMetric, DetectionRulesTypeUsage } from './types'; +import type { RuleMetric, RulesTypeUsage } from './types'; +import { updateRuleUsage } from './update_usage'; +import { getInitialRulesUsage } from './get_initial_usage'; interface StubRuleOptions { ruleType: string; @@ -26,7 +27,7 @@ const createStubRule = ({ caseCount, hasLegacyNotification, hasNotification, -}: StubRuleOptions): DetectionRuleMetric => ({ +}: StubRuleOptions): RuleMetric => ({ rule_name: 'rule-name', rule_id: 'id-123', rule_type: ruleType, @@ -53,10 +54,10 @@ describe('Detections Usage and Metrics', () => { hasLegacyNotification: false, hasNotification: false, }); - const usage = updateDetectionRuleUsage(stubRule, initialDetectionRulesUsage); + const usage = updateRuleUsage(stubRule, getInitialRulesUsage()); - expect(usage).toEqual({ - ...initialDetectionRulesUsage, + expect(usage).toEqual({ + ...getInitialRulesUsage(), elastic_total: { alerts: 1, cases: 1, @@ -127,14 +128,14 @@ describe('Detections Usage and Metrics', () => { hasNotification: false, }); - let usage = updateDetectionRuleUsage(stubEqlRule, initialDetectionRulesUsage); - usage = updateDetectionRuleUsage(stubQueryRuleOne, usage); - usage = updateDetectionRuleUsage(stubQueryRuleTwo, usage); - usage = updateDetectionRuleUsage(stubMachineLearningOne, usage); - usage = updateDetectionRuleUsage(stubMachineLearningTwo, usage); + let usage = updateRuleUsage(stubEqlRule, getInitialRulesUsage()); + usage = updateRuleUsage(stubQueryRuleOne, usage); + usage = updateRuleUsage(stubQueryRuleTwo, usage); + usage = updateRuleUsage(stubMachineLearningOne, usage); + usage = updateRuleUsage(stubMachineLearningTwo, usage); - expect(usage).toEqual({ - ...initialDetectionRulesUsage, + expect(usage).toEqual({ + ...getInitialRulesUsage(), custom_total: { alerts: 5, cases: 12, @@ -242,8 +243,8 @@ describe('Detections Usage and Metrics', () => { alertCount: 0, caseCount: 0, }); - const usage = updateDetectionRuleUsage(rule1, initialDetectionRulesUsage) as ReturnType< - typeof updateDetectionRuleUsage + const usage = updateRuleUsage(rule1, getInitialRulesUsage()) as ReturnType< + typeof updateRuleUsage > & { [key: string]: unknown }; expect(usage[ruleType]).toEqual( expect.objectContaining({ @@ -264,8 +265,8 @@ describe('Detections Usage and Metrics', () => { alertCount: 0, caseCount: 0, }); - const usageAddedByOne = updateDetectionRuleUsage(rule2, usage) as ReturnType< - typeof updateDetectionRuleUsage + const usageAddedByOne = updateRuleUsage(rule2, usage) as ReturnType< + typeof updateRuleUsage > & { [key: string]: unknown }; expect(usageAddedByOne[ruleType]).toEqual( diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/update_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/update_usage.ts new file mode 100644 index 0000000000000..3aa3c3bbc29b0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/update_usage.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesTypeUsage, RuleMetric } from './types'; +import { updateQueryUsage } from './usage_utils/update_query_usage'; +import { updateTotalUsage } from './usage_utils/update_total_usage'; + +export const updateRuleUsage = ( + detectionRuleMetric: RuleMetric, + usage: RulesTypeUsage +): RulesTypeUsage => { + let updatedUsage = usage; + if (detectionRuleMetric.rule_type === 'query') { + updatedUsage = { + ...usage, + query: updateQueryUsage({ + ruleType: detectionRuleMetric.rule_type, + usage, + detectionRuleMetric, + }), + }; + } else if (detectionRuleMetric.rule_type === 'threshold') { + updatedUsage = { + ...usage, + threshold: updateQueryUsage({ + ruleType: detectionRuleMetric.rule_type, + usage, + detectionRuleMetric, + }), + }; + } else if (detectionRuleMetric.rule_type === 'eql') { + updatedUsage = { + ...usage, + eql: updateQueryUsage({ + ruleType: detectionRuleMetric.rule_type, + usage, + detectionRuleMetric, + }), + }; + } else if (detectionRuleMetric.rule_type === 'machine_learning') { + updatedUsage = { + ...usage, + machine_learning: updateQueryUsage({ + ruleType: detectionRuleMetric.rule_type, + usage, + detectionRuleMetric, + }), + }; + } else if (detectionRuleMetric.rule_type === 'threat_match') { + updatedUsage = { + ...usage, + threat_match: updateQueryUsage({ + ruleType: detectionRuleMetric.rule_type, + usage, + detectionRuleMetric, + }), + }; + } + + if (detectionRuleMetric.elastic_rule) { + updatedUsage = { + ...updatedUsage, + elastic_total: updateTotalUsage({ + detectionRuleMetric, + updatedUsage, + totalType: 'elastic_total', + }), + }; + } else { + updatedUsage = { + ...updatedUsage, + custom_total: updateTotalUsage({ + detectionRuleMetric, + updatedUsage, + totalType: 'custom_total', + }), + }; + } + + return updatedUsage; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/get_notifications_enabled_disabled.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/get_notifications_enabled_disabled.ts new file mode 100644 index 0000000000000..aae3f3fe00d0f --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/get_notifications_enabled_disabled.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleMetric } from '../types'; + +export const getNotificationsEnabledDisabled = ( + detectionRuleMetric: RuleMetric +): { + legacyNotificationEnabled: boolean; + legacyNotificationDisabled: boolean; + notificationEnabled: boolean; + notificationDisabled: boolean; +} => { + return { + legacyNotificationEnabled: + detectionRuleMetric.has_legacy_notification && detectionRuleMetric.enabled, + legacyNotificationDisabled: + detectionRuleMetric.has_legacy_notification && !detectionRuleMetric.enabled, + notificationEnabled: detectionRuleMetric.has_notification && detectionRuleMetric.enabled, + notificationDisabled: detectionRuleMetric.has_notification && !detectionRuleMetric.enabled, + }; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/update_query_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/update_query_usage.ts new file mode 100644 index 0000000000000..7f40ceec21c8a --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/update_query_usage.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesTypeUsage, RuleMetric, FeatureTypeUsage } from '../types'; +import { getNotificationsEnabledDisabled } from './get_notifications_enabled_disabled'; + +export interface UpdateQueryUsageOptions { + ruleType: keyof RulesTypeUsage; + usage: RulesTypeUsage; + detectionRuleMetric: RuleMetric; +} + +export const updateQueryUsage = ({ + ruleType, + usage, + detectionRuleMetric, +}: UpdateQueryUsageOptions): FeatureTypeUsage => { + const { + legacyNotificationEnabled, + legacyNotificationDisabled, + notificationEnabled, + notificationDisabled, + } = getNotificationsEnabledDisabled(detectionRuleMetric); + return { + enabled: detectionRuleMetric.enabled ? usage[ruleType].enabled + 1 : usage[ruleType].enabled, + disabled: !detectionRuleMetric.enabled + ? usage[ruleType].disabled + 1 + : usage[ruleType].disabled, + alerts: usage[ruleType].alerts + detectionRuleMetric.alert_count_daily, + cases: usage[ruleType].cases + detectionRuleMetric.cases_count_total, + legacy_notifications_enabled: legacyNotificationEnabled + ? usage[ruleType].legacy_notifications_enabled + 1 + : usage[ruleType].legacy_notifications_enabled, + legacy_notifications_disabled: legacyNotificationDisabled + ? usage[ruleType].legacy_notifications_disabled + 1 + : usage[ruleType].legacy_notifications_disabled, + notifications_enabled: notificationEnabled + ? usage[ruleType].notifications_enabled + 1 + : usage[ruleType].notifications_enabled, + notifications_disabled: notificationDisabled + ? usage[ruleType].notifications_disabled + 1 + : usage[ruleType].notifications_disabled, + }; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/update_total_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/update_total_usage.ts new file mode 100644 index 0000000000000..ed0ff37e2a328 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/update_total_usage.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesTypeUsage, RuleMetric, FeatureTypeUsage } from '../types'; +import { getNotificationsEnabledDisabled } from './get_notifications_enabled_disabled'; + +export interface UpdateTotalUsageOptions { + detectionRuleMetric: RuleMetric; + updatedUsage: RulesTypeUsage; + totalType: 'custom_total' | 'elastic_total'; +} + +export const updateTotalUsage = ({ + detectionRuleMetric, + updatedUsage, + totalType, +}: UpdateTotalUsageOptions): FeatureTypeUsage => { + const { + legacyNotificationEnabled, + legacyNotificationDisabled, + notificationEnabled, + notificationDisabled, + } = getNotificationsEnabledDisabled(detectionRuleMetric); + + return { + enabled: detectionRuleMetric.enabled + ? updatedUsage[totalType].enabled + 1 + : updatedUsage[totalType].enabled, + disabled: !detectionRuleMetric.enabled + ? updatedUsage[totalType].disabled + 1 + : updatedUsage[totalType].disabled, + alerts: updatedUsage[totalType].alerts + detectionRuleMetric.alert_count_daily, + cases: updatedUsage[totalType].cases + detectionRuleMetric.cases_count_total, + legacy_notifications_enabled: legacyNotificationEnabled + ? updatedUsage[totalType].legacy_notifications_enabled + 1 + : updatedUsage[totalType].legacy_notifications_enabled, + legacy_notifications_disabled: legacyNotificationDisabled + ? updatedUsage[totalType].legacy_notifications_disabled + 1 + : updatedUsage[totalType].legacy_notifications_disabled, + notifications_enabled: notificationEnabled + ? updatedUsage[totalType].notifications_enabled + 1 + : updatedUsage[totalType].notifications_enabled, + notifications_disabled: notificationDisabled + ? updatedUsage[totalType].notifications_disabled + 1 + : updatedUsage[totalType].notifications_disabled, + }; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/types.ts b/x-pack/plugins/security_solution/server/usage/detections/types.ts index 8c2459b2eb57d..2895e5c6f8b9a 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/types.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/types.ts @@ -5,165 +5,10 @@ * 2.0. */ -export interface RuleSearchBody { - query: { - bool: { - filter: { - terms: { [key: string]: string[] }; - }; - }; - }; -} - -export interface RuleSearchParams { - body: RuleSearchBody; - filter_path: string[]; - ignore_unavailable: boolean; - index: string; - size: number; -} - -export interface RuleSearchResult { - alert: { - name: string; - enabled: boolean; - tags: string[]; - createdAt: string; - updatedAt: string; - muteAll: boolean | undefined | null; - params: DetectionRuleParms; - actions: unknown[]; - }; -} - -export interface DetectionsMetric { - isElastic: boolean; - isEnabled: boolean; -} - -interface DetectionRuleParms { - ruleId: string; - version: string; - type: string; -} - -interface FeatureUsage { - enabled: number; - disabled: number; -} - -interface FeatureTypeUsage { - enabled: number; - disabled: number; - alerts: number; - cases: number; - legacy_notifications_enabled: number; - legacy_notifications_disabled: number; - notifications_enabled: number; - notifications_disabled: number; -} -export interface DetectionRulesTypeUsage { - query: FeatureTypeUsage; - threshold: FeatureTypeUsage; - eql: FeatureTypeUsage; - machine_learning: FeatureTypeUsage; - threat_match: FeatureTypeUsage; - elastic_total: FeatureTypeUsage; - custom_total: FeatureTypeUsage; -} - -export interface MlJobsUsage { - custom: FeatureUsage; - elastic: FeatureUsage; -} - -export interface DetectionsUsage { - ml_jobs: MlJobsUsage; -} +import type { MlJobUsageMetric } from './ml_jobs/types'; +import type { RuleAdoption } from './rules/types'; export interface DetectionMetrics { - ml_jobs: MlJobUsage; - detection_rules: DetectionRuleAdoption; -} - -export interface MlJobDataCount { - bucket_count: number; - empty_bucket_count: number; - input_bytes: number; - input_record_count: number; - last_data_time: number; - processed_record_count: number; -} - -export interface MlJobModelSize { - bucket_allocation_failures_count: number; - memory_status: string; - model_bytes: number; - model_bytes_exceeded: number; - model_bytes_memory_limit: number; - peak_model_bytes: number; -} - -export interface MlTimingStats { - bucket_count: number; - exponential_average_bucket_processing_time_ms: number; - exponential_average_bucket_processing_time_per_hour_ms: number; - maximum_bucket_processing_time_ms: number; - minimum_bucket_processing_time_ms: number; - total_bucket_processing_time_ms: number; -} - -export interface MlJobMetric { - job_id: string; - open_time: string; - state: string; - data_counts: MlJobDataCount; - model_size_stats: MlJobModelSize; - timing_stats: MlTimingStats; -} - -export interface DetectionRuleMetric { - rule_name: string; - rule_id: string; - rule_type: string; - rule_version: number; - enabled: boolean; - elastic_rule: boolean; - created_on: string; - updated_on: string; - alert_count_daily: number; - cases_count_total: number; - has_legacy_notification: boolean; - has_notification: boolean; -} - -export interface AlertsAggregationResponse { - hits: { - total: { value: number }; - }; - aggregations: { - [aggName: string]: { - buckets: Array<{ key: string; doc_count: number }>; - }; - }; -} - -export interface CasesSavedObject { - type: string; - alertId: string; - index: string; - rule: { - id: string | null; - name: string | null; - }; -} - -export interface MlJobUsage { - ml_job_usage: MlJobsUsage; - ml_job_metrics: MlJobMetric[]; -} - -export interface DetectionRuleAdoption { - detection_rule_detail: DetectionRuleMetric[]; - detection_rule_usage: DetectionRulesTypeUsage; + ml_jobs: MlJobUsageMetric; + detection_rules: RuleAdoption; } diff --git a/x-pack/plugins/security_solution/server/usage/get_internal_saved_objects_client.ts b/x-pack/plugins/security_solution/server/usage/get_internal_saved_objects_client.ts new file mode 100644 index 0000000000000..aea462ecf1fa6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/get_internal_saved_objects_client.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup, SavedObjectsClientContract } from 'kibana/server'; + +import { SAVED_OBJECT_TYPES } from '../../../cases/common/constants'; +// eslint-disable-next-line no-restricted-imports +import { legacyRuleActionsSavedObjectType } from '../lib/detection_engine/rule_actions/legacy_saved_object_mappings'; + +export async function getInternalSavedObjectsClient( + core: CoreSetup +): Promise { + return core.getStartServices().then(async ([coreStart]) => { + // note: we include the "cases" and "alert" hidden types here otherwise we would not be able to query them. If at some point cases and alert is not considered a hidden type this can be removed + return coreStart.savedObjects.createInternalRepository([ + 'alert', + legacyRuleActionsSavedObjectType, + ...SAVED_OBJECT_TYPES, + ]) as unknown as SavedObjectsClientContract; + }); +} diff --git a/x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts b/x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts new file mode 100644 index 0000000000000..1514760b0b251 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { OpenPointInTimeResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { + AggregationsCompositeAggregation, + SearchRequest, +} from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient, Logger } from 'kibana/server'; +import type { AlertBucket, AlertAggs } from '../types'; + +export interface GetAlertsOptions { + esClient: ElasticsearchClient; + signalsIndex: string; + maxSize: number; + maxPerPage: number; + logger: Logger; +} + +export const getAlerts = async ({ + esClient, + signalsIndex, + maxSize, + maxPerPage, + logger, +}: GetAlertsOptions): Promise => { + // default is from looking at Kibana saved objects and online documentation + const keepAlive = '5m'; + + // create and assign an initial point in time + let pitId: OpenPointInTimeResponse['id'] = ( + await esClient.openPointInTime({ + index: signalsIndex, + keep_alive: keepAlive, + }) + ).body.id; + + let after: AggregationsCompositeAggregation['after']; + let buckets: AlertBucket[] = []; + let fetchMore = true; + while (fetchMore) { + const ruleSearchOptions: SearchRequest = { + aggs: { + buckets: { + composite: { + size: Math.min(maxPerPage, maxSize - buckets.length), + sources: [ + { + detectionAlerts: { + terms: { + field: 'kibana.alert.rule.uuid', + }, + }, + }, + ], + after, + }, + }, + }, + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: 'now-24h', + lte: 'now', + }, + }, + }, + ], + }, + }, + track_total_hits: false, + sort: [{ _shard_doc: 'desc' }] as unknown as string[], // Remove this "unknown" once it is typed correctly https://github.com/elastic/elasticsearch-js/issues/1589 + pit: { id: pitId }, + size: 0, + }; + logger.debug( + `Getting alerts with point in time (PIT) query: ${JSON.stringify(ruleSearchOptions)}` + ); + const { body } = await esClient.search(ruleSearchOptions); + if (body.aggregations?.buckets?.buckets != null) { + buckets = [...buckets, ...body.aggregations.buckets.buckets]; + } + if (body.aggregations?.buckets?.after_key != null) { + after = { + detectionAlerts: body.aggregations.buckets.after_key.detectionAlerts, + }; + } + + fetchMore = + body.aggregations?.buckets?.buckets != null && + body.aggregations?.buckets?.buckets.length !== 0 && + buckets.length < maxSize; + if (body.pit_id != null) { + pitId = body.pit_id; + } + } + try { + await esClient.closePointInTime({ id: pitId }); + } catch (error) { + // Don't fail due to a bad point in time closure. We have seen failures in e2e tests during nominal operations. + logger.warn( + `Error trying to close point in time: "${pitId}", it will expire within "${keepAlive}". Error is: "${error}"` + ); + } + logger.debug(`Returning alerts response of length: "${buckets.length}"`); + return buckets; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_cases.ts b/x-pack/plugins/security_solution/server/usage/queries/get_case_comments.ts similarity index 59% rename from x-pack/plugins/security_solution/server/usage/detections/get_cases.ts rename to x-pack/plugins/security_solution/server/usage/queries/get_case_comments.ts index aa93a73fdb652..9e8ceed9fa6d8 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/get_cases.ts +++ b/x-pack/plugins/security_solution/server/usage/queries/get_case_comments.ts @@ -8,28 +8,35 @@ import type { SavedObjectsClientContract, SavedObjectsFindResult, -} from '../../../../../../src/core/server'; -import type { CasesSavedObject } from './types'; + Logger, + SavedObjectsCreatePointInTimeFinderOptions, +} from 'kibana/server'; + import { CASE_COMMENT_SAVED_OBJECT } from '../../../../cases/common/constants'; +import type { CommentAttributes } from '../../../../cases/common/api/cases/comment'; export interface GetCasesOptions { savedObjectsClient: SavedObjectsClientContract; maxSize: number; maxPerPage: number; + logger: Logger; } -export const getCases = async ({ +export const getCaseComments = async ({ savedObjectsClient, maxSize, maxPerPage, -}: GetCasesOptions): Promise>> => { - const finder = savedObjectsClient.createPointInTimeFinder({ + logger, +}: GetCasesOptions): Promise>> => { + const query: SavedObjectsCreatePointInTimeFinderOptions = { type: CASE_COMMENT_SAVED_OBJECT, perPage: maxPerPage, namespaces: ['*'], filter: `${CASE_COMMENT_SAVED_OBJECT}.attributes.type: alert`, - }); - let responses: Array> = []; + }; + logger.debug(`Getting cases with point in time (PIT) query:', ${JSON.stringify(query)}`); + const finder = savedObjectsClient.createPointInTimeFinder(query); + let responses: Array> = []; for await (const response of finder.find()) { const extra = responses.length + response.saved_objects.length - maxSize; if (extra > 0) { @@ -38,5 +45,6 @@ export const getCases = async ({ responses = [...responses, ...response.saved_objects]; } } + logger.debug(`Returning cases response of length: "${responses.length}"`); return responses; }; diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_detection_rules.ts b/x-pack/plugins/security_solution/server/usage/queries/get_detection_rules.ts similarity index 79% rename from x-pack/plugins/security_solution/server/usage/detections/get_detection_rules.ts rename to x-pack/plugins/security_solution/server/usage/queries/get_detection_rules.ts index 5b68ea8c2d25e..8ae8657c4e67d 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/get_detection_rules.ts +++ b/x-pack/plugins/security_solution/server/usage/queries/get_detection_rules.ts @@ -5,7 +5,9 @@ * 2.0. */ -import { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { ElasticsearchClient, Logger } from 'kibana/server'; + import { SIGNALS_ID, EQL_RULE_TYPE_ID, @@ -15,16 +17,15 @@ import { THRESHOLD_RULE_TYPE_ID, SAVED_QUERY_RULE_TYPE_ID, } from '@kbn/securitysolution-rules'; -import { ElasticsearchClient } from 'kibana/server'; -import { fetchWithPit } from './fetch_with_pit'; - -import { RuleSearchResult } from './types'; +import { fetchHitsWithPit } from './utils/fetch_hits_with_pit'; +import { RuleSearchResult } from '../types'; export interface GetDetectionRulesOptions { esClient: ElasticsearchClient; kibanaIndex: string; maxSize: number; maxPerPage: number; + logger: Logger; } export const getDetectionRules = async ({ @@ -32,8 +33,10 @@ export const getDetectionRules = async ({ kibanaIndex, maxSize, maxPerPage, + logger, }: GetDetectionRulesOptions): Promise>> => { - return fetchWithPit({ + return fetchHitsWithPit({ + logger, esClient, index: kibanaIndex, maxSize, diff --git a/x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts b/x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts new file mode 100644 index 0000000000000..9ed8c6a47a0c5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SavedObjectsClientContract, + SavedObjectsFindResult, + Logger, + SavedObjectsCreatePointInTimeFinderOptions, +} from 'kibana/server'; +// eslint-disable-next-line no-restricted-imports +import type { LegacyIRuleActionsAttributesSavedObjectAttributes } from '../../lib/detection_engine/rule_actions/legacy_types'; + +// eslint-disable-next-line no-restricted-imports +import { legacyRuleActionsSavedObjectType } from '../../lib/detection_engine/rule_actions/legacy_saved_object_mappings'; + +export interface LegacyGetRuleActionsOptions { + savedObjectsClient: SavedObjectsClientContract; + maxSize: number; + maxPerPage: number; + logger: Logger; +} + +/** + * Returns the legacy rule actions + * @deprecated Once we are confident all rules relying on side-car actions SO's have been migrated to SO references we should remove "legacyRuleActions" code including this function + */ +export const legacyGetRuleActions = async ({ + savedObjectsClient, + maxSize, + maxPerPage, + logger, +}: LegacyGetRuleActionsOptions) => { + const query: SavedObjectsCreatePointInTimeFinderOptions = { + type: legacyRuleActionsSavedObjectType, + perPage: maxPerPage, + namespaces: ['*'], + }; + logger.debug( + `Getting legacy rule actions with point in time (PIT) query:', ${JSON.stringify(query)}` + ); + const finder = + savedObjectsClient.createPointInTimeFinder( + query + ); + let responses: Array> = + []; + for await (const response of finder.find()) { + const extra = responses.length + response.saved_objects.length - maxSize; + if (extra > 0) { + responses = [...responses, ...response.saved_objects.splice(extra)]; + } else { + responses = [...responses, ...response.saved_objects]; + } + } + logger.debug(`Returning legacy rule actions response of length: "${responses.length}"`); + return responses; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/fetch_with_pit.ts b/x-pack/plugins/security_solution/server/usage/queries/utils/fetch_hits_with_pit.ts similarity index 57% rename from x-pack/plugins/security_solution/server/usage/detections/fetch_with_pit.ts rename to x-pack/plugins/security_solution/server/usage/queries/utils/fetch_hits_with_pit.ts index 59bdb32cb0feb..cc4d8d98df799 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/fetch_with_pit.ts +++ b/x-pack/plugins/security_solution/server/usage/queries/utils/fetch_hits_with_pit.ts @@ -5,13 +5,13 @@ * 2.0. */ -import { +import type { OpenPointInTimeResponse, SearchHit, SortResults, } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { SearchRequest } from '@elastic/elasticsearch/lib/api/types'; -import { ElasticsearchClient } from 'kibana/server'; +import type { SearchRequest } from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient, Logger } from 'kibana/server'; export interface FetchWithPitOptions { esClient: ElasticsearchClient; @@ -19,19 +19,25 @@ export interface FetchWithPitOptions { maxSize: number; maxPerPage: number; searchRequest: SearchRequest; + logger: Logger; } -export const fetchWithPit = async ({ + +export const fetchHitsWithPit = async ({ esClient, index, searchRequest, maxSize, maxPerPage, + logger, }: FetchWithPitOptions): Promise>> => { + // default is from looking at Kibana saved objects and online documentation + const keepAlive = '5m'; + // create and assign an initial point in time let pitId: OpenPointInTimeResponse['id'] = ( await esClient.openPointInTime({ index, - keep_alive: '5m', // default is from looking at Kibana saved objects and online documentation + keep_alive: '5m', }) ).body.id; @@ -41,21 +47,33 @@ export const fetchWithPit = async ({ while (fetchMore) { const ruleSearchOptions: SearchRequest = { ...searchRequest, + track_total_hits: false, search_after: searchAfter, - sort: [{ _shard_doc: 'desc' }] as unknown as string[], // FUNFACT: This is not typed correctly https://github.com/elastic/elasticsearch-js/issues/1589 + sort: [{ _shard_doc: 'desc' }] as unknown as string[], // Remove this "unknown" once it is typed correctly https://github.com/elastic/elasticsearch-js/issues/1589 pit: { id: pitId }, size: Math.min(maxPerPage, maxSize - hits.length), }; + logger.debug( + `Getting hits with point in time (PIT) query of: ${JSON.stringify(ruleSearchOptions)}` + ); const { body } = await esClient.search(ruleSearchOptions); hits = [...hits, ...body.hits.hits]; searchAfter = body.hits.hits.length !== 0 ? body.hits.hits[body.hits.hits.length - 1].sort : undefined; - fetchMore = searchAfter != null && hits.length <= maxSize; + fetchMore = searchAfter != null && body.hits.hits.length > 0 && hits.length < maxSize; if (body.pit_id != null) { pitId = body.pit_id; } } - esClient.closePointInTime({ id: pitId }); + try { + await esClient.closePointInTime({ id: pitId }); + } catch (error) { + // Don't fail due to a bad point in time closure. We have seen failures in e2e tests during nominal operations. + logger.warn( + `Error trying to close point in time: "${pitId}", it will expire within "${keepAlive}". Error is: "${error}"` + ); + } + logger.debug(`Returning hits with point in time (PIT) length of: ${hits.length}`); return hits; }; diff --git a/x-pack/plugins/security_solution/server/usage/queries/utils/is_elastic_rule.ts b/x-pack/plugins/security_solution/server/usage/queries/utils/is_elastic_rule.ts new file mode 100644 index 0000000000000..f08959702b290 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/queries/utils/is_elastic_rule.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; + +export const isElasticRule = (tags: string[] = []) => + tags.includes(`${INTERNAL_IMMUTABLE_KEY}:true`); diff --git a/x-pack/plugins/security_solution/server/usage/schema.ts b/x-pack/plugins/security_solution/server/usage/schema.ts new file mode 100644 index 0000000000000..a70a2be910dec --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/schema.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ml_jobs } from './detections/ml_jobs/schema'; +import { detection_rules } from './detections/rules/schema'; + +export interface UsageData { + detectionMetrics: {}; +} + +export const schema: UsageData = { + detectionMetrics: { + detection_rules, + ml_jobs, + }, +}; diff --git a/x-pack/plugins/security_solution/server/usage/types.ts b/x-pack/plugins/security_solution/server/usage/types.ts index 1a3b5d1e2e29f..bb05c88a0d94e 100644 --- a/x-pack/plugins/security_solution/server/usage/types.ts +++ b/x-pack/plugins/security_solution/server/usage/types.ts @@ -5,11 +5,34 @@ * 2.0. */ -import { CoreSetup } from 'src/core/server'; -import { SetupPlugins } from '../plugin'; +import type { CoreSetup, Logger } from 'src/core/server'; +import type { SanitizedAlert } from '../../../alerting/common/alert'; +import type { RuleParams } from '../lib/detection_engine/schemas/rule_schemas'; +import type { SetupPlugins } from '../plugin'; export type CollectorDependencies = { kibanaIndex: string; signalsIndex: string; core: CoreSetup; + logger: Logger; } & Pick; + +export interface AlertBucket { + key: { + detectionAlerts: string; + }; + doc_count: number; +} + +export interface AlertAggs { + buckets?: { + after_key?: { + detectionAlerts: string; + }; + buckets: AlertBucket[]; + }; +} + +export interface RuleSearchResult { + alert: SanitizedAlert; +} diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/all_types.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/all_types.ts index a8d473597a461..29baea4e4bd90 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/all_types.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/all_types.ts @@ -6,14 +6,14 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../../common/ftr_provider_context'; +import type { FtrProviderContext } from '../../../../common/ftr_provider_context'; import { createSignalsIndex, deleteAllAlerts, deleteSignalsIndex, getStats, } from '../../../../utils'; -import { getInitialDetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/detection_rule_helpers'; +import { getInitialDetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/get_initial_usage'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/detection_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/detection_rules.ts index 8a956d456edec..b93141a1ffe73 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/detection_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/detection_rules.ts @@ -6,12 +6,12 @@ */ import expect from '@kbn/expect'; -import { DetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/types'; -import { +import type { DetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/types'; +import type { ThreatMatchCreateSchema, ThresholdCreateSchema, } from '../../../../../../plugins/security_solution/common/detection_engine/schemas/request'; -import { FtrProviderContext } from '../../../../common/ftr_provider_context'; +import type { FtrProviderContext } from '../../../../common/ftr_provider_context'; import { createLegacyRuleAction, createNewAction, @@ -33,7 +33,7 @@ import { waitForSignalsToBePresent, updateRule, } from '../../../../utils'; -import { getInitialDetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/detection_rule_helpers'; +import { getInitialDetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/get_initial_usage'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { From f306b245f5e6e5c6afca8530a1a41519aff2af96 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Mon, 7 Feb 2022 19:01:18 -0700 Subject: [PATCH 03/10] Fixes schema issue by re-combining it again --- .../server/usage/collector.ts | 487 +++++++++++++++++- .../server/usage/detections/ml_jobs/schema.ts | 202 -------- .../server/usage/detections/rules/schema.ts | 285 ---------- .../security_solution/server/usage/schema.ts | 20 - 4 files changed, 485 insertions(+), 509 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/usage/detections/ml_jobs/schema.ts delete mode 100644 x-pack/plugins/security_solution/server/usage/detections/rules/schema.ts delete mode 100644 x-pack/plugins/security_solution/server/usage/schema.ts diff --git a/x-pack/plugins/security_solution/server/usage/collector.ts b/x-pack/plugins/security_solution/server/usage/collector.ts index 7dd6c23ac97f5..9a2009405b4ef 100644 --- a/x-pack/plugins/security_solution/server/usage/collector.ts +++ b/x-pack/plugins/security_solution/server/usage/collector.ts @@ -8,11 +8,14 @@ import type { CollectorFetchContext } from '../../../../../src/plugins/usage_collection/server'; import type { CollectorDependencies } from './types'; import { getDetectionsMetrics } from './detections/get_metrics'; -import { schema, UsageData } from './schema'; import { getInternalSavedObjectsClient } from './get_internal_saved_objects_client'; export type RegisterCollector = (deps: CollectorDependencies) => void; +export interface UsageData { + detectionMetrics: {}; +} + export const registerCollector: RegisterCollector = ({ core, kibanaIndex, @@ -28,7 +31,487 @@ export const registerCollector: RegisterCollector = ({ const collector = usageCollection.makeUsageCollector({ type: 'security_solution', - schema, + schema: { + detectionMetrics: { + detection_rules: { + detection_rule_usage: { + query: { + enabled: { type: 'long', _meta: { description: 'Number of query rules enabled' } }, + disabled: { type: 'long', _meta: { description: 'Number of query rules disabled' } }, + alerts: { + type: 'long', + _meta: { description: 'Number of alerts generated by query rules' }, + }, + cases: { + type: 'long', + _meta: { description: 'Number of cases attached to query detection rule alerts' }, + }, + legacy_notifications_enabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications enabled' }, + }, + legacy_notifications_disabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications disabled' }, + }, + notifications_enabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + notifications_disabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + }, + threshold: { + enabled: { + type: 'long', + _meta: { description: 'Number of threshold rules enabled' }, + }, + disabled: { + type: 'long', + _meta: { description: 'Number of threshold rules disabled' }, + }, + alerts: { + type: 'long', + _meta: { description: 'Number of alerts generated by threshold rules' }, + }, + cases: { + type: 'long', + _meta: { + description: 'Number of cases attached to threshold detection rule alerts', + }, + }, + legacy_notifications_enabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications enabled' }, + }, + legacy_notifications_disabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications disabled' }, + }, + notifications_enabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + notifications_disabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + }, + eql: { + enabled: { type: 'long', _meta: { description: 'Number of eql rules enabled' } }, + disabled: { type: 'long', _meta: { description: 'Number of eql rules disabled' } }, + alerts: { + type: 'long', + _meta: { description: 'Number of alerts generated by eql rules' }, + }, + cases: { + type: 'long', + _meta: { description: 'Number of cases attached to eql detection rule alerts' }, + }, + legacy_notifications_enabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications enabled' }, + }, + legacy_notifications_disabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications disabled' }, + }, + notifications_enabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + notifications_disabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + }, + machine_learning: { + enabled: { + type: 'long', + _meta: { description: 'Number of machine_learning rules enabled' }, + }, + disabled: { + type: 'long', + _meta: { description: 'Number of machine_learning rules disabled' }, + }, + alerts: { + type: 'long', + _meta: { description: 'Number of alerts generated by machine_learning rules' }, + }, + cases: { + type: 'long', + _meta: { + description: 'Number of cases attached to machine_learning detection rule alerts', + }, + }, + legacy_notifications_enabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications enabled' }, + }, + legacy_notifications_disabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications disabled' }, + }, + notifications_enabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + notifications_disabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + }, + threat_match: { + enabled: { + type: 'long', + _meta: { description: 'Number of threat_match rules enabled' }, + }, + disabled: { + type: 'long', + _meta: { description: 'Number of threat_match rules disabled' }, + }, + alerts: { + type: 'long', + _meta: { description: 'Number of alerts generated by threat_match rules' }, + }, + cases: { + type: 'long', + _meta: { + description: 'Number of cases attached to threat_match detection rule alerts', + }, + }, + legacy_notifications_enabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications enabled' }, + }, + legacy_notifications_disabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications disabled' }, + }, + notifications_enabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + notifications_disabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + }, + elastic_total: { + enabled: { type: 'long', _meta: { description: 'Number of elastic rules enabled' } }, + disabled: { + type: 'long', + _meta: { description: 'Number of elastic rules disabled' }, + }, + alerts: { + type: 'long', + _meta: { description: 'Number of alerts generated by elastic rules' }, + }, + cases: { + type: 'long', + _meta: { description: 'Number of cases attached to elastic detection rule alerts' }, + }, + legacy_notifications_enabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications enabled' }, + }, + legacy_notifications_disabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications disabled' }, + }, + notifications_enabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + notifications_disabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + }, + custom_total: { + enabled: { type: 'long', _meta: { description: 'Number of custom rules enabled' } }, + disabled: { type: 'long', _meta: { description: 'Number of custom rules disabled' } }, + alerts: { + type: 'long', + _meta: { description: 'Number of alerts generated by custom rules' }, + }, + cases: { + type: 'long', + _meta: { description: 'Number of cases attached to custom detection rule alerts' }, + }, + legacy_notifications_enabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications enabled' }, + }, + legacy_notifications_disabled: { + type: 'long', + _meta: { description: 'Number of legacy notifications disabled' }, + }, + notifications_enabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + notifications_disabled: { + type: 'long', + _meta: { description: 'Number of notifications enabled' }, + }, + }, + }, + detection_rule_detail: { + type: 'array', + items: { + rule_name: { + type: 'keyword', + _meta: { description: 'The name of the detection rule' }, + }, + rule_id: { + type: 'keyword', + _meta: { description: 'The UUID id of the detection rule' }, + }, + rule_type: { + type: 'keyword', + _meta: { description: 'The type of detection rule. ie eql, query...' }, + }, + rule_version: { type: 'long', _meta: { description: 'The version of the rule' } }, + enabled: { + type: 'boolean', + _meta: { description: 'If the detection rule has been enabled by the user' }, + }, + elastic_rule: { + type: 'boolean', + _meta: { description: 'If the detection rule has been authored by Elastic' }, + }, + created_on: { + type: 'keyword', + _meta: { description: 'When the detection rule was created on the cluster' }, + }, + updated_on: { + type: 'keyword', + _meta: { description: 'When the detection rule was updated on the cluster' }, + }, + alert_count_daily: { + type: 'long', + _meta: { description: 'The number of daily alerts generated by a rule' }, + }, + cases_count_total: { + type: 'long', + _meta: { description: 'The number of total cases generated by a rule' }, + }, + has_legacy_notification: { + type: 'boolean', + _meta: { description: 'True if this rule has a legacy notification' }, + }, + has_notification: { + type: 'boolean', + _meta: { description: 'True if this rule has a notification' }, + }, + }, + }, + }, + ml_jobs: { + ml_job_usage: { + custom: { + enabled: { + type: 'long', + _meta: { description: 'The number of custom ML jobs rules enabled' }, + }, + disabled: { + type: 'long', + _meta: { description: 'The number of custom ML jobs rules disabled' }, + }, + }, + elastic: { + enabled: { + type: 'long', + _meta: { description: 'The number of elastic provided ML jobs rules enabled' }, + }, + disabled: { + type: 'long', + _meta: { description: 'The number of elastic provided ML jobs rules disabled' }, + }, + }, + }, + ml_job_metrics: { + type: 'array', + items: { + job_id: { + type: 'keyword', + _meta: { description: 'Identifier for the anomaly detection job' }, + }, + open_time: { + type: 'keyword', + _meta: { + description: + 'For open jobs only, the elapsed time for which the job has been open', + }, + }, + create_time: { + type: 'keyword', + _meta: { description: 'The time the job was created' }, + }, + finished_time: { + type: 'keyword', + _meta: { + description: 'If the job closed or failed, this is the time the job finished', + }, + }, + state: { + type: 'keyword', + _meta: { description: 'The status of the anomaly detection job' }, + }, + data_counts: { + bucket_count: { + type: 'long', + _meta: { description: 'The number of buckets processed' }, + }, + empty_bucket_count: { + type: 'long', + _meta: { description: 'The number of buckets which did not contain any data' }, + }, + input_bytes: { + type: 'long', + _meta: { + description: + 'The number of bytes of input data posted to the anomaly detection job', + }, + }, + input_record_count: { + type: 'long', + _meta: { + description: + 'The number of input documents posted to the anomaly detection job', + }, + }, + last_data_time: { + type: 'long', + _meta: { + description: + 'The timestamp at which data was last analyzed, according to server time', + }, + }, + processed_record_count: { + type: 'long', + _meta: { + description: + 'The number of input documents that have been processed by the anomaly detection job', + }, + }, + }, + model_size_stats: { + bucket_allocation_failures_count: { + type: 'long', + _meta: { + description: + 'The number of buckets for which new entities in incoming data were not processed due to insufficient model memory', + }, + }, + model_bytes: { + type: 'long', + _meta: { description: 'The number of bytes of memory used by the models' }, + }, + model_bytes_exceeded: { + type: 'long', + _meta: { + description: + 'The number of bytes over the high limit for memory usage at the last allocation failure', + }, + }, + model_bytes_memory_limit: { + type: 'long', + _meta: { + description: + 'The upper limit for model memory usage, checked on increasing values', + }, + }, + peak_model_bytes: { + type: 'long', + _meta: { + description: 'The peak number of bytes of memory ever used by the models', + }, + }, + }, + timing_stats: { + bucket_count: { + type: 'long', + _meta: { description: 'The number of buckets processed' }, + }, + exponential_average_bucket_processing_time_ms: { + type: 'long', + _meta: { + description: + 'Exponential moving average of all bucket processing times, in milliseconds', + }, + }, + exponential_average_bucket_processing_time_per_hour_ms: { + type: 'long', + _meta: { + description: + 'Exponentially-weighted moving average of bucket processing times calculated in a 1 hour time window, in milliseconds', + }, + }, + maximum_bucket_processing_time_ms: { + type: 'long', + _meta: { + description: 'Maximum among all bucket processing times, in milliseconds', + }, + }, + minimum_bucket_processing_time_ms: { + type: 'long', + _meta: { + description: 'Minimum among all bucket processing times, in milliseconds', + }, + }, + total_bucket_processing_time_ms: { + type: 'long', + _meta: { description: 'Sum of all bucket processing times, in milliseconds' }, + }, + }, + datafeed: { + datafeed_id: { + type: 'keyword', + _meta: { + description: + 'A numerical character string that uniquely identifies the datafeed', + }, + }, + state: { + type: 'keyword', + _meta: { description: 'The status of the datafeed' }, + }, + timing_stats: { + average_search_time_per_bucket_ms: { + type: 'long', + _meta: { description: 'The average search time per bucket, in milliseconds' }, + }, + bucket_count: { + type: 'long', + _meta: { description: 'The number of buckets processed' }, + }, + exponential_average_search_time_per_hour_ms: { + type: 'long', + _meta: { + description: 'The exponential average search time per hour, in milliseconds', + }, + }, + search_count: { + type: 'long', + _meta: { description: 'The number of searches run by the datafeed' }, + }, + total_search_time_ms: { + type: 'long', + _meta: { + description: 'The total time the datafeed spent searching, in milliseconds', + }, + }, + }, + }, + }, + }, + }, + }, + }, isReady: () => true, fetch: async ({ esClient }: CollectorFetchContext): Promise => { const savedObjectsClient = await getInternalSavedObjectsClient(core); diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/schema.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/schema.ts deleted file mode 100644 index 6df24516f3d53..0000000000000 --- a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/schema.ts +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -// eslint-disable-next-line @typescript-eslint/naming-convention -export const ml_jobs = { - ml_job_usage: { - custom: { - enabled: { - type: 'long', - _meta: { description: 'The number of custom ML jobs rules enabled' }, - }, - disabled: { - type: 'long', - _meta: { description: 'The number of custom ML jobs rules disabled' }, - }, - }, - elastic: { - enabled: { - type: 'long', - _meta: { description: 'The number of elastic provided ML jobs rules enabled' }, - }, - disabled: { - type: 'long', - _meta: { description: 'The number of elastic provided ML jobs rules disabled' }, - }, - }, - }, - ml_job_metrics: { - type: 'array', - items: { - job_id: { - type: 'keyword', - _meta: { description: 'Identifier for the anomaly detection job' }, - }, - open_time: { - type: 'keyword', - _meta: { - description: 'For open jobs only, the elapsed time for which the job has been open', - }, - }, - create_time: { - type: 'keyword', - _meta: { description: 'The time the job was created' }, - }, - finished_time: { - type: 'keyword', - _meta: { - description: 'If the job closed or failed, this is the time the job finished', - }, - }, - state: { - type: 'keyword', - _meta: { description: 'The status of the anomaly detection job' }, - }, - data_counts: { - bucket_count: { - type: 'long', - _meta: { description: 'The number of buckets processed' }, - }, - empty_bucket_count: { - type: 'long', - _meta: { description: 'The number of buckets which did not contain any data' }, - }, - input_bytes: { - type: 'long', - _meta: { - description: 'The number of bytes of input data posted to the anomaly detection job', - }, - }, - input_record_count: { - type: 'long', - _meta: { - description: 'The number of input documents posted to the anomaly detection job', - }, - }, - last_data_time: { - type: 'long', - _meta: { - description: 'The timestamp at which data was last analyzed, according to server time', - }, - }, - processed_record_count: { - type: 'long', - _meta: { - description: - 'The number of input documents that have been processed by the anomaly detection job', - }, - }, - }, - model_size_stats: { - bucket_allocation_failures_count: { - type: 'long', - _meta: { - description: - 'The number of buckets for which new entities in incoming data were not processed due to insufficient model memory', - }, - }, - model_bytes: { - type: 'long', - _meta: { description: 'The number of bytes of memory used by the models' }, - }, - model_bytes_exceeded: { - type: 'long', - _meta: { - description: - 'The number of bytes over the high limit for memory usage at the last allocation failure', - }, - }, - model_bytes_memory_limit: { - type: 'long', - _meta: { - description: 'The upper limit for model memory usage, checked on increasing values', - }, - }, - peak_model_bytes: { - type: 'long', - _meta: { - description: 'The peak number of bytes of memory ever used by the models', - }, - }, - }, - timing_stats: { - bucket_count: { - type: 'long', - _meta: { description: 'The number of buckets processed' }, - }, - exponential_average_bucket_processing_time_ms: { - type: 'long', - _meta: { - description: - 'Exponential moving average of all bucket processing times, in milliseconds', - }, - }, - exponential_average_bucket_processing_time_per_hour_ms: { - type: 'long', - _meta: { - description: - 'Exponentially-weighted moving average of bucket processing times calculated in a 1 hour time window, in milliseconds', - }, - }, - maximum_bucket_processing_time_ms: { - type: 'long', - _meta: { - description: 'Maximum among all bucket processing times, in milliseconds', - }, - }, - minimum_bucket_processing_time_ms: { - type: 'long', - _meta: { - description: 'Minimum among all bucket processing times, in milliseconds', - }, - }, - total_bucket_processing_time_ms: { - type: 'long', - _meta: { description: 'Sum of all bucket processing times, in milliseconds' }, - }, - }, - datafeed: { - datafeed_id: { - type: 'keyword', - _meta: { - description: 'A numerical character string that uniquely identifies the datafeed', - }, - }, - state: { - type: 'keyword', - _meta: { description: 'The status of the datafeed' }, - }, - timing_stats: { - average_search_time_per_bucket_ms: { - type: 'long', - _meta: { description: 'The average search time per bucket, in milliseconds' }, - }, - bucket_count: { - type: 'long', - _meta: { description: 'The number of buckets processed' }, - }, - exponential_average_search_time_per_hour_ms: { - type: 'long', - _meta: { - description: 'The exponential average search time per hour, in milliseconds', - }, - }, - search_count: { - type: 'long', - _meta: { description: 'The number of searches run by the datafeed' }, - }, - total_search_time_ms: { - type: 'long', - _meta: { - description: 'The total time the datafeed spent searching, in milliseconds', - }, - }, - }, - }, - }, - }, -}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/schema.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/schema.ts deleted file mode 100644 index 7088c6665e314..0000000000000 --- a/x-pack/plugins/security_solution/server/usage/detections/rules/schema.ts +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -// eslint-disable-next-line @typescript-eslint/naming-convention -export const detection_rules = { - detection_rule_usage: { - query: { - enabled: { type: 'long', _meta: { description: 'Number of query rules enabled' } }, - disabled: { type: 'long', _meta: { description: 'Number of query rules disabled' } }, - alerts: { - type: 'long', - _meta: { description: 'Number of alerts generated by query rules' }, - }, - cases: { - type: 'long', - _meta: { description: 'Number of cases attached to query detection rule alerts' }, - }, - legacy_notifications_enabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications enabled' }, - }, - legacy_notifications_disabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications disabled' }, - }, - notifications_enabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - notifications_disabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - }, - threshold: { - enabled: { - type: 'long', - _meta: { description: 'Number of threshold rules enabled' }, - }, - disabled: { - type: 'long', - _meta: { description: 'Number of threshold rules disabled' }, - }, - alerts: { - type: 'long', - _meta: { description: 'Number of alerts generated by threshold rules' }, - }, - cases: { - type: 'long', - _meta: { - description: 'Number of cases attached to threshold detection rule alerts', - }, - }, - legacy_notifications_enabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications enabled' }, - }, - legacy_notifications_disabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications disabled' }, - }, - notifications_enabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - notifications_disabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - }, - eql: { - enabled: { type: 'long', _meta: { description: 'Number of eql rules enabled' } }, - disabled: { type: 'long', _meta: { description: 'Number of eql rules disabled' } }, - alerts: { - type: 'long', - _meta: { description: 'Number of alerts generated by eql rules' }, - }, - cases: { - type: 'long', - _meta: { description: 'Number of cases attached to eql detection rule alerts' }, - }, - legacy_notifications_enabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications enabled' }, - }, - legacy_notifications_disabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications disabled' }, - }, - notifications_enabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - notifications_disabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - }, - machine_learning: { - enabled: { - type: 'long', - _meta: { description: 'Number of machine_learning rules enabled' }, - }, - disabled: { - type: 'long', - _meta: { description: 'Number of machine_learning rules disabled' }, - }, - alerts: { - type: 'long', - _meta: { description: 'Number of alerts generated by machine_learning rules' }, - }, - cases: { - type: 'long', - _meta: { - description: 'Number of cases attached to machine_learning detection rule alerts', - }, - }, - legacy_notifications_enabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications enabled' }, - }, - legacy_notifications_disabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications disabled' }, - }, - notifications_enabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - notifications_disabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - }, - threat_match: { - enabled: { - type: 'long', - _meta: { description: 'Number of threat_match rules enabled' }, - }, - disabled: { - type: 'long', - _meta: { description: 'Number of threat_match rules disabled' }, - }, - alerts: { - type: 'long', - _meta: { description: 'Number of alerts generated by threat_match rules' }, - }, - cases: { - type: 'long', - _meta: { - description: 'Number of cases attached to threat_match detection rule alerts', - }, - }, - legacy_notifications_enabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications enabled' }, - }, - legacy_notifications_disabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications disabled' }, - }, - notifications_enabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - notifications_disabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - }, - elastic_total: { - enabled: { type: 'long', _meta: { description: 'Number of elastic rules enabled' } }, - disabled: { - type: 'long', - _meta: { description: 'Number of elastic rules disabled' }, - }, - alerts: { - type: 'long', - _meta: { description: 'Number of alerts generated by elastic rules' }, - }, - cases: { - type: 'long', - _meta: { description: 'Number of cases attached to elastic detection rule alerts' }, - }, - legacy_notifications_enabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications enabled' }, - }, - legacy_notifications_disabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications disabled' }, - }, - notifications_enabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - notifications_disabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - }, - custom_total: { - enabled: { type: 'long', _meta: { description: 'Number of custom rules enabled' } }, - disabled: { type: 'long', _meta: { description: 'Number of custom rules disabled' } }, - alerts: { - type: 'long', - _meta: { description: 'Number of alerts generated by custom rules' }, - }, - cases: { - type: 'long', - _meta: { description: 'Number of cases attached to custom detection rule alerts' }, - }, - legacy_notifications_enabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications enabled' }, - }, - legacy_notifications_disabled: { - type: 'long', - _meta: { description: 'Number of legacy notifications disabled' }, - }, - notifications_enabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - notifications_disabled: { - type: 'long', - _meta: { description: 'Number of notifications enabled' }, - }, - }, - }, - detection_rule_detail: { - type: 'array', - items: { - rule_name: { - type: 'keyword', - _meta: { description: 'The name of the detection rule' }, - }, - rule_id: { - type: 'keyword', - _meta: { description: 'The UUID id of the detection rule' }, - }, - rule_type: { - type: 'keyword', - _meta: { description: 'The type of detection rule. ie eql, query...' }, - }, - rule_version: { type: 'long', _meta: { description: 'The version of the rule' } }, - enabled: { - type: 'boolean', - _meta: { description: 'If the detection rule has been enabled by the user' }, - }, - elastic_rule: { - type: 'boolean', - _meta: { description: 'If the detection rule has been authored by Elastic' }, - }, - created_on: { - type: 'keyword', - _meta: { description: 'When the detection rule was created on the cluster' }, - }, - updated_on: { - type: 'keyword', - _meta: { description: 'When the detection rule was updated on the cluster' }, - }, - alert_count_daily: { - type: 'long', - _meta: { description: 'The number of daily alerts generated by a rule' }, - }, - cases_count_total: { - type: 'long', - _meta: { description: 'The number of total cases generated by a rule' }, - }, - has_legacy_notification: { - type: 'boolean', - _meta: { description: 'True if this rule has a legacy notification' }, - }, - has_notification: { - type: 'boolean', - _meta: { description: 'True if this rule has a notification' }, - }, - }, - }, -}; diff --git a/x-pack/plugins/security_solution/server/usage/schema.ts b/x-pack/plugins/security_solution/server/usage/schema.ts deleted file mode 100644 index a70a2be910dec..0000000000000 --- a/x-pack/plugins/security_solution/server/usage/schema.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ml_jobs } from './detections/ml_jobs/schema'; -import { detection_rules } from './detections/rules/schema'; - -export interface UsageData { - detectionMetrics: {}; -} - -export const schema: UsageData = { - detectionMetrics: { - detection_rules, - ml_jobs, - }, -}; From 204d357c8d8328a6968c769196ca85aeba20975a Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Wed, 9 Feb 2022 13:02:31 -0700 Subject: [PATCH 04/10] Updated the constants to pull 1k from queries at a time and did a Promise.all() in one other area. --- .../security_solution/server/usage/constants.ts | 9 ++++----- .../server/usage/detections/rules/get_metrics.ts | 12 +++++++++--- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/server/usage/constants.ts b/x-pack/plugins/security_solution/server/usage/constants.ts index 27b191d7dac9a..d3d526768fcd5 100644 --- a/x-pack/plugins/security_solution/server/usage/constants.ts +++ b/x-pack/plugins/security_solution/server/usage/constants.ts @@ -19,9 +19,8 @@ export const MAX_RESULTS_WINDOW = 10_000; /** - * We arbitrarily choose our max per page based on 100 as that - * appears to be what others are choosing here in documentation: - * https://www.elastic.co/guide/en/elasticsearch/reference/current/point-in-time-api.html - * and within the saved objects client examples and documentation. + * We choose our max per page based on 1k as that + * appears to be what others are choosing here in the other sections of telemetry: + * https://github.com/elastic/kibana/pull/99031 */ -export const MAX_PER_PAGE = 100; +export const MAX_PER_PAGE = 1_000; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts index 8423a7bdeb7bd..f003f3dac6f3d 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts @@ -56,7 +56,7 @@ export const getRuleMetrics = async ({ } // gets the alerts data objects - const detectionAlertsResp = await getAlerts({ + const detectionAlertsRespPromise = getAlerts({ esClient, signalsIndex: `${signalsIndex}*`, maxPerPage: MAX_PER_PAGE, @@ -65,7 +65,7 @@ export const getRuleMetrics = async ({ }); // gets cases saved objects - const caseComments = await getCaseComments({ + const caseCommentsPromise = getCaseComments({ savedObjectsClient, maxSize: MAX_PER_PAGE, maxPerPage: MAX_RESULTS_WINDOW, @@ -73,13 +73,19 @@ export const getRuleMetrics = async ({ }); // gets the legacy rule actions to track legacy notifications. - const legacyRuleActions = await legacyGetRuleActions({ + const legacyRuleActionsPromise = legacyGetRuleActions({ savedObjectsClient, maxSize: MAX_PER_PAGE, maxPerPage: MAX_RESULTS_WINDOW, logger, }); + const [detectionAlertsResp, caseComments, legacyRuleActions] = await Promise.all([ + detectionAlertsRespPromise, + caseCommentsPromise, + legacyRuleActionsPromise, + ]); + // create in-memory maps for correlation const legacyNotificationRuleIds = getRuleIdToEnabledMap(legacyRuleActions); const casesRuleIds = getRuleIdToCasesMap(caseComments); From 1e5698ece58d6862e3f63ea517d9952a193d32c9 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Sat, 12 Feb 2022 13:04:05 -0700 Subject: [PATCH 05/10] Removed detection rules again --- .../detections/detection_rule_helpers.ts | 503 ------------------ 1 file changed, 503 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts diff --git a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts b/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts deleted file mode 100644 index 39c108931e2d7..0000000000000 --- a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts +++ /dev/null @@ -1,503 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - SIGNALS_ID, - EQL_RULE_TYPE_ID, - INDICATOR_RULE_TYPE_ID, - ML_RULE_TYPE_ID, - QUERY_RULE_TYPE_ID, - THRESHOLD_RULE_TYPE_ID, - SAVED_QUERY_RULE_TYPE_ID, -} from '@kbn/securitysolution-rules'; -import { ALERT_RULE_UUID } from '@kbn/rule-data-utils'; -import { CASE_COMMENT_SAVED_OBJECT } from '../../../../cases/common/constants'; - -import { ElasticsearchClient, SavedObjectsClientContract } from '../../../../../../src/core/server'; -import { isElasticRule } from './index'; -import type { - AlertsAggregationResponse, - CasesSavedObject, - DetectionRulesTypeUsage, - DetectionRuleMetric, - DetectionRuleAdoption, - RuleSearchParams, - RuleSearchResult, - DetectionMetrics, -} from './types'; -// eslint-disable-next-line no-restricted-imports -import { legacyRuleActionsSavedObjectType } from '../../lib/detection_engine/rule_actions/legacy_saved_object_mappings'; -// eslint-disable-next-line no-restricted-imports -import { LegacyIRuleActionsAttributesSavedObjectAttributes } from '../../lib/detection_engine/rule_actions/legacy_types'; - -/** - * Initial detection metrics initialized. - */ -export const getInitialDetectionMetrics = (): DetectionMetrics => ({ - ml_jobs: { - ml_job_usage: { - custom: { - enabled: 0, - disabled: 0, - }, - elastic: { - enabled: 0, - disabled: 0, - }, - }, - ml_job_metrics: [], - }, - detection_rules: { - detection_rule_detail: [], - detection_rule_usage: initialDetectionRulesUsage, - }, -}); - -/** - * Default detection rule usage count, split by type + elastic/custom - */ -export const initialDetectionRulesUsage: DetectionRulesTypeUsage = { - query: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - threshold: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - eql: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - machine_learning: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - threat_match: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - elastic_total: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - custom_total: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, -}; - -/* eslint-disable complexity */ -export const updateDetectionRuleUsage = ( - detectionRuleMetric: DetectionRuleMetric, - usage: DetectionRulesTypeUsage -): DetectionRulesTypeUsage => { - let updatedUsage = usage; - - const legacyNotificationEnabled = - detectionRuleMetric.has_legacy_notification && detectionRuleMetric.enabled; - - const legacyNotificationDisabled = - detectionRuleMetric.has_legacy_notification && !detectionRuleMetric.enabled; - - const notificationEnabled = detectionRuleMetric.has_notification && detectionRuleMetric.enabled; - - const notificationDisabled = detectionRuleMetric.has_notification && !detectionRuleMetric.enabled; - - if (detectionRuleMetric.rule_type === 'query') { - updatedUsage = { - ...usage, - query: { - ...usage.query, - enabled: detectionRuleMetric.enabled ? usage.query.enabled + 1 : usage.query.enabled, - disabled: !detectionRuleMetric.enabled ? usage.query.disabled + 1 : usage.query.disabled, - alerts: usage.query.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.query.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.query.legacy_notifications_enabled + 1 - : usage.query.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.query.legacy_notifications_disabled + 1 - : usage.query.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.query.notifications_enabled + 1 - : usage.query.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.query.notifications_disabled + 1 - : usage.query.notifications_disabled, - }, - }; - } else if (detectionRuleMetric.rule_type === 'threshold') { - updatedUsage = { - ...usage, - threshold: { - ...usage.threshold, - enabled: detectionRuleMetric.enabled - ? usage.threshold.enabled + 1 - : usage.threshold.enabled, - disabled: !detectionRuleMetric.enabled - ? usage.threshold.disabled + 1 - : usage.threshold.disabled, - alerts: usage.threshold.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.threshold.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.threshold.legacy_notifications_enabled + 1 - : usage.threshold.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.threshold.legacy_notifications_disabled + 1 - : usage.threshold.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.threshold.notifications_enabled + 1 - : usage.threshold.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.threshold.notifications_disabled + 1 - : usage.threshold.notifications_disabled, - }, - }; - } else if (detectionRuleMetric.rule_type === 'eql') { - updatedUsage = { - ...usage, - eql: { - ...usage.eql, - enabled: detectionRuleMetric.enabled ? usage.eql.enabled + 1 : usage.eql.enabled, - disabled: !detectionRuleMetric.enabled ? usage.eql.disabled + 1 : usage.eql.disabled, - alerts: usage.eql.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.eql.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.eql.legacy_notifications_enabled + 1 - : usage.eql.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.eql.legacy_notifications_disabled + 1 - : usage.eql.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.eql.notifications_enabled + 1 - : usage.eql.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.eql.notifications_disabled + 1 - : usage.eql.notifications_disabled, - }, - }; - } else if (detectionRuleMetric.rule_type === 'machine_learning') { - updatedUsage = { - ...usage, - machine_learning: { - ...usage.machine_learning, - enabled: detectionRuleMetric.enabled - ? usage.machine_learning.enabled + 1 - : usage.machine_learning.enabled, - disabled: !detectionRuleMetric.enabled - ? usage.machine_learning.disabled + 1 - : usage.machine_learning.disabled, - alerts: usage.machine_learning.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.machine_learning.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.machine_learning.legacy_notifications_enabled + 1 - : usage.machine_learning.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.machine_learning.legacy_notifications_disabled + 1 - : usage.machine_learning.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.machine_learning.notifications_enabled + 1 - : usage.machine_learning.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.machine_learning.notifications_disabled + 1 - : usage.machine_learning.notifications_disabled, - }, - }; - } else if (detectionRuleMetric.rule_type === 'threat_match') { - updatedUsage = { - ...usage, - threat_match: { - ...usage.threat_match, - enabled: detectionRuleMetric.enabled - ? usage.threat_match.enabled + 1 - : usage.threat_match.enabled, - disabled: !detectionRuleMetric.enabled - ? usage.threat_match.disabled + 1 - : usage.threat_match.disabled, - alerts: usage.threat_match.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.threat_match.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.threat_match.legacy_notifications_enabled + 1 - : usage.threat_match.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.threat_match.legacy_notifications_disabled + 1 - : usage.threat_match.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.threat_match.notifications_enabled + 1 - : usage.threat_match.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.threat_match.notifications_disabled + 1 - : usage.threat_match.notifications_disabled, - }, - }; - } - - if (detectionRuleMetric.elastic_rule) { - updatedUsage = { - ...updatedUsage, - elastic_total: { - ...updatedUsage.elastic_total, - enabled: detectionRuleMetric.enabled - ? updatedUsage.elastic_total.enabled + 1 - : updatedUsage.elastic_total.enabled, - disabled: !detectionRuleMetric.enabled - ? updatedUsage.elastic_total.disabled + 1 - : updatedUsage.elastic_total.disabled, - alerts: updatedUsage.elastic_total.alerts + detectionRuleMetric.alert_count_daily, - cases: updatedUsage.elastic_total.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? updatedUsage.elastic_total.legacy_notifications_enabled + 1 - : updatedUsage.elastic_total.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? updatedUsage.elastic_total.legacy_notifications_disabled + 1 - : updatedUsage.elastic_total.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? updatedUsage.elastic_total.notifications_enabled + 1 - : updatedUsage.elastic_total.notifications_enabled, - notifications_disabled: notificationDisabled - ? updatedUsage.elastic_total.notifications_disabled + 1 - : updatedUsage.elastic_total.notifications_disabled, - }, - }; - } else { - updatedUsage = { - ...updatedUsage, - custom_total: { - ...updatedUsage.custom_total, - enabled: detectionRuleMetric.enabled - ? updatedUsage.custom_total.enabled + 1 - : updatedUsage.custom_total.enabled, - disabled: !detectionRuleMetric.enabled - ? updatedUsage.custom_total.disabled + 1 - : updatedUsage.custom_total.disabled, - alerts: updatedUsage.custom_total.alerts + detectionRuleMetric.alert_count_daily, - cases: updatedUsage.custom_total.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? updatedUsage.custom_total.legacy_notifications_enabled + 1 - : updatedUsage.custom_total.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? updatedUsage.custom_total.legacy_notifications_disabled + 1 - : updatedUsage.custom_total.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? updatedUsage.custom_total.notifications_enabled + 1 - : updatedUsage.custom_total.notifications_enabled, - notifications_disabled: notificationDisabled - ? updatedUsage.custom_total.notifications_disabled + 1 - : updatedUsage.custom_total.notifications_disabled, - }, - }; - } - - return updatedUsage; -}; - -const MAX_RESULTS_WINDOW = 10_000; // elasticsearch index.max_result_window default value - -export const getDetectionRuleMetrics = async ( - kibanaIndex: string, - signalsIndex: string, - esClient: ElasticsearchClient, - savedObjectClient: SavedObjectsClientContract -): Promise => { - let rulesUsage: DetectionRulesTypeUsage = initialDetectionRulesUsage; - const ruleSearchOptions: RuleSearchParams = { - body: { - query: { - bool: { - filter: { - terms: { - 'alert.alertTypeId': [ - SIGNALS_ID, - EQL_RULE_TYPE_ID, - ML_RULE_TYPE_ID, - QUERY_RULE_TYPE_ID, - SAVED_QUERY_RULE_TYPE_ID, - INDICATOR_RULE_TYPE_ID, - THRESHOLD_RULE_TYPE_ID, - ], - }, - }, - }, - }, - }, - filter_path: [], - ignore_unavailable: true, - index: kibanaIndex, - size: MAX_RESULTS_WINDOW, - }; - - try { - const ruleResults = await esClient.search(ruleSearchOptions); - const detectionAlertsResp = (await esClient.search({ - index: `${signalsIndex}*`, - size: MAX_RESULTS_WINDOW, - body: { - aggs: { - detectionAlerts: { - terms: { field: ALERT_RULE_UUID }, - }, - }, - query: { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: 'now-24h', - lte: 'now', - }, - }, - }, - ], - }, - }, - }, - })) as AlertsAggregationResponse; - - const cases = await savedObjectClient.find({ - type: CASE_COMMENT_SAVED_OBJECT, - page: 1, - perPage: MAX_RESULTS_WINDOW, - namespaces: ['*'], - filter: `${CASE_COMMENT_SAVED_OBJECT}.attributes.type: alert`, - }); - - // Once we are confident all rules relying on side-car actions SO's have been migrated to SO references we should remove this function. - const legacyRuleActions = - await savedObjectClient.find({ - type: legacyRuleActionsSavedObjectType, - page: 1, - perPage: MAX_RESULTS_WINDOW, - namespaces: ['*'], - }); - - const legacyNotificationRuleIds = legacyRuleActions.saved_objects.reduce( - (cache, legacyNotificationsObject) => { - const ruleRef = legacyNotificationsObject.references.find( - (reference) => reference.name === 'alert_0' && reference.type === 'alert' - ); - if (ruleRef != null) { - const enabled = legacyNotificationsObject.attributes.ruleThrottle !== 'no_actions'; - cache.set(ruleRef.id, { enabled }); - } - return cache; - }, - new Map() - ); - - const casesCache = cases.saved_objects.reduce((cache, { attributes: casesObject }) => { - const ruleId = casesObject.rule.id; - if (ruleId != null) { - const cacheCount = cache.get(ruleId); - if (cacheCount === undefined) { - cache.set(ruleId, 1); - } else { - cache.set(ruleId, cacheCount + 1); - } - } - return cache; - }, new Map()); - - const alertBuckets = detectionAlertsResp.aggregations?.detectionAlerts?.buckets ?? []; - - const alertsCache = new Map(); - alertBuckets.map((bucket) => alertsCache.set(bucket.key, bucket.doc_count)); - if (ruleResults.hits?.hits?.length > 0) { - const ruleObjects = ruleResults.hits.hits.map((hit) => { - const ruleId = hit._id.split(':')[1]; - const isElastic = isElasticRule(hit._source?.alert.tags); - - // Even if the legacy notification is set to "no_actions" we still count the rule as having a legacy notification that is not migrated yet. - const hasLegacyNotification = legacyNotificationRuleIds.get(ruleId) != null; - - // We only count a rule as having a notification and being "enabled" if it is _not_ set to "no_actions"/"muteAll" and it has at least one action within its array. - const hasNotification = - !hasLegacyNotification && - hit._source?.alert.actions != null && - hit._source?.alert.actions.length > 0 && - hit._source?.alert.muteAll !== true; - - return { - rule_name: hit._source?.alert.name, - rule_id: hit._source?.alert.params.ruleId, - rule_type: hit._source?.alert.params.type, - rule_version: Number(hit._source?.alert.params.version), - enabled: hit._source?.alert.enabled, - elastic_rule: isElastic, - created_on: hit._source?.alert.createdAt, - updated_on: hit._source?.alert.updatedAt, - alert_count_daily: alertsCache.get(ruleId) || 0, - cases_count_total: casesCache.get(ruleId) || 0, - has_legacy_notification: hasLegacyNotification, - has_notification: hasNotification, - } as DetectionRuleMetric; - }); - - // Only bring back rule detail on elastic prepackaged detection rules - const elasticRuleObjects = ruleObjects.filter((hit) => hit.elastic_rule === true); - - rulesUsage = ruleObjects.reduce((usage, rule) => { - return updateDetectionRuleUsage(rule, usage); - }, rulesUsage); - - return { - detection_rule_detail: elasticRuleObjects, - detection_rule_usage: rulesUsage, - }; - } - } catch (e) { - // ignore failure, usage will be zeroed - } - - return { - detection_rule_detail: [], - detection_rule_usage: rulesUsage, - }; -}; From cdfd72f8794992b71b97e94958a7d26f696ecac7 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Sat, 12 Feb 2022 13:16:10 -0700 Subject: [PATCH 06/10] Fixes issues from merge with new ES client calls --- .../security_solution/server/usage/queries/get_alerts.ts | 7 ++++--- .../server/usage/queries/utils/fetch_hits_with_pit.ts | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts b/x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts index 1514760b0b251..4c0946242b005 100644 --- a/x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts +++ b/x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts @@ -11,6 +11,7 @@ import type { SearchRequest, } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient, Logger } from 'kibana/server'; +import { ALERT_RULE_UUID } from '@kbn/rule-data-utils'; import type { AlertBucket, AlertAggs } from '../types'; export interface GetAlertsOptions { @@ -37,7 +38,7 @@ export const getAlerts = async ({ index: signalsIndex, keep_alive: keepAlive, }) - ).body.id; + ).id; let after: AggregationsCompositeAggregation['after']; let buckets: AlertBucket[] = []; @@ -52,7 +53,7 @@ export const getAlerts = async ({ { detectionAlerts: { terms: { - field: 'kibana.alert.rule.uuid', + field: ALERT_RULE_UUID, }, }, }, @@ -83,7 +84,7 @@ export const getAlerts = async ({ logger.debug( `Getting alerts with point in time (PIT) query: ${JSON.stringify(ruleSearchOptions)}` ); - const { body } = await esClient.search(ruleSearchOptions); + const body = await esClient.search(ruleSearchOptions); if (body.aggregations?.buckets?.buckets != null) { buckets = [...buckets, ...body.aggregations.buckets.buckets]; } diff --git a/x-pack/plugins/security_solution/server/usage/queries/utils/fetch_hits_with_pit.ts b/x-pack/plugins/security_solution/server/usage/queries/utils/fetch_hits_with_pit.ts index cc4d8d98df799..2d84cfe21179d 100644 --- a/x-pack/plugins/security_solution/server/usage/queries/utils/fetch_hits_with_pit.ts +++ b/x-pack/plugins/security_solution/server/usage/queries/utils/fetch_hits_with_pit.ts @@ -39,7 +39,7 @@ export const fetchHitsWithPit = async ({ index, keep_alive: '5m', }) - ).body.id; + ).id; let searchAfter: SortResults | undefined; let hits: Array> = []; @@ -56,7 +56,7 @@ export const fetchHitsWithPit = async ({ logger.debug( `Getting hits with point in time (PIT) query of: ${JSON.stringify(ruleSearchOptions)}` ); - const { body } = await esClient.search(ruleSearchOptions); + const body = await esClient.search(ruleSearchOptions); hits = [...hits, ...body.hits.hits]; searchAfter = body.hits.hits.length !== 0 ? body.hits.hits[body.hits.hits.length - 1].sort : undefined; From b47f2994269362e25bd200cfc52d25c0bf054fde Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Mon, 14 Feb 2022 14:26:41 -0700 Subject: [PATCH 07/10] Fixes all the PR comments from the review --- .../security_solution/server/plugin.ts | 1 - .../server/usage/collector.ts | 2 - .../usage/detections/get_metrics.test.ts | 41 ++-- .../server/usage/detections/get_metrics.ts | 4 +- .../detections/ml_jobs/get_metrics.mocks.ts | 181 +++++++++--------- .../usage/detections/ml_jobs/get_metrics.ts | 10 +- .../detections/rules/get_metrics.mocks.ts | 19 +- .../usage/detections/rules/get_metrics.ts | 9 +- .../get_rule_object_correlations.ts | 34 ++-- .../server/usage/queries/get_alerts.ts | 2 +- .../server/usage/queries/get_case_comments.ts | 14 +- .../usage/queries/get_detection_rules.ts | 88 +++++---- .../usage/queries/legacy_get_rule_actions.ts | 12 +- .../queries/utils/fetch_hits_with_pit.ts | 2 +- .../security_solution/server/usage/types.ts | 36 +++- 15 files changed, 263 insertions(+), 192 deletions(-) diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 7153254a61581..511679ef71a79 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -168,7 +168,6 @@ export class Plugin implements ISecuritySolutionPlugin { initUsageCollectors({ core, - kibanaIndex: core.savedObjects.getKibanaIndex(), signalsIndex: DEFAULT_ALERTS_INDEX, ml: plugins.ml, usageCollection: plugins.usageCollection, diff --git a/x-pack/plugins/security_solution/server/usage/collector.ts b/x-pack/plugins/security_solution/server/usage/collector.ts index 9a2009405b4ef..dc98b68f9f186 100644 --- a/x-pack/plugins/security_solution/server/usage/collector.ts +++ b/x-pack/plugins/security_solution/server/usage/collector.ts @@ -18,7 +18,6 @@ export interface UsageData { export const registerCollector: RegisterCollector = ({ core, - kibanaIndex, signalsIndex, ml, usageCollection, @@ -516,7 +515,6 @@ export const registerCollector: RegisterCollector = ({ fetch: async ({ esClient }: CollectorFetchContext): Promise => { const savedObjectsClient = await getInternalSavedObjectsClient(core); const detectionMetrics = await getDetectionsMetrics({ - kibanaIndex, signalsIndex, esClient, savedObjectsClient, diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_metrics.test.ts b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.test.ts index ef869e4269c7a..65929039bc104 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/get_metrics.test.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.test.ts @@ -21,7 +21,11 @@ import { getMockMlDatafeedStatsResponse, getMockRuleSearchResponse, } from './ml_jobs/get_metrics.mocks'; -import { getMockRuleAlertsResponse, getMockAlertCasesResponse } from './rules/get_metrics.mocks'; +import { + getMockRuleAlertsResponse, + getMockAlertCaseCommentsResponse, + getEmptySavedObjectResponse, +} from './rules/get_metrics.mocks'; import { getInitialDetectionMetrics } from './get_initial_usage'; import { getDetectionsMetrics } from './get_metrics'; import { getInitialRulesUsage } from './rules/get_initial_usage'; @@ -41,7 +45,6 @@ describe('Detections Usage and Metrics', () => { it('returns zeroed counts if calls are empty', async () => { const logger = loggingSystemMock.createLogger(); const result = await getDetectionsMetrics({ - kibanaIndex: '', signalsIndex: '', esClient, savedObjectsClient, @@ -52,13 +55,13 @@ describe('Detections Usage and Metrics', () => { }); it('returns information with rule, alerts and cases', async () => { - esClient.search - .mockResponseOnce(getMockRuleSearchResponse()) - .mockResponseOnce(getMockRuleAlertsResponse(3400)); - savedObjectsClient.find.mockResolvedValue(getMockAlertCasesResponse()); + esClient.search.mockResponseOnce(getMockRuleAlertsResponse(3400)); + savedObjectsClient.find.mockResolvedValueOnce(getMockRuleSearchResponse()); + savedObjectsClient.find.mockResolvedValueOnce(getMockAlertCaseCommentsResponse()); + // Get empty saved object for legacy notification system. + savedObjectsClient.find.mockResolvedValueOnce(getEmptySavedObjectResponse()); const logger = loggingSystemMock.createLogger(); const result = await getDetectionsMetrics({ - kibanaIndex: '', signalsIndex: '', esClient, savedObjectsClient, @@ -113,13 +116,13 @@ describe('Detections Usage and Metrics', () => { }); it('returns information with on non elastic prebuilt rule', async () => { - esClient.search - .mockResponseOnce(getMockRuleSearchResponse('not_immutable')) - .mockResponseOnce(getMockRuleAlertsResponse(800)); - savedObjectsClient.find.mockResolvedValue(getMockAlertCasesResponse()); + esClient.search.mockResponseOnce(getMockRuleAlertsResponse(800)); + savedObjectsClient.find.mockResolvedValueOnce(getMockRuleSearchResponse('not_immutable')); + savedObjectsClient.find.mockResolvedValueOnce(getMockAlertCaseCommentsResponse()); + // Get empty saved object for legacy notification system. + savedObjectsClient.find.mockResolvedValueOnce(getEmptySavedObjectResponse()); const logger = loggingSystemMock.createLogger(); const result = await getDetectionsMetrics({ - kibanaIndex: '', signalsIndex: '', esClient, savedObjectsClient, @@ -159,13 +162,14 @@ describe('Detections Usage and Metrics', () => { }); it('returns information with rule, no alerts and no cases', async () => { - esClient.search - .mockResponseOnce(getMockRuleSearchResponse()) - .mockResponseOnce(getMockRuleAlertsResponse(0)); - savedObjectsClient.find.mockResolvedValue(getMockAlertCasesResponse()); + esClient.search.mockResponseOnce(getMockRuleAlertsResponse(0)); + savedObjectsClient.find.mockResolvedValueOnce(getMockRuleSearchResponse()); + savedObjectsClient.find.mockResolvedValueOnce(getMockAlertCaseCommentsResponse()); + // Get empty saved object for legacy notification system. + savedObjectsClient.find.mockResolvedValueOnce(getEmptySavedObjectResponse()); + const logger = loggingSystemMock.createLogger(); const result = await getDetectionsMetrics({ - kibanaIndex: '', signalsIndex: '', esClient, savedObjectsClient, @@ -225,6 +229,7 @@ describe('Detections Usage and Metrics', () => { esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; mlClient = mlServicesMock.createSetupContract(); savedObjectsClient = savedObjectsClientMock.create(); + savedObjectsClient.find.mockResolvedValue(getEmptySavedObjectResponse()); }); it('returns an empty array if there is no data', async () => { @@ -234,7 +239,6 @@ describe('Detections Usage and Metrics', () => { } as unknown as ReturnType); const logger = loggingSystemMock.createLogger(); const result = await getDetectionsMetrics({ - kibanaIndex: '', signalsIndex: '', esClient, savedObjectsClient, @@ -267,7 +271,6 @@ describe('Detections Usage and Metrics', () => { } as unknown as ReturnType); const result = await getDetectionsMetrics({ - kibanaIndex: '', signalsIndex: '', esClient, savedObjectsClient, diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts index e2abd99fb1812..258945fba662a 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts @@ -15,7 +15,6 @@ import { getInitialRulesUsage } from './rules/get_initial_usage'; import { getInitialMlJobUsage } from './ml_jobs/get_initial_usage'; export interface GetDetectionsMetricsOptions { - kibanaIndex: string; signalsIndex: string; esClient: ElasticsearchClient; savedObjectsClient: SavedObjectsClientContract; @@ -24,7 +23,6 @@ export interface GetDetectionsMetricsOptions { } export const getDetectionsMetrics = async ({ - kibanaIndex, signalsIndex, esClient, savedObjectsClient, @@ -33,7 +31,7 @@ export const getDetectionsMetrics = async ({ }: GetDetectionsMetricsOptions): Promise => { const [mlJobMetrics, detectionRuleMetrics] = await Promise.allSettled([ getMlJobMetrics({ mlClient, savedObjectsClient, logger }), - getRuleMetrics({ kibanaIndex, signalsIndex, esClient, savedObjectsClient, logger }), + getRuleMetrics({ signalsIndex, esClient, savedObjectsClient, logger }), ]); return { diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.mocks.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.mocks.ts index e7870f35a35bc..a507a76e0c4f2 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.mocks.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.mocks.ts @@ -5,7 +5,8 @@ * 2.0. */ -import type { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; +import type { SavedObjectsFindResponse } from 'kibana/server'; +import type { RuleSearchResult } from '../../types'; export const getMockListModulesResponse = () => [ { @@ -291,102 +292,100 @@ export const getMockMlDatafeedStatsResponse = () => ({ export const getMockRuleSearchResponse = ( immutableTag: string = '__internal_immutable:true' -): SearchResponse => ({ - took: 2, - timed_out: false, - _shards: { +): SavedObjectsFindResponse => + ({ + page: 1, + per_page: 1_000, total: 1, - successful: 1, - skipped: 0, - failed: 0, - }, - hits: { - total: { - value: 1093, - relation: 'eq', - }, - max_score: 0, - hits: [ + saved_objects: [ { - _index: '.kibanaindex', - _id: 'alert:6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', - _score: 0, - _source: { - alert: { - name: 'Azure Diagnostic Settings Deletion', - tags: [ - 'Elastic', - 'Cloud', - 'Azure', - 'Continuous Monitoring', - 'SecOps', - 'Monitoring', - '__internal_rule_id:5370d4cd-2bb3-4d71-abf5-1e1d0ff5a2de', - `${immutableTag}`, + type: 'alert', + id: '6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', + namespaces: ['default'], + attributes: { + name: 'Azure Diagnostic Settings Deletion', + tags: [ + 'Elastic', + 'Cloud', + 'Azure', + 'Continuous Monitoring', + 'SecOps', + 'Monitoring', + '__internal_rule_id:5370d4cd-2bb3-4d71-abf5-1e1d0ff5a2de', + `${immutableTag}`, + ], + alertTypeId: 'siem.queryRule', + consumer: 'siem', + params: { + author: ['Elastic'], + description: + 'Identifies the deletion of diagnostic settings in Azure, which send platform logs and metrics to different destinations. An adversary may delete diagnostic settings in an attempt to evade defenses.', + ruleId: '5370d4cd-2bb3-4d71-abf5-1e1d0ff5a2de', + index: ['filebeat-*', 'logs-azure*'], + falsePositives: [ + 'Deletion of diagnostic settings may be done by a system or network administrator. Verify whether the username, hostname, and/or resource name should be making changes in your environment. Diagnostic settings deletion from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule.', ], - alertTypeId: 'siem.signals', - consumer: 'siem', - params: { - author: ['Elastic'], - description: - 'Identifies the deletion of diagnostic settings in Azure, which send platform logs and metrics to different destinations. An adversary may delete diagnostic settings in an attempt to evade defenses.', - ruleId: '5370d4cd-2bb3-4d71-abf5-1e1d0ff5a2de', - index: ['filebeat-*', 'logs-azure*'], - falsePositives: [ - 'Deletion of diagnostic settings may be done by a system or network administrator. Verify whether the username, hostname, and/or resource name should be making changes in your environment. Diagnostic settings deletion from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule.', - ], - from: 'now-25m', - immutable: true, - query: - 'event.dataset:azure.activitylogs and azure.activitylogs.operation_name:"MICROSOFT.INSIGHTS/DIAGNOSTICSETTINGS/DELETE" and event.outcome:(Success or success)', - language: 'kuery', - license: 'Elastic License v2', - outputIndex: '.siem-signals', - maxSignals: 100, - riskScore: 47, - timestampOverride: 'event.ingested', - to: 'now', - type: 'query', - references: [ - 'https://docs.microsoft.com/en-us/azure/azure-monitor/platform/diagnostic-settings', - ], - note: 'The Azure Filebeat module must be enabled to use this rule.', - version: 4, - exceptionsList: [], - }, - schedule: { - interval: '5m', - }, - enabled: false, - actions: [], - throttle: null, - notifyWhen: 'onActiveAlert', - apiKeyOwner: null, - apiKey: null, - createdBy: 'user', - updatedBy: 'user', - createdAt: '2021-03-23T17:15:59.634Z', - updatedAt: '2021-03-23T17:15:59.634Z', - muteAll: false, - mutedInstanceIds: [], - executionStatus: { - status: 'pending', - lastExecutionDate: '2021-03-23T17:15:59.634Z', - error: null, - }, - meta: { - versionApiKeyLastmodified: '8.0.0', + from: 'now-25m', + immutable: true, + query: + 'event.dataset:azure.activitylogs and azure.activitylogs.operation_name:"MICROSOFT.INSIGHTS/DIAGNOSTICSETTINGS/DELETE" and event.outcome:(Success or success)', + language: 'kuery', + license: 'Elastic License v2', + outputIndex: '.siem-signals', + maxSignals: 100, + riskScore: 47, + timestampOverride: 'event.ingested', + to: 'now', + type: 'query', + references: [ + 'https://docs.microsoft.com/en-us/azure/azure-monitor/platform/diagnostic-settings', + ], + note: 'The Azure Filebeat module must be enabled to use this rule.', + version: 4, + exceptionsList: [], + }, + schedule: { + interval: '5m', + }, + enabled: false, + actions: [], + throttle: null, + notifyWhen: 'onActiveAlert', + apiKeyOwner: null, + apiKey: '', + legacyId: null, + createdBy: 'user', + updatedBy: 'user', + createdAt: '2021-03-23T17:15:59.634Z', + updatedAt: '2021-03-23T17:15:59.634Z', + muteAll: true, + mutedInstanceIds: [], + monitoring: { + execution: { + history: [], + calculated_metrics: { + success_ratio: 1, + p99: 7981, + p50: 1653, + p95: 6523.699999999996, + }, }, }, - type: 'alert', - references: [], - migrationVersion: { - alert: '7.13.0', + meta: { + versionApiKeyLastmodified: '8.2.0', }, - coreMigrationVersion: '8.0.0', - updated_at: '2021-03-23T17:15:59.634Z', + scheduledTaskId: '6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', }, + references: [], + migrationVersion: { + alert: '8.0.0', + }, + coreMigrationVersion: '8.2.0', + updated_at: '2021-03-23T17:15:59.634Z', + version: 'Wzk4NTQwLDNd', + score: 0, + sort: ['1644865254209', '19548'], }, ], - }, -}); + // NOTE: We have to cast as "unknown" and then back to "RuleSearchResult" because "RuleSearchResult" isn't an exact type. See notes in the JSDocs fo that type. + } as unknown as SavedObjectsFindResponse); diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts index 61b4e8bc35b21..85c058a31aa6e 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts @@ -5,11 +5,7 @@ * 2.0. */ -import type { - KibanaRequest, - SavedObjectsClientContract, - Logger, -} from '../../../../../../../src/core/server'; +import type { KibanaRequest, SavedObjectsClientContract, Logger } from 'kibana/server'; import type { MlDatafeedStats, MlJob, MlPluginSetup } from '../../../../../ml/server'; import type { MlJobMetric, MlJobUsageMetric } from './types'; @@ -91,8 +87,8 @@ export const getMlJobMetrics = async ({ }; } catch (e) { // ignore failure, usage will be zeroed - logger.error( - `Encountered error in telemetry of message: ${e.message}, error: ${e}. Telemetry for "ml_jobs" will be skipped.` + logger.info( + `Encountered exception in telemetry of message: ${e.message}, error: ${e}. Telemetry for "ml_jobs" will be skipped.` ); return { ml_job_usage: getInitialMlJobUsage(), diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.mocks.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.mocks.ts index cfdc6b0b992e7..1801d5bd67782 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.mocks.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.mocks.ts @@ -6,8 +6,11 @@ */ import type { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; +import type { SavedObjectsFindResponse } from 'kibana/server'; +import type { AlertAggs } from '../../types'; +import { CommentAttributes, CommentType } from '../../../../../cases/common/api/cases/comment'; -export const getMockRuleAlertsResponse = (docCount: number): SearchResponse => ({ +export const getMockRuleAlertsResponse = (docCount: number): SearchResponse => ({ took: 7, timed_out: false, _shards: { @@ -41,7 +44,10 @@ export const getMockRuleAlertsResponse = (docCount: number): SearchResponse ({ +export const getMockAlertCaseCommentsResponse = (): SavedObjectsFindResponse< + Partial, + never +> => ({ page: 1, per_page: 10000, total: 4, @@ -50,7 +56,7 @@ export const getMockAlertCasesResponse = () => ({ type: 'cases-comments', id: '3bb5cc10-9249-11eb-85b7-254c8af1a983', attributes: { - type: 'alert', + type: CommentType.alert, alertId: '54802763917f521249c9f68d0d4be0c26cc538404c26dfed1ae7dcfa94ea2226', index: '.siem-signals-default-000001', rule: { @@ -84,3 +90,10 @@ export const getMockAlertCasesResponse = () => ({ }, ], }); + +export const getEmptySavedObjectResponse = (): SavedObjectsFindResponse => ({ + page: 1, + per_page: 1_000, + total: 0, + saved_objects: [], +}); diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts index f003f3dac6f3d..f38e6dc79a91b 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts @@ -23,7 +23,6 @@ import { getRuleObjectCorrelations } from './transform_utils/get_rule_object_cor import { legacyGetRuleActions } from '../../queries/legacy_get_rule_actions'; export interface GetRuleMetricsOptions { - kibanaIndex: string; signalsIndex: string; esClient: ElasticsearchClient; savedObjectsClient: SavedObjectsClientContract; @@ -31,7 +30,6 @@ export interface GetRuleMetricsOptions { } export const getRuleMetrics = async ({ - kibanaIndex, signalsIndex, esClient, savedObjectsClient, @@ -40,8 +38,7 @@ export const getRuleMetrics = async ({ try { // gets rule saved objects const ruleResults = await getDetectionRules({ - esClient, - kibanaIndex, + savedObjectsClient, maxPerPage: MAX_PER_PAGE, maxSize: MAX_RESULTS_WINDOW, logger, @@ -114,8 +111,8 @@ export const getRuleMetrics = async ({ }; } catch (e) { // ignore failure, usage will be zeroed - logger.error( - `Encountered error in telemetry of message: ${e.message}, error: ${e}. Telemetry for "detection rules" being skipped.` + logger.info( + `Encountered exception in telemetry of message: ${e.message}, error: ${e}. Telemetry for "detection rules" being skipped.` ); return { detection_rule_detail: [], diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_object_correlations.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_object_correlations.ts index 9fb1bca0ed568..0c364efe73bd9 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_object_correlations.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_object_correlations.ts @@ -5,13 +5,14 @@ * 2.0. */ -import { SearchHit } from '@elastic/elasticsearch/lib/api/types'; -import type { RuleSearchResult } from '../../../types'; +import type { SavedObjectsFindResult } from 'kibana/server'; import type { RuleMetric } from '../types'; +import type { RuleSearchResult } from '../../../types'; + import { isElasticRule } from '../../../queries/utils/is_elastic_rule'; export interface RuleObjectCorrelationsOptions { - ruleResults: Array>; + ruleResults: Array>; legacyNotificationRuleIds: Map< string, { @@ -28,9 +29,10 @@ export const getRuleObjectCorrelations = ({ casesRuleIds, alertsCounts, }: RuleObjectCorrelationsOptions): RuleMetric[] => { - return ruleResults.map((hit) => { - const ruleId = hit._id.split(':')[1]; - const isElastic = isElasticRule(hit._source?.alert.tags); + return ruleResults.map((result) => { + const ruleId = result.id; + const { attributes } = result; + const isElastic = isElasticRule(attributes.tags); // Even if the legacy notification is set to "no_actions" we still count the rule as having a legacy notification that is not migrated yet. const hasLegacyNotification = legacyNotificationRuleIds.get(ruleId) != null; @@ -38,19 +40,19 @@ export const getRuleObjectCorrelations = ({ // We only count a rule as having a notification and being "enabled" if it is _not_ set to "no_actions"/"muteAll" and it has at least one action within its array. const hasNotification = !hasLegacyNotification && - hit._source?.alert.actions != null && - hit._source?.alert.actions.length > 0 && - hit._source?.alert.muteAll !== true; + attributes.actions != null && + attributes.actions.length > 0 && + attributes.muteAll !== true; return { - rule_name: String(hit._source?.alert.name), - rule_id: String(hit._source?.alert.params.ruleId), - rule_type: String(hit._source?.alert.params.type), - rule_version: Number(hit._source?.alert.params.version), - enabled: Boolean(hit._source?.alert.enabled), + rule_name: attributes.name, + rule_id: attributes.params.ruleId, + rule_type: attributes.params.type, + rule_version: attributes.params.version, + enabled: attributes.enabled, elastic_rule: isElastic, - created_on: String(hit._source?.alert.createdAt), - updated_on: String(hit._source?.alert.updatedAt), + created_on: attributes.createdAt, + updated_on: attributes.updatedAt, alert_count_daily: alertsCounts.get(ruleId) || 0, cases_count_total: casesRuleIds.get(ruleId) || 0, has_legacy_notification: hasLegacyNotification, diff --git a/x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts b/x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts index 4c0946242b005..792ca28dcfba3 100644 --- a/x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts +++ b/x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts @@ -77,7 +77,7 @@ export const getAlerts = async ({ }, }, track_total_hits: false, - sort: [{ _shard_doc: 'desc' }] as unknown as string[], // Remove this "unknown" once it is typed correctly https://github.com/elastic/elasticsearch-js/issues/1589 + sort: [{ _shard_doc: 'desc' }] as unknown as string[], // TODO: Remove this "unknown" once it is typed correctly https://github.com/elastic/elasticsearch-js/issues/1589 pit: { id: pitId }, size: 0, }; diff --git a/x-pack/plugins/security_solution/server/usage/queries/get_case_comments.ts b/x-pack/plugins/security_solution/server/usage/queries/get_case_comments.ts index 9e8ceed9fa6d8..0a6c7f2fc209a 100644 --- a/x-pack/plugins/security_solution/server/usage/queries/get_case_comments.ts +++ b/x-pack/plugins/security_solution/server/usage/queries/get_case_comments.ts @@ -40,11 +40,23 @@ export const getCaseComments = async ({ for await (const response of finder.find()) { const extra = responses.length + response.saved_objects.length - maxSize; if (extra > 0) { - responses = [...responses, ...response.saved_objects.splice(extra)]; + responses = [ + ...responses, + ...response.saved_objects.slice(-response.saved_objects.length, -extra), + ]; } else { responses = [...responses, ...response.saved_objects]; } } + + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } + logger.debug(`Returning cases response of length: "${responses.length}"`); return responses; }; diff --git a/x-pack/plugins/security_solution/server/usage/queries/get_detection_rules.ts b/x-pack/plugins/security_solution/server/usage/queries/get_detection_rules.ts index 8ae8657c4e67d..864e946c33e47 100644 --- a/x-pack/plugins/security_solution/server/usage/queries/get_detection_rules.ts +++ b/x-pack/plugins/security_solution/server/usage/queries/get_detection_rules.ts @@ -5,9 +5,12 @@ * 2.0. */ -import type { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { ElasticsearchClient, Logger } from 'kibana/server'; - +import type { + Logger, + SavedObjectsCreatePointInTimeFinderOptions, + SavedObjectsClientContract, + SavedObjectsFindResult, +} from 'kibana/server'; import { SIGNALS_ID, EQL_RULE_TYPE_ID, @@ -17,48 +20,61 @@ import { THRESHOLD_RULE_TYPE_ID, SAVED_QUERY_RULE_TYPE_ID, } from '@kbn/securitysolution-rules'; -import { fetchHitsWithPit } from './utils/fetch_hits_with_pit'; -import { RuleSearchResult } from '../types'; +import type { RuleSearchResult } from '../types'; export interface GetDetectionRulesOptions { - esClient: ElasticsearchClient; - kibanaIndex: string; maxSize: number; maxPerPage: number; logger: Logger; + savedObjectsClient: SavedObjectsClientContract; } export const getDetectionRules = async ({ - esClient, - kibanaIndex, maxSize, maxPerPage, logger, -}: GetDetectionRulesOptions): Promise>> => { - return fetchHitsWithPit({ - logger, - esClient, - index: kibanaIndex, - maxSize, - maxPerPage, - searchRequest: { - query: { - bool: { - filter: { - terms: { - 'alert.alertTypeId': [ - SIGNALS_ID, - EQL_RULE_TYPE_ID, - ML_RULE_TYPE_ID, - QUERY_RULE_TYPE_ID, - SAVED_QUERY_RULE_TYPE_ID, - INDICATOR_RULE_TYPE_ID, - THRESHOLD_RULE_TYPE_ID, - ], - }, - }, - }, - }, - }, - }); + savedObjectsClient, +}: GetDetectionRulesOptions): Promise>> => { + const filterAttribute = 'alert.attributes.alertTypeId'; + const filter = [ + `${filterAttribute}: ${SIGNALS_ID}`, + `${filterAttribute}: ${EQL_RULE_TYPE_ID}`, + `${filterAttribute}: ${ML_RULE_TYPE_ID}`, + `${filterAttribute}: ${QUERY_RULE_TYPE_ID}`, + `${filterAttribute}: ${SAVED_QUERY_RULE_TYPE_ID}`, + `${filterAttribute}: ${THRESHOLD_RULE_TYPE_ID}`, + `${filterAttribute}: ${INDICATOR_RULE_TYPE_ID}`, + ].join(' OR '); + + const query: SavedObjectsCreatePointInTimeFinderOptions = { + type: 'alert', + perPage: maxPerPage, + namespaces: ['*'], + filter, + }; + logger.debug( + `Getting detection rules with point in time (PIT) query:', ${JSON.stringify(query)}` + ); + const finder = savedObjectsClient.createPointInTimeFinder(query); + let responses: Array> = []; + for await (const response of finder.find()) { + const extra = responses.length + response.saved_objects.length - maxSize; + if (extra > 0) { + responses = [ + ...responses, + ...response.saved_objects.slice(-response.saved_objects.length, -extra), + ]; + } else { + responses = [...responses, ...response.saved_objects]; + } + } + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } + logger.debug(`Returning cases response of length: "${responses.length}"`); + return responses; }; diff --git a/x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts b/x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts index 9ed8c6a47a0c5..e7864c8340dd3 100644 --- a/x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts +++ b/x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts @@ -51,11 +51,21 @@ export const legacyGetRuleActions = async ({ for await (const response of finder.find()) { const extra = responses.length + response.saved_objects.length - maxSize; if (extra > 0) { - responses = [...responses, ...response.saved_objects.splice(extra)]; + responses = [ + ...responses, + ...response.saved_objects.slice(-response.saved_objects.length, -extra), + ]; } else { responses = [...responses, ...response.saved_objects]; } } logger.debug(`Returning legacy rule actions response of length: "${responses.length}"`); + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } return responses; }; diff --git a/x-pack/plugins/security_solution/server/usage/queries/utils/fetch_hits_with_pit.ts b/x-pack/plugins/security_solution/server/usage/queries/utils/fetch_hits_with_pit.ts index 2d84cfe21179d..34dc545f9b8bf 100644 --- a/x-pack/plugins/security_solution/server/usage/queries/utils/fetch_hits_with_pit.ts +++ b/x-pack/plugins/security_solution/server/usage/queries/utils/fetch_hits_with_pit.ts @@ -49,7 +49,7 @@ export const fetchHitsWithPit = async ({ ...searchRequest, track_total_hits: false, search_after: searchAfter, - sort: [{ _shard_doc: 'desc' }] as unknown as string[], // Remove this "unknown" once it is typed correctly https://github.com/elastic/elasticsearch-js/issues/1589 + sort: [{ _shard_doc: 'desc' }] as unknown as string[], // TODO: Remove this "unknown" once it is typed correctly https://github.com/elastic/elasticsearch-js/issues/1589 pit: { id: pitId }, size: Math.min(maxPerPage, maxSize - hits.length), }; diff --git a/x-pack/plugins/security_solution/server/usage/types.ts b/x-pack/plugins/security_solution/server/usage/types.ts index bb05c88a0d94e..3f3c32c5d054b 100644 --- a/x-pack/plugins/security_solution/server/usage/types.ts +++ b/x-pack/plugins/security_solution/server/usage/types.ts @@ -11,7 +11,6 @@ import type { RuleParams } from '../lib/detection_engine/schemas/rule_schemas'; import type { SetupPlugins } from '../plugin'; export type CollectorDependencies = { - kibanaIndex: string; signalsIndex: string; core: CoreSetup; logger: Logger; @@ -33,6 +32,35 @@ export interface AlertAggs { }; } -export interface RuleSearchResult { - alert: SanitizedAlert; -} +/** + * This type is _very_ similar to "RawRule". However, that type is not exposed in a non-restricted-path + * and it does not support generics well. Trying to use "RawRule" directly with TypeScript Omit does not work well. + * If at some point the rules client API supports cross spaces for gathering metrics, then we should remove our use + * of SavedObject types and this type below and instead natively use the rules client. + * + * NOTE: There are additional types not expressed below such as "apiKey" or there could be other slight differences + * but this will the easiest way to keep these in sync and I see other code that is similar to this pattern. + * {@see RawRule} + */ +export type RuleSearchResult = Omit< + SanitizedAlert, + 'createdBy' | 'updatedBy' | 'createdAt' | 'updatedAt' +> & { + createdBy: string | null; + updatedBy: string | null; + createdAt: string; + updatedAt: string; +}; + +/** + * Same as "RuleSearchResult" just a partial is applied. Useful for unit tests since these types aren't exact. + */ +export type PartialRuleSearchResult = Omit< + SanitizedAlert>, + 'createdBy' | 'updatedBy' | 'createdAt' | 'updatedAt' +> & { + createdBy: string | null; + updatedBy: string | null; + createdAt: string; + updatedAt: string; +}; From 62173ef1a1f267d577a14689843f1f03e0cfeacc Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Mon, 14 Feb 2022 14:37:05 -0700 Subject: [PATCH 08/10] Small white spacing change --- .../server/usage/queries/get_detection_rules.ts | 2 ++ .../server/usage/queries/legacy_get_rule_actions.ts | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/usage/queries/get_detection_rules.ts b/x-pack/plugins/security_solution/server/usage/queries/get_detection_rules.ts index 864e946c33e47..62f5691f73d07 100644 --- a/x-pack/plugins/security_solution/server/usage/queries/get_detection_rules.ts +++ b/x-pack/plugins/security_solution/server/usage/queries/get_detection_rules.ts @@ -68,6 +68,7 @@ export const getDetectionRules = async ({ responses = [...responses, ...response.saved_objects]; } } + try { finder.close(); } catch (exception) { @@ -75,6 +76,7 @@ export const getDetectionRules = async ({ // the response. We have seen this within e2e test containers but nothing happen in normal // operational conditions which is why this try/catch is here. } + logger.debug(`Returning cases response of length: "${responses.length}"`); return responses; }; diff --git a/x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts b/x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts index e7864c8340dd3..6d720bef7d822 100644 --- a/x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts +++ b/x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts @@ -59,7 +59,7 @@ export const legacyGetRuleActions = async ({ responses = [...responses, ...response.saved_objects]; } } - logger.debug(`Returning legacy rule actions response of length: "${responses.length}"`); + try { finder.close(); } catch (exception) { @@ -67,5 +67,7 @@ export const legacyGetRuleActions = async ({ // the response. We have seen this within e2e test containers but nothing happen in normal // operational conditions which is why this try/catch is here. } + + logger.debug(`Returning legacy rule actions response of length: "${responses.length}"`); return responses; }; From 5778b9764e8d7bd021e972e1da9897155a047812 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Mon, 14 Feb 2022 14:47:06 -0700 Subject: [PATCH 09/10] Removed dead type --- .../plugins/security_solution/server/usage/types.ts | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/x-pack/plugins/security_solution/server/usage/types.ts b/x-pack/plugins/security_solution/server/usage/types.ts index 3f3c32c5d054b..f591ffd8f422e 100644 --- a/x-pack/plugins/security_solution/server/usage/types.ts +++ b/x-pack/plugins/security_solution/server/usage/types.ts @@ -51,16 +51,3 @@ export type RuleSearchResult = Omit< createdAt: string; updatedAt: string; }; - -/** - * Same as "RuleSearchResult" just a partial is applied. Useful for unit tests since these types aren't exact. - */ -export type PartialRuleSearchResult = Omit< - SanitizedAlert>, - 'createdBy' | 'updatedBy' | 'createdAt' | 'updatedAt' -> & { - createdBy: string | null; - updatedBy: string | null; - createdAt: string; - updatedAt: string; -}; From 8ea390885aa23d1453494cfb51a7db0980cacd6b Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Tue, 15 Feb 2022 09:11:00 -0700 Subject: [PATCH 10/10] Updates commits with better error handling and debug --- .../server/usage/detections/ml_jobs/get_metrics.ts | 10 ++++++---- .../server/usage/detections/rules/get_metrics.ts | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts index 85c058a31aa6e..2eea42f28d953 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts @@ -86,10 +86,12 @@ export const getMlJobMetrics = async ({ ml_job_metrics: jobMetrics, }; } catch (e) { - // ignore failure, usage will be zeroed - logger.info( - `Encountered exception in telemetry of message: ${e.message}, error: ${e}. Telemetry for "ml_jobs" will be skipped.` - ); + // ignore failure, usage will be zeroed. We don't log the message below as currently ML jobs when it does + // not have a "security" job will cause a throw. If this does not normally throw eventually on normal operations + // we should log a debug message like the following below to not unnecessarily worry users as this will not effect them: + // logger.debug( + // `Encountered unexpected condition in telemetry of message: ${e.message}, object: ${e}. Telemetry for "ml_jobs" will be skipped.` + // ); return { ml_job_usage: getInitialMlJobUsage(), ml_job_metrics: [], diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts index f38e6dc79a91b..b202ea964301c 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts @@ -110,9 +110,9 @@ export const getRuleMetrics = async ({ detection_rule_usage: rulesUsage, }; } catch (e) { - // ignore failure, usage will be zeroed - logger.info( - `Encountered exception in telemetry of message: ${e.message}, error: ${e}. Telemetry for "detection rules" being skipped.` + // ignore failure, usage will be zeroed. We use debug mode to not unnecessarily worry users as this will not effect them. + logger.debug( + `Encountered unexpected condition in telemetry of message: ${e.message}, object: ${e}. Telemetry for "detection rules" being skipped.` ); return { detection_rule_detail: [],