From 775b7de5e0816c8d814e03d1120054d02c1c5951 Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Wed, 23 Sep 2020 07:27:25 -0600 Subject: [PATCH] [Security Solution][Detection Engine] Bubbles up more error messages from ES queries to the UI (#78004) (#78244) ## Summary Fixes: https://github.com/elastic/kibana/issues/77254 Bubbles up error messages from ES queries that have _shards.failures in them. For example if you have errors in your exceptions list you will need to see them bubbled up. Steps to reproduce: Go to a detections rule and add an invalid value within the exceptions such as this one below: Screen Shot 2020-09-21 at 7 52 59 AM Notice that rsa.internal.level value is not a numeric but a text string. You should now see this error message where before you could not: Screen Shot 2020-09-21 at 7 52 44 AM ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../signals/__mocks__/es_results.ts | 4 +- .../signals/find_ml_signals.ts | 8 +- .../signals/find_threshold_signals.ts | 1 + .../signals/search_after_bulk_create.ts | 149 ++++----- .../signals/signal_rule_alert_type.test.ts | 26 +- .../signals/signal_rule_alert_type.ts | 58 ++-- .../signals/single_search_after.test.ts | 74 ++++- .../signals/single_search_after.ts | 12 +- .../threat_mapping/create_threat_signal.ts | 6 +- .../threat_mapping/create_threat_signals.ts | 2 +- .../signals/threat_mapping/types.ts | 2 +- .../signals/threat_mapping/utils.test.ts | 2 +- .../signals/threat_mapping/utils.ts | 2 +- .../lib/detection_engine/signals/types.ts | 50 ++- .../detection_engine/signals/utils.test.ts | 286 +++++++++++++++++- .../lib/detection_engine/signals/utils.ts | 103 ++++++- .../security_solution/server/lib/types.ts | 17 ++ 17 files changed, 647 insertions(+), 155 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index bbdb8ea0a36ed..9ee8c5cf298a1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -337,7 +337,7 @@ export const repeatedSearchResultsWithSortId = ( guids: string[], ips?: string[], destIps?: string[] -) => ({ +): SignalSearchResponse => ({ took: 10, timed_out: false, _shards: { @@ -364,7 +364,7 @@ export const repeatedSearchResultsWithNoSortId = ( pageSize: number, guids: string[], ips?: string[] -) => ({ +): SignalSearchResponse => ({ took: 10, timed_out: false, _shards: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_ml_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_ml_signals.ts index bd9bf50688b58..89e3d28f451e4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_ml_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_ml_signals.ts @@ -8,7 +8,7 @@ import dateMath from '@elastic/datemath'; import { KibanaRequest } from '../../../../../../../src/core/server'; import { MlPluginSetup } from '../../../../../ml/server'; -import { getAnomalies } from '../../machine_learning'; +import { AnomalyResults, getAnomalies } from '../../machine_learning'; export const findMlSignals = async ({ ml, @@ -24,7 +24,7 @@ export const findMlSignals = async ({ anomalyThreshold: number; from: string; to: string; -}) => { +}): Promise => { const { mlAnomalySearch } = ml.mlSystemProvider(request); const params = { jobIds: [jobId], @@ -32,7 +32,5 @@ export const findMlSignals = async ({ earliestMs: dateMath.parse(from)?.valueOf() ?? 0, latestMs: dateMath.parse(to)?.valueOf() ?? 0, }; - const relevantAnomalies = await getAnomalies(params, mlAnomalySearch); - - return relevantAnomalies; + return getAnomalies(params, mlAnomalySearch); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index 251c043adb58b..604b452174045 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -34,6 +34,7 @@ export const findThresholdSignals = async ({ }: FindThresholdSignalsParams): Promise<{ searchResult: SignalSearchResponse; searchDuration: string; + searchErrors: string[]; }> => { const aggregations = threshold && !isEmpty(threshold.field) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index 756aedd5273d3..d369a91335347 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -3,56 +3,18 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import moment from 'moment'; -import { AlertServices } from '../../../../../alerts/server'; -import { ListClient } from '../../../../../lists/server'; -import { RuleAlertAction } from '../../../../common/detection_engine/types'; -import { RuleTypeParams, RefreshTypes } from '../types'; -import { Logger } from '../../../../../../../src/core/server'; import { singleSearchAfter } from './single_search_after'; import { singleBulkCreate } from './single_bulk_create'; -import { BuildRuleMessage } from './rule_messages'; -import { SignalSearchResponse } from './types'; import { filterEventsAgainstList } from './filter_events_with_list'; -import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; -import { getSignalTimeTuples } from './utils'; - -interface SearchAfterAndBulkCreateParams { - gap: moment.Duration | null; - previousStartedAt: Date | null | undefined; - ruleParams: RuleTypeParams; - services: AlertServices; - listClient: ListClient; - exceptionsList: ExceptionListItemSchema[]; - logger: Logger; - id: string; - inputIndexPattern: string[]; - signalsIndex: string; - name: string; - actions: RuleAlertAction[]; - createdAt: string; - createdBy: string; - updatedBy: string; - updatedAt: string; - interval: string; - enabled: boolean; - pageSize: number; - filter: unknown; - refresh: RefreshTypes; - tags: string[]; - throttle: string; - buildRuleMessage: BuildRuleMessage; -} - -export interface SearchAfterAndBulkCreateReturnType { - success: boolean; - searchAfterTimes: string[]; - bulkCreateTimes: string[]; - lastLookBackDate: Date | null | undefined; - createdSignalsCount: number; - errors: string[]; -} +import { + createSearchAfterReturnType, + createSearchAfterReturnTypeFromResponse, + createTotalHitsFromSearchResult, + getSignalTimeTuples, + mergeReturns, +} from './utils'; +import { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType } from './types'; // search_after through documents and re-index using bulk endpoint. export const searchAfterAndBulkCreate = async ({ @@ -81,14 +43,7 @@ export const searchAfterAndBulkCreate = async ({ throttle, buildRuleMessage, }: SearchAfterAndBulkCreateParams): Promise => { - const toReturn: SearchAfterAndBulkCreateReturnType = { - success: true, - searchAfterTimes: [], - bulkCreateTimes: [], - lastLookBackDate: null, - createdSignalsCount: 0, - errors: [], - }; + let toReturn = createSearchAfterReturnType(); // sortId tells us where to start our next consecutive search_after query let sortId: string | undefined; @@ -108,13 +63,15 @@ export const searchAfterAndBulkCreate = async ({ buildRuleMessage, }); logger.debug(buildRuleMessage(`totalToFromTuples: ${totalToFromTuples.length}`)); + while (totalToFromTuples.length > 0) { const tuple = totalToFromTuples.pop(); if (tuple == null || tuple.to == null || tuple.from == null) { logger.error(buildRuleMessage(`[-] malformed date tuple`)); - toReturn.success = false; - toReturn.errors = [...new Set([...toReturn.errors, 'malformed date tuple'])]; - return toReturn; + return createSearchAfterReturnType({ + success: false, + errors: ['malformed date tuple'], + }); } signalsCreatedCount = 0; while (signalsCreatedCount < tuple.maxSignals) { @@ -122,29 +79,27 @@ export const searchAfterAndBulkCreate = async ({ logger.debug(buildRuleMessage(`sortIds: ${sortId}`)); // perform search_after with optionally undefined sortId - const { - searchResult, - searchDuration, - }: { searchResult: SignalSearchResponse; searchDuration: string } = await singleSearchAfter( - { - searchAfterSortId: sortId, - index: inputIndexPattern, - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - filter, - pageSize: tuple.maxSignals < pageSize ? Math.ceil(tuple.maxSignals) : pageSize, // maximum number of docs to receive per search result. - timestampOverride: ruleParams.timestampOverride, - } - ); - toReturn.searchAfterTimes.push(searchDuration); - + const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ + searchAfterSortId: sortId, + index: inputIndexPattern, + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + logger, + filter, + pageSize: tuple.maxSignals < pageSize ? Math.ceil(tuple.maxSignals) : pageSize, // maximum number of docs to receive per search result. + timestampOverride: ruleParams.timestampOverride, + }); + toReturn = mergeReturns([ + toReturn, + createSearchAfterReturnTypeFromResponse({ searchResult }), + createSearchAfterReturnType({ + searchAfterTimes: [searchDuration], + errors: searchErrors, + }), + ]); // determine if there are any candidate signals to be processed - const totalHits = - typeof searchResult.hits.total === 'number' - ? searchResult.hits.total - : searchResult.hits.total.value; + const totalHits = createTotalHitsFromSearchResult({ searchResult }); logger.debug(buildRuleMessage(`totalHits: ${totalHits}`)); logger.debug( buildRuleMessage(`searchResult.hit.hits.length: ${searchResult.hits.hits.length}`) @@ -168,17 +123,11 @@ export const searchAfterAndBulkCreate = async ({ ); break; } - toReturn.lastLookBackDate = - searchResult.hits.hits.length > 0 - ? new Date( - searchResult.hits.hits[searchResult.hits.hits.length - 1]?._source['@timestamp'] - ) - : null; // filter out the search results that match with the values found in the list. // the resulting set are signals to be indexed, given they are not duplicates // of signals already present in the signals index. - const filteredEvents: SignalSearchResponse = await filterEventsAgainstList({ + const filteredEvents = await filterEventsAgainstList({ listClient, exceptionsList, logger, @@ -222,19 +171,21 @@ export const searchAfterAndBulkCreate = async ({ tags, throttle, }); - logger.debug(buildRuleMessage(`created ${createdCount} signals`)); - toReturn.createdSignalsCount += createdCount; + toReturn = mergeReturns([ + toReturn, + createSearchAfterReturnType({ + success: bulkSuccess, + createdSignalsCount: createdCount, + bulkCreateTimes: bulkDuration ? [bulkDuration] : undefined, + errors: bulkErrors, + }), + ]); signalsCreatedCount += createdCount; + logger.debug(buildRuleMessage(`created ${createdCount} signals`)); logger.debug(buildRuleMessage(`signalsCreatedCount: ${signalsCreatedCount}`)); - if (bulkDuration) { - toReturn.bulkCreateTimes.push(bulkDuration); - } - logger.debug( buildRuleMessage(`filteredEvents.hits.hits: ${filteredEvents.hits.hits.length}`) ); - toReturn.success = toReturn.success && bulkSuccess; - toReturn.errors = [...new Set([...toReturn.errors, ...bulkErrors])]; } // we are guaranteed to have searchResult hits at this point @@ -249,9 +200,13 @@ export const searchAfterAndBulkCreate = async ({ } } catch (exc: unknown) { logger.error(buildRuleMessage(`[-] search_after and bulk threw an error ${exc}`)); - toReturn.success = false; - toReturn.errors = [...new Set([...toReturn.errors, `${exc}`])]; - return toReturn; + return mergeReturns([ + toReturn, + createSearchAfterReturnType({ + success: false, + errors: [`${exc}`], + }), + ]); } } } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 3ff5d5d2a6e13..382acf2f38245 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -18,11 +18,8 @@ import { sortExceptionItems, } from './utils'; import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; -import { RuleExecutorOptions } from './types'; -import { - searchAfterAndBulkCreate, - SearchAfterAndBulkCreateReturnType, -} from './search_after_bulk_create'; +import { RuleExecutorOptions, SearchAfterAndBulkCreateReturnType } from './types'; +import { searchAfterAndBulkCreate } from './search_after_bulk_create'; import { scheduleNotificationActions } from '../notifications/schedule_notification_actions'; import { RuleAlertType } from '../rules/types'; import { findMlSignals } from './find_ml_signals'; @@ -36,7 +33,17 @@ jest.mock('./rule_status_saved_objects_client'); jest.mock('./rule_status_service'); jest.mock('./search_after_bulk_create'); jest.mock('./get_filter'); -jest.mock('./utils'); +jest.mock('./utils', () => { + const original = jest.requireActual('./utils'); + return { + ...original, + getGapBetweenRuns: jest.fn(), + getGapMaxCatchupRatio: jest.fn(), + getListsClient: jest.fn(), + getExceptions: jest.fn(), + sortExceptionItems: jest.fn(), + }; +}); jest.mock('../notifications/schedule_notification_actions'); jest.mock('./find_ml_signals'); jest.mock('./bulk_create_ml_signals'); @@ -383,6 +390,7 @@ describe('rules_notification_alert_type', () => { }, ]); (findMlSignals as jest.Mock).mockResolvedValue({ + _shards: {}, hits: { hits: [], }, @@ -401,6 +409,7 @@ describe('rules_notification_alert_type', () => { payload = getPayload(ruleAlert, alertServices) as jest.Mocked; jobsSummaryMock.mockResolvedValue([]); (findMlSignals as jest.Mock).mockResolvedValue({ + _shards: {}, hits: { hits: [], }, @@ -409,6 +418,7 @@ describe('rules_notification_alert_type', () => { success: true, bulkCreateDuration: 0, createdItemsCount: 0, + errors: [], }); await alert.executor(payload); expect(ruleStatusService.success).not.toHaveBeenCalled(); @@ -425,6 +435,7 @@ describe('rules_notification_alert_type', () => { }, ]); (findMlSignals as jest.Mock).mockResolvedValue({ + _shards: { failed: 0 }, hits: { hits: [{}], }, @@ -433,6 +444,7 @@ describe('rules_notification_alert_type', () => { success: true, bulkCreateDuration: 1, createdItemsCount: 1, + errors: [], }); await alert.executor(payload); expect(ruleStatusService.success).toHaveBeenCalled(); @@ -460,6 +472,7 @@ describe('rules_notification_alert_type', () => { }); jobsSummaryMock.mockResolvedValue([]); (findMlSignals as jest.Mock).mockResolvedValue({ + _shards: { failed: 0 }, hits: { hits: [{}], }, @@ -468,6 +481,7 @@ describe('rules_notification_alert_type', () => { success: true, bulkCreateDuration: 1, createdItemsCount: 1, + errors: [], }); await alert.executor(payload); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 196c17b42221b..97ab12f905358 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -22,10 +22,7 @@ import { import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; import { SetupPlugins } from '../../../plugin'; import { getInputIndex } from './get_input_output_index'; -import { - searchAfterAndBulkCreate, - SearchAfterAndBulkCreateReturnType, -} from './search_after_bulk_create'; +import { searchAfterAndBulkCreate } from './search_after_bulk_create'; import { getFilter } from './get_filter'; import { SignalRuleAlertTypeDefinition, RuleAlertAttributes } from './types'; import { @@ -34,6 +31,10 @@ import { getExceptions, getGapMaxCatchupRatio, MAX_RULE_GAP_RATIO, + createErrorsFromShard, + createSearchAfterReturnType, + mergeReturns, + createSearchAfterReturnTypeFromResponse, } from './utils'; import { signalParamsSchema } from './signal_params_schema'; import { siemRuleActionGroups } from './siem_rule_action_groups'; @@ -104,14 +105,7 @@ export const signalRulesAlertType = ({ } = params; const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); let hasError: boolean = false; - let result: SearchAfterAndBulkCreateReturnType = { - success: false, - bulkCreateTimes: [], - searchAfterTimes: [], - lastLookBackDate: null, - createdSignalsCount: 0, - errors: [], - }; + let result = createSearchAfterReturnType(); const ruleStatusClient = ruleStatusSavedObjectsClientFactory(services.savedObjectsClient); const ruleStatusService = await ruleStatusServiceFactory({ alertId, @@ -255,12 +249,22 @@ export const signalRulesAlertType = ({ refresh, tags, }); - result.success = success; - result.errors = errors; - result.createdSignalsCount = createdItemsCount; - if (bulkCreateDuration) { - result.bulkCreateTimes.push(bulkCreateDuration); - } + // The legacy ES client does not define failures when it can be present on the structure, hence why I have the & { failures: [] } + const shardFailures = + (anomalyResults._shards as typeof anomalyResults._shards & { failures: [] }).failures ?? + []; + const searchErrors = createErrorsFromShard({ + errors: shardFailures, + }); + result = mergeReturns([ + result, + createSearchAfterReturnType({ + success: success && anomalyResults._shards.failed === 0, + errors: [...errors, ...searchErrors], + createdSignalsCount: createdItemsCount, + bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], + }), + ]); } else if (isEqlRule(type)) { throw new Error('EQL Rules are under development, execution is not yet implemented'); } else if (isThresholdRule(type) && threshold) { @@ -276,7 +280,7 @@ export const signalRulesAlertType = ({ lists: exceptionItems ?? [], }); - const { searchResult: thresholdResults } = await findThresholdSignals({ + const { searchResult: thresholdResults, searchErrors } = await findThresholdSignals({ inputIndexPattern: inputIndex, from, to, @@ -313,12 +317,16 @@ export const signalRulesAlertType = ({ refresh, tags, }); - result.success = success; - result.errors = errors; - result.createdSignalsCount = createdItemsCount; - if (bulkCreateDuration) { - result.bulkCreateTimes.push(bulkCreateDuration); - } + result = mergeReturns([ + result, + createSearchAfterReturnTypeFromResponse({ searchResult: thresholdResults }), + createSearchAfterReturnType({ + success, + errors: [...errors, ...searchErrors], + createdSignalsCount: createdItemsCount, + bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], + }), + ]); } else if (isThreatMatchRule(type)) { if ( threatQuery == null || diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts index 250b891eb1f2c..da81911f07ad9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts @@ -11,6 +11,7 @@ import { } from './__mocks__/es_results'; import { singleSearchAfter } from './single_search_after'; import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks'; +import { ShardError } from '../../types'; describe('singleSearchAfter', () => { const mockService: AlertServicesMock = alertsMock.createAlertServices(); @@ -20,10 +21,71 @@ describe('singleSearchAfter', () => { }); test('if singleSearchAfter works without a given sort id', async () => { - let searchAfterSortId; - mockService.callCluster.mockResolvedValue(sampleDocSearchResultsNoSortId); + mockService.callCluster.mockResolvedValue(sampleDocSearchResultsNoSortId()); const { searchResult } = await singleSearchAfter({ - searchAfterSortId, + searchAfterSortId: undefined, + index: [], + from: 'now-360s', + to: 'now', + services: mockService, + logger: mockLogger, + pageSize: 1, + filter: undefined, + timestampOverride: undefined, + }); + expect(searchResult).toEqual(sampleDocSearchResultsNoSortId()); + }); + test('if singleSearchAfter returns an empty failure array', async () => { + mockService.callCluster.mockResolvedValue(sampleDocSearchResultsNoSortId()); + const { searchErrors } = await singleSearchAfter({ + searchAfterSortId: undefined, + index: [], + from: 'now-360s', + to: 'now', + services: mockService, + logger: mockLogger, + pageSize: 1, + filter: undefined, + timestampOverride: undefined, + }); + expect(searchErrors).toEqual([]); + }); + test('if singleSearchAfter will return an error array', async () => { + const errors: ShardError[] = [ + { + shard: 1, + index: 'index-123', + node: 'node-123', + reason: { + type: 'some type', + reason: 'some reason', + index_uuid: 'uuid-123', + index: 'index-123', + caused_by: { + type: 'some type', + reason: 'some reason', + }, + }, + }, + ]; + mockService.callCluster.mockResolvedValue({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 1, + skipped: 0, + failures: errors, + }, + hits: { + total: 100, + max_score: 100, + hits: [], + }, + }); + const { searchErrors } = await singleSearchAfter({ + searchAfterSortId: undefined, index: [], from: 'now-360s', to: 'now', @@ -33,11 +95,11 @@ describe('singleSearchAfter', () => { filter: undefined, timestampOverride: undefined, }); - expect(searchResult).toEqual(sampleDocSearchResultsNoSortId); + expect(searchErrors).toEqual(['reason: some reason, type: some type, caused by: some reason']); }); test('if singleSearchAfter works with a given sort id', async () => { const searchAfterSortId = '1234567891111'; - mockService.callCluster.mockResolvedValue(sampleDocSearchResultsWithSortId); + mockService.callCluster.mockResolvedValue(sampleDocSearchResultsWithSortId()); const { searchResult } = await singleSearchAfter({ searchAfterSortId, index: [], @@ -49,7 +111,7 @@ describe('singleSearchAfter', () => { filter: undefined, timestampOverride: undefined, }); - expect(searchResult).toEqual(sampleDocSearchResultsWithSortId); + expect(searchResult).toEqual(sampleDocSearchResultsWithSortId()); }); test('if singleSearchAfter throws error', async () => { const searchAfterSortId = '1234567891111'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index 92ce7a2836115..f758adb21611c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -9,7 +9,7 @@ import { AlertServices } from '../../../../../alerts/server'; import { Logger } from '../../../../../../../src/core/server'; import { SignalSearchResponse } from './types'; import { buildEventsSearchQuery } from './build_events_query'; -import { makeFloatString } from './utils'; +import { createErrorsFromShard, makeFloatString } from './utils'; import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas'; interface SingleSearchAfterParams { @@ -40,6 +40,7 @@ export const singleSearchAfter = async ({ }: SingleSearchAfterParams): Promise<{ searchResult: SignalSearchResponse; searchDuration: string; + searchErrors: string[]; }> => { try { const searchAfterQuery = buildEventsSearchQuery({ @@ -59,7 +60,14 @@ export const singleSearchAfter = async ({ searchAfterQuery ); const end = performance.now(); - return { searchResult: nextSearchAfterResult, searchDuration: makeFloatString(end - start) }; + const searchErrors = createErrorsFromShard({ + errors: nextSearchAfterResult._shards.failures ?? [], + }); + return { + searchResult: nextSearchAfterResult, + searchDuration: makeFloatString(end - start), + searchErrors, + }; } catch (exc) { logger.error(`[-] nextSearchAfter threw an error ${exc}`); throw exc; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index 7542128d83769..a6d4a2ba58ddd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -9,12 +9,10 @@ import { getThreatList } from './get_threat_list'; import { buildThreatMappingFilter } from './build_threat_mapping_filter'; import { getFilter } from '../get_filter'; -import { - searchAfterAndBulkCreate, - SearchAfterAndBulkCreateReturnType, -} from '../search_after_bulk_create'; +import { searchAfterAndBulkCreate } from '../search_after_bulk_create'; import { CreateThreatSignalOptions, ThreatListItem } from './types'; import { combineResults } from './utils'; +import { SearchAfterAndBulkCreateReturnType } from '../types'; export const createThreatSignal = async ({ threatMapping, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 9027475d71c4a..f416ae6703b66 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -6,9 +6,9 @@ import { getThreatList } from './get_threat_list'; -import { SearchAfterAndBulkCreateReturnType } from '../search_after_bulk_create'; import { CreateThreatSignalsOptions } from './types'; import { createThreatSignal } from './create_threat_signal'; +import { SearchAfterAndBulkCreateReturnType } from '../types'; export const createThreatSignals = async ({ threatMapping, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 4c3cd9943adb4..d63f2d2b3b6aa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -19,10 +19,10 @@ import { import { PartialFilter, RuleTypeParams } from '../../types'; import { AlertServices } from '../../../../../../alerts/server'; import { ExceptionListItemSchema } from '../../../../../../lists/common/schemas'; -import { SearchAfterAndBulkCreateReturnType } from '../search_after_bulk_create'; import { ILegacyScopedClusterClient, Logger } from '../../../../../../../../src/core/server'; import { RuleAlertAction } from '../../../../../common/detection_engine/types'; import { BuildRuleMessage } from '../rule_messages'; +import { SearchAfterAndBulkCreateReturnType } from '../types'; export interface CreateThreatSignalsOptions { threatMapping: ThreatMapping; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts index 48bdf430b940e..27593b40b0c8f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SearchAfterAndBulkCreateReturnType } from '../search_after_bulk_create'; +import { SearchAfterAndBulkCreateReturnType } from '../types'; import { calculateAdditiveMax, combineResults } from './utils'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts index 38bbb70b6c4ec..401a4a1acb065 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SearchAfterAndBulkCreateReturnType } from '../search_after_bulk_create'; +import { SearchAfterAndBulkCreateReturnType } from '../types'; /** * Given two timers this will take the max of each and add them to each other and return that addition. diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 23aa786558a99..6ebdca0764e9d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -5,12 +5,22 @@ */ import { DslQuery, Filter } from 'src/plugins/data/common'; +import moment from 'moment'; import { Status } from '../../../../common/detection_engine/schemas/common/schemas'; import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; -import { AlertType, AlertTypeState, AlertExecutorOptions } from '../../../../../alerts/server'; +import { + AlertType, + AlertTypeState, + AlertExecutorOptions, + AlertServices, +} from '../../../../../alerts/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; -import { RuleTypeParams } from '../types'; +import { RuleTypeParams, RefreshTypes } from '../types'; import { SearchResponse } from '../../types'; +import { ListClient } from '../../../../../lists/server'; +import { Logger } from '../../../../../../../src/core/server'; +import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; +import { BuildRuleMessage } from './rule_messages'; // used for gap detection code // eslint-disable-next-line @typescript-eslint/naming-convention @@ -179,3 +189,39 @@ export interface QueryFilter { must_not: Filter[]; }; } + +export interface SearchAfterAndBulkCreateParams { + gap: moment.Duration | null; + previousStartedAt: Date | null | undefined; + ruleParams: RuleTypeParams; + services: AlertServices; + listClient: ListClient; + exceptionsList: ExceptionListItemSchema[]; + logger: Logger; + id: string; + inputIndexPattern: string[]; + signalsIndex: string; + name: string; + actions: RuleAlertAction[]; + createdAt: string; + createdBy: string; + updatedBy: string; + updatedAt: string; + interval: string; + enabled: boolean; + pageSize: number; + filter: unknown; + refresh: RefreshTypes; + tags: string[]; + throttle: string; + buildRuleMessage: BuildRuleMessage; +} + +export interface SearchAfterAndBulkCreateReturnType { + success: boolean; + searchAfterTimes: string[]; + bulkCreateTimes: string[]; + lastLookBackDate: Date | null | undefined; + createdSignalsCount: number; + errors: string[]; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 123b9c9bdffa2..97f3dbeaf4489 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -25,15 +25,25 @@ import { getListsClient, getSignalTimeTuples, getExceptions, + createErrorsFromShard, + createSearchAfterReturnTypeFromResponse, + createSearchAfterReturnType, + mergeReturns, + createTotalHitsFromSearchResult, } from './utils'; -import { BulkResponseErrorAggregation } from './types'; +import { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types'; import { sampleBulkResponse, sampleEmptyBulkResponse, sampleBulkError, sampleBulkErrorItem, mockLogger, + sampleDocSearchResultsWithSortId, + sampleEmptyDocSearchResults, + sampleDocSearchResultsNoSortIdNoHits, + repeatedSearchResultsWithSortId, } from './__mocks__/es_results'; +import { ShardError } from '../../types'; const buildRuleMessage = buildRuleMessageFactory({ id: 'fake id', @@ -783,4 +793,278 @@ describe('utils', () => { expect(exceptions).toEqual([]); }); }); + + describe('createErrorsFromShard', () => { + test('empty errors will return an empty array', () => { + const createdErrors = createErrorsFromShard({ errors: [] }); + expect(createdErrors).toEqual([]); + }); + + test('single error will return single converted array of a string of a reason', () => { + const errors: ShardError[] = [ + { + shard: 1, + index: 'index-123', + node: 'node-123', + reason: { + type: 'some type', + reason: 'some reason', + index_uuid: 'uuid-123', + index: 'index-123', + caused_by: { + type: 'some type', + reason: 'some reason', + }, + }, + }, + ]; + const createdErrors = createErrorsFromShard({ errors }); + expect(createdErrors).toEqual([ + 'reason: some reason, type: some type, caused by: some reason', + ]); + }); + + test('two errors will return two converted arrays to a string of a reason', () => { + const errors: ShardError[] = [ + { + shard: 1, + index: 'index-123', + node: 'node-123', + reason: { + type: 'some type', + reason: 'some reason', + index_uuid: 'uuid-123', + index: 'index-123', + caused_by: { + type: 'some type', + reason: 'some reason', + }, + }, + }, + { + shard: 2, + index: 'index-345', + node: 'node-345', + reason: { + type: 'some type 2', + reason: 'some reason 2', + index_uuid: 'uuid-345', + index: 'index-345', + caused_by: { + type: 'some type 2', + reason: 'some reason 2', + }, + }, + }, + ]; + const createdErrors = createErrorsFromShard({ errors }); + expect(createdErrors).toEqual([ + 'reason: some reason, type: some type, caused by: some reason', + 'reason: some reason 2, type: some type 2, caused by: some reason 2', + ]); + }); + }); + + describe('createSearchAfterReturnTypeFromResponse', () => { + test('empty results will return successful type', () => { + const searchResult = sampleEmptyDocSearchResults(); + const newSearchResult = createSearchAfterReturnTypeFromResponse({ searchResult }); + const expected: SearchAfterAndBulkCreateReturnType = { + bulkCreateTimes: [], + createdSignalsCount: 0, + errors: [], + lastLookBackDate: null, + searchAfterTimes: [], + success: true, + }; + expect(newSearchResult).toEqual(expected); + }); + + test('multiple results will return successful type with expected success', () => { + const searchResult = sampleDocSearchResultsWithSortId(); + const newSearchResult = createSearchAfterReturnTypeFromResponse({ searchResult }); + const expected: SearchAfterAndBulkCreateReturnType = { + bulkCreateTimes: [], + createdSignalsCount: 0, + errors: [], + lastLookBackDate: new Date('2020-04-20T21:27:45.000Z'), + searchAfterTimes: [], + success: true, + }; + expect(newSearchResult).toEqual(expected); + }); + + test('result with error will create success: false within the result set', () => { + const searchResult = sampleDocSearchResultsNoSortIdNoHits(); + searchResult._shards.failed = 1; + const { success } = createSearchAfterReturnTypeFromResponse({ searchResult }); + expect(success).toEqual(false); + }); + + test('result with error will create success: false within the result set if failed is 2 or more', () => { + const searchResult = sampleDocSearchResultsNoSortIdNoHits(); + searchResult._shards.failed = 2; + const { success } = createSearchAfterReturnTypeFromResponse({ searchResult }); + expect(success).toEqual(false); + }); + + test('result with error will create success: true within the result set if failed is 0', () => { + const searchResult = sampleDocSearchResultsNoSortIdNoHits(); + searchResult._shards.failed = 0; + const { success } = createSearchAfterReturnTypeFromResponse({ searchResult }); + expect(success).toEqual(true); + }); + }); + + describe('createSearchAfterReturnType', () => { + test('createSearchAfterReturnType will return full object when nothing is passed', () => { + const searchAfterReturnType = createSearchAfterReturnType(); + const expected: SearchAfterAndBulkCreateReturnType = { + bulkCreateTimes: [], + createdSignalsCount: 0, + errors: [], + lastLookBackDate: null, + searchAfterTimes: [], + success: true, + }; + expect(searchAfterReturnType).toEqual(expected); + }); + + test('createSearchAfterReturnType can override all values', () => { + const searchAfterReturnType = createSearchAfterReturnType({ + bulkCreateTimes: ['123'], + createdSignalsCount: 5, + errors: ['error 1'], + lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), + searchAfterTimes: ['123'], + success: false, + }); + const expected: SearchAfterAndBulkCreateReturnType = { + bulkCreateTimes: ['123'], + createdSignalsCount: 5, + errors: ['error 1'], + lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), + searchAfterTimes: ['123'], + success: false, + }; + expect(searchAfterReturnType).toEqual(expected); + }); + + test('createSearchAfterReturnType can override select values', () => { + const searchAfterReturnType = createSearchAfterReturnType({ + createdSignalsCount: 5, + errors: ['error 1'], + }); + const expected: SearchAfterAndBulkCreateReturnType = { + bulkCreateTimes: [], + createdSignalsCount: 5, + errors: ['error 1'], + lastLookBackDate: null, + searchAfterTimes: [], + success: true, + }; + expect(searchAfterReturnType).toEqual(expected); + }); + }); + + describe('mergeReturns', () => { + test('it merges a default "prev" and "next" correctly ', () => { + const merged = mergeReturns([createSearchAfterReturnType(), createSearchAfterReturnType()]); + const expected: SearchAfterAndBulkCreateReturnType = { + bulkCreateTimes: [], + createdSignalsCount: 0, + errors: [], + lastLookBackDate: null, + searchAfterTimes: [], + success: true, + }; + expect(merged).toEqual(expected); + }); + + test('it merges search in with two default search results where "prev" "success" is false correctly', () => { + const { success } = mergeReturns([ + createSearchAfterReturnType({ success: false }), + createSearchAfterReturnType(), + ]); + expect(success).toEqual(false); + }); + + test('it merges search in with two default search results where "next" "success" is false correctly', () => { + const { success } = mergeReturns([ + createSearchAfterReturnType(), + createSearchAfterReturnType({ success: false }), + ]); + expect(success).toEqual(false); + }); + + test('it merges search where the lastLookBackDate is the "next" date when given', () => { + const { lastLookBackDate } = mergeReturns([ + createSearchAfterReturnType({ + lastLookBackDate: new Date('2020-08-21T19:21:46.194Z'), + }), + createSearchAfterReturnType({ + lastLookBackDate: new Date('2020-09-21T19:21:46.194Z'), + }), + ]); + expect(lastLookBackDate).toEqual(new Date('2020-09-21T19:21:46.194Z')); + }); + + test('it merges search where the lastLookBackDate is the "prev" if given undefined for "next', () => { + const { lastLookBackDate } = mergeReturns([ + createSearchAfterReturnType({ + lastLookBackDate: new Date('2020-08-21T19:21:46.194Z'), + }), + createSearchAfterReturnType({ + lastLookBackDate: undefined, + }), + ]); + expect(lastLookBackDate).toEqual(new Date('2020-08-21T19:21:46.194Z')); + }); + + test('it merges search where values from "next" and "prev" are computed together', () => { + const merged = mergeReturns([ + createSearchAfterReturnType({ + bulkCreateTimes: ['123'], + createdSignalsCount: 3, + errors: ['error 1', 'error 2'], + lastLookBackDate: new Date('2020-08-21T18:51:25.193Z'), + searchAfterTimes: ['123'], + success: true, + }), + createSearchAfterReturnType({ + bulkCreateTimes: ['456'], + createdSignalsCount: 2, + errors: ['error 3'], + lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), + searchAfterTimes: ['567'], + success: true, + }), + ]); + const expected: SearchAfterAndBulkCreateReturnType = { + bulkCreateTimes: ['123', '456'], // concatenates the prev and next together + createdSignalsCount: 5, // Adds the 3 and 2 together + errors: ['error 1', 'error 2', 'error 3'], // concatenates the prev and next together + lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), // takes the next lastLookBackDate + searchAfterTimes: ['123', '567'], // concatenates the searchAfterTimes together + success: true, // Defaults to success true is all of it was successful + }; + expect(merged).toEqual(expected); + }); + }); + + describe('createTotalHitsFromSearchResult', () => { + test('it should return 0 for empty results', () => { + const result = createTotalHitsFromSearchResult({ + searchResult: sampleEmptyDocSearchResults(), + }); + expect(result).toEqual(0); + }); + + test('it should return 4 for 4 result sets', () => { + const result = createTotalHitsFromSearchResult({ + searchResult: repeatedSearchResultsWithSortId(4, 1, ['1', '2', '3', '4']), + }); + expect(result).toEqual(4); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 9f1e5d6980466..2eabc03dccad7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -12,11 +12,18 @@ import { AlertServices, parseDuration } from '../../../../../alerts/server'; import { ExceptionListClient, ListClient, ListPluginSetup } from '../../../../../lists/server'; import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; import { ListArray } from '../../../../common/detection_engine/schemas/types/lists'; -import { BulkResponse, BulkResponseErrorAggregation, isValidUnit } from './types'; +import { + BulkResponse, + BulkResponseErrorAggregation, + isValidUnit, + SearchAfterAndBulkCreateReturnType, + SignalSearchResponse, +} from './types'; import { BuildRuleMessage } from './rule_messages'; import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; import { hasLargeValueList } from '../../../../common/detection_engine/utils'; import { MAX_EXCEPTION_LIST_SIZE } from '../../../../../lists/common/constants'; +import { ShardError } from '../../types'; interface SortExceptionsReturn { exceptionsWithValueLists: ExceptionListItemSchema[]; @@ -439,3 +446,97 @@ export const getSignalTimeTuples = ({ ); return totalToFromTuples; }; + +/** + * Given errors from a search query this will return an array of strings derived from the errors. + * @param errors The errors to derive the strings from + */ +export const createErrorsFromShard = ({ errors }: { errors: ShardError[] }): string[] => { + return errors.map((error) => { + return `reason: ${error.reason.reason}, type: ${error.reason.caused_by.type}, caused by: ${error.reason.caused_by.reason}`; + }); +}; + +export const createSearchAfterReturnTypeFromResponse = ({ + searchResult, +}: { + searchResult: SignalSearchResponse; +}): SearchAfterAndBulkCreateReturnType => { + return createSearchAfterReturnType({ + success: searchResult._shards.failed === 0, + lastLookBackDate: + searchResult.hits.hits.length > 0 + ? new Date(searchResult.hits.hits[searchResult.hits.hits.length - 1]?._source['@timestamp']) + : undefined, + }); +}; + +export const createSearchAfterReturnType = ({ + success, + searchAfterTimes, + bulkCreateTimes, + lastLookBackDate, + createdSignalsCount, + errors, +}: { + success?: boolean | undefined; + searchAfterTimes?: string[] | undefined; + bulkCreateTimes?: string[] | undefined; + lastLookBackDate?: Date | undefined; + createdSignalsCount?: number | undefined; + errors?: string[] | undefined; +} = {}): SearchAfterAndBulkCreateReturnType => { + return { + success: success ?? true, + searchAfterTimes: searchAfterTimes ?? [], + bulkCreateTimes: bulkCreateTimes ?? [], + lastLookBackDate: lastLookBackDate ?? null, + createdSignalsCount: createdSignalsCount ?? 0, + errors: errors ?? [], + }; +}; + +export const mergeReturns = ( + searchAfters: SearchAfterAndBulkCreateReturnType[] +): SearchAfterAndBulkCreateReturnType => { + return searchAfters.reduce((prev, next) => { + const { + success: existingSuccess, + searchAfterTimes: existingSearchAfterTimes, + bulkCreateTimes: existingBulkCreateTimes, + lastLookBackDate: existingLastLookBackDate, + createdSignalsCount: existingCreatedSignalsCount, + errors: existingErrors, + } = prev; + + const { + success: newSuccess, + searchAfterTimes: newSearchAfterTimes, + bulkCreateTimes: newBulkCreateTimes, + lastLookBackDate: newLastLookBackDate, + createdSignalsCount: newCreatedSignalsCount, + errors: newErrors, + } = next; + + return { + success: existingSuccess && newSuccess, + searchAfterTimes: [...existingSearchAfterTimes, ...newSearchAfterTimes], + bulkCreateTimes: [...existingBulkCreateTimes, ...newBulkCreateTimes], + lastLookBackDate: newLastLookBackDate ?? existingLastLookBackDate, + createdSignalsCount: existingCreatedSignalsCount + newCreatedSignalsCount, + errors: [...new Set([...existingErrors, ...newErrors])], + }; + }); +}; + +export const createTotalHitsFromSearchResult = ({ + searchResult, +}: { + searchResult: SignalSearchResponse; +}): number => { + const totalHits = + typeof searchResult.hits.total === 'number' + ? searchResult.hits.total + : searchResult.hits.total.value; + return totalHits; +}; diff --git a/x-pack/plugins/security_solution/server/lib/types.ts b/x-pack/plugins/security_solution/server/lib/types.ts index ff89512124b66..435bcd9d61d89 100644 --- a/x-pack/plugins/security_solution/server/lib/types.ts +++ b/x-pack/plugins/security_solution/server/lib/types.ts @@ -98,6 +98,23 @@ export interface ShardsResponse { successful: number; failed: number; skipped: number; + failures?: ShardError[]; +} + +export interface ShardError { + shard: number; + index: string; + node: string; + reason: { + type: string; + reason: string; + index_uuid: string; + index: string; + caused_by: { + type: string; + reason: string; + }; + }; } export interface Explanation {