Skip to content

Commit

Permalink
[Security Solution] Converge detection engine on single schema repres…
Browse files Browse the repository at this point in the history 
…entation (#96186)

* Replace validation function in signal executor

* Remove more RuleTypeParams usage

* Add security solution rules migration to alerting plugin

* Handle and test null value in threshold.field

* Remove runtime normalization of threshold field

* Remove signalParamsSchema

Co-authored-by: Davis Plumlee <davis.plumlee@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
  • Loading branch information
@marshallmain @dplumlee @kibanamachine
3 people committed Apr 14, 2021
1 parent 4c33bfd commit ff7c533
Show file tree
Hide file tree
Showing 84 changed files with 1,200 additions and 3,319 deletions.
276 changes: 276 additions & 0 deletions x-pack/plugins/alerting/server/saved_objects/migrations.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -699,6 +699,282 @@ describe('7.11.2', () => {
});
});

describe('7.13.0', () => {
beforeEach(() => {
jest.resetAllMocks();
encryptedSavedObjectsSetup.createMigration.mockImplementation(
(shouldMigrateWhenPredicate, migration) => migration
);
});
test('security solution alerts get migrated and remove null values', () => {
const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0'];
const alert = getMockData({
alertTypeId: 'siem.signals',
params: {
author: ['Elastic'],
buildingBlockType: null,
description:
"This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.",
ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3',
index: ['packetbeat-*'],
falsePositives: [
"This rule could identify benign domains that are formatted similarly to FIN7's command and control algorithm. Alerts should be investigated by an analyst to assess the validity of the individual observations.",
],
from: 'now-6m',
immutable: true,
query:
'event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-zA-Z]{4,5}.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us',
language: 'lucene',
license: 'Elastic License',
outputIndex: '.siem-signals-rylandherrick_2-default',
savedId: null,
timelineId: null,
timelineTitle: null,
meta: null,
filters: null,
maxSignals: 100,
riskScore: 73,
riskScoreMapping: [],
ruleNameOverride: null,
severity: 'high',
severityMapping: null,
threat: null,
threatFilters: null,
timestampOverride: null,
to: 'now',
type: 'query',
references: [
'https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html',
],
note:
'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.',
version: 1,
exceptionsList: null,
threshold: {
field: null,
value: 5,
},
},
});

expect(migration713(alert, migrationContext)).toEqual({
...alert,
attributes: {
...alert.attributes,
params: {
author: ['Elastic'],
description:
"This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.",
ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3',
index: ['packetbeat-*'],
falsePositives: [
"This rule could identify benign domains that are formatted similarly to FIN7's command and control algorithm. Alerts should be investigated by an analyst to assess the validity of the individual observations.",
],
from: 'now-6m',
immutable: true,
query:
'event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-zA-Z]{4,5}.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us',
language: 'lucene',
license: 'Elastic License',
outputIndex: '.siem-signals-rylandherrick_2-default',
maxSignals: 100,
riskScore: 73,
riskScoreMapping: [],
severity: 'high',
severityMapping: [],
threat: [],
to: 'now',
type: 'query',
references: [
'https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html',
],
note:
'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.',
version: 1,
exceptionsList: [],
threshold: {
field: [],
value: 5,
cardinality: [],
},
},
},
});
});

test('non-null values in security solution alerts are not modified', () => {
const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0'];
const alert = getMockData({
alertTypeId: 'siem.signals',
params: {
author: ['Elastic'],
buildingBlockType: 'default',
description:
"This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.",
ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3',
index: ['packetbeat-*'],
falsePositives: [
"This rule could identify benign domains that are formatted similarly to FIN7's command and control algorithm. Alerts should be investigated by an analyst to assess the validity of the individual observations.",
],
from: 'now-6m',
immutable: true,
query:
'event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-zA-Z]{4,5}.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us',
language: 'lucene',
license: 'Elastic License',
outputIndex: '.siem-signals-rylandherrick_2-default',
savedId: 'saved-id',
timelineId: 'timeline-id',
timelineTitle: 'timeline-title',
meta: {
field: 'value',
},
filters: ['filters'],
maxSignals: 100,
riskScore: 73,
riskScoreMapping: ['risk-score-mapping'],
ruleNameOverride: 'field.name',
severity: 'high',
severityMapping: ['severity-mapping'],
threat: [
{
framework: 'MITRE ATT&CK',
tactic: {
id: 'TA0011',
name: 'Command and Control',
reference: 'https://attack.mitre.org/tactics/TA0011/',
},
technique: [
{
id: 'T1483',
name: 'Domain Generation Algorithms',
reference: 'https://attack.mitre.org/techniques/T1483/',
},
],
},
],
threatFilters: ['threat-filter'],
timestampOverride: 'event.ingested',
to: 'now',
type: 'query',
references: [
'https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html',
],
note:
'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.',
version: 1,
exceptionsList: ['exceptions-list'],
},
});

expect(migration713(alert, migrationContext)).toEqual(alert);
});

test('security solution threshold alert with string in threshold.field is migrated to array', () => {
const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0'];
const alert = getMockData({
alertTypeId: 'siem.signals',
params: {
threshold: {
field: 'host.id',
value: 5,
},
},
});

expect(migration713(alert, migrationContext)).toEqual({
...alert,
attributes: {
...alert.attributes,
params: {
threshold: {
field: ['host.id'],
value: 5,
cardinality: [],
},
exceptionsList: [],
riskScoreMapping: [],
severityMapping: [],
threat: [],
},
},
});
});

test('security solution threshold alert with empty string in threshold.field is migrated to empty array', () => {
const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0'];
const alert = getMockData({
alertTypeId: 'siem.signals',
params: {
threshold: {
field: '',
value: 5,
},
},
});

expect(migration713(alert, migrationContext)).toEqual({
...alert,
attributes: {
...alert.attributes,
params: {
threshold: {
field: [],
value: 5,
cardinality: [],
},
exceptionsList: [],
riskScoreMapping: [],
severityMapping: [],
threat: [],
},
},
});
});

test('security solution threshold alert with array in threshold.field and cardinality is left alone', () => {
const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0'];
const alert = getMockData({
alertTypeId: 'siem.signals',
params: {
threshold: {
field: ['host.id'],
value: 5,
cardinality: [
{
field: 'source.ip',
value: 10,
},
],
},
},
});

expect(migration713(alert, migrationContext)).toEqual({
...alert,
attributes: {
...alert.attributes,
params: {
threshold: {
field: ['host.id'],
value: 5,
cardinality: [
{
field: 'source.ip',
value: 10,
},
],
},
exceptionsList: [],
riskScoreMapping: [],
severityMapping: [],
threat: [],
},
},
});
});
});

function getUpdatedAt(): string {
const updatedAt = new Date();
updatedAt.setHours(updatedAt.getHours() + 2);
Expand Down
72 changes: 72 additions & 0 deletions x-pack/plugins/alerting/server/saved_objects/migrations.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
SavedObjectMigrationFn,
SavedObjectMigrationContext,
SavedObjectAttributes,
SavedObjectAttribute,
} from '../../../../../src/core/server';
import { RawAlert, RawAlertAction } from '../types';
import { EncryptedSavedObjectsPluginSetup } from '../../../encrypted_saved_objects/server';
Expand All @@ -30,6 +31,9 @@ export const isAnyActionSupportIncidents = (doc: SavedObjectUnsanitizedDoc<RawAl
SUPPORT_INCIDENTS_ACTION_TYPES.includes(action.actionTypeId)
);

export const isSecuritySolutionRule = (doc: SavedObjectUnsanitizedDoc<RawAlert>): boolean =>
doc.attributes.alertTypeId === 'siem.signals';

export function getMigrations(
encryptedSavedObjects: EncryptedSavedObjectsPluginSetup
): SavedObjectMigrationMap {
Expand Down Expand Up @@ -59,10 +63,16 @@ export function getMigrations(
pipeMigrations(restructureConnectorsThatSupportIncident)
);

const migrationSecurityRules713 = encryptedSavedObjects.createMigration<RawAlert, RawAlert>(
(doc): doc is SavedObjectUnsanitizedDoc<RawAlert> => isSecuritySolutionRule(doc),
pipeMigrations(removeNullsFromSecurityRules)
);

return {
'7.10.0': executeMigrationWithErrorHandling(migrationWhenRBACWasIntroduced, '7.10.0'),
'7.11.0': executeMigrationWithErrorHandling(migrationAlertUpdatedAtAndNotifyWhen, '7.11.0'),
'7.11.2': executeMigrationWithErrorHandling(migrationActions7112, '7.11.2'),
'7.13.0': executeMigrationWithErrorHandling(migrationSecurityRules713, '7.13.0'),
};
}

Expand Down Expand Up @@ -333,6 +343,68 @@ function restructureConnectorsThatSupportIncident(
};
}

function convertNullToUndefined(attribute: SavedObjectAttribute) {
return attribute != null ? attribute : undefined;
}

function removeNullsFromSecurityRules(
doc: SavedObjectUnsanitizedDoc<RawAlert>
): SavedObjectUnsanitizedDoc<RawAlert> {
const {
attributes: { params },
} = doc;
return {
...doc,
attributes: {
...doc.attributes,
params: {
...params,
buildingBlockType: convertNullToUndefined(params.buildingBlockType),
note: convertNullToUndefined(params.note),
index: convertNullToUndefined(params.index),
language: convertNullToUndefined(params.language),
license: convertNullToUndefined(params.license),
outputIndex: convertNullToUndefined(params.outputIndex),
savedId: convertNullToUndefined(params.savedId),
timelineId: convertNullToUndefined(params.timelineId),
timelineTitle: convertNullToUndefined(params.timelineTitle),
meta: convertNullToUndefined(params.meta),
query: convertNullToUndefined(params.query),
filters: convertNullToUndefined(params.filters),
riskScoreMapping: params.riskScoreMapping != null ? params.riskScoreMapping : [],
ruleNameOverride: convertNullToUndefined(params.ruleNameOverride),
severityMapping: params.severityMapping != null ? params.severityMapping : [],
threat: params.threat != null ? params.threat : [],
threshold:
params.threshold != null &&
typeof params.threshold === 'object' &&
!Array.isArray(params.threshold)
? {
field: Array.isArray(params.threshold.field)
? params.threshold.field
: params.threshold.field === '' || params.threshold.field == null
? []
: [params.threshold.field],
value: params.threshold.value,
cardinality:
params.threshold.cardinality != null ? params.threshold.cardinality : [],
}
: undefined,
timestampOverride: convertNullToUndefined(params.timestampOverride),
exceptionsList:
params.exceptionsList != null
? params.exceptionsList
: params.exceptions_list != null
? params.exceptions_list
: params.lists != null
? params.lists
: [],
threatFilters: convertNullToUndefined(params.threatFilters),
},
},
};
}

function pipeMigrations(...migrations: AlertMigration[]): AlertMigration {
return (doc: SavedObjectUnsanitizedDoc<RawAlert>) =>
migrations.reduce((migratedDoc, nextMigration) => nextMigration(migratedDoc), doc);
Expand Down
Loading

0 comments on commit ff7c533

Please sign in to comment.