Skip to content

Commit

Permalink
Add allowlist filter for Exception list data (#112668)
Browse files Browse the repository at this point in the history 
* Add allowlist filter to exception list telemetry.

* Refactor away union type.

* Bump interval back to 24h.

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
  • Loading branch information
@pjhampton @kibanamachine
pjhampton and kibanamachine committed Sep 21, 2021
1 parent 8cf0efe commit feae8e7
Show file tree
Hide file tree
Showing 10 changed files with 266 additions and 217 deletions.
2 changes: 2 additions & 0 deletions x-pack/plugins/security_solution/server/lib/telemetry/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ export const TELEMETRY_CHANNEL_LISTS = 'security-lists';

export const TELEMETRY_CHANNEL_ENDPOINT_META = 'endpoint-metadata';

export const LIST_TRUSTED_APPLICATION = 'trusted_application';

export const LIST_ENDPOINT_EXCEPTION = 'endpoint_exception';

export const LIST_ENDPOINT_EVENT_FILTER = 'endpoint_event_filter';
127 changes: 127 additions & 0 deletions x-pack/plugins/security_solution/server/lib/telemetry/filters.test.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { copyAllowlistedFields } from './filters';

describe('Security Telemetry filters', () => {
describe('allowlistEventFields', () => {
const allowlist = {
a: true,
b: true,
c: {
d: true,
},
};

it('filters top level', () => {
const event = {
a: 'a',
a1: 'a1',
b: 'b',
b1: 'b1',
};
expect(copyAllowlistedFields(allowlist, event)).toStrictEqual({
a: 'a',
b: 'b',
});
});

it('filters nested', () => {
const event = {
a: {
a1: 'a1',
},
a1: 'a1',
b: {
b1: 'b1',
},
b1: 'b1',
c: {
d: 'd',
e: 'e',
f: 'f',
},
};
expect(copyAllowlistedFields(allowlist, event)).toStrictEqual({
a: {
a1: 'a1',
},
b: {
b1: 'b1',
},
c: {
d: 'd',
},
});
});

it('filters arrays of objects', () => {
const event = {
a: [
{
a1: 'a1',
},
],
b: {
b1: 'b1',
},
c: [
{
d: 'd1',
e: 'e1',
f: 'f1',
},
{
d: 'd2',
e: 'e2',
f: 'f2',
},
{
d: 'd3',
e: 'e3',
f: 'f3',
},
],
};
expect(copyAllowlistedFields(allowlist, event)).toStrictEqual({
a: [
{
a1: 'a1',
},
],
b: {
b1: 'b1',
},
c: [
{
d: 'd1',
},
{
d: 'd2',
},
{
d: 'd3',
},
],
});
});

it("doesn't create empty objects", () => {
const event = {
a: 'a',
b: 'b',
c: {
e: 'e',
},
};
expect(copyAllowlistedFields(allowlist, event)).toStrictEqual({
a: 'a',
b: 'b',
});
});
});
});
47 changes: 47 additions & 0 deletions x-pack/plugins/security_solution/server/lib/telemetry/filters.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
* 2.0.
*/

import { TelemetryEvent } from './types';

export interface AllowlistFields {
[key: string]: boolean | AllowlistFields;
}
Expand Down Expand Up @@ -124,3 +126,48 @@ export const allowlistEventFields: AllowlistFields = {
},
...allowlistBaseEventFields,
};

export const exceptionListEventFields: AllowlistFields = {
created_at: true,
description: true,
effectScope: true,
entries: true,
id: true,
name: true,
os: true,
os_types: true,
};

/**
* Filters out information not required for downstream analysis
*
* @param allowlist
* @param event
* @returns
*/
export function copyAllowlistedFields(
allowlist: AllowlistFields,
event: TelemetryEvent
): TelemetryEvent {
return Object.entries(allowlist).reduce<TelemetryEvent>((newEvent, [allowKey, allowValue]) => {
const eventValue = event[allowKey];
if (eventValue !== null && eventValue !== undefined) {
if (allowValue === true) {
return { ...newEvent, [allowKey]: eventValue };
} else if (typeof allowValue === 'object' && Array.isArray(eventValue)) {
const subValues = eventValue.filter((v) => typeof v === 'object');
return {
...newEvent,
[allowKey]: subValues.map((v) => copyAllowlistedFields(allowValue, v as TelemetryEvent)),
};
} else if (typeof allowValue === 'object' && typeof eventValue === 'object') {
const values = copyAllowlistedFields(allowValue, eventValue as TelemetryEvent);
return {
...newEvent,
...(Object.keys(values).length > 0 ? { [allowKey]: values } : {}),
};
}
}
return newEvent;
}, {});
}
40 changes: 19 additions & 21 deletions x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,19 @@

import moment from 'moment';
import { createMockPackagePolicy } from './mocks';
import { TrustedApp } from '../../../common/endpoint/types';
import { LIST_ENDPOINT_EXCEPTION, LIST_ENDPOINT_EVENT_FILTER } from './constants';
import {
LIST_ENDPOINT_EXCEPTION,
LIST_ENDPOINT_EVENT_FILTER,
LIST_TRUSTED_APPLICATION,
} from './constants';
import {
getPreviousDiagTaskTimestamp,
getPreviousEpMetaTaskTimestamp,
batchTelemetryRecords,
isPackagePolicyList,
templateTrustedApps,
templateEndpointExceptions,
templateExceptionList,
} from './helpers';
import { EndpointExceptionListItem } from './types';
import { ExceptionListItem } from './types';

describe('test diagnostic telemetry scheduled task timing helper', () => {
test('test -5 mins is returned when there is no previous task run', async () => {
Expand Down Expand Up @@ -133,17 +135,17 @@ describe('test package policy type guard', () => {

describe('list telemetry schema', () => {
test('trusted apps document is correctly formed', () => {
const data = [{ id: 'test_1' }] as TrustedApp[];
const templatedItems = templateTrustedApps(data);
const data = [{ id: 'test_1' }] as ExceptionListItem[];
const templatedItems = templateExceptionList(data, LIST_TRUSTED_APPLICATION);

expect(templatedItems[0]?.trusted_application.length).toEqual(1);
expect(templatedItems[0]?.endpoint_exception.length).toEqual(0);
expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0);
});

test('trusted apps document is correctly formed with multiple entries', () => {
const data = [{ id: 'test_2' }, { id: 'test_2' }] as TrustedApp[];
const templatedItems = templateTrustedApps(data);
const data = [{ id: 'test_2' }, { id: 'test_2' }] as ExceptionListItem[];
const templatedItems = templateExceptionList(data, LIST_TRUSTED_APPLICATION);

expect(templatedItems[0]?.trusted_application.length).toEqual(1);
expect(templatedItems[1]?.trusted_application.length).toEqual(1);
Expand All @@ -152,21 +154,17 @@ describe('list telemetry schema', () => {
});

test('endpoint exception document is correctly formed', () => {
const data = [{ id: 'test_3' }] as EndpointExceptionListItem[];
const templatedItems = templateEndpointExceptions(data, LIST_ENDPOINT_EXCEPTION);
const data = [{ id: 'test_3' }] as ExceptionListItem[];
const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EXCEPTION);

expect(templatedItems[0]?.trusted_application.length).toEqual(0);
expect(templatedItems[0]?.endpoint_exception.length).toEqual(1);
expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(0);
});

test('endpoint exception document is correctly formed with multiple entries', () => {
const data = [
{ id: 'test_4' },
{ id: 'test_4' },
{ id: 'test_4' },
] as EndpointExceptionListItem[];
const templatedItems = templateEndpointExceptions(data, LIST_ENDPOINT_EXCEPTION);
const data = [{ id: 'test_4' }, { id: 'test_4' }, { id: 'test_4' }] as ExceptionListItem[];
const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EXCEPTION);

expect(templatedItems[0]?.trusted_application.length).toEqual(0);
expect(templatedItems[0]?.endpoint_exception.length).toEqual(1);
Expand All @@ -176,17 +174,17 @@ describe('list telemetry schema', () => {
});

test('endpoint event filters document is correctly formed', () => {
const data = [{ id: 'test_5' }] as EndpointExceptionListItem[];
const templatedItems = templateEndpointExceptions(data, LIST_ENDPOINT_EVENT_FILTER);
const data = [{ id: 'test_5' }] as ExceptionListItem[];
const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EVENT_FILTER);

expect(templatedItems[0]?.trusted_application.length).toEqual(0);
expect(templatedItems[0]?.endpoint_exception.length).toEqual(0);
expect(templatedItems[0]?.endpoint_event_filter.length).toEqual(1);
});

test('endpoint event filters document is correctly formed with multiple entries', () => {
const data = [{ id: 'test_6' }, { id: 'test_6' }] as EndpointExceptionListItem[];
const templatedItems = templateEndpointExceptions(data, LIST_ENDPOINT_EVENT_FILTER);
const data = [{ id: 'test_6' }, { id: 'test_6' }] as ExceptionListItem[];
const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EVENT_FILTER);

expect(templatedItems[0]?.trusted_application.length).toEqual(0);
expect(templatedItems[0]?.endpoint_exception.length).toEqual(0);
Expand Down
Loading

0 comments on commit feae8e7

Please sign in to comment.