Skip to content

Commit

Permalink
[SIEM] [Detection Engine] Search signals index (#52661) (#52772)
Browse files Browse the repository at this point in the history
* adds route for querying signals index, also updates signal status type names

* first pass at happy path tests

* fixes stuff after rebase with master

* utilizes removes search_query from payload and replaces it with just query, adds aggs to signals search api, updates route and validation tests

* removes _headers parameter from route handler and updates comment for aggs script
  • Loading branch information
dhurley14 authored Dec 11, 2019
1 parent 3bf86b9 commit 4bc5a9d
Show file tree
Hide file tree
Showing 12 changed files with 323 additions and 17 deletions.
1 change: 1 addition & 0 deletions x-pack/legacy/plugins/siem/common/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,4 @@ export const DETECTION_ENGINE_INDEX_URL = `${DETECTION_ENGINE_URL}/index`;
export const SIGNALS_INDEX_KEY = 'signalsIndex';
export const DETECTION_ENGINE_SIGNALS_URL = `${DETECTION_ENGINE_URL}/signals`;
export const DETECTION_ENGINE_SIGNALS_STATUS_URL = `${DETECTION_ENGINE_SIGNALS_URL}/status`;
export const DETECTION_ENGINE_QUERY_SIGNALS_URL = `${DETECTION_ENGINE_SIGNALS_URL}/search`;
2 changes: 2 additions & 0 deletions x-pack/legacy/plugins/siem/server/kibana.index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import { findRulesRoute } from './lib/detection_engine/routes/rules/find_rules_r
import { deleteRulesRoute } from './lib/detection_engine/routes/rules/delete_rules_route';
import { updateRulesRoute } from './lib/detection_engine/routes/rules/update_rules_route';
import { setSignalsStatusRoute } from './lib/detection_engine/routes/signals/open_close_signals_route';
import { querySignalsRoute } from './lib/detection_engine/routes/signals/query_signals_route';
import { ServerFacade } from './types';
import { deleteIndexRoute } from './lib/detection_engine/routes/index/delete_index_route';
import { isAlertExecutor } from './lib/detection_engine/signals/types';
Expand Down Expand Up @@ -44,6 +45,7 @@ export const initServerWithKibana = (context: PluginInitializerContext, __legacy
// POST /api/detection_engine/signals/status
// Example usage can be found in siem/server/lib/detection_engine/scripts/signals
setSignalsStatusRoute(__legacy);
querySignalsRoute(__legacy);

// Detection Engine index routes that have the REST endpoints of /api/detection_engine/index
// All REST index creation, policy management for spaces
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@

import { ServerInjectOptions } from 'hapi';
import { ActionResult } from '../../../../../../actions/server/types';
import { SignalsRestParams } from '../../signals/types';
import { SignalsStatusRestParams, SignalsQueryRestParams } from '../../signals/types';
import {
DETECTION_ENGINE_RULES_URL,
DETECTION_ENGINE_SIGNALS_STATUS_URL,
DETECTION_ENGINE_QUERY_SIGNALS_URL,
} from '../../../../../common/constants';
import { RuleAlertType } from '../../rules/types';
import { RuleAlertParamsRest } from '../../types';
Expand Down Expand Up @@ -40,17 +41,25 @@ export const typicalPayload = (): Partial<Omit<RuleAlertParamsRest, 'filter'>> =
],
});

export const typicalSetStatusSignalByIdsPayload = (): Partial<SignalsRestParams> => ({
export const typicalSetStatusSignalByIdsPayload = (): Partial<SignalsStatusRestParams> => ({
signal_ids: ['somefakeid1', 'somefakeid2'],
status: 'closed',
});

export const typicalSetStatusSignalByQueryPayload = (): Partial<SignalsRestParams> => ({
export const typicalSetStatusSignalByQueryPayload = (): Partial<SignalsStatusRestParams> => ({
query: { range: { '@timestamp': { gte: 'now-2M', lte: 'now/M' } } },
status: 'closed',
});

export const setStatusSignalMissingIdsAndQueryPayload = (): Partial<SignalsRestParams> => ({
export const typicalSignalsQuery = (): Partial<SignalsQueryRestParams> => ({
query: { match_all: {} },
});

export const typicalSignalsQueryAggs = (): Partial<SignalsQueryRestParams> => ({
aggs: { statuses: { terms: { field: 'signal.status', size: 10 } } },
});

export const setStatusSignalMissingIdsAndQueryPayload = (): Partial<SignalsStatusRestParams> => ({
status: 'closed',
});

Expand Down Expand Up @@ -134,6 +143,18 @@ export const getSetSignalStatusByQueryRequest = (): ServerInjectOptions => ({
},
});

export const getSignalsQueryRequest = (): ServerInjectOptions => ({
method: 'POST',
url: DETECTION_ENGINE_QUERY_SIGNALS_URL,
payload: { ...typicalSignalsQuery() },
});

export const getSignalsAggsQueryRequest = (): ServerInjectOptions => ({
method: 'POST',
url: DETECTION_ENGINE_QUERY_SIGNALS_URL,
payload: { ...typicalSignalsQueryAggs() },
});

export const createActionResult = (): ActionResult => ({
id: 'result-1',
actionTypeId: 'action-id-1',
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { querySignalsSchema } from './query_signals_index_schema';
import { SignalsQueryRestParams } from '../../signals/types';

describe('query and aggs on signals index', () => {
test('query and aggs simultaneously', () => {
expect(
querySignalsSchema.validate<Partial<SignalsQueryRestParams>>({
query: {},
aggs: {},
}).error
).toBeFalsy();
});

test('query only', () => {
expect(
querySignalsSchema.validate<Partial<SignalsQueryRestParams>>({
query: {},
}).error
).toBeFalsy();
});

test('aggs only', () => {
expect(
querySignalsSchema.validate<Partial<SignalsQueryRestParams>>({
aggs: {},
}).error
).toBeFalsy();
});

test('missing query and aggs is invalid', () => {
expect(querySignalsSchema.validate<Partial<SignalsQueryRestParams>>({}).error).toBeTruthy();
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import Joi from 'joi';

export const querySignalsSchema = Joi.object({
query: Joi.object(),
aggs: Joi.object(),
}).min(1);
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
*/

import { setSignalsStatusSchema } from './set_signal_status_schema';
import { SignalsRestParams } from '../../signals/types';
import { SignalsStatusRestParams } from '../../signals/types';

describe('set signal status schema', () => {
test('signal_ids and status is valid', () => {
expect(
setSignalsStatusSchema.validate<Partial<SignalsRestParams>>({
setSignalsStatusSchema.validate<Partial<SignalsStatusRestParams>>({
signal_ids: ['somefakeid'],
status: 'open',
}).error
Expand All @@ -19,7 +19,7 @@ describe('set signal status schema', () => {

test('query and status is valid', () => {
expect(
setSignalsStatusSchema.validate<Partial<SignalsRestParams>>({
setSignalsStatusSchema.validate<Partial<SignalsStatusRestParams>>({
query: {},
status: 'open',
}).error
Expand All @@ -28,23 +28,23 @@ describe('set signal status schema', () => {

test('signal_ids and missing status is invalid', () => {
expect(
setSignalsStatusSchema.validate<Partial<SignalsRestParams>>({
setSignalsStatusSchema.validate<Partial<SignalsStatusRestParams>>({
signal_ids: ['somefakeid'],
}).error
).toBeTruthy();
});

test('query and missing status is invalid', () => {
expect(
setSignalsStatusSchema.validate<Partial<SignalsRestParams>>({
setSignalsStatusSchema.validate<Partial<SignalsStatusRestParams>>({
query: {},
}).error
).toBeTruthy();
});

test('status is present but query or signal_ids is missing is invalid', () => {
expect(
setSignalsStatusSchema.validate<Partial<SignalsRestParams>>({
setSignalsStatusSchema.validate<Partial<SignalsStatusRestParams>>({
status: 'closed',
}).error
).toBeTruthy();
Expand All @@ -54,7 +54,7 @@ describe('set signal status schema', () => {
expect(
setSignalsStatusSchema.validate<
Partial<
Omit<SignalsRestParams, 'status'> & {
Omit<SignalsStatusRestParams, 'status'> & {
status: string;
}
>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import Hapi from 'hapi';
import { DETECTION_ENGINE_SIGNALS_STATUS_URL } from '../../../../../common/constants';
import { SignalsRequest } from '../../signals/types';
import { SignalsStatusRequest } from '../../signals/types';
import { setSignalsStatusSchema } from '../schemas/set_signal_status_schema';
import { ServerFacade } from '../../../../types';
import { transformError, getIndex } from '../utils';
Expand All @@ -24,7 +24,7 @@ export const setSignalsStatusRouteDef = (server: ServerFacade): Hapi.ServerRoute
payload: setSignalsStatusSchema,
},
},
async handler(request: SignalsRequest, headers) {
async handler(request: SignalsStatusRequest) {
const { signal_ids: signalIds, query, status } = request.payload;
const index = getIndex(request, server);
const { callWithRequest } = server.plugins.elasticsearch.getCluster('data');
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { createMockServer } from '../__mocks__/_mock_server';
import { querySignalsRoute } from './query_signals_route';
import * as myUtils from '../utils';
import { ServerInjectOptions } from 'hapi';
import {
getSignalsQueryRequest,
getSignalsAggsQueryRequest,
typicalSignalsQuery,
typicalSignalsQueryAggs,
} from '../__mocks__/request_responses';
import { DETECTION_ENGINE_QUERY_SIGNALS_URL } from '../../../../../common/constants';

describe('query for signal', () => {
let { server, elasticsearch } = createMockServer();

beforeEach(() => {
jest.resetAllMocks();
jest.spyOn(myUtils, 'getIndex').mockReturnValue('fakeindex');
({ server, elasticsearch } = createMockServer());
elasticsearch.getCluster = jest.fn(() => ({
callWithRequest: jest.fn(() => true),
}));
querySignalsRoute(server);
});

describe('query and agg on signals index', () => {
test('returns 200 when using single query', async () => {
elasticsearch.getCluster = jest.fn(() => ({
callWithRequest: jest.fn(
(_req, _type: string, queryBody: { index: string; body: object }) => {
expect(queryBody.body).toMatchObject({ ...typicalSignalsQueryAggs() });
return true;
}
),
}));
const { statusCode } = await server.inject(getSignalsAggsQueryRequest());
expect(statusCode).toBe(200);
expect(myUtils.getIndex).toHaveReturnedWith('fakeindex');
});

test('returns 200 when using single agg', async () => {
elasticsearch.getCluster = jest.fn(() => ({
callWithRequest: jest.fn(
(_req, _type: string, queryBody: { index: string; body: object }) => {
expect(queryBody.body).toMatchObject({ ...typicalSignalsQueryAggs() });
return true;
}
),
}));
const { statusCode } = await server.inject(getSignalsAggsQueryRequest());
expect(statusCode).toBe(200);
expect(myUtils.getIndex).toHaveReturnedWith('fakeindex');
});

test('returns 200 when using aggs and query together', async () => {
const allTogether = getSignalsQueryRequest();
allTogether.payload = { ...typicalSignalsQueryAggs(), ...typicalSignalsQuery() };
elasticsearch.getCluster = jest.fn(() => ({
callWithRequest: jest.fn(
(_req, _type: string, queryBody: { index: string; body: object }) => {
expect(queryBody.body).toMatchObject({
...typicalSignalsQueryAggs(),
...typicalSignalsQuery(),
});
return true;
}
),
}));
const { statusCode } = await server.inject(allTogether);
expect(statusCode).toBe(200);
expect(myUtils.getIndex).toHaveReturnedWith('fakeindex');
});

test('returns 400 when missing aggs and query', async () => {
const allTogether = getSignalsQueryRequest();
allTogether.payload = {};
const { statusCode } = await server.inject(allTogether);
expect(statusCode).toBe(400);
});
});

describe('validation', () => {
test('returns 200 if query present', async () => {
const request: ServerInjectOptions = {
method: 'POST',
url: DETECTION_ENGINE_QUERY_SIGNALS_URL,
payload: typicalSignalsQuery(),
};
const { statusCode } = await server.inject(request);
expect(statusCode).toBe(200);
});

test('returns 200 if aggs is present', async () => {
const request: ServerInjectOptions = {
method: 'POST',
url: DETECTION_ENGINE_QUERY_SIGNALS_URL,
payload: typicalSignalsQueryAggs(),
};
const { statusCode } = await server.inject(request);
expect(statusCode).toBe(200);
});

test('returns 200 if aggs and query are present', async () => {
const request: ServerInjectOptions = {
method: 'POST',
url: DETECTION_ENGINE_QUERY_SIGNALS_URL,
payload: { ...typicalSignalsQueryAggs(), ...typicalSignalsQuery() },
};
const { statusCode } = await server.inject(request);
expect(statusCode).toBe(200);
});

test('returns 400 if aggs and query are NOT present', async () => {
const request: ServerInjectOptions = {
method: 'POST',
url: DETECTION_ENGINE_QUERY_SIGNALS_URL,
payload: {},
};
const { statusCode } = await server.inject(request);
expect(statusCode).toBe(400);
});
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import Hapi from 'hapi';
import { DETECTION_ENGINE_QUERY_SIGNALS_URL } from '../../../../../common/constants';
import { SignalsQueryRequest } from '../../signals/types';
import { querySignalsSchema } from '../schemas/query_signals_index_schema';
import { ServerFacade } from '../../../../types';
import { transformError, getIndex } from '../utils';

export const querySignalsRouteDef = (server: ServerFacade): Hapi.ServerRoute => {
return {
method: 'POST',
path: DETECTION_ENGINE_QUERY_SIGNALS_URL,
options: {
tags: ['access:siem'],
validate: {
options: {
abortEarly: false,
},
payload: querySignalsSchema,
},
},
async handler(request: SignalsQueryRequest) {
const { query, aggs } = request.payload;
const body = { query, aggs };
const index = getIndex(request, server);
const { callWithRequest } = server.plugins.elasticsearch.getCluster('data');
try {
return callWithRequest(request, 'search', {
index,
body,
});
} catch (exc) {
// error while getting or updating signal with id: id in signal index .siem-signals
return transformError(exc);
}
},
};
};

export const querySignalsRoute = (server: ServerFacade) => {
server.route(querySignalsRouteDef(server));
};
Loading

0 comments on commit 4bc5a9d

Please sign in to comment.