From 8c9a5fc98cdc554b5a7eb46df5d43e0cd2644174 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Tue, 8 Oct 2019 17:09:34 -0600 Subject: [PATCH 01/15] Added API endpoints and more scripts --- x-pack/legacy/plugins/siem/index.ts | 4 +- .../plugins/siem/server/kibana.index.ts | 8 ++ .../server/lib/detection_engine/README.md | 8 +- .../detection_engine/alerts/delete_signals.ts | 57 +++++++++ .../detection_engine/alerts/find_signals.ts | 30 +++++ .../detection_engine/alerts/read_signals.ts | 18 +++ .../detection_engine/alerts/update_signals.ts | 103 +++++++++++++++++ .../routes/delete_signals_route.ts | 40 +++++++ .../routes/find_signals_route.ts | 63 ++++++++++ .../routes/read_signals_route.ts | 36 ++++++ .../routes/updated_signals_route.ts | 108 ++++++++++++++++++ .../lib/detection_engine/scripts/README.md | 1 + .../scripts/check_env_variables.sh | 46 ++++++++ .../scripts/delete_all_actions.sh | 34 ++++++ .../scripts/delete_all_alert_tasks.sh | 23 ++++ .../scripts/delete_all_alerts.sh | 23 ++++ .../detection_engine/scripts/delete_signal.sh | 16 +++ .../scripts/delete_signal_index.sh | 9 ++ .../scripts/find_saved_object.sh | 9 ++ .../detection_engine/scripts/find_signals.sh | 15 +++ .../scripts/get_action_instances.sh | 9 ++ .../scripts/get_action_types.sh | 9 ++ .../scripts/get_alert_instances.sh | 9 ++ .../scripts/get_alert_tasks.sh | 9 ++ .../scripts/get_alert_types.sh | 9 ++ .../scripts/get_saved_objects.sh | 10 ++ .../scripts/get_signal_mapping.sh | 9 ++ .../detection_engine/scripts/hard_reset.sh | 16 +++ .../detection_engine/scripts/post_signal.sh | 9 ++ .../scripts/put_signal_index.sh | 10 +- .../detection_engine/scripts/read_signals.sh | 15 +++ .../scripts/signals/temp_update_1.json | 12 ++ .../detection_engine/scripts/update_signal.sh | 25 ++++ 33 files changed, 796 insertions(+), 6 deletions(-) create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/find_signals.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/read_signals.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/delete_signals_route.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/find_signals_route.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/read_signals_route.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/updated_signals_route.ts create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/check_env_variables.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_all_actions.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_all_alert_tasks.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_all_alerts.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_signals.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/read_signals.sh create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/temp_update_1.json create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/update_signal.sh diff --git a/x-pack/legacy/plugins/siem/index.ts b/x-pack/legacy/plugins/siem/index.ts index c3e2c2b0e119d..4b12afd80cf77 100644 --- a/x-pack/legacy/plugins/siem/index.ts +++ b/x-pack/legacy/plugins/siem/index.ts @@ -31,12 +31,12 @@ export function siem(kibana: any) { id: APP_ID, configPrefix: 'xpack.siem', publicDir: resolve(__dirname, 'public'), - require: ['kibana', 'elasticsearch'], + // require: ['kibana', 'elasticsearch'], // Uncomment these lines to turn on alerting and action for detection engine and comment the other // require statement out. These are hidden behind feature flags at the moment so if you turn // these on without the feature flags turned on then Kibana will crash since we are a legacy plugin // and legacy plugins cannot have optional requirements. - // require: ['kibana', 'elasticsearch', 'alerting', 'actions'], + require: ['kibana', 'elasticsearch', 'alerting', 'actions'], uiExports: { app: { description: i18n.translate('xpack.siem.securityDescription', { diff --git a/x-pack/legacy/plugins/siem/server/kibana.index.ts b/x-pack/legacy/plugins/siem/server/kibana.index.ts index ea29b7cdeef81..0e69cfda7af1a 100644 --- a/x-pack/legacy/plugins/siem/server/kibana.index.ts +++ b/x-pack/legacy/plugins/siem/server/kibana.index.ts @@ -17,6 +17,10 @@ import { } from './saved_objects'; import { createSignalsRoute } from './lib/detection_engine/routes/create_signals_route'; +import { readSignalsRoute } from './lib/detection_engine/routes/read_signals_route'; +import { findSignalsRoute } from './lib/detection_engine/routes/find_signals_route'; +import { deleteSignalsRoute } from './lib/detection_engine/routes/delete_signals_route'; +import { updateSignalsRoute } from './lib/detection_engine/routes/updated_signals_route'; const APP_ID = 'siem'; @@ -46,6 +50,10 @@ export const initServerWithKibana = (kbnServer: Server) => { 'Detected feature flags for actions and alerting and enabling signals API endpoints' ); createSignalsRoute(kbnServer); + readSignalsRoute(kbnServer); + updateSignalsRoute(kbnServer); + deleteSignalsRoute(kbnServer); + findSignalsRoute(kbnServer); } logger.info('Plugin done initializing'); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md b/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md index 3f18f95223408..1dc88b87ef9dc 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md @@ -17,6 +17,7 @@ export ELASTICSEARCH_URL=https://${ip}:9200 export KIBANA_URL=http://localhost:5601 export SIGNALS_INDEX=.siem-signals-${your user id} export TASK_MANAGER_INDEX=.kibana-task-manager-${your user id} +export KIBANA_INDEX=.kibana-${your user id} # This is for the kbn-action and kbn-alert tool export KBN_URLBASE=http://${user}:${password}@localhost:5601 @@ -62,18 +63,19 @@ server log [11:39:05.561] [info][siem] Detected feature flags for actions a Open a terminal and go into the scripts folder `cd kibana/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts` and run: ``` -./delete_signal_index.sh -./put_signal_index.sh +./hard_reset.sh ./post_signal.sh ``` which will: +* Delete any existing actions you have +* Delete any existing alerts you have +* Delete any existing alert tasks you have * Delete any existing signal mapping you might have had. * Add the latest signal index and its mappings * Posts a sample signal which checks for root or admin every 5 minutes - Now you can run ```sh diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts new file mode 100644 index 0000000000000..5f964343ae772 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { RawAlertAction, AlertAction } from '../../../../../alerting/server/types'; +import { ActionsClient } from '../../../../../actions/server/actions_client'; +import { AlertsClient } from '../../../../../alerting/server/alerts_client'; + +export interface DeleteSignalParams { + alertsClient: AlertsClient; + actionsClient: ActionsClient; + id: string; +} + +interface ObjectWithId { + id: AlertAction['id']; +} + +export const isObjectWithId = (obj: unknown): obj is ObjectWithId => { + return Object.getOwnPropertyDescriptor(obj, 'id') != null; +}; + +export const getObjectsWithId = (actions: unknown[] = []): ObjectWithId[] => { + return actions.reduce((accum, current) => { + if (isObjectWithId(current)) { + return [...accum, current]; + } else { + return accum; + } + }, []); +}; + +export const deleteAllSignalActions = async ( + actionsClient: ActionsClient, + actions: RawAlertAction[] | AlertAction[] | undefined +): Promise => { + const actionsWithIds = getObjectsWithId(actions); + try { + await Promise.all(actionsWithIds.map(async ({ id }) => actionsClient.delete({ id }))); + return null; + } catch (error) { + return error; + } +}; + +export const deleteSignals = async ({ alertsClient, actionsClient, id }: DeleteSignalParams) => { + const alert = await alertsClient.get({ id }); + const actionsErrors = await deleteAllSignalActions(actionsClient, alert.actions); + const deletedAlert = alertsClient.delete({ id }); + if (actionsErrors != null) { + throw actionsErrors; + } else { + return deletedAlert; + } +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/find_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/find_signals.ts new file mode 100644 index 0000000000000..0dd784d83f31f --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/find_signals.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SIGNALS_ID } from '../../../../common/constants'; +import { AlertsClient } from '../../../../../alerting/server/alerts_client'; + +export interface GetSignalParams { + alertsClient: AlertsClient; + perPage?: number; + page?: number; + sortField?: string; + fields?: string[]; +} + +// TODO: Change this from a search to a filter once this ticket is solved: +// https://github.com/elastic/kibana/projects/26#card-27462236 +export const findSignals = async ({ alertsClient, perPage, page, fields }: GetSignalParams) => { + return alertsClient.find({ + options: { + fields, + page, + perPage, + searchFields: ['alertTypeId'], + search: SIGNALS_ID, + }, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/read_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/read_signals.ts new file mode 100644 index 0000000000000..15b4cd57e3aa2 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/read_signals.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { AlertsClient } from '../../../../../alerting/server/alerts_client'; + +export interface ReadSignalParams { + alertsClient: AlertsClient; + id: string; +} + +// TODO: Change this from a search to a filter once this ticket is solved: +// https://github.com/elastic/kibana/projects/26#card-27462236 +export const readSignals = async ({ alertsClient, id }: ReadSignalParams) => { + return alertsClient.get({ id }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts new file mode 100644 index 0000000000000..52eaca707295d --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { AlertsClient } from '../../../../../alerting/server/alerts_client'; +import { ActionsClient } from '../../../../../actions/server/actions_client'; +import { readSignals } from './read_signals'; + +export interface SignalParams { + alertsClient: AlertsClient; + actionsClient: ActionsClient; + description: string; + from: string; + id: string; + index: string[]; + interval: string; + enabled: boolean; + filter: Record | undefined; + kql: string | undefined; + maxSignals: string; + name: string; + severity: number; + type: string; // TODO: Replace this type with a static enum type + to: string; + references: string[]; +} + +export const updateSignal = async ({ + alertsClient, + actionsClient, + description, + enabled, + filter, + from, + id, + index, + interval, + kql, + name, + severity, + to, + type, + references, +}: SignalParams) => { + // TODO: Error handling and abstraction. Right now if this is an error then what happens is we get the error of + // "message": "Saved object [alert/{id}] not found" + + const signal = await readSignals({ alertsClient, id }); + + /* + alertsClient.update({ + id, + data: { + interval, + actions: signal.actions, + }, + }); + */ + return signal; + /* + const actionResults = await actionsClient.create({ + action: { + actionTypeId: '.server-log', + description: 'SIEM Alerts Log', + config: {}, + secrets: {}, + }, + }); + + return alertsClient.create({ + data: { + alertTypeId: SIGNALS_ID, + alertTypeParams: { + description, + id, + index, + from, + filter, + kql, + name, + severity, + to, + type, + references, + }, + interval, + enabled, + actions: [ + { + group: 'default', + id: actionResults.id, + params: { + message: 'SIEM Alert Fired', + }, + }, + ], + throttle: null, + }, + }); + */ +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/delete_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/delete_signals_route.ts new file mode 100644 index 0000000000000..559077c862c1e --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/delete_signals_route.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Hapi from 'hapi'; +import { isFunction } from 'lodash/fp'; + +import { deleteSignals } from '../alerts/delete_signals'; + +export const deleteSignalsRoute = (server: Hapi.Server) => { + server.route({ + method: 'DELETE', + path: '/api/siem/signals/{id}', + options: { + tags: ['access:signals-all'], + validate: { + options: { + abortEarly: false, + }, + }, + }, + async handler(request: Hapi.Request, headers) { + const { id } = request.params; + const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null; + const actionsClient = isFunction(request.getActionsClient) + ? request.getActionsClient() + : null; + if (alertsClient == null || actionsClient == null) { + return headers.response().code(404); + } + return deleteSignals({ + actionsClient, + alertsClient, + id, + }); + }, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/find_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/find_signals_route.ts new file mode 100644 index 0000000000000..f774f8d76797d --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/find_signals_route.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Hapi from 'hapi'; +import Joi from 'joi'; +import { isFunction } from 'lodash/fp'; +import { findSignals } from '../alerts/find_signals'; + +interface FindSignalsRequest extends Omit { + query: { + per_page: number; + page: number; + search?: string; + sort_field?: string; + fields?: string[]; + }; +} + +export const findSignalsRoute = (server: Hapi.Server) => { + server.route({ + method: 'GET', + path: '/api/siem/signals/_find', + options: { + tags: ['access:signals-all'], + validate: { + options: { + abortEarly: false, + }, + query: Joi.object() + .keys({ + per_page: Joi.number() + .min(0) + .default(20), + page: Joi.number() + .min(1) + .default(1), + sort_field: Joi.string(), + fields: Joi.array() + .items(Joi.string()) + .single(), + }) + .default(), + }, + }, + async handler(request: FindSignalsRequest, headers) { + const { query } = request; + const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null; + if (alertsClient == null) { + return headers.response().code(404); + } + + return findSignals({ + alertsClient, + perPage: query.per_page, + page: query.page, + sortField: query.sort_field, + }); + }, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/read_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/read_signals_route.ts new file mode 100644 index 0000000000000..0d4f9ac0ef1fc --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/read_signals_route.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Hapi from 'hapi'; +import { isFunction } from 'lodash/fp'; + +import { readSignals } from '../alerts/read_signals'; + +export const readSignalsRoute = (server: Hapi.Server) => { + server.route({ + method: 'GET', + path: '/api/siem/signals/{id}', + options: { + tags: ['access:signals-all'], + validate: { + options: { + abortEarly: false, + }, + }, + }, + async handler(request: Hapi.Request, headers) { + const { id } = request.params; + const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null; + if (alertsClient == null) { + return headers.response().code(404); + } + return readSignals({ + alertsClient, + id, + }); + }, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/updated_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/updated_signals_route.ts new file mode 100644 index 0000000000000..2b672035df168 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/updated_signals_route.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Hapi from 'hapi'; +import Joi from 'joi'; +import { isFunction } from 'lodash/fp'; +import { updateSignal } from '../alerts/update_signals'; + +interface SignalsRequest extends Hapi.Request { + payload: { + description: string; + enabled: boolean; + filter: Record | undefined; + from: string; + id: string; + index: string[]; + interval: string; + kql: string | undefined; + max_signals: string; + name: string; + severity: number; + type: string; + to: string; + references: string[]; + }; +} + +export const updateSignalsRoute = (server: Hapi.Server) => { + server.route({ + method: 'PUT', + path: '/api/siem/signals', + options: { + tags: ['access:signals-all'], + validate: { + options: { + abortEarly: false, + }, + payload: Joi.object({ + description: Joi.string().required(), + enabled: Joi.boolean().default(true), + filter: Joi.object(), + from: Joi.string().required(), + id: Joi.string().required(), + index: Joi.array().required(), + interval: Joi.string().default('5m'), + kql: Joi.string(), + max_signals: Joi.array().default([]), + name: Joi.string().required(), + severity: Joi.number().required(), + to: Joi.string().required(), + type: Joi.string().required(), // TODO: Restrict this to only be kql or filter for the moment + references: Joi.array().default([]), + }).xor('filter', 'kql'), + }, + }, + async handler(request: SignalsRequest, headers) { + const { + description, + enabled, + filter, + kql, + from, + id, + index, + interval, + // eslint-disable-next-line @typescript-eslint/camelcase + max_signals: maxSignals, + name, + severity, + to, + type, + references, + } = request.payload; + + const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null; + + const actionsClient = isFunction(request.getActionsClient) + ? request.getActionsClient() + : null; + + if (!alertsClient || !actionsClient) { + return headers.response().code(404); + } + + return updateSignal({ + alertsClient, + actionsClient, + description, + enabled, + filter, + from, + id, + index, + interval, + kql, + maxSignals, + name, + severity, + to, + type, + references, + }); + }, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/README.md b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/README.md index 0258f5455ca87..b3ab0011e1f8f 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/README.md +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/README.md @@ -11,6 +11,7 @@ export ELASTICSEARCH_URL=https://${ip}:9200 export KIBANA_URL=http://localhost:5601 export SIGNALS_INDEX=.siem-signals-${your user id} export TASK_MANAGER_INDEX=.kibana-task-manager-${your user id} +export KIBANA_INDEX=.kibana-${your user id} # This is for the kbn-action and kbn-alert tool export KBN_URLBASE=http://${user}:${password}@localhost:5601 diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/check_env_variables.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/check_env_variables.sh new file mode 100755 index 0000000000000..c534b33d28413 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/check_env_variables.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# Add this to the start of any scripts to detect if env variables are set + +set -e + +if [ -z "${ELASTICSEARCH_USERNAME}" ]; then + echo "Set ELASTICSEARCH_USERNAME in your enviornment" + exit 1 +fi + +if [ -z "${ELASTICSEARCH_PASSWORD}" ]; then + echo "Set ELASTICSEARCH_PASSWORD in your enviornment" + exit 1 +fi + +if [ -z "${ELASTICSEARCH_URL}" ]; then + echo "Set ELASTICSEARCH_URL in your enviornment" + exit 1 +fi + +if [ -z "${KIBANA_URL}" ]; then + echo "Set KIBANA_URL in your enviornment" + exit 1 +fi + +if [ -z "${SIGNALS_INDEX}" ]; then + echo "Set SIGNALS_INDEX in your enviornment" + exit 1 +fi + +if [ -z "${TASK_MANAGER_INDEX}" ]; then + echo "Set TASK_MANAGER_INDEX in your enviornment" + exit 1 +fi + +if [ -z "${KIBANA_INDEX}" ]; then + echo "Set KIBANA_INDEX in your enviornment" + exit 1 +fi diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_all_actions.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_all_actions.sh new file mode 100755 index 0000000000000..e9818980c7506 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_all_actions.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./delete_all_actions.sh +# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html +curl -s -k \ + -H "Content-Type: application/json" \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${ELASTICSEARCH_URL}/${KIBANA_INDEX}*/_delete_by_query \ + --data '{ + "query": { + "exists": { "field": "action" } + } + }' \ + | jq . + +curl -s -k \ + -H "Content-Type: application/json" \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${ELASTICSEARCH_URL}/${KIBANA_INDEX}*/_delete_by_query \ + --data '{ + "query": { + "exists": { "field": "action_task_params" } + } + }' \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_all_alert_tasks.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_all_alert_tasks.sh new file mode 100755 index 0000000000000..cd285ac180219 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_all_alert_tasks.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./delete_all_alert_tasks.sh +# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html +curl -s -k \ + -H "Content-Type: application/json" \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${ELASTICSEARCH_URL}/${TASK_MANAGER_INDEX}*/_delete_by_query \ + --data '{ + "query": { + "term" : { "task.scope" : "alerting" } + } + }' \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_all_alerts.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_all_alerts.sh new file mode 100755 index 0000000000000..f4038d4afe0cb --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_all_alerts.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./delete_all_alerts.sh +# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html +curl -s -k \ + -H "Content-Type: application/json" \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${ELASTICSEARCH_URL}/${KIBANA_INDEX}*/_delete_by_query \ + --data '{ + "query": { + "exists": { "field": "alert" } + } + }' \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal.sh new file mode 100755 index 0000000000000..5d1b7551aa3c3 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./read_signals.sh ${id} +curl -s -k \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X DELETE ${KIBANA_URL}/api/siem/signals/$1 | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal_index.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal_index.sh index edb88d8b9338c..8d5deec1ba3a1 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal_index.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal_index.sh @@ -1,5 +1,14 @@ #!/bin/sh +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + # Example: ./delete_signal_index.sh # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-delete-index.html curl -s -k \ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_saved_object.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_saved_object.sh index 3b9b8686df9a6..2b26c939a924c 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_saved_object.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_saved_object.sh @@ -1,5 +1,14 @@ #!/bin/sh +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + # Uses a default of alert if no argument is specified TYPE=${1:-alert} diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_signals.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_signals.sh new file mode 100755 index 0000000000000..f851bda0c12c9 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_signals.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./find_signals.sh +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${KIBANA_URL}/api/siem/signals/_find | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_instances.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_instances.sh index e04bdba4f1073..e2177bb750057 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_instances.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_instances.sh @@ -1,5 +1,14 @@ #!/bin/sh +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + # Example: ./get_action_instances.sh # https://github.com/elastic/kibana/blob/master/x-pack/legacy/plugins/actions/README.md#get-apiaction_find-find-actions curl -s -k \ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_types.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_types.sh index 47a571f47bf68..7937f2f99a37f 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_types.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_types.sh @@ -1,5 +1,14 @@ #!/bin/sh +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + # Example: ./get_action_types.sh # https://github.com/elastic/kibana/blob/master/x-pack/legacy/plugins/actions/README.md curl -s -k \ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_instances.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_instances.sh index a3c03d56d6506..3abc8c9adee62 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_instances.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_instances.sh @@ -1,5 +1,14 @@ #!/bin/sh +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + # Example: ./get_alert_instances.sh # https://github.com/elastic/kibana/blob/master/x-pack/legacy/plugins/alerting/README.md#get-apialert_find-find-alerts curl -s -k \ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_tasks.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_tasks.sh index 210fc2d532b7d..8a94c254ab656 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_tasks.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_tasks.sh @@ -1,5 +1,14 @@ #!/bin/sh +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + # Example: ./get_alert_tasks.sh # https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html curl -s -k \ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_types.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_types.sh index e70bbb9e882c3..7f7361a6252bc 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_types.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_types.sh @@ -1,5 +1,14 @@ #!/bin/sh +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + # Example: ./get_alert_types.sh # https://github.com/elastic/kibana/blob/master/x-pack/legacy/plugins/alerting/README.md#get-apialerttypes-list-alert-types curl -s -k \ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_saved_objects.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_saved_objects.sh index 71725ed308a85..4829beba86743 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_saved_objects.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_saved_objects.sh @@ -1,6 +1,16 @@ #!/bin/sh +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + # Example: ./get_saved_object.sh +# Example: ./get_saved_objects.sh alert 836dab88-edff-42a5-a219-4aae46fcd385 # https://www.elastic.co/guide/en/kibana/master/saved-objects-api-get.html curl -s -k \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_signal_mapping.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_signal_mapping.sh index d30f66df72eda..8b384fcc76f72 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_signal_mapping.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_signal_mapping.sh @@ -1,5 +1,14 @@ #!/bin/sh +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + # Example: ./get_signal_mapping.sh # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-mapping.html curl -s -k \ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh new file mode 100755 index 0000000000000..ee8fa18e1234d --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +./delete_all_actions.sh +./delete_all_alerts.sh +./delete_all_alert_tasks.sh +./delete_signal_index.sh +./put_signal_index.sh diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh index 3393084c56c96..961953c71b2dc 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh @@ -1,5 +1,14 @@ #!/bin/sh +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + # Uses a default if no argument is specified SIGNAL=${1:-./signals/root_or_admin_1.json} diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/put_signal_index.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/put_signal_index.sh index 54b60c7ccdbe6..1b3b148a99161 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/put_signal_index.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/put_signal_index.sh @@ -1,7 +1,15 @@ #!/bin/sh +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + # Example: ./put_signal_index.sh -# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-put-mapping.html curl -s -k \ -H "Content-Type: application/json" \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/read_signals.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/read_signals.sh new file mode 100755 index 0000000000000..fa8c9941337f4 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/read_signals.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./read_signals.sh {id} +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${KIBANA_URL}/api/siem/signals/$1 | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/temp_update_1.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/temp_update_1.json new file mode 100644 index 0000000000000..2f1e917cd3124 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/temp_update_1.json @@ -0,0 +1,12 @@ +{ + "id": "85e55248-7b90-4e9c-afa2-4ff97baf3de3", + "description": "Detecting root and admin users", + "index": ["auditbeat-*", "filebeat-*", "packetbeat-*", "winlogbeat-*"], + "interval": "5m", + "name": "Detect Root/Admin Users", + "severity": 1, + "type": "kql", + "from": "now-6m", + "to": "now", + "kql": "user.name: root or user.name: admin" +} diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/update_signal.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/update_signal.sh new file mode 100755 index 0000000000000..0980999156f11 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/update_signal.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# TODO: Since we only have GUID's at the moment, you have to use ./find_signals.sh and then copy and paste that +# into the temp_update_1.json as your ID in order to run this script. + +# Uses a default if no argument is specified +SIGNAL=${1:-./signals/temp_update_1.json} + +# Example: ./update_signal.sh {id} ./signals/root_or_admin_1.json +curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X PUT ${KIBANA_URL}/api/siem/signals \ + -d @${SIGNAL} \ + | jq . From 91953a1a9373aff3fbcbfc71a4d3c16a5f1f2a2f Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Wed, 9 Oct 2019 14:12:19 -0600 Subject: [PATCH 02/15] Added the update endpoint and scripts for the support of it. Changed some of the data types around --- .../detection_engine/alerts/create_signal.ts | 1 + .../alerts/signals_alert_type.ts | 4 +- .../detection_engine/alerts/update_signals.ts | 138 ++++++++++-------- .../routes/create_signals_route.ts | 2 +- .../routes/updated_signals_route.ts | 22 +-- .../scripts/signals/temp_update_1.json | 21 +-- 6 files changed, 103 insertions(+), 85 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts index 6ebdecfeeba83..8685b4f082fa5 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts @@ -82,6 +82,7 @@ export const createSignal = async ({ id: actionResults.id, params: { message: 'SIEM Alert Fired', + level: 'info', }, }, ], diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts index 5872e555ca344..21e84eae5ca2e 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts @@ -25,10 +25,10 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): AlertType => { params: schema.object({ description: schema.string(), from: schema.string(), - filter: schema.maybe(schema.object({}, { allowUnknowns: true })), + filter: schema.nullable(schema.object({}, { allowUnknowns: true })), id: schema.number(), index: schema.arrayOf(schema.string()), - kql: schema.maybe(schema.string({ defaultValue: undefined })), + kql: schema.nullable(schema.string()), maxSignals: schema.number({ defaultValue: 100 }), name: schema.string(), severity: schema.number(), diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts index 52eaca707295d..b1932825bf37a 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { defaults } from 'lodash/fp'; +import { AlertAction } from '../../../../../alerting/server/types'; import { AlertsClient } from '../../../../../alerting/server/alerts_client'; import { ActionsClient } from '../../../../../actions/server/actions_client'; import { readSignals } from './read_signals'; @@ -11,25 +13,51 @@ import { readSignals } from './read_signals'; export interface SignalParams { alertsClient: AlertsClient; actionsClient: ActionsClient; - description: string; - from: string; + description?: string; + from?: string; id: string; - index: string[]; - interval: string; - enabled: boolean; - filter: Record | undefined; - kql: string | undefined; - maxSignals: string; - name: string; - severity: number; - type: string; // TODO: Replace this type with a static enum type - to: string; - references: string[]; + index?: string[]; + interval?: string; + enabled?: boolean; + filter?: Record | undefined; + kql?: string | undefined; + maxSignals?: string; + name?: string; + severity?: number; + type?: string; // TODO: Replace this type with a static enum type + to?: string; + references?: string[]; } +export const calculateInterval = ( + interval: string | undefined, + signalInterval: string | undefined +): string => { + if (interval != null) { + return interval; + } else if (signalInterval != null) { + return signalInterval; + } else { + return '5m'; + } +}; + +export const calculateKqlAndFilter = ( + kql: string | undefined, + filter: {} | undefined +): { kql: string | null | undefined; filter: {} | null | undefined } => { + if (filter != null) { + return { kql: null, filter }; + } else if (kql != null) { + return { kql, filter: null }; + } else { + return { kql: undefined, filter: undefined }; + } +}; + export const updateSignal = async ({ alertsClient, - actionsClient, + actionsClient, // TODO: Use this whenever we add feature support for different action types description, enabled, filter, @@ -46,58 +74,46 @@ export const updateSignal = async ({ }: SignalParams) => { // TODO: Error handling and abstraction. Right now if this is an error then what happens is we get the error of // "message": "Saved object [alert/{id}] not found" - const signal = await readSignals({ alertsClient, id }); - /* - alertsClient.update({ - id, - data: { - interval, - actions: signal.actions, - }, - }); - */ - return signal; - /* - const actionResults = await actionsClient.create({ - action: { - actionTypeId: '.server-log', - description: 'SIEM Alerts Log', - config: {}, - secrets: {}, + // TODO: Remove this as cast as soon as signal.actions TypeScript bug is fixed + // where it is trying to return AlertAction[] or RawAlertAction[] + const actions = (signal.actions as AlertAction[]) || []; + + const alertTypeParams = signal.alertTypeParams || {}; + + const { kql: nextKql, filter: nextFilter } = calculateKqlAndFilter(kql, filter); + + const nextAlertTypeParams = defaults( + { + ...alertTypeParams, }, - }); + { + description, + filter: nextFilter, + from, + index, + kql: nextKql, + name, + severity, + to, + type, + references, + } + ); + + if (signal.enabled && !enabled) { + await alertsClient.disable({ id }); + } else if (!signal.enabled && enabled) { + await alertsClient.enable({ id }); + } - return alertsClient.create({ + return alertsClient.update({ + id, data: { - alertTypeId: SIGNALS_ID, - alertTypeParams: { - description, - id, - index, - from, - filter, - kql, - name, - severity, - to, - type, - references, - }, - interval, - enabled, - actions: [ - { - group: 'default', - id: actionResults.id, - params: { - message: 'SIEM Alert Fired', - }, - }, - ], - throttle: null, + interval: calculateInterval(interval, signal.interval), + actions, + alertTypeParams: nextAlertTypeParams, }, }); - */ }; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts index 19070243520b8..5ba8e9e363209 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts @@ -47,7 +47,7 @@ export const createSignalsRoute = (server: Hapi.Server) => { index: Joi.array().required(), interval: Joi.string().default('5m'), kql: Joi.string(), - max_signals: Joi.array().default([]), + max_signals: Joi.number().default(100), name: Joi.string().required(), severity: Joi.number().required(), to: Joi.string().required(), diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/updated_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/updated_signals_route.ts index 2b672035df168..6ebb925e8ef6e 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/updated_signals_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/updated_signals_route.ts @@ -39,21 +39,21 @@ export const updateSignalsRoute = (server: Hapi.Server) => { abortEarly: false, }, payload: Joi.object({ - description: Joi.string().required(), - enabled: Joi.boolean().default(true), + description: Joi.string(), + enabled: Joi.boolean(), filter: Joi.object(), - from: Joi.string().required(), + from: Joi.string(), id: Joi.string().required(), - index: Joi.array().required(), - interval: Joi.string().default('5m'), + index: Joi.array(), + interval: Joi.string(), kql: Joi.string(), - max_signals: Joi.array().default([]), - name: Joi.string().required(), - severity: Joi.number().required(), - to: Joi.string().required(), - type: Joi.string().required(), // TODO: Restrict this to only be kql or filter for the moment + max_signals: Joi.number().default(100), + name: Joi.string(), + severity: Joi.number(), + to: Joi.string(), + type: Joi.string(), // TODO: Restrict this to only be kql or filter for the moment references: Joi.array().default([]), - }).xor('filter', 'kql'), + }).nand('filter', 'kql'), }, }, async handler(request: SignalsRequest, headers) { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/temp_update_1.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/temp_update_1.json index 2f1e917cd3124..8ca571666ea4e 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/temp_update_1.json +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/temp_update_1.json @@ -1,12 +1,13 @@ { - "id": "85e55248-7b90-4e9c-afa2-4ff97baf3de3", - "description": "Detecting root and admin users", - "index": ["auditbeat-*", "filebeat-*", "packetbeat-*", "winlogbeat-*"], - "interval": "5m", - "name": "Detect Root/Admin Users", - "severity": 1, - "type": "kql", - "from": "now-6m", - "to": "now", - "kql": "user.name: root or user.name: admin" + "id": "f47e5de7-0023-4898-b856-cf3874e883ea", + "description": "Only watch winlogbeat users", + "index": ["winlogbeat-*"], + "interval": "9m", + "name": "Just watch other winlogbeat users", + "severity": 500, + "enabled": false, + "type": "filter", + "from": "now-5d", + "to": "now-1d", + "kql": "user.name: something_else" } From 84739d469d2a012f75c6f193c3a3166bcf7c500b Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Wed, 9 Oct 2019 14:13:37 -0600 Subject: [PATCH 03/15] put back the require statement for the builds to work correctly --- x-pack/legacy/plugins/siem/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/siem/index.ts b/x-pack/legacy/plugins/siem/index.ts index 7ac09a5393930..eadb680c030e5 100644 --- a/x-pack/legacy/plugins/siem/index.ts +++ b/x-pack/legacy/plugins/siem/index.ts @@ -32,12 +32,12 @@ export function siem(kibana: any) { id: APP_ID, configPrefix: 'xpack.siem', publicDir: resolve(__dirname, 'public'), - // require: ['kibana', 'elasticsearch'], + require: ['kibana', 'elasticsearch'], // Uncomment these lines to turn on alerting and action for detection engine and comment the other // require statement out. These are hidden behind feature flags at the moment so if you turn // these on without the feature flags turned on then Kibana will crash since we are a legacy plugin // and legacy plugins cannot have optional requirements. - require: ['kibana', 'elasticsearch', 'alerting', 'actions'], + // require: ['kibana', 'elasticsearch', 'alerting', 'actions'], uiExports: { app: { description: i18n.translate('xpack.siem.securityDescription', { From 91f2797d04816d6342b59eb865ed1d0ca14d4bfc Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Wed, 9 Oct 2019 14:41:58 -0600 Subject: [PATCH 04/15] Added casting magic with a TODO block --- .../detection_engine/alerts/delete_signals.ts | 32 ++++++------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts index 5f964343ae772..70cd9ed126765 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { RawAlertAction, AlertAction } from '../../../../../alerting/server/types'; +import { AlertAction } from '../../../../../alerting/server/types'; import { ActionsClient } from '../../../../../actions/server/actions_client'; import { AlertsClient } from '../../../../../alerting/server/alerts_client'; @@ -14,31 +14,12 @@ export interface DeleteSignalParams { id: string; } -interface ObjectWithId { - id: AlertAction['id']; -} - -export const isObjectWithId = (obj: unknown): obj is ObjectWithId => { - return Object.getOwnPropertyDescriptor(obj, 'id') != null; -}; - -export const getObjectsWithId = (actions: unknown[] = []): ObjectWithId[] => { - return actions.reduce((accum, current) => { - if (isObjectWithId(current)) { - return [...accum, current]; - } else { - return accum; - } - }, []); -}; - export const deleteAllSignalActions = async ( actionsClient: ActionsClient, - actions: RawAlertAction[] | AlertAction[] | undefined + actions: AlertAction[] | undefined = [] ): Promise => { - const actionsWithIds = getObjectsWithId(actions); try { - await Promise.all(actionsWithIds.map(async ({ id }) => actionsClient.delete({ id }))); + await Promise.all(actions.map(async ({ id }) => actionsClient.delete({ id }))); return null; } catch (error) { return error; @@ -47,7 +28,12 @@ export const deleteAllSignalActions = async ( export const deleteSignals = async ({ alertsClient, actionsClient, id }: DeleteSignalParams) => { const alert = await alertsClient.get({ id }); - const actionsErrors = await deleteAllSignalActions(actionsClient, alert.actions); + + // TODO: Remove this as cast as soon as signal.actions TypeScript bug is fixed + // where it is trying to return AlertAction[] or RawAlertAction[] + const actions = alert.actions as (AlertAction[] | undefined); + + const actionsErrors = await deleteAllSignalActions(actionsClient, actions); const deletedAlert = alertsClient.delete({ id }); if (actionsErrors != null) { throw actionsErrors; From ee7c2c300f8a618442c580237f9927b7a7ac0480 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Wed, 9 Oct 2019 14:44:13 -0600 Subject: [PATCH 05/15] cleaned up more types --- .../siem/server/lib/detection_engine/alerts/delete_signals.ts | 4 ++-- .../siem/server/lib/detection_engine/alerts/update_signals.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts index 70cd9ed126765..a08db6145836c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts @@ -16,7 +16,7 @@ export interface DeleteSignalParams { export const deleteAllSignalActions = async ( actionsClient: ActionsClient, - actions: AlertAction[] | undefined = [] + actions: AlertAction[] ): Promise => { try { await Promise.all(actions.map(async ({ id }) => actionsClient.delete({ id }))); @@ -31,7 +31,7 @@ export const deleteSignals = async ({ alertsClient, actionsClient, id }: DeleteS // TODO: Remove this as cast as soon as signal.actions TypeScript bug is fixed // where it is trying to return AlertAction[] or RawAlertAction[] - const actions = alert.actions as (AlertAction[] | undefined); + const actions = (alert.actions as (AlertAction[] | undefined)) || []; const actionsErrors = await deleteAllSignalActions(actionsClient, actions); const deletedAlert = alertsClient.delete({ id }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts index b1932825bf37a..7c307695dd2a5 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts @@ -78,7 +78,7 @@ export const updateSignal = async ({ // TODO: Remove this as cast as soon as signal.actions TypeScript bug is fixed // where it is trying to return AlertAction[] or RawAlertAction[] - const actions = (signal.actions as AlertAction[]) || []; + const actions = (signal.actions as AlertAction[] | undefined) || []; const alertTypeParams = signal.alertTypeParams || {}; From 5fe3a77e032d734d277b6bb5bbc12b45ff125aa1 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Wed, 9 Oct 2019 15:14:55 -0600 Subject: [PATCH 06/15] Updated per code review --- .../siem/server/lib/detection_engine/scripts/delete_signal.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal.sh index 5d1b7551aa3c3..c393665315e25 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal.sh @@ -9,7 +9,7 @@ set -e ./check_env_variables.sh -# Example: ./read_signals.sh ${id} +# Example: ./delete_signal.sh ${id} curl -s -k \ -H 'kbn-xsrf: 123' \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ From 42fbb44b6d760c08cb6c7256c0ae863a04f3912a Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Wed, 9 Oct 2019 15:17:15 -0600 Subject: [PATCH 07/15] Changed per code review --- .../scripts/{read_signals.sh => read_signal.sh} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/{read_signals.sh => read_signal.sh} (92%) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/read_signals.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/read_signal.sh similarity index 92% rename from x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/read_signals.sh rename to x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/read_signal.sh index fa8c9941337f4..7eb07e6e2dedf 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/read_signals.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/read_signal.sh @@ -9,7 +9,7 @@ set -e ./check_env_variables.sh -# Example: ./read_signals.sh {id} +# Example: ./read_signal.sh {id} curl -s -k \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ -X GET ${KIBANA_URL}/api/siem/signals/$1 | jq . From a5f544839512824854587f0a95fdb65eb6c54dad Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Wed, 9 Oct 2019 17:17:49 -0600 Subject: [PATCH 08/15] Updated per code review --- .../siem/server/lib/detection_engine/alerts/delete_signals.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts index a08db6145836c..dad9147d9eb68 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts @@ -34,7 +34,7 @@ export const deleteSignals = async ({ alertsClient, actionsClient, id }: DeleteS const actions = (alert.actions as (AlertAction[] | undefined)) || []; const actionsErrors = await deleteAllSignalActions(actionsClient, actions); - const deletedAlert = alertsClient.delete({ id }); + const deletedAlert = await alertsClient.delete({ id }); if (actionsErrors != null) { throw actionsErrors; } else { From aa7108b15bdd975459beb74b96cf77e60c21c9a0 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Wed, 9 Oct 2019 18:40:08 -0600 Subject: [PATCH 09/15] Added optional id parameter in the URL that can be sent --- .../routes/updated_signals_route.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/updated_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/updated_signals_route.ts index 6ebb925e8ef6e..be567695f730d 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/updated_signals_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/updated_signals_route.ts @@ -31,19 +31,26 @@ interface SignalsRequest extends Hapi.Request { export const updateSignalsRoute = (server: Hapi.Server) => { server.route({ method: 'PUT', - path: '/api/siem/signals', + path: '/api/siem/signals/{id?}', options: { tags: ['access:signals-all'], validate: { options: { abortEarly: false, }, + params: { + id: Joi.when(Joi.ref('$payload.id'), { + is: Joi.exist(), + then: Joi.string().optional(), + otherwise: Joi.string().required(), + }), + }, payload: Joi.object({ description: Joi.string(), enabled: Joi.boolean(), filter: Joi.object(), from: Joi.string(), - id: Joi.string().required(), + id: Joi.string(), index: Joi.array(), interval: Joi.string(), kql: Joi.string(), @@ -74,7 +81,6 @@ export const updateSignalsRoute = (server: Hapi.Server) => { type, references, } = request.payload; - const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null; const actionsClient = isFunction(request.getActionsClient) @@ -92,7 +98,7 @@ export const updateSignalsRoute = (server: Hapi.Server) => { enabled, filter, from, - id, + id: request.params.id ? request.params.id : id, index, interval, kql, From eadac5d2d3399f6c87eb81b08996b528b63649f2 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Wed, 9 Oct 2019 18:55:22 -0600 Subject: [PATCH 10/15] added some unit tests --- .../alerts/update_signals.test.ts | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.test.ts diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.test.ts new file mode 100644 index 0000000000000..e3e00e5cea5ce --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.test.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { calculateInterval, calculateKqlAndFilter } from './update_signals'; + +describe('update_signals', () => { + describe('#calculateInterval', () => { + test('given a undefined interval, it returns the signalInterval ', () => { + const interval = calculateInterval(undefined, '10m'); + expect(interval).toEqual('10m'); + }); + + test('given a undefined signalInterval, it returns a undefined interval ', () => { + const interval = calculateInterval('10m', undefined); + expect(interval).toEqual('10m'); + }); + + test('given both an undefined signalInterval and a undefined interval, it returns 5m', () => { + const interval = calculateInterval(undefined, undefined); + expect(interval).toEqual('5m'); + }); + }); + + describe('#calculateKqlAndFilter', () => { + test('given a undefined kql filter it returns a null kql', () => { + const kqlFilter = calculateKqlAndFilter(undefined, {}); + expect(kqlFilter).toEqual({ + filter: {}, + kql: null, + }); + }); + + test('given a undefined filter it returns a null filter', () => { + const kqlFilter = calculateKqlAndFilter('some kql string', undefined); + expect(kqlFilter).toEqual({ + filter: null, + kql: 'some kql string', + }); + }); + + test('given both a undefined filter and undefined kql it returns both as undefined', () => { + const kqlFilter = calculateKqlAndFilter(undefined, undefined); + expect(kqlFilter).toEqual({ + filter: undefined, + kql: undefined, + }); + }); + }); +}); From 9a3c0367b065d26ed93a78c7a3a13eb25cf489f4 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Thu, 10 Oct 2019 17:13:17 -0600 Subject: [PATCH 11/15] Does all crud through a request params called alert_id instead of the _id --- .../detection_engine/alerts/create_signal.ts | 154 +++++++++++++----- .../detection_engine/alerts/delete_signals.ts | 7 +- .../detection_engine/alerts/read_signals.ts | 56 ++++++- .../lib/detection_engine/alerts/types.ts | 40 +++++ .../detection_engine/alerts/update_signals.ts | 6 +- .../scripts/post_x_signals.sh | 36 ++++ ...ate_1.json => root_or_admin_update_1.json} | 2 +- .../detection_engine/scripts/update_signal.sh | 5 +- 8 files changed, 252 insertions(+), 54 deletions(-) create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_x_signals.sh rename x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/{temp_update_1.json => root_or_admin_update_1.json} (85%) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts index 8685b4f082fa5..008526e0688bd 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts @@ -7,6 +7,7 @@ import { SIGNALS_ID } from '../../../../common/constants'; import { AlertsClient } from '../../../../../alerting/server/alerts_client'; import { ActionsClient } from '../../../../../actions/server/actions_client'; +import { updateSignal } from './update_signals'; export interface SignalParams { alertsClient: AlertsClient; @@ -27,7 +28,9 @@ export interface SignalParams { references: string[]; } -export const createSignal = async ({ +// TODO: This updateIfIdExists should be temporary and we will remove it once we can POST id's directly to +// the alerting framework. +export const updateIfIdExists = async ({ alertsClient, actionsClient, description, @@ -44,49 +47,116 @@ export const createSignal = async ({ type, references, }: SignalParams) => { - // TODO: Right now we are using the .server-log as the default action as each alert has to have - // at least one action or it will not be able to do in-memory persistence. When adding in actions - // such as email, slack, etc... this should be the default action if not action is specified to - // create signals + try { + const signal = await updateSignal({ + alertsClient, + actionsClient, + description, + enabled, + filter, + from, + id, + index, + interval, + kql, + name, + severity, + to, + type, + references, + }); + return signal; + } catch (err) { + // This happens when we cannot get a saved object back from reading a signal. + // So we continue normally as we have nothing we can upsert. + } + return null; +}; - const actionResults = await actionsClient.create({ - action: { - actionTypeId: '.server-log', - description: 'SIEM Alerts Log', - config: {}, - secrets: {}, - }, +export const createSignal = async ({ + alertsClient, + actionsClient, + description, + enabled, + filter, + from, + id, + index, + interval, + kql, + maxSignals, + name, + severity, + to, + type, + references, +}: SignalParams) => { + // TODO: Once we can post directly to _id we will not have to do this part anymore. + const signalUpdating = await updateIfIdExists({ + alertsClient, + actionsClient, + description, + enabled, + filter, + from, + id, + index, + interval, + kql, + maxSignals, + name, + severity, + to, + type, + references, }); - - return alertsClient.create({ - data: { - alertTypeId: SIGNALS_ID, - alertTypeParams: { - description, - id, - index, - from, - filter, - kql, - name, - severity, - to, - type, - references, + if (signalUpdating == null) { + // TODO: Right now we are using the .server-log as the default action as each alert has to have + // at least one action or it will not be able to do in-memory persistence. When adding in actions + // such as email, slack, etc... this should be the default action if not action is specified to + // create signals + const actionResults = await actionsClient.create({ + action: { + actionTypeId: '.server-log', + description: 'SIEM Alerts Log', + config: {}, + secrets: {}, }, - interval, - enabled, - actions: [ - { - group: 'default', - id: actionResults.id, - params: { - message: 'SIEM Alert Fired', - level: 'info', - }, + }); + + return alertsClient.create({ + data: { + alertTypeId: SIGNALS_ID, + alertTypeParams: { + description, + id, + index, + from, + filter, + kql, + maxSignals, + name, + severity, + to, + type, + references, }, - ], - throttle: null, - }, - }); + interval, + enabled, + actions: [ + { + group: 'default', + id: actionResults.id, + params: { + message: 'SIEM Alert Fired', + level: 'info', + }, + }, + ], + throttle: null, + }, + }); + } else { + return signalUpdating; + } }; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts index dad9147d9eb68..7c49a8d49b1ea 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts @@ -7,6 +7,7 @@ import { AlertAction } from '../../../../../alerting/server/types'; import { ActionsClient } from '../../../../../actions/server/actions_client'; import { AlertsClient } from '../../../../../alerting/server/alerts_client'; +import { readSignals } from './read_signals'; export interface DeleteSignalParams { alertsClient: AlertsClient; @@ -27,14 +28,14 @@ export const deleteAllSignalActions = async ( }; export const deleteSignals = async ({ alertsClient, actionsClient, id }: DeleteSignalParams) => { - const alert = await alertsClient.get({ id }); + const signal = await readSignals({ alertsClient, id }); // TODO: Remove this as cast as soon as signal.actions TypeScript bug is fixed // where it is trying to return AlertAction[] or RawAlertAction[] - const actions = (alert.actions as (AlertAction[] | undefined)) || []; + const actions = (signal.actions as (AlertAction[] | undefined)) || []; const actionsErrors = await deleteAllSignalActions(actionsClient, actions); - const deletedAlert = await alertsClient.delete({ id }); + const deletedAlert = await alertsClient.delete({ id: signal.id }); if (actionsErrors != null) { throw actionsErrors; } else { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/read_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/read_signals.ts index 15b4cd57e3aa2..fe122cedbb5de 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/read_signals.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/read_signals.ts @@ -5,14 +5,68 @@ */ import { AlertsClient } from '../../../../../alerting/server/alerts_client'; +import { findSignals } from './find_signals'; +import { SignalAlertType, isAlertTypeArray } from './types'; export interface ReadSignalParams { alertsClient: AlertsClient; id: string; } +export interface ReadSignalByIdParams { + alertsClient: AlertsClient; + id: string; +} + +export const findSignalInArrayById = (objects: object[], id: string): SignalAlertType | null => { + if (isAlertTypeArray(objects)) { + const signals: SignalAlertType[] = objects; + const signal: SignalAlertType[] = signals.filter(datum => { + // TODO: Change String(datum.alertTypeParams.id) below once the data type is a string + // in the alert params schema + return String(datum.alertTypeParams.id) === id; + }); + if (signal.length !== 0) { + return signal[0]; + } else { + return null; + } + } else { + return null; + } +}; + +// This an extremely slow and inefficient way of getting a signal by its id. +// I have to manually query every single record since the Signal Params are +// not indexed and I cannot push in my own _id when I create an alert at the moment. +// TODO: Once we can directly push in the _id, then we should no longer need this way. +// TODO: This is meant to be _very_ temporary. +export const readSignalById = async ({ + alertsClient, + id, +}: ReadSignalByIdParams): Promise => { + let length: number = 0; + let page: number = 1; + do { + const signals = await findSignals({ alertsClient, page }); + const signal = findSignalInArrayById(signals.data, id); + if (signal != null) { + return signal; + } else { + length = signals.data.length; + page++; + } + } while (length !== 0); + return null; +}; + // TODO: Change this from a search to a filter once this ticket is solved: // https://github.com/elastic/kibana/projects/26#card-27462236 export const readSignals = async ({ alertsClient, id }: ReadSignalParams) => { - return alertsClient.get({ id }); + const signalById = await readSignalById({ alertsClient, id }); + if (signalById != null) { + return signalById; + } else { + return alertsClient.get({ id }); + } }; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts new file mode 100644 index 0000000000000..f3dfef7f94681 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get } from 'lodash/fp'; + +import { SIGNALS_ID } from '../../../../common/constants'; +import { Alert } from '../../../../../alerting/server/types'; + +// TODO: Migrate the other types over to using this one +export interface SignalAlertParams { + description: string; + from: string; + id: string; + index: string[]; + interval: string; + enabled: boolean; + filter: Record | undefined; + kql: string | undefined; + maxSignals: string; + name: string; + severity: number; + type: string; // TODO: Replace this type with a static enum type + to: string; +} + +export type SignalAlertType = Alert & { + id: string; + alertTypeParams: SignalAlertParams; +}; + +export const isAlertType = (obj: unknown): obj is SignalAlertType => { + return get('alertTypeId', obj) === SIGNALS_ID; +}; + +export const isAlertTypeArray = (objArray: unknown[]): objArray is SignalAlertType[] => { + return objArray.length === 0 || isAlertType(objArray[0]); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts index 7c307695dd2a5..ac183790fd5a1 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts @@ -103,13 +103,13 @@ export const updateSignal = async ({ ); if (signal.enabled && !enabled) { - await alertsClient.disable({ id }); + await alertsClient.disable({ id: signal.id }); } else if (!signal.enabled && enabled) { - await alertsClient.enable({ id }); + await alertsClient.enable({ id: signal.id }); } return alertsClient.update({ - id, + id: signal.id, data: { interval: calculateInterval(interval, signal.interval), actions, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_x_signals.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_x_signals.sh new file mode 100755 index 0000000000000..ae305ab8ac758 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_x_signals.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Uses a default of 100 if no argument is specified +NUMBER=${1:-100} + +# Example: ./post_x_signals.sh +# Example: ./post_x_signals.sh 200 +for i in $(seq 1 $NUMBER); +do curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${KIBANA_URL}/api/siem/signals \ + --data "{ + \"id\": \"${i}\", + \"description\": \"Detecting root and admin users\", + \"index\": [\"auditbeat-*\", \"filebeat-*\", \"packetbeat-*\", \"winlogbeat-*\"], + \"interval\": \"24h\", + \"name\": \"Detect Root/Admin Users\", + \"severity\": 1, + \"type\": \"kql\", + \"from\": \"now-6m\", + \"to\": \"now\", + \"kql\": \"user.name: root or user.name: admin\" + }" \ + | jq .; +done diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/temp_update_1.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_update_1.json similarity index 85% rename from x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/temp_update_1.json rename to x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_update_1.json index 8ca571666ea4e..02a1bda23fd1c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/temp_update_1.json +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_update_1.json @@ -1,5 +1,5 @@ { - "id": "f47e5de7-0023-4898-b856-cf3874e883ea", + "id": "1", "description": "Only watch winlogbeat users", "index": ["winlogbeat-*"], "interval": "9m", diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/update_signal.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/update_signal.sh index 0980999156f11..8cf69dc41e0be 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/update_signal.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/update_signal.sh @@ -9,11 +9,8 @@ set -e ./check_env_variables.sh -# TODO: Since we only have GUID's at the moment, you have to use ./find_signals.sh and then copy and paste that -# into the temp_update_1.json as your ID in order to run this script. - # Uses a default if no argument is specified -SIGNAL=${1:-./signals/temp_update_1.json} +SIGNAL=${1:-./signals/root_or_admin_update_1.json} # Example: ./update_signal.sh {id} ./signals/root_or_admin_1.json curl -s -k \ From 5ab372a628a315e5cd5db76b50c960e4b73aa5c7 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Thu, 10 Oct 2019 17:25:26 -0600 Subject: [PATCH 12/15] Removed weird wording from a function --- .../siem/server/lib/detection_engine/alerts/read_signals.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/read_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/read_signals.ts index fe122cedbb5de..f8f4b1267c10f 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/read_signals.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/read_signals.ts @@ -60,8 +60,6 @@ export const readSignalById = async ({ return null; }; -// TODO: Change this from a search to a filter once this ticket is solved: -// https://github.com/elastic/kibana/projects/26#card-27462236 export const readSignals = async ({ alertsClient, id }: ReadSignalParams) => { const signalById = await readSignalById({ alertsClient, id }); if (signalById != null) { From acd9ff04f7d0087a41f34beaab467a64efc49a4c Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Fri, 11 Oct 2019 19:58:22 -0600 Subject: [PATCH 13/15] Added scripts to convert from saved objects to signals for posting --- .../convert_saved_search_to_signals.js | 131 ++++++++++++++++++ .../convert_saved_search_to_signals.sh | 12 ++ .../detection_engine/scripts/post_signal.sh | 25 ++-- .../scripts/post_x_signals.sh | 6 +- 4 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/convert_saved_search_to_signals.sh diff --git a/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js b/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js new file mode 100644 index 0000000000000..dd6f2fead549a --- /dev/null +++ b/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +require('../../../../../src/setup_node_env'); + +const fs = require('fs'); +const path = require('path'); + +/* + * This script is used to parse a set of saved searches on a file system + * and output signal data compatible json files. + * Example: + * node saved_query_to_signals.js ${HOME}/saved_searches ${HOME}/saved_signals + * + * After editing any changes in the files of ${HOME}/saved_signals/*.json + * you can then post the signals with a CURL post script such as: + * + * ./post_signal.sh ${HOME}/saved_signals/*.json + * + * Note: This script is recursive and but does not preserve folder structure + * when it outputs the saved signals. + */ + +// Defaults of the outputted signals since the saved KQL searches do not have +// this type of information. You usually will want to make any hand edits after +// doing a search to KQL conversion before posting it as a signal or checking it +// into another repository. +const INTERVAL = '24h'; +const SEVERITY = 1; +const TYPE = 'kql'; +const FROM = 'now-24h'; +const TO = 'now'; +const INDEX = ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*']; + +const walk = dir => { + const list = fs.readdirSync(dir); + return list.reduce((accum, file) => { + const fileWithDir = dir + '/' + file; + const stat = fs.statSync(fileWithDir); + if (stat && stat.isDirectory()) { + return [...accum, ...walk(fileWithDir)]; + } else { + return [...accum, fileWithDir]; + } + }, []); +}; + +// Temporary hash function for converting string to numbers. +// TODO: Once we move from numbers to pure strings for id's this can be removed +// and the file name used as the id (or a GUID), etc... +const hashFunc = str => { + let chr; + let hash = 0; + if (str.length === 0) return 0; + for (let i = 0; i < str.length; i++) { + chr = str.charCodeAt(i); + // eslint-disable-next-line no-bitwise + hash = (hash << 5) - hash + chr; + // eslint-disable-next-line no-bitwise + hash |= 0; + } + return hash; +}; + +//clean up the file system characters +const cleanupFileName = file => { + return path + .basename(file, path.extname(file)) + .replace(/\s+/g, '_') + .replace(/,/g, '') + .replace(/\+s/g, '') + .replace(/-/g, '') + .replace(/__/g, '_') + .toLowerCase(); +}; + +async function main() { + if (process.argv.length !== 4) { + throw new Error( + 'usage: saved_query_to_signals [input directory with saved searches] [output directory]' + ); + } + + const files = process.argv[2]; + const outputDir = process.argv[3]; + + const savedSearchesJson = walk(files).filter(file => file.endsWith('.ndjson')); + + const savedSearchesParsed = savedSearchesJson.reduce((accum, json) => { + const jsonFile = fs.readFileSync(json, 'utf8'); + try { + const parsedFile = JSON.parse(jsonFile); + parsedFile._file = json; + parsedFile.attributes.kibanaSavedObjectMeta.searchSourceJSON = JSON.parse( + parsedFile.attributes.kibanaSavedObjectMeta.searchSourceJSON + ); + return [...accum, parsedFile]; + } catch (err) { + return accum; + } + }, []); + + savedSearchesParsed.forEach(savedSearch => { + const fileToWrite = cleanupFileName(savedSearch._file); + + const query = savedSearch.attributes.kibanaSavedObjectMeta.searchSourceJSON.query.query; + if (query != null && query.trim() !== '') { + const outputMessage = { + id: `${hashFunc(fileToWrite)}`, // TODO: Remove this once we change id to a string + description: savedSearch.attributes.description || savedSearch.attributes.title, + index: INDEX, + interval: INTERVAL, + name: savedSearch.attributes.title, + severity: SEVERITY, + type: TYPE, + from: FROM, + to: TO, + kql: savedSearch.attributes.kibanaSavedObjectMeta.searchSourceJSON.query.query, + }; + + fs.writeFileSync(`${outputDir}/${fileToWrite}.json`, JSON.stringify(outputMessage, null, 2)); + } + }); +} + +if (require.main === module) { + main(); +} diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/convert_saved_search_to_signals.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/convert_saved_search_to_signals.sh new file mode 100755 index 0000000000000..802273c67849d --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/convert_saved_search_to_signals.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +node ../../../../scripts/convert_saved_search_to_signals.js $1 $2 diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh index 961953c71b2dc..ee76fe244d54e 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh @@ -10,14 +10,23 @@ set -e ./check_env_variables.sh # Uses a default if no argument is specified -SIGNAL=${1:-./signals/root_or_admin_1.json} +SIGNALS=(${@:-./signals/root_or_admin_1.json}) # Example: ./post_signal.sh # Example: ./post_signal.sh ./signals/root_or_admin_1.json -curl -s -k \ - -H 'Content-Type: application/json' \ - -H 'kbn-xsrf: 123' \ - -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ - -X POST ${KIBANA_URL}/api/siem/signals \ - -d @${SIGNAL} \ - | jq . +# Example glob: ./post_signal.sh ./signals/* +for SIGNAL in "${SIGNALS[@]}" +do { + [ -e "$SIGNAL" ] || continue + echo "Posting $SIGNAL signal..." + curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${KIBANA_URL}/api/siem/signals \ + -d @${SIGNAL} \ + | jq .; +} & +done + +wait diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_x_signals.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_x_signals.sh index ae305ab8ac758..1aaecca37ba6f 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_x_signals.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_x_signals.sh @@ -15,7 +15,8 @@ NUMBER=${1:-100} # Example: ./post_x_signals.sh # Example: ./post_x_signals.sh 200 for i in $(seq 1 $NUMBER); -do curl -s -k \ +do { + curl -s -k \ -H 'Content-Type: application/json' \ -H 'kbn-xsrf: 123' \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ @@ -33,4 +34,7 @@ do curl -s -k \ \"kql\": \"user.name: root or user.name: admin\" }" \ | jq .; +} & done + +wait \ No newline at end of file From 93e888eeb4808e845d5e9a92fef35679adeae47f Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Fri, 11 Oct 2019 20:07:15 -0600 Subject: [PATCH 14/15] Remove echo statement --- .../siem/server/lib/detection_engine/scripts/post_signal.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh index ee76fe244d54e..6d79856ffd4fb 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh @@ -18,7 +18,6 @@ SIGNALS=(${@:-./signals/root_or_admin_1.json}) for SIGNAL in "${SIGNALS[@]}" do { [ -e "$SIGNAL" ] || continue - echo "Posting $SIGNAL signal..." curl -s -k \ -H 'Content-Type: application/json' \ -H 'kbn-xsrf: 123' \ From 17394a2d904cf07c785b607944a8a32777014ec5 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Tue, 15 Oct 2019 11:51:22 -0600 Subject: [PATCH 15/15] Fix minor wording --- .../siem/server/lib/detection_engine/alerts/create_signal.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts index 008526e0688bd..a28e38a083c98 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts @@ -113,7 +113,7 @@ export const createSignal = async ({ if (signalUpdating == null) { // TODO: Right now we are using the .server-log as the default action as each alert has to have // at least one action or it will not be able to do in-memory persistence. When adding in actions - // such as email, slack, etc... this should be the default action if not action is specified to + // such as email, slack, etc... this should be the default action if no action is specified to // create signals const actionResults = await actionsClient.create({ action: {