From afb60f5176d0956a1969894faf3ea948ef6f59c3 Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Mon, 7 Oct 2019 12:58:28 -0600 Subject: [PATCH] [SIEM] [Detection Engine] Adds a create signal API (#47384) ## Summary * https://github.com/elastic/kibana/issues/47013 This adds a basic create signal API rout API at: ``` /api/siem/signals ``` At the moment this will create an action/alert as a one for one. This also introduces several scripts which allow developers to: * Re-create signals index for development velocity * post signals to the `/api/siem/signals` endpoint * Check different indexes, saved objects, and task manager for trouble shooting ### Checklist Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR. ~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~ ~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~ ~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~ ~~- [ ] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios~~ ~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~ ### For maintainers ~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~ ~~- [ ] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~ --- .../legacy/plugins/siem/common/constants.ts | 5 + x-pack/legacy/plugins/siem/index.test.ts | 30 +++++ .../plugins/siem/server/kibana.index.ts | 14 ++- .../server/lib/detection_engine/README.md | 74 ++++++------ .../alerts/build_events_reindex.ts | 42 ++++--- .../detection_engine/alerts/create_signal.ts | 91 +++++++++++++++ .../alerts/signals_alert_type.ts | 81 +++++++------ .../routes/create_signals_route.ts | 108 ++++++++++++++++++ .../lib/detection_engine/scripts/README.md | 37 ++++++ .../scripts/delete_signal_index.sh | 9 ++ .../scripts/find_saved_object.sh | 13 +++ .../scripts/get_action_instances.sh | 8 ++ .../scripts/get_action_types.sh | 8 ++ .../scripts/get_alert_instances.sh | 8 ++ .../scripts/get_alert_tasks.sh | 16 +++ .../scripts/get_alert_types.sh | 8 ++ .../scripts/get_saved_objects.sh | 8 ++ .../scripts/get_signal_mapping.sh | 8 ++ .../detection_engine/scripts/post_signal.sh | 14 +++ .../scripts/put_signal_index.sh | 10 ++ .../scripts/signals/root_or_admin_1.json | 12 ++ .../scripts/signals/root_or_admin_2.json | 12 ++ .../signals/root_or_admin_filter_9999.json | 42 +++++++ .../scripts/signals/watch_longmont_3.json | 12 ++ 24 files changed, 584 insertions(+), 86 deletions(-) create mode 100644 x-pack/legacy/plugins/siem/index.test.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/README.md create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal_index.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_saved_object.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_instances.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_types.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_instances.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_tasks.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_types.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_saved_objects.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_signal_mapping.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/put_signal_index.sh create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_1.json create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_2.json create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_filter_9999.json create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/watch_longmont_3.json diff --git a/x-pack/legacy/plugins/siem/common/constants.ts b/x-pack/legacy/plugins/siem/common/constants.ts index 1641ee983bee9..6845648ee921d 100644 --- a/x-pack/legacy/plugins/siem/common/constants.ts +++ b/x-pack/legacy/plugins/siem/common/constants.ts @@ -25,3 +25,8 @@ export const DEFAULT_TO = 'now'; export const DEFAULT_INTERVAL_PAUSE = true; export const DEFAULT_INTERVAL_TYPE = 'manual'; export const DEFAULT_INTERVAL_VALUE = 300000; // ms + +/** + * Id for the SIGNALS alerting type + */ +export const SIGNALS_ID = `${APP_ID}.signals`; diff --git a/x-pack/legacy/plugins/siem/index.test.ts b/x-pack/legacy/plugins/siem/index.test.ts new file mode 100644 index 0000000000000..702676cc202d5 --- /dev/null +++ b/x-pack/legacy/plugins/siem/index.test.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { siem } from '.'; + +describe('siem plugin tests', () => { + // This test is a temporary test which is so we do not accidentally check-in + // feature flags turned on from "alerting" and "actions". If those get + // turned on during a check-in it will cause everyone's Kibana to not start. + // Once alerting and actions are part of the plugins by default this test + // should be removed. + test(` + You have accidentally tried to check-in a feature flag with alerting located + here: x-pack/legacy/plugins/siem/index.ts, please change the plugin require to + NOT have these two inside of the require array." + `, () => { + class MockPlugin { + require: string[]; + constructor({ require }: { require: string[] }) { + this.require = require; + } + } + const plugin = siem({ Plugin: MockPlugin }); + expect(plugin.require.includes('alerting')).toBe(false); + expect(plugin.require.includes('actions')).toBe(false); + }); +}); diff --git a/x-pack/legacy/plugins/siem/server/kibana.index.ts b/x-pack/legacy/plugins/siem/server/kibana.index.ts index 8d0e8d56cfaf3..ea29b7cdeef81 100644 --- a/x-pack/legacy/plugins/siem/server/kibana.index.ts +++ b/x-pack/legacy/plugins/siem/server/kibana.index.ts @@ -16,6 +16,8 @@ import { timelineSavedObjectType, } from './saved_objects'; +import { createSignalsRoute } from './lib/detection_engine/routes/create_signals_route'; + const APP_ID = 'siem'; export const amMocking = (): boolean => process.env.INGEST_MOCKS === 'true'; @@ -34,7 +36,17 @@ export const initServerWithKibana = (kbnServer: Server) => { const libs = compose(kbnServer); initServer(libs, { mocking, logger }); - + if ( + kbnServer.config().has('xpack.actions.enabled') && + kbnServer.config().get('xpack.actions.enabled') === true && + kbnServer.config().has('xpack.alerting.enabled') && + kbnServer.config().has('xpack.alerting.enabled') === true + ) { + logger.info( + 'Detected feature flags for actions and alerting and enabling signals API endpoints' + ); + createSignalsRoute(kbnServer); + } logger.info('Plugin done initializing'); const xpackMainPlugin = kbnServer.plugins.xpack_main; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md b/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md index 063e7c1975b25..3f18f95223408 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md @@ -9,23 +9,20 @@ Since there is no UI yet and a lot of backend areas that are not created, you should install the kbn-action and kbn-alert project from here: https://github.com/pmuellr/kbn-action -Add your signal mappings into your Kibana instance manually by opening - -``` -x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json +Open up your .zshrc/.bashrc and add these lines with the variables filled in: ``` +export ELASTICSEARCH_USERNAME=${user} +export ELASTICSEARCH_PASSWORD=${password} +export ELASTICSEARCH_URL=https://${ip}:9200 +export KIBANA_URL=http://localhost:5601 +export SIGNALS_INDEX=.siem-signals-${your user id} +export TASK_MANAGER_INDEX=.kibana-task-manager-${your user id} -And copying that to your DEV tools so it looks something like: -``` -PUT /.siem-signals-10-01-2019 -{ - "mappings": { - "dynamic": false, -... +# This is for the kbn-action and kbn-alert tool +export KBN_URLBASE=http://${user}:${password}@localhost:5601 ``` -We will solve the above issue here: -https://github.com/elastic/kibana/issues/47002 +source your .zhsrc/.bashrc or open a new terminal to ensure you get the new values set. Add these lines to your `kibana.dev.yml` to turn on the feature toggles of alerting and actions: ``` @@ -56,42 +53,34 @@ server log [22:05:22.277] [info][status][plugin:alerting@8.0.0] Status chan server log [22:05:22.270] [info][status][plugin:actions@8.0.0] Status changed from uninitialized to green - Ready ``` -Open a terminal and run +You should also see the SIEM detect the feature flags and start the API endpoints for signals -```sh -kbn-alert ls-types +``` +server log [11:39:05.561] [info][siem] Detected feature flags for actions and alerting and enabling signals API endpoints ``` -You should see the new alert type of: +Open a terminal and go into the scripts folder `cd kibana/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts` and run: -```ts -[ - { - "id": "siem.signals", - "name": "SIEM Signals" - } -] +``` +./delete_signal_index.sh +./put_signal_index.sh +./post_signal.sh ``` -Setup SIEM Alerts Log action through +which will: -```ts -kbn-action create .server-log "SIEM Alerts Log" {} {} -{ - "id": "7edd7e98-9286-4fdb-a5c5-16de776bc7c7", - "actionTypeId": ".server-log", - "description": "SIEM Alerts Log", - "config": {} -} -``` +* Delete any existing signal mapping you might have had. +* Add the latest signal index and its mappings +* Posts a sample signal which checks for root or admin every 5 minutes -Take note of the `id` GUID above and copy and paste that into a create alert like so -```ts -kbn-alert create siem.signals 5m '{}' "[{group:default id:'7edd7e98-9286-4fdb-a5c5-16de776bc7c7' params:{message: 'SIEM Alert Fired'}}]" +Now you can run + +```sh +./get_alert_instances.sh ``` -You should get back a response like so +You should see the new alert instance created like so: ```ts { "id": "908a6af1-ac63-4d52-a856-fc635a00db0f", @@ -122,5 +111,10 @@ Every 5 minutes you should see this message in your terminal now: server log [22:17:33.945] [info][alerting] SIEM Alert Fired ``` -Add the `.siem-signals-10-01-2019` to your advanced SIEM settings to see any signals -created which should update once every 5 minutes at this point. \ No newline at end of file +See the scripts folder and the tools for more command line fun. + +Add the `.siem-signals-${your user id}` to your advanced SIEM settings to see any signals +created which should update once every 5 minutes at this point. + +Also add the `.siem-signals-${your user id}` as a kibana index for Maps to be able to see the +signals diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_reindex.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_reindex.ts index 84cedca0855e0..c48be695756af 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_reindex.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_reindex.ts @@ -4,48 +4,64 @@ * you may not use this file except in compliance with the Elastic License. */ +import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; + // TODO: Re-index is just a temporary solution in order to speed up development // of any front end pieces. This should be replaced with a combination of the file // build_events_query.ts and any scrolling/scaling solutions from that particular // file. interface BuildEventsReIndexParams { + description: string; index: string[]; from: number; to: number; signalsIndex: string; maxDocs: number; - kqlFilter: {}; + filter: Record | undefined; + kql: string | undefined; severity: number; - description: string; name: string; timeDetected: number; ruleRevision: number; - ruleId: string; - ruleType: string; + id: string; + type: string; references: string[]; } +export const getFilter = (kql: string | undefined, filter: Record | undefined) => { + if (kql != null) { + return toElasticsearchQuery(fromKueryExpression(kql), null); + } else if (filter != null) { + return filter; + } else { + // TODO: Re-visit this error (which should never happen) when we do signal errors for the UI + throw new TypeError('either kql or filter should be set'); + } +}; + export const buildEventsReIndex = ({ + description, index, from, to, signalsIndex, maxDocs, - kqlFilter, + filter, + kql, severity, - description, name, timeDetected, ruleRevision, - ruleId, - ruleType, + id, + type, references, }: BuildEventsReIndexParams) => { + const kqlOrFilter = getFilter(kql, filter); const indexPatterns = index.map(element => `"${element}"`).join(','); const refs = references.map(element => `"${element}"`).join(','); - const filter = [ - kqlFilter, + const filterWithTime = [ + kqlOrFilter, { bool: { filter: [ @@ -96,7 +112,7 @@ export const buildEventsReIndex = ({ query: { bool: { filter: [ - ...filter, + ...filterWithTime, { match_all: {}, }, @@ -120,8 +136,8 @@ export const buildEventsReIndex = ({ def signal = [ "rule_revision": "${ruleRevision}", - "rule_id": "${ruleId}", - "rule_type": "${ruleType}", + "rule_id": "${id}", + "rule_type": "${type}", "parent": parent, "name": "${name}", "severity": ${severity}, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts new file mode 100644 index 0000000000000..6ebdecfeeba83 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SIGNALS_ID } from '../../../../common/constants'; +import { AlertsClient } from '../../../../../alerting/server/alerts_client'; +import { ActionsClient } from '../../../../../actions/server/actions_client'; + +export interface SignalParams { + alertsClient: AlertsClient; + actionsClient: ActionsClient; + description: string; + from: string; + id: string; + index: string[]; + interval: string; + enabled: boolean; + filter: Record | undefined; + kql: string | undefined; + maxSignals: string; + name: string; + severity: number; + type: string; // TODO: Replace this type with a static enum type + to: string; + references: string[]; +} + +export const createSignal = async ({ + alertsClient, + actionsClient, + description, + enabled, + filter, + from, + id, + index, + interval, + kql, + name, + severity, + to, + type, + references, +}: SignalParams) => { + // TODO: Right now we are using the .server-log as the default action as each alert has to have + // at least one action or it will not be able to do in-memory persistence. When adding in actions + // such as email, slack, etc... this should be the default action if not action is specified to + // create signals + + const actionResults = await actionsClient.create({ + action: { + actionTypeId: '.server-log', + description: 'SIEM Alerts Log', + config: {}, + secrets: {}, + }, + }); + + return alertsClient.create({ + data: { + alertTypeId: SIGNALS_ID, + alertTypeParams: { + description, + id, + index, + from, + filter, + kql, + name, + severity, + to, + type, + references, + }, + interval, + enabled, + actions: [ + { + group: 'default', + id: actionResults.id, + params: { + message: 'SIEM Alert Fired', + }, + }, + ], + throttle: null, + }, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts index d0b631f66d54d..3f8658f839dfa 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import moment from 'moment'; -import { APP_ID } from '../../../../common/constants'; +import { schema } from '@kbn/config-schema'; +import { SIGNALS_ID } from '../../../../common/constants'; import { AlertType, AlertExecutorOptions } from '../../../../../alerting'; // TODO: Remove this for the build_events_query call eventually @@ -16,47 +16,65 @@ import { buildEventsReIndex } from './build_events_reindex'; // import { buildEventsQuery } from './build_events_query'; export const signalsAlertType: AlertType = { - id: `${APP_ID}.signals`, + id: SIGNALS_ID, name: 'SIEM Signals', actionGroups: ['default'], + validate: { + params: schema.object({ + description: schema.string(), + from: schema.string(), + filter: schema.maybe(schema.object({}, { allowUnknowns: true })), + id: schema.number(), + index: schema.arrayOf(schema.string()), + kql: schema.maybe(schema.string({ defaultValue: undefined })), + maxSignals: schema.number({ defaultValue: 100 }), + name: schema.string(), + severity: schema.number(), + to: schema.string(), + type: schema.string(), + references: schema.arrayOf(schema.string(), { defaultValue: [] }), + }), + }, + // TODO: Type the params as it is all filled with any async executor({ services, params, state }: AlertExecutorOptions) { - // TODO: We need to swap out this arbitrary number of siem-signal id for an injected - // data driven instance id through passed in parameters. const instance = services.alertInstanceFactory('siem-signals'); // TODO: Comment this in eventually and use the buildEventsQuery() // for scrolling and other fun stuff instead of using the buildEventsReIndex() // const query = buildEventsQuery(); - // TODO: Turn these options being sent in into a template for the alert type + const { + description, + filter, + from, + id, + index, + kql, + maxSignals, + name, + references, + severity, + to, + type, + } = params; const reIndex = buildEventsReIndex({ - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - from: moment() - .subtract(5, 'minutes') - .valueOf(), - to: Date.now(), - signalsIndex: '.siem-signals-10-01-2019', - severity: 2, - description: 'User root activity', - name: 'User Rule', + index, + from, + kql, + to, + // TODO: Change this out once we have solved + // https://github.com/elastic/kibana/issues/47002 + signalsIndex: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019', + severity, + description, + name, timeDetected: Date.now(), - kqlFilter: { - bool: { - should: [ - { - match_phrase: { - 'user.name': 'root', - }, - }, - ], - minimum_should_match: 1, - }, - }, - maxDocs: 100, + filter, + maxDocs: maxSignals, ruleRevision: 1, - ruleId: '1', - ruleType: 'KQL', - references: ['https://www.elastic.co', 'https://example.com'], + id, + type, + references, }); try { @@ -65,7 +83,6 @@ export const signalsAlertType: AlertType = { // TODO: Comment this in eventually and use this for manual insertion of the // signals instead of the ReIndex() api // const result = await services.callCluster('search', query); - // eslint-disable-next-line const result = await services.callCluster('reindex', reIndex); // TODO: Error handling here and writing of any errors that come back from ES by diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts new file mode 100644 index 0000000000000..19070243520b8 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Hapi from 'hapi'; +import Joi from 'joi'; +import { isFunction } from 'lodash/fp'; +import { createSignal } from '../alerts/create_signal'; + +interface SignalsRequest extends Hapi.Request { + payload: { + description: string; + enabled: boolean; + filter: Record | undefined; + from: string; + id: string; + index: string[]; + interval: string; + kql: string | undefined; + max_signals: string; + name: string; + severity: number; + type: string; + to: string; + references: string[]; + }; +} + +export const createSignalsRoute = (server: Hapi.Server) => { + server.route({ + method: 'POST', + path: '/api/siem/signals', + options: { + tags: ['access:signals-all'], + validate: { + options: { + abortEarly: false, + }, + payload: Joi.object({ + description: Joi.string().required(), + enabled: Joi.boolean().default(true), + filter: Joi.object(), + from: Joi.string().required(), + id: Joi.string().required(), + index: Joi.array().required(), + interval: Joi.string().default('5m'), + kql: Joi.string(), + max_signals: Joi.array().default([]), + name: Joi.string().required(), + severity: Joi.number().required(), + to: Joi.string().required(), + type: Joi.string().required(), // TODO: Restrict this to only be kql or filter for the moment + references: Joi.array().default([]), + }).xor('filter', 'kql'), + }, + }, + async handler(request: SignalsRequest, headers) { + const { + description, + enabled, + filter, + kql, + from, + id, + index, + interval, + // eslint-disable-next-line @typescript-eslint/camelcase + max_signals: maxSignals, + name, + severity, + to, + type, + references, + } = request.payload; + + const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null; + + const actionsClient = isFunction(request.getActionsClient) + ? request.getActionsClient() + : null; + + if (!alertsClient || !actionsClient) { + return headers.response().code(404); + } + + return createSignal({ + alertsClient, + actionsClient, + description, + enabled, + filter, + from, + id, + index, + interval, + kql, + maxSignals, + name, + severity, + to, + type, + references, + }); + }, + }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/README.md b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/README.md new file mode 100644 index 0000000000000..0258f5455ca87 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/README.md @@ -0,0 +1,37 @@ +A set of scripts for developers to utilize command line functionality of Kibana/Elastic +search which is not available in the DEV console for the detection engine. + +Before beginning ensure in your .zshrc/.bashrc you have your user, password, and url set: + +Open up your .zshrc/.bashrc and add these lines with the variables filled in: +``` +export ELASTICSEARCH_USERNAME=${user} +export ELASTICSEARCH_PASSWORD=${password} +export ELASTICSEARCH_URL=https://${ip}:9200 +export KIBANA_URL=http://localhost:5601 +export SIGNALS_INDEX=.siem-signals-${your user id} +export TASK_MANAGER_INDEX=.kibana-task-manager-${your user id} + +# This is for the kbn-action and kbn-alert tool +export KBN_URLBASE=http://${user}:${password}@localhost:5601 +``` + +And that you have the latest version of [NodeJS](https://nodejs.org/en/), +[CURL](https://curl.haxx.se), and [jq](https://stedolan.github.io/jq/) installed. + +If you have homebrew you can install using brew like so +``` +brew install jq +``` + +After that you can execute scripts within this folder by first ensuring +your current working directory is `./scripts` and then running any scripts within +that folder. + +Example to add a signal to the system + +``` +cd ./scripts +./post_signal.sh ./signals/root_or_admin_1.json +``` + diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal_index.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal_index.sh new file mode 100755 index 0000000000000..edb88d8b9338c --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_signal_index.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +# Example: ./delete_signal_index.sh +# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-delete-index.html +curl -s -k \ + -H "Content-Type: application/json" \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X DELETE ${ELASTICSEARCH_URL}/${SIGNALS_INDEX} \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_saved_object.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_saved_object.sh new file mode 100755 index 0000000000000..3b9b8686df9a6 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_saved_object.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +# Uses a default of alert if no argument is specified +TYPE=${1:-alert} + +# Example: ./find_saved_object.sh alert +# Example: ./find_saved_object.sh action +# Example: ./find_saved_object.sh action_task_params +# https://www.elastic.co/guide/en/kibana/master/saved-objects-api-find.html#saved-objects-api-find-request +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${KIBANA_URL}/api/saved_objects/_find?type=$TYPE \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_instances.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_instances.sh new file mode 100755 index 0000000000000..e04bdba4f1073 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_instances.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# Example: ./get_action_instances.sh +# https://github.com/elastic/kibana/blob/master/x-pack/legacy/plugins/actions/README.md#get-apiaction_find-find-actions +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${KIBANA_URL}/api/action/_find \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_types.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_types.sh new file mode 100755 index 0000000000000..47a571f47bf68 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_action_types.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# Example: ./get_action_types.sh +# https://github.com/elastic/kibana/blob/master/x-pack/legacy/plugins/actions/README.md +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${KIBANA_URL}/api/action/types \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_instances.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_instances.sh new file mode 100755 index 0000000000000..a3c03d56d6506 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_instances.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# Example: ./get_alert_instances.sh +# https://github.com/elastic/kibana/blob/master/x-pack/legacy/plugins/alerting/README.md#get-apialert_find-find-alerts +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${KIBANA_URL}/api/alert/_find \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_tasks.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_tasks.sh new file mode 100755 index 0000000000000..210fc2d532b7d --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_tasks.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# Example: ./get_alert_tasks.sh +# https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html +curl -s -k \ + -H "Content-Type: application/json" \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${ELASTICSEARCH_URL}/${TASK_MANAGER_INDEX}*/_search \ + --data '{ + "query": { + "term" : { "task.taskType" : "alerting:siem.signals" } + }, + "size": 100 + } + ' \ +| jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_types.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_types.sh new file mode 100755 index 0000000000000..e70bbb9e882c3 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_alert_types.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# Example: ./get_alert_types.sh +# https://github.com/elastic/kibana/blob/master/x-pack/legacy/plugins/alerting/README.md#get-apialerttypes-list-alert-types +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${KIBANA_URL}/api/alert/types \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_saved_objects.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_saved_objects.sh new file mode 100755 index 0000000000000..71725ed308a85 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_saved_objects.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# Example: ./get_saved_object.sh +# https://www.elastic.co/guide/en/kibana/master/saved-objects-api-get.html +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${KIBANA_URL}/api/saved_objects/$1/$2 \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_signal_mapping.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_signal_mapping.sh new file mode 100755 index 0000000000000..d30f66df72eda --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/get_signal_mapping.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# Example: ./get_signal_mapping.sh +# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-mapping.html +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${ELASTICSEARCH_URL}/${SIGNALS_INDEX}/_mapping \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh new file mode 100755 index 0000000000000..3393084c56c96 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_signal.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +# Uses a default if no argument is specified +SIGNAL=${1:-./signals/root_or_admin_1.json} + +# Example: ./post_signal.sh +# Example: ./post_signal.sh ./signals/root_or_admin_1.json +curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${KIBANA_URL}/api/siem/signals \ + -d @${SIGNAL} \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/put_signal_index.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/put_signal_index.sh new file mode 100755 index 0000000000000..54b60c7ccdbe6 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/put_signal_index.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# Example: ./put_signal_index.sh +# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-put-mapping.html +curl -s -k \ + -H "Content-Type: application/json" \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -d @../signals_mapping.json \ + -X PUT ${ELASTICSEARCH_URL}/${SIGNALS_INDEX} \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_1.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_1.json new file mode 100644 index 0000000000000..8b07d2d0ee86c --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_1.json @@ -0,0 +1,12 @@ +{ + "id": "1", + "description": "Detecting root and admin users", + "index": ["auditbeat-*", "filebeat-*", "packetbeat-*", "winlogbeat-*"], + "interval": "5m", + "name": "Detect Root/Admin Users", + "severity": 1, + "type": "kql", + "from": "now-6m", + "to": "now", + "kql": "user.name: root or user.name: admin" +} diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_2.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_2.json new file mode 100644 index 0000000000000..bc206cb8b846e --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_2.json @@ -0,0 +1,12 @@ +{ + "id": "2", + "description": "Detecting root and admin users over a long period of time", + "index": ["auditbeat-*", "filebeat-*", "packetbeat-*", "winlogbeat-*"], + "interval": "24h", + "name": "Detect Root/Admin Users over a long period of time", + "severity": 1, + "type": "kql", + "from": "now-1y", + "to": "now", + "kql": "user.name: root or user.name: admin" +} diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_filter_9999.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_filter_9999.json new file mode 100644 index 0000000000000..04d4315f8a868 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_filter_9999.json @@ -0,0 +1,42 @@ +{ + "id": "9999", + "description": "Detecting root and admin users", + "index": ["auditbeat-*", "filebeat-*", "packetbeat-*", "winlogbeat-*"], + "interval": "5m", + "name": "Detect Root/Admin Users", + "severity": 1, + "type": "filter", + "from": "now-6m", + "to": "now", + "filter": { + "bool": { + "should": [ + { + "bool": { + "should": [ + { + "match_phrase": { + "user.name": "root" + } + } + ], + "minimum_should_match": 1 + } + }, + { + "bool": { + "should": [ + { + "match_phrase": { + "user.name": "admin" + } + } + ], + "minimum_should_match": 1 + } + } + ], + "minimum_should_match": 1 + } + } +} diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/watch_longmont_3.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/watch_longmont_3.json new file mode 100644 index 0000000000000..21ca9f8ba73b8 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/watch_longmont_3.json @@ -0,0 +1,12 @@ +{ + "id": "3", + "description": "Detect Longmont activity", + "index": ["auditbeat-*", "filebeat-*", "packetbeat-*", "winlogbeat-*"], + "interval": "24h", + "name": "Detect Longmont activity", + "severity": 2, + "type": "kql", + "from": "now-1y", + "to": "now", + "kql": "source.as.organization.name: \"Longmont Power & Communications\"" +}