[7.x] [SIEM] [Detection Engine] Adds a create signal API (#47384)

x-pack/legacy/plugins/siem/common/constants.ts

@@ -25,3 +25,8 @@ export const DEFAULT_TO = 'now';
 export const DEFAULT_INTERVAL_PAUSE = true;
 export const DEFAULT_INTERVAL_TYPE = 'manual';
 export const DEFAULT_INTERVAL_VALUE = 300000; // ms
+
+/**
+ * Id for the SIGNALS alerting type
+ */
+export const SIGNALS_ID = `${APP_ID}.signals`;

x-pack/legacy/plugins/siem/index.test.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { siem } from '.';
+
+describe('siem plugin tests', () => {
+  // This test is a temporary test which is so we do not accidentally check-in
+  // feature flags turned on from "alerting" and "actions". If those get
+  // turned on during a check-in it will cause everyone's Kibana to not start.
+  // Once alerting and actions are part of the plugins by default this test
+  // should be removed.
+  test(`
+    You have accidentally tried to check-in a feature flag with alerting located
+    here: x-pack/legacy/plugins/siem/index.ts, please change the plugin require to
+    NOT have these two inside of the require array."
+  `, () => {
+    class MockPlugin {
+      require: string[];
+      constructor({ require }: { require: string[] }) {
+        this.require = require;
+      }
+    }
+    const plugin = siem({ Plugin: MockPlugin });
+    expect(plugin.require.includes('alerting')).toBe(false);
+    expect(plugin.require.includes('actions')).toBe(false);
+  });
+});

x-pack/legacy/plugins/siem/server/kibana.index.ts

@@ -16,6 +16,8 @@ import {
   timelineSavedObjectType,
 } from './saved_objects';
 
+import { createSignalsRoute } from './lib/detection_engine/routes/create_signals_route';
+
 const APP_ID = 'siem';
 
 export const amMocking = (): boolean => process.env.INGEST_MOCKS === 'true';
@@ -34,7 +36,17 @@ export const initServerWithKibana = (kbnServer: Server) => {
 
   const libs = compose(kbnServer);
   initServer(libs, { mocking, logger });
-
+  if (
+    kbnServer.config().has('xpack.actions.enabled') &&
+    kbnServer.config().get('xpack.actions.enabled') === true &&
+    kbnServer.config().has('xpack.alerting.enabled') &&
+    kbnServer.config().has('xpack.alerting.enabled') === true
+  ) {
+    logger.info(
+      'Detected feature flags for actions and alerting and enabling signals API endpoints'
+    );
+    createSignalsRoute(kbnServer);
+  }
   logger.info('Plugin done initializing');
 
   const xpackMainPlugin = kbnServer.plugins.xpack_main;

x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md

@@ -9,23 +9,20 @@ Since there is no UI yet and a lot of backend areas that are not created, you
 should install the kbn-action and kbn-alert project from here:
 https://github.com/pmuellr/kbn-action
 
-Add your signal mappings into your Kibana instance manually by opening
-
-```
-x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json
+Open up your .zshrc/.bashrc and add these lines with the variables filled in:
 ```
+export ELASTICSEARCH_USERNAME=${user}
+export ELASTICSEARCH_PASSWORD=${password}
+export ELASTICSEARCH_URL=https://${ip}:9200
+export KIBANA_URL=http://localhost:5601
+export SIGNALS_INDEX=.siem-signals-${your user id}
+export TASK_MANAGER_INDEX=.kibana-task-manager-${your user id}
 
-And copying that to your DEV tools so it looks something like:
-```
-PUT /.siem-signals-10-01-2019
-{
-  "mappings": {
-    "dynamic": false,
-...
+# This is for the kbn-action and kbn-alert tool
+export KBN_URLBASE=http://${user}:${password}@localhost:5601
 ```
 
-We will solve the above issue here:
-https://github.com/elastic/kibana/issues/47002
+source your .zhsrc/.bashrc or open a new terminal to ensure you get the new values set.
 
 Add these lines to your `kibana.dev.yml` to turn on the feature toggles of alerting and actions:
 ```
@@ -56,42 +53,34 @@ server    log   [22:05:22.277] [info][status][plugin:alerting@8.0.0] Status chan
 server    log   [22:05:22.270] [info][status][plugin:actions@8.0.0] Status changed from uninitialized to green - Ready
 ```
 
-Open a terminal and run
+You should also see the SIEM detect the feature flags and start the API endpoints for signals
 
-```sh
-kbn-alert ls-types
+```
+server    log   [11:39:05.561] [info][siem] Detected feature flags for actions and alerting and enabling signals API endpoints
 ```
 
-You should see the new alert type of:
+Open a terminal and go into the scripts folder `cd kibana/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts` and run:
 
-```ts
-[
-    {
-        "id": "siem.signals",
-        "name": "SIEM Signals"
-    }
-]
+```
+./delete_signal_index.sh
+./put_signal_index.sh
+./post_signal.sh
 ```
 
-Setup SIEM Alerts Log action through
+which will:
 
-```ts
-kbn-action create .server-log "SIEM Alerts Log" {} {}
-{
-    "id": "7edd7e98-9286-4fdb-a5c5-16de776bc7c7",
-    "actionTypeId": ".server-log",
-    "description": "SIEM Alerts Log",
-    "config": {}
-}
-```
+* Delete any existing signal mapping you might have had.
+* Add the latest signal index and its mappings
+* Posts a sample signal which checks for root or admin every 5 minutes
 
-Take note of the `id` GUID above and copy and paste that into a create alert like so
 
-```ts
-kbn-alert create siem.signals 5m '{}' "[{group:default id:'7edd7e98-9286-4fdb-a5c5-16de776bc7c7' params:{message: 'SIEM Alert Fired'}}]"
+Now you can run
+
+```sh
+./get_alert_instances.sh
 ```
 
-You should get back a response like so
+You should see the new alert instance created like so:
 ```ts
 {
     "id": "908a6af1-ac63-4d52-a856-fc635a00db0f",
@@ -122,5 +111,10 @@ Every 5 minutes you should see this message in your terminal now:
 server    log   [22:17:33.945] [info][alerting] SIEM Alert Fired
 ```
 
-Add the `.siem-signals-10-01-2019` to your advanced SIEM settings to see any signals
-created which should update once every 5 minutes at this point.
+See the scripts folder and the tools for more command line fun. 
+
+Add the `.siem-signals-${your user id}` to your advanced SIEM settings to see any signals
+created which should update once every 5 minutes at this point.
+
+Also add the `.siem-signals-${your user id}` as a kibana index for Maps to be able to see the
+signals 

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_reindex.ts

@@ -4,48 +4,64 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
+
 // TODO: Re-index is just a temporary solution in order to speed up development
 // of any front end pieces. This should be replaced with a combination of the file
 // build_events_query.ts and any scrolling/scaling solutions from that particular
 // file.
 
 interface BuildEventsReIndexParams {
+  description: string;
   index: string[];
   from: number;
   to: number;
   signalsIndex: string;
   maxDocs: number;
-  kqlFilter: {};
+  filter: Record<string, {}> | undefined;
+  kql: string | undefined;
   severity: number;
-  description: string;
   name: string;
   timeDetected: number;
   ruleRevision: number;
-  ruleId: string;
-  ruleType: string;
+  id: string;
+  type: string;
   references: string[];
 }
 
+export const getFilter = (kql: string | undefined, filter: Record<string, {}> | undefined) => {
+  if (kql != null) {
+    return toElasticsearchQuery(fromKueryExpression(kql), null);
+  } else if (filter != null) {
+    return filter;
+  } else {
+    // TODO: Re-visit this error (which should never happen) when we do signal errors for the UI
+    throw new TypeError('either kql or filter should be set');
+  }
+};
+
 export const buildEventsReIndex = ({
+  description,
   index,
   from,
   to,
   signalsIndex,
   maxDocs,
-  kqlFilter,
+  filter,
+  kql,
   severity,
-  description,
   name,
   timeDetected,
   ruleRevision,
-  ruleId,
-  ruleType,
+  id,
+  type,
   references,
 }: BuildEventsReIndexParams) => {
+  const kqlOrFilter = getFilter(kql, filter);
   const indexPatterns = index.map(element => `"${element}"`).join(',');
   const refs = references.map(element => `"${element}"`).join(',');
-  const filter = [
-    kqlFilter,
+  const filterWithTime = [
+    kqlOrFilter,
     {
       bool: {
         filter: [
@@ -96,7 +112,7 @@ export const buildEventsReIndex = ({
         query: {
           bool: {
             filter: [
-              ...filter,
+              ...filterWithTime,
               {
                 match_all: {},
               },
@@ -120,8 +136,8 @@ export const buildEventsReIndex = ({
 
           def signal = [
             "rule_revision": "${ruleRevision}",
-            "rule_id": "${ruleId}",
-            "rule_type": "${ruleType}",
+            "rule_id": "${id}",
+            "rule_type": "${type}",
             "parent": parent,
             "name": "${name}",
             "severity": ${severity},

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SIGNALS_ID } from '../../../../common/constants';
+import { AlertsClient } from '../../../../../alerting/server/alerts_client';
+import { ActionsClient } from '../../../../../actions/server/actions_client';
+
+export interface SignalParams {
+  alertsClient: AlertsClient;
+  actionsClient: ActionsClient;
+  description: string;
+  from: string;
+  id: string;
+  index: string[];
+  interval: string;
+  enabled: boolean;
+  filter: Record<string, {}> | undefined;
+  kql: string | undefined;
+  maxSignals: string;
+  name: string;
+  severity: number;
+  type: string; // TODO: Replace this type with a static enum type
+  to: string;
+  references: string[];
+}
+
+export const createSignal = async ({
+  alertsClient,
+  actionsClient,
+  description,
+  enabled,
+  filter,
+  from,
+  id,
+  index,
+  interval,
+  kql,
+  name,
+  severity,
+  to,
+  type,
+  references,
+}: SignalParams) => {
+  // TODO: Right now we are using the .server-log as the default action as each alert has to have
+  // at least one action or it will not be able to do in-memory persistence. When adding in actions
+  // such as email, slack, etc... this should be the default action if not action is specified to
+  // create signals
+
+  const actionResults = await actionsClient.create({
+    action: {
+      actionTypeId: '.server-log',
+      description: 'SIEM Alerts Log',
+      config: {},
+      secrets: {},
+    },
+  });
+
+  return alertsClient.create({
+    data: {
+      alertTypeId: SIGNALS_ID,
+      alertTypeParams: {
+        description,
+        id,
+        index,
+        from,
+        filter,
+        kql,
+        name,
+        severity,
+        to,
+        type,
+        references,
+      },
+      interval,
+      enabled,
+      actions: [
+        {
+          group: 'default',
+          id: actionResults.id,
+          params: {
+            message: 'SIEM Alert Fired',
+          },
+        },
+      ],
+      throttle: null,
+    },
+  });
+};