[SIEM][Detection Engine] Adds Read, Update, Delete API endpoints (#47765

)

## Summary

* Adds a Read, Update, Delete, and Find API endpoints.
* Adds several scripts to exercise the endpoints as well as improves the helper scripts
* Fixes a bad assumption with the way alert params works with `null`
* Fixes a bug where I was using an array instead of a number of `max_signals` 
* Fixes a bug with the log level since it upgraded recently and requires a log level
 

IMPORTANT NOTE:
---
This still uses auto-generated GUID's and not the alert id so there are not stable id's just yet. However, either we will add that capability through alert params or the alerting team will add the capability to do a POST of the {id} into the create endpoints.

Testing:
---
Follow the `README.md` for initial setup of our temporary environment variables. Use the scripts and post a signal after a hard reset like so:

```ts
./hard_reset.sh
./post_signal.sh
``` 

Then run the following scripts to test each piece:

```sh
# Creates a new signal
./post_signal.sh
{
    "id": "908a6af1-ac63-4d52-a856-fc635a00db0f",
    "alertTypeId": "siem.signals",
    "interval": "5m",
    "actions": [
        {
            "group": "default",
            "params": {
                "message": "SIEM Alert Fired"
            },
            "id": "7edd7e98-9286-4fdb-a5c5-16de776bc7c7"
        }
    ],
    "alertTypeParams": {},
    "enabled": true,
    "throttle": null,
    "createdBy": "elastic",
    "updatedBy": "elastic",
    "apiKeyOwner": "elastic",
    "scheduledTaskId": "4f401ca0-e402-11e9-94ed-051d758a6c79"
}

# Read a signal that is from the result 
./read_signal.sh 908a6af1-ac63-4d52-a856-fc635a00db0f

# Edit the file `vim signals/temp_update_1.json` (manually) and add the ID into 
# it like so since we don't have stable alert ID's just yet
{
  "id": "908a6af1-ac63-4d52-a856-fc635a00db0f",
  "description": "Only watch winlogbeat users",
  "index": ["winlogbeat-*"],
  "interval": "9m",
  "name": "Just watch other winlogbeat users",
  "severity": 500,
  "enabled": false,
  "type": "filter",
  "from": "now-5d",
  "to": "now-1d",
  "kql": "user.name: something_else"
}

# Then update it
./update_signal.sh

# Then run a find to see all signals
./find_signals.sh

# Delete the signal
./delete_signal.sh 908a6af1-ac63-4d52-a856-fc635a00db0f
```

Take a look at the arguments and play around with removing fields from the update document as well as using different features of the API to see if something is broken for the initial roll out.

### Checklist

Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR.

~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~

~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~

~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~

~~- [ ] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios~~

~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~

### For maintainers

~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~

~~- [ ] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~

x-pack/legacy/plugins/siem/server/kibana.index.ts

@@ -17,6 +17,10 @@ import {
 } from './saved_objects';
 
 import { createSignalsRoute } from './lib/detection_engine/routes/create_signals_route';
+import { readSignalsRoute } from './lib/detection_engine/routes/read_signals_route';
+import { findSignalsRoute } from './lib/detection_engine/routes/find_signals_route';
+import { deleteSignalsRoute } from './lib/detection_engine/routes/delete_signals_route';
+import { updateSignalsRoute } from './lib/detection_engine/routes/updated_signals_route';
 
 const APP_ID = 'siem';
 
@@ -46,6 +50,10 @@ export const initServerWithKibana = (kbnServer: Server) => {
       'Detected feature flags for actions and alerting and enabling signals API endpoints'
     );
     createSignalsRoute(kbnServer);
+    readSignalsRoute(kbnServer);
+    updateSignalsRoute(kbnServer);
+    deleteSignalsRoute(kbnServer);
+    findSignalsRoute(kbnServer);
   }
   logger.info('Plugin done initializing');
 

x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md

@@ -17,6 +17,7 @@ export ELASTICSEARCH_URL=https://${ip}:9200
 export KIBANA_URL=http://localhost:5601
 export SIGNALS_INDEX=.siem-signals-${your user id}
 export TASK_MANAGER_INDEX=.kibana-task-manager-${your user id}
+export KIBANA_INDEX=.kibana-${your user id}
 
 # This is for the kbn-action and kbn-alert tool
 export KBN_URLBASE=http://${user}:${password}@localhost:5601
@@ -69,18 +70,19 @@ server    log   [11:39:05.561] [info][siem] Detected feature flags for actions a
 Open a terminal and go into the scripts folder `cd kibana/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts` and run:
 
 ```
-./delete_signal_index.sh
-./put_signal_index.sh
+./hard_reset.sh
 ./post_signal.sh
 ```
 
 which will:
 
+* Delete any existing actions you have
+* Delete any existing alerts you have
+* Delete any existing alert tasks you have
 * Delete any existing signal mapping you might have had.
 * Add the latest signal index and its mappings
 * Posts a sample signal which checks for root or admin every 5 minutes
 
-
 Now you can run
 
 ```sh

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts

@@ -82,6 +82,7 @@ export const createSignal = async ({
           id: actionResults.id,
           params: {
             message: 'SIEM Alert Fired',
+            level: 'info',
           },
         },
       ],

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { AlertAction } from '../../../../../alerting/server/types';
+import { ActionsClient } from '../../../../../actions/server/actions_client';
+import { AlertsClient } from '../../../../../alerting/server/alerts_client';
+
+export interface DeleteSignalParams {
+  alertsClient: AlertsClient;
+  actionsClient: ActionsClient;
+  id: string;
+}
+
+export const deleteAllSignalActions = async (
+  actionsClient: ActionsClient,
+  actions: AlertAction[]
+): Promise<Error | null> => {
+  try {
+    await Promise.all(actions.map(async ({ id }) => actionsClient.delete({ id })));
+    return null;
+  } catch (error) {
+    return error;
+  }
+};
+
+export const deleteSignals = async ({ alertsClient, actionsClient, id }: DeleteSignalParams) => {
+  const alert = await alertsClient.get({ id });
+
+  // TODO: Remove this as cast as soon as signal.actions TypeScript bug is fixed
+  // where it is trying to return AlertAction[] or RawAlertAction[]
+  const actions = (alert.actions as (AlertAction[] | undefined)) || [];
+
+  const actionsErrors = await deleteAllSignalActions(actionsClient, actions);
+  const deletedAlert = await alertsClient.delete({ id });
+  if (actionsErrors != null) {
+    throw actionsErrors;
+  } else {
+    return deletedAlert;
+  }
+};

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/find_signals.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SIGNALS_ID } from '../../../../common/constants';
+import { AlertsClient } from '../../../../../alerting/server/alerts_client';
+
+export interface GetSignalParams {
+  alertsClient: AlertsClient;
+  perPage?: number;
+  page?: number;
+  sortField?: string;
+  fields?: string[];
+}
+
+// TODO: Change this from a search to a filter once this ticket is solved:
+// https://github.com/elastic/kibana/projects/26#card-27462236
+export const findSignals = async ({ alertsClient, perPage, page, fields }: GetSignalParams) => {
+  return alertsClient.find({
+    options: {
+      fields,
+      page,
+      perPage,
+      searchFields: ['alertTypeId'],
+      search: SIGNALS_ID,
+    },
+  });
+};

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/read_signals.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { AlertsClient } from '../../../../../alerting/server/alerts_client';
+
+export interface ReadSignalParams {
+  alertsClient: AlertsClient;
+  id: string;
+}
+
+// TODO: Change this from a search to a filter once this ticket is solved:
+// https://github.com/elastic/kibana/projects/26#card-27462236
+export const readSignals = async ({ alertsClient, id }: ReadSignalParams) => {
+  return alertsClient.get({ id });
+};

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts

@@ -28,10 +28,10 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): AlertType => {
       params: schema.object({
         description: schema.string(),
         from: schema.string(),
-        filter: schema.maybe(schema.object({}, { allowUnknowns: true })),
+        filter: schema.nullable(schema.object({}, { allowUnknowns: true })),
         id: schema.number(),
         index: schema.arrayOf(schema.string()),
-        kql: schema.maybe(schema.string({ defaultValue: undefined })),
+        kql: schema.nullable(schema.string()),
         maxSignals: schema.number({ defaultValue: 100 }),
         name: schema.string(),
         severity: schema.number(),

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.test.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { calculateInterval, calculateKqlAndFilter } from './update_signals';
+
+describe('update_signals', () => {
+  describe('#calculateInterval', () => {
+    test('given a undefined interval, it returns the signalInterval ', () => {
+      const interval = calculateInterval(undefined, '10m');
+      expect(interval).toEqual('10m');
+    });
+
+    test('given a undefined signalInterval, it returns a undefined interval ', () => {
+      const interval = calculateInterval('10m', undefined);
+      expect(interval).toEqual('10m');
+    });
+
+    test('given both an undefined signalInterval and a undefined interval, it returns 5m', () => {
+      const interval = calculateInterval(undefined, undefined);
+      expect(interval).toEqual('5m');
+    });
+  });
+
+  describe('#calculateKqlAndFilter', () => {
+    test('given a undefined kql filter it returns a null kql', () => {
+      const kqlFilter = calculateKqlAndFilter(undefined, {});
+      expect(kqlFilter).toEqual({
+        filter: {},
+        kql: null,
+      });
+    });
+
+    test('given a undefined filter it returns a null filter', () => {
+      const kqlFilter = calculateKqlAndFilter('some kql string', undefined);
+      expect(kqlFilter).toEqual({
+        filter: null,
+        kql: 'some kql string',
+      });
+    });
+
+    test('given both a undefined filter and undefined kql it returns both as undefined', () => {
+      const kqlFilter = calculateKqlAndFilter(undefined, undefined);
+      expect(kqlFilter).toEqual({
+        filter: undefined,
+        kql: undefined,
+      });
+    });
+  });
+});

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts

@@ -0,0 +1,119 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { defaults } from 'lodash/fp';
+import { AlertAction } from '../../../../../alerting/server/types';
+import { AlertsClient } from '../../../../../alerting/server/alerts_client';
+import { ActionsClient } from '../../../../../actions/server/actions_client';
+import { readSignals } from './read_signals';
+
+export interface SignalParams {
+  alertsClient: AlertsClient;
+  actionsClient: ActionsClient;
+  description?: string;
+  from?: string;
+  id: string;
+  index?: string[];
+  interval?: string;
+  enabled?: boolean;
+  filter?: Record<string, {}> | undefined;
+  kql?: string | undefined;
+  maxSignals?: string;
+  name?: string;
+  severity?: number;
+  type?: string; // TODO: Replace this type with a static enum type
+  to?: string;
+  references?: string[];
+}
+
+export const calculateInterval = (
+  interval: string | undefined,
+  signalInterval: string | undefined
+): string => {
+  if (interval != null) {
+    return interval;
+  } else if (signalInterval != null) {
+    return signalInterval;
+  } else {
+    return '5m';
+  }
+};
+
+export const calculateKqlAndFilter = (
+  kql: string | undefined,
+  filter: {} | undefined
+): { kql: string | null | undefined; filter: {} | null | undefined } => {
+  if (filter != null) {
+    return { kql: null, filter };
+  } else if (kql != null) {
+    return { kql, filter: null };
+  } else {
+    return { kql: undefined, filter: undefined };
+  }
+};
+
+export const updateSignal = async ({
+  alertsClient,
+  actionsClient, // TODO: Use this whenever we add feature support for different action types
+  description,
+  enabled,
+  filter,
+  from,
+  id,
+  index,
+  interval,
+  kql,
+  name,
+  severity,
+  to,
+  type,
+  references,
+}: SignalParams) => {
+  // TODO: Error handling and abstraction. Right now if this is an error then what happens is we get the error of
+  // "message": "Saved object [alert/{id}] not found"
+  const signal = await readSignals({ alertsClient, id });
+
+  // TODO: Remove this as cast as soon as signal.actions TypeScript bug is fixed
+  // where it is trying to return AlertAction[] or RawAlertAction[]
+  const actions = (signal.actions as AlertAction[] | undefined) || [];
+
+  const alertTypeParams = signal.alertTypeParams || {};
+
+  const { kql: nextKql, filter: nextFilter } = calculateKqlAndFilter(kql, filter);
+
+  const nextAlertTypeParams = defaults(
+    {
+      ...alertTypeParams,
+    },
+    {
+      description,
+      filter: nextFilter,
+      from,
+      index,
+      kql: nextKql,
+      name,
+      severity,
+      to,
+      type,
+      references,
+    }
+  );
+
+  if (signal.enabled && !enabled) {
+    await alertsClient.disable({ id });
+  } else if (!signal.enabled && enabled) {
+    await alertsClient.enable({ id });
+  }
+
+  return alertsClient.update({
+    id,
+    data: {
+      interval: calculateInterval(interval, signal.interval),
+      actions,
+      alertTypeParams: nextAlertTypeParams,
+    },
+  });
+};

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts

@@ -47,7 +47,7 @@ export const createSignalsRoute = (server: Hapi.Server) => {
           index: Joi.array().required(),
           interval: Joi.string().default('5m'),
           kql: Joi.string(),
-          max_signals: Joi.array().default([]),
+          max_signals: Joi.number().default(100),
           name: Joi.string().required(),
           severity: Joi.number().required(),
           to: Joi.string().required(),

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/delete_signals_route.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import Hapi from 'hapi';
+import { isFunction } from 'lodash/fp';
+
+import { deleteSignals } from '../alerts/delete_signals';
+
+export const deleteSignalsRoute = (server: Hapi.Server) => {
+  server.route({
+    method: 'DELETE',
+    path: '/api/siem/signals/{id}',
+    options: {
+      tags: ['access:signals-all'],
+      validate: {
+        options: {
+          abortEarly: false,
+        },
+      },
+    },
+    async handler(request: Hapi.Request, headers) {
+      const { id } = request.params;
+      const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null;
+      const actionsClient = isFunction(request.getActionsClient)
+        ? request.getActionsClient()
+        : null;
+      if (alertsClient == null || actionsClient == null) {
+        return headers.response().code(404);
+      }
+      return deleteSignals({
+        actionsClient,
+        alertsClient,
+        id,
+      });
+    },
+  });
+};