[SIEM] Adds stable alerting ids and more scripting for product testing

x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js

@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+require('../../../../../src/setup_node_env');
+
+const fs = require('fs');
+const path = require('path');
+
+/*
+ * This script is used to parse a set of saved searches on a file system
+ * and output signal data compatible json files.
+ * Example:
+ * node saved_query_to_signals.js ${HOME}/saved_searches ${HOME}/saved_signals
+ *
+ * After editing any changes in the files of ${HOME}/saved_signals/*.json
+ * you can then post the signals with a CURL post script such as:
+ *
+ * ./post_signal.sh ${HOME}/saved_signals/*.json
+ *
+ * Note: This script is recursive and but does not preserve folder structure
+ * when it outputs the saved signals.
+ */
+
+// Defaults of the outputted signals since the saved KQL searches do not have
+// this type of information. You usually will want to make any hand edits after
+// doing a search to KQL conversion before posting it as a signal or checking it
+// into another repository.
+const INTERVAL = '24h';
+const SEVERITY = 1;
+const TYPE = 'kql';
+const FROM = 'now-24h';
+const TO = 'now';
+const INDEX = ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'];
+
+const walk = dir => {
+  const list = fs.readdirSync(dir);
+  return list.reduce((accum, file) => {
+    const fileWithDir = dir + '/' + file;
+    const stat = fs.statSync(fileWithDir);
+    if (stat && stat.isDirectory()) {
+      return [...accum, ...walk(fileWithDir)];
+    } else {
+      return [...accum, fileWithDir];
+    }
+  }, []);
+};
+
+// Temporary hash function for converting string to numbers.
+// TODO: Once we move from numbers to pure strings for id's this can be removed
+// and the file name used as the id (or a GUID), etc...
+const hashFunc = str => {
+  let chr;
+  let hash = 0;
+  if (str.length === 0) return 0;
+  for (let i = 0; i < str.length; i++) {
+    chr = str.charCodeAt(i);
+    // eslint-disable-next-line no-bitwise
+    hash = (hash << 5) - hash + chr;
+    // eslint-disable-next-line no-bitwise
+    hash |= 0;
+  }
+  return hash;
+};
+
+//clean up the file system characters
+const cleanupFileName = file => {
+  return path
+    .basename(file, path.extname(file))
+    .replace(/\s+/g, '_')
+    .replace(/,/g, '')
+    .replace(/\+s/g, '')
+    .replace(/-/g, '')
+    .replace(/__/g, '_')
+    .toLowerCase();
+};
+
+async function main() {
+  if (process.argv.length !== 4) {
+    throw new Error(
+      'usage: saved_query_to_signals [input directory with saved searches] [output directory]'
+    );
+  }
+
+  const files = process.argv[2];
+  const outputDir = process.argv[3];
+
+  const savedSearchesJson = walk(files).filter(file => file.endsWith('.ndjson'));
+
+  const savedSearchesParsed = savedSearchesJson.reduce((accum, json) => {
+    const jsonFile = fs.readFileSync(json, 'utf8');
+    try {
+      const parsedFile = JSON.parse(jsonFile);
+      parsedFile._file = json;
+      parsedFile.attributes.kibanaSavedObjectMeta.searchSourceJSON = JSON.parse(
+        parsedFile.attributes.kibanaSavedObjectMeta.searchSourceJSON
+      );
+      return [...accum, parsedFile];
+    } catch (err) {
+      return accum;
+    }
+  }, []);
+
+  savedSearchesParsed.forEach(savedSearch => {
+    const fileToWrite = cleanupFileName(savedSearch._file);
+
+    const query = savedSearch.attributes.kibanaSavedObjectMeta.searchSourceJSON.query.query;
+    if (query != null && query.trim() !== '') {
+      const outputMessage = {
+        id: `${hashFunc(fileToWrite)}`, // TODO: Remove this once we change id to a string
+        description: savedSearch.attributes.description || savedSearch.attributes.title,
+        index: INDEX,
+        interval: INTERVAL,
+        name: savedSearch.attributes.title,
+        severity: SEVERITY,
+        type: TYPE,
+        from: FROM,
+        to: TO,
+        kql: savedSearch.attributes.kibanaSavedObjectMeta.searchSourceJSON.query.query,
+      };
+
+      fs.writeFileSync(`${outputDir}/${fileToWrite}.json`, JSON.stringify(outputMessage, null, 2));
+    }
+  });
+}
+
+if (require.main === module) {
+  main();
+}

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signal.ts

@@ -7,6 +7,7 @@
 import { SIGNALS_ID } from '../../../../common/constants';
 import { AlertsClient } from '../../../../../alerting/server/alerts_client';
 import { ActionsClient } from '../../../../../actions/server/actions_client';
+import { updateSignal } from './update_signals';
 
 export interface SignalParams {
   alertsClient: AlertsClient;
@@ -27,7 +28,9 @@ export interface SignalParams {
   references: string[];
 }
 
-export const createSignal = async ({
+// TODO: This updateIfIdExists should be temporary and we will remove it once we can POST id's directly to
+// the alerting framework.
+export const updateIfIdExists = async ({
   alertsClient,
   actionsClient,
   description,
@@ -44,49 +47,116 @@ export const createSignal = async ({
   type,
   references,
 }: SignalParams) => {
-  // TODO: Right now we are using the .server-log as the default action as each alert has to have
-  // at least one action or it will not be able to do in-memory persistence. When adding in actions
-  // such as email, slack, etc... this should be the default action if not action is specified to
-  // create signals
+  try {
+    const signal = await updateSignal({
+      alertsClient,
+      actionsClient,
+      description,
+      enabled,
+      filter,
+      from,
+      id,
+      index,
+      interval,
+      kql,
+      name,
+      severity,
+      to,
+      type,
+      references,
+    });
+    return signal;
+  } catch (err) {
+    // This happens when we cannot get a saved object back from reading a signal.
+    // So we continue normally as we have nothing we can upsert.
+  }
+  return null;
+};
 
-  const actionResults = await actionsClient.create({
-    action: {
-      actionTypeId: '.server-log',
-      description: 'SIEM Alerts Log',
-      config: {},
-      secrets: {},
-    },
+export const createSignal = async ({
+  alertsClient,
+  actionsClient,
+  description,
+  enabled,
+  filter,
+  from,
+  id,
+  index,
+  interval,
+  kql,
+  maxSignals,
+  name,
+  severity,
+  to,
+  type,
+  references,
+}: SignalParams) => {
+  // TODO: Once we can post directly to _id we will not have to do this part anymore.
+  const signalUpdating = await updateIfIdExists({
+    alertsClient,
+    actionsClient,
+    description,
+    enabled,
+    filter,
+    from,
+    id,
+    index,
+    interval,
+    kql,
+    maxSignals,
+    name,
+    severity,
+    to,
+    type,
+    references,
   });
-
-  return alertsClient.create({
-    data: {
-      alertTypeId: SIGNALS_ID,
-      alertTypeParams: {
-        description,
-        id,
-        index,
-        from,
-        filter,
-        kql,
-        name,
-        severity,
-        to,
-        type,
-        references,
+  if (signalUpdating == null) {
+    // TODO: Right now we are using the .server-log as the default action as each alert has to have
+    // at least one action or it will not be able to do in-memory persistence. When adding in actions
+    // such as email, slack, etc... this should be the default action if no action is specified to
+    // create signals
+    const actionResults = await actionsClient.create({
+      action: {
+        actionTypeId: '.server-log',
+        description: 'SIEM Alerts Log',
+        config: {},
+        secrets: {},
       },
-      interval,
-      enabled,
-      actions: [
-        {
-          group: 'default',
-          id: actionResults.id,
-          params: {
-            message: 'SIEM Alert Fired',
-            level: 'info',
-          },
+    });
+
+    return alertsClient.create({
+      data: {
+        alertTypeId: SIGNALS_ID,
+        alertTypeParams: {
+          description,
+          id,
+          index,
+          from,
+          filter,
+          kql,
+          maxSignals,
+          name,
+          severity,
+          to,
+          type,
+          references,
         },
-      ],
-      throttle: null,
-    },
-  });
+        interval,
+        enabled,
+        actions: [
+          {
+            group: 'default',
+            id: actionResults.id,
+            params: {
+              message: 'SIEM Alert Fired',
+              level: 'info',
+            },
+          },
+        ],
+        throttle: null,
+      },
+    });
+  } else {
+    return signalUpdating;
+  }
 };

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/delete_signals.ts

@@ -7,6 +7,7 @@
 import { AlertAction } from '../../../../../alerting/server/types';
 import { ActionsClient } from '../../../../../actions/server/actions_client';
 import { AlertsClient } from '../../../../../alerting/server/alerts_client';
+import { readSignals } from './read_signals';
 
 export interface DeleteSignalParams {
   alertsClient: AlertsClient;
@@ -27,14 +28,14 @@ export const deleteAllSignalActions = async (
 };
 
 export const deleteSignals = async ({ alertsClient, actionsClient, id }: DeleteSignalParams) => {
-  const alert = await alertsClient.get({ id });
+  const signal = await readSignals({ alertsClient, id });
 
   // TODO: Remove this as cast as soon as signal.actions TypeScript bug is fixed
   // where it is trying to return AlertAction[] or RawAlertAction[]
-  const actions = (alert.actions as (AlertAction[] | undefined)) || [];
+  const actions = (signal.actions as (AlertAction[] | undefined)) || [];
 
   const actionsErrors = await deleteAllSignalActions(actionsClient, actions);
-  const deletedAlert = await alertsClient.delete({ id });
+  const deletedAlert = await alertsClient.delete({ id: signal.id });
   if (actionsErrors != null) {
     throw actionsErrors;
   } else {