[SIEM] [Detection Engine] Adds stable alerting ids, more scripting fo…

…r product testing, and more unit tests (#48471) (#48580)

* Adds stable alerting id's by using the alert params.
* Currently does a manual walk through of the alert params to find the stable id
* Updated all of the endpoints to take either of the two id's.
* Added several scripts to support performance testing ad-hoc such as `post_x_signals.sh`
* Added scripts to support converting from saved searches to alerts.
* Consolidated and fixed a lot of the backend types
* Added unit tests against the router endpoints
* #47013

Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR.

~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~

~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~

~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~

- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~

- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)
- [ ] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)

x-pack/legacy/plugins/siem/index.ts

@@ -26,6 +26,7 @@ import {
 } from './common/constants';
 import { signalsAlertType } from './server/lib/detection_engine/alerts/signals_alert_type';
 import { defaultIndexPattern } from './default_index_pattern';
+import { isAlertExecutor } from './server/lib/detection_engine/alerts/types';
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export function siem(kibana: any) {
@@ -126,9 +127,12 @@ export function siem(kibana: any) {
     init(server: Server) {
       const newPlatform = ((server as unknown) as KbnServer).newPlatform;
       if (server.plugins.alerting != null) {
-        server.plugins.alerting.setup.registerType(
-          signalsAlertType({ logger: newPlatform.coreContext.logger.get('plugins', APP_ID) })
-        );
+        const type = signalsAlertType({
+          logger: newPlatform.coreContext.logger.get('plugins', APP_ID),
+        });
+        if (isAlertExecutor(type)) {
+          server.plugins.alerting.setup.registerType(type);
+        }
       }
       server.injectUiAppVars('siem', async () => server.getInjectedUiAppVars('kibana'));
       initServerWithKibana(server);

x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js

@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+require('../../../../../src/setup_node_env');
+
+const fs = require('fs');
+const path = require('path');
+
+/*
+ * This script is used to parse a set of saved searches on a file system
+ * and output signal data compatible json files.
+ * Example:
+ * node saved_query_to_signals.js ${HOME}/saved_searches ${HOME}/saved_signals
+ *
+ * After editing any changes in the files of ${HOME}/saved_signals/*.json
+ * you can then post the signals with a CURL post script such as:
+ *
+ * ./post_signal.sh ${HOME}/saved_signals/*.json
+ *
+ * Note: This script is recursive and but does not preserve folder structure
+ * when it outputs the saved signals.
+ */
+
+// Defaults of the outputted signals since the saved KQL searches do not have
+// this type of information. You usually will want to make any hand edits after
+// doing a search to KQL conversion before posting it as a signal or checking it
+// into another repository.
+const INTERVAL = '24h';
+const SEVERITY = 1;
+const TYPE = 'kql';
+const FROM = 'now-24h';
+const TO = 'now';
+const INDEX = ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'];
+
+const walk = dir => {
+  const list = fs.readdirSync(dir);
+  return list.reduce((accum, file) => {
+    const fileWithDir = dir + '/' + file;
+    const stat = fs.statSync(fileWithDir);
+    if (stat && stat.isDirectory()) {
+      return [...accum, ...walk(fileWithDir)];
+    } else {
+      return [...accum, fileWithDir];
+    }
+  }, []);
+};
+
+//clean up the file system characters
+const cleanupFileName = file => {
+  return path
+    .basename(file, path.extname(file))
+    .replace(/\s+/g, '_')
+    .replace(/,/g, '')
+    .replace(/\+s/g, '')
+    .replace(/-/g, '')
+    .replace(/__/g, '_')
+    .toLowerCase();
+};
+
+async function main() {
+  if (process.argv.length !== 4) {
+    throw new Error(
+      'usage: saved_query_to_signals [input directory with saved searches] [output directory]'
+    );
+  }
+
+  const files = process.argv[2];
+  const outputDir = process.argv[3];
+
+  const savedSearchesJson = walk(files).filter(file => file.endsWith('.ndjson'));
+
+  const savedSearchesParsed = savedSearchesJson.reduce((accum, json) => {
+    const jsonFile = fs.readFileSync(json, 'utf8');
+    try {
+      const parsedFile = JSON.parse(jsonFile);
+      parsedFile._file = json;
+      parsedFile.attributes.kibanaSavedObjectMeta.searchSourceJSON = JSON.parse(
+        parsedFile.attributes.kibanaSavedObjectMeta.searchSourceJSON
+      );
+      return [...accum, parsedFile];
+    } catch (err) {
+      return accum;
+    }
+  }, []);
+
+  savedSearchesParsed.forEach(savedSearch => {
+    const fileToWrite = cleanupFileName(savedSearch._file);
+
+    const query = savedSearch.attributes.kibanaSavedObjectMeta.searchSourceJSON.query.query;
+    if (query != null && query.trim() !== '') {
+      const outputMessage = {
+        id: fileToWrite,
+        description: savedSearch.attributes.description || savedSearch.attributes.title,
+        index: INDEX,
+        interval: INTERVAL,
+        name: savedSearch.attributes.title,
+        severity: SEVERITY,
+        type: TYPE,
+        from: FROM,
+        to: TO,
+        kql: savedSearch.attributes.kibanaSavedObjectMeta.searchSourceJSON.query.query,
+      };
+
+      fs.writeFileSync(`${outputDir}/${fileToWrite}.json`, JSON.stringify(outputMessage, null, 2));
+    }
+  });
+}
+
+if (require.main === module) {
+  main();
+}

x-pack/legacy/plugins/siem/server/kibana.index.ts

@@ -20,7 +20,7 @@ import { createSignalsRoute } from './lib/detection_engine/routes/create_signals
 import { readSignalsRoute } from './lib/detection_engine/routes/read_signals_route';
 import { findSignalsRoute } from './lib/detection_engine/routes/find_signals_route';
 import { deleteSignalsRoute } from './lib/detection_engine/routes/delete_signals_route';
-import { updateSignalsRoute } from './lib/detection_engine/routes/updated_signals_route';
+import { updateSignalsRoute } from './lib/detection_engine/routes/update_signals_route';
 
 const APP_ID = 'siem';
 

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts

@@ -8,8 +8,8 @@ import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
 
 interface BuildEventsScrollQuery {
   index: string[];
-  from: number;
-  to: number;
+  from: string;
+  to: string;
   kql: string | undefined;
   filter: Record<string, {}> | undefined;
   size: number;

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_reindex.ts

@@ -14,10 +14,10 @@ import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
 interface BuildEventsReIndexParams {
   description: string;
   index: string[];
-  from: number;
-  to: number;
+  from: string;
+  to: string;
   signalsIndex: string;
-  maxDocs: number;
+  maxDocs: string;
   filter: Record<string, {}> | undefined;
   kql: string | undefined;
   severity: number;