[SIEM] [Detection Engine] Adds filtering abilities to the KQL REST API (

#49451) (#49583) ## Summary * Removes the older beginner KQL type of signal creation in favor of newer version with filtering * Adds ability to create KQL or lucene queries that will work with the UI filters * UI state with the filters are now savable to re-hydrate UI's on the front end * Adds `saved_id` ability so the UI can tether dynamic saved queries with signals * Changed `it` to `test` as `it` is not the alias we use for tests * Updated script which converts older saved searches to work with newer mechanism * Fixed script to accept proper ndjson lines * Adds validation unit tests for the endpoint * Increases validation strictness of the endpoints * Adds more data scripts for testing scenarios * #47013 ## Testing * Run `./hard_reset.sh` script * Test with both algorithms through this toggle before starting kibana: `export USE_REINDEX_API=true` * Convert older saved searches to compatible new query filters by running: `./convert_saved_search_to_signals.sh ~/projects/saved_searches /tmp/signals` * Post them`./post_signal.sh /tmp/signals/*.json` * Hard reset again * Test smaller set of signals and REST endpoints using the typical scripts of: ```sh ./post_signal.sh ./read_signal.sh ./find_signals.sh ./update_signal.sh ./delete_signal.sh ``` or test using POSTMAN, etc... If you want to test validation. If you see any validation issues let me know as I have validation testing files and can easily fix them add another unit test to the growing large collection we have now. Change in your advanced settings of SIEM to use your signals index you configured for verification that the signals show up. ### Checklist Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR. ~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~ ~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~ ~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~ - [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios ~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~ ### For maintainers ~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~ ~~- [ ] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~
elastic · Oct 29, 2019 · 7a755d0 · 7a755d0
1 parent a11f180
commit 7a755d0
Show file tree

Hide file tree

Showing 34 changed files with 2,324 additions and 360 deletions.
diff --git a/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js b/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js
@@ -30,7 +30,7 @@ const path = require('path');
 // into another repository.
 const INTERVAL = '24h';
 const SEVERITY = 'low';
-const TYPE = 'kql';
+const TYPE = 'query';
 const FROM = 'now-24h';
 const TO = 'now';
 const INDEX = ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'];
@@ -70,43 +70,74 @@ async function main() {
   const files = process.argv[2];
   const outputDir = process.argv[3];
 
-  const savedSearchesJson = walk(files).filter(file => file.endsWith('.ndjson'));
+  const savedSearchesJson = walk(files).filter(file => {
+    return !path.basename(file).startsWith('.') && file.endsWith('.ndjson');
+  });
 
   const savedSearchesParsed = savedSearchesJson.reduce((accum, json) => {
     const jsonFile = fs.readFileSync(json, 'utf8');
-    try {
-      const parsedFile = JSON.parse(jsonFile);
-      parsedFile._file = json;
-      parsedFile.attributes.kibanaSavedObjectMeta.searchSourceJSON = JSON.parse(
-        parsedFile.attributes.kibanaSavedObjectMeta.searchSourceJSON
-      );
-      return [...accum, parsedFile];
-    } catch (err) {
-      return accum;
-    }
+    const jsonLines = jsonFile.split(/\r{0,1}\n/);
+    const parsedLines = jsonLines.reduce((accum, line, index) => {
+      try {
+        const parsedLine = JSON.parse(line);
+        if (index !== 0) {
+          parsedLine._file = `${json.substring(0, json.length - '.ndjson'.length)}_${String(
+            index
+          )}.ndjson`;
+        } else {
+          parsedLine._file = json;
+        }
+        parsedLine.attributes.kibanaSavedObjectMeta.searchSourceJSON = JSON.parse(
+          parsedLine.attributes.kibanaSavedObjectMeta.searchSourceJSON
+        );
+        return [...accum, parsedLine];
+      } catch (err) {
+        console.log('error parsing a line in this file:', json);
+        return accum;
+      }
+    }, []);
+    return [...accum, ...parsedLines];
   }, []);
 
-  savedSearchesParsed.forEach(savedSearch => {
-    const fileToWrite = cleanupFileName(savedSearch._file);
+  savedSearchesParsed.forEach(
+    ({
+      _file,
+      attributes: {
+        description,
+        title,
+        kibanaSavedObjectMeta: {
+          searchSourceJSON: {
+            query: { query, language },
+            filter,
+          },
+        },
+      },
+    }) => {
+      const fileToWrite = cleanupFileName(_file);
 
-    const query = savedSearch.attributes.kibanaSavedObjectMeta.searchSourceJSON.query.query;
-    if (query != null && query.trim() !== '') {
-      const outputMessage = {
-        id: fileToWrite,
-        description: savedSearch.attributes.description || savedSearch.attributes.title,
-        index: INDEX,
-        interval: INTERVAL,
-        name: savedSearch.attributes.title,
-        severity: SEVERITY,
-        type: TYPE,
-        from: FROM,
-        to: TO,
-        kql: savedSearch.attributes.kibanaSavedObjectMeta.searchSourceJSON.query.query,
-      };
+      if (query != null && query.trim() !== '') {
+        const outputMessage = {
+          id: fileToWrite,
+          description: description || title,
+          index: INDEX,
+          interval: INTERVAL,
+          name: title,
+          severity: SEVERITY,
+          type: TYPE,
+          from: FROM,
+          to: TO,
+          query,
+          language,
+          filters: filter,
+        };
 
-      fs.writeFileSync(`${outputDir}/${fileToWrite}.json`, JSON.stringify(outputMessage, null, 2));
+        fs.writeFileSync(
+          `${outputDir}/${fileToWrite}.json`,
+          JSON.stringify(outputMessage, null, 2)
+        );
+      }
     }
-  });
+  );
 }
 
 if (require.main === module) {

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts
@@ -4,41 +4,25 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
-
 interface BuildEventsScrollQuery {
   index: string[];
   from: string;
   to: string;
-  kql: string | undefined;
-  filter: Record<string, {}> | undefined;
+  filter: unknown;
   size: number;
   scroll: string;
 }
 
-export const getFilter = (kql: string | undefined, filter: Record<string, {}> | undefined) => {
-  if (kql != null) {
-    return toElasticsearchQuery(fromKueryExpression(kql), null);
-  } else if (filter != null) {
-    return filter;
-  } else {
-    // TODO: Re-visit this error (which should never happen) when we do signal errors for the UI
-    throw new TypeError('either kql or filter should be set');
-  }
-};
-
 export const buildEventsScrollQuery = ({
   index,
   from,
   to,
-  kql,
   filter,
   size,
   scroll,
 }: BuildEventsScrollQuery) => {
-  const kqlOrFilter = getFilter(kql, filter);
   const filterWithTime = [
-    kqlOrFilter,
+    filter,
     {
       bool: {
         filter: [

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_reindex.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_reindex.ts
@@ -4,8 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
-
 // TODO: Re-index is just a temporary solution in order to speed up development
 // of any front end pieces. This should be replaced with a combination of the file
 // build_events_query.ts and any scrolling/scaling solutions from that particular
@@ -17,9 +15,8 @@ interface BuildEventsReIndexParams {
   from: string;
   to: string;
   signalsIndex: string;
-  maxDocs: string;
-  filter: Record<string, {}> | undefined;
-  kql: string | undefined;
+  maxDocs: number;
+  filter: unknown;
   severity: string;
   name: string;
   timeDetected: string;
@@ -29,17 +26,6 @@ interface BuildEventsReIndexParams {
   references: string[];
 }
 
-export const getFilter = (kql: string | undefined, filter: Record<string, {}> | undefined) => {
-  if (kql != null) {
-    return toElasticsearchQuery(fromKueryExpression(kql), null);
-  } else if (filter != null) {
-    return filter;
-  } else {
-    // TODO: Re-visit this error (which should never happen) when we do signal errors for the UI
-    throw new TypeError('either kql or filter should be set');
-  }
-};
-
 export const buildEventsReIndex = ({
   description,
   index,
@@ -48,7 +34,6 @@ export const buildEventsReIndex = ({
   signalsIndex,
   maxDocs,
   filter,
-  kql,
   severity,
   name,
   timeDetected,
@@ -57,11 +42,10 @@ export const buildEventsReIndex = ({
   type,
   references,
 }: BuildEventsReIndexParams) => {
-  const kqlOrFilter = getFilter(kql, filter);
   const indexPatterns = index.map(element => `"${element}"`).join(',');
   const refs = references.map(element => `"${element}"`).join(',');
   const filterWithTime = [
-    kqlOrFilter,
+    filter,
     {
       bool: {
         filter: [

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signals.ts
@@ -17,10 +17,13 @@ export const updateIfIdExists = async ({
   enabled,
   filter,
   from,
+  query,
+  language,
+  savedId,
+  filters,
   id,
   index,
   interval,
-  kql,
   maxSignals,
   name,
   severity,
@@ -36,10 +39,13 @@ export const updateIfIdExists = async ({
       enabled,
       filter,
       from,
+      query,
+      language,
+      savedId,
+      filters,
       id,
       index,
       interval,
-      kql,
       maxSignals,
       name,
       severity,
@@ -62,10 +68,13 @@ export const createSignals = async ({
   enabled,
   filter,
   from,
+  query,
+  language,
+  savedId,
+  filters,
   id,
   index,
   interval,
-  kql,
   maxSignals,
   name,
   severity,
@@ -81,10 +90,13 @@ export const createSignals = async ({
     enabled,
     filter,
     from,
+    query,
+    language,
+    savedId,
+    filters,
     id,
     index,
     interval,
-    kql,
     maxSignals,
     name,
     severity,
@@ -115,7 +127,10 @@ export const createSignals = async ({
           index,
           from,
           filter,
-          kql,
+          query,
+          language,
+          savedId,
+          filters,
           maxSignals,
           name,
           severity,