[8.x] [ML] Fix query for pattern analysis and change point analysis (e…

…lastic#194742) (elastic#195246) # Backport This will backport the following commits from `main` to `8.x`: - [[ML] Fix query for pattern analysis and change point analysis (elastic#194742)](elastic#194742)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: James Gowdy <jgowdy@elastic.co>
kibanamachine · Oct 7, 2024 · 1963138 · 1963138
1 parent c099f33
commit 1963138
Show file tree

Hide file tree

Showing 6 changed files with 177 additions and 24 deletions.
diff --git a/x-pack/packages/ml/aiops_common/create_default_query.test.ts b/x-pack/packages/ml/aiops_common/create_default_query.test.ts
@@ -0,0 +1,156 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createDefaultQuery } from './create_default_query';
+
+describe('createDefaultQuery', () => {
+  it('should create a default match_all query when no input query is provided', () => {
+    const result = createDefaultQuery(undefined, 'timestamp', undefined);
+    expect(result).toEqual({
+      bool: {
+        must: [{ match_all: {} }],
+      },
+    });
+  });
+
+  it('should wrap an existing match_all query in a bool must clause', () => {
+    const inputQuery = { match_all: {} };
+    const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
+    expect(result).toEqual({
+      bool: {
+        must: [{ match_all: {} }],
+      },
+    });
+  });
+
+  it('should wrap an existing query_string query in a bool must clause', () => {
+    const inputQuery = { query_string: { query: '*' } };
+    const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
+    expect(result).toEqual({
+      bool: {
+        must: [{ query_string: { query: '*' } }],
+      },
+    });
+  });
+
+  it('should wrap an existing multi_match query in a bool should clause', () => {
+    const inputQuery = { multi_match: { query: 'test', fields: ['field1', 'field2'] } };
+    const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
+    expect(result).toEqual({
+      bool: {
+        must: [],
+        should: { multi_match: { query: 'test', fields: ['field1', 'field2'] } },
+      },
+    });
+  });
+
+  it('should add a time range filter to the query', () => {
+    const timeRange = { from: 1609459200000, to: 1609545600000 };
+    const result = createDefaultQuery(undefined, 'timestamp', timeRange);
+    expect(result).toEqual({
+      bool: {
+        must: [
+          { match_all: {} },
+          {
+            range: {
+              timestamp: {
+                gte: 1609459200000,
+                lte: 1609545600000,
+                format: 'epoch_millis',
+              },
+            },
+          },
+        ],
+      },
+    });
+  });
+
+  it('should merge existing bool query with new time range filter', () => {
+    const inputQuery = { bool: { must: [{ term: { field: 'value' } }] } };
+    const timeRange = { from: 1609459200000, to: 1609545600000 };
+    const result = createDefaultQuery(inputQuery, 'timestamp', timeRange);
+    expect(result).toEqual({
+      bool: {
+        must: [
+          { term: { field: 'value' } },
+          {
+            range: {
+              timestamp: {
+                gte: 1609459200000,
+                lte: 1609545600000,
+                format: 'epoch_millis',
+              },
+            },
+          },
+        ],
+      },
+    });
+  });
+
+  it('should handle an existing bool query with must clause', () => {
+    const inputQuery = { bool: { must: [{ term: { field: 'value' } }] } };
+    const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
+    expect(result).toEqual({
+      bool: {
+        must: [{ term: { field: 'value' } }],
+      },
+    });
+  });
+
+  it('should handle an existing bool query with should clause', () => {
+    const inputQuery = { bool: { should: [{ term: { field: 'value' } }] } };
+    const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
+    expect(result).toEqual({
+      bool: {
+        must: [],
+        should: [{ term: { field: 'value' } }],
+      },
+    });
+  });
+
+  it('should handle an existing bool query with must_not clause', () => {
+    const inputQuery = { bool: { must_not: [{ term: { field: 'value' } }] } };
+    const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
+    expect(result).toEqual({
+      bool: {
+        must: [],
+        must_not: [{ term: { field: 'value' } }],
+      },
+    });
+  });
+
+  it('should handle an existing bool query with filter clause', () => {
+    const inputQuery = { bool: { filter: [{ term: { field: 'value' } }] } };
+    const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
+    expect(result).toEqual({
+      bool: {
+        must: [],
+        filter: [{ term: { field: 'value' } }],
+      },
+    });
+  });
+
+  it('should handle an input query with multiple clauses', () => {
+    const inputQuery = {
+      bool: {
+        must: [{ term: { field1: 'value1' } }],
+        should: [{ term: { field2: 'value2' } }],
+        must_not: [{ term: { field3: 'value3' } }],
+        filter: [{ term: { field4: 'value4' } }],
+      },
+    };
+    const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
+    expect(result).toEqual({
+      bool: {
+        must: [{ term: { field1: 'value1' } }],
+        should: [{ term: { field2: 'value2' } }],
+        must_not: [{ term: { field3: 'value3' } }],
+        filter: [{ term: { field4: 'value4' } }],
+      },
+    });
+  });
+});
diff --git a/...ttern_analysis/create_categorize_query.ts → ...s/ml/aiops_common/create_default_query.ts b/...ttern_analysis/create_categorize_query.ts → ...s/ml/aiops_common/create_default_query.ts
@@ -7,24 +7,32 @@
 
 import { cloneDeep } from 'lodash';
 
-import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
+import type {
+  QueryDslBoolQuery,
+  QueryDslQueryContainer,
+} from '@elastic/elasticsearch/lib/api/types';
 
-export function createCategorizeQuery(
+export function createDefaultQuery(
   queryIn: QueryDslQueryContainer | undefined,
   timeField: string,
   timeRange: { from: number; to: number } | undefined
 ) {
   const query = cloneDeep(queryIn ?? { match_all: {} });
 
   if (query.bool === undefined) {
-    query.bool = {};
+    query.bool = Object.create(null) as QueryDslBoolQuery;
   }
+
   if (query.bool.must === undefined) {
     query.bool.must = [];
     if (query.match_all !== undefined) {
       query.bool.must.push({ match_all: query.match_all });
       delete query.match_all;
     }
+    if (query.query_string !== undefined) {
+      query.bool.must.push({ query_string: query.query_string });
+      delete query.query_string;
+    }
   }
   if (query.multi_match !== undefined) {
     query.bool.should = {

diff --git a/x-pack/packages/ml/aiops_log_pattern_analysis/create_category_request.ts b/x-pack/packages/ml/aiops_log_pattern_analysis/create_category_request.ts
@@ -14,7 +14,7 @@ import { isPopulatedObject } from '@kbn/ml-is-populated-object/src/is_populated_
 
 import type { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils';
 
-import { createCategorizeQuery } from './create_categorize_query';
+import { createDefaultQuery } from '@kbn/aiops-common/create_default_query';
 
 const CATEGORY_LIMIT = 1000;
 const EXAMPLE_LIMIT = 4;
@@ -38,7 +38,7 @@ export function createCategoryRequest(
   useStandardTokenizer: boolean = true,
   includeSparkline: boolean = true
 ) {
-  const query = createCategorizeQuery(queryIn, timeField, timeRange);
+  const query = createDefaultQuery(queryIn, timeField, timeRange);
   const aggs = {
     categories: {
       categorize_text: {

diff --git a/x-pack/packages/ml/aiops_log_pattern_analysis/tsconfig.json b/x-pack/packages/ml/aiops_log_pattern_analysis/tsconfig.json
@@ -23,5 +23,6 @@
     "@kbn/saved-search-plugin",
     "@kbn/data-views-plugin",
     "@kbn/ml-is-populated-object",
+    "@kbn/aiops-common",
   ]
 }
diff --git a/...plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx b/...plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx
@@ -16,6 +16,7 @@ import { ES_FIELD_TYPES } from '@kbn/field-types';
 import { type QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
 import type { TimeBuckets, TimeBucketsInterval } from '@kbn/ml-time-buckets';
 import { useTimeBuckets } from '@kbn/ml-time-buckets';
+import { createDefaultQuery } from '@kbn/aiops-common/create_default_query';
 import { useFilterQueryUpdates } from '../../hooks/use_filters_query';
 import { type ChangePointType, DEFAULT_AGG_FUNCTION } from './constants';
 import {
@@ -261,23 +262,10 @@ export const ChangePointDetectionContextProvider: FC<PropsWithChildren<unknown>>
 
   const combinedQuery = useMemo(() => {
     const mergedQuery = createMergedEsQuery(resultQuery, resultFilters, dataView, uiSettings);
-    if (!Array.isArray(mergedQuery.bool?.filter)) {
-      if (!mergedQuery.bool) {
-        mergedQuery.bool = {};
-      }
-      mergedQuery.bool.filter = [];
-    }
-
-    mergedQuery.bool!.filter.push({
-      range: {
-        [dataView.timeFieldName!]: {
-          from: searchBounds.min?.valueOf(),
-          to: searchBounds.max?.valueOf(),
-        },
-      },
-    });
-
-    return mergedQuery;
+    const to = searchBounds.max?.valueOf();
+    const from = searchBounds.min?.valueOf();
+    const timeRange = to !== undefined && from !== undefined ? { from, to } : undefined;
+    return createDefaultQuery(mergedQuery, dataView.timeFieldName!, timeRange);
   }, [resultFilters, resultQuery, uiSettings, dataView, searchBounds]);
 
   if (!bucketInterval) return null;

diff --git a/x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts b/x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts
@@ -14,7 +14,7 @@ import type { FieldValidationResults } from '@kbn/ml-category-validator';
 import type { HttpFetchOptions } from '@kbn/core/public';
 import { AIOPS_API_ENDPOINT } from '@kbn/aiops-common/constants';
 
-import { createCategorizeQuery } from '@kbn/aiops-log-pattern-analysis/create_categorize_query';
+import { createDefaultQuery } from '@kbn/aiops-common/create_default_query';
 
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 
@@ -32,7 +32,7 @@ export function useValidateFieldRequest() {
       runtimeMappings: MappingRuntimeFields | undefined,
       headers?: HttpFetchOptions['headers']
     ) => {
-      const query = createCategorizeQuery(queryIn, timeField, timeRange);
+      const query = createDefaultQuery(queryIn, timeField, timeRange);
       const resp = await http.post<FieldValidationResults>(
         AIOPS_API_ENDPOINT.CATEGORIZATION_FIELD_VALIDATION,
         {